From 38bc1fe8e9d4601e6a5879e7763fe4755a89c988 Mon Sep 17 00:00:00 2001 From: henryzhx8 Date: Thu, 19 Sep 2024 09:30:40 +0800 Subject: [PATCH 1/9] add metric for flusher component (#1762) --- core/CMakeLists.txt | 2 +- core/common/common.cmake | 2 +- .../compression/CompressType.h | 10 +- core/common/compression/Compressor.cpp | 59 +++++++++ .../compression/Compressor.h | 23 +++- .../compression/CompressorFactory.cpp | 30 +++-- .../compression/CompressorFactory.h | 8 +- .../compression/LZ4Compressor.cpp | 2 +- .../compression/LZ4Compressor.h | 9 +- .../compression/ZstdCompressor.cpp | 2 +- .../compression/ZstdCompressor.h | 8 +- core/go_pipeline/LogtailPlugin.cpp | 16 ++- core/monitor/LogtailMetric.h | 18 +++ core/monitor/MetricConstants.cpp | 12 +- core/monitor/MetricConstants.h | 12 +- core/pipeline/Pipeline.cpp | 26 +++- core/pipeline/Pipeline.h | 14 ++- core/pipeline/batch/BatchItem.h | 34 +++++- core/pipeline/batch/BatchStatus.h | 2 +- core/pipeline/batch/BatchedEvents.h | 10 +- core/pipeline/batch/Batcher.h | 88 +++++++++++++- .../plugin/instance/FlusherInstance.cpp | 8 +- .../plugin/instance/FlusherInstance.h | 4 +- .../plugin/instance/ProcessorInstance.cpp | 6 +- core/pipeline/queue/QueueInterface.h | 8 +- core/pipeline/route/Router.cpp | 11 ++ core/pipeline/route/Router.h | 5 + core/pipeline/serializer/SLSSerializer.cpp | 41 +++++-- core/pipeline/serializer/SLSSerializer.h | 5 + core/pipeline/serializer/Serializer.h | 73 ++++++++++- core/plugin/flusher/sls/FlusherSLS.cpp | 16 +-- core/plugin/flusher/sls/FlusherSLS.h | 6 +- core/profile_sender/ProfileSender.cpp | 6 +- core/unittest/batch/BatchItemUnittest.cpp | 31 ++++- core/unittest/batch/BatchStatusUnittest.cpp | 5 +- core/unittest/batch/BatcherUnittest.cpp | 63 +++++++++- core/unittest/compression/CMakeLists.txt | 4 + .../compression/CompressorFactoryUnittest.cpp | 42 +++++-- .../compression/CompressorUnittest.cpp | 79 ++++++++++++ .../compression/LZ4CompressorUnittest.cpp | 4 +- .../compression/ZstdCompressorUnittest.cpp | 4 +- core/unittest/flusher/FlusherSLSUnittest.cpp | 22 ++-- core/unittest/pipeline/PipelineUnittest.cpp | 29 +++-- core/unittest/route/RouterUnittest.cpp | 41 +++++++ core/unittest/serializer/CMakeLists.txt | 4 + .../serializer/SLSSerializerUnittest.cpp | 18 +-- .../serializer/SerializerUnittest.cpp | 114 ++++++++++++++++++ 47 files changed, 882 insertions(+), 154 deletions(-) rename core/{pipeline => common}/compression/CompressType.h (85%) create mode 100644 core/common/compression/Compressor.cpp rename core/{pipeline => common}/compression/Compressor.h (64%) rename core/{pipeline => common}/compression/CompressorFactory.cpp (71%) rename core/{pipeline => common}/compression/CompressorFactory.h (87%) rename core/{pipeline => common}/compression/LZ4Compressor.cpp (97%) rename core/{pipeline => common}/compression/LZ4Compressor.h (89%) rename core/{pipeline => common}/compression/ZstdCompressor.cpp (97%) rename core/{pipeline => common}/compression/ZstdCompressor.h (92%) create mode 100644 core/unittest/compression/CompressorUnittest.cpp create mode 100644 core/unittest/serializer/SerializerUnittest.cpp diff --git a/core/CMakeLists.txt b/core/CMakeLists.txt index 162ed8a82b..6c36c26eeb 100644 --- a/core/CMakeLists.txt +++ b/core/CMakeLists.txt @@ -113,7 +113,7 @@ include(${CMAKE_CURRENT_SOURCE_DIR}/plugin/flusher/links.cmake) set(SUB_DIRECTORIES_LIST application app_config checkpoint container_manager logger go_pipeline monitor profile_sender models config config/feedbacker config/provider config/watcher - pipeline pipeline/batch pipeline/compression pipeline/limiter pipeline/plugin pipeline/plugin/creator pipeline/plugin/instance pipeline/plugin/interface pipeline/queue pipeline/route pipeline/serializer + pipeline pipeline/batch pipeline/limiter pipeline/plugin pipeline/plugin/creator pipeline/plugin/instance pipeline/plugin/interface pipeline/queue pipeline/route pipeline/serializer runner runner/sink/http protobuf/config_server/v1 protobuf/config_server/v2 protobuf/sls file_server file_server/event file_server/event_handler file_server/event_listener file_server/reader file_server/polling diff --git a/core/common/common.cmake b/core/common/common.cmake index e905a95fa3..caf8e45b98 100644 --- a/core/common/common.cmake +++ b/core/common/common.cmake @@ -30,7 +30,7 @@ list(APPEND THIS_SOURCE_FILES_LIST ${XX_HASH_SOURCE_FILES}) list(APPEND THIS_SOURCE_FILES_LIST ${CMAKE_SOURCE_DIR}/common/memory/SourceBuffer.h) list(APPEND THIS_SOURCE_FILES_LIST ${CMAKE_SOURCE_DIR}/common/http/AsynCurlRunner.cpp ${CMAKE_SOURCE_DIR}/common/http/Curl.cpp ${CMAKE_SOURCE_DIR}/common/http/HttpResponse.cpp) list(APPEND THIS_SOURCE_FILES_LIST ${CMAKE_SOURCE_DIR}/common/timer/Timer.cpp ${CMAKE_SOURCE_DIR}/common/timer/HttpRequestTimerEvent.cpp) - +list(APPEND THIS_SOURCE_FILES_LIST ${CMAKE_SOURCE_DIR}/common/compression/Compressor.cpp ${CMAKE_SOURCE_DIR}/common/compression/CompressorFactory.cpp ${CMAKE_SOURCE_DIR}/common/compression/LZ4Compressor.cpp ${CMAKE_SOURCE_DIR}/common/compression/ZstdCompressor.cpp) # remove several files in common list(REMOVE_ITEM THIS_SOURCE_FILES_LIST ${CMAKE_SOURCE_DIR}/common/BoostRegexValidator.cpp ${CMAKE_SOURCE_DIR}/common/GetUUID.cpp) diff --git a/core/pipeline/compression/CompressType.h b/core/common/compression/CompressType.h similarity index 85% rename from core/pipeline/compression/CompressType.h rename to core/common/compression/CompressType.h index 96f996ee9a..1580bb5967 100644 --- a/core/pipeline/compression/CompressType.h +++ b/core/common/compression/CompressType.h @@ -18,6 +18,14 @@ namespace logtail { -enum class CompressType { NONE, LZ4, ZSTD }; +enum class CompressType { + NONE, + LZ4, + ZSTD +#ifdef APSARA_UNIT_TEST_MAIN + , + MOCK +#endif +}; } // namespace logtail diff --git a/core/common/compression/Compressor.cpp b/core/common/compression/Compressor.cpp new file mode 100644 index 0000000000..39a7af4b57 --- /dev/null +++ b/core/common/compression/Compressor.cpp @@ -0,0 +1,59 @@ +// Copyright 2024 iLogtail Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "common/compression/Compressor.h" + +#include + +#include "monitor/MetricConstants.h" + +using namespace std; + +namespace logtail { + +void Compressor::SetMetricRecordRef(MetricLabels&& labels, DynamicMetricLabels&& dynamicLabels) { + WriteMetrics::GetInstance()->PrepareMetricsRecordRef( + mMetricsRecordRef, std::move(labels), std::move(dynamicLabels)); + mInItemsCnt = mMetricsRecordRef.CreateCounter(METRIC_IN_ITEMS_CNT); + mInItemSizeBytes = mMetricsRecordRef.CreateCounter(METRIC_IN_ITEM_SIZE_BYTES); + mOutItemsCnt = mMetricsRecordRef.CreateCounter(METRIC_OUT_ITEMS_CNT); + mOutItemSizeBytes = mMetricsRecordRef.CreateCounter(METRIC_OUT_ITEM_SIZE_BYTES); + mDiscardedItemsCnt = mMetricsRecordRef.CreateCounter("discarded_items_cnt"); + mDiscardedItemSizeBytes = mMetricsRecordRef.CreateCounter("discarded_item_size_bytes"); + mTotalDelayMs = mMetricsRecordRef.CreateCounter(METRIC_TOTAL_DELAY_MS); +} + +bool Compressor::DoCompress(const string& input, string& output, string& errorMsg) { + if (mMetricsRecordRef != nullptr) { + mInItemsCnt->Add(1); + mInItemSizeBytes->Add(input.size()); + } + + auto before = chrono::system_clock::now(); + auto res = Compress(input, output, errorMsg); + + if (mMetricsRecordRef != nullptr) { + mTotalDelayMs->Add(chrono::duration_cast(chrono::system_clock::now() - before).count()); + if (res) { + mOutItemsCnt->Add(1); + mOutItemSizeBytes->Add(output.size()); + } else { + mDiscardedItemsCnt->Add(1); + mDiscardedItemSizeBytes->Add(input.size()); + } + } + return res; +} + +} // namespace logtail diff --git a/core/pipeline/compression/Compressor.h b/core/common/compression/Compressor.h similarity index 64% rename from core/pipeline/compression/Compressor.h rename to core/common/compression/Compressor.h index 1694f6e6ad..0c32358ca5 100644 --- a/core/pipeline/compression/Compressor.h +++ b/core/common/compression/Compressor.h @@ -18,7 +18,8 @@ #include -#include "pipeline/compression/CompressType.h" +#include "monitor/LogtailMetric.h" +#include "common/compression/CompressType.h" namespace logtail { @@ -27,7 +28,7 @@ class Compressor { Compressor(CompressType type) : mType(type) {} virtual ~Compressor() = default; - virtual bool Compress(const std::string& input, std::string& output, std::string& errorMsg) = 0; + bool DoCompress(const std::string& input, std::string& output, std::string& errorMsg); #ifdef APSARA_UNIT_TEST_MAIN // buffer shoudl be reserved for output before calling this function @@ -35,9 +36,27 @@ class Compressor { #endif CompressType GetCompressType() const { return mType; } + void SetMetricRecordRef(MetricLabels&& labels, DynamicMetricLabels&& dynamicLabels = {}); + +protected: + mutable MetricsRecordRef mMetricsRecordRef; + CounterPtr mInItemsCnt; + CounterPtr mInItemSizeBytes; + CounterPtr mOutItemsCnt; + CounterPtr mOutItemSizeBytes; + CounterPtr mDiscardedItemsCnt; + CounterPtr mDiscardedItemSizeBytes; + CounterPtr mTotalDelayMs; private: + virtual bool Compress(const std::string& input, std::string& output, std::string& errorMsg) = 0; + CompressType mType = CompressType::NONE; + +#ifdef APSARA_UNIT_TEST_MAIN + friend class CompressorUnittest; + friend class CompressorFactoryUnittest; +#endif }; } // namespace logtail diff --git a/core/pipeline/compression/CompressorFactory.cpp b/core/common/compression/CompressorFactory.cpp similarity index 71% rename from core/pipeline/compression/CompressorFactory.cpp rename to core/common/compression/CompressorFactory.cpp index 1c91f24035..0e3b27197b 100644 --- a/core/pipeline/compression/CompressorFactory.cpp +++ b/core/common/compression/CompressorFactory.cpp @@ -12,21 +12,24 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include "pipeline/compression/CompressorFactory.h" +#include "common/compression/CompressorFactory.h" #include "common/ParamExtractor.h" -#include "pipeline/compression/LZ4Compressor.h" -#include "pipeline/compression/ZstdCompressor.h" +#include "monitor/MetricConstants.h" +#include "common/compression/LZ4Compressor.h" +#include "common/compression/ZstdCompressor.h" using namespace std; namespace logtail { unique_ptr CompressorFactory::Create(const Json::Value& config, - const PipelineContext& ctx, - const string& pluginType, - CompressType defaultType) { + const PipelineContext& ctx, + const string& pluginType, + const string& flusherId, + CompressType defaultType) { string compressType, errorMsg; + unique_ptr compressor; if (!GetOptionalStringParam(config, "CompressType", compressType, errorMsg)) { PARAM_WARNING_DEFAULT(ctx.GetLogger(), ctx.GetAlarm(), @@ -37,11 +40,11 @@ unique_ptr CompressorFactory::Create(const Json::Value& config, ctx.GetProjectName(), ctx.GetLogstoreName(), ctx.GetRegion()); - return Create(defaultType); + compressor = Create(defaultType); } else if (compressType == "lz4") { - return Create(CompressType::LZ4); + compressor = Create(CompressType::LZ4); } else if (compressType == "zstd") { - return Create(CompressType::ZSTD); + compressor = Create(CompressType::ZSTD); } else if (compressType == "none") { return nullptr; } else if (!compressType.empty()) { @@ -54,10 +57,15 @@ unique_ptr CompressorFactory::Create(const Json::Value& config, ctx.GetProjectName(), ctx.GetLogstoreName(), ctx.GetRegion()); - return Create(defaultType); + compressor = Create(defaultType); } else { - return Create(defaultType); + compressor = Create(defaultType); } + compressor->SetMetricRecordRef({{METRIC_LABEL_PROJECT, ctx.GetProjectName()}, + {METRIC_LABEL_CONFIG_NAME, ctx.GetConfigName()}, + {METRIC_LABEL_KEY_COMPONENT_NAME, "compressor"}, + {METRIC_LABEL_KEY_FLUSHER_NODE_ID, flusherId}}); + return compressor; } unique_ptr CompressorFactory::Create(CompressType type) { diff --git a/core/pipeline/compression/CompressorFactory.h b/core/common/compression/CompressorFactory.h similarity index 87% rename from core/pipeline/compression/CompressorFactory.h rename to core/common/compression/CompressorFactory.h index dbdc826f16..8311332b7a 100644 --- a/core/pipeline/compression/CompressorFactory.h +++ b/core/common/compression/CompressorFactory.h @@ -21,9 +21,9 @@ #include #include -#include "pipeline/compression/CompressType.h" -#include "pipeline/compression/Compressor.h" #include "pipeline/PipelineContext.h" +#include "common/compression/CompressType.h" +#include "common/compression/Compressor.h" namespace logtail { @@ -42,13 +42,13 @@ class CompressorFactory { std::unique_ptr Create(const Json::Value& config, const PipelineContext& ctx, const std::string& pluginType, + const std::string& flusherId, CompressType defaultType); + std::unique_ptr Create(CompressType type); private: CompressorFactory() = default; ~CompressorFactory() = default; - - std::unique_ptr Create(CompressType defaultType); }; } // namespace logtail diff --git a/core/pipeline/compression/LZ4Compressor.cpp b/core/common/compression/LZ4Compressor.cpp similarity index 97% rename from core/pipeline/compression/LZ4Compressor.cpp rename to core/common/compression/LZ4Compressor.cpp index 7063812610..ec54c1c4e4 100644 --- a/core/pipeline/compression/LZ4Compressor.cpp +++ b/core/common/compression/LZ4Compressor.cpp @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include "pipeline/compression/LZ4Compressor.h" +#include "common/compression/LZ4Compressor.h" #include diff --git a/core/pipeline/compression/LZ4Compressor.h b/core/common/compression/LZ4Compressor.h similarity index 89% rename from core/pipeline/compression/LZ4Compressor.h rename to core/common/compression/LZ4Compressor.h index 6a64ea2af6..9fd453c163 100644 --- a/core/pipeline/compression/LZ4Compressor.h +++ b/core/common/compression/LZ4Compressor.h @@ -16,19 +16,20 @@ #pragma once -#include "pipeline/compression/Compressor.h" +#include "common/compression/Compressor.h" namespace logtail { class LZ4Compressor : public Compressor { public: - LZ4Compressor(CompressType type) : Compressor(type){}; + LZ4Compressor(CompressType type) : Compressor(type) {}; - bool Compress(const std::string& input, std::string& output, std::string& errorMsg) override; - #ifdef APSARA_UNIT_TEST_MAIN bool UnCompress(const std::string& input, std::string& output, std::string& errorMsg) override; #endif + +private: + bool Compress(const std::string& input, std::string& output, std::string& errorMsg) override; }; } // namespace logtail diff --git a/core/pipeline/compression/ZstdCompressor.cpp b/core/common/compression/ZstdCompressor.cpp similarity index 97% rename from core/pipeline/compression/ZstdCompressor.cpp rename to core/common/compression/ZstdCompressor.cpp index 6e47985b5e..c482c569a1 100644 --- a/core/pipeline/compression/ZstdCompressor.cpp +++ b/core/common/compression/ZstdCompressor.cpp @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include "pipeline/compression/ZstdCompressor.h" +#include "common/compression/ZstdCompressor.h" #include diff --git a/core/pipeline/compression/ZstdCompressor.h b/core/common/compression/ZstdCompressor.h similarity index 92% rename from core/pipeline/compression/ZstdCompressor.h rename to core/common/compression/ZstdCompressor.h index 1e910a2b0d..bf53c7d96d 100644 --- a/core/pipeline/compression/ZstdCompressor.h +++ b/core/common/compression/ZstdCompressor.h @@ -16,21 +16,21 @@ #pragma once -#include "pipeline/compression/Compressor.h" +#include "common/compression/Compressor.h" namespace logtail { class ZstdCompressor : public Compressor { public: - ZstdCompressor(CompressType type, int32_t level = 1) : Compressor(type), mCompressionLevel(level){}; - - bool Compress(const std::string& input, std::string& output, std::string& errorMsg) override; + ZstdCompressor(CompressType type, int32_t level = 1) : Compressor(type), mCompressionLevel(level) {}; #ifdef APSARA_UNIT_TEST_MAIN bool UnCompress(const std::string& input, std::string& output, std::string& errorMsg) override; #endif private: + bool Compress(const std::string& input, std::string& output, std::string& errorMsg) override; + int32_t mCompressionLevel = 1; }; diff --git a/core/go_pipeline/LogtailPlugin.cpp b/core/go_pipeline/LogtailPlugin.cpp index a84ff893c9..bb90dcdecc 100644 --- a/core/go_pipeline/LogtailPlugin.cpp +++ b/core/go_pipeline/LogtailPlugin.cpp @@ -22,15 +22,15 @@ #include "common/JsonUtil.h" #include "common/LogtailCommonFlags.h" #include "common/TimeUtil.h" -#include "pipeline/compression/CompressorFactory.h" +#include "common/compression/CompressorFactory.h" #include "container_manager/ConfigContainerInfoUpdateCmd.h" #include "file_server/ConfigManager.h" #include "logger/Logger.h" #include "monitor/LogFileProfiler.h" #include "monitor/LogtailAlarm.h" #include "pipeline/PipelineManager.h" -#include "profile_sender/ProfileSender.h" #include "pipeline/queue/SenderQueueManager.h" +#include "profile_sender/ProfileSender.h" DEFINE_FLAG_BOOL(enable_sls_metrics_format, "if enable format metrics in SLS metricstore log pattern", false); DEFINE_FLAG_BOOL(enable_containerd_upper_dir_detect, @@ -53,16 +53,13 @@ LogtailPlugin::LogtailPlugin() { mPluginValid = false; mPluginAlarmConfig.mLogstore = "logtail_alarm"; mPluginAlarmConfig.mAliuid = STRING_FLAG(logtail_profile_aliuid); - mPluginAlarmConfig.mCompressor - = CompressorFactory::GetInstance()->Create(Json::Value(), PipelineContext(), "flusher_sls", CompressType::ZSTD); + mPluginAlarmConfig.mCompressor = CompressorFactory::GetInstance()->Create(CompressType::ZSTD); mPluginProfileConfig.mLogstore = "shennong_log_profile"; mPluginProfileConfig.mAliuid = STRING_FLAG(logtail_profile_aliuid); - mPluginProfileConfig.mCompressor - = CompressorFactory::GetInstance()->Create(Json::Value(), PipelineContext(), "flusher_sls", CompressType::ZSTD); + mPluginProfileConfig.mCompressor = CompressorFactory::GetInstance()->Create(CompressType::ZSTD); mPluginContainerConfig.mLogstore = "logtail_containers"; mPluginContainerConfig.mAliuid = STRING_FLAG(logtail_profile_aliuid); - mPluginContainerConfig.mCompressor - = CompressorFactory::GetInstance()->Create(Json::Value(), PipelineContext(), "flusher_sls", CompressType::ZSTD); + mPluginContainerConfig.mCompressor = CompressorFactory::GetInstance()->Create(CompressType::ZSTD); mPluginCfg["LogtailSysConfDir"] = AppConfig::GetInstance()->GetLogtailSysConfDir(); mPluginCfg["HostIP"] = LogFileProfiler::mIpAddr; @@ -470,7 +467,8 @@ void LogtailPlugin::ProcessLogGroup(const std::string& configName, } } -void LogtailPlugin::GetGoMetrics(std::vector>& metircsList, const string& metricType) { +void LogtailPlugin::GetGoMetrics(std::vector>& metircsList, + const string& metricType) { if (mGetGoMetricsFun != nullptr) { GoString type; type.n = metricType.size(); diff --git a/core/monitor/LogtailMetric.h b/core/monitor/LogtailMetric.h index d6d76666ed..6c98081b2a 100644 --- a/core/monitor/LogtailMetric.h +++ b/core/monitor/LogtailMetric.h @@ -58,6 +58,8 @@ class MetricsRecord { class MetricsRecordRef { friend class WriteMetrics; + friend bool operator==(const MetricsRecordRef& lhs, std::nullptr_t rhs); + friend bool operator==(std::nullptr_t rhs, const MetricsRecordRef& lhs); private: MetricsRecord* mMetrics = nullptr; @@ -83,6 +85,22 @@ class MetricsRecordRef { #endif }; +inline bool operator==(const MetricsRecordRef& lhs, std::nullptr_t rhs) { + return lhs.mMetrics == rhs; +} + +inline bool operator==(std::nullptr_t lhs, const MetricsRecordRef& rhs) { + return lhs == rhs.mMetrics; +} + +inline bool operator!=(const MetricsRecordRef& lhs, std::nullptr_t rhs) { + return !(lhs == rhs); +} + +inline bool operator!=(std::nullptr_t lhs, const MetricsRecordRef& rhs) { + return !(lhs == rhs); +} + class ReentrantMetricsRecord { private: MetricsRecordRef mMetricsRecordRef; diff --git a/core/monitor/MetricConstants.cpp b/core/monitor/MetricConstants.cpp index 52c8d1158b..24212c9d3c 100644 --- a/core/monitor/MetricConstants.cpp +++ b/core/monitor/MetricConstants.cpp @@ -137,8 +137,6 @@ const std::string METRIC_PROC_PARSE_STDOUT_TOTAL = "proc_parse_stdout_total"; const std::string METRIC_PROC_PARSE_STDERR_TOTAL = "proc_parse_stderr_total"; // flusher common metrics -const std::string METRIC_FLUSHER_IN_RECORDS_TOTAL = "flusher_in_records_total"; -const std::string METRIC_FLUSHER_IN_RECORDS_SIZE_BYTES = "flusher_in_records_size_bytes"; const std::string METRIC_FLUSHER_ERROR_TOTAL = "flusher_error_total"; const std::string METRIC_FLUSHER_DISCARD_RECORDS_TOTAL = "flusher_discard_records_total"; const std::string METRIC_FLUSHER_SUCCESS_RECORDS_TOTAL = "flusher_success_records_total"; @@ -151,4 +149,14 @@ const std::string METRIC_FLUSHER_QUOTA_ERROR_TOTAL = "flusher_quota_error_total" const std::string METRIC_FLUSHER_RETRIES_TOTAL = "flusher_retries_total"; const std::string METRIC_FLUSHER_RETRIES_ERROR_TOTAL = "flusher_retries_error_total"; +const std::string METRIC_IN_EVENTS_CNT = "in_events_cnt"; +const std::string METRIC_IN_ITEMS_CNT = "in_items_cnt"; +const std::string METRIC_IN_EVENT_GROUP_SIZE_BYTES = "in_event_group_data_size_bytes"; +const std::string METRIC_IN_ITEM_SIZE_BYTES = "in_item_data_size_bytes"; +const std::string METRIC_OUT_EVENTS_CNT = "out_events_cnt"; +const std::string METRIC_OUT_ITEMS_CNT = "out_items_cnt"; +const std::string METRIC_OUT_EVENT_GROUP_SIZE_BYTES = "out_event_group_data_size_bytes"; +const std::string METRIC_OUT_ITEM_SIZE_BYTES = "out_item_data_size_bytes"; +const std::string METRIC_TOTAL_DELAY_MS = "total_delay_ms"; + } // namespace logtail diff --git a/core/monitor/MetricConstants.h b/core/monitor/MetricConstants.h index 27f3a762e3..3b78a5eae0 100644 --- a/core/monitor/MetricConstants.h +++ b/core/monitor/MetricConstants.h @@ -130,8 +130,6 @@ extern const std::string METRIC_PROC_PARSE_STDOUT_TOTAL; extern const std::string METRIC_PROC_PARSE_STDERR_TOTAL; // flusher common metrics -extern const std::string METRIC_FLUSHER_IN_RECORDS_TOTAL; -extern const std::string METRIC_FLUSHER_IN_RECORDS_SIZE_BYTES; extern const std::string METRIC_FLUSHER_ERROR_TOTAL; extern const std::string METRIC_FLUSHER_DISCARD_RECORDS_TOTAL; extern const std::string METRIC_FLUSHER_SUCCESS_RECORDS_TOTAL; @@ -144,4 +142,14 @@ extern const std::string METRIC_FLUSHER_QUOTA_ERROR_TOTAL; extern const std::string METRIC_FLUSHER_RETRIES_TOTAL; extern const std::string METRIC_FLUSHER_RETRIES_ERROR_TOTAL; +extern const std::string METRIC_IN_EVENTS_CNT; +extern const std::string METRIC_IN_ITEMS_CNT; +extern const std::string METRIC_IN_EVENT_GROUP_SIZE_BYTES; +extern const std::string METRIC_IN_ITEM_SIZE_BYTES; +extern const std::string METRIC_OUT_EVENTS_CNT; +extern const std::string METRIC_OUT_ITEMS_CNT; +extern const std::string METRIC_OUT_EVENT_GROUP_SIZE_BYTES; +extern const std::string METRIC_OUT_ITEM_SIZE_BYTES; +extern const std::string METRIC_TOTAL_DELAY_MS; + } // namespace logtail diff --git a/core/pipeline/Pipeline.cpp b/core/pipeline/Pipeline.cpp index 511b642b96..7d38880084 100644 --- a/core/pipeline/Pipeline.cpp +++ b/core/pipeline/Pipeline.cpp @@ -16,6 +16,7 @@ #include "pipeline/Pipeline.h" +#include #include #include @@ -314,10 +315,19 @@ bool Pipeline::Init(PipelineConfig&& config) { ProcessQueueManager::GetInstance()->SetDownStreamQueues(mContext.GetProcessQueueKey(), std::move(senderQueues)); } + WriteMetrics::GetInstance()->PrepareMetricsRecordRef( + mMetricsRecordRef, {{METRIC_LABEL_PROJECT, mContext.GetProjectName()}, {METRIC_LABEL_CONFIG_NAME, mName}}); + mStartTime = mMetricsRecordRef.CreateIntGauge("start_time"); + mProcessorsInEventsCnt = mMetricsRecordRef.CreateCounter("processors_in_events_cnt"); + mProcessorsInGroupsCnt = mMetricsRecordRef.CreateCounter("processors_in_event_groups_cnt"); + mProcessorsInGroupDataSizeBytes = mMetricsRecordRef.CreateCounter("processors_in_event_group_data_size_bytes"); + mProcessorsTotalDelayMs = mMetricsRecordRef.CreateCounter("processors_total_delay_ms"); + return true; } void Pipeline::Start() { +#ifndef APSARA_UNIT_TEST_MAIN // TODO: 应该保证指定时间内返回，如果无法返回，将配置放入startDisabled里 for (const auto& flusher : mFlushers) { flusher->Start(); @@ -337,16 +347,27 @@ void Pipeline::Start() { input->Start(); } - LOG_INFO(sLogger, ("pipeline start", "succeeded")("config", mName)); + mStartTime->Set(chrono::duration_cast(chrono::system_clock::now().time_since_epoch()).count()); +#endif + LOG_INFO(sLogger, ("pipeline start", "succeeded")("config", mName)("ptr", mStartTime.get())); } void Pipeline::Process(vector& logGroupList, size_t inputIndex) { + for (const auto& logGroup : logGroupList) { + mProcessorsInEventsCnt->Add(logGroup.GetEvents().size()); + mProcessorsInGroupDataSizeBytes->Add(logGroup.DataSize()); + } + mProcessorsInGroupsCnt->Add(logGroupList.size()); + + auto before = chrono::system_clock::now(); for (auto& p : mInputs[inputIndex]->GetInnerProcessors()) { p->Process(logGroupList); } for (auto& p : mProcessorLine) { p->Process(logGroupList); } + mProcessorsTotalDelayMs->Add( + chrono::duration_cast(chrono::system_clock::now() - before).count()); } bool Pipeline::Send(vector&& groupList) { @@ -381,6 +402,7 @@ bool Pipeline::FlushBatch() { } void Pipeline::Stop(bool isRemoving) { +#ifndef APSARA_UNIT_TEST_MAIN // TODO: 应该保证指定时间内返回，如果无法返回，将配置放入stopDisabled里 for (const auto& input : mInputs) { input->Stop(isRemoving); @@ -403,7 +425,7 @@ void Pipeline::Stop(bool isRemoving) { for (const auto& flusher : mFlushers) { flusher->Stop(isRemoving); } - +#endif LOG_INFO(sLogger, ("pipeline stop", "succeeded")("config", mName)); } diff --git a/core/pipeline/Pipeline.h b/core/pipeline/Pipeline.h index 90bc5ada01..e3775f7b48 100644 --- a/core/pipeline/Pipeline.h +++ b/core/pipeline/Pipeline.h @@ -24,14 +24,15 @@ #include #include "config/PipelineConfig.h" -#include "plugin/input/InputContainerStdio.h" -#include "plugin/input/InputFile.h" #include "models/PipelineEventGroup.h" +#include "monitor/LogtailMetric.h" #include "pipeline/PipelineContext.h" #include "pipeline/plugin/instance/FlusherInstance.h" #include "pipeline/plugin/instance/InputInstance.h" #include "pipeline/plugin/instance/ProcessorInstance.h" #include "pipeline/route/Router.h" +#include "plugin/input/InputContainerStdio.h" +#include "plugin/input/InputFile.h" namespace logtail { @@ -84,8 +85,15 @@ class Pipeline { std::unique_ptr mConfig; std::atomic_uint16_t mPluginID; + mutable MetricsRecordRef mMetricsRecordRef; + IntGaugePtr mStartTime; + CounterPtr mProcessorsInEventsCnt; + CounterPtr mProcessorsInGroupsCnt; + CounterPtr mProcessorsInGroupDataSizeBytes; + CounterPtr mProcessorsTotalDelayMs; + #ifdef APSARA_UNIT_TEST_MAIN - friend class PipelineMock; + friend class PipelineMock; friend class PipelineUnittest; friend class InputContainerStdioUnittest; friend class InputFileUnittest; diff --git a/core/pipeline/batch/BatchItem.h b/core/pipeline/batch/BatchItem.h index b27fa0f5e8..9fa956dbef 100644 --- a/core/pipeline/batch/BatchItem.h +++ b/core/pipeline/batch/BatchItem.h @@ -20,11 +20,11 @@ #include #include +#include "models/PipelineEventGroup.h" +#include "models/StringView.h" #include "pipeline/batch/BatchStatus.h" #include "pipeline/batch/BatchedEvents.h" #include "pipeline/batch/FlushStrategy.h" -#include "models/PipelineEventGroup.h" -#include "models/StringView.h" namespace logtail { @@ -32,7 +32,9 @@ class GroupBatchItem { public: GroupBatchItem() { mStatus.Reset(); } - void Add(BatchedEvents&& g) { + void Add(BatchedEvents&& g, int64_t totalEnqueTimeMs) { + mEventsCnt += g.mEvents.size(); + mTotalEnqueTimeMs += totalEnqueTimeMs; mGroups.emplace_back(std::move(g)); mStatus.Update(mGroups.back()); } @@ -56,6 +58,10 @@ class GroupBatchItem { } GroupBatchStatus& GetStatus() { return mStatus; } + size_t GroupSize() const { return mGroups.size(); } + size_t EventSize() const { return mEventsCnt; } + size_t DataSize() const { return mStatus.GetSize(); } + int64_t TotalEnqueTimeMs() const { return mTotalEnqueTimeMs; } bool IsEmpty() { return mGroups.empty(); } @@ -63,11 +69,17 @@ class GroupBatchItem { void Clear() { mGroups.clear(); mStatus.Reset(); + mEventsCnt = 0; + mTotalEnqueTimeMs = 0; } std::vector mGroups; GroupBatchStatus mStatus; + size_t mEventsCnt = 0; + // if more than 10^6 events are contained in the batch, the value may overflow + // however, this is almost impossible in practice + int64_t mTotalEnqueTimeMs = 0; #ifdef APSARA_UNIT_TEST_MAIN friend class EventBatchItemUnittest; @@ -82,6 +94,9 @@ class EventBatchItem { void Add(PipelineEventPtr&& e) { mBatch.mEvents.emplace_back(std::move(e)); mStatus.Update(mBatch.mEvents.back()); + mTotalEnqueTimeMs += std::chrono::time_point_cast(std::chrono::system_clock::now()) + .time_since_epoch() + .count(); } void Flush(GroupBatchItem& res) { @@ -91,7 +106,8 @@ class EventBatchItem { if (mBatch.mExactlyOnceCheckpoint) { UpdateExactlyOnceLogPosition(); } - res.Add(std::move(mBatch)); + mBatch.mSizeBytes = DataSize(); + res.Add(std::move(mBatch), mTotalEnqueTimeMs); Clear(); } @@ -102,6 +118,7 @@ class EventBatchItem { if (mBatch.mExactlyOnceCheckpoint) { UpdateExactlyOnceLogPosition(); } + mBatch.mSizeBytes = DataSize(); res.emplace_back(std::move(mBatch)); Clear(); } @@ -114,6 +131,7 @@ class EventBatchItem { if (mBatch.mExactlyOnceCheckpoint) { UpdateExactlyOnceLogPosition(); } + mBatch.mSizeBytes = DataSize(); res.back().emplace_back(std::move(mBatch)); Clear(); } @@ -141,11 +159,16 @@ class EventBatchItem { bool IsEmpty() { return mBatch.mEvents.empty(); } + size_t DataSize() const { return sizeof(decltype(mBatch.mEvents)) + mStatus.GetSize() + mBatch.mTags.DataSize(); } + size_t EventSize() const { return mBatch.mEvents.size(); } + int64_t TotalEnqueTimeMs() const { return mTotalEnqueTimeMs; } + private: void Clear() { mBatch.Clear(); mSourceBuffers.clear(); mStatus.Reset(); + mTotalEnqueTimeMs = 0; } void UpdateExactlyOnceLogPosition() { @@ -159,6 +182,9 @@ class EventBatchItem { BatchedEvents mBatch; std::unordered_set mSourceBuffers; T mStatus; + // if more than 10^6 events are contained in the batch, the value may overflow + // however, this is almost impossible in practice + int64_t mTotalEnqueTimeMs = 0; #ifdef APSARA_UNIT_TEST_MAIN friend class EventBatchItemUnittest; diff --git a/core/pipeline/batch/BatchStatus.h b/core/pipeline/batch/BatchStatus.h index 18c2b7db6c..224a83dc78 100644 --- a/core/pipeline/batch/BatchStatus.h +++ b/core/pipeline/batch/BatchStatus.h @@ -67,7 +67,7 @@ class GroupBatchStatus { if (mCreateTime == 0) { mCreateTime = time(nullptr); } - mSizeBytes += g.DataSize(); + mSizeBytes += g.mSizeBytes; } uint32_t GetSize() const { return mSizeBytes; } diff --git a/core/pipeline/batch/BatchedEvents.h b/core/pipeline/batch/BatchedEvents.h index 9ec5d0ff37..e16d15be15 100644 --- a/core/pipeline/batch/BatchedEvents.h +++ b/core/pipeline/batch/BatchedEvents.h @@ -28,6 +28,7 @@ struct BatchedEvents { EventsContainer mEvents; SizedMap mTags; std::vector> mSourceBuffers; + size_t mSizeBytes = 0; // only set on completion // for flusher_sls only RangeCheckpointPtr mExactlyOnceCheckpoint; StringView mPackIdPrefix; @@ -45,20 +46,17 @@ struct BatchedEvents { mExactlyOnceCheckpoint(std::move(eoo)), mPackIdPrefix(packIdPrefix) { mSourceBuffers.emplace_back(std::move(sourceBuffer)); - } - - size_t DataSize() const { - size_t eventsSize = sizeof(decltype(mEvents)); + mSizeBytes = sizeof(decltype(mEvents)) + mTags.DataSize(); for (const auto& item : mEvents) { - eventsSize += item->DataSize(); + mSizeBytes += item->DataSize(); } - return eventsSize + mTags.DataSize(); } void Clear() { mEvents.clear(); mTags.Clear(); mSourceBuffers.clear(); + mSizeBytes = 0; mExactlyOnceCheckpoint.reset(); mPackIdPrefix = StringView(); } diff --git a/core/pipeline/batch/Batcher.h b/core/pipeline/batch/Batcher.h index b165bab370..ca01986ca5 100644 --- a/core/pipeline/batch/Batcher.h +++ b/core/pipeline/batch/Batcher.h @@ -24,14 +24,16 @@ #include #include -#include "pipeline/batch/BatchItem.h" -#include "pipeline/batch/BatchStatus.h" -#include "pipeline/batch/FlushStrategy.h" -#include "pipeline/batch/TimeoutFlushManager.h" #include "common/Flags.h" #include "common/ParamExtractor.h" #include "models/PipelineEventGroup.h" +#include "monitor/LogtailMetric.h" +#include "monitor/MetricConstants.h" #include "pipeline/PipelineContext.h" +#include "pipeline/batch/BatchItem.h" +#include "pipeline/batch/BatchStatus.h" +#include "pipeline/batch/FlushStrategy.h" +#include "pipeline/batch/TimeoutFlushManager.h" namespace logtail { @@ -97,6 +99,26 @@ class Batcher { mFlusher = flusher; + std::vector> labels{ + {METRIC_LABEL_PROJECT, ctx.GetProjectName()}, + {METRIC_LABEL_CONFIG_NAME, ctx.GetConfigName()}, + {METRIC_LABEL_KEY_COMPONENT_NAME, "batcher"}, + {METRIC_LABEL_KEY_FLUSHER_NODE_ID, flusher->GetNodeID()}}; + if (enableGroupBatch) { + labels.emplace_back("enable_group_batch", "true"); + } else { + labels.emplace_back("enable_group_batch", "false"); + } + WriteMetrics::GetInstance()->PrepareMetricsRecordRef(mMetricsRecordRef, std::move(labels)); + mInEventsCnt = mMetricsRecordRef.CreateCounter(METRIC_IN_EVENTS_CNT); + mInGroupDataSizeBytes = mMetricsRecordRef.CreateCounter(METRIC_IN_EVENT_GROUP_SIZE_BYTES); + mOutEventsCnt = mMetricsRecordRef.CreateCounter(METRIC_OUT_EVENTS_CNT); + mTotalDelayMs = mMetricsRecordRef.CreateCounter(METRIC_TOTAL_DELAY_MS); + mEventBatchItemsCnt = mMetricsRecordRef.CreateIntGauge("event_batches_cnt"); + mBufferedGroupsCnt = mMetricsRecordRef.CreateIntGauge("buffered_groups_cnt"); + mBufferedEventsCnt = mMetricsRecordRef.CreateIntGauge("buffered_events_cnt"); + mBufferedDataSizeByte = mMetricsRecordRef.CreateIntGauge("buffered_data_size_bytes"); + return true; } @@ -105,15 +127,20 @@ class Batcher { std::lock_guard lock(mMux); size_t key = g.GetTagsHash(); EventBatchItem& item = mEventQueueMap[key]; + mInEventsCnt->Add(g.GetEvents().size()); + mInGroupDataSizeBytes->Add(g.DataSize()); + mEventBatchItemsCnt->Set(mEventQueueMap.size()); size_t eventsSize = g.GetEvents().size(); for (size_t i = 0; i < eventsSize; ++i) { PipelineEventPtr& e = g.MutableEvents()[i]; if (!item.IsEmpty() && mEventFlushStrategy.NeedFlushByTime(item.GetStatus(), e)) { if (!mGroupQueue) { + UpdateMetricsOnFlushingEventQueue(item); item.Flush(res); } else { if (!mGroupQueue->IsEmpty() && mGroupFlushStrategy->NeedFlushByTime(mGroupQueue->GetStatus())) { + UpdateMetricsOnFlushingGroupQueue(); mGroupQueue->Flush(res); } if (mGroupQueue->IsEmpty()) { @@ -125,6 +152,7 @@ class Batcher { } item.Flush(mGroupQueue.value()); if (mGroupFlushStrategy->NeedFlushBySize(mGroupQueue->GetStatus())) { + UpdateMetricsOnFlushingGroupQueue(); mGroupQueue->Flush(res); } } @@ -136,12 +164,17 @@ class Batcher { g.GetMetadata(EventGroupMetaKey::SOURCE_ID)); TimeoutFlushManager::GetInstance()->UpdateRecord( mFlusher->GetContext().GetConfigName(), 0, key, mEventFlushStrategy.GetTimeoutSecs(), mFlusher); + mBufferedGroupsCnt->Add(1); + mBufferedDataSizeByte->Add(item.DataSize()); } else if (i == 0) { item.AddSourceBuffer(g.GetSourceBuffer()); } + mBufferedEventsCnt->Add(1); + mBufferedDataSizeByte->Add(e->DataSize()); item.Add(std::move(e)); if (mEventFlushStrategy.NeedFlushBySize(item.GetStatus()) || mEventFlushStrategy.NeedFlushByCnt(item.GetStatus())) { + UpdateMetricsOnFlushingEventQueue(item); item.Flush(res); } } @@ -155,6 +188,7 @@ class Batcher { if (!mGroupQueue) { return; } + UpdateMetricsOnFlushingGroupQueue(); return mGroupQueue->Flush(res); } @@ -164,12 +198,15 @@ class Batcher { } if (!mGroupQueue) { + UpdateMetricsOnFlushingEventQueue(iter->second); iter->second.Flush(res); mEventQueueMap.erase(iter); + mEventBatchItemsCnt->Set(mEventQueueMap.size()); return; } if (!mGroupQueue->IsEmpty() && mGroupFlushStrategy->NeedFlushByTime(mGroupQueue->GetStatus())) { + UpdateMetricsOnFlushingGroupQueue(); mGroupQueue->Flush(res); } if (mGroupQueue->IsEmpty()) { @@ -178,7 +215,9 @@ class Batcher { } iter->second.Flush(mGroupQueue.value()); mEventQueueMap.erase(iter); + mEventBatchItemsCnt->Set(mEventQueueMap.size()); if (mGroupFlushStrategy->NeedFlushBySize(mGroupQueue->GetStatus())) { + UpdateMetricsOnFlushingGroupQueue(); mGroupQueue->Flush(res); } } @@ -187,20 +226,25 @@ class Batcher { std::lock_guard lock(mMux); for (auto& item : mEventQueueMap) { if (!mGroupQueue) { + UpdateMetricsOnFlushingEventQueue(item.second); item.second.Flush(res); } else { if (!mGroupQueue->IsEmpty() && mGroupFlushStrategy->NeedFlushByTime(mGroupQueue->GetStatus())) { + UpdateMetricsOnFlushingGroupQueue(); mGroupQueue->Flush(res); } item.second.Flush(mGroupQueue.value()); if (mGroupFlushStrategy->NeedFlushBySize(mGroupQueue->GetStatus())) { + UpdateMetricsOnFlushingGroupQueue(); mGroupQueue->Flush(res); } } } if (mGroupQueue) { + UpdateMetricsOnFlushingGroupQueue(); mGroupQueue->Flush(res); } + mEventBatchItemsCnt->Set(0); mEventQueueMap.clear(); } @@ -210,6 +254,32 @@ class Batcher { #endif private: + void UpdateMetricsOnFlushingEventQueue(const EventBatchItem& item) { + mOutEventsCnt->Add(item.EventSize()); + mTotalDelayMs->Add( + item.EventSize() + * std::chrono::time_point_cast(std::chrono::system_clock::now()) + .time_since_epoch() + .count() + - item.TotalEnqueTimeMs()); + mBufferedGroupsCnt->Sub(1); + mBufferedEventsCnt->Sub(item.EventSize()); + mBufferedDataSizeByte->Sub(item.DataSize()); + } + + void UpdateMetricsOnFlushingGroupQueue() { + mOutEventsCnt->Add(mGroupQueue->EventSize()); + mTotalDelayMs->Add( + mGroupQueue->EventSize() + * std::chrono::time_point_cast(std::chrono::system_clock::now()) + .time_since_epoch() + .count() + - mGroupQueue->TotalEnqueTimeMs()); + mBufferedGroupsCnt->Sub(mGroupQueue->GroupSize()); + mBufferedEventsCnt->Sub(mGroupQueue->EventSize()); + mBufferedDataSizeByte->Sub(mGroupQueue->DataSize()); + } + std::mutex mMux; std::map> mEventQueueMap; EventFlushStrategy mEventFlushStrategy; @@ -219,6 +289,16 @@ class Batcher { Flusher* mFlusher = nullptr; + mutable MetricsRecordRef mMetricsRecordRef; + CounterPtr mInEventsCnt; + CounterPtr mInGroupDataSizeBytes; + CounterPtr mOutEventsCnt; + CounterPtr mTotalDelayMs; + IntGaugePtr mEventBatchItemsCnt; + IntGaugePtr mBufferedGroupsCnt; + IntGaugePtr mBufferedEventsCnt; + IntGaugePtr mBufferedDataSizeByte; + #ifdef APSARA_UNIT_TEST_MAIN friend class BatcherUnittest; #endif diff --git a/core/pipeline/plugin/instance/FlusherInstance.cpp b/core/pipeline/plugin/instance/FlusherInstance.cpp index 9623a96b75..997f75fe4b 100644 --- a/core/pipeline/plugin/instance/FlusherInstance.cpp +++ b/core/pipeline/plugin/instance/FlusherInstance.cpp @@ -25,14 +25,14 @@ bool FlusherInstance::Init(const Json::Value& config, PipelineContext& context, return false; } - mFlusherInRecordsTotal = mPlugin->GetMetricsRecordRef().CreateCounter(METRIC_FLUSHER_IN_RECORDS_TOTAL); - mFlusherInRecordsSizeBytes = mPlugin->GetMetricsRecordRef().CreateCounter(METRIC_FLUSHER_IN_RECORDS_SIZE_BYTES); + mInEventsCnt = mPlugin->GetMetricsRecordRef().CreateCounter(METRIC_IN_EVENTS_CNT); + mInGroupDataSizeBytes = mPlugin->GetMetricsRecordRef().CreateCounter(METRIC_IN_EVENT_GROUP_SIZE_BYTES); return true; } bool FlusherInstance::Send(PipelineEventGroup&& g) { - mFlusherInRecordsTotal->Add(g.GetEvents().size()); - mFlusherInRecordsSizeBytes->Add(g.DataSize()); + mInEventsCnt->Add(g.GetEvents().size()); + mInGroupDataSizeBytes->Add(g.DataSize()); return mPlugin->Send(std::move(g)); } diff --git a/core/pipeline/plugin/instance/FlusherInstance.h b/core/pipeline/plugin/instance/FlusherInstance.h index 2900a594df..4abec0d766 100644 --- a/core/pipeline/plugin/instance/FlusherInstance.h +++ b/core/pipeline/plugin/instance/FlusherInstance.h @@ -46,8 +46,8 @@ class FlusherInstance : public PluginInstance { private: std::unique_ptr mPlugin; - CounterPtr mFlusherInRecordsTotal; - CounterPtr mFlusherInRecordsSizeBytes; + CounterPtr mInEventsCnt; + CounterPtr mInGroupDataSizeBytes; }; } // namespace logtail diff --git a/core/pipeline/plugin/instance/ProcessorInstance.cpp b/core/pipeline/plugin/instance/ProcessorInstance.cpp index 8790886cd4..70dbc5a172 100644 --- a/core/pipeline/plugin/instance/ProcessorInstance.cpp +++ b/core/pipeline/plugin/instance/ProcessorInstance.cpp @@ -49,11 +49,9 @@ void ProcessorInstance::Process(vector& logGroupList) { mProcInRecordsTotal->Add(logGroup.GetEvents().size()); } - uint64_t startTime = GetCurrentTimeInMilliSeconds(); + auto before = chrono::system_clock::now(); mPlugin->Process(logGroupList); - uint64_t durationTime = GetCurrentTimeInMilliSeconds() - startTime; - - mProcTimeMS->Add(durationTime); + mProcTimeMS->Add(chrono::duration_cast(chrono::system_clock::now() - before).count()); for (const auto& logGroup : logGroupList) { mProcOutRecordsTotal->Add(logGroup.GetEvents().size()); diff --git a/core/pipeline/queue/QueueInterface.h b/core/pipeline/queue/QueueInterface.h index 5c2e3b82cb..1c450013ad 100644 --- a/core/pipeline/queue/QueueInterface.h +++ b/core/pipeline/queue/QueueInterface.h @@ -33,10 +33,10 @@ class QueueInterface { {METRIC_LABEL_CONFIG_NAME, ctx.GetConfigName()}, }); - mInItemsCnt = mMetricsRecordRef.CreateCounter("in_items_cnt"); - mInItemDataSizeBytes = mMetricsRecordRef.CreateCounter("in_item_data_size_bytes"); - mOutItemsCnt = mMetricsRecordRef.CreateCounter("out_items_cnt"); - mTotalDelayMs = mMetricsRecordRef.CreateCounter("total_delay_ms"); + mInItemsCnt = mMetricsRecordRef.CreateCounter(METRIC_IN_ITEMS_CNT); + mInItemDataSizeBytes = mMetricsRecordRef.CreateCounter(METRIC_IN_ITEM_SIZE_BYTES); + mOutItemsCnt = mMetricsRecordRef.CreateCounter(METRIC_OUT_ITEMS_CNT); + mTotalDelayMs = mMetricsRecordRef.CreateCounter(METRIC_TOTAL_DELAY_MS); mQueueSize = mMetricsRecordRef.CreateIntGauge("queue_size"); mQueueDataSizeByte = mMetricsRecordRef.CreateIntGauge("queue_data_size_bytes"); } diff --git a/core/pipeline/route/Router.cpp b/core/pipeline/route/Router.cpp index e9c4a817cd..73de16bcda 100644 --- a/core/pipeline/route/Router.cpp +++ b/core/pipeline/route/Router.cpp @@ -15,6 +15,7 @@ #include "pipeline/route/Router.h" #include "common/ParamExtractor.h" +#include "monitor/MetricConstants.h" #include "pipeline/Pipeline.h" #include "pipeline/plugin/interface/Flusher.h" @@ -33,10 +34,20 @@ bool Router::Init(std::vector> configs, const P mAlwaysMatchedFlusherIdx.push_back(item.first); } } + + WriteMetrics::GetInstance()->PrepareMetricsRecordRef(mMetricsRecordRef, + {{METRIC_LABEL_PROJECT, ctx.GetProjectName()}, + {METRIC_LABEL_CONFIG_NAME, ctx.GetConfigName()}, + {METRIC_LABEL_KEY_COMPONENT_NAME, "router"}}); + mInEventsCnt = mMetricsRecordRef.CreateCounter(METRIC_IN_EVENTS_CNT); + mInGroupDataSizeBytes = mMetricsRecordRef.CreateCounter(METRIC_IN_EVENT_GROUP_SIZE_BYTES); return true; } vector Router::Route(const PipelineEventGroup& g) const { + mInEventsCnt->Add(g.GetEvents().size()); + mInGroupDataSizeBytes->Add(g.DataSize()); + vector res(mAlwaysMatchedFlusherIdx); for (size_t i = 0; i < mConditions.size(); ++i) { if (mConditions[i].second.Check(g)) { diff --git a/core/pipeline/route/Router.h b/core/pipeline/route/Router.h index c46dedca6e..b036256551 100644 --- a/core/pipeline/route/Router.h +++ b/core/pipeline/route/Router.h @@ -22,6 +22,7 @@ #include #include "models/PipelineEventGroup.h" +#include "monitor/LogtailMetric.h" #include "pipeline/route/Condition.h" namespace logtail { @@ -37,6 +38,10 @@ class Router { std::vector> mConditions; std::vector mAlwaysMatchedFlusherIdx; + mutable MetricsRecordRef mMetricsRecordRef; + CounterPtr mInEventsCnt; + CounterPtr mInGroupDataSizeBytes; + #ifdef APSARA_UNIT_TEST_MAIN friend class RouterUnittest; friend class PipelineUnittest; diff --git a/core/pipeline/serializer/SLSSerializer.cpp b/core/pipeline/serializer/SLSSerializer.cpp index 600e709f68..f24030c100 100644 --- a/core/pipeline/serializer/SLSSerializer.cpp +++ b/core/pipeline/serializer/SLSSerializer.cpp @@ -17,14 +17,14 @@ #include "application/Application.h" #include "common/Flags.h" #include "common/TimeUtil.h" -#include "pipeline/compression/CompressType.h" +#include "common/compression/CompressType.h" #include "plugin/flusher/sls/FlusherSLS.h" DEFINE_FLAG_INT32(max_send_log_group_size, "bytes", 10 * 1024 * 1024); const std::string METRIC_RESERVED_KEY_NAME = "__name__"; -const std::string METRIC_RESERVED_KEY_LABELS = "__labels__"; +const std::string METRIC_RESERVED_KEY_LABELS = "__labels__"; const std::string METRIC_RESERVED_KEY_VALUE = "__value__"; const std::string METRIC_RESERVED_KEY_TIME_NANO = "__time_nano__"; @@ -35,7 +35,31 @@ using namespace std; namespace logtail { +template <> +bool Serializer>::DoSerialize(vector&& p, + std::string& output, + std::string& errorMsg) { + auto inputSize = 0; + for (auto& item : p) { + inputSize += item.mData.size(); + } + mInItemsCnt->Add(1); + mInItemSizeBytes->Add(inputSize); + auto before = std::chrono::system_clock::now(); + auto res = Serialize(std::move(p), output, errorMsg); + mTotalDelayMs->Add( + std::chrono::duration_cast(std::chrono::system_clock::now() - before).count()); + + if (res) { + mOutItemsCnt->Add(1); + mOutItemSizeBytes->Add(output.size()); + } else { + mDiscardedItemsCnt->Add(1); + mDiscardedItemSizeBytes->Add(inputSize); + } + return res; +} bool SLSEventGroupSerializer::Serialize(BatchedEvents&& group, string& res, string& errorMsg) { sls_logs::LogGroup logGroup; @@ -56,7 +80,7 @@ bool SLSEventGroupSerializer::Serialize(BatchedEvents&& group, string& res, stri } else if (e.Is()) { const auto& metricEvent = e.Cast(); if (metricEvent.Is()) { - continue; + continue; } auto log = logGroup.add_logs(); std::ostringstream oss; @@ -78,9 +102,10 @@ bool SLSEventGroupSerializer::Serialize(BatchedEvents&& group, string& res, stri logPtr = log->add_contents(); logPtr->set_key(METRIC_RESERVED_KEY_TIME_NANO); if (metricEvent.GetTimestampNanosecond()) { - logPtr->set_value(std::to_string(metricEvent.GetTimestamp()) + NumberToDigitString(metricEvent.GetTimestampNanosecond().value(), 9)); + logPtr->set_value(std::to_string(metricEvent.GetTimestamp()) + + NumberToDigitString(metricEvent.GetTimestampNanosecond().value(), 9)); } else { - logPtr->set_value(std::to_string(metricEvent.GetTimestamp())); + logPtr->set_value(std::to_string(metricEvent.GetTimestamp())); } // set __value__ if (metricEvent.Is()) { @@ -88,7 +113,7 @@ bool SLSEventGroupSerializer::Serialize(BatchedEvents&& group, string& res, stri logPtr = log->add_contents(); logPtr->set_key(METRIC_RESERVED_KEY_VALUE); logPtr->set_value(std::to_string(value)); - } + } // set __name__ logPtr = log->add_contents(); logPtr->set_key(METRIC_RESERVED_KEY_NAME); @@ -122,9 +147,7 @@ bool SLSEventGroupSerializer::Serialize(BatchedEvents&& group, string& res, stri return true; } -bool SLSEventGroupListSerializer::Serialize(vector&& v, - string& res, - string& errorMsg) { +bool SLSEventGroupListSerializer::Serialize(vector&& v, string& res, string& errorMsg) { sls_logs::SlsLogPackageList logPackageList; for (const auto& item : v) { auto package = logPackageList.add_packages(); diff --git a/core/pipeline/serializer/SLSSerializer.h b/core/pipeline/serializer/SLSSerializer.h index f19d0f8e3a..c9bdb56798 100644 --- a/core/pipeline/serializer/SLSSerializer.h +++ b/core/pipeline/serializer/SLSSerializer.h @@ -27,6 +27,7 @@ class SLSEventGroupSerializer : public Serializer { public: SLSEventGroupSerializer(Flusher* f) : Serializer(f) {} +private: bool Serialize(BatchedEvents&& p, std::string& res, std::string& errorMsg) override; }; @@ -37,10 +38,14 @@ struct CompressedLogGroup { CompressedLogGroup(std::string&& data, size_t rawSize) : mData(std::move(data)), mRawSize(rawSize) {} }; +template<> +bool Serializer>::DoSerialize(std::vector&& p, std::string& output, std::string& errorMsg); + class SLSEventGroupListSerializer : public Serializer> { public: SLSEventGroupListSerializer(Flusher* f) : Serializer>(f) {} +private: bool Serialize(std::vector&& v, std::string& res, std::string& errorMsg) override; }; diff --git a/core/pipeline/serializer/Serializer.h b/core/pipeline/serializer/Serializer.h index 968f882a91..b632124b7a 100644 --- a/core/pipeline/serializer/Serializer.h +++ b/core/pipeline/serializer/Serializer.h @@ -16,27 +16,92 @@ #pragma once +#include #include -#include "pipeline/batch/BatchedEvents.h" #include "models/PipelineEventPtr.h" +#include "monitor/MetricConstants.h" +#include "pipeline/batch/BatchedEvents.h" #include "pipeline/plugin/interface/Flusher.h" namespace logtail { +inline size_t GetInputSize(const PipelineEventPtr& p) { + return p->DataSize(); +} + +inline size_t GetInputSize(const BatchedEvents& p) { + return p.mSizeBytes; +} + +inline size_t GetInputSize(const BatchedEventsList& p) { + size_t size = 0; + for (const auto& e : p) { + size += GetInputSize(e); + } + return size; +} + // T: PipelineEventPtr, BatchedEvents, BatchedEventsList template class Serializer { public: - Serializer() = default; - Serializer(Flusher* f) : mFlusher(f) {} + Serializer(Flusher* f) : mFlusher(f) { + WriteMetrics::GetInstance()->PrepareMetricsRecordRef( + mMetricsRecordRef, + {{METRIC_LABEL_PROJECT, f->GetContext().GetProjectName()}, + {METRIC_LABEL_CONFIG_NAME, f->GetContext().GetConfigName()}, + {METRIC_LABEL_KEY_COMPONENT_NAME, "serializer"}, + {METRIC_LABEL_KEY_FLUSHER_NODE_ID, f->GetNodeID()}}); + mInItemsCnt = mMetricsRecordRef.CreateCounter(METRIC_IN_ITEMS_CNT); + mInItemSizeBytes = mMetricsRecordRef.CreateCounter(METRIC_IN_ITEM_SIZE_BYTES); + mOutItemsCnt = mMetricsRecordRef.CreateCounter(METRIC_OUT_ITEMS_CNT); + mOutItemSizeBytes = mMetricsRecordRef.CreateCounter(METRIC_OUT_ITEM_SIZE_BYTES); + mDiscardedItemsCnt = mMetricsRecordRef.CreateCounter("discarded_items_cnt"); + mDiscardedItemSizeBytes = mMetricsRecordRef.CreateCounter("discarded_item_data_size_bytes"); + mTotalDelayMs = mMetricsRecordRef.CreateCounter(METRIC_TOTAL_DELAY_MS); + } virtual ~Serializer() = default; - virtual bool Serialize(T&& p, std::string& res, std::string& errorMsg) = 0; + bool DoSerialize(T&& p, std::string& output, std::string& errorMsg) { + auto inputSize = GetInputSize(p); + mInItemsCnt->Add(1); + mInItemSizeBytes->Add(inputSize); + + auto before = std::chrono::system_clock::now(); + auto res = Serialize(std::move(p), output, errorMsg); + mTotalDelayMs->Add( + std::chrono::duration_cast(std::chrono::system_clock::now() - before).count()); + + if (res) { + mOutItemsCnt->Add(1); + mOutItemSizeBytes->Add(output.size()); + } else { + mDiscardedItemsCnt->Add(1); + mDiscardedItemSizeBytes->Add(inputSize); + } + return res; + } protected: // if serialized output contains output related info, it can be obtained via this member const Flusher* mFlusher = nullptr; + + mutable MetricsRecordRef mMetricsRecordRef; + CounterPtr mInItemsCnt; + CounterPtr mInItemSizeBytes; + CounterPtr mOutItemsCnt; + CounterPtr mOutItemSizeBytes; + CounterPtr mDiscardedItemsCnt; + CounterPtr mDiscardedItemSizeBytes; + CounterPtr mTotalDelayMs; + +private: + virtual bool Serialize(T&& p, std::string& res, std::string& errorMsg) = 0; + +#ifdef APSARA_UNIT_TEST_MAIN + friend class SerializerUnittest; +#endif }; using EventSerializer = Serializer; diff --git a/core/plugin/flusher/sls/FlusherSLS.cpp b/core/plugin/flusher/sls/FlusherSLS.cpp index 5df36854c9..5f965a4535 100644 --- a/core/plugin/flusher/sls/FlusherSLS.cpp +++ b/core/plugin/flusher/sls/FlusherSLS.cpp @@ -23,9 +23,9 @@ #include "common/LogtailCommonFlags.h" #include "common/ParamExtractor.h" #include "common/TimeUtil.h" +#include "common/compression/CompressorFactory.h" #include "pipeline/Pipeline.h" #include "pipeline/batch/FlushStrategy.h" -#include "pipeline/compression/CompressorFactory.h" #include "pipeline/queue/QueueKeyManager.h" #include "pipeline/queue/SLSSenderQueueItem.h" #include "pipeline/queue/SenderQueueManager.h" @@ -445,7 +445,7 @@ bool FlusherSLS::Init(const Json::Value& config, Json::Value& optionalGoPipeline // CompressType if (BOOL_FLAG(sls_client_send_compress)) { - mCompressor = CompressorFactory::GetInstance()->Create(config, *mContext, sName, CompressType::LZ4); + mCompressor = CompressorFactory::GetInstance()->Create(config, *mContext, sName, mNodeID, CompressType::LZ4); } mGroupSerializer = make_unique(this); @@ -809,7 +809,7 @@ bool FlusherSLS::Send(string&& data, const string& shardHashKey, const string& l string compressedData; if (mCompressor) { string errorMsg; - if (!mCompressor->Compress(data, compressedData, errorMsg)) { + if (!mCompressor->DoCompress(data, compressedData, errorMsg)) { LOG_WARNING(mContext->GetLogger(), ("failed to compress data", errorMsg)("action", "discard data")("plugin", sName)("config", mContext->GetConfigName())); @@ -866,7 +866,7 @@ bool FlusherSLS::SerializeAndPush(PipelineEventGroup&& group) { std::move(group.GetExactlyOnceCheckpoint())); AddPackId(g); string errorMsg; - if (!mGroupSerializer->Serialize(std::move(g), serializedData, errorMsg)) { + if (!mGroupSerializer->DoSerialize(std::move(g), serializedData, errorMsg)) { LOG_WARNING(mContext->GetLogger(), ("failed to serialize event group", errorMsg)("action", "discard data")("plugin", sName)("config", mContext->GetConfigName())); @@ -880,7 +880,7 @@ bool FlusherSLS::SerializeAndPush(PipelineEventGroup&& group) { return false; } if (mCompressor) { - if (!mCompressor->Compress(serializedData, compressedData, errorMsg)) { + if (!mCompressor->DoCompress(serializedData, compressedData, errorMsg)) { LOG_WARNING(mContext->GetLogger(), ("failed to compress event group", errorMsg)("action", "discard data")("plugin", sName)("config", mContext->GetConfigName())); @@ -926,7 +926,7 @@ bool FlusherSLS::SerializeAndPush(BatchedEventsList&& groupList) { } AddPackId(group); string errorMsg; - if (!mGroupSerializer->Serialize(std::move(group), serializedData, errorMsg)) { + if (!mGroupSerializer->DoSerialize(std::move(group), serializedData, errorMsg)) { LOG_WARNING(mContext->GetLogger(), ("failed to serialize event group", errorMsg)("action", "discard data")("plugin", sName)("config", mContext->GetConfigName())); @@ -941,7 +941,7 @@ bool FlusherSLS::SerializeAndPush(BatchedEventsList&& groupList) { continue; } if (mCompressor) { - if (!mCompressor->Compress(serializedData, compressedData, errorMsg)) { + if (!mCompressor->DoCompress(serializedData, compressedData, errorMsg)) { LOG_WARNING(mContext->GetLogger(), ("failed to compress event group", errorMsg)("action", "discard data")("plugin", sName)("config", mContext->GetConfigName())); @@ -991,7 +991,7 @@ bool FlusherSLS::SerializeAndPush(BatchedEventsList&& groupList) { } if (enablePackageList) { string errorMsg; - mGroupListSerializer->Serialize(std::move(compressedLogGroups), serializedData, errorMsg); + mGroupListSerializer->DoSerialize(std::move(compressedLogGroups), serializedData, errorMsg); allSucceeded = Flusher::PushToQueue(make_unique( std::move(serializedData), packageSize, this, mQueueKey, mLogstore, RawDataType::EVENT_GROUP_LIST)) diff --git a/core/plugin/flusher/sls/FlusherSLS.h b/core/plugin/flusher/sls/FlusherSLS.h index 2840b94fe4..7e3d4658a1 100644 --- a/core/plugin/flusher/sls/FlusherSLS.h +++ b/core/plugin/flusher/sls/FlusherSLS.h @@ -24,12 +24,12 @@ #include #include +#include "common/compression/Compressor.h" +#include "models/PipelineEventGroup.h" #include "pipeline/batch/BatchStatus.h" #include "pipeline/batch/Batcher.h" -#include "pipeline/compression/Compressor.h" -#include "models/PipelineEventGroup.h" -#include "pipeline/plugin/interface/HttpFlusher.h" #include "pipeline/limiter/ConcurrencyLimiter.h" +#include "pipeline/plugin/interface/HttpFlusher.h" #include "pipeline/serializer/SLSSerializer.h" namespace logtail { diff --git a/core/profile_sender/ProfileSender.cpp b/core/profile_sender/ProfileSender.cpp index 1121e91216..7098f31ce1 100644 --- a/core/profile_sender/ProfileSender.cpp +++ b/core/profile_sender/ProfileSender.cpp @@ -28,7 +28,7 @@ #include "sdk/Exception.h" #include "sls_control/SLSControl.h" // TODO: temporarily used -#include "pipeline/compression/CompressorFactory.h" +#include "common/compression/CompressorFactory.h" using namespace std; @@ -101,8 +101,8 @@ void ProfileSender::SetProfileProjectName(const string& region, const string& pr flusher.mAliuid = STRING_FLAG(logtail_profile_aliuid); // logstore is given at send time // TODO: temporarily used - flusher.mCompressor - = CompressorFactory::GetInstance()->Create(Json::Value(), PipelineContext(), "flusher_sls", CompressType::ZSTD); + flusher.mCompressor = CompressorFactory::GetInstance()->Create( + Json::Value(), PipelineContext(), "flusher_sls", "", CompressType::ZSTD); } FlusherSLS* ProfileSender::GetFlusher(const string& region) { diff --git a/core/unittest/batch/BatchItemUnittest.cpp b/core/unittest/batch/BatchItemUnittest.cpp index 71d2d7fa59..0aac5d30b3 100644 --- a/core/unittest/batch/BatchItemUnittest.cpp +++ b/core/unittest/batch/BatchItemUnittest.cpp @@ -92,6 +92,7 @@ void EventBatchItemUnittest::TestAdd() { APSARA_TEST_EQUAL(1U, mItem.mBatch.mEvents.size()); APSARA_TEST_EQUAL(1U, mItem.GetStatus().GetCnt()); APSARA_TEST_EQUAL(size, mItem.GetStatus().GetSize()); + APSARA_TEST_NOT_EQUAL(0, mItem.mTotalEnqueTimeMs); } void EventBatchItemUnittest::TestFlushEmpty() { @@ -116,10 +117,12 @@ void EventBatchItemUnittest::TestFlushGroupBatchItem() { sEventGroup->AddLogEvent(); PipelineEventPtr& e = sEventGroup->MutableEvents().back(); mItem.Add(std::move(e)); + auto size = mItem.DataSize(); GroupBatchItem res; mItem.Flush(res); APSARA_TEST_EQUAL(1U, res.mGroups.size()); + APSARA_TEST_EQUAL(size, res.mGroups[0].mSizeBytes); APSARA_TEST_TRUE(mItem.IsEmpty()); APSARA_TEST_TRUE(mItem.mBatch.mTags.mInner.empty()); @@ -130,12 +133,14 @@ void EventBatchItemUnittest::TestFlushGroupBatchItem() { APSARA_TEST_EQUAL(0U, mItem.GetStatus().GetCnt()); APSARA_TEST_EQUAL(0U, mItem.GetStatus().GetSize()); APSARA_TEST_EQUAL(0, mItem.GetStatus().GetCreateTime()); + APSARA_TEST_EQUAL(0, mItem.mTotalEnqueTimeMs); } void EventBatchItemUnittest::TestFlushBatchedEvensList() { sEventGroup->AddLogEvent(); PipelineEventPtr& e = sEventGroup->MutableEvents().back(); mItem.Add(std::move(e)); + auto size = mItem.DataSize(); BatchedEventsList res; mItem.Flush(res); @@ -145,6 +150,7 @@ void EventBatchItemUnittest::TestFlushBatchedEvensList() { APSARA_TEST_NOT_EQUAL(nullptr, res[0].mExactlyOnceCheckpoint); APSARA_TEST_STREQ("pack_id", res[0].mPackIdPrefix.data()); APSARA_TEST_EQUAL(1U, res[0].mSourceBuffers.size()); + APSARA_TEST_EQUAL(size, res[0].mSizeBytes); APSARA_TEST_TRUE(mItem.IsEmpty()); APSARA_TEST_TRUE(mItem.mBatch.mTags.mInner.empty()); @@ -155,12 +161,14 @@ void EventBatchItemUnittest::TestFlushBatchedEvensList() { APSARA_TEST_EQUAL(0U, mItem.GetStatus().GetCnt()); APSARA_TEST_EQUAL(0U, mItem.GetStatus().GetSize()); APSARA_TEST_EQUAL(0, mItem.GetStatus().GetCreateTime()); + APSARA_TEST_EQUAL(0, mItem.mTotalEnqueTimeMs); } void EventBatchItemUnittest::TestFlushBatchedEvensLists() { sEventGroup->AddLogEvent(); PipelineEventPtr& e = sEventGroup->MutableEvents().back(); mItem.Add(std::move(e)); + auto size = mItem.DataSize(); vector res; mItem.Flush(res); @@ -171,6 +179,7 @@ void EventBatchItemUnittest::TestFlushBatchedEvensLists() { APSARA_TEST_NOT_EQUAL(nullptr, res[0][0].mExactlyOnceCheckpoint); APSARA_TEST_STREQ("pack_id", res[0][0].mPackIdPrefix.data()); APSARA_TEST_EQUAL(1U, res[0][0].mSourceBuffers.size()); + APSARA_TEST_EQUAL(size, res[0][0].mSizeBytes); APSARA_TEST_TRUE(mItem.IsEmpty()); APSARA_TEST_TRUE(mItem.mBatch.mTags.mInner.empty()); @@ -181,6 +190,7 @@ void EventBatchItemUnittest::TestFlushBatchedEvensLists() { APSARA_TEST_EQUAL(0U, mItem.GetStatus().GetCnt()); APSARA_TEST_EQUAL(0U, mItem.GetStatus().GetSize()); APSARA_TEST_EQUAL(0, mItem.GetStatus().GetCreateTime()); + APSARA_TEST_EQUAL(0, mItem.mTotalEnqueTimeMs); } void EventBatchItemUnittest::TestExactlyOnce() { @@ -223,6 +233,7 @@ class GroupBatchItemUnittest : public ::testing::Test { mBatch.mEvents = std::move(eventGroup.MutableEvents()); mBatch.mSourceBuffers.emplace_back(std::move(eventGroup.GetSourceBuffer())); mBatch.mTags = std::move(eventGroup.GetSizedTags()); + mBatch.mSizeBytes = 100; } void TearDown() override { @@ -236,11 +247,15 @@ class GroupBatchItemUnittest : public ::testing::Test { }; void GroupBatchItemUnittest::TestAdd() { - size_t size = mBatch.DataSize(); - mItem.Add(std::move(mBatch)); + size_t size = mBatch.mSizeBytes; + mItem.Add(std::move(mBatch), 1234567890000); APSARA_TEST_EQUAL(1U, mItem.mGroups.size()); APSARA_TEST_EQUAL(size, mItem.GetStatus().GetSize()); + APSARA_TEST_EQUAL(1234567890000, mItem.TotalEnqueTimeMs()); + APSARA_TEST_EQUAL(1U, mItem.EventSize()); + APSARA_TEST_EQUAL(1U, mItem.GroupSize()); + APSARA_TEST_EQUAL(100U, mItem.DataSize()); } void GroupBatchItemUnittest::TestFlushEmpty() { @@ -257,7 +272,7 @@ void GroupBatchItemUnittest::TestFlushEmpty() { } void GroupBatchItemUnittest::TestFlushBatchedEvensList() { - mItem.Add(std::move(mBatch)); + mItem.Add(std::move(mBatch), 1234567890000); BatchedEventsList res; mItem.Flush(res); @@ -266,10 +281,14 @@ void GroupBatchItemUnittest::TestFlushBatchedEvensList() { APSARA_TEST_TRUE(mItem.IsEmpty()); APSARA_TEST_EQUAL(0U, mItem.GetStatus().GetSize()); APSARA_TEST_EQUAL(0, mItem.GetStatus().GetCreateTime()); + APSARA_TEST_EQUAL(0, mItem.TotalEnqueTimeMs()); + APSARA_TEST_EQUAL(0U, mItem.EventSize()); + APSARA_TEST_EQUAL(0U, mItem.GroupSize()); + APSARA_TEST_EQUAL(0U, mItem.DataSize()); } void GroupBatchItemUnittest::TestFlushBatchedEvensLists() { - mItem.Add(std::move(mBatch)); + mItem.Add(std::move(mBatch), 1234567890000); vector res; mItem.Flush(res); @@ -279,6 +298,10 @@ void GroupBatchItemUnittest::TestFlushBatchedEvensLists() { APSARA_TEST_TRUE(mItem.IsEmpty()); APSARA_TEST_EQUAL(0U, mItem.GetStatus().GetSize()); APSARA_TEST_EQUAL(0, mItem.GetStatus().GetCreateTime()); + APSARA_TEST_EQUAL(0, mItem.TotalEnqueTimeMs()); + APSARA_TEST_EQUAL(0U, mItem.EventSize()); + APSARA_TEST_EQUAL(0U, mItem.GroupSize()); + APSARA_TEST_EQUAL(0U, mItem.DataSize()); } UNIT_TEST_CASE(GroupBatchItemUnittest, TestAdd) diff --git a/core/unittest/batch/BatchStatusUnittest.cpp b/core/unittest/batch/BatchStatusUnittest.cpp index 23932bef46..094333fd3d 100644 --- a/core/unittest/batch/BatchStatusUnittest.cpp +++ b/core/unittest/batch/BatchStatusUnittest.cpp @@ -124,6 +124,7 @@ class GroupBatchStatusUnittest : public ::testing::Test { void TestUpdate(); protected: + static void SetUpTestCase() { sBatch.mSizeBytes = 100; } void SetUp() override { mStatus.Reset(); } private: @@ -142,10 +143,10 @@ void GroupBatchStatusUnittest::TestReset() { void GroupBatchStatusUnittest::TestUpdate() { mStatus.Update(sBatch); time_t createTime = mStatus.GetCreateTime(); - APSARA_TEST_EQUAL(sBatch.DataSize(), mStatus.GetSize()); + APSARA_TEST_EQUAL(sBatch.mSizeBytes, mStatus.GetSize()); mStatus.Update(sBatch); - APSARA_TEST_EQUAL(2 * sBatch.DataSize(), mStatus.GetSize()); + APSARA_TEST_EQUAL(2 * sBatch.mSizeBytes, mStatus.GetSize()); APSARA_TEST_EQUAL(createTime, mStatus.GetCreateTime()); } diff --git a/core/unittest/batch/BatcherUnittest.cpp b/core/unittest/batch/BatcherUnittest.cpp index 8d40721565..799153692d 100644 --- a/core/unittest/batch/BatcherUnittest.cpp +++ b/core/unittest/batch/BatcherUnittest.cpp @@ -12,9 +12,8 @@ // See the License for the specific language governing permissions and // limitations under the License. - -#include "pipeline/batch/Batcher.h" #include "common/JsonUtil.h" +#include "pipeline/batch/Batcher.h" #include "unittest/Unittest.h" #include "unittest/plugin/PluginMock.h" @@ -34,6 +33,7 @@ class BatcherUnittest : public ::testing::Test { void TestFlushGroupQueue(); void TestFlushAllWithoutGroupBatch(); void TestFlushAllWithGroupBatch(); + void TestMetric(); protected: static void SetUpTestCase() { sFlusher = make_unique(); } @@ -42,6 +42,7 @@ class BatcherUnittest : public ::testing::Test { mCtx.SetConfigName("test_config"); sFlusher->SetContext(mCtx); sFlusher->SetMetricsRecordRef(FlusherMock::sName, "1", "1", "1"); + sFlusher->SetNodeID("1"); } void TearDown() override { TimeoutFlushManager::GetInstance()->mTimeoutRecords.clear(); } @@ -559,6 +560,63 @@ void BatcherUnittest::TestFlushAllWithGroupBatch() { APSARA_TEST_STREQ("pack_id", res[1][0].mPackIdPrefix.data()); } +void BatcherUnittest::TestMetric() { + { + DefaultFlushStrategyOptions strategy; + strategy.mMaxCnt = 2; + strategy.mMaxSizeBytes = 1000; + strategy.mTimeoutSecs = 3; + + Batcher<> batch; + batch.Init(Json::Value(), sFlusher.get(), strategy, false); + + PipelineEventGroup g = CreateEventGroup(3); + auto groupSize = g.DataSize(); + auto batchSize = groupSize - 2 * g.GetEvents()[0]->DataSize(); + + vector res; + batch.Add(std::move(g), res); + APSARA_TEST_EQUAL(5U, batch.mMetricsRecordRef->GetLabels()->size()); + APSARA_TEST_TRUE(batch.mMetricsRecordRef.HasLabel(METRIC_LABEL_PROJECT, "")); + APSARA_TEST_TRUE(batch.mMetricsRecordRef.HasLabel(METRIC_LABEL_CONFIG_NAME, "test_config")); + APSARA_TEST_TRUE(batch.mMetricsRecordRef.HasLabel(METRIC_LABEL_KEY_COMPONENT_NAME, "batcher")); + APSARA_TEST_TRUE(batch.mMetricsRecordRef.HasLabel(METRIC_LABEL_KEY_FLUSHER_NODE_ID, "1")); + APSARA_TEST_TRUE(batch.mMetricsRecordRef.HasLabel("enable_group_batch", "false")); + APSARA_TEST_EQUAL(3U, batch.mInEventsCnt->GetValue()); + APSARA_TEST_EQUAL(groupSize, batch.mInGroupDataSizeBytes->GetValue()); + APSARA_TEST_EQUAL(2U, batch.mOutEventsCnt->GetValue()); + APSARA_TEST_EQUAL(1U, batch.mEventBatchItemsCnt->GetValue()); + APSARA_TEST_EQUAL(1U, batch.mBufferedGroupsCnt->GetValue()); + APSARA_TEST_EQUAL(1U, batch.mBufferedEventsCnt->GetValue()); + APSARA_TEST_EQUAL(batchSize, batch.mBufferedDataSizeByte->GetValue()); + } + { + DefaultFlushStrategyOptions strategy; + strategy.mMaxCnt = 2; + strategy.mMaxSizeBytes = 1000; + strategy.mTimeoutSecs = 3; + + Batcher<> batch; + batch.Init(Json::Value(), sFlusher.get(), strategy, true); + + PipelineEventGroup g = CreateEventGroup(3); + auto groupSize = g.DataSize(); + auto batchSize = groupSize - 2 * g.GetEvents()[0]->DataSize(); + + vector res; + batch.Add(std::move(g), res); + batch.FlushQueue(0, res[0]); + APSARA_TEST_TRUE(batch.mMetricsRecordRef.HasLabel("enable_group_batch", "true")); + APSARA_TEST_EQUAL(3U, batch.mInEventsCnt->GetValue()); + APSARA_TEST_EQUAL(groupSize, batch.mInGroupDataSizeBytes->GetValue()); + APSARA_TEST_EQUAL(2U, batch.mOutEventsCnt->GetValue()); + APSARA_TEST_EQUAL(1U, batch.mEventBatchItemsCnt->GetValue()); + APSARA_TEST_EQUAL(1U, batch.mBufferedGroupsCnt->GetValue()); + APSARA_TEST_EQUAL(1U, batch.mBufferedEventsCnt->GetValue()); + APSARA_TEST_EQUAL(batchSize, batch.mBufferedDataSizeByte->GetValue()); + } +} + PipelineEventGroup BatcherUnittest::CreateEventGroup(size_t cnt) { PipelineEventGroup group(make_shared()); group.SetTag(string("key"), string("val")); @@ -581,6 +639,7 @@ UNIT_TEST_CASE(BatcherUnittest, TestFlushEventQueueWithGroupBatch) UNIT_TEST_CASE(BatcherUnittest, TestFlushGroupQueue) UNIT_TEST_CASE(BatcherUnittest, TestFlushAllWithoutGroupBatch) UNIT_TEST_CASE(BatcherUnittest, TestFlushAllWithGroupBatch) +UNIT_TEST_CASE(BatcherUnittest, TestMetric) } // namespace logtail diff --git a/core/unittest/compression/CMakeLists.txt b/core/unittest/compression/CMakeLists.txt index 442a8db893..60f1655e5d 100644 --- a/core/unittest/compression/CMakeLists.txt +++ b/core/unittest/compression/CMakeLists.txt @@ -18,6 +18,9 @@ project(compression_unittest) add_executable(compressor_factory_unittest CompressorFactoryUnittest.cpp) target_link_libraries(compressor_factory_unittest ${UT_BASE_TARGET}) +add_executable(compressor_unittest CompressorUnittest.cpp) +target_link_libraries(compressor_unittest ${UT_BASE_TARGET}) + add_executable(lz4_compressor_unittest LZ4CompressorUnittest.cpp) target_link_libraries(lz4_compressor_unittest ${UT_BASE_TARGET}) @@ -26,5 +29,6 @@ target_link_libraries(zstd_compressor_unittest ${UT_BASE_TARGET}) include(GoogleTest) gtest_discover_tests(compressor_factory_unittest) +gtest_discover_tests(compressor_unittest) gtest_discover_tests(lz4_compressor_unittest) gtest_discover_tests(zstd_compressor_unittest) diff --git a/core/unittest/compression/CompressorFactoryUnittest.cpp b/core/unittest/compression/CompressorFactoryUnittest.cpp index 7093a37ff3..84e159ee34 100644 --- a/core/unittest/compression/CompressorFactoryUnittest.cpp +++ b/core/unittest/compression/CompressorFactoryUnittest.cpp @@ -12,7 +12,8 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include "pipeline/compression/CompressorFactory.h" +#include "monitor/MetricConstants.h" +#include "common/compression/CompressorFactory.h" #include "unittest/Unittest.h" using namespace std; @@ -23,56 +24,64 @@ class CompressorFactoryUnittest : public ::testing::Test { public: void TestCreate(); void TestCompressTypeToString(); + void TestMetric(); protected: - void SetUp() { mCtx.SetConfigName("test_config"); } + void SetUp() { + mCtx.SetConfigName("test_config"); + mFlusherId = "1"; + } private: PipelineContext mCtx; + string mFlusherId; }; void CompressorFactoryUnittest::TestCreate() { { // use default - auto compressor - = CompressorFactory::GetInstance()->Create(Json::Value(), mCtx, "test_plugin", CompressType::LZ4); + auto compressor = CompressorFactory::GetInstance()->Create( + Json::Value(), mCtx, "test_plugin", mFlusherId, CompressType::LZ4); APSARA_TEST_EQUAL(CompressType::LZ4, compressor->GetCompressType()); } { // lz4 Json::Value config; config["CompressType"] = "lz4"; - auto compressor = CompressorFactory::GetInstance()->Create(config, mCtx, "test_plugin", CompressType::ZSTD); + auto compressor + = CompressorFactory::GetInstance()->Create(config, mCtx, "test_plugin", mFlusherId, CompressType::ZSTD); APSARA_TEST_EQUAL(CompressType::LZ4, compressor->GetCompressType()); } { // zstd Json::Value config; config["CompressType"] = "zstd"; - auto compressor = CompressorFactory::GetInstance()->Create(config, mCtx, "test_plugin", CompressType::LZ4); + auto compressor + = CompressorFactory::GetInstance()->Create(config, mCtx, "test_plugin", mFlusherId, CompressType::LZ4); APSARA_TEST_EQUAL(CompressType::ZSTD, compressor->GetCompressType()); } { // none Json::Value config; config["CompressType"] = "none"; - auto compressor = CompressorFactory::GetInstance()->Create(config, mCtx, "test_plugin", CompressType::LZ4); + auto compressor + = CompressorFactory::GetInstance()->Create(config, mCtx, "test_plugin", mFlusherId, CompressType::LZ4); APSARA_TEST_EQUAL(nullptr, compressor); } { // unknown Json::Value config; config["CompressType"] = "unknown"; - auto compressor - = CompressorFactory::GetInstance()->Create(Json::Value(), mCtx, "test_plugin", CompressType::LZ4); + auto compressor = CompressorFactory::GetInstance()->Create( + Json::Value(), mCtx, "test_plugin", mFlusherId, CompressType::LZ4); APSARA_TEST_EQUAL(CompressType::LZ4, compressor->GetCompressType()); } { // invalid Json::Value config; config["CompressType"] = 123; - auto compressor - = CompressorFactory::GetInstance()->Create(Json::Value(), mCtx, "test_plugin", CompressType::LZ4); + auto compressor = CompressorFactory::GetInstance()->Create( + Json::Value(), mCtx, "test_plugin", mFlusherId, CompressType::LZ4); APSARA_TEST_EQUAL(CompressType::LZ4, compressor->GetCompressType()); } } @@ -83,8 +92,19 @@ void CompressorFactoryUnittest::TestCompressTypeToString() { APSARA_TEST_STREQ("none", CompressTypeToString(CompressType::NONE).data()); } +void CompressorFactoryUnittest::TestMetric() { + auto compressor + = CompressorFactory::GetInstance()->Create(Json::Value(), mCtx, "test_plugin", mFlusherId, CompressType::LZ4); + APSARA_TEST_EQUAL(4U, compressor->mMetricsRecordRef->GetLabels()->size()); + APSARA_TEST_TRUE(compressor->mMetricsRecordRef.HasLabel(METRIC_LABEL_PROJECT, "")); + APSARA_TEST_TRUE(compressor->mMetricsRecordRef.HasLabel(METRIC_LABEL_CONFIG_NAME, "test_config")); + APSARA_TEST_TRUE(compressor->mMetricsRecordRef.HasLabel(METRIC_LABEL_KEY_COMPONENT_NAME, "compressor")); + APSARA_TEST_TRUE(compressor->mMetricsRecordRef.HasLabel(METRIC_LABEL_KEY_FLUSHER_NODE_ID, mFlusherId)); +} + UNIT_TEST_CASE(CompressorFactoryUnittest, TestCreate) UNIT_TEST_CASE(CompressorFactoryUnittest, TestCompressTypeToString) +UNIT_TEST_CASE(CompressorFactoryUnittest, TestMetric) } // namespace logtail diff --git a/core/unittest/compression/CompressorUnittest.cpp b/core/unittest/compression/CompressorUnittest.cpp new file mode 100644 index 0000000000..54576c63a4 --- /dev/null +++ b/core/unittest/compression/CompressorUnittest.cpp @@ -0,0 +1,79 @@ +// Copyright 2024 iLogtail Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "common/compression/LZ4Compressor.h" +#include "monitor/MetricConstants.h" +#include "unittest/Unittest.h" + +using namespace std; + +namespace logtail { + +class CompressorMock : public Compressor { +public: + CompressorMock(CompressType type) : Compressor(type) {}; + + bool UnCompress(const std::string& input, std::string& output, std::string& errorMsg) override { return true; } + +private: + bool Compress(const std::string& input, std::string& output, std::string& errorMsg) override { + if (input == "failed") { + return false; + } + output = input.substr(0, input.size() / 2); + return true; + } +}; + +class CompressorUnittest : public ::testing::Test { +public: + void TestMetric(); +}; + +void CompressorUnittest::TestMetric() { + { + CompressorMock compressor(CompressType::MOCK); + compressor.SetMetricRecordRef({}); + string input = "hello world"; + string output; + string errorMsg; + compressor.DoCompress(input, output, errorMsg); + APSARA_TEST_EQUAL(1U, compressor.mInItemsCnt->GetValue()); + APSARA_TEST_EQUAL(input.size(), compressor.mInItemSizeBytes->GetValue()); + APSARA_TEST_EQUAL(1U, compressor.mOutItemsCnt->GetValue()); + APSARA_TEST_EQUAL(output.size(), compressor.mOutItemSizeBytes->GetValue()); + APSARA_TEST_EQUAL(0U, compressor.mDiscardedItemsCnt->GetValue()); + APSARA_TEST_EQUAL(0U, compressor.mDiscardedItemSizeBytes->GetValue()); + } + { + CompressorMock compressor(CompressType::MOCK); + compressor.SetMetricRecordRef({}); + string input = "failed"; + string output; + string errorMsg; + compressor.DoCompress(input, output, errorMsg); + APSARA_TEST_EQUAL(1U, compressor.mInItemsCnt->GetValue()); + APSARA_TEST_EQUAL(input.size(), compressor.mInItemSizeBytes->GetValue()); + APSARA_TEST_EQUAL(0U, compressor.mOutItemsCnt->GetValue()); + APSARA_TEST_EQUAL(0U, compressor.mOutItemSizeBytes->GetValue()); + APSARA_TEST_EQUAL(1U, compressor.mDiscardedItemsCnt->GetValue()); + APSARA_TEST_EQUAL(input.size(), compressor.mDiscardedItemSizeBytes->GetValue()); + } +} + +UNIT_TEST_CASE(CompressorUnittest, TestMetric) + +} // namespace logtail + +UNIT_TEST_MAIN diff --git a/core/unittest/compression/LZ4CompressorUnittest.cpp b/core/unittest/compression/LZ4CompressorUnittest.cpp index 7bd62300f7..e852c063cf 100644 --- a/core/unittest/compression/LZ4CompressorUnittest.cpp +++ b/core/unittest/compression/LZ4CompressorUnittest.cpp @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include "pipeline/compression/LZ4Compressor.h" +#include "common/compression/LZ4Compressor.h" #include "unittest/Unittest.h" using namespace std; @@ -29,7 +29,7 @@ void LZ4CompressorUnittest::TestCompress() { string input = "hello world"; string output; string errorMsg; - APSARA_TEST_TRUE(compressor.Compress(input, output, errorMsg)); + APSARA_TEST_TRUE(compressor.DoCompress(input, output, errorMsg)); string decompressed; decompressed.resize(input.size()); APSARA_TEST_TRUE(compressor.UnCompress(output, decompressed, errorMsg)); diff --git a/core/unittest/compression/ZstdCompressorUnittest.cpp b/core/unittest/compression/ZstdCompressorUnittest.cpp index 6362907173..0e165b1987 100644 --- a/core/unittest/compression/ZstdCompressorUnittest.cpp +++ b/core/unittest/compression/ZstdCompressorUnittest.cpp @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include "pipeline/compression/ZstdCompressor.h" +#include "common/compression/ZstdCompressor.h" #include "unittest/Unittest.h" using namespace std; @@ -29,7 +29,7 @@ void ZstdCompressorUnittest::TestCompress() { string input = "hello world"; string output; string errorMsg; - APSARA_TEST_TRUE(compressor.Compress(input, output, errorMsg)); + APSARA_TEST_TRUE(compressor.DoCompress(input, output, errorMsg)); string decompressed; decompressed.resize(input.size()); APSARA_TEST_TRUE(compressor.UnCompress(output, decompressed, errorMsg)); diff --git a/core/unittest/flusher/FlusherSLSUnittest.cpp b/core/unittest/flusher/FlusherSLSUnittest.cpp index 52a3d2f510..71e13cbcc6 100644 --- a/core/unittest/flusher/FlusherSLSUnittest.cpp +++ b/core/unittest/flusher/FlusherSLSUnittest.cpp @@ -22,10 +22,7 @@ #ifdef __ENTERPRISE__ #include "config/provider/EnterpriseConfigProvider.h" #endif -#include "pipeline/compression/CompressorFactory.h" -#include "plugin/flusher/sls/FlusherSLS.h" -#include "plugin/flusher/sls/PackIdManager.h" -#include "plugin/flusher/sls/SLSClientManager.h" +#include "common/compression/CompressorFactory.h" #include "pipeline/Pipeline.h" #include "pipeline/PipelineContext.h" #include "pipeline/queue/ExactlyOnceQueueManager.h" @@ -33,6 +30,9 @@ #include "pipeline/queue/QueueKeyManager.h" #include "pipeline/queue/SLSSenderQueueItem.h" #include "pipeline/queue/SenderQueueManager.h" +#include "plugin/flusher/sls/FlusherSLS.h" +#include "plugin/flusher/sls/PackIdManager.h" +#include "plugin/flusher/sls/SLSClientManager.h" #include "unittest/Unittest.h" DECLARE_FLAG_INT32(batch_send_interval); @@ -617,7 +617,7 @@ void FlusherSLSUnittest::TestSend() { APSARA_TEST_EQUAL(cpt, item->mExactlyOnceCheckpoint); auto compressor - = CompressorFactory::GetInstance()->Create(Json::Value(), ctx, "flusher_sls", CompressType::LZ4); + = CompressorFactory::GetInstance()->Create(Json::Value(), ctx, "flusher_sls", "1", CompressType::LZ4); string output, errorMsg; output.resize(item->mRawSize); APSARA_TEST_TRUE(compressor->UnCompress(item->mData, output, errorMsg)); @@ -671,7 +671,7 @@ void FlusherSLSUnittest::TestSend() { APSARA_TEST_EQUAL(checkpoints[0], item->mExactlyOnceCheckpoint); auto compressor - = CompressorFactory::GetInstance()->Create(Json::Value(), ctx, "flusher_sls", CompressType::LZ4); + = CompressorFactory::GetInstance()->Create(Json::Value(), ctx, "flusher_sls", "1", CompressType::LZ4); string output, errorMsg; output.resize(item->mRawSize); APSARA_TEST_TRUE(compressor->UnCompress(item->mData, output, errorMsg)); @@ -750,7 +750,7 @@ void FlusherSLSUnittest::TestSend() { APSARA_TEST_EQUAL(flusher.mLogstore, item->mLogstore); auto compressor - = CompressorFactory::GetInstance()->Create(Json::Value(), ctx, "flusher_sls", CompressType::LZ4); + = CompressorFactory::GetInstance()->Create(Json::Value(), ctx, "flusher_sls", "1", CompressType::LZ4); string output, errorMsg; output.resize(item->mRawSize); APSARA_TEST_TRUE(compressor->UnCompress(item->mData, output, errorMsg)); @@ -844,7 +844,7 @@ void FlusherSLSUnittest::TestSend() { APSARA_TEST_EQUAL(flusher.mLogstore, item->mLogstore); auto compressor - = CompressorFactory::GetInstance()->Create(Json::Value(), ctx, "flusher_sls", CompressType::LZ4); + = CompressorFactory::GetInstance()->Create(Json::Value(), ctx, "flusher_sls", "1", CompressType::LZ4); sls_logs::SlsLogPackageList packageList; APSARA_TEST_TRUE(packageList.ParseFromString(item->mData)); @@ -1017,7 +1017,7 @@ void FlusherSLSUnittest::OnGoPipelineSend() { APSARA_TEST_EQUAL("other_logstore", item->mLogstore); auto compressor - = CompressorFactory::GetInstance()->Create(Json::Value(), ctx, "flusher_sls", CompressType::LZ4); + = CompressorFactory::GetInstance()->Create(Json::Value(), ctx, "flusher_sls", "1", CompressType::LZ4); string output; output.resize(item->mRawSize); APSARA_TEST_TRUE(compressor->UnCompress(item->mData, output, errorMsg)); @@ -1040,7 +1040,7 @@ void FlusherSLSUnittest::OnGoPipelineSend() { flusher.mProject = "test_project"; flusher.mLogstore = "test_logstore"; flusher.mCompressor = CompressorFactory::GetInstance()->Create( - Json::Value(), PipelineContext(), "flusher_sls", CompressType::LZ4); + Json::Value(), PipelineContext(), "flusher_sls", "1", CompressType::LZ4); APSARA_TEST_TRUE(flusher.Send("content", "")); @@ -1060,7 +1060,7 @@ void FlusherSLSUnittest::OnGoPipelineSend() { APSARA_TEST_EQUAL("test_logstore", item->mLogstore); auto compressor - = CompressorFactory::GetInstance()->Create(Json::Value(), ctx, "flusher_sls", CompressType::LZ4); + = CompressorFactory::GetInstance()->Create(Json::Value(), ctx, "flusher_sls", "1", CompressType::LZ4); string output; output.resize(item->mRawSize); string errorMsg; diff --git a/core/unittest/pipeline/PipelineUnittest.cpp b/core/unittest/pipeline/PipelineUnittest.cpp index 98b41a5ff6..d4d76b5138 100644 --- a/core/unittest/pipeline/PipelineUnittest.cpp +++ b/core/unittest/pipeline/PipelineUnittest.cpp @@ -18,17 +18,17 @@ #include #include "app_config/AppConfig.h" -#include "pipeline/batch/TimeoutFlushManager.h" #include "common/JsonUtil.h" #include "config/PipelineConfig.h" -#include "plugin/input/InputFeedbackInterfaceRegistry.h" #include "pipeline/Pipeline.h" +#include "pipeline/batch/TimeoutFlushManager.h" #include "pipeline/plugin/PluginRegistry.h" -#include "plugin/processor/inner/ProcessorSplitLogStringNative.h" -#include "plugin/processor/inner/ProcessorSplitMultilineLogStringNative.h" #include "pipeline/queue/BoundedProcessQueue.h" #include "pipeline/queue/ProcessQueueManager.h" #include "pipeline/queue/QueueKeyManager.h" +#include "plugin/input/InputFeedbackInterfaceRegistry.h" +#include "plugin/processor/inner/ProcessorSplitLogStringNative.h" +#include "plugin/processor/inner/ProcessorSplitMultilineLogStringNative.h" #include "unittest/Unittest.h" #include "unittest/plugin/PluginMock.h" @@ -112,6 +112,9 @@ void PipelineUnittest::OnSuccessfulInit() const { APSARA_TEST_EQUAL("test_region", pipeline->GetContext().GetRegion()); APSARA_TEST_EQUAL(QueueKeyManager::GetInstance()->GetKey("test_config-flusher_sls-test_project#test_logstore"), pipeline->GetContext().GetLogstoreKey()); + APSARA_TEST_EQUAL(2U, pipeline->mMetricsRecordRef->GetLabels()->size()); + APSARA_TEST_TRUE(pipeline->mMetricsRecordRef.HasLabel(METRIC_LABEL_CONFIG_NAME, configName)); + APSARA_TEST_TRUE(pipeline->mMetricsRecordRef.HasLabel(METRIC_LABEL_PROJECT, "test_project")); // without sls flusher configStr = R"( @@ -2684,12 +2687,24 @@ void PipelineUnittest::TestProcess() const { processor->Init(Json::Value(), ctx); pipeline.mProcessorLine.emplace_back(std::move(processor)); - vector group; - group.emplace_back(make_shared()); - pipeline.Process(group, 0); + WriteMetrics::GetInstance()->PrepareMetricsRecordRef(pipeline.mMetricsRecordRef, {}); + pipeline.mProcessorsInEventsCnt = pipeline.mMetricsRecordRef.CreateCounter("processors_in_events_cnt"); + pipeline.mProcessorsInGroupsCnt = pipeline.mMetricsRecordRef.CreateCounter("processors_in_event_groups_cnt"); + pipeline.mProcessorsInGroupDataSizeBytes + = pipeline.mMetricsRecordRef.CreateCounter("processors_in_event_group_data_size_bytes"); + pipeline.mProcessorsTotalDelayMs = pipeline.mMetricsRecordRef.CreateCounter("processors_total_delay_ms"); + + vector groups; + groups.emplace_back(make_shared()); + groups.back().AddLogEvent(); + auto size = groups.back().DataSize(); + pipeline.Process(groups, 0); APSARA_TEST_EQUAL( 1U, static_cast(pipeline.mInputs[0]->GetInnerProcessors()[0]->mPlugin.get())->mCnt); APSARA_TEST_EQUAL(1U, static_cast(pipeline.mProcessorLine[0]->mPlugin.get())->mCnt); + APSARA_TEST_EQUAL(1U, pipeline.mProcessorsInEventsCnt->GetValue()); + APSARA_TEST_EQUAL(1U, pipeline.mProcessorsInGroupsCnt->GetValue()); + APSARA_TEST_EQUAL(size, pipeline.mProcessorsInGroupDataSizeBytes->GetValue()); } void PipelineUnittest::TestSend() const { diff --git a/core/unittest/route/RouterUnittest.cpp b/core/unittest/route/RouterUnittest.cpp index bee7bb1803..3e42ff722a 100644 --- a/core/unittest/route/RouterUnittest.cpp +++ b/core/unittest/route/RouterUnittest.cpp @@ -13,6 +13,7 @@ // limitations under the License. #include "common/JsonUtil.h" +#include "monitor/MetricConstants.h" #include "pipeline/Pipeline.h" #include "pipeline/route/Router.h" #include "unittest/Unittest.h" @@ -25,6 +26,10 @@ class RouterUnittest : public testing::Test { public: void TestInit(); void TestRoute(); + void TestMetric(); + +protected: + void SetUp() override { ctx.SetConfigName("test_config"); } private: PipelineContext ctx; @@ -124,8 +129,44 @@ void RouterUnittest::TestRoute() { } } +void RouterUnittest::TestMetric() { + Json::Value configJson; + string errorMsg; + string configStr = R"( + [ + { + "Type": "event_type", + "Value": "log" + } + ] + )"; + APSARA_TEST_TRUE(ParseJsonTable(configStr, configJson, errorMsg)); + vector> configs; + for (Json::Value::ArrayIndex i = 0; i < configJson.size(); ++i) { + configs.emplace_back(i, &configJson[i]); + } + configs.emplace_back(configJson.size(), nullptr); + + Router router; + router.Init(configs, ctx); + + APSARA_TEST_EQUAL(3U, router.mMetricsRecordRef->GetLabels()->size()); + APSARA_TEST_TRUE(router.mMetricsRecordRef.HasLabel(METRIC_LABEL_PROJECT, "")); + APSARA_TEST_TRUE(router.mMetricsRecordRef.HasLabel(METRIC_LABEL_CONFIG_NAME, "test_config")); + APSARA_TEST_TRUE(router.mMetricsRecordRef.HasLabel(METRIC_LABEL_KEY_COMPONENT_NAME, "router")); + + PipelineEventGroup g(make_shared()); + g.AddLogEvent(); + auto size = g.DataSize(); + router.Route(g); + + APSARA_TEST_EQUAL(1U, router.mInEventsCnt->GetValue()); + APSARA_TEST_EQUAL(size, router.mInGroupDataSizeBytes->GetValue()); +} + UNIT_TEST_CASE(RouterUnittest, TestInit) UNIT_TEST_CASE(RouterUnittest, TestRoute) +UNIT_TEST_CASE(RouterUnittest, TestMetric) } // namespace logtail diff --git a/core/unittest/serializer/CMakeLists.txt b/core/unittest/serializer/CMakeLists.txt index 11c1436ee5..b471e3b81a 100644 --- a/core/unittest/serializer/CMakeLists.txt +++ b/core/unittest/serializer/CMakeLists.txt @@ -15,8 +15,12 @@ cmake_minimum_required(VERSION 3.22) project(serializer_unittest) +add_executable(serializer_unittest SerializerUnittest.cpp) +target_link_libraries(serializer_unittest ${UT_BASE_TARGET}) + add_executable(sls_serializer_unittest SLSSerializerUnittest.cpp) target_link_libraries(sls_serializer_unittest ${UT_BASE_TARGET}) include(GoogleTest) +gtest_discover_tests(serializer_unittest) gtest_discover_tests(sls_serializer_unittest) diff --git a/core/unittest/serializer/SLSSerializerUnittest.cpp b/core/unittest/serializer/SLSSerializerUnittest.cpp index a3695df4d1..7d57cef290 100644 --- a/core/unittest/serializer/SLSSerializerUnittest.cpp +++ b/core/unittest/serializer/SLSSerializerUnittest.cpp @@ -52,7 +52,7 @@ void SLSSerializerUnittest::TestSerializeEventGroup() { { // nano second disabled, and set string res, errorMsg; - APSARA_TEST_TRUE(serializer.Serialize(CreateBatchedEvents(false), res, errorMsg)); + APSARA_TEST_TRUE(serializer.DoSerialize(CreateBatchedEvents(false), res, errorMsg)); sls_logs::LogGroup logGroup; APSARA_TEST_TRUE(logGroup.ParseFromString(res)); APSARA_TEST_EQUAL(1, logGroup.logs_size()); @@ -72,7 +72,7 @@ void SLSSerializerUnittest::TestSerializeEventGroup() { // nano second enabled, and set const_cast(mCtx.GetGlobalConfig()).mEnableTimestampNanosecond = true; string res, errorMsg; - APSARA_TEST_TRUE(serializer.Serialize(CreateBatchedEvents(true), res, errorMsg)); + APSARA_TEST_TRUE(serializer.DoSerialize(CreateBatchedEvents(true), res, errorMsg)); sls_logs::LogGroup logGroup; APSARA_TEST_TRUE(logGroup.ParseFromString(res)); APSARA_TEST_EQUAL(1234567890U, logGroup.logs(0).time()); @@ -83,7 +83,7 @@ void SLSSerializerUnittest::TestSerializeEventGroup() { // nano second enabled, not set const_cast(mCtx.GetGlobalConfig()).mEnableTimestampNanosecond = true; string res, errorMsg; - APSARA_TEST_TRUE(serializer.Serialize(CreateBatchedEvents(false), res, errorMsg)); + APSARA_TEST_TRUE(serializer.DoSerialize(CreateBatchedEvents(false), res, errorMsg)); sls_logs::LogGroup logGroup; APSARA_TEST_TRUE(logGroup.ParseFromString(res)); APSARA_TEST_EQUAL(1234567890U, logGroup.logs(0).time()); @@ -94,13 +94,13 @@ void SLSSerializerUnittest::TestSerializeEventGroup() { // log group exceed size limit INT32_FLAG(max_send_log_group_size) = 0; string res, errorMsg; - APSARA_TEST_FALSE(serializer.Serialize(CreateBatchedEvents(true), res, errorMsg)); + APSARA_TEST_FALSE(serializer.DoSerialize(CreateBatchedEvents(true), res, errorMsg)); INT32_FLAG(max_send_log_group_size) = 10 * 1024 * 1024; } { // metric event string res, errorMsg; - APSARA_TEST_TRUE(serializer.Serialize(CreateBatchedMetricEvents(false, 0, false), res, errorMsg)); + APSARA_TEST_TRUE(serializer.DoSerialize(CreateBatchedMetricEvents(false, 0, false), res, errorMsg)); sls_logs::LogGroup logGroup; APSARA_TEST_TRUE(logGroup.ParseFromString(res)); @@ -125,7 +125,7 @@ void SLSSerializerUnittest::TestSerializeEventGroup() { const_cast(mCtx.GetGlobalConfig()).mEnableTimestampNanosecond = true; string res, errorMsg; - APSARA_TEST_TRUE(serializer.Serialize(CreateBatchedMetricEvents(true, 1, false), res, errorMsg)); + APSARA_TEST_TRUE(serializer.DoSerialize(CreateBatchedMetricEvents(true, 1, false), res, errorMsg)); sls_logs::LogGroup logGroup; APSARA_TEST_TRUE(logGroup.ParseFromString(res)); @@ -151,7 +151,7 @@ void SLSSerializerUnittest::TestSerializeEventGroup() { const_cast(mCtx.GetGlobalConfig()).mEnableTimestampNanosecond = true; string res, errorMsg; - APSARA_TEST_TRUE(serializer.Serialize(CreateBatchedMetricEvents(true, 1999999999, false), res, errorMsg)); + APSARA_TEST_TRUE(serializer.DoSerialize(CreateBatchedMetricEvents(true, 1999999999, false), res, errorMsg)); sls_logs::LogGroup logGroup; APSARA_TEST_TRUE(logGroup.ParseFromString(res)); @@ -178,7 +178,7 @@ void SLSSerializerUnittest::TestSerializeEventGroup() { const_cast(mCtx.GetGlobalConfig()).mEnableTimestampNanosecond = true; string res, errorMsg; - APSARA_TEST_TRUE(serializer.Serialize(CreateBatchedMetricEvents(false, 0, true), res, errorMsg)); + APSARA_TEST_TRUE(serializer.DoSerialize(CreateBatchedMetricEvents(false, 0, true), res, errorMsg)); sls_logs::LogGroup logGroup; APSARA_TEST_TRUE(logGroup.ParseFromString(res)); @@ -192,7 +192,7 @@ void SLSSerializerUnittest::TestSerializeEventGroupList() { SLSEventGroupListSerializer serializer(sFlusher.get()); string res, errorMsg; - APSARA_TEST_TRUE(serializer.Serialize(std::move(v), res, errorMsg)); + APSARA_TEST_TRUE(serializer.DoSerialize(std::move(v), res, errorMsg)); sls_logs::SlsLogPackageList logPackageList; APSARA_TEST_TRUE(logPackageList.ParseFromString(res)); APSARA_TEST_EQUAL(1, logPackageList.packages_size()); diff --git a/core/unittest/serializer/SerializerUnittest.cpp b/core/unittest/serializer/SerializerUnittest.cpp new file mode 100644 index 0000000000..ed632e02ae --- /dev/null +++ b/core/unittest/serializer/SerializerUnittest.cpp @@ -0,0 +1,114 @@ +// Copyright 2024 iLogtail Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "monitor/MetricConstants.h" +#include "pipeline/plugin/interface/Flusher.h" +#include "pipeline/serializer/Serializer.h" +#include "unittest/Unittest.h" +#include "unittest/plugin/PluginMock.h" + +using namespace std; + +namespace logtail { + +class SerializerMock : public Serializer { +public: + SerializerMock(Flusher* f) : Serializer(f) {}; + +private: + bool Serialize(BatchedEvents&& p, std::string& res, std::string& errorMsg) override { + if (p.mEvents.empty()) { + return false; + } + res = "result"; + return true; + } +}; + +class SerializerUnittest : public ::testing::Test { +public: + void TestMetric(); + +protected: + static void SetUpTestCase() { sFlusher = make_unique(); } + + void SetUp() override { + mCtx.SetConfigName("test_config"); + sFlusher->SetContext(mCtx); + sFlusher->SetMetricsRecordRef(FlusherMock::sName, "1", "1", "1"); + } + +private: + static unique_ptr sFlusher; + + BatchedEvents CreateBatchedMetricEvents(bool withEvents = true); + + PipelineContext mCtx; +}; + +unique_ptr SerializerUnittest::sFlusher; + +void SerializerUnittest::TestMetric() { + { + SerializerMock serializer(sFlusher.get()); + auto input = CreateBatchedMetricEvents(); + auto inputSize = input.mSizeBytes; + string output; + string errorMsg; + serializer.DoSerialize(std::move(input), output, errorMsg); + APSARA_TEST_EQUAL(1U, serializer.mInItemsCnt->GetValue()); + APSARA_TEST_EQUAL(inputSize, serializer.mInItemSizeBytes->GetValue()); + APSARA_TEST_EQUAL(1U, serializer.mOutItemsCnt->GetValue()); + APSARA_TEST_EQUAL(output.size(), serializer.mOutItemSizeBytes->GetValue()); + APSARA_TEST_EQUAL(0U, serializer.mDiscardedItemsCnt->GetValue()); + APSARA_TEST_EQUAL(0U, serializer.mDiscardedItemSizeBytes->GetValue()); + } + { + SerializerMock serializer(sFlusher.get()); + auto input = CreateBatchedMetricEvents(false); + auto inputSize = input.mSizeBytes; + string output; + string errorMsg; + serializer.DoSerialize(std::move(input), output, errorMsg); + APSARA_TEST_EQUAL(1U, serializer.mInItemsCnt->GetValue()); + APSARA_TEST_EQUAL(inputSize, serializer.mInItemSizeBytes->GetValue()); + APSARA_TEST_EQUAL(0U, serializer.mOutItemsCnt->GetValue()); + APSARA_TEST_EQUAL(0U, serializer.mOutItemSizeBytes->GetValue()); + APSARA_TEST_EQUAL(1U, serializer.mDiscardedItemsCnt->GetValue()); + APSARA_TEST_EQUAL(inputSize, serializer.mDiscardedItemSizeBytes->GetValue()); + } +} + +BatchedEvents SerializerUnittest::CreateBatchedMetricEvents(bool withEvents) { + PipelineEventGroup group(make_shared()); + group.SetTag(string("key"), string("value")); + StringBuffer b = group.GetSourceBuffer()->CopyString(string("pack_id")); + group.SetMetadataNoCopy(EventGroupMetaKey::SOURCE_ID, StringView(b.data, b.size)); + group.SetExactlyOnceCheckpoint(RangeCheckpointPtr(new RangeCheckpoint)); + if (withEvents) { + group.AddLogEvent(); + } + BatchedEvents batch(std::move(group.MutableEvents()), + std::move(group.GetSizedTags()), + std::move(group.GetSourceBuffer()), + group.GetMetadata(EventGroupMetaKey::SOURCE_ID), + std::move(group.GetExactlyOnceCheckpoint())); + return batch; +} + +UNIT_TEST_CASE(SerializerUnittest, TestMetric) + +} // namespace logtail + +UNIT_TEST_MAIN From 7106316ca201b9d0084a6d083eb13fd8873bd7a9 Mon Sep 17 00:00:00 2001 From: dog Date: Thu, 19 Sep 2024 10:55:58 +0800 Subject: [PATCH 2/9] feat: 1. support prom scrape_protocols, 2. support prom enable_compression (#1719) --- core/prometheus/Constants.h | 15 ++ core/prometheus/schedulers/ScrapeConfig.cpp | 112 +++++++++- core/prometheus/schedulers/ScrapeConfig.h | 9 +- .../prometheus/schedulers/ScrapeScheduler.cpp | 4 +- .../prometheus/ScrapeConfigUnittest.cpp | 201 +++++++++++++++++- .../prometheus/ScrapeSchedulerUnittest.cpp | 2 +- 6 files changed, 327 insertions(+), 16 deletions(-) diff --git a/core/prometheus/Constants.h b/core/prometheus/Constants.h index fbd33597ae..6a60e3a161 100644 --- a/core/prometheus/Constants.h +++ b/core/prometheus/Constants.h @@ -50,6 +50,7 @@ const char* const SCHEME = "scheme"; const char* const METRICS_PATH = "metrics_path"; const char* const SCRAPE_INTERVAL = "scrape_interval"; const char* const SCRAPE_TIMEOUT = "scrape_timeout"; +const char* const SCRAPE_PROTOCOLS = "scrape_protocols"; const char* const HEADERS = "headers"; const char* const PARAMS = "params"; const char* const QUERY_STRING = "query_string"; @@ -71,6 +72,14 @@ const char* const PASSWORD = "password"; const char* const PASSWORD_FILE = "password_file"; const char* const BASIC_PREFIX = "Basic "; +// scrape protocols, from https://prometheus.io/docs/prometheus/latest/configuration/configuration/#scrape_config +// text/plain, application/openmetrics-text will be used +// version of openmetrics is 1.0.0 or 0.0.1, from +// https://github.com/OpenObservability/OpenMetrics/blob/main/specification/OpenMetrics.md#extensions-and-improvements +const char* const PrometheusProto = "PrometheusProto"; +const char* const PrometheusText0_0_4 = "PrometheusText0.0.4"; +const char* const OpenMetricsText0_0_1 = "OpenMetricsText0.0.1"; +const char* const OpenMetricsText1_0_0 = "OpenMetricsText1.0.0"; // metric labels const char* const JOB = "job"; @@ -94,4 +103,10 @@ const char* const UP = "up"; const char* const SCRAPE_TIMESTAMP_MILLISEC = "scrape_timestamp_millisec"; +// scrape config compression +const char* const ENABLE_COMPRESSION = "enable_compression"; +const char* const ACCEPT_ENCODING = "Accept-Encoding"; +const char* const GZIP = "gzip"; +const char* const IDENTITY = "identity"; + } // namespace logtail::prometheus diff --git a/core/prometheus/schedulers/ScrapeConfig.cpp b/core/prometheus/schedulers/ScrapeConfig.cpp index 177c9a7e27..ff6ee77b15 100644 --- a/core/prometheus/schedulers/ScrapeConfig.cpp +++ b/core/prometheus/schedulers/ScrapeConfig.cpp @@ -35,6 +35,7 @@ bool ScrapeConfig::Init(const Json::Value& scrapeConfig) { return false; } + if (scrapeConfig.isMember(prometheus::SCRAPE_INTERVAL) && scrapeConfig[prometheus::SCRAPE_INTERVAL].isString()) { string tmpScrapeIntervalString = scrapeConfig[prometheus::SCRAPE_INTERVAL].asString(); mScrapeIntervalSeconds = DurationToSecond(tmpScrapeIntervalString); @@ -43,6 +44,23 @@ bool ScrapeConfig::Init(const Json::Value& scrapeConfig) { string tmpScrapeTimeoutString = scrapeConfig[prometheus::SCRAPE_TIMEOUT].asString(); mScrapeTimeoutSeconds = DurationToSecond(tmpScrapeTimeoutString); } + if (scrapeConfig.isMember(prometheus::SCRAPE_PROTOCOLS) && scrapeConfig[prometheus::SCRAPE_PROTOCOLS].isArray()) { + if (!InitScrapeProtocols(scrapeConfig[prometheus::SCRAPE_PROTOCOLS])) { + LOG_ERROR(sLogger, ("scrape protocol config error", scrapeConfig[prometheus::SCRAPE_PROTOCOLS])); + return false; + } + } else { + Json::Value nullJson; + InitScrapeProtocols(nullJson); + } + + if (scrapeConfig.isMember(prometheus::ENABLE_COMPRESSION) + && scrapeConfig[prometheus::ENABLE_COMPRESSION].isBool()) { + InitEnableCompression(scrapeConfig[prometheus::ENABLE_COMPRESSION].asBool()); + } else { + InitEnableCompression(true); + } + if (scrapeConfig.isMember(prometheus::METRICS_PATH) && scrapeConfig[prometheus::METRICS_PATH].isString()) { mMetricsPath = scrapeConfig[prometheus::METRICS_PATH].asString(); } @@ -185,7 +203,7 @@ bool ScrapeConfig::InitBasicAuth(const Json::Value& basicAuth) { auto token = username + ":" + password; auto token64 = sdk::Base64Enconde(token); - mAuthHeaders[prometheus::A_UTHORIZATION] = prometheus::BASIC_PREFIX + token64; + mRequestHeaders[prometheus::A_UTHORIZATION] = prometheus::BASIC_PREFIX + token64; return true; } @@ -219,8 +237,98 @@ bool ScrapeConfig::InitAuthorization(const Json::Value& authorization) { return false; } - mAuthHeaders[prometheus::A_UTHORIZATION] = type + " " + credentials; + mRequestHeaders[prometheus::A_UTHORIZATION] = type + " " + credentials; + return true; +} + +bool ScrapeConfig::InitScrapeProtocols(const Json::Value& scrapeProtocols) { + static auto sScrapeProtocolsHeaders = std::map{ + {prometheus::PrometheusProto, + "application/vnd.google.protobuf;proto=io.prometheus.client.MetricFamily;encoding=delimited"}, + {prometheus::PrometheusText0_0_4, "text/plain;version=0.0.4"}, + {prometheus::OpenMetricsText0_0_1, "application/openmetrics-text;version=0.0.1"}, + {prometheus::OpenMetricsText1_0_0, "application/openmetrics-text;version=1.0.0"}, + }; + static auto sDefaultScrapeProtocols = vector{ + prometheus::PrometheusText0_0_4, + prometheus::PrometheusProto, + prometheus::OpenMetricsText0_0_1, + prometheus::OpenMetricsText1_0_0, + }; + + auto join = [](const vector& strs, const string& sep) { + string result; + for (const auto& str : strs) { + if (!result.empty()) { + result += sep; + } + result += str; + } + return result; + }; + + auto getScrapeProtocols = [](const Json::Value& scrapeProtocols, vector& res) { + for (const auto& scrapeProtocol : scrapeProtocols) { + if (scrapeProtocol.isString()) { + res.push_back(scrapeProtocol.asString()); + } else { + LOG_ERROR(sLogger, ("scrape_protocols config error", "")); + return false; + } + } + return true; + }; + + auto validateScrapeProtocols = [](const vector& scrapeProtocols) { + set dups; + for (const auto& scrapeProtocol : scrapeProtocols) { + if (!sScrapeProtocolsHeaders.count(scrapeProtocol)) { + LOG_ERROR(sLogger, + ("unknown scrape protocol prometheusproto", scrapeProtocol)( + "supported", + "[OpenMetricsText0.0.1 OpenMetricsText1.0.0 PrometheusProto PrometheusText0.0.4]")); + return false; + } + if (dups.count(scrapeProtocol)) { + LOG_ERROR(sLogger, ("duplicated protocol in scrape_protocols", scrapeProtocol)); + return false; + } + dups.insert(scrapeProtocol); + } + return true; + }; + + vector tmpScrapeProtocols; + + if (!getScrapeProtocols(scrapeProtocols, tmpScrapeProtocols)) { + return false; + } + + // if scrape_protocols is empty, use default protocols + if (tmpScrapeProtocols.empty()) { + tmpScrapeProtocols = sDefaultScrapeProtocols; + } + if (!validateScrapeProtocols(tmpScrapeProtocols)) { + return false; + } + + auto weight = tmpScrapeProtocols.size() + 1; + for (auto& tmpScrapeProtocol : tmpScrapeProtocols) { + auto val = sScrapeProtocolsHeaders[tmpScrapeProtocol]; + val += ";q=0." + std::to_string(weight--); + tmpScrapeProtocol = val; + } + tmpScrapeProtocols.push_back("*/*;q=0." + ToString(weight)); + mRequestHeaders[prometheus::ACCEPT] = join(tmpScrapeProtocols, ","); return true; } +void ScrapeConfig::InitEnableCompression(bool enableCompression) { + if (enableCompression) { + mRequestHeaders[prometheus::ACCEPT_ENCODING] = prometheus::GZIP; + } else { + mRequestHeaders[prometheus::ACCEPT_ENCODING] = prometheus::IDENTITY; + } +} + } // namespace logtail \ No newline at end of file diff --git a/core/prometheus/schedulers/ScrapeConfig.h b/core/prometheus/schedulers/ScrapeConfig.h index 4f8e866cbf..eac6d9615a 100644 --- a/core/prometheus/schedulers/ScrapeConfig.h +++ b/core/prometheus/schedulers/ScrapeConfig.h @@ -12,6 +12,7 @@ namespace logtail { + class ScrapeConfig { public: std::string mJobName; @@ -20,7 +21,10 @@ class ScrapeConfig { std::string mMetricsPath; std::string mScheme; - std::map mAuthHeaders; + // auth header + // scrape_protocols Accept header: PrometheusProto, OpenMetricsText0.0.1, OpenMetricsText1.0.0, PrometheusText0.0.4 + // enable_compression Accept-Encoding header: gzip, identity + std::map mRequestHeaders; int64_t mMaxScrapeSizeBytes; int64_t mSampleLimit; @@ -37,10 +41,13 @@ class ScrapeConfig { private: bool InitBasicAuth(const Json::Value& basicAuth); bool InitAuthorization(const Json::Value& authorization); + bool InitScrapeProtocols(const Json::Value& scrapeProtocols); + void InitEnableCompression(bool enableCompression); #ifdef APSARA_UNIT_TEST_MAIN friend class ScrapeConfigUnittest; #endif }; + } // namespace logtail \ No newline at end of file diff --git a/core/prometheus/schedulers/ScrapeScheduler.cpp b/core/prometheus/schedulers/ScrapeScheduler.cpp index f0494c60ae..f5013eebea 100644 --- a/core/prometheus/schedulers/ScrapeScheduler.cpp +++ b/core/prometheus/schedulers/ScrapeScheduler.cpp @@ -67,7 +67,7 @@ void ScrapeScheduler::OnMetricResult(const HttpResponse& response, uint64_t time if (response.mStatusCode != 200) { mScrapeResponseSizeBytes = 0; string headerStr; - for (const auto& [k, v] : mScrapeConfigPtr->mAuthHeaders) { + for (const auto& [k, v] : mScrapeConfigPtr->mRequestHeaders) { headerStr.append(k).append(":").append(v).append(";"); } LOG_WARNING(sLogger, @@ -144,7 +144,7 @@ std::unique_ptr ScrapeScheduler::BuildScrapeTimerEvent(std::chrono:: mPort, mScrapeConfigPtr->mMetricsPath, mScrapeConfigPtr->mQueryString, - mScrapeConfigPtr->mAuthHeaders, + mScrapeConfigPtr->mRequestHeaders, "", mScrapeConfigPtr->mScrapeTimeoutSeconds, mScrapeConfigPtr->mScrapeIntervalSeconds diff --git a/core/unittest/prometheus/ScrapeConfigUnittest.cpp b/core/unittest/prometheus/ScrapeConfigUnittest.cpp index 8560892071..65e7aeb8da 100644 --- a/core/unittest/prometheus/ScrapeConfigUnittest.cpp +++ b/core/unittest/prometheus/ScrapeConfigUnittest.cpp @@ -17,6 +17,8 @@ class ScrapeConfigUnittest : public testing::Test { void TestAuth(); void TestBasicAuth(); void TestAuthorization(); + void TestScrapeProtocols(); + void TestEnableCompression(); private: void SetUp() override; @@ -55,6 +57,12 @@ void ScrapeConfigUnittest::TestInit() { "scrape_interval": "30s", "scrape_timeout": "30s", "metrics_path": "/metrics", + "scrape_protocols": [ + "PrometheusText0.0.4", + "PrometheusProto", + "OpenMetricsText0.0.1" + ], + "enable_compression": false, "scheme": "http", "basic_auth": { "username": "test_user", @@ -92,8 +100,17 @@ void ScrapeConfigUnittest::TestInit() { APSARA_TEST_EQUAL(scrapeConfig.mMetricsPath, "/metrics"); APSARA_TEST_EQUAL(scrapeConfig.mScheme, "http"); + // scrape protocols + APSARA_TEST_EQUAL(scrapeConfig.mRequestHeaders["Accept"], + "text/plain;version=0.0.4;q=0.4,application/" + "vnd.google.protobuf;proto=io.prometheus.client.MetricFamily;encoding=delimited;q=0.3," + "application/openmetrics-text;version=0.0.1;q=0.2,*/*;q=0.1"); + + // disable compression + APSARA_TEST_EQUAL(scrapeConfig.mRequestHeaders["Accept-Encoding"], "identity"); + // basic auth - APSARA_TEST_EQUAL(scrapeConfig.mAuthHeaders["Authorization"], "Basic dGVzdF91c2VyOnRlc3RfcGFzc3dvcmQ="); + APSARA_TEST_EQUAL(scrapeConfig.mRequestHeaders["Authorization"], "Basic dGVzdF91c2VyOnRlc3RfcGFzc3dvcmQ="); APSARA_TEST_EQUAL(scrapeConfig.mMaxScrapeSizeBytes, 1024 * 1024 * 1024); APSARA_TEST_EQUAL(scrapeConfig.mSampleLimit, 10000); @@ -150,9 +167,9 @@ void ScrapeConfigUnittest::TestBasicAuth() { APSARA_TEST_TRUE(ParseJsonTable(configStr, config, errorMsg)); APSARA_TEST_TRUE(scrapeConfig.Init(config)); - APSARA_TEST_EQUAL(scrapeConfig.mAuthHeaders["Authorization"], "Basic dGVzdF91c2VyOnRlc3RfcGFzc3dvcmQ="); + APSARA_TEST_EQUAL(scrapeConfig.mRequestHeaders["Authorization"], "Basic dGVzdF91c2VyOnRlc3RfcGFzc3dvcmQ="); - scrapeConfig.mAuthHeaders.clear(); + scrapeConfig.mRequestHeaders.clear(); configStr = R"JSON({ "job_name": "test_job", "scrape_interval": "30s", @@ -167,10 +184,10 @@ void ScrapeConfigUnittest::TestBasicAuth() { APSARA_TEST_TRUE(ParseJsonTable(configStr, config, errorMsg)); APSARA_TEST_TRUE(scrapeConfig.Init(config)); - APSARA_TEST_EQUAL(scrapeConfig.mAuthHeaders["Authorization"], "Basic dGVzdF91c2VyOnRlc3RfcGFzc3dvcmQuZmlsZQ=="); + APSARA_TEST_EQUAL(scrapeConfig.mRequestHeaders["Authorization"], "Basic dGVzdF91c2VyOnRlc3RfcGFzc3dvcmQuZmlsZQ=="); // error - scrapeConfig.mAuthHeaders.clear(); + scrapeConfig.mRequestHeaders.clear(); configStr = R"JSON({ "job_name": "test_job", "scrape_interval": "30s", @@ -206,12 +223,12 @@ void ScrapeConfigUnittest::TestAuthorization() { })JSON"; APSARA_TEST_TRUE(ParseJsonTable(configStr, config, errorMsg)); - scrapeConfig.mAuthHeaders.clear(); + scrapeConfig.mRequestHeaders.clear(); APSARA_TEST_TRUE(scrapeConfig.Init(config)); // bearer auth - APSARA_TEST_EQUAL(scrapeConfig.mAuthHeaders["Authorization"], "Bearer test_token"); + APSARA_TEST_EQUAL(scrapeConfig.mRequestHeaders["Authorization"], "Bearer test_token"); - scrapeConfig.mAuthHeaders.clear(); + scrapeConfig.mRequestHeaders.clear(); // default Bearer auth configStr = R"JSON({ @@ -226,15 +243,179 @@ void ScrapeConfigUnittest::TestAuthorization() { })JSON"; APSARA_TEST_TRUE(ParseJsonTable(configStr, config, errorMsg)); - scrapeConfig.mAuthHeaders.clear(); + scrapeConfig.mRequestHeaders.clear(); + APSARA_TEST_TRUE(scrapeConfig.Init(config)); + APSARA_TEST_EQUAL(scrapeConfig.mRequestHeaders["Authorization"], "Bearer " + mKey); +} + +void ScrapeConfigUnittest::TestScrapeProtocols() { + Json::Value config; + ScrapeConfig scrapeConfig; + string errorMsg; + string configStr; + + // default + configStr = R"JSON({ + "job_name": "test_job", + "scrape_interval": "30s", + "scrape_timeout": "30s", + "metrics_path": "/metrics", + "scheme": "http" + })JSON"; + APSARA_TEST_TRUE(ParseJsonTable(configStr, config, errorMsg)); + scrapeConfig.mRequestHeaders.clear(); + APSARA_TEST_TRUE(scrapeConfig.Init(config)); + APSARA_TEST_EQUAL( + "text/plain;version=0.0.4;q=0.5,application/" + "vnd.google.protobuf;proto=io.prometheus.client.MetricFamily;encoding=delimited;q=0.4,application/" + "openmetrics-text;version=0.0.1;q=0.3,application/openmetrics-text;version=1.0.0;q=0.2,*/*;q=0.1", + scrapeConfig.mRequestHeaders["Accept"]); + + // custom quality protocols + configStr = R"JSON({ + "job_name": "test_job", + "scrape_interval": "30s", + "scrape_timeout": "30s", + "metrics_path": "/metrics", + "scheme": "http", + "scrape_protocols": ["PrometheusProto", "OpenMetricsText1.0.0", "PrometheusText0.0.4", "OpenMetricsText0.0.1"] + })JSON"; + APSARA_TEST_TRUE(ParseJsonTable(configStr, config, errorMsg)); + scrapeConfig.mRequestHeaders.clear(); + APSARA_TEST_TRUE(scrapeConfig.Init(config)); + APSARA_TEST_EQUAL( + "application/vnd.google.protobuf;proto=io.prometheus.client.MetricFamily;encoding=delimited;q=0.5," + "application/openmetrics-text;version=1.0.0;q=0.4," + "text/plain;version=0.0.4;q=0.3,application/openmetrics-text;version=0.0.1;q=0.2,*/*;q=0.1", + scrapeConfig.mRequestHeaders["Accept"]); + + // only prometheus0.0.4 protocols + configStr = R"JSON({ + "job_name": "test_job", + "scrape_interval": "30s", + "scrape_timeout": "30s", + "metrics_path": "/metrics", + "scheme": "http", + "scrape_protocols": ["PrometheusText0.0.4"] + })JSON"; + APSARA_TEST_TRUE(ParseJsonTable(configStr, config, errorMsg)); + scrapeConfig.mRequestHeaders.clear(); + APSARA_TEST_TRUE(scrapeConfig.Init(config)); + APSARA_TEST_EQUAL( + "text/plain;version=0.0.4;q=0.2,*/*;q=0.1", + scrapeConfig.mRequestHeaders["Accept"]); + + // Capital error + configStr = R"JSON({ + "job_name": "test_job", + "scrape_interval": "30s", + "scrape_timeout": "30s", + "metrics_path": "/metrics", + "scheme": "http", + "scrape_protocols": ["prometheusproto"] + })JSON"; + + APSARA_TEST_TRUE(ParseJsonTable(configStr, config, errorMsg)); + scrapeConfig.mRequestHeaders.clear(); + APSARA_TEST_FALSE(scrapeConfig.Init(config)); + + // OpenMetricsText1.0.0 duplication error + configStr = R"JSON({ + "job_name": "test_job", + "scrape_interval": "30s", + "scrape_timeout": "30s", + "metrics_path": "/metrics", + "scheme": "http", + "scrape_protocols": ["OpenMetricsText1.0.0", "PrometheusProto", "OpenMetricsText1.0.0"] + })JSON"; + + APSARA_TEST_TRUE(ParseJsonTable(configStr, config, errorMsg)); + scrapeConfig.mRequestHeaders.clear(); + APSARA_TEST_FALSE(scrapeConfig.Init(config)); + + // protocols invalid + configStr = R"JSON({ + "job_name": "test_job", + "scrape_interval": "30s", + "scrape_timeout": "30s", + "metrics_path": "/metrics", + "scheme": "http", + "scrape_protocols": ["OpenMetricsText1.0.0", "PrometheusProto", 999] + })JSON"; + + APSARA_TEST_TRUE(ParseJsonTable(configStr, config, errorMsg)); + scrapeConfig.mRequestHeaders.clear(); + APSARA_TEST_FALSE(scrapeConfig.Init(config)); + + // unknown protocol + configStr = R"JSON({ + "job_name": "test_job", + "scrape_interval": "30s", + "scrape_timeout": "30s", + "metrics_path": "/metrics", + "scheme": "http", + "scrape_protocols": ["OpenMetricsText"] + })JSON"; + + APSARA_TEST_TRUE(ParseJsonTable(configStr, config, errorMsg)); + scrapeConfig.mRequestHeaders.clear(); + APSARA_TEST_FALSE(scrapeConfig.Init(config)); +} + +void ScrapeConfigUnittest::TestEnableCompression() { + Json::Value config; + ScrapeConfig scrapeConfig; + string errorMsg; + string configStr; + + // default + configStr = R"JSON({ + "job_name": "test_job", + "scrape_interval": "30s", + "scrape_timeout": "30s", + "metrics_path": "/metrics", + "scheme": "http" + })JSON"; + APSARA_TEST_TRUE(ParseJsonTable(configStr, config, errorMsg)); + scrapeConfig.mRequestHeaders.clear(); + APSARA_TEST_TRUE(scrapeConfig.Init(config)); + APSARA_TEST_EQUAL("gzip", scrapeConfig.mRequestHeaders["Accept-Encoding"]); + + // disable + configStr = R"JSON({ + "job_name": "test_job", + "scrape_interval": "30s", + "scrape_timeout": "30s", + "metrics_path": "/metrics", + "scheme": "http", + "enable_compression": false + })JSON"; + APSARA_TEST_TRUE(ParseJsonTable(configStr, config, errorMsg)); + scrapeConfig.mRequestHeaders.clear(); + APSARA_TEST_TRUE(scrapeConfig.Init(config)); + APSARA_TEST_EQUAL("identity", scrapeConfig.mRequestHeaders["Accept-Encoding"]); + + // enable + configStr = R"JSON({ + "job_name": "test_job", + "scrape_interval": "30s", + "scrape_timeout": "30s", + "metrics_path": "/metrics", + "scheme": "http", + "enable_compression": true + })JSON"; + APSARA_TEST_TRUE(ParseJsonTable(configStr, config, errorMsg)); + scrapeConfig.mRequestHeaders.clear(); APSARA_TEST_TRUE(scrapeConfig.Init(config)); - APSARA_TEST_EQUAL(scrapeConfig.mAuthHeaders["Authorization"], "Bearer " + mKey); + APSARA_TEST_EQUAL("gzip", scrapeConfig.mRequestHeaders["Accept-Encoding"]); } UNIT_TEST_CASE(ScrapeConfigUnittest, TestInit); UNIT_TEST_CASE(ScrapeConfigUnittest, TestAuth); UNIT_TEST_CASE(ScrapeConfigUnittest, TestBasicAuth); UNIT_TEST_CASE(ScrapeConfigUnittest, TestAuthorization); +UNIT_TEST_CASE(ScrapeConfigUnittest, TestScrapeProtocols); +UNIT_TEST_CASE(ScrapeConfigUnittest, TestEnableCompression); } // namespace logtail diff --git a/core/unittest/prometheus/ScrapeSchedulerUnittest.cpp b/core/unittest/prometheus/ScrapeSchedulerUnittest.cpp index 951507b97b..fd3d540a92 100644 --- a/core/unittest/prometheus/ScrapeSchedulerUnittest.cpp +++ b/core/unittest/prometheus/ScrapeSchedulerUnittest.cpp @@ -56,7 +56,7 @@ class ScrapeSchedulerUnittest : public testing::Test { mScrapeConfig->mScrapeIntervalSeconds = 10; mScrapeConfig->mScrapeTimeoutSeconds = 10; mScrapeConfig->mMetricsPath = "/metrics"; - mScrapeConfig->mAuthHeaders = {{"Authorization", "Bearer xxxxx"}}; + mScrapeConfig->mRequestHeaders = {{"Authorization", "Bearer xxxxx"}}; mHttpResponse.mBody = "# HELP go_gc_duration_seconds A summary of the pause duration of garbage collection cycles.\n" From 96afc79e01d1fb3b821f21f4e2fa9265f106b8a1 Mon Sep 17 00:00:00 2001 From: Zhu Shunjia Date: Thu, 19 Sep 2024 11:13:57 +0800 Subject: [PATCH 3/9] feat: provider APIs return multi remote config providers (#1708) * provider APIs return multi remote config providers * restore virtual to VIRTUAL * polish cmake files * add unittest path * update build.md * change vector to map and store the path info Change-Id: I0d049a051c971469e749e6c6478d1e7b43fc5e75 * refactor RemoteConfigProvider APIs Change-Id: I52a04db594048da93014d133632ffb578597aa0d * fix typo Change-Id: Iec15ca29fcbe49eb07e1cd7aa8cf632ad0f815bb * simpify GetRemoteConfigProviders API Change-Id: I18096a8bbc2f62776401bc56ad8893946a9b1b18 --- core/CMakeLists.txt | 20 +++++---- core/application/Application.cpp | 12 +++--- .../CommonConfigProvider.cpp | 2 +- .../CommonConfigProvider.h | 2 +- .../LegacyCommonConfigProvider.cpp | 2 +- .../LegacyCommonConfigProvider.h | 0 core/go_pipeline/LogtailPlugin.cpp | 14 +++---- core/monitor/LogFileProfiler.cpp | 4 +- core/monitor/LogtailAlarm.cpp | 8 ++-- core/monitor/MetricExportor.cpp | 4 +- core/monitor/MetricExportor.h | 2 +- core/monitor/Monitor.cpp | 8 ++-- core/monitor/Monitor.h | 1 + core/plugin/flusher/sls/DiskBufferWriter.cpp | 4 +- core/plugin/flusher/sls/FlusherSLS.cpp | 4 +- core/provider/CMakeLists.txt | 39 +++++++++++++++++ core/provider/Provider.cpp | 41 ++++++++++++++++++ core/provider/Provider.h | 37 ++++++++++++++++ core/unittest/CMakeLists.txt | 2 + .../config/CommonConfigProviderUnittest.cpp | 2 +- core/unittest/provider/CMakeLists.txt | 22 ++++++++++ core/unittest/provider/ProviderUnittest.cpp | 42 +++++++++++++++++++ docs/cn/installation/sources/build.md | 10 +++++ 23 files changed, 240 insertions(+), 42 deletions(-) rename core/config/{provider => common_provider}/CommonConfigProvider.cpp (99%) rename core/config/{provider => common_provider}/CommonConfigProvider.h (98%) rename core/config/{provider => common_provider}/LegacyCommonConfigProvider.cpp (99%) rename core/config/{provider => common_provider}/LegacyCommonConfigProvider.h (100%) create mode 100644 core/provider/CMakeLists.txt create mode 100644 core/provider/Provider.cpp create mode 100644 core/provider/Provider.h create mode 100644 core/unittest/provider/CMakeLists.txt create mode 100644 core/unittest/provider/ProviderUnittest.cpp diff --git a/core/CMakeLists.txt b/core/CMakeLists.txt index 6c36c26eeb..39073d2271 100644 --- a/core/CMakeLists.txt +++ b/core/CMakeLists.txt @@ -33,6 +33,8 @@ cmake_dependent_option(ENABLE_STATIC_LINK_CRT "Build Logtail by linking CRT stat option(WITHOUTGDB "Build Logtail without gdb") option(WITHSPL "Build Logtail and UT with SPL" ON) option(BUILD_LOGTAIL_UT "Build unit test for Logtail") +set(PROVIDER_PATH "provider" CACHE PATH "Path to the provider module") # external provider path can be set with -DPROVIDER_PATH +set(UNITTEST_PATH "unittest" CACHE PATH "Path to the unittest module") # external unittest path can be set with -DUNITTEST_PATH if (BUILD_LOGTAIL_SHARED_LIBRARY AND WITHSPL) message(FATEL_ERROR, "Generating logtail shared library is not supported to be linked with SPL. WITHSPL should be set OFF.") @@ -109,13 +111,13 @@ include(${CMAKE_CURRENT_SOURCE_DIR}/plugin/processor/links.cmake) include(${CMAKE_CURRENT_SOURCE_DIR}/plugin/flusher/flusher.cmake) include(${CMAKE_CURRENT_SOURCE_DIR}/plugin/flusher/links.cmake) -# Subdirectories (modules). except for common, input, processor, flusher, observer, helper and spl. +# Subdirectories (modules). except for common, input, processor, flusher, observer, helper, spl, and provider. set(SUB_DIRECTORIES_LIST application app_config checkpoint container_manager logger go_pipeline monitor profile_sender models - config config/feedbacker config/provider config/watcher + config config/watcher pipeline pipeline/batch pipeline/limiter pipeline/plugin pipeline/plugin/creator pipeline/plugin/instance pipeline/plugin/interface pipeline/queue pipeline/route pipeline/serializer runner runner/sink/http - protobuf/config_server/v1 protobuf/config_server/v2 protobuf/sls + protobuf/sls file_server file_server/event file_server/event_handler file_server/event_listener file_server/reader file_server/polling prometheus prometheus/labels prometheus/schedulers prometheus/async ebpf ebpf/observer ebpf/security ebpf/handler @@ -161,6 +163,10 @@ endif() # remove several files in go_pipeline list(REMOVE_ITEM FRAMEWORK_SOURCE_FILES ${CMAKE_CURRENT_SOURCE_DIR}/go_pipeline/LogtailPluginAdapter.cpp ${CMAKE_CURRENT_SOURCE_DIR}/go_pipeline/LogtailPluginAdapter.h) + +# add provider +add_subdirectory("${PROVIDER_PATH}" "${CMAKE_BINARY_DIR}/provider") + if(MSVC) # remove linux event listener file(GLOB REMOVE_EVENT_LISTENER_SOURCES file_server/event_listener/*_Linux.cpp file_server/event_listener/*_Linux.h) @@ -231,12 +237,12 @@ if(BUILD_LOGTAIL OR BUILD_LOGTAIL_SHARED_LIBRARY) flusher_link(${LOGTAIL_TARGET}) all_link(${LOGTAIL_TARGET}) common_link(${LOGTAIL_TARGET}) + target_link_libraries(${LOGTAIL_TARGET} provider) endif() # Logtail UT. if (BUILD_LOGTAIL_UT) - message(STATUS "Build unittest.") - add_definitions(-DVIRTUAL=virtual) + message(STATUS "Build unittest.") function(delete_gcda_files target_directory) if(EXISTS "${target_directory}") message(STATUS "Deleting .gcda files in ${target_directory}") @@ -249,7 +255,5 @@ if (BUILD_LOGTAIL_UT) delete_gcda_files(".") include(CTest) enable_testing() - add_subdirectory(unittest) -else() - add_definitions(-DVIRTUAL= ) + add_subdirectory("${UNITTEST_PATH}" "${CMAKE_BINARY_DIR}/unittest") endif () \ No newline at end of file diff --git a/core/application/Application.cpp b/core/application/Application.cpp index d86fc6a92d..3ed86d1e65 100644 --- a/core/application/Application.cpp +++ b/core/application/Application.cpp @@ -60,8 +60,7 @@ #include "streamlog/StreamLogManager.h" #endif #else -#include "config/provider/CommonConfigProvider.h" -#include "config/provider/LegacyCommonConfigProvider.h" +#include "provider/Provider.h" #endif DEFINE_FLAG_BOOL(ilogtail_disable_core, "disable core in worker process", true); @@ -242,8 +241,7 @@ void Application::Start() { // GCOVR_EXCL_START EnterpriseConfigProvider::GetInstance()->Init("enterprise"); LegacyConfigProvider::GetInstance()->Init("legacy"); #else - CommonConfigProvider::GetInstance()->Init("common_v2"); - LegacyCommonConfigProvider::GetInstance()->Init("common"); + InitRemoteConfigProviders(); #endif LogtailAlarm::GetInstance()->Init(); @@ -370,8 +368,10 @@ void Application::Exit() { EnterpriseConfigProvider::GetInstance()->Stop(); LegacyConfigProvider::GetInstance()->Stop(); #else - CommonConfigProvider::GetInstance()->Stop(); - LegacyCommonConfigProvider::GetInstance()->Stop(); + auto remoteConfigProviders = GetRemoteConfigProviders(); + for (auto& provider : remoteConfigProviders) { + provider->Stop(); + } #endif LogtailMonitor::GetInstance()->Stop(); diff --git a/core/config/provider/CommonConfigProvider.cpp b/core/config/common_provider/CommonConfigProvider.cpp similarity index 99% rename from core/config/provider/CommonConfigProvider.cpp rename to core/config/common_provider/CommonConfigProvider.cpp index 77e67b2c5f..c96d5e8399 100644 --- a/core/config/provider/CommonConfigProvider.cpp +++ b/core/config/common_provider/CommonConfigProvider.cpp @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include "config/provider/CommonConfigProvider.h" +#include "CommonConfigProvider.h" #include diff --git a/core/config/provider/CommonConfigProvider.h b/core/config/common_provider/CommonConfigProvider.h similarity index 98% rename from core/config/provider/CommonConfigProvider.h rename to core/config/common_provider/CommonConfigProvider.h index 75f20521b7..804e8cb50c 100644 --- a/core/config/provider/CommonConfigProvider.h +++ b/core/config/common_provider/CommonConfigProvider.h @@ -125,7 +125,7 @@ class CommonConfigProvider : public ConfigProvider, ConfigFeedbackable { ConfigServerAddress GetOneConfigServerAddress(bool changeConfigServer); - VIRTUAL bool SendHttpRequest(const std::string& operation, + virtual bool SendHttpRequest(const std::string& operation, const std::string& reqBody, const std::string& configType, const std::string& requestId, diff --git a/core/config/provider/LegacyCommonConfigProvider.cpp b/core/config/common_provider/LegacyCommonConfigProvider.cpp similarity index 99% rename from core/config/provider/LegacyCommonConfigProvider.cpp rename to core/config/common_provider/LegacyCommonConfigProvider.cpp index 1a017758b6..f5d000711f 100644 --- a/core/config/provider/LegacyCommonConfigProvider.cpp +++ b/core/config/common_provider/LegacyCommonConfigProvider.cpp @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include "config/provider/LegacyCommonConfigProvider.h" +#include "LegacyCommonConfigProvider.h" #include diff --git a/core/config/provider/LegacyCommonConfigProvider.h b/core/config/common_provider/LegacyCommonConfigProvider.h similarity index 100% rename from core/config/provider/LegacyCommonConfigProvider.h rename to core/config/common_provider/LegacyCommonConfigProvider.h diff --git a/core/go_pipeline/LogtailPlugin.cpp b/core/go_pipeline/LogtailPlugin.cpp index bb90dcdecc..ccff0763fc 100644 --- a/core/go_pipeline/LogtailPlugin.cpp +++ b/core/go_pipeline/LogtailPlugin.cpp @@ -30,7 +30,7 @@ #include "monitor/LogtailAlarm.h" #include "pipeline/PipelineManager.h" #include "pipeline/queue/SenderQueueManager.h" -#include "profile_sender/ProfileSender.h" +#include "provider/Provider.h" DEFINE_FLAG_BOOL(enable_sls_metrics_format, "if enable format metrics in SLS metricstore log pattern", false); DEFINE_FLAG_BOOL(enable_containerd_upper_dir_detect, @@ -171,22 +171,22 @@ int LogtailPlugin::SendPbV2(const char* configName, FlusherSLS* pConfig = NULL; if (configNameStr == alarmConfig->mLogstore) { pConfig = alarmConfig; - pConfig->mProject = ProfileSender::GetInstance()->GetDefaultProfileProjectName(); - pConfig->mRegion = ProfileSender::GetInstance()->GetDefaultProfileRegion(); + pConfig->mProject = GetProfileSender()->GetDefaultProfileProjectName(); + pConfig->mRegion = GetProfileSender()->GetDefaultProfileRegion(); if (pConfig->mProject.empty()) { return 0; } } else if (configNameStr == profileConfig->mLogstore) { pConfig = profileConfig; - pConfig->mProject = ProfileSender::GetInstance()->GetDefaultProfileProjectName(); - pConfig->mRegion = ProfileSender::GetInstance()->GetDefaultProfileRegion(); + pConfig->mProject = GetProfileSender()->GetDefaultProfileProjectName(); + pConfig->mRegion = GetProfileSender()->GetDefaultProfileRegion(); if (pConfig->mProject.empty()) { return 0; } } else if (configNameStr == containerConfig->mLogstore) { pConfig = containerConfig; - pConfig->mProject = ProfileSender::GetInstance()->GetDefaultProfileProjectName(); - pConfig->mRegion = ProfileSender::GetInstance()->GetDefaultProfileRegion(); + pConfig->mProject = GetProfileSender()->GetDefaultProfileProjectName(); + pConfig->mRegion = GetProfileSender()->GetDefaultProfileRegion(); if (pConfig->mProject.empty()) { return 0; } diff --git a/core/monitor/LogFileProfiler.cpp b/core/monitor/LogFileProfiler.cpp index 701cd03a33..b9dbaf2794 100644 --- a/core/monitor/LogFileProfiler.cpp +++ b/core/monitor/LogFileProfiler.cpp @@ -27,7 +27,7 @@ #include "common/version.h" #include "file_server/ConfigManager.h" #include "logger/Logger.h" -#include "profile_sender/ProfileSender.h" +#include "provider/Provider.h" #include "pipeline/queue/QueueKeyManager.h" DEFINE_FLAG_INT32(profile_data_send_interval, "interval of send LogFile/DomainSocket profile data, seconds", 600); @@ -261,7 +261,7 @@ void LogFileProfiler::SendProfileData(bool forceSend) { } } UpdateDumpData(logGroup, detail, logstore); - ProfileSender::GetInstance()->SendToProfileProject(region, logGroup); + GetProfileSender()->SendToProfileProject(region, logGroup); } while (true); DumpToLocal(curTime, forceSend, detail, logstore); mLastSendTime = curTime; diff --git a/core/monitor/LogtailAlarm.cpp b/core/monitor/LogtailAlarm.cpp index 65cf05314d..cde2320161 100644 --- a/core/monitor/LogtailAlarm.cpp +++ b/core/monitor/LogtailAlarm.cpp @@ -23,7 +23,7 @@ #include "common/TimeUtil.h" #include "common/version.h" #include "protobuf/sls/sls_logs.pb.h" -#include "profile_sender/ProfileSender.h" +#include "provider/Provider.h" #include "pipeline/queue/QueueKeyManager.h" #include "pipeline/queue/SenderQueueManager.h" @@ -195,7 +195,7 @@ void LogtailAlarm::SendAllRegionAlarm() { // check sender queue status, if invalid jump this region QueueKey alarmPrjLogstoreKey = QueueKeyManager::GetInstance()->GetKey( - "-flusher_sls-" + ProfileSender::GetInstance()->GetProfileProjectName(region) + "#logtail_alarm"); + "-flusher_sls-" + GetProfileSender()->GetProfileProjectName(region) + "#logtail_alarm"); if (!SenderQueueManager::GetInstance()->IsValidToPush(alarmPrjLogstoreKey)) { // jump this region ++sendRegionIndex; @@ -264,7 +264,7 @@ void LogtailAlarm::SendAllRegionAlarm() { continue; } // this is an anonymous send and non lock send - ProfileSender::GetInstance()->SendToProfileProject(region, logGroup); + GetProfileSender()->SendToProfileProject(region, logGroup); } while (true); } @@ -298,7 +298,7 @@ void LogtailAlarm::SendAlarm(const LogtailAlarmType alarmType, } // ignore alarm for profile data - if (ProfileSender::GetInstance()->IsProfileData(region, projectName, category)) { + if (GetProfileSender()->IsProfileData(region, projectName, category)) { return; } // LOG_DEBUG(sLogger, ("Add Alarm", region)("projectName", projectName)("alarm index", diff --git a/core/monitor/MetricExportor.cpp b/core/monitor/MetricExportor.cpp index f8a1b10298..e8198d8150 100644 --- a/core/monitor/MetricExportor.cpp +++ b/core/monitor/MetricExportor.cpp @@ -90,10 +90,10 @@ void MetricExportor::SendToSLS(std::map& logGr logGroup->set_source(LogFileProfiler::mIpAddr); logGroup->set_topic(METRIC_TOPIC_TYPE); if (METRIC_REGION_DEFAULT == iter->first) { - ProfileSender::GetInstance()->SendToProfileProject(ProfileSender::GetInstance()->GetDefaultProfileRegion(), + GetProfileSender()->SendToProfileProject(GetProfileSender()->GetDefaultProfileRegion(), *logGroup); } else { - ProfileSender::GetInstance()->SendToProfileProject(iter->first, *logGroup); + GetProfileSender()->SendToProfileProject(iter->first, *logGroup); } delete logGroup; } diff --git a/core/monitor/MetricExportor.h b/core/monitor/MetricExportor.h index dfb524814a..09c3908a3f 100644 --- a/core/monitor/MetricExportor.h +++ b/core/monitor/MetricExportor.h @@ -17,7 +17,7 @@ #pragma once #include "Monitor.h" -#include "profile_sender/ProfileSender.h" +#include "provider/Provider.h" namespace logtail { diff --git a/core/monitor/Monitor.cpp b/core/monitor/Monitor.cpp index 54cf59b0fb..f3bfe2b391 100644 --- a/core/monitor/Monitor.cpp +++ b/core/monitor/Monitor.cpp @@ -50,7 +50,7 @@ #include "config/provider/EnterpriseConfigProvider.h" #endif #include "pipeline/PipelineManager.h" -#include "profile_sender/ProfileSender.h" +#include "provider/Provider.h" using namespace std; using namespace sls_logs; @@ -299,7 +299,7 @@ bool LogtailMonitor::SendStatusProfile(bool suicide) { AddLogContent(logPtr, "plugin_stats", PipelineManager::GetInstance()->GetPluginStatistics()); // Metrics. vector allProfileRegion; - ProfileSender::GetInstance()->GetAllProfileRegion(allProfileRegion); + GetProfileSender()->GetAllProfileRegion(allProfileRegion); UpdateMetric("region", allProfileRegion); #ifdef __ENTERPRISE__ UpdateMetric("config_update_count", EnterpriseConfigProvider::GetInstance()->GetConfigUpdateTotalCount()); @@ -347,10 +347,10 @@ bool LogtailMonitor::SendStatusProfile(bool suicide) { } if (i == allProfileRegion.size() - 1) { - ProfileSender::GetInstance()->SendToProfileProject(allProfileRegion[i], logGroup); + GetProfileSender()->SendToProfileProject(allProfileRegion[i], logGroup); } else { LogGroup copyLogGroup = logGroup; - ProfileSender::GetInstance()->SendToProfileProject(allProfileRegion[i], copyLogGroup); + GetProfileSender()->SendToProfileProject(allProfileRegion[i], copyLogGroup); } } return true; diff --git a/core/monitor/Monitor.h b/core/monitor/Monitor.h index d0cac3314e..5fcc551dd5 100644 --- a/core/monitor/Monitor.h +++ b/core/monitor/Monitor.h @@ -24,6 +24,7 @@ #include "LogtailMetric.h" #include "MetricConstants.h" #include "MetricStore.h" + #if defined(_MSC_VER) #include #endif diff --git a/core/plugin/flusher/sls/DiskBufferWriter.cpp b/core/plugin/flusher/sls/DiskBufferWriter.cpp index 49bb2789d0..94471e1e54 100644 --- a/core/plugin/flusher/sls/DiskBufferWriter.cpp +++ b/core/plugin/flusher/sls/DiskBufferWriter.cpp @@ -27,7 +27,7 @@ #include "protobuf/sls/sls_logs.pb.h" #include "logger/Logger.h" #include "monitor/LogtailAlarm.h" -#include "profile_sender/ProfileSender.h" +#include "provider/Provider.h" #include "pipeline/queue/QueueKeyManager.h" #include "pipeline/queue/SLSSenderQueueItem.h" #include "sdk/Exception.h" @@ -765,7 +765,7 @@ SendResult DiskBufferWriter::SendToNetSync(sdk::Client* sendClient, bufferMeta.logstore(), ""); // no region - if (!ProfileSender::GetInstance()->IsProfileData("", bufferMeta.project(), bufferMeta.logstore())) + if (!GetProfileSender()->IsProfileData("", bufferMeta.project(), bufferMeta.logstore())) LOG_ERROR(sLogger, ("send data to SLS fail, error_code", errorCode)("error_message", ex.GetMessage())( "endpoint", sendClient->GetRawSlsHost())("projectName", bufferMeta.project())( diff --git a/core/plugin/flusher/sls/FlusherSLS.cpp b/core/plugin/flusher/sls/FlusherSLS.cpp index 5f965a4535..1650d9fb79 100644 --- a/core/plugin/flusher/sls/FlusherSLS.cpp +++ b/core/plugin/flusher/sls/FlusherSLS.cpp @@ -33,7 +33,7 @@ #include "plugin/flusher/sls/SLSClientManager.h" #include "plugin/flusher/sls/SLSResponse.h" #include "plugin/flusher/sls/SendResult.h" -#include "profile_sender/ProfileSender.h" +#include "provider/Provider.h" #include "runner/FlusherRunner.h" #include "sdk/Common.h" #include "sls_control/SLSControl.h" @@ -610,7 +610,7 @@ void FlusherSLS::OnSendDone(const HttpResponse& response, SenderQueueItem* item) auto data = static_cast(item); string configName = HasContext() ? GetContext().GetConfigName() : ""; - bool isProfileData = ProfileSender::GetInstance()->IsProfileData(mRegion, mProject, data->mLogstore); + bool isProfileData = GetProfileSender()->IsProfileData(mRegion, mProject, data->mLogstore); int32_t curTime = time(NULL); auto curSystemTime = chrono::system_clock::now(); if (slsResponse.mStatusCode == 200) { diff --git a/core/provider/CMakeLists.txt b/core/provider/CMakeLists.txt new file mode 100644 index 0000000000..b0e0feb6c3 --- /dev/null +++ b/core/provider/CMakeLists.txt @@ -0,0 +1,39 @@ +# Copyright 2024 iLogtail Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +cmake_minimum_required(VERSION 3.22) +project(provider) + +file(GLOB LIB_SOURCE_FILES *.cpp *.h) + +set(PROVIDER_SUB_DIRECTORIES_LIST + profile_sender config/feedbacker config/provider config/common_provider protobuf/config_server/v1 protobuf/config_server/v2 +) + +foreach(DIR_NAME IN LISTS PROVIDER_SUB_DIRECTORIES_LIST) + file(GLOB TEMP_SOURCE_FILES + "${CMAKE_CURRENT_SOURCE_DIR}/../${DIR_NAME}/*.cpp" + "${CMAKE_CURRENT_SOURCE_DIR}/../${DIR_NAME}/*.cc" + "${CMAKE_CURRENT_SOURCE_DIR}/../${DIR_NAME}/*.c" + ) + list(APPEND LIB_SOURCE_FILES ${TEMP_SOURCE_FILES}) +endforeach() + +add_library(${PROJECT_NAME} STATIC ${LIB_SOURCE_FILES}) + +# message(STATUS "PROVIDER SOURCE FILES: ${LIB_SOURCE_FILES}") + +foreach(DIR_NAME IN LISTS PROVIDER_SUB_DIRECTORIES_LIST) + target_include_directories(provider PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/../${DIR_NAME}) +endforeach() \ No newline at end of file diff --git a/core/provider/Provider.cpp b/core/provider/Provider.cpp new file mode 100644 index 0000000000..52523b256c --- /dev/null +++ b/core/provider/Provider.cpp @@ -0,0 +1,41 @@ +/* + * Copyright 2024 iLogtail Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "Provider.h" + +#include "config/common_provider/CommonConfigProvider.h" +#include "config/common_provider/LegacyCommonConfigProvider.h" + +namespace logtail { + + +std::vector GetRemoteConfigProviders() { + std::vector providers; + providers.push_back(LegacyCommonConfigProvider::GetInstance()); + providers.push_back(CommonConfigProvider::GetInstance()); + return providers; +} + +void InitRemoteConfigProviders() { + CommonConfigProvider::GetInstance()->Init("common_v2"); + LegacyCommonConfigProvider::GetInstance()->Init("common"); +} + +ProfileSender* GetProfileSender() { + return ProfileSender::GetInstance(); +} + +} // namespace logtail \ No newline at end of file diff --git a/core/provider/Provider.h b/core/provider/Provider.h new file mode 100644 index 0000000000..999aaa8a62 --- /dev/null +++ b/core/provider/Provider.h @@ -0,0 +1,37 @@ +/* + * Copyright 2024 iLogtail Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include "config/provider/ConfigProvider.h" +#include "profile_sender/ProfileSender.h" + +namespace logtail { +/* + providers modules are used to replace the default implementation of ilogtail. +*/ + +// GetRemoteConfigProviders returns a vector of pairs for remote config providers. +// It currently returns two providers: LegacyCommonConfigProvider and CommonConfigProvider. +std::vector GetRemoteConfigProviders(); + +// InitRemoteConfigProviders initializes the remote config providers. +// It currently initializes the LegacyCommonConfigProvider and CommonConfigProvider. +void InitRemoteConfigProviders(); + +// GetProfileSender returns the ProfileSender instance. +ProfileSender* GetProfileSender(); +} // namespace logtail \ No newline at end of file diff --git a/core/unittest/CMakeLists.txt b/core/unittest/CMakeLists.txt index 111bff640b..ba42e5bf98 100644 --- a/core/unittest/CMakeLists.txt +++ b/core/unittest/CMakeLists.txt @@ -44,6 +44,7 @@ macro(add_core_subdir) add_subdirectory(plugin) add_subdirectory(polling) add_subdirectory(processor) + add_subdirectory(provider) add_subdirectory(queue) add_subdirectory(reader) add_subdirectory(sdk) @@ -68,6 +69,7 @@ macro(ut_link ut_link_withspl) flusher_link(${UT_BASE_TARGET}) all_link(${UT_BASE_TARGET}) common_link(${UT_BASE_TARGET}) + target_link_libraries(${UT_BASE_TARGET} provider) endmacro() set(SOURCE_FILES_CORE ${FRAMEWORK_SOURCE_FILES} ${PLUGIN_SOURCE_FILES_CORE}) diff --git a/core/unittest/config/CommonConfigProviderUnittest.cpp b/core/unittest/config/CommonConfigProviderUnittest.cpp index d68f049327..e434210b7c 100644 --- a/core/unittest/config/CommonConfigProviderUnittest.cpp +++ b/core/unittest/config/CommonConfigProviderUnittest.cpp @@ -19,7 +19,7 @@ #include "common/FileSystemUtil.h" #include "common/version.h" #include "config/ConfigDiff.h" -#include "config/provider/CommonConfigProvider.h" +#include "config/common_provider/CommonConfigProvider.h" #include "config/watcher/ConfigWatcher.h" #include "gmock/gmock.h" #include "pipeline/PipelineManager.h" diff --git a/core/unittest/provider/CMakeLists.txt b/core/unittest/provider/CMakeLists.txt new file mode 100644 index 0000000000..e3f7f7abaf --- /dev/null +++ b/core/unittest/provider/CMakeLists.txt @@ -0,0 +1,22 @@ +# Copyright 2024 iLogtail Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +cmake_minimum_required(VERSION 3.22) +project(provider_unittest) + +add_executable(provider_unittest ProviderUnittest.cpp) +target_link_libraries(provider_unittest ${UT_BASE_TARGET}) + +include(GoogleTest) +gtest_discover_tests(provider_unittest) diff --git a/core/unittest/provider/ProviderUnittest.cpp b/core/unittest/provider/ProviderUnittest.cpp new file mode 100644 index 0000000000..7fb455699b --- /dev/null +++ b/core/unittest/provider/ProviderUnittest.cpp @@ -0,0 +1,42 @@ +// Copyright 2023 iLogtail Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "unittest/Unittest.h" +#include "provider/Provider.h" + + +namespace logtail { + +class ProviderUnittest : public testing::Test { +public: + void TestGetRemoteConfigProvider(); + void TestGetProfileSender(); + +}; +void ProviderUnittest::TestGetRemoteConfigProvider() { + auto remoteConfigProviders = GetRemoteConfigProviders(); + APSARA_TEST_GT(remoteConfigProviders.size(), 0); +} + +void ProviderUnittest::TestGetProfileSender() { + auto profileSender = GetProfileSender(); + APSARA_TEST_NOT_EQUAL(nullptr, profileSender); +} + + +UNIT_TEST_CASE(ProviderUnittest, TestGetRemoteConfigProvider) +UNIT_TEST_CASE(ProviderUnittest, TestGetProfileSender) +} // namespace logtail + +UNIT_TEST_MAIN \ No newline at end of file diff --git a/docs/cn/installation/sources/build.md b/docs/cn/installation/sources/build.md index 8eca9dded5..60703e3c0a 100644 --- a/docs/cn/installation/sources/build.md +++ b/docs/cn/installation/sources/build.md @@ -119,3 +119,13 @@ cd 到ilogtail/scripts目录下，执行windows64_build.bat(windows32_build.bat) - PluginAdapter.dll (插件接口) - PluginBase.dll (插件lib) - PluginBase.h + + +## 编译时替换外部模块 +iLogtail通过 Provider 模块暴露出一些拓展点，这些拓展点可以由用户自行实现，并通过编译时CMAKE DPROVIDER_PATH选项替换掉默认的实现。 + +示例： + +```bash +cmake -DPROVIDER_PATH=../../../core_extensions/provider .. +``` \ No newline at end of file From 832d52d7a5db91de3fa4ba3bc43c909bbbd68543 Mon Sep 17 00:00:00 2001 From: dog Date: Thu, 19 Sep 2024 16:41:23 +0800 Subject: [PATCH 4/9] fix: pass wrong interval seconds when GetRandSleepMilliSec for target scheduler (#1770) --- core/prometheus/schedulers/TargetSubscriberScheduler.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/prometheus/schedulers/TargetSubscriberScheduler.cpp b/core/prometheus/schedulers/TargetSubscriberScheduler.cpp index 8a77c200fd..c8bc2d501e 100644 --- a/core/prometheus/schedulers/TargetSubscriberScheduler.cpp +++ b/core/prometheus/schedulers/TargetSubscriberScheduler.cpp @@ -217,7 +217,7 @@ TargetSubscriberScheduler::BuildScrapeSchedulerSet(std::vector& targetGr scrapeScheduler->SetTimer(mTimer); auto randSleepMilliSec = GetRandSleepMilliSec( - scrapeScheduler->GetId(), prometheus::RefeshIntervalSeconds, GetCurrentTimeInMilliSeconds()); + scrapeScheduler->GetId(), mScrapeConfigPtr->mScrapeIntervalSeconds, GetCurrentTimeInMilliSeconds()); auto firstExecTime = std::chrono::steady_clock::now() + std::chrono::milliseconds(randSleepMilliSec); scrapeScheduler->SetFirstExecTime(firstExecTime); From 4f9411bc0b74260da98bae0717f4fd9dfc3c35e5 Mon Sep 17 00:00:00 2001 From: henryzhx8 Date: Fri, 20 Sep 2024 15:50:31 +0800 Subject: [PATCH 5/9] add setpipelineforitems for process queues (#1769) --- core/pipeline/Pipeline.cpp | 2 +- core/pipeline/PipelineManager.h | 4 + core/pipeline/queue/BoundedProcessQueue.cpp | 15 +- core/pipeline/queue/BoundedProcessQueue.h | 3 +- core/pipeline/queue/CircularProcessQueue.cpp | 10 ++ core/pipeline/queue/CircularProcessQueue.h | 1 + .../queue/ExactlyOnceQueueManager.cpp | 11 +- core/pipeline/queue/ExactlyOnceQueueManager.h | 4 +- core/pipeline/queue/ProcessQueueInterface.h | 8 +- core/pipeline/queue/ProcessQueueManager.cpp | 15 +- core/pipeline/queue/ProcessQueueManager.h | 4 +- .../queue/BoundedProcessQueueUnittest.cpp | 28 +++- .../queue/CircularProcessQueueUnittest.cpp | 28 +++- .../queue/ExactlyOnceQueueManagerUnittest.cpp | 61 ++++++- .../queue/ProcessQueueManagerUnittest.cpp | 154 +++++++++++++++--- core/unittest/reader/ForceReadUnittest.cpp | 1 + 16 files changed, 294 insertions(+), 55 deletions(-) diff --git a/core/pipeline/Pipeline.cpp b/core/pipeline/Pipeline.cpp index 7d38880084..0f841798f1 100644 --- a/core/pipeline/Pipeline.cpp +++ b/core/pipeline/Pipeline.cpp @@ -337,7 +337,7 @@ void Pipeline::Start() { // TODO: 加载该Go流水线 } - // TODO: 启用Process中改流水线对应的输入队列 + ProcessQueueManager::GetInstance()->EnablePop(mName); if (!mGoPipelineWithInput.isNull()) { // TODO: 加载该Go流水线 diff --git a/core/pipeline/PipelineManager.h b/core/pipeline/PipelineManager.h index 5f63b0fd25..171a1e7084 100644 --- a/core/pipeline/PipelineManager.h +++ b/core/pipeline/PipelineManager.h @@ -71,6 +71,10 @@ class PipelineManager { #ifdef APSARA_UNIT_TEST_MAIN friend class PipelineManagerMock; friend class PipelineManagerUnittest; + friend class ProcessQueueManagerUnittest; + friend class ExactlyOnceQueueManagerUnittest; + friend class BoundedProcessQueueUnittest; + friend class CircularProcessQueueUnittest; friend class CommonConfigProviderUnittest; #endif }; diff --git a/core/pipeline/queue/BoundedProcessQueue.cpp b/core/pipeline/queue/BoundedProcessQueue.cpp index ced855e43e..b577cf79fc 100644 --- a/core/pipeline/queue/BoundedProcessQueue.cpp +++ b/core/pipeline/queue/BoundedProcessQueue.cpp @@ -14,6 +14,8 @@ #include "pipeline/queue/BoundedProcessQueue.h" +#include "pipeline/PipelineManager.h" + using namespace std; namespace logtail { @@ -35,7 +37,7 @@ bool BoundedProcessQueue::Push(unique_ptr&& item) { } item->mEnqueTime = chrono::system_clock::now(); auto size = item->mEventGroup.DataSize(); - mQueue.push(std::move(item)); + mQueue.push_back(std::move(item)); ChangeStateIfNeededAfterPush(); mInItemsCnt->Add(1); @@ -51,7 +53,7 @@ bool BoundedProcessQueue::Pop(unique_ptr& item) { return false; } item = std::move(mQueue.front()); - mQueue.pop(); + mQueue.pop_front(); if (ChangeStateIfNeededAfterPop()) { GiveFeedback(); } @@ -65,6 +67,15 @@ bool BoundedProcessQueue::Pop(unique_ptr& item) { return true; } +void BoundedProcessQueue::SetPipelineForItems(const std::string& name) const { + auto p = PipelineManager::GetInstance()->FindConfigByName(name); + for (auto& item : mQueue) { + if (!item->mPipeline) { + item->mPipeline = p; + } + } +} + void BoundedProcessQueue::SetUpStreamFeedbacks(vector&& feedbacks) { mUpStreamFeedbacks.clear(); for (auto& item : feedbacks) { diff --git a/core/pipeline/queue/BoundedProcessQueue.h b/core/pipeline/queue/BoundedProcessQueue.h index 78c7772fd0..1a4c793e9f 100644 --- a/core/pipeline/queue/BoundedProcessQueue.h +++ b/core/pipeline/queue/BoundedProcessQueue.h @@ -35,6 +35,7 @@ class BoundedProcessQueue : public BoundedQueueInterface&& item) override; bool Pop(std::unique_ptr& item) override; + void SetPipelineForItems(const std::string& name) const override; void SetUpStreamFeedbacks(std::vector&& feedbacks); @@ -43,7 +44,7 @@ class BoundedProcessQueue : public BoundedQueueInterface> mQueue; + std::deque> mQueue; std::vector mUpStreamFeedbacks; #ifdef APSARA_UNIT_TEST_MAIN diff --git a/core/pipeline/queue/CircularProcessQueue.cpp b/core/pipeline/queue/CircularProcessQueue.cpp index c6df833e7f..e7f8b279f7 100644 --- a/core/pipeline/queue/CircularProcessQueue.cpp +++ b/core/pipeline/queue/CircularProcessQueue.cpp @@ -15,6 +15,7 @@ #include "pipeline/queue/CircularProcessQueue.h" #include "logger/Logger.h" +#include "pipeline/PipelineManager.h" #include "pipeline/queue/QueueKeyManager.h" using namespace std; @@ -71,6 +72,15 @@ bool CircularProcessQueue::Pop(unique_ptr& item) { return true; } +void CircularProcessQueue::SetPipelineForItems(const std::string& name) const { + auto p = PipelineManager::GetInstance()->FindConfigByName(name); + for (auto& item : mQueue) { + if (!item->mPipeline) { + item->mPipeline = p; + } + } +} + void CircularProcessQueue::Reset(size_t cap) { // it seems more reasonable to retain extra items and process them immediately, however this contray to current // framework design so we simply discard extra items, considering that it is a rare case to change capacity diff --git a/core/pipeline/queue/CircularProcessQueue.h b/core/pipeline/queue/CircularProcessQueue.h index 813929c799..a58a34e035 100644 --- a/core/pipeline/queue/CircularProcessQueue.h +++ b/core/pipeline/queue/CircularProcessQueue.h @@ -33,6 +33,7 @@ class CircularProcessQueue : virtual public QueueInterface&& item) override; bool Pop(std::unique_ptr& item) override; + void SetPipelineForItems(const std::string& name) const override; void Reset(size_t cap); diff --git a/core/pipeline/queue/ExactlyOnceQueueManager.cpp b/core/pipeline/queue/ExactlyOnceQueueManager.cpp index 3c66b636e3..b78aba82d8 100644 --- a/core/pipeline/queue/ExactlyOnceQueueManager.cpp +++ b/core/pipeline/queue/ExactlyOnceQueueManager.cpp @@ -148,20 +148,23 @@ bool ExactlyOnceQueueManager::IsAllProcessQueueEmpty() const { return true; } -void ExactlyOnceQueueManager::InvalidatePopProcessQueue(const string& configName) { +void ExactlyOnceQueueManager::DisablePopProcessQueue(const string& configName, bool isPipelineRemoving) { lock_guard lock(mProcessQueueMux); for (auto& iter : mProcessQueues) { if (iter.second->GetConfigName() == configName) { - iter.second->InvalidatePop(); + iter.second->DisablePop(); + if (!isPipelineRemoving) { + iter.second->SetPipelineForItems(configName); + } } } } -void ExactlyOnceQueueManager::ValidatePopProcessQueue(const string& configName) { +void ExactlyOnceQueueManager::EnablePopProcessQueue(const string& configName) { lock_guard lock(mProcessQueueMux); for (auto& iter : mProcessQueues) { if (iter.second->GetConfigName() == configName) { - iter.second->ValidatePop(); + iter.second->EnablePop(); } } } diff --git a/core/pipeline/queue/ExactlyOnceQueueManager.h b/core/pipeline/queue/ExactlyOnceQueueManager.h index 7f5e8a70ea..7ca468f84d 100644 --- a/core/pipeline/queue/ExactlyOnceQueueManager.h +++ b/core/pipeline/queue/ExactlyOnceQueueManager.h @@ -59,8 +59,8 @@ class ExactlyOnceQueueManager { // 0: success, 1: queue is full, 2: queue not found int PushProcessQueue(QueueKey key, std::unique_ptr&& item); bool IsAllProcessQueueEmpty() const; - void InvalidatePopProcessQueue(const std::string& configName); - void ValidatePopProcessQueue(const std::string& configName); + void DisablePopProcessQueue(const std::string& configName, bool isPipelineRemoving); + void EnablePopProcessQueue(const std::string& configName); // 0: success, 1: queue is full, 2: queue not found int PushSenderQueue(QueueKey key, std::unique_ptr&& item); diff --git a/core/pipeline/queue/ProcessQueueInterface.h b/core/pipeline/queue/ProcessQueueInterface.h index 6a3b8833c8..12baec4029 100644 --- a/core/pipeline/queue/ProcessQueueInterface.h +++ b/core/pipeline/queue/ProcessQueueInterface.h @@ -42,8 +42,10 @@ class ProcessQueueInterface : virtual public QueueInterface&& ques); - void InvalidatePop() { mValidToPop = false; } - void ValidatePop() { mValidToPop = true; } + void DisablePop() { mValidToPop = false; } + void EnablePop() { mValidToPop = true; } + + virtual void SetPipelineForItems(const std::string& name) const = 0; void Reset() { mDownStreamQueues.clear(); } @@ -57,7 +59,7 @@ class ProcessQueueInterface : virtual public QueueInterface mDownStreamQueues; - bool mValidToPop = true; + bool mValidToPop = false; #ifdef APSARA_UNIT_TEST_MAIN friend class BoundedProcessQueueUnittest; diff --git a/core/pipeline/queue/ProcessQueueManager.cpp b/core/pipeline/queue/ProcessQueueManager.cpp index 0b66f6769b..26bc1b8072 100644 --- a/core/pipeline/queue/ProcessQueueManager.cpp +++ b/core/pipeline/queue/ProcessQueueManager.cpp @@ -229,29 +229,32 @@ bool ProcessQueueManager::SetFeedbackInterface(QueueKey key, vectorHasKey(configName)) { auto key = QueueKeyManager::GetInstance()->GetKey(configName); lock_guard lock(mQueueMux); auto iter = mQueues.find(key); if (iter != mQueues.end()) { - (*iter->second.first)->InvalidatePop(); + (*iter->second.first)->DisablePop(); + if (!isPipelineRemoving) { + (*iter->second.first)->SetPipelineForItems(configName); + } } } else { - ExactlyOnceQueueManager::GetInstance()->InvalidatePopProcessQueue(configName); + ExactlyOnceQueueManager::GetInstance()->DisablePopProcessQueue(configName, isPipelineRemoving); } } -void ProcessQueueManager::ValidatePop(const string& configName) { +void ProcessQueueManager::EnablePop(const string& configName) { if (QueueKeyManager::GetInstance()->HasKey(configName)) { auto key = QueueKeyManager::GetInstance()->GetKey(configName); lock_guard lock(mQueueMux); auto iter = mQueues.find(key); if (iter != mQueues.end()) { - (*iter->second.first)->ValidatePop(); + (*iter->second.first)->EnablePop(); } } else { - ExactlyOnceQueueManager::GetInstance()->ValidatePopProcessQueue(configName); + ExactlyOnceQueueManager::GetInstance()->EnablePopProcessQueue(configName); } } diff --git a/core/pipeline/queue/ProcessQueueManager.h b/core/pipeline/queue/ProcessQueueManager.h index dd0198c4b4..ab877a2c7d 100644 --- a/core/pipeline/queue/ProcessQueueManager.h +++ b/core/pipeline/queue/ProcessQueueManager.h @@ -62,8 +62,8 @@ class ProcessQueueManager : public FeedbackInterface { bool IsAllQueueEmpty() const; bool SetDownStreamQueues(QueueKey key, std::vector&& ques); bool SetFeedbackInterface(QueueKey key, std::vector&& feedback); - void InvalidatePop(const std::string& configName); - void ValidatePop(const std::string& configName); + void DisablePop(const std::string& configName, bool isPipelineRemoving); + void EnablePop(const std::string& configName); bool Wait(uint64_t ms); void Trigger(); diff --git a/core/unittest/queue/BoundedProcessQueueUnittest.cpp b/core/unittest/queue/BoundedProcessQueueUnittest.cpp index e2228108d5..bd68ccb470 100644 --- a/core/unittest/queue/BoundedProcessQueueUnittest.cpp +++ b/core/unittest/queue/BoundedProcessQueueUnittest.cpp @@ -16,6 +16,7 @@ #include "common/FeedbackInterface.h" #include "models/PipelineEventGroup.h" +#include "pipeline/PipelineManager.h" #include "pipeline/queue/BoundedProcessQueue.h" #include "pipeline/queue/SenderQueue.h" #include "unittest/Unittest.h" @@ -30,6 +31,7 @@ class BoundedProcessQueueUnittest : public testing::Test { void TestPush(); void TestPop(); void TestMetric(); + void TestSetPipeline(); protected: static void SetUpTestCase() { sCtx.SetConfigName("test_config"); } @@ -44,6 +46,7 @@ class BoundedProcessQueueUnittest : public testing::Test { mFeedback1.reset(new FeedbackInterfaceMock); mFeedback2.reset(new FeedbackInterfaceMock); mQueue->SetUpStreamFeedbacks(vector{mFeedback1.get(), mFeedback2.get()}); + mQueue->EnablePop(); } private: @@ -91,9 +94,9 @@ void BoundedProcessQueueUnittest::TestPop() { mQueue->Push(GenerateItem()); // invalidate pop - mQueue->InvalidatePop(); + mQueue->DisablePop(); APSARA_TEST_EQUAL(0, mQueue->Pop(item)); - mQueue->ValidatePop(); + mQueue->EnablePop(); // downstream queues are not valid to push mSenderQueue1->mValidToPush = false; @@ -140,9 +143,30 @@ void BoundedProcessQueueUnittest::TestMetric() { APSARA_TEST_EQUAL(1U, mQueue->mValidToPushFlag->GetValue()); } +void BoundedProcessQueueUnittest::TestSetPipeline() { + auto pipeline = make_shared(); + PipelineManager::GetInstance()->mPipelineNameEntityMap["test_config"] = pipeline; + + auto item1 = GenerateItem(); + auto p1 = item1.get(); + auto pipelineTmp = make_shared(); + item1->mPipeline = pipelineTmp; + + auto item2 = GenerateItem(); + auto p2 = item2.get(); + + mQueue->Push(std::move(item1)); + mQueue->Push(std::move(item2)); + mQueue->SetPipelineForItems("test_config"); + + APSARA_TEST_EQUAL(pipelineTmp, p1->mPipeline); + APSARA_TEST_EQUAL(pipeline, p2->mPipeline); +} + UNIT_TEST_CASE(BoundedProcessQueueUnittest, TestPush) UNIT_TEST_CASE(BoundedProcessQueueUnittest, TestPop) UNIT_TEST_CASE(BoundedProcessQueueUnittest, TestMetric) +UNIT_TEST_CASE(BoundedProcessQueueUnittest, TestSetPipeline) } // namespace logtail diff --git a/core/unittest/queue/CircularProcessQueueUnittest.cpp b/core/unittest/queue/CircularProcessQueueUnittest.cpp index 7b5ef9d0d3..82fce43912 100644 --- a/core/unittest/queue/CircularProcessQueueUnittest.cpp +++ b/core/unittest/queue/CircularProcessQueueUnittest.cpp @@ -15,6 +15,7 @@ #include #include "models/PipelineEventGroup.h" +#include "pipeline/PipelineManager.h" #include "pipeline/queue/CircularProcessQueue.h" #include "pipeline/queue/SenderQueue.h" #include "unittest/Unittest.h" @@ -28,6 +29,7 @@ class CircularProcessQueueUnittest : public testing::Test { void TestPop(); void TestReset(); void TestMetric(); + void TestSetPipeline(); protected: static void SetUpTestCase() { sCtx.SetConfigName("test_config"); } @@ -38,6 +40,7 @@ class CircularProcessQueueUnittest : public testing::Test { mSenderQueue1.reset(new SenderQueue(10, 0, 10, 0, "", sCtx)); mSenderQueue2.reset(new SenderQueue(10, 0, 10, 0, "", sCtx)); mQueue->SetDownStreamQueues(vector{mSenderQueue1.get(), mSenderQueue2.get()}); + mQueue->EnablePop(); } private: @@ -97,9 +100,9 @@ void CircularProcessQueueUnittest::TestPop() { mQueue->Push(GenerateItem(1)); // invalidate pop - mQueue->InvalidatePop(); + mQueue->DisablePop(); APSARA_TEST_FALSE(mQueue->Pop(item)); - mQueue->ValidatePop(); + mQueue->EnablePop(); // downstream queues are not valid to push mSenderQueue1->mValidToPush = false; @@ -177,10 +180,31 @@ void CircularProcessQueueUnittest::TestMetric() { APSARA_TEST_EQUAL(0U, mQueue->mQueueDataSizeByte->GetValue()); } +void CircularProcessQueueUnittest::TestSetPipeline() { + auto pipeline = make_shared(); + PipelineManager::GetInstance()->mPipelineNameEntityMap["test_config"] = pipeline; + + auto item1 = GenerateItem(1); + auto p1 = item1.get(); + auto pipelineTmp = make_shared(); + item1->mPipeline = pipelineTmp; + + auto item2 = GenerateItem(1); + auto p2 = item2.get(); + + mQueue->Push(std::move(item1)); + mQueue->Push(std::move(item2)); + mQueue->SetPipelineForItems("test_config"); + + APSARA_TEST_EQUAL(pipelineTmp, p1->mPipeline); + APSARA_TEST_EQUAL(pipeline, p2->mPipeline); +} + UNIT_TEST_CASE(CircularProcessQueueUnittest, TestPush) UNIT_TEST_CASE(CircularProcessQueueUnittest, TestPop) UNIT_TEST_CASE(CircularProcessQueueUnittest, TestReset) UNIT_TEST_CASE(CircularProcessQueueUnittest, TestMetric) +UNIT_TEST_CASE(CircularProcessQueueUnittest, TestSetPipeline) } // namespace logtail diff --git a/core/unittest/queue/ExactlyOnceQueueManagerUnittest.cpp b/core/unittest/queue/ExactlyOnceQueueManagerUnittest.cpp index 5e929eb477..06f1f58b10 100644 --- a/core/unittest/queue/ExactlyOnceQueueManagerUnittest.cpp +++ b/core/unittest/queue/ExactlyOnceQueueManagerUnittest.cpp @@ -15,6 +15,7 @@ #include #include "models/PipelineEventGroup.h" +#include "pipeline/PipelineManager.h" #include "pipeline/queue/ExactlyOnceQueueManager.h" #include "pipeline/queue/QueueKeyManager.h" #include "pipeline/queue/SLSSenderQueueItem.h" @@ -278,16 +279,66 @@ void ExactlyOnceQueueManagerUnittest::TestIsAllSenderQueueEmpty() { void ExactlyOnceQueueManagerUnittest::OnPipelineUpdate() { PipelineContext ctx; ctx.SetConfigName("test_config"); - sManager->CreateOrUpdateQueue(0, 0, ctx, sCheckpoints); sManager->CreateOrUpdateQueue(1, 0, ctx, sCheckpoints); + sManager->CreateOrUpdateQueue(2, 0, ctx, sCheckpoints); - sManager->InvalidatePopProcessQueue("test_config"); - APSARA_TEST_FALSE(sManager->mProcessQueues[0]->mValidToPop); + auto pipeline1 = make_shared(); + PipelineManager::GetInstance()->mPipelineNameEntityMap["test_config"] = pipeline1; + + auto item1 = GenerateProcessItem(); + auto p1 = item1.get(); + sManager->PushProcessQueue(1, std::move(item1)); + + auto item2 = GenerateProcessItem(); + auto p2 = item2.get(); + sManager->PushProcessQueue(2, std::move(item2)); + + sManager->DisablePopProcessQueue("test_config", false); + APSARA_TEST_FALSE(ExactlyOnceQueueManager::GetInstance()->mProcessQueues[1]->mValidToPop); + APSARA_TEST_FALSE(ExactlyOnceQueueManager::GetInstance()->mProcessQueues[2]->mValidToPop); + APSARA_TEST_EQUAL(pipeline1, p1->mPipeline); + APSARA_TEST_EQUAL(pipeline1, p2->mPipeline); + + auto item3 = GenerateProcessItem(); + auto p3 = item3.get(); + sManager->PushProcessQueue(1, std::move(item3)); + + auto item4 = GenerateProcessItem(); + auto p4 = item4.get(); + sManager->PushProcessQueue(2, std::move(item4)); + + auto pipeline2 = make_shared(); + PipelineManager::GetInstance()->mPipelineNameEntityMap["test_config"] = pipeline2; + + sManager->DisablePopProcessQueue("test_config", false); APSARA_TEST_FALSE(sManager->mProcessQueues[1]->mValidToPop); + APSARA_TEST_FALSE(sManager->mProcessQueues[2]->mValidToPop); + APSARA_TEST_EQUAL(pipeline1, p1->mPipeline); + APSARA_TEST_EQUAL(pipeline1, p2->mPipeline); + APSARA_TEST_EQUAL(pipeline2, p3->mPipeline); + APSARA_TEST_EQUAL(pipeline2, p4->mPipeline); + + auto item5 = GenerateProcessItem(); + auto p5 = item5.get(); + sManager->PushProcessQueue(1, std::move(item5)); - sManager->ValidatePopProcessQueue("test_config"); - APSARA_TEST_TRUE(sManager->mProcessQueues[0]->mValidToPop); + auto item6 = GenerateProcessItem(); + auto p6 = item6.get(); + sManager->PushProcessQueue(2, std::move(item6)); + + sManager->DisablePopProcessQueue("test_config", true); + APSARA_TEST_FALSE(sManager->mProcessQueues[1]->mValidToPop); + APSARA_TEST_FALSE(sManager->mProcessQueues[2]->mValidToPop); + APSARA_TEST_EQUAL(pipeline1, p1->mPipeline); + APSARA_TEST_EQUAL(pipeline1, p2->mPipeline); + APSARA_TEST_EQUAL(pipeline2, p3->mPipeline); + APSARA_TEST_EQUAL(pipeline2, p4->mPipeline); + APSARA_TEST_EQUAL(nullptr, p5->mPipeline); + APSARA_TEST_EQUAL(nullptr, p6->mPipeline); + + sManager->EnablePopProcessQueue("test_config"); APSARA_TEST_TRUE(sManager->mProcessQueues[1]->mValidToPop); + APSARA_TEST_TRUE(sManager->mProcessQueues[2]->mValidToPop); } unique_ptr ExactlyOnceQueueManagerUnittest::GenerateProcessItem() { diff --git a/core/unittest/queue/ProcessQueueManagerUnittest.cpp b/core/unittest/queue/ProcessQueueManagerUnittest.cpp index d87bf88e96..21226fe19b 100644 --- a/core/unittest/queue/ProcessQueueManagerUnittest.cpp +++ b/core/unittest/queue/ProcessQueueManagerUnittest.cpp @@ -15,6 +15,7 @@ #include #include "models/PipelineEventGroup.h" +#include "pipeline/PipelineManager.h" #include "pipeline/queue/ExactlyOnceQueueManager.h" #include "pipeline/queue/ProcessQueueManager.h" #include "pipeline/queue/QueueKeyManager.h" @@ -267,17 +268,22 @@ void ProcessQueueManagerUnittest::TestPopItem() { ctx.SetConfigName("test_config_1"); QueueKey key1 = QueueKeyManager::GetInstance()->GetKey("test_config_1"); sProcessQueueManager->CreateOrUpdateBoundedQueue(key1, 0, ctx); + sProcessQueueManager->EnablePop("test_config_1"); ctx.SetConfigName("test_config_2"); QueueKey key2 = QueueKeyManager::GetInstance()->GetKey("test_config_2"); sProcessQueueManager->CreateOrUpdateBoundedQueue(key2, 1, ctx); + sProcessQueueManager->EnablePop("test_config_2"); ctx.SetConfigName("test_config_3"); QueueKey key3 = QueueKeyManager::GetInstance()->GetKey("test_config_3"); sProcessQueueManager->CreateOrUpdateBoundedQueue(key3, 1, ctx); + sProcessQueueManager->EnablePop("test_config_3"); ctx.SetConfigName("test_config_4"); QueueKey key4 = QueueKeyManager::GetInstance()->GetKey("test_config_4"); sProcessQueueManager->CreateOrUpdateBoundedQueue(key4, 1, ctx); + sProcessQueueManager->EnablePop("test_config_4"); ctx.SetConfigName("test_config_5"); ExactlyOnceQueueManager::GetInstance()->CreateOrUpdateQueue(5, 0, ctx, vector(5)); + ExactlyOnceQueueManager::GetInstance()->EnablePopProcessQueue("test_config_5"); sProcessQueueManager->PushQueue(key2, GenerateItem()); sProcessQueueManager->PushQueue(key3, GenerateItem()); @@ -318,10 +324,26 @@ void ProcessQueueManagerUnittest::TestPopItem() { } void ProcessQueueManagerUnittest::TestIsAllQueueEmpty() { - sProcessQueueManager->CreateOrUpdateBoundedQueue(0, 0, sCtx); - sProcessQueueManager->CreateOrUpdateBoundedQueue(1, 1, sCtx); - ExactlyOnceQueueManager::GetInstance()->CreateOrUpdateQueue(2, 0, sCtx, vector(5)); - ExactlyOnceQueueManager::GetInstance()->CreateOrUpdateQueue(3, 2, sCtx, vector(5)); + PipelineContext ctx; + ctx.SetConfigName("test_config_1"); + QueueKey key1 = QueueKeyManager::GetInstance()->GetKey("test_config_1"); + sProcessQueueManager->CreateOrUpdateBoundedQueue(key1, 0, ctx); + sProcessQueueManager->EnablePop("test_config_1"); + + ctx.SetConfigName("test_config_2"); + QueueKey key2 = QueueKeyManager::GetInstance()->GetKey("test_config_2"); + sProcessQueueManager->CreateOrUpdateBoundedQueue(key2, 1, ctx); + sProcessQueueManager->EnablePop("test_config_2"); + + ctx.SetConfigName("test_config_3"); + QueueKey key3 = QueueKeyManager::GetInstance()->GetKey("test_config_3"); + ExactlyOnceQueueManager::GetInstance()->CreateOrUpdateQueue(key3, 0, ctx, vector(5)); + ExactlyOnceQueueManager::GetInstance()->EnablePopProcessQueue("test_config_3"); + + ctx.SetConfigName("test_config_4"); + QueueKey key4 = QueueKeyManager::GetInstance()->GetKey("test_config_4"); + ExactlyOnceQueueManager::GetInstance()->CreateOrUpdateQueue(key4, 2, ctx, vector(5)); + ExactlyOnceQueueManager::GetInstance()->EnablePopProcessQueue("test_config_4"); APSARA_TEST_TRUE(sProcessQueueManager->IsAllQueueEmpty()); // non empty normal process queue @@ -334,7 +356,7 @@ void ProcessQueueManagerUnittest::TestIsAllQueueEmpty() { APSARA_TEST_TRUE(sProcessQueueManager->IsAllQueueEmpty()); // non empty exactly once process queue - sProcessQueueManager->PushQueue(2, GenerateItem()); + sProcessQueueManager->PushQueue(key3, GenerateItem()); APSARA_TEST_FALSE(sProcessQueueManager->IsAllQueueEmpty()); sProcessQueueManager->PopItem(0, item, configName); @@ -342,27 +364,109 @@ void ProcessQueueManagerUnittest::TestIsAllQueueEmpty() { } void ProcessQueueManagerUnittest::OnPipelineUpdate() { - PipelineContext ctx; - ctx.SetConfigName("test_config_1"); + PipelineContext ctx1, ctx2; + ctx1.SetConfigName("test_config_1"); + ctx2.SetConfigName("test_config_2"); QueueKey key = QueueKeyManager::GetInstance()->GetKey("test_config_1"); - sProcessQueueManager->CreateOrUpdateBoundedQueue(key, 0, ctx); - ctx.SetConfigName("test_config_2"); - ExactlyOnceQueueManager::GetInstance()->CreateOrUpdateQueue(1, 0, ctx, vector(5)); - ExactlyOnceQueueManager::GetInstance()->CreateOrUpdateQueue(2, 0, ctx, vector(5)); - - sProcessQueueManager->InvalidatePop("test_config_1"); - APSARA_TEST_FALSE((*sProcessQueueManager->mQueues[key].first)->mValidToPop); - - sProcessQueueManager->InvalidatePop("test_config_2"); - APSARA_TEST_FALSE(ExactlyOnceQueueManager::GetInstance()->mProcessQueues[1]->mValidToPop); - APSARA_TEST_FALSE(ExactlyOnceQueueManager::GetInstance()->mProcessQueues[2]->mValidToPop); - - sProcessQueueManager->ValidatePop("test_config_1"); - APSARA_TEST_TRUE((*sProcessQueueManager->mQueues[key].first)->mValidToPop); - - sProcessQueueManager->ValidatePop("test_config_2"); - APSARA_TEST_TRUE(ExactlyOnceQueueManager::GetInstance()->mProcessQueues[1]->mValidToPop); - APSARA_TEST_TRUE(ExactlyOnceQueueManager::GetInstance()->mProcessQueues[2]->mValidToPop); + sProcessQueueManager->CreateOrUpdateBoundedQueue(key, 0, ctx1); + ExactlyOnceQueueManager::GetInstance()->CreateOrUpdateQueue(1, 0, ctx2, vector(5)); + ExactlyOnceQueueManager::GetInstance()->CreateOrUpdateQueue(2, 0, ctx2, vector(5)); + + auto pipeline1 = make_shared(); + auto pipeline2 = make_shared(); + PipelineManager::GetInstance()->mPipelineNameEntityMap["test_config_1"] = pipeline1; + PipelineManager::GetInstance()->mPipelineNameEntityMap["test_config_2"] = pipeline2; + + { + auto item1 = GenerateItem(); + auto p1 = item1.get(); + sProcessQueueManager->PushQueue(key, std::move(item1)); + + sProcessQueueManager->DisablePop("test_config_1", false); + APSARA_TEST_FALSE((*sProcessQueueManager->mQueues[key].first)->mValidToPop); + APSARA_TEST_EQUAL(pipeline1, p1->mPipeline); + + auto item2 = GenerateItem(); + auto p2 = item2.get(); + sProcessQueueManager->PushQueue(key, std::move(item2)); + + auto pipeline3 = make_shared(); + PipelineManager::GetInstance()->mPipelineNameEntityMap["test_config_1"] = pipeline3; + + sProcessQueueManager->DisablePop("test_config_1", false); + APSARA_TEST_FALSE((*sProcessQueueManager->mQueues[key].first)->mValidToPop); + APSARA_TEST_EQUAL(pipeline1, p1->mPipeline); + APSARA_TEST_EQUAL(pipeline3, p2->mPipeline); + + auto item3 = GenerateItem(); + auto p3 = item3.get(); + sProcessQueueManager->PushQueue(key, std::move(item3)); + + sProcessQueueManager->DisablePop("test_config_1", true); + APSARA_TEST_FALSE((*sProcessQueueManager->mQueues[key].first)->mValidToPop); + APSARA_TEST_EQUAL(pipeline1, p1->mPipeline); + APSARA_TEST_EQUAL(pipeline3, p2->mPipeline); + APSARA_TEST_EQUAL(nullptr, p3->mPipeline); + + sProcessQueueManager->EnablePop("test_config_1"); + APSARA_TEST_TRUE((*sProcessQueueManager->mQueues[key].first)->mValidToPop); + } + { + auto item1 = GenerateItem(); + auto p1 = item1.get(); + sProcessQueueManager->PushQueue(1, std::move(item1)); + + auto item2 = GenerateItem(); + auto p2 = item2.get(); + sProcessQueueManager->PushQueue(2, std::move(item2)); + + sProcessQueueManager->DisablePop("test_config_2", false); + APSARA_TEST_FALSE(ExactlyOnceQueueManager::GetInstance()->mProcessQueues[1]->mValidToPop); + APSARA_TEST_FALSE(ExactlyOnceQueueManager::GetInstance()->mProcessQueues[2]->mValidToPop); + APSARA_TEST_EQUAL(pipeline2, p1->mPipeline); + APSARA_TEST_EQUAL(pipeline2, p2->mPipeline); + + auto item3 = GenerateItem(); + auto p3 = item3.get(); + sProcessQueueManager->PushQueue(1, std::move(item3)); + + auto item4 = GenerateItem(); + auto p4 = item4.get(); + sProcessQueueManager->PushQueue(2, std::move(item4)); + + auto pipeline3 = make_shared(); + PipelineManager::GetInstance()->mPipelineNameEntityMap["test_config_2"] = pipeline3; + + sProcessQueueManager->DisablePop("test_config_2", false); + APSARA_TEST_FALSE(ExactlyOnceQueueManager::GetInstance()->mProcessQueues[1]->mValidToPop); + APSARA_TEST_FALSE(ExactlyOnceQueueManager::GetInstance()->mProcessQueues[2]->mValidToPop); + APSARA_TEST_EQUAL(pipeline2, p1->mPipeline); + APSARA_TEST_EQUAL(pipeline2, p2->mPipeline); + APSARA_TEST_EQUAL(pipeline3, p3->mPipeline); + APSARA_TEST_EQUAL(pipeline3, p4->mPipeline); + + auto item5 = GenerateItem(); + auto p5 = item5.get(); + sProcessQueueManager->PushQueue(1, std::move(item5)); + + auto item6 = GenerateItem(); + auto p6 = item6.get(); + sProcessQueueManager->PushQueue(2, std::move(item6)); + + sProcessQueueManager->DisablePop("test_config_2", true); + APSARA_TEST_FALSE(ExactlyOnceQueueManager::GetInstance()->mProcessQueues[1]->mValidToPop); + APSARA_TEST_FALSE(ExactlyOnceQueueManager::GetInstance()->mProcessQueues[2]->mValidToPop); + APSARA_TEST_EQUAL(pipeline2, p1->mPipeline); + APSARA_TEST_EQUAL(pipeline2, p2->mPipeline); + APSARA_TEST_EQUAL(pipeline3, p3->mPipeline); + APSARA_TEST_EQUAL(pipeline3, p4->mPipeline); + APSARA_TEST_EQUAL(nullptr, p5->mPipeline); + APSARA_TEST_EQUAL(nullptr, p6->mPipeline); + + sProcessQueueManager->EnablePop("test_config_2"); + APSARA_TEST_TRUE(ExactlyOnceQueueManager::GetInstance()->mProcessQueues[1]->mValidToPop); + APSARA_TEST_TRUE(ExactlyOnceQueueManager::GetInstance()->mProcessQueues[2]->mValidToPop); + } } UNIT_TEST_CASE(ProcessQueueManagerUnittest, TestUpdateSameTypeQueue) diff --git a/core/unittest/reader/ForceReadUnittest.cpp b/core/unittest/reader/ForceReadUnittest.cpp index aac74460f1..fa99c10eec 100644 --- a/core/unittest/reader/ForceReadUnittest.cpp +++ b/core/unittest/reader/ForceReadUnittest.cpp @@ -119,6 +119,7 @@ class ForceReadUnittest : public testing::Test { FileServer::GetInstance()->AddFileReaderConfig(mConfigName, &readerOpts, &ctx); FileServer::GetInstance()->AddMultilineConfig(mConfigName, &multilineOpts, &ctx); ProcessQueueManager::GetInstance()->CreateOrUpdateBoundedQueue(0, 0, ctx); + ProcessQueueManager::GetInstance()->EnablePop(mConfigName); } void TearDown() override { remove(utf8File.c_str()); } From 02e1a409e535eda92c9da47d039383140180052f Mon Sep 17 00:00:00 2001 From: henryzhx8 Date: Tue, 24 Sep 2024 14:26:34 +0800 Subject: [PATCH 6/9] add metric for runner (#1766) --- core/common/compression/Compressor.cpp | 14 +-- core/file_server/FileServer.cpp | 9 +- core/file_server/FileServer.h | 9 +- core/file_server/event_handler/LogInput.cpp | 22 +++-- core/file_server/event_handler/LogInput.h | 6 +- core/file_server/polling/PollingDirFile.cpp | 19 ++-- core/file_server/polling/PollingDirFile.h | 4 +- core/file_server/polling/PollingModify.cpp | 7 +- core/file_server/polling/PollingModify.h | 2 +- core/monitor/LogtailMetric.cpp | 27 +----- core/monitor/LogtailMetric.h | 10 +- core/monitor/MetricConstants.cpp | 96 +++++++++++++++---- core/monitor/MetricConstants.h | 94 ++++++++++++++---- core/monitor/Monitor.cpp | 19 ---- core/monitor/Monitor.h | 2 - core/pipeline/Pipeline.cpp | 13 +-- core/pipeline/batch/Batcher.h | 16 ++-- .../plugin/instance/FlusherInstance.cpp | 4 +- core/pipeline/plugin/interface/Plugin.h | 20 ++-- core/pipeline/queue/BoundedQueueInterface.h | 2 +- .../queue/BoundedSenderQueueInterface.cpp | 4 +- core/pipeline/queue/CircularProcessQueue.cpp | 4 +- core/pipeline/queue/CircularProcessQueue.h | 2 +- core/pipeline/queue/QueueInterface.h | 12 +-- core/pipeline/route/Router.cpp | 4 +- core/pipeline/serializer/Serializer.h | 14 +-- core/runner/FlusherRunner.cpp | 22 +++++ core/runner/FlusherRunner.h | 10 ++ core/runner/LogProcess.cpp | 65 +++++-------- core/runner/LogProcess.h | 8 +- core/runner/sink/http/HttpSink.cpp | 44 +++++++-- core/runner/sink/http/HttpSink.h | 10 ++ core/unittest/pipeline/PipelineUnittest.cpp | 6 +- .../queue/CircularProcessQueueUnittest.cpp | 2 +- 34 files changed, 372 insertions(+), 230 deletions(-) diff --git a/core/common/compression/Compressor.cpp b/core/common/compression/Compressor.cpp index 39a7af4b57..7a6d95951e 100644 --- a/core/common/compression/Compressor.cpp +++ b/core/common/compression/Compressor.cpp @@ -25,13 +25,13 @@ namespace logtail { void Compressor::SetMetricRecordRef(MetricLabels&& labels, DynamicMetricLabels&& dynamicLabels) { WriteMetrics::GetInstance()->PrepareMetricsRecordRef( mMetricsRecordRef, std::move(labels), std::move(dynamicLabels)); - mInItemsCnt = mMetricsRecordRef.CreateCounter(METRIC_IN_ITEMS_CNT); - mInItemSizeBytes = mMetricsRecordRef.CreateCounter(METRIC_IN_ITEM_SIZE_BYTES); - mOutItemsCnt = mMetricsRecordRef.CreateCounter(METRIC_OUT_ITEMS_CNT); - mOutItemSizeBytes = mMetricsRecordRef.CreateCounter(METRIC_OUT_ITEM_SIZE_BYTES); - mDiscardedItemsCnt = mMetricsRecordRef.CreateCounter("discarded_items_cnt"); - mDiscardedItemSizeBytes = mMetricsRecordRef.CreateCounter("discarded_item_size_bytes"); - mTotalDelayMs = mMetricsRecordRef.CreateCounter(METRIC_TOTAL_DELAY_MS); + mInItemsCnt = mMetricsRecordRef.CreateCounter(METRIC_COMPONENT_IN_ITEMS_CNT); + mInItemSizeBytes = mMetricsRecordRef.CreateCounter(METRIC_COMPONENT_IN_ITEM_SIZE_BYTES); + mOutItemsCnt = mMetricsRecordRef.CreateCounter(METRIC_COMPONENT_OUT_ITEMS_CNT); + mOutItemSizeBytes = mMetricsRecordRef.CreateCounter(METRIC_COMPONENT_OUT_ITEM_SIZE_BYTES); + mTotalDelayMs = mMetricsRecordRef.CreateCounter(METRIC_COMPONENT_TOTAL_DELAY_MS); + mDiscardedItemsCnt = mMetricsRecordRef.CreateCounter(METRIC_COMPONENT_DISCARDED_ITEMS_CNT); + mDiscardedItemSizeBytes = mMetricsRecordRef.CreateCounter(METRIC_COMPONENT_DISCARDED_ITEMS_SIZE_BYTES); } bool Compressor::DoCompress(const string& input, string& output, string& errorMsg) { diff --git a/core/file_server/FileServer.cpp b/core/file_server/FileServer.cpp index 49b1953b72..0e03c3f80d 100644 --- a/core/file_server/FileServer.cpp +++ b/core/file_server/FileServer.cpp @@ -18,12 +18,12 @@ #include "common/Flags.h" #include "common/StringTools.h" #include "common/TimeUtil.h" +#include "file_server/ConfigManager.h" #include "file_server/EventDispatcher.h" #include "file_server/event_handler/LogInput.h" -#include "file_server/ConfigManager.h" -#include "plugin/input/InputFile.h" #include "file_server/polling/PollingDirFile.h" #include "file_server/polling/PollingModify.h" +#include "plugin/input/InputFile.h" DEFINE_FLAG_BOOL(enable_polling_discovery, "", true); @@ -31,6 +31,11 @@ using namespace std; namespace logtail { +FileServer::FileServer() { + WriteMetrics::GetInstance()->PrepareMetricsRecordRef(mMetricsRecordRef, + {{METRIC_LABEL_KEY_RUNNER_NAME, "file_server"}}); +} + // 启动文件服务，包括加载配置、处理检查点、注册事件等 void FileServer::Start() { ConfigManager::GetInstance()->LoadDockerConfig(); diff --git a/core/file_server/FileServer.h b/core/file_server/FileServer.h index 7daee73fc5..d77b6a13a1 100644 --- a/core/file_server/FileServer.h +++ b/core/file_server/FileServer.h @@ -23,9 +23,11 @@ #include "common/Lock.h" #include "file_server/FileDiscoveryOptions.h" #include "file_server/MultilineOptions.h" +#include "file_server/reader/FileReaderOptions.h" +#include "monitor/LogtailMetric.h" #include "monitor/PluginMetricManager.h" #include "pipeline/PipelineContext.h" -#include "file_server/reader/FileReaderOptions.h" + namespace logtail { @@ -78,6 +80,7 @@ class FileServer { // for reader, event_handler ... ReentrantMetricsRecordRef GetOrCreateReentrantMetricsRecordRef(const std::string& name, MetricLabels& labels); void ReleaseReentrantMetricsRecordRef(const std::string& name, MetricLabels& labels); + MetricsRecordRef& GetMetricsRecordRef() { return mMetricsRecordRef; } // 过渡使用 void Resume(bool isConfigUpdate = true); @@ -88,7 +91,7 @@ class FileServer { void RemoveExactlyOnceConcurrency(const std::string& name); private: - FileServer() = default; + FileServer(); ~FileServer() = default; void PauseInner(); @@ -102,6 +105,8 @@ class FileServer { std::unordered_map mPipelineNamePluginMetricManagersMap; // 过渡使用 std::unordered_map mPipelineNameEOConcurrencyMap; + + mutable MetricsRecordRef mMetricsRecordRef; }; } // namespace logtail diff --git a/core/file_server/event_handler/LogInput.cpp b/core/file_server/event_handler/LogInput.cpp index a5ea529bec..96a3ac31af 100644 --- a/core/file_server/event_handler/LogInput.cpp +++ b/core/file_server/event_handler/LogInput.cpp @@ -16,8 +16,6 @@ #include -#include "file_server/event_handler/EventHandler.h" -#include "file_server/event_handler/HistoryFileImporter.h" #include "app_config/AppConfig.h" #include "application/Application.h" #include "checkpoint/CheckPointManager.h" @@ -27,18 +25,20 @@ #include "common/RuntimeUtil.h" #include "common/StringTools.h" #include "common/TimeUtil.h" +#include "file_server/ConfigManager.h" #include "file_server/EventDispatcher.h" #include "file_server/event/BlockEventManager.h" -#include "file_server/ConfigManager.h" -#include "logger/Logger.h" -#include "monitor/LogtailAlarm.h" -#include "monitor/Monitor.h" +#include "file_server/event_handler/EventHandler.h" +#include "file_server/event_handler/HistoryFileImporter.h" #include "file_server/polling/PollingCache.h" #include "file_server/polling/PollingDirFile.h" #include "file_server/polling/PollingEventQueue.h" #include "file_server/polling/PollingModify.h" #include "file_server/reader/GloablFileDescriptorManager.h" #include "file_server/reader/LogFileReader.h" +#include "logger/Logger.h" +#include "monitor/LogtailAlarm.h" +#include "monitor/Monitor.h" #ifdef __ENTERPRISE__ #include "config/provider/EnterpriseConfigProvider.h" #endif @@ -88,9 +88,11 @@ void LogInput::Start() { mInteruptFlag = false; + mLastRunTime = FileServer::GetInstance()->GetMetricsRecordRef().CreateIntGauge(METRIC_RUNNER_LAST_RUN_TIME); mAgentOpenFdTotal = LoongCollectorMonitor::GetInstance()->GetIntGauge(METRIC_AGENT_OPEN_FD_TOTAL); - mAgentRegisterHandlerTotal - = LoongCollectorMonitor::GetInstance()->GetIntGauge(METRIC_AGENT_REGISTER_HANDLER_TOTAL); + mRegisterdHandlersCnt = FileServer::GetInstance()->GetMetricsRecordRef().CreateIntGauge(METRIC_RUNNER_FILE_WATCHED_DIRS_CNT); + mActiveReadersCnt = FileServer::GetInstance()->GetMetricsRecordRef().CreateIntGauge(METRIC_RUNNER_FILE_ACTIVE_READERS_CNT); + mEnableFileIncludedByMultiConfigs = FileServer::GetInstance()->GetMetricsRecordRef().CreateIntGauge(METRIC_RUNNER_FILE_ENABLE_FILE_INCLUDED_BY_MULTI_CONFIGS_FLAG); new Thread([this]() { ProcessLoop(); }); } @@ -342,6 +344,7 @@ void LogInput::ProcessEvent(EventDispatcher* dispatcher, Event* ev) { void LogInput::UpdateCriticalMetric(int32_t curTime) { LogtailMonitor::GetInstance()->UpdateMetric("last_read_event_time", GetTimeStamp(mLastReadEventTime, "%Y-%m-%d %H:%M:%S")); + mLastRunTime->Set(mLastReadEventTime.load()); LogtailMonitor::GetInstance()->UpdateMetric("event_tps", 1.0 * mEventProcessCount / (curTime - mLastUpdateMetricTime)); @@ -350,8 +353,9 @@ void LogInput::UpdateCriticalMetric(int32_t curTime) { mAgentOpenFdTotal->Set(openFdTotal); size_t handlerCount = EventDispatcher::GetInstance()->GetHandlerCount(); LogtailMonitor::GetInstance()->UpdateMetric("register_handler", handlerCount); - mAgentRegisterHandlerTotal->Set(handlerCount); + mRegisterdHandlersCnt->Set(handlerCount); LogtailMonitor::GetInstance()->UpdateMetric("reader_count", CheckPointManager::Instance()->GetReaderCount()); + mActiveReadersCnt->Set(CheckPointManager::Instance()->GetReaderCount()); LogtailMonitor::GetInstance()->UpdateMetric("multi_config", AppConfig::GetInstance()->IsAcceptMultiConfig()); mEventProcessCount = 0; } diff --git a/core/file_server/event_handler/LogInput.h b/core/file_server/event_handler/LogInput.h index 21217086a3..69f2ffb077 100644 --- a/core/file_server/event_handler/LogInput.h +++ b/core/file_server/event_handler/LogInput.h @@ -79,8 +79,12 @@ class LogInput : public LogRunnable { volatile bool mIdleFlag; int32_t mEventProcessCount; int32_t mLastUpdateMetricTime; + + IntGaugePtr mLastRunTime; IntGaugePtr mAgentOpenFdTotal; - IntGaugePtr mAgentRegisterHandlerTotal; + IntGaugePtr mRegisterdHandlersCnt; + IntGaugePtr mActiveReadersCnt; + IntGaugePtr mEnableFileIncludedByMultiConfigs; std::atomic_int mLastReadEventTime{0}; mutable std::mutex mThreadRunningMux; diff --git a/core/file_server/polling/PollingDirFile.cpp b/core/file_server/polling/PollingDirFile.cpp index 2bc809de7c..0a578f5ce8 100644 --- a/core/file_server/polling/PollingDirFile.cpp +++ b/core/file_server/polling/PollingDirFile.cpp @@ -21,8 +21,6 @@ #endif #include -#include "file_server/polling/PollingEventQueue.h" -#include "file_server/polling/PollingModify.h" #include "app_config/AppConfig.h" #include "common/ErrorUtil.h" #include "common/FileSystemUtil.h" @@ -30,10 +28,13 @@ #include "common/StringTools.h" #include "common/TimeUtil.h" #include "file_server/ConfigManager.h" -#include "file_server/event/Event.h" #include "file_server/FileServer.h" +#include "file_server/event/Event.h" +#include "file_server/polling/PollingEventQueue.h" +#include "file_server/polling/PollingModify.h" #include "logger/Logger.h" #include "monitor/LogtailAlarm.h" +#include "monitor/MetricConstants.h" // Control the check frequency to call ClearUnavailableFileAndDir. DEFINE_FLAG_INT32(check_not_exist_file_dir_round, "clear not exist file dir cache, round", 20); @@ -69,10 +70,10 @@ static const int64_t NANO_CONVERTING = 1000000000; void PollingDirFile::Start() { ClearCache(); mAgentConfigTotal = LoongCollectorMonitor::GetInstance()->GetIntGauge(METRIC_AGENT_PIPELINE_CONFIG_TOTAL); - mAgentPollingDirCacheSizeTotal - = LoongCollectorMonitor::GetInstance()->GetIntGauge(METRIC_AGENT_POLLING_DIR_CACHE_SIZE_TOTAL); - mAgentPollingFileCacheSizeTotal - = LoongCollectorMonitor::GetInstance()->GetIntGauge(METRIC_AGENT_POLLING_FILE_CACHE_SIZE_TOTAL); + mPollingDirCacheSize + = FileServer::GetInstance()->GetMetricsRecordRef().CreateIntGauge(METRIC_RUNNER_FILE_POLLING_DIR_CACHE_SIZE); + mPollingFileCacheSize + = FileServer::GetInstance()->GetMetricsRecordRef().CreateIntGauge(METRIC_RUNNER_FILE_POLLING_FILE_CACHE_SIZE); mRuningFlag = true; mThreadPtr = CreateThread([this]() { Polling(); }); } @@ -157,10 +158,10 @@ void PollingDirFile::Polling() { ScopedSpinLock lock(mCacheLock); size_t pollingDirCacheSize = mDirCacheMap.size(); LogtailMonitor::GetInstance()->UpdateMetric("polling_dir_cache", pollingDirCacheSize); - mAgentPollingDirCacheSizeTotal->Set(pollingDirCacheSize); + mPollingDirCacheSize->Set(pollingDirCacheSize); size_t pollingFileCacheSize = mFileCacheMap.size(); LogtailMonitor::GetInstance()->UpdateMetric("polling_file_cache", pollingFileCacheSize); - mAgentPollingFileCacheSizeTotal->Set(pollingFileCacheSize); + mPollingFileCacheSize->Set(pollingFileCacheSize); } // Iterate all normal configs, make sure stat count will not exceed limit. diff --git a/core/file_server/polling/PollingDirFile.h b/core/file_server/polling/PollingDirFile.h index 40a4ebfb8b..3e381f80fa 100644 --- a/core/file_server/polling/PollingDirFile.h +++ b/core/file_server/polling/PollingDirFile.h @@ -137,8 +137,8 @@ class PollingDirFile : public LogRunnable { uint64_t mCurrentRound; IntGaugePtr mAgentConfigTotal; - IntGaugePtr mAgentPollingDirCacheSizeTotal; - IntGaugePtr mAgentPollingFileCacheSizeTotal; + IntGaugePtr mPollingDirCacheSize; + IntGaugePtr mPollingFileCacheSize; #ifdef APSARA_UNIT_TEST_MAIN friend class PollingUnittest; diff --git a/core/file_server/polling/PollingModify.cpp b/core/file_server/polling/PollingModify.cpp index 6dac3afad4..73ff76b3cd 100644 --- a/core/file_server/polling/PollingModify.cpp +++ b/core/file_server/polling/PollingModify.cpp @@ -24,9 +24,11 @@ #include "common/Flags.h" #include "common/StringTools.h" #include "common/TimeUtil.h" +#include "file_server/FileServer.h" #include "file_server/event/Event.h" #include "logger/Logger.h" #include "monitor/LogtailAlarm.h" +#include "monitor/MetricConstants.h" using namespace std; @@ -47,7 +49,8 @@ PollingModify::~PollingModify() { void PollingModify::Start() { ClearCache(); - mAgentPollingModifySizeTotal = LoongCollectorMonitor::GetInstance()->GetIntGauge(METRIC_AGENT_POLLING_MODIFY_SIZE_TOTAL); + mPollingModifySize + = FileServer::GetInstance()->GetMetricsRecordRef().CreateIntGauge(METRIC_RUNNER_FILE_POLLING_MODIFY_CACHE_SIZE); mRuningFlag = true; mThreadPtr = CreateThread([this]() { Polling(); }); @@ -251,7 +254,7 @@ void PollingModify::Polling() { int32_t statCount = 0; size_t pollingModifySizeTotal = mModifyCacheMap.size(); LogtailMonitor::GetInstance()->UpdateMetric("polling_modify_size", pollingModifySizeTotal); - mAgentPollingModifySizeTotal->Set(pollingModifySizeTotal); + mPollingModifySize->Set(pollingModifySizeTotal); for (auto iter = mModifyCacheMap.begin(); iter != mModifyCacheMap.end(); ++iter) { if (!mRuningFlag || mHoldOnFlag) break; diff --git a/core/file_server/polling/PollingModify.h b/core/file_server/polling/PollingModify.h index c82337a898..5e3738a92a 100644 --- a/core/file_server/polling/PollingModify.h +++ b/core/file_server/polling/PollingModify.h @@ -101,7 +101,7 @@ class PollingModify : public LogRunnable { ModifyCheckCacheMap mModifyCacheMap; - IntGaugePtr mAgentPollingModifySizeTotal; + IntGaugePtr mPollingModifySize; #ifdef APSARA_UNIT_TEST_MAIN friend class PollingUnittest; diff --git a/core/monitor/LogtailMetric.cpp b/core/monitor/LogtailMetric.cpp index d6ddc5e768..6b26300d8c 100644 --- a/core/monitor/LogtailMetric.cpp +++ b/core/monitor/LogtailMetric.cpp @@ -205,34 +205,11 @@ WriteMetrics::~WriteMetrics() { Clear(); } -void WriteMetrics::PreparePluginCommonLabels(const std::string& projectName, - const std::string& logstoreName, - const std::string& region, - const std::string& configName, - const std::string& pluginType, - const std::string& pluginID, - const std::string& nodeID, - const std::string& childNodeID, - MetricLabels& labels) { - labels.emplace_back(std::make_pair(METRIC_LABEL_PROJECT, projectName)); - labels.emplace_back(std::make_pair(METRIC_LABEL_LOGSTORE, logstoreName)); - labels.emplace_back(std::make_pair(METRIC_LABEL_REGION, region)); - labels.emplace_back(std::make_pair(METRIC_LABEL_CONFIG_NAME, configName)); - labels.emplace_back(std::make_pair(METRIC_LABEL_PLUGIN_NAME, pluginType)); - labels.emplace_back(std::make_pair(METRIC_LABEL_PLUGIN_ID, pluginID)); - labels.emplace_back(std::make_pair(METRIC_LABEL_NODE_ID, nodeID)); - labels.emplace_back(std::make_pair(METRIC_LABEL_CHILD_NODE_ID, childNodeID)); -} - void WriteMetrics::PrepareMetricsRecordRef(MetricsRecordRef& ref, MetricLabels&& labels, DynamicMetricLabels&& dynamicLabels) { - MetricsRecord* cur = new MetricsRecord(std::make_shared(labels), - std::make_shared(dynamicLabels)); - ref.SetMetricsRecord(cur); - std::lock_guard lock(mMutex); - cur->SetNext(mHead); - mHead = cur; + CreateMetricsRecordRef(ref, std::move(labels), std::move(dynamicLabels)); + CommitMetricsRecordRef(ref); } void WriteMetrics::CreateMetricsRecordRef(MetricsRecordRef& ref, diff --git a/core/monitor/LogtailMetric.h b/core/monitor/LogtailMetric.h index 6c98081b2a..6f4ef9e8ef 100644 --- a/core/monitor/LogtailMetric.h +++ b/core/monitor/LogtailMetric.h @@ -133,15 +133,7 @@ class WriteMetrics { static WriteMetrics* ptr = new WriteMetrics(); return ptr; } - void PreparePluginCommonLabels(const std::string& projectName, - const std::string& logstoreName, - const std::string& region, - const std::string& configName, - const std::string& pluginType, - const std::string& pluginID, - const std::string& nodeID, - const std::string& childNodeID, - MetricLabels& labels); + void PrepareMetricsRecordRef(MetricsRecordRef& ref, MetricLabels&& labels, DynamicMetricLabels&& dynamicLabels = {}); void CreateMetricsRecordRef(MetricsRecordRef& ref, MetricLabels&& labels, DynamicMetricLabels&& dynamicLabels = {}); diff --git a/core/monitor/MetricConstants.cpp b/core/monitor/MetricConstants.cpp index 24212c9d3c..e0b43e87e3 100644 --- a/core/monitor/MetricConstants.cpp +++ b/core/monitor/MetricConstants.cpp @@ -45,21 +45,12 @@ const std::string METRIC_AGENT_MEMORY = "agent_memory_used_mb"; const std::string METRIC_AGENT_MEMORY_GO = "agent_go_memory_used_mb"; const std::string METRIC_AGENT_GO_ROUTINES_TOTAL = "agent_go_routines_total"; const std::string METRIC_AGENT_OPEN_FD_TOTAL = "agent_open_fd_total"; -const std::string METRIC_AGENT_POLLING_DIR_CACHE_SIZE_TOTAL = "agent_polling_dir_cache_size_total"; -const std::string METRIC_AGENT_POLLING_FILE_CACHE_SIZE_TOTAL = "agent_polling_file_cache_size_total"; -const std::string METRIC_AGENT_POLLING_MODIFY_SIZE_TOTAL = "agent_polling_modify_size_total"; -const std::string METRIC_AGENT_REGISTER_HANDLER_TOTAL = "agent_register_handler_total"; const std::string METRIC_AGENT_INSTANCE_CONFIG_TOTAL = "agent_instance_config_total"; const std::string METRIC_AGENT_PIPELINE_CONFIG_TOTAL = "agent_pipeline_config_total"; const std::string METRIC_AGENT_ENV_PIPELINE_CONFIG_TOTAL = "agent_env_pipeline_config_total"; const std::string METRIC_AGENT_CRD_PIPELINE_CONFIG_TOTAL = "agent_crd_pipeline_config_total"; const std::string METRIC_AGENT_CONSOLE_PIPELINE_CONFIG_TOTAL = "agent_console_pipeline_config_total"; const std::string METRIC_AGENT_PLUGIN_TOTAL = "agent_plugin_total"; -const std::string METRIC_AGENT_PROCESS_QUEUE_FULL_TOTAL = "agent_process_queue_full_total"; -const std::string METRIC_AGENT_PROCESS_QUEUE_TOTAL = "agent_process_queue_total"; -const std::string METRIC_AGENT_SEND_QUEUE_FULL_TOTAL = "agent_send_queue_full_total"; -const std::string METRIC_AGENT_SEND_QUEUE_TOTAL = "agent_send_queue_total"; -const std::string METRIC_AGENT_USED_SENDING_CONCURRENCY = "agent_used_sending_concurrency"; // common plugin labels const std::string METRIC_LABEL_PROJECT = "project"; @@ -72,6 +63,7 @@ const std::string METRIC_LABEL_NODE_ID = "node_id"; const std::string METRIC_LABEL_CHILD_NODE_ID = "child_node_id"; const std::string METRIC_LABEL_KEY_COMPONENT_NAME = "component_name"; +const std::string METRIC_LABEL_KEY_RUNNER_NAME = "runner_name"; const std::string METRIC_LABEL_KEY_QUEUE_TYPE = "queue_type"; const std::string METRIC_LABEL_KEY_EXACTLY_ONCE_FLAG = "is_exactly_once"; const std::string METRIC_LABEL_KEY_FLUSHER_NODE_ID = "flusher_node_id"; @@ -149,14 +141,82 @@ const std::string METRIC_FLUSHER_QUOTA_ERROR_TOTAL = "flusher_quota_error_total" const std::string METRIC_FLUSHER_RETRIES_TOTAL = "flusher_retries_total"; const std::string METRIC_FLUSHER_RETRIES_ERROR_TOTAL = "flusher_retries_error_total"; -const std::string METRIC_IN_EVENTS_CNT = "in_events_cnt"; -const std::string METRIC_IN_ITEMS_CNT = "in_items_cnt"; -const std::string METRIC_IN_EVENT_GROUP_SIZE_BYTES = "in_event_group_data_size_bytes"; -const std::string METRIC_IN_ITEM_SIZE_BYTES = "in_item_data_size_bytes"; -const std::string METRIC_OUT_EVENTS_CNT = "out_events_cnt"; -const std::string METRIC_OUT_ITEMS_CNT = "out_items_cnt"; -const std::string METRIC_OUT_EVENT_GROUP_SIZE_BYTES = "out_event_group_data_size_bytes"; -const std::string METRIC_OUT_ITEM_SIZE_BYTES = "out_item_data_size_bytes"; -const std::string METRIC_TOTAL_DELAY_MS = "total_delay_ms"; +////////////////////////////////////////////////////////////////////////// +// plugin +////////////////////////////////////////////////////////////////////////// +// common metrics +const std::string METRIC_PLUGIN_IN_EVENTS_CNT = "plugin_in_events_total"; +const std::string METRIC_PLUGIN_IN_EVENT_GROUP_SIZE_BYTES = "plugin_in_event_group_size_bytes"; + +////////////////////////////////////////////////////////////////////////// +// component +////////////////////////////////////////////////////////////////////////// +// common metrics +const std::string METRIC_COMPONENT_IN_EVENTS_CNT = "component_in_events_total"; +const std::string METRIC_COMPONENT_IN_ITEMS_CNT = "component_in_items_total"; +const std::string METRIC_COMPONENT_IN_EVENT_GROUP_SIZE_BYTES = "component_in_event_group_size_bytes"; +const std::string METRIC_COMPONENT_IN_ITEM_SIZE_BYTES = "component_in_item_size_bytes"; +const std::string METRIC_COMPONENT_OUT_EVENTS_CNT = "component_out_events_total"; +const std::string METRIC_COMPONENT_OUT_ITEMS_CNT = "component_out_items_total"; +const std::string METRIC_COMPONENT_OUT_ITEM_SIZE_BYTES = "component_out_item_size_bytes"; +const std::string METRIC_COMPONENT_TOTAL_DELAY_MS = "component_total_delay_ms"; +const std::string METRIC_COMPONENT_DISCARDED_ITEMS_CNT = "component_discarded_items_total"; +const std::string METRIC_COMPONENT_DISCARDED_ITEMS_SIZE_BYTES = "component_discarded_item_size_bytes"; + +// batcher metrics +const std::string METRIC_COMPONENT_BATCHER_EVENT_BATCHES_CNT = "component_event_batches_total"; +const std::string METRIC_COMPONENT_BATCHER_BUFFERED_GROUPS_CNT = "component_buffered_groups_total"; +const std::string METRIC_COMPONENT_BATCHER_BUFFERED_EVENTS_CNT = "component_buffered_events_total"; +const std::string METRIC_COMPONENT_BATCHER_BUFFERED_SIZE_BYTES = "component_buffered_size_bytes"; + +// queue metrics +const std::string METRIC_COMPONENT_QUEUE_SIZE_CNT = "component_queue_size"; +const std::string METRIC_COMPONENT_QUEUE_SIZE_BYTES = "component_queue_size_bytes"; +const std::string METRIC_COMPONENT_QUEUE_VALID_TO_PUSH_FLAG = "component_valid_to_push"; +const std::string METRIC_COMPONENT_QUEUE_EXTRA_BUFFER_SIZE = "component_extra_buffer_size"; +const std::string METRIC_COMPONENT_QUEUE_EXTRA_BUFFER_SIZE_BYTES = "component_extra_buffer_size"; +const std::string METRIC_COMPONENT_QUEUE_DISCARDED_EVENTS_CNT = "component_discarded_events_total"; + +////////////////////////////////////////////////////////////////////////// +// pipeline +////////////////////////////////////////////////////////////////////////// +const std::string METRIC_PIPELINE_START_TIME = "pipeline_start_time"; +const std::string METRIC_PIPELINE_PROCESSORS_IN_EVENTS_CNT = "pipeline_processors_in_events_total"; +const std::string METRIC_PIPELINE_PROCESSORS_IN_EVENT_GROUPS_CNT = "pipeline_processors_in_event_groups_total"; +const std::string METRIC_PIPELINE_PROCESSORS_IN_EVENT_GROUP_SIZE_BYTES + = "pipeline_processors_in_event_group_size_bytes"; +const std::string METRIC_PIPELINE_PROCESSORS_TOTAL_DELAY_MS = "pipeline_processors_total_delay_ms"; + +////////////////////////////////////////////////////////////////////////// +// runner +////////////////////////////////////////////////////////////////////////// +// common metrics +const std::string METRIC_RUNNER_IN_EVENTS_CNT = "runner_in_events_total"; +const std::string METRIC_RUNNER_IN_EVENT_GROUPS_CNT = "runner_in_event_groups_total"; +const std::string METRIC_RUNNER_IN_ITEMS_CNT = "runner_in_items_total"; +const std::string METRIC_RUNNER_IN_EVENT_GROUP_SIZE_BYTES = "runner_in_event_group_size_bytes"; +const std::string METRIC_RUNNER_IN_ITEM_SIZE_BYTES = "runner_in_item_size_bytes"; +const std::string METRIC_RUNNER_OUT_ITEMS_CNT = "runner_out_items_total"; +const std::string METRIC_RUNNER_TOTAL_DELAY_MS = "runner_total_delay_ms"; +const std::string METRIC_RUNNER_LAST_RUN_TIME = "runner_last_run_time"; + +// http sink metrics +const std::string METRIC_RUNNER_HTTP_SINK_OUT_SUCCESSFUL_ITEMS_CNT = "runner_out_successful_items_total"; +const std::string METRIC_RUNNER_HTTP_SINK_OUT_FAILED_ITEMS_CNT = "runner_out_failed_items_total"; +const std::string METRIC_RUNNER_HTTP_SINK_SENDING_ITEMS_CNT = "runner_sending_items_total"; +const std::string METRIC_RUNNER_HTTP_SINK_SEND_CONCURRENCY = "runner_send_concurrency"; + +// flusher runner metrics +const std::string METRIC_RUNNER_FLUSHER_IN_ITEM_RAW_SIZE_BYTES = "runner_in_item_raw_size_bytes"; +const std::string METRIC_RUNNER_FLUSHER_WAITING_ITEMS_CNT = "runner_waiting_items_total"; + +// file server metrics +const std::string METRIC_RUNNER_FILE_WATCHED_DIRS_CNT = "runner_watched_dirs_total"; +const std::string METRIC_RUNNER_FILE_ACTIVE_READERS_CNT = "runner_active_readers_total"; +const std::string METRIC_RUNNER_FILE_ENABLE_FILE_INCLUDED_BY_MULTI_CONFIGS_FLAG + = "runner_enable_file_included_by_multi_configs"; +const std::string METRIC_RUNNER_FILE_POLLING_MODIFY_CACHE_SIZE = "runner_polling_modify_cache_size"; +const std::string METRIC_RUNNER_FILE_POLLING_DIR_CACHE_SIZE = "runner_polling_dir_cache_size"; +const std::string METRIC_RUNNER_FILE_POLLING_FILE_CACHE_SIZE = "runner_polling_file_cache_size"; } // namespace logtail diff --git a/core/monitor/MetricConstants.h b/core/monitor/MetricConstants.h index 3b78a5eae0..cdf82bc734 100644 --- a/core/monitor/MetricConstants.h +++ b/core/monitor/MetricConstants.h @@ -47,21 +47,12 @@ extern const std::string METRIC_AGENT_MEMORY; extern const std::string METRIC_AGENT_MEMORY_GO; extern const std::string METRIC_AGENT_GO_ROUTINES_TOTAL; extern const std::string METRIC_AGENT_OPEN_FD_TOTAL; -extern const std::string METRIC_AGENT_POLLING_DIR_CACHE_SIZE_TOTAL; -extern const std::string METRIC_AGENT_POLLING_FILE_CACHE_SIZE_TOTAL; -extern const std::string METRIC_AGENT_POLLING_MODIFY_SIZE_TOTAL; -extern const std::string METRIC_AGENT_REGISTER_HANDLER_TOTAL; extern const std::string METRIC_AGENT_INSTANCE_CONFIG_TOTAL; extern const std::string METRIC_AGENT_PIPELINE_CONFIG_TOTAL; extern const std::string METRIC_AGENT_ENV_PIPELINE_CONFIG_TOTAL; extern const std::string METRIC_AGENT_CRD_PIPELINE_CONFIG_TOTAL; extern const std::string METRIC_AGENT_CONSOLE_PIPELINE_CONFIG_TOTAL; extern const std::string METRIC_AGENT_PLUGIN_TOTAL; -extern const std::string METRIC_AGENT_PROCESS_QUEUE_FULL_TOTAL; -extern const std::string METRIC_AGENT_PROCESS_QUEUE_TOTAL; -extern const std::string METRIC_AGENT_SEND_QUEUE_FULL_TOTAL; -extern const std::string METRIC_AGENT_SEND_QUEUE_TOTAL; -extern const std::string METRIC_AGENT_USED_SENDING_CONCURRENCY; // common plugin labels extern const std::string METRIC_LABEL_PROJECT; @@ -79,6 +70,7 @@ extern const std::string METRIC_LABEL_FILE_INODE; extern const std::string METRIC_LABEL_FILE_NAME; extern const std::string METRIC_LABEL_KEY_COMPONENT_NAME; +extern const std::string METRIC_LABEL_KEY_RUNNER_NAME; extern const std::string METRIC_LABEL_KEY_QUEUE_TYPE; extern const std::string METRIC_LABEL_KEY_EXACTLY_ONCE_FLAG; extern const std::string METRIC_LABEL_KEY_FLUSHER_NODE_ID; @@ -142,14 +134,80 @@ extern const std::string METRIC_FLUSHER_QUOTA_ERROR_TOTAL; extern const std::string METRIC_FLUSHER_RETRIES_TOTAL; extern const std::string METRIC_FLUSHER_RETRIES_ERROR_TOTAL; -extern const std::string METRIC_IN_EVENTS_CNT; -extern const std::string METRIC_IN_ITEMS_CNT; -extern const std::string METRIC_IN_EVENT_GROUP_SIZE_BYTES; -extern const std::string METRIC_IN_ITEM_SIZE_BYTES; -extern const std::string METRIC_OUT_EVENTS_CNT; -extern const std::string METRIC_OUT_ITEMS_CNT; -extern const std::string METRIC_OUT_EVENT_GROUP_SIZE_BYTES; -extern const std::string METRIC_OUT_ITEM_SIZE_BYTES; -extern const std::string METRIC_TOTAL_DELAY_MS; +////////////////////////////////////////////////////////////////////////// +// plugin +////////////////////////////////////////////////////////////////////////// +// common metrics +extern const std::string METRIC_PLUGIN_IN_EVENTS_CNT; +extern const std::string METRIC_PLUGIN_IN_EVENT_GROUP_SIZE_BYTES; + +////////////////////////////////////////////////////////////////////////// +// component +////////////////////////////////////////////////////////////////////////// +// common metrics +extern const std::string METRIC_COMPONENT_IN_EVENTS_CNT; +extern const std::string METRIC_COMPONENT_IN_ITEMS_CNT; +extern const std::string METRIC_COMPONENT_IN_EVENT_GROUP_SIZE_BYTES; +extern const std::string METRIC_COMPONENT_IN_ITEM_SIZE_BYTES; +extern const std::string METRIC_COMPONENT_OUT_EVENTS_CNT; +extern const std::string METRIC_COMPONENT_OUT_ITEMS_CNT; +extern const std::string METRIC_COMPONENT_OUT_ITEM_SIZE_BYTES; +extern const std::string METRIC_COMPONENT_TOTAL_DELAY_MS; +extern const std::string METRIC_COMPONENT_DISCARDED_ITEMS_CNT; +extern const std::string METRIC_COMPONENT_DISCARDED_ITEMS_SIZE_BYTES; + +// batcher metrics +extern const std::string METRIC_COMPONENT_BATCHER_EVENT_BATCHES_CNT; +extern const std::string METRIC_COMPONENT_BATCHER_BUFFERED_GROUPS_CNT; +extern const std::string METRIC_COMPONENT_BATCHER_BUFFERED_EVENTS_CNT; +extern const std::string METRIC_COMPONENT_BATCHER_BUFFERED_SIZE_BYTES; + +// queue metrics +extern const std::string METRIC_COMPONENT_QUEUE_SIZE_CNT; +extern const std::string METRIC_COMPONENT_QUEUE_SIZE_BYTES; +extern const std::string METRIC_COMPONENT_QUEUE_VALID_TO_PUSH_FLAG; +extern const std::string METRIC_COMPONENT_QUEUE_EXTRA_BUFFER_SIZE; +extern const std::string METRIC_COMPONENT_QUEUE_EXTRA_BUFFER_SIZE_BYTES; +extern const std::string METRIC_COMPONENT_QUEUE_DISCARDED_EVENTS_CNT; + +////////////////////////////////////////////////////////////////////////// +// pipeline +////////////////////////////////////////////////////////////////////////// +extern const std::string METRIC_PIPELINE_START_TIME; +extern const std::string METRIC_PIPELINE_PROCESSORS_IN_EVENTS_CNT; +extern const std::string METRIC_PIPELINE_PROCESSORS_IN_EVENT_GROUPS_CNT; +extern const std::string METRIC_PIPELINE_PROCESSORS_IN_EVENT_GROUP_SIZE_BYTES; +extern const std::string METRIC_PIPELINE_PROCESSORS_TOTAL_DELAY_MS; + +////////////////////////////////////////////////////////////////////////// +// runner +////////////////////////////////////////////////////////////////////////// +// common metrics +extern const std::string METRIC_RUNNER_IN_EVENTS_CNT; +extern const std::string METRIC_RUNNER_IN_EVENT_GROUPS_CNT; +extern const std::string METRIC_RUNNER_IN_ITEMS_CNT; +extern const std::string METRIC_RUNNER_IN_EVENT_GROUP_SIZE_BYTES; +extern const std::string METRIC_RUNNER_IN_ITEM_SIZE_BYTES; +extern const std::string METRIC_RUNNER_OUT_ITEMS_CNT; +extern const std::string METRIC_RUNNER_TOTAL_DELAY_MS; +extern const std::string METRIC_RUNNER_LAST_RUN_TIME; + +// http sink metrics +extern const std::string METRIC_RUNNER_HTTP_SINK_OUT_SUCCESSFUL_ITEMS_CNT; +extern const std::string METRIC_RUNNER_HTTP_SINK_OUT_FAILED_ITEMS_CNT; +extern const std::string METRIC_RUNNER_HTTP_SINK_SENDING_ITEMS_CNT; +extern const std::string METRIC_RUNNER_HTTP_SINK_SEND_CONCURRENCY; + +// flusher runner metrics +extern const std::string METRIC_RUNNER_FLUSHER_IN_ITEM_RAW_SIZE_BYTES; +extern const std::string METRIC_RUNNER_FLUSHER_WAITING_ITEMS_CNT; + +// file server metrics +extern const std::string METRIC_RUNNER_FILE_WATCHED_DIRS_CNT; +extern const std::string METRIC_RUNNER_FILE_ACTIVE_READERS_CNT; +extern const std::string METRIC_RUNNER_FILE_ENABLE_FILE_INCLUDED_BY_MULTI_CONFIGS_FLAG; +extern const std::string METRIC_RUNNER_FILE_POLLING_MODIFY_CACHE_SIZE; +extern const std::string METRIC_RUNNER_FILE_POLLING_DIR_CACHE_SIZE; +extern const std::string METRIC_RUNNER_FILE_POLLING_FILE_CACHE_SIZE; } // namespace logtail diff --git a/core/monitor/Monitor.cpp b/core/monitor/Monitor.cpp index f3bfe2b391..2888af75e5 100644 --- a/core/monitor/Monitor.cpp +++ b/core/monitor/Monitor.cpp @@ -116,8 +116,6 @@ bool LogtailMonitor::Init() { // init metrics mAgentCpuGauge = LoongCollectorMonitor::GetInstance()->GetDoubleGauge(METRIC_AGENT_CPU); mAgentMemoryGauge = LoongCollectorMonitor::GetInstance()->GetIntGauge(METRIC_AGENT_MEMORY); - mAgentUsedSendingConcurrency - = LoongCollectorMonitor::GetInstance()->GetIntGauge(METRIC_AGENT_USED_SENDING_CONCURRENCY); // Initialize monitor thread. mThreadRes = async(launch::async, &LogtailMonitor::Monitor, this); @@ -320,7 +318,6 @@ bool LogtailMonitor::SendStatusProfile(bool suicide) { } int32_t usedSendingConcurrency = FlusherRunner::GetInstance()->GetSendingBufferCount(); UpdateMetric("used_sending_concurrency", usedSendingConcurrency); - mAgentUsedSendingConcurrency->Set(usedSendingConcurrency); AddLogContent(logPtr, "metric_json", MetricToString()); AddLogContent(logPtr, "status", CheckLogtailStatus()); @@ -731,14 +728,6 @@ void LoongCollectorMonitor::Init() { mIntGauges[METRIC_AGENT_MEMORY_GO] = mMetricsRecordRef.CreateIntGauge(METRIC_AGENT_MEMORY_GO); mIntGauges[METRIC_AGENT_GO_ROUTINES_TOTAL] = mMetricsRecordRef.CreateIntGauge(METRIC_AGENT_GO_ROUTINES_TOTAL); mIntGauges[METRIC_AGENT_OPEN_FD_TOTAL] = mMetricsRecordRef.CreateIntGauge(METRIC_AGENT_OPEN_FD_TOTAL); - mIntGauges[METRIC_AGENT_POLLING_DIR_CACHE_SIZE_TOTAL] - = mMetricsRecordRef.CreateIntGauge(METRIC_AGENT_POLLING_DIR_CACHE_SIZE_TOTAL); - mIntGauges[METRIC_AGENT_POLLING_FILE_CACHE_SIZE_TOTAL] - = mMetricsRecordRef.CreateIntGauge(METRIC_AGENT_POLLING_FILE_CACHE_SIZE_TOTAL); - mIntGauges[METRIC_AGENT_POLLING_MODIFY_SIZE_TOTAL] - = mMetricsRecordRef.CreateIntGauge(METRIC_AGENT_POLLING_MODIFY_SIZE_TOTAL); - mIntGauges[METRIC_AGENT_REGISTER_HANDLER_TOTAL] - = mMetricsRecordRef.CreateIntGauge(METRIC_AGENT_REGISTER_HANDLER_TOTAL); // mIntGauges[METRIC_AGENT_INSTANCE_CONFIG_TOTAL] = // mMetricsRecordRef.CreateIntGauge(METRIC_AGENT_INSTANCE_CONFIG_TOTAL); mIntGauges[METRIC_AGENT_PIPELINE_CONFIG_TOTAL] @@ -750,14 +739,6 @@ void LoongCollectorMonitor::Init() { // mIntGauges[METRIC_AGENT_CONSOLE_PIPELINE_CONFIG_TOTAL] // = mMetricsRecordRef.CreateIntGauge(METRIC_AGENT_CONSOLE_PIPELINE_CONFIG_TOTAL); // mIntGauges[METRIC_AGENT_PLUGIN_TOTAL] = mMetricsRecordRef.CreateIntGauge(METRIC_AGENT_PLUGIN_TOTAL); - mIntGauges[METRIC_AGENT_PROCESS_QUEUE_FULL_TOTAL] - = mMetricsRecordRef.CreateIntGauge(METRIC_AGENT_PROCESS_QUEUE_FULL_TOTAL); - mIntGauges[METRIC_AGENT_PROCESS_QUEUE_TOTAL] = mMetricsRecordRef.CreateIntGauge(METRIC_AGENT_PROCESS_QUEUE_TOTAL); - mIntGauges[METRIC_AGENT_SEND_QUEUE_FULL_TOTAL] - = mMetricsRecordRef.CreateIntGauge(METRIC_AGENT_SEND_QUEUE_FULL_TOTAL); - mIntGauges[METRIC_AGENT_SEND_QUEUE_TOTAL] = mMetricsRecordRef.CreateIntGauge(METRIC_AGENT_SEND_QUEUE_TOTAL); - mIntGauges[METRIC_AGENT_USED_SENDING_CONCURRENCY] - = mMetricsRecordRef.CreateIntGauge(METRIC_AGENT_USED_SENDING_CONCURRENCY); LOG_INFO(sLogger, ("LoongCollectorMonitor", "started")); } diff --git a/core/monitor/Monitor.h b/core/monitor/Monitor.h index 5fcc551dd5..0a4d557361 100644 --- a/core/monitor/Monitor.h +++ b/core/monitor/Monitor.h @@ -167,8 +167,6 @@ class LogtailMonitor : public MetricStore { MemStat mMemStat; IntGaugePtr mAgentMemoryGauge; - IntGaugePtr mAgentUsedSendingConcurrency; - // Current scale up level, updated by CheckScaledCpuUsageUpLimit. float mScaledCpuUsageUpLimit; #if defined(__linux__) diff --git a/core/pipeline/Pipeline.cpp b/core/pipeline/Pipeline.cpp index 0f841798f1..1b58a5a044 100644 --- a/core/pipeline/Pipeline.cpp +++ b/core/pipeline/Pipeline.cpp @@ -317,11 +317,12 @@ bool Pipeline::Init(PipelineConfig&& config) { WriteMetrics::GetInstance()->PrepareMetricsRecordRef( mMetricsRecordRef, {{METRIC_LABEL_PROJECT, mContext.GetProjectName()}, {METRIC_LABEL_CONFIG_NAME, mName}}); - mStartTime = mMetricsRecordRef.CreateIntGauge("start_time"); - mProcessorsInEventsCnt = mMetricsRecordRef.CreateCounter("processors_in_events_cnt"); - mProcessorsInGroupsCnt = mMetricsRecordRef.CreateCounter("processors_in_event_groups_cnt"); - mProcessorsInGroupDataSizeBytes = mMetricsRecordRef.CreateCounter("processors_in_event_group_data_size_bytes"); - mProcessorsTotalDelayMs = mMetricsRecordRef.CreateCounter("processors_total_delay_ms"); + mStartTime = mMetricsRecordRef.CreateIntGauge(METRIC_PIPELINE_START_TIME); + mProcessorsInEventsCnt = mMetricsRecordRef.CreateCounter(METRIC_PIPELINE_PROCESSORS_IN_EVENTS_CNT); + mProcessorsInGroupsCnt = mMetricsRecordRef.CreateCounter(METRIC_PIPELINE_PROCESSORS_IN_EVENT_GROUPS_CNT); + mProcessorsInGroupDataSizeBytes + = mMetricsRecordRef.CreateCounter(METRIC_PIPELINE_PROCESSORS_IN_EVENT_GROUP_SIZE_BYTES); + mProcessorsTotalDelayMs = mMetricsRecordRef.CreateCounter(METRIC_PIPELINE_PROCESSORS_TOTAL_DELAY_MS); return true; } @@ -349,7 +350,7 @@ void Pipeline::Start() { mStartTime->Set(chrono::duration_cast(chrono::system_clock::now().time_since_epoch()).count()); #endif - LOG_INFO(sLogger, ("pipeline start", "succeeded")("config", mName)("ptr", mStartTime.get())); + LOG_INFO(sLogger, ("pipeline start", "succeeded")("config", mName)); } void Pipeline::Process(vector& logGroupList, size_t inputIndex) { diff --git a/core/pipeline/batch/Batcher.h b/core/pipeline/batch/Batcher.h index ca01986ca5..0c09990649 100644 --- a/core/pipeline/batch/Batcher.h +++ b/core/pipeline/batch/Batcher.h @@ -110,14 +110,14 @@ class Batcher { labels.emplace_back("enable_group_batch", "false"); } WriteMetrics::GetInstance()->PrepareMetricsRecordRef(mMetricsRecordRef, std::move(labels)); - mInEventsCnt = mMetricsRecordRef.CreateCounter(METRIC_IN_EVENTS_CNT); - mInGroupDataSizeBytes = mMetricsRecordRef.CreateCounter(METRIC_IN_EVENT_GROUP_SIZE_BYTES); - mOutEventsCnt = mMetricsRecordRef.CreateCounter(METRIC_OUT_EVENTS_CNT); - mTotalDelayMs = mMetricsRecordRef.CreateCounter(METRIC_TOTAL_DELAY_MS); - mEventBatchItemsCnt = mMetricsRecordRef.CreateIntGauge("event_batches_cnt"); - mBufferedGroupsCnt = mMetricsRecordRef.CreateIntGauge("buffered_groups_cnt"); - mBufferedEventsCnt = mMetricsRecordRef.CreateIntGauge("buffered_events_cnt"); - mBufferedDataSizeByte = mMetricsRecordRef.CreateIntGauge("buffered_data_size_bytes"); + mInEventsCnt = mMetricsRecordRef.CreateCounter(METRIC_COMPONENT_IN_EVENTS_CNT); + mInGroupDataSizeBytes = mMetricsRecordRef.CreateCounter(METRIC_COMPONENT_IN_EVENT_GROUP_SIZE_BYTES); + mOutEventsCnt = mMetricsRecordRef.CreateCounter(METRIC_COMPONENT_OUT_EVENTS_CNT); + mTotalDelayMs = mMetricsRecordRef.CreateCounter(METRIC_COMPONENT_TOTAL_DELAY_MS); + mEventBatchItemsCnt = mMetricsRecordRef.CreateIntGauge(METRIC_COMPONENT_BATCHER_EVENT_BATCHES_CNT); + mBufferedGroupsCnt = mMetricsRecordRef.CreateIntGauge(METRIC_COMPONENT_BATCHER_BUFFERED_GROUPS_CNT); + mBufferedEventsCnt = mMetricsRecordRef.CreateIntGauge(METRIC_COMPONENT_BATCHER_BUFFERED_EVENTS_CNT); + mBufferedDataSizeByte = mMetricsRecordRef.CreateIntGauge(METRIC_COMPONENT_BATCHER_BUFFERED_SIZE_BYTES); return true; } diff --git a/core/pipeline/plugin/instance/FlusherInstance.cpp b/core/pipeline/plugin/instance/FlusherInstance.cpp index 997f75fe4b..003b279022 100644 --- a/core/pipeline/plugin/instance/FlusherInstance.cpp +++ b/core/pipeline/plugin/instance/FlusherInstance.cpp @@ -25,8 +25,8 @@ bool FlusherInstance::Init(const Json::Value& config, PipelineContext& context, return false; } - mInEventsCnt = mPlugin->GetMetricsRecordRef().CreateCounter(METRIC_IN_EVENTS_CNT); - mInGroupDataSizeBytes = mPlugin->GetMetricsRecordRef().CreateCounter(METRIC_IN_EVENT_GROUP_SIZE_BYTES); + mInEventsCnt = mPlugin->GetMetricsRecordRef().CreateCounter(METRIC_PLUGIN_IN_EVENTS_CNT); + mInGroupDataSizeBytes = mPlugin->GetMetricsRecordRef().CreateCounter(METRIC_PLUGIN_IN_EVENT_GROUP_SIZE_BYTES); return true; } diff --git a/core/pipeline/plugin/interface/Plugin.h b/core/pipeline/plugin/interface/Plugin.h index e63d098ae3..1ee9464022 100644 --- a/core/pipeline/plugin/interface/Plugin.h +++ b/core/pipeline/plugin/interface/Plugin.h @@ -21,6 +21,7 @@ #include #include "monitor/LogtailMetric.h" +#include "monitor/MetricConstants.h" #include "pipeline/PipelineContext.h" namespace logtail { @@ -39,18 +40,13 @@ class Plugin { const std::string& id, const std::string& nodeID, const std::string& childNodeID) { - std::vector> labels; - WriteMetrics::GetInstance()->PreparePluginCommonLabels(mContext->GetProjectName(), - mContext->GetLogstoreName(), - mContext->GetRegion(), - mContext->GetConfigName(), - name, - id, - nodeID, - childNodeID, - labels); - - WriteMetrics::GetInstance()->PrepareMetricsRecordRef(mMetricsRecordRef, std::move(labels)); + WriteMetrics::GetInstance()->PrepareMetricsRecordRef(mMetricsRecordRef, + {{METRIC_LABEL_PROJECT, mContext->GetProjectName()}, + {METRIC_LABEL_CONFIG_NAME, mContext->GetConfigName()}, + {METRIC_LABEL_PLUGIN_NAME, name}, + {METRIC_LABEL_PLUGIN_ID, id}, + {METRIC_LABEL_NODE_ID, nodeID}, + {METRIC_LABEL_CHILD_NODE_ID, childNodeID}}); } protected: diff --git a/core/pipeline/queue/BoundedQueueInterface.h b/core/pipeline/queue/BoundedQueueInterface.h index 350e22be37..c8102a5fbd 100644 --- a/core/pipeline/queue/BoundedQueueInterface.h +++ b/core/pipeline/queue/BoundedQueueInterface.h @@ -26,7 +26,7 @@ class BoundedQueueInterface : virtual public QueueInterface { BoundedQueueInterface(QueueKey key, size_t cap, size_t low, size_t high, const PipelineContext& ctx) : QueueInterface(key, cap, ctx), mLowWatermark(low), mHighWatermark(high) { this->mMetricsRecordRef.AddLabels({{METRIC_LABEL_KEY_QUEUE_TYPE, "bounded"}}); - mValidToPushFlag = this->mMetricsRecordRef.CreateIntGauge("valid_to_push"); + mValidToPushFlag = this->mMetricsRecordRef.CreateIntGauge(METRIC_COMPONENT_QUEUE_VALID_TO_PUSH_FLAG); } virtual ~BoundedQueueInterface() = default; diff --git a/core/pipeline/queue/BoundedSenderQueueInterface.cpp b/core/pipeline/queue/BoundedSenderQueueInterface.cpp index 03e26ebac3..3e58d48e49 100644 --- a/core/pipeline/queue/BoundedSenderQueueInterface.cpp +++ b/core/pipeline/queue/BoundedSenderQueueInterface.cpp @@ -26,8 +26,8 @@ BoundedSenderQueueInterface::BoundedSenderQueueInterface( : QueueInterface(key, cap, ctx), BoundedQueueInterface>(key, cap, low, high, ctx) { mMetricsRecordRef.AddLabels({{METRIC_LABEL_KEY_COMPONENT_NAME, "sender_queue"}}); mMetricsRecordRef.AddLabels({{METRIC_LABEL_KEY_FLUSHER_NODE_ID, flusherId}}); - mExtraBufferSize = mMetricsRecordRef.CreateIntGauge("extra_buffer_size"); - mExtraBufferDataSizeBytes = mMetricsRecordRef.CreateIntGauge("extra_buffer_data_size_bytes"); + mExtraBufferSize = mMetricsRecordRef.CreateIntGauge(METRIC_COMPONENT_QUEUE_EXTRA_BUFFER_SIZE); + mExtraBufferDataSizeBytes = mMetricsRecordRef.CreateIntGauge(METRIC_COMPONENT_QUEUE_EXTRA_BUFFER_SIZE_BYTES); } void BoundedSenderQueueInterface::SetFeedback(FeedbackInterface* feedback) { diff --git a/core/pipeline/queue/CircularProcessQueue.cpp b/core/pipeline/queue/CircularProcessQueue.cpp index e7f8b279f7..d9b7b551c8 100644 --- a/core/pipeline/queue/CircularProcessQueue.cpp +++ b/core/pipeline/queue/CircularProcessQueue.cpp @@ -25,7 +25,7 @@ namespace logtail { CircularProcessQueue::CircularProcessQueue(size_t cap, int64_t key, uint32_t priority, const PipelineContext& ctx) : QueueInterface>(key, cap, ctx), ProcessQueueInterface(key, cap, priority, ctx) { mMetricsRecordRef.AddLabels({{METRIC_LABEL_KEY_QUEUE_TYPE, "circular"}}); - mDroppedEventsCnt = mMetricsRecordRef.CreateCounter("dropped_events_cnt"); + mDiscardedEventsCnt = mMetricsRecordRef.CreateCounter(METRIC_COMPONENT_QUEUE_DISCARDED_EVENTS_CNT); WriteMetrics::GetInstance()->CommitMetricsRecordRef(mMetricsRecordRef); } @@ -38,7 +38,7 @@ bool CircularProcessQueue::Push(unique_ptr&& item) { mQueue.pop_front(); mQueueSize->Set(Size()); mQueueDataSizeByte->Sub(size); - mDroppedEventsCnt->Add(cnt); + mDiscardedEventsCnt->Add(cnt); } if (mEventCnt + newCnt > mCapacity) { return false; diff --git a/core/pipeline/queue/CircularProcessQueue.h b/core/pipeline/queue/CircularProcessQueue.h index a58a34e035..8a7c8c1ee9 100644 --- a/core/pipeline/queue/CircularProcessQueue.h +++ b/core/pipeline/queue/CircularProcessQueue.h @@ -43,7 +43,7 @@ class CircularProcessQueue : virtual public QueueInterface> mQueue; size_t mEventCnt = 0; - CounterPtr mDroppedEventsCnt; + CounterPtr mDiscardedEventsCnt; #ifdef APSARA_UNIT_TEST_MAIN friend class CircularProcessQueueUnittest; diff --git a/core/pipeline/queue/QueueInterface.h b/core/pipeline/queue/QueueInterface.h index 1c450013ad..7754190000 100644 --- a/core/pipeline/queue/QueueInterface.h +++ b/core/pipeline/queue/QueueInterface.h @@ -33,12 +33,12 @@ class QueueInterface { {METRIC_LABEL_CONFIG_NAME, ctx.GetConfigName()}, }); - mInItemsCnt = mMetricsRecordRef.CreateCounter(METRIC_IN_ITEMS_CNT); - mInItemDataSizeBytes = mMetricsRecordRef.CreateCounter(METRIC_IN_ITEM_SIZE_BYTES); - mOutItemsCnt = mMetricsRecordRef.CreateCounter(METRIC_OUT_ITEMS_CNT); - mTotalDelayMs = mMetricsRecordRef.CreateCounter(METRIC_TOTAL_DELAY_MS); - mQueueSize = mMetricsRecordRef.CreateIntGauge("queue_size"); - mQueueDataSizeByte = mMetricsRecordRef.CreateIntGauge("queue_data_size_bytes"); + mInItemsCnt = mMetricsRecordRef.CreateCounter(METRIC_COMPONENT_IN_ITEMS_CNT); + mInItemDataSizeBytes = mMetricsRecordRef.CreateCounter(METRIC_COMPONENT_IN_ITEM_SIZE_BYTES); + mOutItemsCnt = mMetricsRecordRef.CreateCounter(METRIC_COMPONENT_OUT_ITEMS_CNT); + mTotalDelayMs = mMetricsRecordRef.CreateCounter(METRIC_COMPONENT_TOTAL_DELAY_MS); + mQueueSize = mMetricsRecordRef.CreateIntGauge(METRIC_COMPONENT_QUEUE_SIZE_CNT); + mQueueDataSizeByte = mMetricsRecordRef.CreateIntGauge(METRIC_COMPONENT_QUEUE_SIZE_BYTES); } virtual ~QueueInterface() = default; diff --git a/core/pipeline/route/Router.cpp b/core/pipeline/route/Router.cpp index 73de16bcda..a83e1b4225 100644 --- a/core/pipeline/route/Router.cpp +++ b/core/pipeline/route/Router.cpp @@ -39,8 +39,8 @@ bool Router::Init(std::vector> configs, const P {{METRIC_LABEL_PROJECT, ctx.GetProjectName()}, {METRIC_LABEL_CONFIG_NAME, ctx.GetConfigName()}, {METRIC_LABEL_KEY_COMPONENT_NAME, "router"}}); - mInEventsCnt = mMetricsRecordRef.CreateCounter(METRIC_IN_EVENTS_CNT); - mInGroupDataSizeBytes = mMetricsRecordRef.CreateCounter(METRIC_IN_EVENT_GROUP_SIZE_BYTES); + mInEventsCnt = mMetricsRecordRef.CreateCounter(METRIC_COMPONENT_IN_EVENTS_CNT); + mInGroupDataSizeBytes = mMetricsRecordRef.CreateCounter(METRIC_COMPONENT_IN_EVENT_GROUP_SIZE_BYTES); return true; } diff --git a/core/pipeline/serializer/Serializer.h b/core/pipeline/serializer/Serializer.h index b632124b7a..c4e7c674ae 100644 --- a/core/pipeline/serializer/Serializer.h +++ b/core/pipeline/serializer/Serializer.h @@ -53,13 +53,13 @@ class Serializer { {METRIC_LABEL_CONFIG_NAME, f->GetContext().GetConfigName()}, {METRIC_LABEL_KEY_COMPONENT_NAME, "serializer"}, {METRIC_LABEL_KEY_FLUSHER_NODE_ID, f->GetNodeID()}}); - mInItemsCnt = mMetricsRecordRef.CreateCounter(METRIC_IN_ITEMS_CNT); - mInItemSizeBytes = mMetricsRecordRef.CreateCounter(METRIC_IN_ITEM_SIZE_BYTES); - mOutItemsCnt = mMetricsRecordRef.CreateCounter(METRIC_OUT_ITEMS_CNT); - mOutItemSizeBytes = mMetricsRecordRef.CreateCounter(METRIC_OUT_ITEM_SIZE_BYTES); - mDiscardedItemsCnt = mMetricsRecordRef.CreateCounter("discarded_items_cnt"); - mDiscardedItemSizeBytes = mMetricsRecordRef.CreateCounter("discarded_item_data_size_bytes"); - mTotalDelayMs = mMetricsRecordRef.CreateCounter(METRIC_TOTAL_DELAY_MS); + mInItemsCnt = mMetricsRecordRef.CreateCounter(METRIC_COMPONENT_IN_ITEMS_CNT); + mInItemSizeBytes = mMetricsRecordRef.CreateCounter(METRIC_COMPONENT_IN_ITEM_SIZE_BYTES); + mOutItemsCnt = mMetricsRecordRef.CreateCounter(METRIC_COMPONENT_OUT_ITEMS_CNT); + mOutItemSizeBytes = mMetricsRecordRef.CreateCounter(METRIC_COMPONENT_OUT_ITEM_SIZE_BYTES); + mTotalDelayMs = mMetricsRecordRef.CreateCounter(METRIC_COMPONENT_TOTAL_DELAY_MS); + mDiscardedItemsCnt = mMetricsRecordRef.CreateCounter(METRIC_COMPONENT_DISCARDED_ITEMS_CNT); + mDiscardedItemSizeBytes = mMetricsRecordRef.CreateCounter(METRIC_COMPONENT_DISCARDED_ITEMS_SIZE_BYTES); } virtual ~Serializer() = default; diff --git a/core/runner/FlusherRunner.cpp b/core/runner/FlusherRunner.cpp index 16af0a51df..3d176b75ef 100644 --- a/core/runner/FlusherRunner.cpp +++ b/core/runner/FlusherRunner.cpp @@ -41,6 +41,16 @@ namespace logtail { bool FlusherRunner::Init() { srand(time(nullptr)); + WriteMetrics::GetInstance()->PrepareMetricsRecordRef(mMetricsRecordRef, + {{METRIC_LABEL_KEY_RUNNER_NAME, "flusher_runner"}}); + mInItemsCnt = mMetricsRecordRef.CreateCounter(METRIC_RUNNER_IN_ITEMS_CNT); + mInItemDataSizeBytes = mMetricsRecordRef.CreateCounter(METRIC_RUNNER_IN_ITEM_SIZE_BYTES); + mOutItemsCnt = mMetricsRecordRef.CreateCounter(METRIC_RUNNER_OUT_ITEMS_CNT); + mTotalDelayMs = mMetricsRecordRef.CreateCounter(METRIC_RUNNER_TOTAL_DELAY_MS); + mLastRunTime = mMetricsRecordRef.CreateIntGauge(METRIC_RUNNER_LAST_RUN_TIME); + mInItemRawDataSizeBytes = mMetricsRecordRef.CreateCounter(METRIC_RUNNER_FLUSHER_IN_ITEM_RAW_SIZE_BYTES); + mWaitingItemsCnt = mMetricsRecordRef.CreateIntGauge(METRIC_RUNNER_FLUSHER_WAITING_ITEMS_CNT); + mThreadRes = async(launch::async, &FlusherRunner::Run, this); mLastCheckSendClientTime = time(nullptr); return true; @@ -99,12 +109,20 @@ void FlusherRunner::Run() { LOG_INFO(sLogger, ("flusher runner", "started")); while (true) { auto curTime = chrono::system_clock::now(); + mLastRunTime->Set(chrono::duration_cast(curTime.time_since_epoch()).count()); vector items; SenderQueueManager::GetInstance()->GetAllAvailableItems(items, !Application::GetInstance()->IsExiting()); if (items.empty()) { SenderQueueManager::GetInstance()->Wait(1000); } else { + for (auto itr = items.begin(); itr != items.end(); ++itr) { + mInItemDataSizeBytes->Add((*itr)->mData.size()); + mInItemRawDataSizeBytes->Add((*itr)->mRawSize); + } + mInItemsCnt->Add(items.size()); + mWaitingItemsCnt->Add(items.size()); + // smoothing send tps, walk around webserver load burst uint32_t bufferPackageCount = items.size(); if (!Application::GetInstance()->IsExiting() && AppConfig::GetInstance()->IsSendRandomSleep()) { @@ -132,6 +150,10 @@ void FlusherRunner::Run() { } Dispatch(*itr); + mWaitingItemsCnt->Sub(1); + mOutItemsCnt->Add(1); + mTotalDelayMs->Add( + chrono::duration_cast(chrono::system_clock::now() - curTime).count()); } // TODO: move the following logic to scheduler diff --git a/core/runner/FlusherRunner.h b/core/runner/FlusherRunner.h index fb7347cddb..5681c4b263 100644 --- a/core/runner/FlusherRunner.h +++ b/core/runner/FlusherRunner.h @@ -20,6 +20,7 @@ #include #include +#include "monitor/LogtailMetric.h" #include "pipeline/plugin/interface/Flusher.h" #include "pipeline/queue/SenderQueueItem.h" #include "runner/sink/SinkType.h" @@ -63,6 +64,15 @@ class FlusherRunner { int64_t mSendLastTime = 0; int32_t mSendLastByte = 0; + mutable MetricsRecordRef mMetricsRecordRef; + CounterPtr mInItemsCnt; + CounterPtr mInItemDataSizeBytes; + CounterPtr mInItemRawDataSizeBytes; + CounterPtr mOutItemsCnt; + CounterPtr mTotalDelayMs; + IntGaugePtr mWaitingItemsCnt; + IntGaugePtr mLastRunTime; + #ifdef APSARA_UNIT_TEST_MAIN friend class PluginRegistryUnittest; friend class FlusherRunnerUnittest; diff --git a/core/runner/LogProcess.cpp b/core/runner/LogProcess.cpp index cd65998408..dc3cb84ad1 100644 --- a/core/runner/LogProcess.cpp +++ b/core/runner/LogProcess.cpp @@ -15,12 +15,13 @@ #include "runner/LogProcess.h" #include "app_config/AppConfig.h" -#include "pipeline/batch/TimeoutFlushManager.h" #include "common/Flags.h" #include "go_pipeline/LogtailPlugin.h" #include "monitor/LogFileProfiler.h" #include "monitor/LogtailAlarm.h" +#include "monitor/MetricConstants.h" #include "pipeline/PipelineManager.h" +#include "pipeline/batch/TimeoutFlushManager.h" #include "pipeline/queue/ExactlyOnceQueueManager.h" #include "pipeline/queue/ProcessQueueManager.h" #include "pipeline/queue/QueueKeyManager.h" @@ -39,6 +40,12 @@ DEFINE_FLAG_INT32(default_flush_merged_buffer_interval, "default flush merged bu namespace logtail { +thread_local MetricsRecordRef LogProcess::sMetricsRecordRef; +thread_local CounterPtr LogProcess::sInGroupsCnt; +thread_local CounterPtr LogProcess::sInEventsCnt; +thread_local CounterPtr LogProcess::sInGroupDataSizeBytes; +thread_local IntGaugePtr LogProcess::sLastRunTime; + LogProcess::LogProcess() : mAccessProcessThreadRWL(ReadWriteLock::PREFER_WRITER) { } @@ -56,8 +63,6 @@ LogProcess::~LogProcess() { void LogProcess::Start() { if (mInitialized) return; - mAgentProcessQueueFullTotal = LoongCollectorMonitor::GetInstance()->GetIntGauge(METRIC_AGENT_PROCESS_QUEUE_FULL_TOTAL); - mAgentProcessQueueTotal = LoongCollectorMonitor::GetInstance()->GetIntGauge(METRIC_AGENT_PROCESS_QUEUE_TOTAL); mInitialized = true; mThreadCount = AppConfig::GetInstance()->GetProcessThreadCount(); @@ -137,13 +142,16 @@ bool LogProcess::FlushOut(int32_t waitMs) { } void* LogProcess::ProcessLoop(int32_t threadNo) { - LOG_DEBUG(sLogger, ("runner/LogProcess.hread", "Start")("threadNo", threadNo)); + LOG_DEBUG(sLogger, ("LogProcess", "Start")("threadNo", threadNo)); + // thread local metrics should be initialized in each thread + WriteMetrics::GetInstance()->PrepareMetricsRecordRef( + sMetricsRecordRef, {{METRIC_LABEL_KEY_RUNNER_NAME, "processor_runner"}, {"thread_no", ToString(threadNo)}}); + sInGroupsCnt = sMetricsRecordRef.CreateCounter(METRIC_RUNNER_IN_EVENT_GROUPS_CNT); + sInEventsCnt = sMetricsRecordRef.CreateCounter(METRIC_RUNNER_IN_EVENTS_CNT); + sInGroupDataSizeBytes = sMetricsRecordRef.CreateCounter(METRIC_RUNNER_IN_EVENT_GROUP_SIZE_BYTES); + sLastRunTime = sMetricsRecordRef.CreateIntGauge(METRIC_RUNNER_LAST_RUN_TIME); + static int32_t lastMergeTime = 0; - static atomic_int s_processCount{0}; - static atomic_long s_processBytes{0}; - static atomic_int s_processLines{0}; - // only thread 0 update metric - int32_t lastUpdateMetricTime = time(NULL); while (true) { mThreadFlags[threadNo] = false; @@ -153,35 +161,9 @@ void* LogProcess::ProcessLoop(int32_t threadNo) { lastMergeTime = curTime; } - if (threadNo == 0 && curTime - lastUpdateMetricTime >= 40) { - static auto sMonitor = LogtailMonitor::GetInstance(); - - // atomic counter will be negative if process speed is too fast. - sMonitor->UpdateMetric("process_tps", 1.0 * s_processCount / (curTime - lastUpdateMetricTime)); - sMonitor->UpdateMetric("process_bytes_ps", 1.0 * s_processBytes / (curTime - lastUpdateMetricTime)); - sMonitor->UpdateMetric("process_lines_ps", 1.0 * s_processLines / (curTime - lastUpdateMetricTime)); - lastUpdateMetricTime = curTime; - s_processCount = 0; - s_processBytes = 0; - s_processLines = 0; - - // update process queue status - uint32_t InvalidProcessQueueTotal = ProcessQueueManager::GetInstance()->GetInvalidCnt(); - sMonitor->UpdateMetric("process_queue_full", InvalidProcessQueueTotal); - mAgentProcessQueueFullTotal->Set(InvalidProcessQueueTotal); - uint32_t ProcessQueueTotal = ProcessQueueManager::GetInstance()->GetCnt(); - sMonitor->UpdateMetric("process_queue_total", ProcessQueueTotal); - mAgentProcessQueueTotal->Set(ProcessQueueTotal); - if (ExactlyOnceQueueManager::GetInstance()->GetProcessQueueCnt() > 0) { - sMonitor->UpdateMetric("eo_process_queue_full", - ExactlyOnceQueueManager::GetInstance()->GetInvalidProcessQueueCnt()); - sMonitor->UpdateMetric("eo_process_queue_total", - ExactlyOnceQueueManager::GetInstance()->GetProcessQueueCnt()); - } - } - { ReadLock lock(mAccessProcessThreadRWL); + sLastRunTime->Set(curTime); unique_ptr item; string configName; @@ -189,6 +171,9 @@ void* LogProcess::ProcessLoop(int32_t threadNo) { ProcessQueueManager::GetInstance()->Wait(100); continue; } + sInEventsCnt->Add(item->mEventGroup.GetEvents().size()); + sInGroupsCnt->Add(1); + sInGroupDataSizeBytes->Add(item->mEventGroup.DataSize()); mThreadFlags[threadNo] = true; auto pipeline = PipelineManager::GetInstance()->FindConfigByName(configName); @@ -227,12 +212,6 @@ void* LogProcess::ProcessLoop(int32_t threadNo) { pipeline->GetContext().GetRegion()); } - s_processCount++; - if (isLog) { - s_processBytes += profile.readBytes; - s_processLines += profile.splitLines; - } - if (eventGroupList.empty()) { continue; } @@ -296,7 +275,7 @@ void* LogProcess::ProcessLoop(int32_t threadNo) { } } } - LOG_WARNING(sLogger, ("runner/LogProcess.hread", "Exit")("threadNo", threadNo)); + LOG_WARNING(sLogger, ("LogProcess", "Exit")("threadNo", threadNo)); return NULL; } diff --git a/core/runner/LogProcess.h b/core/runner/LogProcess.h index c0bd77e9ad..624bb09be8 100644 --- a/core/runner/LogProcess.h +++ b/core/runner/LogProcess.h @@ -23,6 +23,7 @@ #include "common/LogRunnable.h" #include "common/Thread.h" #include "models/PipelineEventGroup.h" +#include "monitor/LogtailMetric.h" #include "monitor/Monitor.h" #include "pipeline/queue/QueueKey.h" @@ -60,8 +61,11 @@ class LogProcess : public LogRunnable { std::atomic_bool* mThreadFlags; ReadWriteLock mAccessProcessThreadRWL; - IntGaugePtr mAgentProcessQueueFullTotal; - IntGaugePtr mAgentProcessQueueTotal; + thread_local static MetricsRecordRef sMetricsRecordRef; + thread_local static CounterPtr sInGroupsCnt; + thread_local static CounterPtr sInEventsCnt; + thread_local static CounterPtr sInGroupDataSizeBytes; + thread_local static IntGaugePtr sLastRunTime; }; } // namespace logtail \ No newline at end of file diff --git a/core/runner/sink/http/HttpSink.cpp b/core/runner/sink/http/HttpSink.cpp index 0a5e7de3d6..68e276397f 100644 --- a/core/runner/sink/http/HttpSink.cpp +++ b/core/runner/sink/http/HttpSink.cpp @@ -18,6 +18,7 @@ #include "common/StringTools.h" #include "common/http/Curl.h" #include "logger/Logger.h" +#include "monitor/MetricConstants.h" #include "pipeline/plugin/interface/HttpFlusher.h" #include "pipeline/queue/QueueKeyManager.h" #include "pipeline/queue/SenderQueueItem.h" @@ -33,6 +34,19 @@ bool HttpSink::Init() { LOG_ERROR(sLogger, ("failed to init http sink", "failed to init curl multi client")); return false; } + + WriteMetrics::GetInstance()->PrepareMetricsRecordRef(mMetricsRecordRef, + {{METRIC_LABEL_KEY_RUNNER_NAME, "http sink"}}); + mInItemsCnt = mMetricsRecordRef.CreateCounter(METRIC_RUNNER_IN_ITEMS_CNT); + mLastRunTime = mMetricsRecordRef.CreateIntGauge(METRIC_RUNNER_LAST_RUN_TIME); + mOutSuccessfulItemsCnt = mMetricsRecordRef.CreateCounter(METRIC_RUNNER_HTTP_SINK_OUT_SUCCESSFUL_ITEMS_CNT); + mOutFailedItemsCnt = mMetricsRecordRef.CreateCounter(METRIC_RUNNER_HTTP_SINK_OUT_FAILED_ITEMS_CNT); + mSendingItemsCnt = mMetricsRecordRef.CreateIntGauge(METRIC_RUNNER_HTTP_SINK_SENDING_ITEMS_CNT); + mSendConcurrency = mMetricsRecordRef.CreateIntGauge(METRIC_RUNNER_HTTP_SINK_SEND_CONCURRENCY); + + // TODO: should be dynamic + mSendConcurrency->Set(AppConfig::GetInstance()->GetSendRequestConcurrency()); + mThreadRes = async(launch::async, &HttpSink::Run, this); return true; } @@ -48,9 +62,13 @@ void HttpSink::Stop() { } void HttpSink::Run() { + LOG_INFO(sLogger, ("http sink", "started")); while (true) { + mLastRunTime->Set( + chrono::duration_cast(chrono::system_clock::now().time_since_epoch()).count()); unique_ptr request; if (mQueue.WaitAndPop(request, 500)) { + mInItemsCnt->Add(1); LOG_DEBUG( sLogger, ("got item from flusher runner, item address", request->mItem)( @@ -59,6 +77,7 @@ void HttpSink::Run() { if (!AddRequestToClient(std::move(request))) { continue; } + mSendingItemsCnt->Add(1); } else if (mIsFlush && mQueue.Empty()) { break; } else { @@ -72,7 +91,7 @@ void HttpSink::Run() { } } -bool HttpSink::AddRequestToClient(std::unique_ptr&& request) { +bool HttpSink::AddRequestToClient(unique_ptr&& request) { curl_slist* headers = nullptr; CURL* curl = CreateCurlHandler(request->mMethod, request->mHTTPSFlag, @@ -90,6 +109,7 @@ bool HttpSink::AddRequestToClient(std::unique_ptr&& request) { if (curl == nullptr) { request->mItem->mStatus = SendingStatus::IDLE; FlusherRunner::GetInstance()->DecreaseHttpSendingCnt(); + mOutFailedItemsCnt->Add(1); LOG_ERROR(sLogger, ("failed to send request", "failed to init curl handler")( "action", "put sender queue item back to sender queue")("item address", request->mItem)( @@ -99,13 +119,14 @@ bool HttpSink::AddRequestToClient(std::unique_ptr&& request) { request->mPrivateData = headers; curl_easy_setopt(curl, CURLOPT_PRIVATE, request.get()); - request->mLastSendTime = std::chrono::system_clock::now(); + request->mLastSendTime = chrono::system_clock::now(); auto res = curl_multi_add_handle(mClient, curl); if (res != CURLM_OK) { request->mItem->mStatus = SendingStatus::IDLE; FlusherRunner::GetInstance()->DecreaseHttpSendingCnt(); curl_easy_cleanup(curl); + mOutFailedItemsCnt->Add(1); LOG_ERROR(sLogger, ("failed to send request", "failed to add the easy curl handle to multi_handle")("errMsg", curl_multi_strerror(res))( @@ -122,6 +143,8 @@ void HttpSink::DoRun() { CURLMcode mc; int runningHandlers = 1; while (runningHandlers) { + auto curTime = chrono::system_clock::now(); + mLastRunTime->Set(chrono::duration_cast(curTime.time_since_epoch()).count()); if ((mc = curl_multi_perform(mClient, &runningHandlers)) != CURLM_OK) { LOG_ERROR( sLogger, @@ -133,6 +156,7 @@ void HttpSink::DoRun() { unique_ptr request; if (mQueue.TryPop(request)) { + mInItemsCnt->Add(1); LOG_DEBUG( sLogger, ("got item from flusher runner, item address", request->mItem)( @@ -140,6 +164,7 @@ void HttpSink::DoRun() { "wait time", ToString(time(nullptr) - request->mEnqueTime))("try cnt", ToString(request->mTryCnt))); if (AddRequestToClient(std::move(request))) { ++runningHandlers; + mSendingItemsCnt->Add(1); } } @@ -191,10 +216,13 @@ void HttpSink::HandleCompletedRequests() { HttpSinkRequest* request = nullptr; curl_easy_getinfo(handler, CURLINFO_PRIVATE, &request); LOG_DEBUG(sLogger, - ("send http request completed, item address", request->mItem) - ("config-flusher-dst", QueueKeyManager::GetInstance()->GetName(request->mItem->mQueueKey)) - ("response time",ToString(chrono::duration_cast(chrono::system_clock::now()- request->mLastSendTime).count()) + "ms") - ("try cnt", ToString(request->mTryCnt))); + ("send http request completed, item address", request->mItem)( + "config-flusher-dst", QueueKeyManager::GetInstance()->GetName(request->mItem->mQueueKey))( + "response time", + ToString(chrono::duration_cast(chrono::system_clock::now() + - request->mLastSendTime) + .count()) + + "ms")("try cnt", ToString(request->mTryCnt))); switch (msg->data.result) { case CURLE_OK: { long statusCode = 0; @@ -202,6 +230,8 @@ void HttpSink::HandleCompletedRequests() { request->mResponse.mStatusCode = (int32_t)statusCode; static_cast(request->mItem->mFlusher)->OnSendDone(request->mResponse, request->mItem); FlusherRunner::GetInstance()->DecreaseHttpSendingCnt(); + mOutSuccessfulItemsCnt->Add(1); + mSendingItemsCnt->Sub(1); break; } default: @@ -225,6 +255,8 @@ void HttpSink::HandleCompletedRequests() { ->OnSendDone(request->mResponse, request->mItem); FlusherRunner::GetInstance()->DecreaseHttpSendingCnt(); } + mOutFailedItemsCnt->Add(1); + mSendingItemsCnt->Sub(1); break; } curl_multi_remove_handle(mClient, handler); diff --git a/core/runner/sink/http/HttpSink.h b/core/runner/sink/http/HttpSink.h index 6e11b84855..56d9e01302 100644 --- a/core/runner/sink/http/HttpSink.h +++ b/core/runner/sink/http/HttpSink.h @@ -25,6 +25,7 @@ #include "runner/sink/Sink.h" #include "runner/sink/http/HttpSinkRequest.h" +#include "monitor/LogtailMetric.h" namespace logtail { @@ -55,6 +56,15 @@ class HttpSink : public Sink { std::future mThreadRes; std::atomic_bool mIsFlush = false; + mutable MetricsRecordRef mMetricsRecordRef; + CounterPtr mInItemsCnt; + CounterPtr mOutSuccessfulItemsCnt; + CounterPtr mOutFailedItemsCnt; + // CounterPtr mTotalDelayMs; // TODO: should record distribution instead of average + IntGaugePtr mSendingItemsCnt; + IntGaugePtr mSendConcurrency; + IntGaugePtr mLastRunTime; + #ifdef APSARA_UNIT_TEST_MAIN friend class FlusherRunnerUnittest; #endif diff --git a/core/unittest/pipeline/PipelineUnittest.cpp b/core/unittest/pipeline/PipelineUnittest.cpp index d4d76b5138..4c061b255d 100644 --- a/core/unittest/pipeline/PipelineUnittest.cpp +++ b/core/unittest/pipeline/PipelineUnittest.cpp @@ -2688,10 +2688,10 @@ void PipelineUnittest::TestProcess() const { pipeline.mProcessorLine.emplace_back(std::move(processor)); WriteMetrics::GetInstance()->PrepareMetricsRecordRef(pipeline.mMetricsRecordRef, {}); - pipeline.mProcessorsInEventsCnt = pipeline.mMetricsRecordRef.CreateCounter("processors_in_events_cnt"); - pipeline.mProcessorsInGroupsCnt = pipeline.mMetricsRecordRef.CreateCounter("processors_in_event_groups_cnt"); + pipeline.mProcessorsInEventsCnt = pipeline.mMetricsRecordRef.CreateCounter("pipeline_processors_in_events_total"); + pipeline.mProcessorsInGroupsCnt = pipeline.mMetricsRecordRef.CreateCounter("pipeline_processors_in_event_groups_total"); pipeline.mProcessorsInGroupDataSizeBytes - = pipeline.mMetricsRecordRef.CreateCounter("processors_in_event_group_data_size_bytes"); + = pipeline.mMetricsRecordRef.CreateCounter("processors_in_event_group_size_bytes"); pipeline.mProcessorsTotalDelayMs = pipeline.mMetricsRecordRef.CreateCounter("processors_total_delay_ms"); vector groups; diff --git a/core/unittest/queue/CircularProcessQueueUnittest.cpp b/core/unittest/queue/CircularProcessQueueUnittest.cpp index 82fce43912..2e6e1411f5 100644 --- a/core/unittest/queue/CircularProcessQueueUnittest.cpp +++ b/core/unittest/queue/CircularProcessQueueUnittest.cpp @@ -172,7 +172,7 @@ void CircularProcessQueueUnittest::TestMetric() { APSARA_TEST_EQUAL(dataSize1 + dataSize2, mQueue->mInItemDataSizeBytes->GetValue()); APSARA_TEST_EQUAL(1U, mQueue->mQueueSize->GetValue()); APSARA_TEST_EQUAL(dataSize2, mQueue->mQueueDataSizeByte->GetValue()); - APSARA_TEST_EQUAL(2U, mQueue->mDroppedEventsCnt->GetValue()); + APSARA_TEST_EQUAL(2U, mQueue->mDiscardedEventsCnt->GetValue()); mQueue->Pop(item); APSARA_TEST_EQUAL(1U, mQueue->mOutItemsCnt->GetValue()); From 7cd66f4c61f34e9b4245ae1a336e6ea5faa03c0d Mon Sep 17 00:00:00 2001 From: henryzhx8 Date: Tue, 24 Sep 2024 15:20:07 +0800 Subject: [PATCH 7/9] fix deadlock in full drain mode on config update (#1776) --- core/file_server/event_handler/LogInput.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/file_server/event_handler/LogInput.cpp b/core/file_server/event_handler/LogInput.cpp index 96a3ac31af..0ee7970ee7 100644 --- a/core/file_server/event_handler/LogInput.cpp +++ b/core/file_server/event_handler/LogInput.cpp @@ -106,7 +106,7 @@ void LogInput::Resume() { void LogInput::HoldOn() { LOG_INFO(sLogger, ("event handle daemon pause", "starts")); - if (BOOL_FLAG(enable_full_drain_mode)) { + if (BOOL_FLAG(enable_full_drain_mode) && Application::GetInstance()->IsExiting()) { unique_lock lock(mThreadRunningMux); mStopCV.wait(lock, [this]() { return mInteruptFlag; }); } else { From f051364dba562795e8860ec734c1271414920aef Mon Sep 17 00:00:00 2001 From: henryzhx8 Date: Wed, 25 Sep 2024 09:42:35 +0800 Subject: [PATCH 8/9] remove InputStream (#1777) --- core/CMakeLists.txt | 2 +- core/app_config/AppConfig.cpp | 9 ------ core/app_config/AppConfig.h | 4 +-- core/application/Application.cpp | 1 - core/config/PipelineConfig.cpp | 40 ----------------------- core/monitor/Monitor.cpp | 1 - core/pipeline/PipelineManager.cpp | 43 +++---------------------- core/pipeline/PipelineManager.h | 1 - core/pipeline/plugin/PluginRegistry.cpp | 6 ---- core/plugin/input/input.cmake | 7 ---- 10 files changed, 8 insertions(+), 106 deletions(-) diff --git a/core/CMakeLists.txt b/core/CMakeLists.txt index 39073d2271..c6fa28e92a 100644 --- a/core/CMakeLists.txt +++ b/core/CMakeLists.txt @@ -125,7 +125,7 @@ set(SUB_DIRECTORIES_LIST ) if (LINUX) if (ENABLE_ENTERPRISE) - set(SUB_DIRECTORIES_LIST ${SUB_DIRECTORIES_LIST} shennong shennong/sdk streamlog aggregator) + set(SUB_DIRECTORIES_LIST ${SUB_DIRECTORIES_LIST} shennong shennong/sdk) endif() elseif(MSVC) if (ENABLE_ENTERPRISE) diff --git a/core/app_config/AppConfig.cpp b/core/app_config/AppConfig.cpp index e95ef285c0..f6950069bc 100644 --- a/core/app_config/AppConfig.cpp +++ b/core/app_config/AppConfig.cpp @@ -851,15 +851,6 @@ bool AppConfig::CheckAndResetProxyAddress(const char* envKey, string& address) { } void AppConfig::LoadOtherConf(const Json::Value& confJson) { - // if (confJson.isMember("mapping_conf_path") && confJson["mapping_conf_path"].isString()) - // mMappingConfigPath = confJson["mapping_conf_path"].asString(); - // else - // mMappingConfigPath = STRING_FLAG(default_mapping_config_path); - - if (confJson.isMember("streamlog_open") && confJson["streamlog_open"].isBool()) { - mOpenStreamLog = confJson["streamlog_open"].asBool(); - } - { int32_t oasConnectTimeout = 0; if (LoadInt32Parameter( diff --git a/core/app_config/AppConfig.h b/core/app_config/AppConfig.h index 69a129c068..c52b67a16b 100644 --- a/core/app_config/AppConfig.h +++ b/core/app_config/AppConfig.h @@ -57,7 +57,7 @@ class AppConfig { // uint32_t mStreamLogTcpPort; // uint32_t mStreamLogPoolSizeInMb; // uint32_t mStreamLogRcvLenPerCall; - bool mOpenStreamLog; + // bool mOpenStreamLog; // performance float mCpuUsageUpLimit; @@ -285,7 +285,7 @@ class AppConfig { // uint32_t GetStreamLogRcvLenPerCall() const { return mStreamLogRcvLenPerCall; } - bool GetOpenStreamLog() const { return mOpenStreamLog; } + // bool GetOpenStreamLog() const { return mOpenStreamLog; } std::string GetIlogtailConfigJson() { ScopedSpinLock lock(mAppConfigLock); diff --git a/core/application/Application.cpp b/core/application/Application.cpp index 3ed86d1e65..f0a2fd66ae 100644 --- a/core/application/Application.cpp +++ b/core/application/Application.cpp @@ -57,7 +57,6 @@ #if defined(__linux__) && !defined(__ANDROID__) #include "common/LinuxDaemonUtil.h" #include "shennong/ShennongManager.h" -#include "streamlog/StreamLogManager.h" #endif #else #include "provider/Provider.h" diff --git a/core/config/PipelineConfig.cpp b/core/config/PipelineConfig.cpp index 83f3c581b2..b44d05df87 100644 --- a/core/config/PipelineConfig.cpp +++ b/core/config/PipelineConfig.cpp @@ -140,9 +140,6 @@ bool PipelineConfig::Parse() { // extensions module parsing will rely on their results. bool hasObserverInput = false; bool hasFileInput = false; -#ifdef __ENTERPRISE__ - bool hasStreamInput = false; -#endif key = "inputs"; itr = mDetail->find(key.c_str(), key.c_str() + key.size()); if (!itr) { @@ -244,21 +241,10 @@ bool PipelineConfig::Parse() { hasObserverInput = true; } else if (pluginType == "input_file" || pluginType == "input_container_stdio") { hasFileInput = true; -#ifdef __ENTERPRISE__ - } else if (pluginType == "input_stream") { - if (!AppConfig::GetInstance()->GetOpenStreamLog()) { - PARAM_ERROR_RETURN( - sLogger, alarm, "stream log is not enabled", noModule, mName, mProject, mLogstore, mRegion); - } - hasStreamInput = true; -#endif } } // TODO: remove these special restrictions bool hasSpecialInput = hasObserverInput || hasFileInput; -#ifdef __ENTERPRISE__ - hasSpecialInput = hasSpecialInput || hasStreamInput; -#endif if (hasSpecialInput && (*mDetail)["inputs"].size() > 1) { PARAM_ERROR_RETURN(sLogger, alarm, @@ -283,19 +269,6 @@ bool PipelineConfig::Parse() { mLogstore, mRegion); } -#ifdef __ENTERPRISE__ - // TODO: remove these special restrictions - if (hasStreamInput && !itr->empty()) { - PARAM_ERROR_RETURN(sLogger, - alarm, - "processor plugins coexist with input_stream", - noModule, - mName, - mProject, - mLogstore, - mRegion); - } -#endif bool isCurrentPluginNative = true; for (Json::Value::ArrayIndex i = 0; i < itr->size(); ++i) { const Json::Value& plugin = (*itr)[i]; @@ -520,19 +493,6 @@ bool PipelineConfig::Parse() { PARAM_ERROR_RETURN( sLogger, alarm, "unsupported flusher plugin", pluginType, mName, mProject, mLogstore, mRegion); } -#ifdef __ENTERPRISE__ - // TODO: remove these special restrictions - if (hasStreamInput && pluginType != "flusher_sls") { - PARAM_ERROR_RETURN(sLogger, - alarm, - "flusher plugins other than flusher_sls coexist with input_stream", - noModule, - mName, - mProject, - mLogstore, - mRegion); - } -#endif mFlushers.push_back(&plugin); } // TODO: remove these special restrictions diff --git a/core/monitor/Monitor.cpp b/core/monitor/Monitor.cpp index 2888af75e5..580b6e7e1d 100644 --- a/core/monitor/Monitor.cpp +++ b/core/monitor/Monitor.cpp @@ -284,7 +284,6 @@ bool LogtailMonitor::SendStatusProfile(bool suicide) { AddLogContent(logPtr, "projects", FlusherSLS::GetAllProjects()); AddLogContent(logPtr, "instance_id", Application::GetInstance()->GetInstanceId()); AddLogContent(logPtr, "instance_key", id); - AddLogContent(logPtr, "syslog_open", AppConfig::GetInstance()->GetOpenStreamLog()); // Host informations. AddLogContent(logPtr, "ip", LogFileProfiler::mIpAddr); AddLogContent(logPtr, "hostname", LogFileProfiler::mHostname); diff --git a/core/pipeline/PipelineManager.cpp b/core/pipeline/PipelineManager.cpp index 9a0a69f60c..f5cac36598 100644 --- a/core/pipeline/PipelineManager.cpp +++ b/core/pipeline/PipelineManager.cpp @@ -28,7 +28,6 @@ #if defined(__ENTERPRISE__) && defined(__linux__) && !defined(__ANDROID__) #include "app_config/AppConfig.h" #include "shennong/ShennongManager.h" -#include "streamlog/StreamLogManager.h" #endif #include "config/feedbacker/ConfigFeedbackReceiver.h" #include "pipeline/queue/ProcessQueueManager.h" @@ -42,40 +41,26 @@ void logtail::PipelineManager::UpdatePipelines(PipelineConfigDiff& diff) { #ifndef APSARA_UNIT_TEST_MAIN // 过渡使用 static bool isFileServerStarted = false, isInputObserverStarted = false; -#if defined(__ENTERPRISE__) && defined(__linux__) && !defined(__ANDROID__) - static bool isInputStreamStarted = false; -#endif - bool isInputObserverChanged = false, isInputFileChanged = false, isInputStreamChanged = false, - isInputContainerStdioChanged = false; + bool isInputObserverChanged = false, isInputFileChanged = false, isInputContainerStdioChanged = false; for (const auto& name : diff.mRemoved) { CheckIfInputUpdated(mPipelineNameEntityMap[name]->GetConfig()["inputs"][0], isInputObserverChanged, isInputFileChanged, - isInputStreamChanged, isInputContainerStdioChanged); } for (const auto& config : diff.mModified) { - CheckIfInputUpdated(*config.mInputs[0], - isInputObserverChanged, - isInputFileChanged, - isInputStreamChanged, - isInputContainerStdioChanged); + CheckIfInputUpdated( + *config.mInputs[0], isInputObserverChanged, isInputFileChanged, isInputContainerStdioChanged); } for (const auto& config : diff.mAdded) { - CheckIfInputUpdated(*config.mInputs[0], - isInputObserverChanged, - isInputFileChanged, - isInputStreamChanged, - isInputContainerStdioChanged); + CheckIfInputUpdated( + *config.mInputs[0], isInputObserverChanged, isInputFileChanged, isInputContainerStdioChanged); } #if defined(__ENTERPRISE__) && defined(__linux__) && !defined(__ANDROID__) if (AppConfig::GetInstance()->ShennongSocketEnabled()) { ShennongManager::GetInstance()->Pause(); } - if (isInputStreamStarted && isInputStreamChanged) { - StreamLogManager::GetInstance()->ShutdownConfigUsage(); - } #endif #if defined(__linux__) && !defined(__ANDROID__) if (isInputObserverStarted && isInputObserverChanged) { @@ -182,16 +167,6 @@ void logtail::PipelineManager::UpdatePipelines(PipelineConfigDiff& diff) { } #endif #if defined(__ENTERPRISE__) && defined(__linux__) && !defined(__ANDROID__) - if (isInputStreamChanged) { - if (isInputStreamStarted) { - StreamLogManager::GetInstance()->StartupConfigUsage(); - } else { - if (AppConfig::GetInstance()->GetOpenStreamLog()) { - StreamLogManager::GetInstance()->Init(); - isInputStreamStarted = true; - } - } - } if (AppConfig::GetInstance()->ShennongSocketEnabled()) { ShennongManager::GetInstance()->Resume(); } @@ -228,11 +203,6 @@ string PipelineManager::GetPluginStatistics() const { void PipelineManager::StopAllPipelines() { LOG_INFO(sLogger, ("stop all pipelines", "starts")); -#if defined(__ENTERPRISE__) && defined(__linux__) && !defined(__ANDROID__) - if (AppConfig::GetInstance()->GetOpenStreamLog()) { - StreamLogManager::GetInstance()->Shutdown(); - } -#endif PrometheusInputRunner::GetInstance()->Stop(); #if defined(__linux__) && !defined(__ANDROID__) ObserverManager::GetInstance()->HoldOn(true); @@ -296,15 +266,12 @@ void PipelineManager::DecreasePluginUsageCnt(const unordered_map> mPipelineNameEntityMap; diff --git a/core/pipeline/plugin/PluginRegistry.cpp b/core/pipeline/plugin/PluginRegistry.cpp index 5beeb2ab0e..ba04e1eedd 100644 --- a/core/pipeline/plugin/PluginRegistry.cpp +++ b/core/pipeline/plugin/PluginRegistry.cpp @@ -36,9 +36,6 @@ #include "plugin/input/InputNetworkSecurity.h" #include "plugin/input/InputProcessSecurity.h" #include "plugin/input/InputObserverNetwork.h" -#ifdef __ENTERPRISE__ -#include "plugin/input/InputStream.h" -#endif #endif #include "logger/Logger.h" #include "pipeline/plugin/creator/CProcessor.h" @@ -136,9 +133,6 @@ void PluginRegistry::LoadStaticPlugins() { RegisterInputCreator(new StaticInputCreator()); RegisterInputCreator(new StaticInputCreator()); RegisterInputCreator(new StaticInputCreator()); -#ifdef __ENTERPRISE__ - RegisterInputCreator(new StaticInputCreator()); -#endif #endif RegisterProcessorCreator(new StaticProcessorCreator()); diff --git a/core/plugin/input/input.cmake b/core/plugin/input/input.cmake index eb213b46b5..b986267eca 100644 --- a/core/plugin/input/input.cmake +++ b/core/plugin/input/input.cmake @@ -23,17 +23,10 @@ list(APPEND THIS_SOURCE_FILES_LIST ${THIS_SOURCE_FILES}) if(MSVC) # remove observer related files in input list(REMOVE_ITEM THIS_SOURCE_FILES_LIST ${CMAKE_SOURCE_DIR}/plugin/input/InputObserverNetwork.cpp ${CMAKE_SOURCE_DIR}/plugin/input/InputObserverNetwork.h) - if (ENABLE_ENTERPRISE) - list(REMOVE_ITEM THIS_SOURCE_FILES_LIST ${CMAKE_SOURCE_DIR}/plugin/input/InputStream.cpp ${CMAKE_SOURCE_DIR}/plugin/input/InputStream.h) - endif () elseif(UNIX) if (NOT LINUX) # remove observer related files in input list(REMOVE_ITEM THIS_SOURCE_FILES_LIST ${CMAKE_SOURCE_DIR}/plugin/input/InputObserverNetwork.cpp ${CMAKE_SOURCE_DIR}/plugin/input/InputObserverNetwork.h) - # remove inputStream in input - if (ENABLE_ENTERPRISE) - list(REMOVE_ITEM THIS_SOURCE_FILES_LIST ${CMAKE_SOURCE_DIR}/plugin/input/InputStream.cpp ${CMAKE_SOURCE_DIR}/plugin/input/InputStream.h) - endif () endif() endif() From 5862d33f682195e0f040291a3b5de8a9bebecc28 Mon Sep 17 00:00:00 2001 From: Tom Yu Date: Wed, 25 Sep 2024 10:08:08 +0800 Subject: [PATCH 9/9] fix: do not use gtid if gtid is disabled (#1781) Fixed an issue where the input_canal plugin used GTID as the sync method even when gtid_enabled was set to false. This led to syncing from the beginning due to receiving an empty GTID set. Syncing from the beginning is usually undesirable as it consumes significant CPU resources on the server side and deviates from the default plugin behavior. This commit ensures that the plugin respects the gtid_enabled switch and the gtid_mode query result. --- plugins/input/canal/input_canal.go | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/plugins/input/canal/input_canal.go b/plugins/input/canal/input_canal.go index 3f0b4d7b3b..0f35d9d52a 100644 --- a/plugins/input/canal/input_canal.go +++ b/plugins/input/canal/input_canal.go @@ -759,9 +759,13 @@ func (sc *ServiceCanal) Start(c pipeline.Collector) error { startPos.Pos = sc.checkpoint.Offset } if nil == gtid && 0 == len(startPos.Name) && !sc.StartFromBegining { - gtid, err = sc.getLatestGTID() - if err != nil { - logger.Warning(sc.context.GetRuntimeContext(), "CANAL_START_ALARM", "Call getLatestGTID failed, error", err) + if sc.isGTIDEnabled { + gtid, err = sc.getLatestGTID() + if err != nil { + logger.Warning(sc.context.GetRuntimeContext(), "CANAL_START_ALARM", "Call getLatestGTID failed, error", err) + } + } + if gtid == nil { startPos = sc.GetBinlogLatestPos() } logger.Infof(sc.context.GetRuntimeContext(), "Get latest checkpoint GTID: %v Position: %v", gtid, startPos)