diff --git a/core/CMakeLists.txt b/core/CMakeLists.txt index 162ed8a82b..c6fa28e92a 100644 --- a/core/CMakeLists.txt +++ b/core/CMakeLists.txt @@ -33,6 +33,8 @@ cmake_dependent_option(ENABLE_STATIC_LINK_CRT "Build Logtail by linking CRT stat option(WITHOUTGDB "Build Logtail without gdb") option(WITHSPL "Build Logtail and UT with SPL" ON) option(BUILD_LOGTAIL_UT "Build unit test for Logtail") +set(PROVIDER_PATH "provider" CACHE PATH "Path to the provider module") # external provider path can be set with -DPROVIDER_PATH +set(UNITTEST_PATH "unittest" CACHE PATH "Path to the unittest module") # external unittest path can be set with -DUNITTEST_PATH if (BUILD_LOGTAIL_SHARED_LIBRARY AND WITHSPL) message(FATEL_ERROR, "Generating logtail shared library is not supported to be linked with SPL. WITHSPL should be set OFF.") @@ -109,13 +111,13 @@ include(${CMAKE_CURRENT_SOURCE_DIR}/plugin/processor/links.cmake) include(${CMAKE_CURRENT_SOURCE_DIR}/plugin/flusher/flusher.cmake) include(${CMAKE_CURRENT_SOURCE_DIR}/plugin/flusher/links.cmake) -# Subdirectories (modules). except for common, input, processor, flusher, observer, helper and spl. +# Subdirectories (modules). except for common, input, processor, flusher, observer, helper, spl, and provider. set(SUB_DIRECTORIES_LIST application app_config checkpoint container_manager logger go_pipeline monitor profile_sender models - config config/feedbacker config/provider config/watcher - pipeline pipeline/batch pipeline/compression pipeline/limiter pipeline/plugin pipeline/plugin/creator pipeline/plugin/instance pipeline/plugin/interface pipeline/queue pipeline/route pipeline/serializer + config config/watcher + pipeline pipeline/batch pipeline/limiter pipeline/plugin pipeline/plugin/creator pipeline/plugin/instance pipeline/plugin/interface pipeline/queue pipeline/route pipeline/serializer runner runner/sink/http - protobuf/config_server/v1 protobuf/config_server/v2 protobuf/sls + protobuf/sls file_server file_server/event file_server/event_handler file_server/event_listener file_server/reader file_server/polling prometheus prometheus/labels prometheus/schedulers prometheus/async ebpf ebpf/observer ebpf/security ebpf/handler @@ -123,7 +125,7 @@ set(SUB_DIRECTORIES_LIST ) if (LINUX) if (ENABLE_ENTERPRISE) - set(SUB_DIRECTORIES_LIST ${SUB_DIRECTORIES_LIST} shennong shennong/sdk streamlog aggregator) + set(SUB_DIRECTORIES_LIST ${SUB_DIRECTORIES_LIST} shennong shennong/sdk) endif() elseif(MSVC) if (ENABLE_ENTERPRISE) @@ -161,6 +163,10 @@ endif() # remove several files in go_pipeline list(REMOVE_ITEM FRAMEWORK_SOURCE_FILES ${CMAKE_CURRENT_SOURCE_DIR}/go_pipeline/LogtailPluginAdapter.cpp ${CMAKE_CURRENT_SOURCE_DIR}/go_pipeline/LogtailPluginAdapter.h) + +# add provider +add_subdirectory("${PROVIDER_PATH}" "${CMAKE_BINARY_DIR}/provider") + if(MSVC) # remove linux event listener file(GLOB REMOVE_EVENT_LISTENER_SOURCES file_server/event_listener/*_Linux.cpp file_server/event_listener/*_Linux.h) @@ -231,12 +237,12 @@ if(BUILD_LOGTAIL OR BUILD_LOGTAIL_SHARED_LIBRARY) flusher_link(${LOGTAIL_TARGET}) all_link(${LOGTAIL_TARGET}) common_link(${LOGTAIL_TARGET}) + target_link_libraries(${LOGTAIL_TARGET} provider) endif() # Logtail UT. if (BUILD_LOGTAIL_UT) - message(STATUS "Build unittest.") - add_definitions(-DVIRTUAL=virtual) + message(STATUS "Build unittest.") function(delete_gcda_files target_directory) if(EXISTS "${target_directory}") message(STATUS "Deleting .gcda files in ${target_directory}") @@ -249,7 +255,5 @@ if (BUILD_LOGTAIL_UT) delete_gcda_files(".") include(CTest) enable_testing() - add_subdirectory(unittest) -else() - add_definitions(-DVIRTUAL= ) + add_subdirectory("${UNITTEST_PATH}" "${CMAKE_BINARY_DIR}/unittest") endif () \ No newline at end of file diff --git a/core/app_config/AppConfig.cpp b/core/app_config/AppConfig.cpp index e95ef285c0..f6950069bc 100644 --- a/core/app_config/AppConfig.cpp +++ b/core/app_config/AppConfig.cpp @@ -851,15 +851,6 @@ bool AppConfig::CheckAndResetProxyAddress(const char* envKey, string& address) { } void AppConfig::LoadOtherConf(const Json::Value& confJson) { - // if (confJson.isMember("mapping_conf_path") && confJson["mapping_conf_path"].isString()) - // mMappingConfigPath = confJson["mapping_conf_path"].asString(); - // else - // mMappingConfigPath = STRING_FLAG(default_mapping_config_path); - - if (confJson.isMember("streamlog_open") && confJson["streamlog_open"].isBool()) { - mOpenStreamLog = confJson["streamlog_open"].asBool(); - } - { int32_t oasConnectTimeout = 0; if (LoadInt32Parameter( diff --git a/core/app_config/AppConfig.h b/core/app_config/AppConfig.h index 69a129c068..c52b67a16b 100644 --- a/core/app_config/AppConfig.h +++ b/core/app_config/AppConfig.h @@ -57,7 +57,7 @@ class AppConfig { // uint32_t mStreamLogTcpPort; // uint32_t mStreamLogPoolSizeInMb; // uint32_t mStreamLogRcvLenPerCall; - bool mOpenStreamLog; + // bool mOpenStreamLog; // performance float mCpuUsageUpLimit; @@ -285,7 +285,7 @@ class AppConfig { // uint32_t GetStreamLogRcvLenPerCall() const { return mStreamLogRcvLenPerCall; } - bool GetOpenStreamLog() const { return mOpenStreamLog; } + // bool GetOpenStreamLog() const { return mOpenStreamLog; } std::string GetIlogtailConfigJson() { ScopedSpinLock lock(mAppConfigLock); diff --git a/core/application/Application.cpp b/core/application/Application.cpp index 9cf0924b9d..8dd0124181 100644 --- a/core/application/Application.cpp +++ b/core/application/Application.cpp @@ -57,11 +57,9 @@ #if defined(__linux__) && !defined(__ANDROID__) #include "common/LinuxDaemonUtil.h" #include "shennong/ShennongManager.h" -#include "streamlog/StreamLogManager.h" #endif #else -#include "config/provider/CommonConfigProvider.h" -#include "config/provider/LegacyCommonConfigProvider.h" +#include "provider/Provider.h" #endif DEFINE_FLAG_BOOL(ilogtail_disable_core, "disable core in worker process", true); @@ -242,8 +240,7 @@ void Application::Start() { // GCOVR_EXCL_START EnterpriseConfigProvider::GetInstance()->Init("enterprise"); LegacyConfigProvider::GetInstance()->Init("legacy"); #else - CommonConfigProvider::GetInstance()->Init("common_v2"); - LegacyCommonConfigProvider::GetInstance()->Init("common"); + InitRemoteConfigProviders(); #endif LogtailAlarm::GetInstance()->Init(); @@ -377,8 +374,10 @@ void Application::Exit() { EnterpriseConfigProvider::GetInstance()->Stop(); LegacyConfigProvider::GetInstance()->Stop(); #else - CommonConfigProvider::GetInstance()->Stop(); - LegacyCommonConfigProvider::GetInstance()->Stop(); + auto remoteConfigProviders = GetRemoteConfigProviders(); + for (auto& provider : remoteConfigProviders) { + provider->Stop(); + } #endif LogtailMonitor::GetInstance()->Stop(); diff --git a/core/common/common.cmake b/core/common/common.cmake index e905a95fa3..caf8e45b98 100644 --- a/core/common/common.cmake +++ b/core/common/common.cmake @@ -30,7 +30,7 @@ list(APPEND THIS_SOURCE_FILES_LIST ${XX_HASH_SOURCE_FILES}) list(APPEND THIS_SOURCE_FILES_LIST ${CMAKE_SOURCE_DIR}/common/memory/SourceBuffer.h) list(APPEND THIS_SOURCE_FILES_LIST ${CMAKE_SOURCE_DIR}/common/http/AsynCurlRunner.cpp ${CMAKE_SOURCE_DIR}/common/http/Curl.cpp ${CMAKE_SOURCE_DIR}/common/http/HttpResponse.cpp) list(APPEND THIS_SOURCE_FILES_LIST ${CMAKE_SOURCE_DIR}/common/timer/Timer.cpp ${CMAKE_SOURCE_DIR}/common/timer/HttpRequestTimerEvent.cpp) - +list(APPEND THIS_SOURCE_FILES_LIST ${CMAKE_SOURCE_DIR}/common/compression/Compressor.cpp ${CMAKE_SOURCE_DIR}/common/compression/CompressorFactory.cpp ${CMAKE_SOURCE_DIR}/common/compression/LZ4Compressor.cpp ${CMAKE_SOURCE_DIR}/common/compression/ZstdCompressor.cpp) # remove several files in common list(REMOVE_ITEM THIS_SOURCE_FILES_LIST ${CMAKE_SOURCE_DIR}/common/BoostRegexValidator.cpp ${CMAKE_SOURCE_DIR}/common/GetUUID.cpp) diff --git a/core/pipeline/compression/CompressType.h b/core/common/compression/CompressType.h similarity index 85% rename from core/pipeline/compression/CompressType.h rename to core/common/compression/CompressType.h index 96f996ee9a..1580bb5967 100644 --- a/core/pipeline/compression/CompressType.h +++ b/core/common/compression/CompressType.h @@ -18,6 +18,14 @@ namespace logtail { -enum class CompressType { NONE, LZ4, ZSTD }; +enum class CompressType { + NONE, + LZ4, + ZSTD +#ifdef APSARA_UNIT_TEST_MAIN + , + MOCK +#endif +}; } // namespace logtail diff --git a/core/common/compression/Compressor.cpp b/core/common/compression/Compressor.cpp new file mode 100644 index 0000000000..7a6d95951e --- /dev/null +++ b/core/common/compression/Compressor.cpp @@ -0,0 +1,59 @@ +// Copyright 2024 iLogtail Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "common/compression/Compressor.h" + +#include + +#include "monitor/MetricConstants.h" + +using namespace std; + +namespace logtail { + +void Compressor::SetMetricRecordRef(MetricLabels&& labels, DynamicMetricLabels&& dynamicLabels) { + WriteMetrics::GetInstance()->PrepareMetricsRecordRef( + mMetricsRecordRef, std::move(labels), std::move(dynamicLabels)); + mInItemsCnt = mMetricsRecordRef.CreateCounter(METRIC_COMPONENT_IN_ITEMS_CNT); + mInItemSizeBytes = mMetricsRecordRef.CreateCounter(METRIC_COMPONENT_IN_ITEM_SIZE_BYTES); + mOutItemsCnt = mMetricsRecordRef.CreateCounter(METRIC_COMPONENT_OUT_ITEMS_CNT); + mOutItemSizeBytes = mMetricsRecordRef.CreateCounter(METRIC_COMPONENT_OUT_ITEM_SIZE_BYTES); + mTotalDelayMs = mMetricsRecordRef.CreateCounter(METRIC_COMPONENT_TOTAL_DELAY_MS); + mDiscardedItemsCnt = mMetricsRecordRef.CreateCounter(METRIC_COMPONENT_DISCARDED_ITEMS_CNT); + mDiscardedItemSizeBytes = mMetricsRecordRef.CreateCounter(METRIC_COMPONENT_DISCARDED_ITEMS_SIZE_BYTES); +} + +bool Compressor::DoCompress(const string& input, string& output, string& errorMsg) { + if (mMetricsRecordRef != nullptr) { + mInItemsCnt->Add(1); + mInItemSizeBytes->Add(input.size()); + } + + auto before = chrono::system_clock::now(); + auto res = Compress(input, output, errorMsg); + + if (mMetricsRecordRef != nullptr) { + mTotalDelayMs->Add(chrono::duration_cast(chrono::system_clock::now() - before).count()); + if (res) { + mOutItemsCnt->Add(1); + mOutItemSizeBytes->Add(output.size()); + } else { + mDiscardedItemsCnt->Add(1); + mDiscardedItemSizeBytes->Add(input.size()); + } + } + return res; +} + +} // namespace logtail diff --git a/core/pipeline/compression/Compressor.h b/core/common/compression/Compressor.h similarity index 64% rename from core/pipeline/compression/Compressor.h rename to core/common/compression/Compressor.h index 1694f6e6ad..0c32358ca5 100644 --- a/core/pipeline/compression/Compressor.h +++ b/core/common/compression/Compressor.h @@ -18,7 +18,8 @@ #include -#include "pipeline/compression/CompressType.h" +#include "monitor/LogtailMetric.h" +#include "common/compression/CompressType.h" namespace logtail { @@ -27,7 +28,7 @@ class Compressor { Compressor(CompressType type) : mType(type) {} virtual ~Compressor() = default; - virtual bool Compress(const std::string& input, std::string& output, std::string& errorMsg) = 0; + bool DoCompress(const std::string& input, std::string& output, std::string& errorMsg); #ifdef APSARA_UNIT_TEST_MAIN // buffer shoudl be reserved for output before calling this function @@ -35,9 +36,27 @@ class Compressor { #endif CompressType GetCompressType() const { return mType; } + void SetMetricRecordRef(MetricLabels&& labels, DynamicMetricLabels&& dynamicLabels = {}); + +protected: + mutable MetricsRecordRef mMetricsRecordRef; + CounterPtr mInItemsCnt; + CounterPtr mInItemSizeBytes; + CounterPtr mOutItemsCnt; + CounterPtr mOutItemSizeBytes; + CounterPtr mDiscardedItemsCnt; + CounterPtr mDiscardedItemSizeBytes; + CounterPtr mTotalDelayMs; private: + virtual bool Compress(const std::string& input, std::string& output, std::string& errorMsg) = 0; + CompressType mType = CompressType::NONE; + +#ifdef APSARA_UNIT_TEST_MAIN + friend class CompressorUnittest; + friend class CompressorFactoryUnittest; +#endif }; } // namespace logtail diff --git a/core/pipeline/compression/CompressorFactory.cpp b/core/common/compression/CompressorFactory.cpp similarity index 71% rename from core/pipeline/compression/CompressorFactory.cpp rename to core/common/compression/CompressorFactory.cpp index 1c91f24035..0e3b27197b 100644 --- a/core/pipeline/compression/CompressorFactory.cpp +++ b/core/common/compression/CompressorFactory.cpp @@ -12,21 +12,24 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include "pipeline/compression/CompressorFactory.h" +#include "common/compression/CompressorFactory.h" #include "common/ParamExtractor.h" -#include "pipeline/compression/LZ4Compressor.h" -#include "pipeline/compression/ZstdCompressor.h" +#include "monitor/MetricConstants.h" +#include "common/compression/LZ4Compressor.h" +#include "common/compression/ZstdCompressor.h" using namespace std; namespace logtail { unique_ptr CompressorFactory::Create(const Json::Value& config, - const PipelineContext& ctx, - const string& pluginType, - CompressType defaultType) { + const PipelineContext& ctx, + const string& pluginType, + const string& flusherId, + CompressType defaultType) { string compressType, errorMsg; + unique_ptr compressor; if (!GetOptionalStringParam(config, "CompressType", compressType, errorMsg)) { PARAM_WARNING_DEFAULT(ctx.GetLogger(), ctx.GetAlarm(), @@ -37,11 +40,11 @@ unique_ptr CompressorFactory::Create(const Json::Value& config, ctx.GetProjectName(), ctx.GetLogstoreName(), ctx.GetRegion()); - return Create(defaultType); + compressor = Create(defaultType); } else if (compressType == "lz4") { - return Create(CompressType::LZ4); + compressor = Create(CompressType::LZ4); } else if (compressType == "zstd") { - return Create(CompressType::ZSTD); + compressor = Create(CompressType::ZSTD); } else if (compressType == "none") { return nullptr; } else if (!compressType.empty()) { @@ -54,10 +57,15 @@ unique_ptr CompressorFactory::Create(const Json::Value& config, ctx.GetProjectName(), ctx.GetLogstoreName(), ctx.GetRegion()); - return Create(defaultType); + compressor = Create(defaultType); } else { - return Create(defaultType); + compressor = Create(defaultType); } + compressor->SetMetricRecordRef({{METRIC_LABEL_PROJECT, ctx.GetProjectName()}, + {METRIC_LABEL_CONFIG_NAME, ctx.GetConfigName()}, + {METRIC_LABEL_KEY_COMPONENT_NAME, "compressor"}, + {METRIC_LABEL_KEY_FLUSHER_NODE_ID, flusherId}}); + return compressor; } unique_ptr CompressorFactory::Create(CompressType type) { diff --git a/core/pipeline/compression/CompressorFactory.h b/core/common/compression/CompressorFactory.h similarity index 87% rename from core/pipeline/compression/CompressorFactory.h rename to core/common/compression/CompressorFactory.h index dbdc826f16..8311332b7a 100644 --- a/core/pipeline/compression/CompressorFactory.h +++ b/core/common/compression/CompressorFactory.h @@ -21,9 +21,9 @@ #include #include -#include "pipeline/compression/CompressType.h" -#include "pipeline/compression/Compressor.h" #include "pipeline/PipelineContext.h" +#include "common/compression/CompressType.h" +#include "common/compression/Compressor.h" namespace logtail { @@ -42,13 +42,13 @@ class CompressorFactory { std::unique_ptr Create(const Json::Value& config, const PipelineContext& ctx, const std::string& pluginType, + const std::string& flusherId, CompressType defaultType); + std::unique_ptr Create(CompressType type); private: CompressorFactory() = default; ~CompressorFactory() = default; - - std::unique_ptr Create(CompressType defaultType); }; } // namespace logtail diff --git a/core/pipeline/compression/LZ4Compressor.cpp b/core/common/compression/LZ4Compressor.cpp similarity index 97% rename from core/pipeline/compression/LZ4Compressor.cpp rename to core/common/compression/LZ4Compressor.cpp index 7063812610..ec54c1c4e4 100644 --- a/core/pipeline/compression/LZ4Compressor.cpp +++ b/core/common/compression/LZ4Compressor.cpp @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include "pipeline/compression/LZ4Compressor.h" +#include "common/compression/LZ4Compressor.h" #include diff --git a/core/pipeline/compression/LZ4Compressor.h b/core/common/compression/LZ4Compressor.h similarity index 89% rename from core/pipeline/compression/LZ4Compressor.h rename to core/common/compression/LZ4Compressor.h index 6a64ea2af6..9fd453c163 100644 --- a/core/pipeline/compression/LZ4Compressor.h +++ b/core/common/compression/LZ4Compressor.h @@ -16,19 +16,20 @@ #pragma once -#include "pipeline/compression/Compressor.h" +#include "common/compression/Compressor.h" namespace logtail { class LZ4Compressor : public Compressor { public: - LZ4Compressor(CompressType type) : Compressor(type){}; + LZ4Compressor(CompressType type) : Compressor(type) {}; - bool Compress(const std::string& input, std::string& output, std::string& errorMsg) override; - #ifdef APSARA_UNIT_TEST_MAIN bool UnCompress(const std::string& input, std::string& output, std::string& errorMsg) override; #endif + +private: + bool Compress(const std::string& input, std::string& output, std::string& errorMsg) override; }; } // namespace logtail diff --git a/core/pipeline/compression/ZstdCompressor.cpp b/core/common/compression/ZstdCompressor.cpp similarity index 97% rename from core/pipeline/compression/ZstdCompressor.cpp rename to core/common/compression/ZstdCompressor.cpp index 6e47985b5e..c482c569a1 100644 --- a/core/pipeline/compression/ZstdCompressor.cpp +++ b/core/common/compression/ZstdCompressor.cpp @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include "pipeline/compression/ZstdCompressor.h" +#include "common/compression/ZstdCompressor.h" #include diff --git a/core/pipeline/compression/ZstdCompressor.h b/core/common/compression/ZstdCompressor.h similarity index 92% rename from core/pipeline/compression/ZstdCompressor.h rename to core/common/compression/ZstdCompressor.h index 1e910a2b0d..bf53c7d96d 100644 --- a/core/pipeline/compression/ZstdCompressor.h +++ b/core/common/compression/ZstdCompressor.h @@ -16,21 +16,21 @@ #pragma once -#include "pipeline/compression/Compressor.h" +#include "common/compression/Compressor.h" namespace logtail { class ZstdCompressor : public Compressor { public: - ZstdCompressor(CompressType type, int32_t level = 1) : Compressor(type), mCompressionLevel(level){}; - - bool Compress(const std::string& input, std::string& output, std::string& errorMsg) override; + ZstdCompressor(CompressType type, int32_t level = 1) : Compressor(type), mCompressionLevel(level) {}; #ifdef APSARA_UNIT_TEST_MAIN bool UnCompress(const std::string& input, std::string& output, std::string& errorMsg) override; #endif private: + bool Compress(const std::string& input, std::string& output, std::string& errorMsg) override; + int32_t mCompressionLevel = 1; }; diff --git a/core/config/PipelineConfig.cpp b/core/config/PipelineConfig.cpp index 83f3c581b2..b44d05df87 100644 --- a/core/config/PipelineConfig.cpp +++ b/core/config/PipelineConfig.cpp @@ -140,9 +140,6 @@ bool PipelineConfig::Parse() { // extensions module parsing will rely on their results. bool hasObserverInput = false; bool hasFileInput = false; -#ifdef __ENTERPRISE__ - bool hasStreamInput = false; -#endif key = "inputs"; itr = mDetail->find(key.c_str(), key.c_str() + key.size()); if (!itr) { @@ -244,21 +241,10 @@ bool PipelineConfig::Parse() { hasObserverInput = true; } else if (pluginType == "input_file" || pluginType == "input_container_stdio") { hasFileInput = true; -#ifdef __ENTERPRISE__ - } else if (pluginType == "input_stream") { - if (!AppConfig::GetInstance()->GetOpenStreamLog()) { - PARAM_ERROR_RETURN( - sLogger, alarm, "stream log is not enabled", noModule, mName, mProject, mLogstore, mRegion); - } - hasStreamInput = true; -#endif } } // TODO: remove these special restrictions bool hasSpecialInput = hasObserverInput || hasFileInput; -#ifdef __ENTERPRISE__ - hasSpecialInput = hasSpecialInput || hasStreamInput; -#endif if (hasSpecialInput && (*mDetail)["inputs"].size() > 1) { PARAM_ERROR_RETURN(sLogger, alarm, @@ -283,19 +269,6 @@ bool PipelineConfig::Parse() { mLogstore, mRegion); } -#ifdef __ENTERPRISE__ - // TODO: remove these special restrictions - if (hasStreamInput && !itr->empty()) { - PARAM_ERROR_RETURN(sLogger, - alarm, - "processor plugins coexist with input_stream", - noModule, - mName, - mProject, - mLogstore, - mRegion); - } -#endif bool isCurrentPluginNative = true; for (Json::Value::ArrayIndex i = 0; i < itr->size(); ++i) { const Json::Value& plugin = (*itr)[i]; @@ -520,19 +493,6 @@ bool PipelineConfig::Parse() { PARAM_ERROR_RETURN( sLogger, alarm, "unsupported flusher plugin", pluginType, mName, mProject, mLogstore, mRegion); } -#ifdef __ENTERPRISE__ - // TODO: remove these special restrictions - if (hasStreamInput && pluginType != "flusher_sls") { - PARAM_ERROR_RETURN(sLogger, - alarm, - "flusher plugins other than flusher_sls coexist with input_stream", - noModule, - mName, - mProject, - mLogstore, - mRegion); - } -#endif mFlushers.push_back(&plugin); } // TODO: remove these special restrictions diff --git a/core/config/provider/CommonConfigProvider.cpp b/core/config/common_provider/CommonConfigProvider.cpp similarity index 99% rename from core/config/provider/CommonConfigProvider.cpp rename to core/config/common_provider/CommonConfigProvider.cpp index 77e67b2c5f..c96d5e8399 100644 --- a/core/config/provider/CommonConfigProvider.cpp +++ b/core/config/common_provider/CommonConfigProvider.cpp @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include "config/provider/CommonConfigProvider.h" +#include "CommonConfigProvider.h" #include diff --git a/core/config/provider/CommonConfigProvider.h b/core/config/common_provider/CommonConfigProvider.h similarity index 98% rename from core/config/provider/CommonConfigProvider.h rename to core/config/common_provider/CommonConfigProvider.h index 75f20521b7..804e8cb50c 100644 --- a/core/config/provider/CommonConfigProvider.h +++ b/core/config/common_provider/CommonConfigProvider.h @@ -125,7 +125,7 @@ class CommonConfigProvider : public ConfigProvider, ConfigFeedbackable { ConfigServerAddress GetOneConfigServerAddress(bool changeConfigServer); - VIRTUAL bool SendHttpRequest(const std::string& operation, + virtual bool SendHttpRequest(const std::string& operation, const std::string& reqBody, const std::string& configType, const std::string& requestId, diff --git a/core/config/provider/LegacyCommonConfigProvider.cpp b/core/config/common_provider/LegacyCommonConfigProvider.cpp similarity index 99% rename from core/config/provider/LegacyCommonConfigProvider.cpp rename to core/config/common_provider/LegacyCommonConfigProvider.cpp index 1a017758b6..f5d000711f 100644 --- a/core/config/provider/LegacyCommonConfigProvider.cpp +++ b/core/config/common_provider/LegacyCommonConfigProvider.cpp @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include "config/provider/LegacyCommonConfigProvider.h" +#include "LegacyCommonConfigProvider.h" #include diff --git a/core/config/provider/LegacyCommonConfigProvider.h b/core/config/common_provider/LegacyCommonConfigProvider.h similarity index 100% rename from core/config/provider/LegacyCommonConfigProvider.h rename to core/config/common_provider/LegacyCommonConfigProvider.h diff --git a/core/file_server/FileServer.cpp b/core/file_server/FileServer.cpp index 49b1953b72..0e03c3f80d 100644 --- a/core/file_server/FileServer.cpp +++ b/core/file_server/FileServer.cpp @@ -18,12 +18,12 @@ #include "common/Flags.h" #include "common/StringTools.h" #include "common/TimeUtil.h" +#include "file_server/ConfigManager.h" #include "file_server/EventDispatcher.h" #include "file_server/event_handler/LogInput.h" -#include "file_server/ConfigManager.h" -#include "plugin/input/InputFile.h" #include "file_server/polling/PollingDirFile.h" #include "file_server/polling/PollingModify.h" +#include "plugin/input/InputFile.h" DEFINE_FLAG_BOOL(enable_polling_discovery, "", true); @@ -31,6 +31,11 @@ using namespace std; namespace logtail { +FileServer::FileServer() { + WriteMetrics::GetInstance()->PrepareMetricsRecordRef(mMetricsRecordRef, + {{METRIC_LABEL_KEY_RUNNER_NAME, "file_server"}}); +} + // 启动文件服务，包括加载配置、处理检查点、注册事件等 void FileServer::Start() { ConfigManager::GetInstance()->LoadDockerConfig(); diff --git a/core/file_server/FileServer.h b/core/file_server/FileServer.h index 7daee73fc5..d77b6a13a1 100644 --- a/core/file_server/FileServer.h +++ b/core/file_server/FileServer.h @@ -23,9 +23,11 @@ #include "common/Lock.h" #include "file_server/FileDiscoveryOptions.h" #include "file_server/MultilineOptions.h" +#include "file_server/reader/FileReaderOptions.h" +#include "monitor/LogtailMetric.h" #include "monitor/PluginMetricManager.h" #include "pipeline/PipelineContext.h" -#include "file_server/reader/FileReaderOptions.h" + namespace logtail { @@ -78,6 +80,7 @@ class FileServer { // for reader, event_handler ... ReentrantMetricsRecordRef GetOrCreateReentrantMetricsRecordRef(const std::string& name, MetricLabels& labels); void ReleaseReentrantMetricsRecordRef(const std::string& name, MetricLabels& labels); + MetricsRecordRef& GetMetricsRecordRef() { return mMetricsRecordRef; } // 过渡使用 void Resume(bool isConfigUpdate = true); @@ -88,7 +91,7 @@ class FileServer { void RemoveExactlyOnceConcurrency(const std::string& name); private: - FileServer() = default; + FileServer(); ~FileServer() = default; void PauseInner(); @@ -102,6 +105,8 @@ class FileServer { std::unordered_map mPipelineNamePluginMetricManagersMap; // 过渡使用 std::unordered_map mPipelineNameEOConcurrencyMap; + + mutable MetricsRecordRef mMetricsRecordRef; }; } // namespace logtail diff --git a/core/file_server/event_handler/LogInput.cpp b/core/file_server/event_handler/LogInput.cpp index a5ea529bec..0ee7970ee7 100644 --- a/core/file_server/event_handler/LogInput.cpp +++ b/core/file_server/event_handler/LogInput.cpp @@ -16,8 +16,6 @@ #include -#include "file_server/event_handler/EventHandler.h" -#include "file_server/event_handler/HistoryFileImporter.h" #include "app_config/AppConfig.h" #include "application/Application.h" #include "checkpoint/CheckPointManager.h" @@ -27,18 +25,20 @@ #include "common/RuntimeUtil.h" #include "common/StringTools.h" #include "common/TimeUtil.h" +#include "file_server/ConfigManager.h" #include "file_server/EventDispatcher.h" #include "file_server/event/BlockEventManager.h" -#include "file_server/ConfigManager.h" -#include "logger/Logger.h" -#include "monitor/LogtailAlarm.h" -#include "monitor/Monitor.h" +#include "file_server/event_handler/EventHandler.h" +#include "file_server/event_handler/HistoryFileImporter.h" #include "file_server/polling/PollingCache.h" #include "file_server/polling/PollingDirFile.h" #include "file_server/polling/PollingEventQueue.h" #include "file_server/polling/PollingModify.h" #include "file_server/reader/GloablFileDescriptorManager.h" #include "file_server/reader/LogFileReader.h" +#include "logger/Logger.h" +#include "monitor/LogtailAlarm.h" +#include "monitor/Monitor.h" #ifdef __ENTERPRISE__ #include "config/provider/EnterpriseConfigProvider.h" #endif @@ -88,9 +88,11 @@ void LogInput::Start() { mInteruptFlag = false; + mLastRunTime = FileServer::GetInstance()->GetMetricsRecordRef().CreateIntGauge(METRIC_RUNNER_LAST_RUN_TIME); mAgentOpenFdTotal = LoongCollectorMonitor::GetInstance()->GetIntGauge(METRIC_AGENT_OPEN_FD_TOTAL); - mAgentRegisterHandlerTotal - = LoongCollectorMonitor::GetInstance()->GetIntGauge(METRIC_AGENT_REGISTER_HANDLER_TOTAL); + mRegisterdHandlersCnt = FileServer::GetInstance()->GetMetricsRecordRef().CreateIntGauge(METRIC_RUNNER_FILE_WATCHED_DIRS_CNT); + mActiveReadersCnt = FileServer::GetInstance()->GetMetricsRecordRef().CreateIntGauge(METRIC_RUNNER_FILE_ACTIVE_READERS_CNT); + mEnableFileIncludedByMultiConfigs = FileServer::GetInstance()->GetMetricsRecordRef().CreateIntGauge(METRIC_RUNNER_FILE_ENABLE_FILE_INCLUDED_BY_MULTI_CONFIGS_FLAG); new Thread([this]() { ProcessLoop(); }); } @@ -104,7 +106,7 @@ void LogInput::Resume() { void LogInput::HoldOn() { LOG_INFO(sLogger, ("event handle daemon pause", "starts")); - if (BOOL_FLAG(enable_full_drain_mode)) { + if (BOOL_FLAG(enable_full_drain_mode) && Application::GetInstance()->IsExiting()) { unique_lock lock(mThreadRunningMux); mStopCV.wait(lock, [this]() { return mInteruptFlag; }); } else { @@ -342,6 +344,7 @@ void LogInput::ProcessEvent(EventDispatcher* dispatcher, Event* ev) { void LogInput::UpdateCriticalMetric(int32_t curTime) { LogtailMonitor::GetInstance()->UpdateMetric("last_read_event_time", GetTimeStamp(mLastReadEventTime, "%Y-%m-%d %H:%M:%S")); + mLastRunTime->Set(mLastReadEventTime.load()); LogtailMonitor::GetInstance()->UpdateMetric("event_tps", 1.0 * mEventProcessCount / (curTime - mLastUpdateMetricTime)); @@ -350,8 +353,9 @@ void LogInput::UpdateCriticalMetric(int32_t curTime) { mAgentOpenFdTotal->Set(openFdTotal); size_t handlerCount = EventDispatcher::GetInstance()->GetHandlerCount(); LogtailMonitor::GetInstance()->UpdateMetric("register_handler", handlerCount); - mAgentRegisterHandlerTotal->Set(handlerCount); + mRegisterdHandlersCnt->Set(handlerCount); LogtailMonitor::GetInstance()->UpdateMetric("reader_count", CheckPointManager::Instance()->GetReaderCount()); + mActiveReadersCnt->Set(CheckPointManager::Instance()->GetReaderCount()); LogtailMonitor::GetInstance()->UpdateMetric("multi_config", AppConfig::GetInstance()->IsAcceptMultiConfig()); mEventProcessCount = 0; } diff --git a/core/file_server/event_handler/LogInput.h b/core/file_server/event_handler/LogInput.h index 21217086a3..69f2ffb077 100644 --- a/core/file_server/event_handler/LogInput.h +++ b/core/file_server/event_handler/LogInput.h @@ -79,8 +79,12 @@ class LogInput : public LogRunnable { volatile bool mIdleFlag; int32_t mEventProcessCount; int32_t mLastUpdateMetricTime; + + IntGaugePtr mLastRunTime; IntGaugePtr mAgentOpenFdTotal; - IntGaugePtr mAgentRegisterHandlerTotal; + IntGaugePtr mRegisterdHandlersCnt; + IntGaugePtr mActiveReadersCnt; + IntGaugePtr mEnableFileIncludedByMultiConfigs; std::atomic_int mLastReadEventTime{0}; mutable std::mutex mThreadRunningMux; diff --git a/core/file_server/polling/PollingDirFile.cpp b/core/file_server/polling/PollingDirFile.cpp index 2bc809de7c..0a578f5ce8 100644 --- a/core/file_server/polling/PollingDirFile.cpp +++ b/core/file_server/polling/PollingDirFile.cpp @@ -21,8 +21,6 @@ #endif #include -#include "file_server/polling/PollingEventQueue.h" -#include "file_server/polling/PollingModify.h" #include "app_config/AppConfig.h" #include "common/ErrorUtil.h" #include "common/FileSystemUtil.h" @@ -30,10 +28,13 @@ #include "common/StringTools.h" #include "common/TimeUtil.h" #include "file_server/ConfigManager.h" -#include "file_server/event/Event.h" #include "file_server/FileServer.h" +#include "file_server/event/Event.h" +#include "file_server/polling/PollingEventQueue.h" +#include "file_server/polling/PollingModify.h" #include "logger/Logger.h" #include "monitor/LogtailAlarm.h" +#include "monitor/MetricConstants.h" // Control the check frequency to call ClearUnavailableFileAndDir. DEFINE_FLAG_INT32(check_not_exist_file_dir_round, "clear not exist file dir cache, round", 20); @@ -69,10 +70,10 @@ static const int64_t NANO_CONVERTING = 1000000000; void PollingDirFile::Start() { ClearCache(); mAgentConfigTotal = LoongCollectorMonitor::GetInstance()->GetIntGauge(METRIC_AGENT_PIPELINE_CONFIG_TOTAL); - mAgentPollingDirCacheSizeTotal - = LoongCollectorMonitor::GetInstance()->GetIntGauge(METRIC_AGENT_POLLING_DIR_CACHE_SIZE_TOTAL); - mAgentPollingFileCacheSizeTotal - = LoongCollectorMonitor::GetInstance()->GetIntGauge(METRIC_AGENT_POLLING_FILE_CACHE_SIZE_TOTAL); + mPollingDirCacheSize + = FileServer::GetInstance()->GetMetricsRecordRef().CreateIntGauge(METRIC_RUNNER_FILE_POLLING_DIR_CACHE_SIZE); + mPollingFileCacheSize + = FileServer::GetInstance()->GetMetricsRecordRef().CreateIntGauge(METRIC_RUNNER_FILE_POLLING_FILE_CACHE_SIZE); mRuningFlag = true; mThreadPtr = CreateThread([this]() { Polling(); }); } @@ -157,10 +158,10 @@ void PollingDirFile::Polling() { ScopedSpinLock lock(mCacheLock); size_t pollingDirCacheSize = mDirCacheMap.size(); LogtailMonitor::GetInstance()->UpdateMetric("polling_dir_cache", pollingDirCacheSize); - mAgentPollingDirCacheSizeTotal->Set(pollingDirCacheSize); + mPollingDirCacheSize->Set(pollingDirCacheSize); size_t pollingFileCacheSize = mFileCacheMap.size(); LogtailMonitor::GetInstance()->UpdateMetric("polling_file_cache", pollingFileCacheSize); - mAgentPollingFileCacheSizeTotal->Set(pollingFileCacheSize); + mPollingFileCacheSize->Set(pollingFileCacheSize); } // Iterate all normal configs, make sure stat count will not exceed limit. diff --git a/core/file_server/polling/PollingDirFile.h b/core/file_server/polling/PollingDirFile.h index 40a4ebfb8b..3e381f80fa 100644 --- a/core/file_server/polling/PollingDirFile.h +++ b/core/file_server/polling/PollingDirFile.h @@ -137,8 +137,8 @@ class PollingDirFile : public LogRunnable { uint64_t mCurrentRound; IntGaugePtr mAgentConfigTotal; - IntGaugePtr mAgentPollingDirCacheSizeTotal; - IntGaugePtr mAgentPollingFileCacheSizeTotal; + IntGaugePtr mPollingDirCacheSize; + IntGaugePtr mPollingFileCacheSize; #ifdef APSARA_UNIT_TEST_MAIN friend class PollingUnittest; diff --git a/core/file_server/polling/PollingModify.cpp b/core/file_server/polling/PollingModify.cpp index 6dac3afad4..73ff76b3cd 100644 --- a/core/file_server/polling/PollingModify.cpp +++ b/core/file_server/polling/PollingModify.cpp @@ -24,9 +24,11 @@ #include "common/Flags.h" #include "common/StringTools.h" #include "common/TimeUtil.h" +#include "file_server/FileServer.h" #include "file_server/event/Event.h" #include "logger/Logger.h" #include "monitor/LogtailAlarm.h" +#include "monitor/MetricConstants.h" using namespace std; @@ -47,7 +49,8 @@ PollingModify::~PollingModify() { void PollingModify::Start() { ClearCache(); - mAgentPollingModifySizeTotal = LoongCollectorMonitor::GetInstance()->GetIntGauge(METRIC_AGENT_POLLING_MODIFY_SIZE_TOTAL); + mPollingModifySize + = FileServer::GetInstance()->GetMetricsRecordRef().CreateIntGauge(METRIC_RUNNER_FILE_POLLING_MODIFY_CACHE_SIZE); mRuningFlag = true; mThreadPtr = CreateThread([this]() { Polling(); }); @@ -251,7 +254,7 @@ void PollingModify::Polling() { int32_t statCount = 0; size_t pollingModifySizeTotal = mModifyCacheMap.size(); LogtailMonitor::GetInstance()->UpdateMetric("polling_modify_size", pollingModifySizeTotal); - mAgentPollingModifySizeTotal->Set(pollingModifySizeTotal); + mPollingModifySize->Set(pollingModifySizeTotal); for (auto iter = mModifyCacheMap.begin(); iter != mModifyCacheMap.end(); ++iter) { if (!mRuningFlag || mHoldOnFlag) break; diff --git a/core/file_server/polling/PollingModify.h b/core/file_server/polling/PollingModify.h index c82337a898..5e3738a92a 100644 --- a/core/file_server/polling/PollingModify.h +++ b/core/file_server/polling/PollingModify.h @@ -101,7 +101,7 @@ class PollingModify : public LogRunnable { ModifyCheckCacheMap mModifyCacheMap; - IntGaugePtr mAgentPollingModifySizeTotal; + IntGaugePtr mPollingModifySize; #ifdef APSARA_UNIT_TEST_MAIN friend class PollingUnittest; diff --git a/core/go_pipeline/LogtailPlugin.cpp b/core/go_pipeline/LogtailPlugin.cpp index a84ff893c9..ccff0763fc 100644 --- a/core/go_pipeline/LogtailPlugin.cpp +++ b/core/go_pipeline/LogtailPlugin.cpp @@ -22,15 +22,15 @@ #include "common/JsonUtil.h" #include "common/LogtailCommonFlags.h" #include "common/TimeUtil.h" -#include "pipeline/compression/CompressorFactory.h" +#include "common/compression/CompressorFactory.h" #include "container_manager/ConfigContainerInfoUpdateCmd.h" #include "file_server/ConfigManager.h" #include "logger/Logger.h" #include "monitor/LogFileProfiler.h" #include "monitor/LogtailAlarm.h" #include "pipeline/PipelineManager.h" -#include "profile_sender/ProfileSender.h" #include "pipeline/queue/SenderQueueManager.h" +#include "provider/Provider.h" DEFINE_FLAG_BOOL(enable_sls_metrics_format, "if enable format metrics in SLS metricstore log pattern", false); DEFINE_FLAG_BOOL(enable_containerd_upper_dir_detect, @@ -53,16 +53,13 @@ LogtailPlugin::LogtailPlugin() { mPluginValid = false; mPluginAlarmConfig.mLogstore = "logtail_alarm"; mPluginAlarmConfig.mAliuid = STRING_FLAG(logtail_profile_aliuid); - mPluginAlarmConfig.mCompressor - = CompressorFactory::GetInstance()->Create(Json::Value(), PipelineContext(), "flusher_sls", CompressType::ZSTD); + mPluginAlarmConfig.mCompressor = CompressorFactory::GetInstance()->Create(CompressType::ZSTD); mPluginProfileConfig.mLogstore = "shennong_log_profile"; mPluginProfileConfig.mAliuid = STRING_FLAG(logtail_profile_aliuid); - mPluginProfileConfig.mCompressor - = CompressorFactory::GetInstance()->Create(Json::Value(), PipelineContext(), "flusher_sls", CompressType::ZSTD); + mPluginProfileConfig.mCompressor = CompressorFactory::GetInstance()->Create(CompressType::ZSTD); mPluginContainerConfig.mLogstore = "logtail_containers"; mPluginContainerConfig.mAliuid = STRING_FLAG(logtail_profile_aliuid); - mPluginContainerConfig.mCompressor - = CompressorFactory::GetInstance()->Create(Json::Value(), PipelineContext(), "flusher_sls", CompressType::ZSTD); + mPluginContainerConfig.mCompressor = CompressorFactory::GetInstance()->Create(CompressType::ZSTD); mPluginCfg["LogtailSysConfDir"] = AppConfig::GetInstance()->GetLogtailSysConfDir(); mPluginCfg["HostIP"] = LogFileProfiler::mIpAddr; @@ -174,22 +171,22 @@ int LogtailPlugin::SendPbV2(const char* configName, FlusherSLS* pConfig = NULL; if (configNameStr == alarmConfig->mLogstore) { pConfig = alarmConfig; - pConfig->mProject = ProfileSender::GetInstance()->GetDefaultProfileProjectName(); - pConfig->mRegion = ProfileSender::GetInstance()->GetDefaultProfileRegion(); + pConfig->mProject = GetProfileSender()->GetDefaultProfileProjectName(); + pConfig->mRegion = GetProfileSender()->GetDefaultProfileRegion(); if (pConfig->mProject.empty()) { return 0; } } else if (configNameStr == profileConfig->mLogstore) { pConfig = profileConfig; - pConfig->mProject = ProfileSender::GetInstance()->GetDefaultProfileProjectName(); - pConfig->mRegion = ProfileSender::GetInstance()->GetDefaultProfileRegion(); + pConfig->mProject = GetProfileSender()->GetDefaultProfileProjectName(); + pConfig->mRegion = GetProfileSender()->GetDefaultProfileRegion(); if (pConfig->mProject.empty()) { return 0; } } else if (configNameStr == containerConfig->mLogstore) { pConfig = containerConfig; - pConfig->mProject = ProfileSender::GetInstance()->GetDefaultProfileProjectName(); - pConfig->mRegion = ProfileSender::GetInstance()->GetDefaultProfileRegion(); + pConfig->mProject = GetProfileSender()->GetDefaultProfileProjectName(); + pConfig->mRegion = GetProfileSender()->GetDefaultProfileRegion(); if (pConfig->mProject.empty()) { return 0; } @@ -470,7 +467,8 @@ void LogtailPlugin::ProcessLogGroup(const std::string& configName, } } -void LogtailPlugin::GetGoMetrics(std::vector>& metircsList, const string& metricType) { +void LogtailPlugin::GetGoMetrics(std::vector>& metircsList, + const string& metricType) { if (mGetGoMetricsFun != nullptr) { GoString type; type.n = metricType.size(); diff --git a/core/monitor/LogFileProfiler.cpp b/core/monitor/LogFileProfiler.cpp index 701cd03a33..b9dbaf2794 100644 --- a/core/monitor/LogFileProfiler.cpp +++ b/core/monitor/LogFileProfiler.cpp @@ -27,7 +27,7 @@ #include "common/version.h" #include "file_server/ConfigManager.h" #include "logger/Logger.h" -#include "profile_sender/ProfileSender.h" +#include "provider/Provider.h" #include "pipeline/queue/QueueKeyManager.h" DEFINE_FLAG_INT32(profile_data_send_interval, "interval of send LogFile/DomainSocket profile data, seconds", 600); @@ -261,7 +261,7 @@ void LogFileProfiler::SendProfileData(bool forceSend) { } } UpdateDumpData(logGroup, detail, logstore); - ProfileSender::GetInstance()->SendToProfileProject(region, logGroup); + GetProfileSender()->SendToProfileProject(region, logGroup); } while (true); DumpToLocal(curTime, forceSend, detail, logstore); mLastSendTime = curTime; diff --git a/core/monitor/LogtailAlarm.cpp b/core/monitor/LogtailAlarm.cpp index 65cf05314d..cde2320161 100644 --- a/core/monitor/LogtailAlarm.cpp +++ b/core/monitor/LogtailAlarm.cpp @@ -23,7 +23,7 @@ #include "common/TimeUtil.h" #include "common/version.h" #include "protobuf/sls/sls_logs.pb.h" -#include "profile_sender/ProfileSender.h" +#include "provider/Provider.h" #include "pipeline/queue/QueueKeyManager.h" #include "pipeline/queue/SenderQueueManager.h" @@ -195,7 +195,7 @@ void LogtailAlarm::SendAllRegionAlarm() { // check sender queue status, if invalid jump this region QueueKey alarmPrjLogstoreKey = QueueKeyManager::GetInstance()->GetKey( - "-flusher_sls-" + ProfileSender::GetInstance()->GetProfileProjectName(region) + "#logtail_alarm"); + "-flusher_sls-" + GetProfileSender()->GetProfileProjectName(region) + "#logtail_alarm"); if (!SenderQueueManager::GetInstance()->IsValidToPush(alarmPrjLogstoreKey)) { // jump this region ++sendRegionIndex; @@ -264,7 +264,7 @@ void LogtailAlarm::SendAllRegionAlarm() { continue; } // this is an anonymous send and non lock send - ProfileSender::GetInstance()->SendToProfileProject(region, logGroup); + GetProfileSender()->SendToProfileProject(region, logGroup); } while (true); } @@ -298,7 +298,7 @@ void LogtailAlarm::SendAlarm(const LogtailAlarmType alarmType, } // ignore alarm for profile data - if (ProfileSender::GetInstance()->IsProfileData(region, projectName, category)) { + if (GetProfileSender()->IsProfileData(region, projectName, category)) { return; } // LOG_DEBUG(sLogger, ("Add Alarm", region)("projectName", projectName)("alarm index", diff --git a/core/monitor/LogtailMetric.cpp b/core/monitor/LogtailMetric.cpp index d6ddc5e768..6b26300d8c 100644 --- a/core/monitor/LogtailMetric.cpp +++ b/core/monitor/LogtailMetric.cpp @@ -205,34 +205,11 @@ WriteMetrics::~WriteMetrics() { Clear(); } -void WriteMetrics::PreparePluginCommonLabels(const std::string& projectName, - const std::string& logstoreName, - const std::string& region, - const std::string& configName, - const std::string& pluginType, - const std::string& pluginID, - const std::string& nodeID, - const std::string& childNodeID, - MetricLabels& labels) { - labels.emplace_back(std::make_pair(METRIC_LABEL_PROJECT, projectName)); - labels.emplace_back(std::make_pair(METRIC_LABEL_LOGSTORE, logstoreName)); - labels.emplace_back(std::make_pair(METRIC_LABEL_REGION, region)); - labels.emplace_back(std::make_pair(METRIC_LABEL_CONFIG_NAME, configName)); - labels.emplace_back(std::make_pair(METRIC_LABEL_PLUGIN_NAME, pluginType)); - labels.emplace_back(std::make_pair(METRIC_LABEL_PLUGIN_ID, pluginID)); - labels.emplace_back(std::make_pair(METRIC_LABEL_NODE_ID, nodeID)); - labels.emplace_back(std::make_pair(METRIC_LABEL_CHILD_NODE_ID, childNodeID)); -} - void WriteMetrics::PrepareMetricsRecordRef(MetricsRecordRef& ref, MetricLabels&& labels, DynamicMetricLabels&& dynamicLabels) { - MetricsRecord* cur = new MetricsRecord(std::make_shared(labels), - std::make_shared(dynamicLabels)); - ref.SetMetricsRecord(cur); - std::lock_guard lock(mMutex); - cur->SetNext(mHead); - mHead = cur; + CreateMetricsRecordRef(ref, std::move(labels), std::move(dynamicLabels)); + CommitMetricsRecordRef(ref); } void WriteMetrics::CreateMetricsRecordRef(MetricsRecordRef& ref, diff --git a/core/monitor/LogtailMetric.h b/core/monitor/LogtailMetric.h index d6d76666ed..6f4ef9e8ef 100644 --- a/core/monitor/LogtailMetric.h +++ b/core/monitor/LogtailMetric.h @@ -58,6 +58,8 @@ class MetricsRecord { class MetricsRecordRef { friend class WriteMetrics; + friend bool operator==(const MetricsRecordRef& lhs, std::nullptr_t rhs); + friend bool operator==(std::nullptr_t rhs, const MetricsRecordRef& lhs); private: MetricsRecord* mMetrics = nullptr; @@ -83,6 +85,22 @@ class MetricsRecordRef { #endif }; +inline bool operator==(const MetricsRecordRef& lhs, std::nullptr_t rhs) { + return lhs.mMetrics == rhs; +} + +inline bool operator==(std::nullptr_t lhs, const MetricsRecordRef& rhs) { + return lhs == rhs.mMetrics; +} + +inline bool operator!=(const MetricsRecordRef& lhs, std::nullptr_t rhs) { + return !(lhs == rhs); +} + +inline bool operator!=(std::nullptr_t lhs, const MetricsRecordRef& rhs) { + return !(lhs == rhs); +} + class ReentrantMetricsRecord { private: MetricsRecordRef mMetricsRecordRef; @@ -115,15 +133,7 @@ class WriteMetrics { static WriteMetrics* ptr = new WriteMetrics(); return ptr; } - void PreparePluginCommonLabels(const std::string& projectName, - const std::string& logstoreName, - const std::string& region, - const std::string& configName, - const std::string& pluginType, - const std::string& pluginID, - const std::string& nodeID, - const std::string& childNodeID, - MetricLabels& labels); + void PrepareMetricsRecordRef(MetricsRecordRef& ref, MetricLabels&& labels, DynamicMetricLabels&& dynamicLabels = {}); void CreateMetricsRecordRef(MetricsRecordRef& ref, MetricLabels&& labels, DynamicMetricLabels&& dynamicLabels = {}); diff --git a/core/monitor/MetricConstants.cpp b/core/monitor/MetricConstants.cpp index 52c8d1158b..e0b43e87e3 100644 --- a/core/monitor/MetricConstants.cpp +++ b/core/monitor/MetricConstants.cpp @@ -45,21 +45,12 @@ const std::string METRIC_AGENT_MEMORY = "agent_memory_used_mb"; const std::string METRIC_AGENT_MEMORY_GO = "agent_go_memory_used_mb"; const std::string METRIC_AGENT_GO_ROUTINES_TOTAL = "agent_go_routines_total"; const std::string METRIC_AGENT_OPEN_FD_TOTAL = "agent_open_fd_total"; -const std::string METRIC_AGENT_POLLING_DIR_CACHE_SIZE_TOTAL = "agent_polling_dir_cache_size_total"; -const std::string METRIC_AGENT_POLLING_FILE_CACHE_SIZE_TOTAL = "agent_polling_file_cache_size_total"; -const std::string METRIC_AGENT_POLLING_MODIFY_SIZE_TOTAL = "agent_polling_modify_size_total"; -const std::string METRIC_AGENT_REGISTER_HANDLER_TOTAL = "agent_register_handler_total"; const std::string METRIC_AGENT_INSTANCE_CONFIG_TOTAL = "agent_instance_config_total"; const std::string METRIC_AGENT_PIPELINE_CONFIG_TOTAL = "agent_pipeline_config_total"; const std::string METRIC_AGENT_ENV_PIPELINE_CONFIG_TOTAL = "agent_env_pipeline_config_total"; const std::string METRIC_AGENT_CRD_PIPELINE_CONFIG_TOTAL = "agent_crd_pipeline_config_total"; const std::string METRIC_AGENT_CONSOLE_PIPELINE_CONFIG_TOTAL = "agent_console_pipeline_config_total"; const std::string METRIC_AGENT_PLUGIN_TOTAL = "agent_plugin_total"; -const std::string METRIC_AGENT_PROCESS_QUEUE_FULL_TOTAL = "agent_process_queue_full_total"; -const std::string METRIC_AGENT_PROCESS_QUEUE_TOTAL = "agent_process_queue_total"; -const std::string METRIC_AGENT_SEND_QUEUE_FULL_TOTAL = "agent_send_queue_full_total"; -const std::string METRIC_AGENT_SEND_QUEUE_TOTAL = "agent_send_queue_total"; -const std::string METRIC_AGENT_USED_SENDING_CONCURRENCY = "agent_used_sending_concurrency"; // common plugin labels const std::string METRIC_LABEL_PROJECT = "project"; @@ -72,6 +63,7 @@ const std::string METRIC_LABEL_NODE_ID = "node_id"; const std::string METRIC_LABEL_CHILD_NODE_ID = "child_node_id"; const std::string METRIC_LABEL_KEY_COMPONENT_NAME = "component_name"; +const std::string METRIC_LABEL_KEY_RUNNER_NAME = "runner_name"; const std::string METRIC_LABEL_KEY_QUEUE_TYPE = "queue_type"; const std::string METRIC_LABEL_KEY_EXACTLY_ONCE_FLAG = "is_exactly_once"; const std::string METRIC_LABEL_KEY_FLUSHER_NODE_ID = "flusher_node_id"; @@ -137,8 +129,6 @@ const std::string METRIC_PROC_PARSE_STDOUT_TOTAL = "proc_parse_stdout_total"; const std::string METRIC_PROC_PARSE_STDERR_TOTAL = "proc_parse_stderr_total"; // flusher common metrics -const std::string METRIC_FLUSHER_IN_RECORDS_TOTAL = "flusher_in_records_total"; -const std::string METRIC_FLUSHER_IN_RECORDS_SIZE_BYTES = "flusher_in_records_size_bytes"; const std::string METRIC_FLUSHER_ERROR_TOTAL = "flusher_error_total"; const std::string METRIC_FLUSHER_DISCARD_RECORDS_TOTAL = "flusher_discard_records_total"; const std::string METRIC_FLUSHER_SUCCESS_RECORDS_TOTAL = "flusher_success_records_total"; @@ -151,4 +141,82 @@ const std::string METRIC_FLUSHER_QUOTA_ERROR_TOTAL = "flusher_quota_error_total" const std::string METRIC_FLUSHER_RETRIES_TOTAL = "flusher_retries_total"; const std::string METRIC_FLUSHER_RETRIES_ERROR_TOTAL = "flusher_retries_error_total"; +////////////////////////////////////////////////////////////////////////// +// plugin +////////////////////////////////////////////////////////////////////////// +// common metrics +const std::string METRIC_PLUGIN_IN_EVENTS_CNT = "plugin_in_events_total"; +const std::string METRIC_PLUGIN_IN_EVENT_GROUP_SIZE_BYTES = "plugin_in_event_group_size_bytes"; + +////////////////////////////////////////////////////////////////////////// +// component +////////////////////////////////////////////////////////////////////////// +// common metrics +const std::string METRIC_COMPONENT_IN_EVENTS_CNT = "component_in_events_total"; +const std::string METRIC_COMPONENT_IN_ITEMS_CNT = "component_in_items_total"; +const std::string METRIC_COMPONENT_IN_EVENT_GROUP_SIZE_BYTES = "component_in_event_group_size_bytes"; +const std::string METRIC_COMPONENT_IN_ITEM_SIZE_BYTES = "component_in_item_size_bytes"; +const std::string METRIC_COMPONENT_OUT_EVENTS_CNT = "component_out_events_total"; +const std::string METRIC_COMPONENT_OUT_ITEMS_CNT = "component_out_items_total"; +const std::string METRIC_COMPONENT_OUT_ITEM_SIZE_BYTES = "component_out_item_size_bytes"; +const std::string METRIC_COMPONENT_TOTAL_DELAY_MS = "component_total_delay_ms"; +const std::string METRIC_COMPONENT_DISCARDED_ITEMS_CNT = "component_discarded_items_total"; +const std::string METRIC_COMPONENT_DISCARDED_ITEMS_SIZE_BYTES = "component_discarded_item_size_bytes"; + +// batcher metrics +const std::string METRIC_COMPONENT_BATCHER_EVENT_BATCHES_CNT = "component_event_batches_total"; +const std::string METRIC_COMPONENT_BATCHER_BUFFERED_GROUPS_CNT = "component_buffered_groups_total"; +const std::string METRIC_COMPONENT_BATCHER_BUFFERED_EVENTS_CNT = "component_buffered_events_total"; +const std::string METRIC_COMPONENT_BATCHER_BUFFERED_SIZE_BYTES = "component_buffered_size_bytes"; + +// queue metrics +const std::string METRIC_COMPONENT_QUEUE_SIZE_CNT = "component_queue_size"; +const std::string METRIC_COMPONENT_QUEUE_SIZE_BYTES = "component_queue_size_bytes"; +const std::string METRIC_COMPONENT_QUEUE_VALID_TO_PUSH_FLAG = "component_valid_to_push"; +const std::string METRIC_COMPONENT_QUEUE_EXTRA_BUFFER_SIZE = "component_extra_buffer_size"; +const std::string METRIC_COMPONENT_QUEUE_EXTRA_BUFFER_SIZE_BYTES = "component_extra_buffer_size"; +const std::string METRIC_COMPONENT_QUEUE_DISCARDED_EVENTS_CNT = "component_discarded_events_total"; + +////////////////////////////////////////////////////////////////////////// +// pipeline +////////////////////////////////////////////////////////////////////////// +const std::string METRIC_PIPELINE_START_TIME = "pipeline_start_time"; +const std::string METRIC_PIPELINE_PROCESSORS_IN_EVENTS_CNT = "pipeline_processors_in_events_total"; +const std::string METRIC_PIPELINE_PROCESSORS_IN_EVENT_GROUPS_CNT = "pipeline_processors_in_event_groups_total"; +const std::string METRIC_PIPELINE_PROCESSORS_IN_EVENT_GROUP_SIZE_BYTES + = "pipeline_processors_in_event_group_size_bytes"; +const std::string METRIC_PIPELINE_PROCESSORS_TOTAL_DELAY_MS = "pipeline_processors_total_delay_ms"; + +////////////////////////////////////////////////////////////////////////// +// runner +////////////////////////////////////////////////////////////////////////// +// common metrics +const std::string METRIC_RUNNER_IN_EVENTS_CNT = "runner_in_events_total"; +const std::string METRIC_RUNNER_IN_EVENT_GROUPS_CNT = "runner_in_event_groups_total"; +const std::string METRIC_RUNNER_IN_ITEMS_CNT = "runner_in_items_total"; +const std::string METRIC_RUNNER_IN_EVENT_GROUP_SIZE_BYTES = "runner_in_event_group_size_bytes"; +const std::string METRIC_RUNNER_IN_ITEM_SIZE_BYTES = "runner_in_item_size_bytes"; +const std::string METRIC_RUNNER_OUT_ITEMS_CNT = "runner_out_items_total"; +const std::string METRIC_RUNNER_TOTAL_DELAY_MS = "runner_total_delay_ms"; +const std::string METRIC_RUNNER_LAST_RUN_TIME = "runner_last_run_time"; + +// http sink metrics +const std::string METRIC_RUNNER_HTTP_SINK_OUT_SUCCESSFUL_ITEMS_CNT = "runner_out_successful_items_total"; +const std::string METRIC_RUNNER_HTTP_SINK_OUT_FAILED_ITEMS_CNT = "runner_out_failed_items_total"; +const std::string METRIC_RUNNER_HTTP_SINK_SENDING_ITEMS_CNT = "runner_sending_items_total"; +const std::string METRIC_RUNNER_HTTP_SINK_SEND_CONCURRENCY = "runner_send_concurrency"; + +// flusher runner metrics +const std::string METRIC_RUNNER_FLUSHER_IN_ITEM_RAW_SIZE_BYTES = "runner_in_item_raw_size_bytes"; +const std::string METRIC_RUNNER_FLUSHER_WAITING_ITEMS_CNT = "runner_waiting_items_total"; + +// file server metrics +const std::string METRIC_RUNNER_FILE_WATCHED_DIRS_CNT = "runner_watched_dirs_total"; +const std::string METRIC_RUNNER_FILE_ACTIVE_READERS_CNT = "runner_active_readers_total"; +const std::string METRIC_RUNNER_FILE_ENABLE_FILE_INCLUDED_BY_MULTI_CONFIGS_FLAG + = "runner_enable_file_included_by_multi_configs"; +const std::string METRIC_RUNNER_FILE_POLLING_MODIFY_CACHE_SIZE = "runner_polling_modify_cache_size"; +const std::string METRIC_RUNNER_FILE_POLLING_DIR_CACHE_SIZE = "runner_polling_dir_cache_size"; +const std::string METRIC_RUNNER_FILE_POLLING_FILE_CACHE_SIZE = "runner_polling_file_cache_size"; + } // namespace logtail diff --git a/core/monitor/MetricConstants.h b/core/monitor/MetricConstants.h index 27f3a762e3..cdf82bc734 100644 --- a/core/monitor/MetricConstants.h +++ b/core/monitor/MetricConstants.h @@ -47,21 +47,12 @@ extern const std::string METRIC_AGENT_MEMORY; extern const std::string METRIC_AGENT_MEMORY_GO; extern const std::string METRIC_AGENT_GO_ROUTINES_TOTAL; extern const std::string METRIC_AGENT_OPEN_FD_TOTAL; -extern const std::string METRIC_AGENT_POLLING_DIR_CACHE_SIZE_TOTAL; -extern const std::string METRIC_AGENT_POLLING_FILE_CACHE_SIZE_TOTAL; -extern const std::string METRIC_AGENT_POLLING_MODIFY_SIZE_TOTAL; -extern const std::string METRIC_AGENT_REGISTER_HANDLER_TOTAL; extern const std::string METRIC_AGENT_INSTANCE_CONFIG_TOTAL; extern const std::string METRIC_AGENT_PIPELINE_CONFIG_TOTAL; extern const std::string METRIC_AGENT_ENV_PIPELINE_CONFIG_TOTAL; extern const std::string METRIC_AGENT_CRD_PIPELINE_CONFIG_TOTAL; extern const std::string METRIC_AGENT_CONSOLE_PIPELINE_CONFIG_TOTAL; extern const std::string METRIC_AGENT_PLUGIN_TOTAL; -extern const std::string METRIC_AGENT_PROCESS_QUEUE_FULL_TOTAL; -extern const std::string METRIC_AGENT_PROCESS_QUEUE_TOTAL; -extern const std::string METRIC_AGENT_SEND_QUEUE_FULL_TOTAL; -extern const std::string METRIC_AGENT_SEND_QUEUE_TOTAL; -extern const std::string METRIC_AGENT_USED_SENDING_CONCURRENCY; // common plugin labels extern const std::string METRIC_LABEL_PROJECT; @@ -79,6 +70,7 @@ extern const std::string METRIC_LABEL_FILE_INODE; extern const std::string METRIC_LABEL_FILE_NAME; extern const std::string METRIC_LABEL_KEY_COMPONENT_NAME; +extern const std::string METRIC_LABEL_KEY_RUNNER_NAME; extern const std::string METRIC_LABEL_KEY_QUEUE_TYPE; extern const std::string METRIC_LABEL_KEY_EXACTLY_ONCE_FLAG; extern const std::string METRIC_LABEL_KEY_FLUSHER_NODE_ID; @@ -130,8 +122,6 @@ extern const std::string METRIC_PROC_PARSE_STDOUT_TOTAL; extern const std::string METRIC_PROC_PARSE_STDERR_TOTAL; // flusher common metrics -extern const std::string METRIC_FLUSHER_IN_RECORDS_TOTAL; -extern const std::string METRIC_FLUSHER_IN_RECORDS_SIZE_BYTES; extern const std::string METRIC_FLUSHER_ERROR_TOTAL; extern const std::string METRIC_FLUSHER_DISCARD_RECORDS_TOTAL; extern const std::string METRIC_FLUSHER_SUCCESS_RECORDS_TOTAL; @@ -144,4 +134,80 @@ extern const std::string METRIC_FLUSHER_QUOTA_ERROR_TOTAL; extern const std::string METRIC_FLUSHER_RETRIES_TOTAL; extern const std::string METRIC_FLUSHER_RETRIES_ERROR_TOTAL; +////////////////////////////////////////////////////////////////////////// +// plugin +////////////////////////////////////////////////////////////////////////// +// common metrics +extern const std::string METRIC_PLUGIN_IN_EVENTS_CNT; +extern const std::string METRIC_PLUGIN_IN_EVENT_GROUP_SIZE_BYTES; + +////////////////////////////////////////////////////////////////////////// +// component +////////////////////////////////////////////////////////////////////////// +// common metrics +extern const std::string METRIC_COMPONENT_IN_EVENTS_CNT; +extern const std::string METRIC_COMPONENT_IN_ITEMS_CNT; +extern const std::string METRIC_COMPONENT_IN_EVENT_GROUP_SIZE_BYTES; +extern const std::string METRIC_COMPONENT_IN_ITEM_SIZE_BYTES; +extern const std::string METRIC_COMPONENT_OUT_EVENTS_CNT; +extern const std::string METRIC_COMPONENT_OUT_ITEMS_CNT; +extern const std::string METRIC_COMPONENT_OUT_ITEM_SIZE_BYTES; +extern const std::string METRIC_COMPONENT_TOTAL_DELAY_MS; +extern const std::string METRIC_COMPONENT_DISCARDED_ITEMS_CNT; +extern const std::string METRIC_COMPONENT_DISCARDED_ITEMS_SIZE_BYTES; + +// batcher metrics +extern const std::string METRIC_COMPONENT_BATCHER_EVENT_BATCHES_CNT; +extern const std::string METRIC_COMPONENT_BATCHER_BUFFERED_GROUPS_CNT; +extern const std::string METRIC_COMPONENT_BATCHER_BUFFERED_EVENTS_CNT; +extern const std::string METRIC_COMPONENT_BATCHER_BUFFERED_SIZE_BYTES; + +// queue metrics +extern const std::string METRIC_COMPONENT_QUEUE_SIZE_CNT; +extern const std::string METRIC_COMPONENT_QUEUE_SIZE_BYTES; +extern const std::string METRIC_COMPONENT_QUEUE_VALID_TO_PUSH_FLAG; +extern const std::string METRIC_COMPONENT_QUEUE_EXTRA_BUFFER_SIZE; +extern const std::string METRIC_COMPONENT_QUEUE_EXTRA_BUFFER_SIZE_BYTES; +extern const std::string METRIC_COMPONENT_QUEUE_DISCARDED_EVENTS_CNT; + +////////////////////////////////////////////////////////////////////////// +// pipeline +////////////////////////////////////////////////////////////////////////// +extern const std::string METRIC_PIPELINE_START_TIME; +extern const std::string METRIC_PIPELINE_PROCESSORS_IN_EVENTS_CNT; +extern const std::string METRIC_PIPELINE_PROCESSORS_IN_EVENT_GROUPS_CNT; +extern const std::string METRIC_PIPELINE_PROCESSORS_IN_EVENT_GROUP_SIZE_BYTES; +extern const std::string METRIC_PIPELINE_PROCESSORS_TOTAL_DELAY_MS; + +////////////////////////////////////////////////////////////////////////// +// runner +////////////////////////////////////////////////////////////////////////// +// common metrics +extern const std::string METRIC_RUNNER_IN_EVENTS_CNT; +extern const std::string METRIC_RUNNER_IN_EVENT_GROUPS_CNT; +extern const std::string METRIC_RUNNER_IN_ITEMS_CNT; +extern const std::string METRIC_RUNNER_IN_EVENT_GROUP_SIZE_BYTES; +extern const std::string METRIC_RUNNER_IN_ITEM_SIZE_BYTES; +extern const std::string METRIC_RUNNER_OUT_ITEMS_CNT; +extern const std::string METRIC_RUNNER_TOTAL_DELAY_MS; +extern const std::string METRIC_RUNNER_LAST_RUN_TIME; + +// http sink metrics +extern const std::string METRIC_RUNNER_HTTP_SINK_OUT_SUCCESSFUL_ITEMS_CNT; +extern const std::string METRIC_RUNNER_HTTP_SINK_OUT_FAILED_ITEMS_CNT; +extern const std::string METRIC_RUNNER_HTTP_SINK_SENDING_ITEMS_CNT; +extern const std::string METRIC_RUNNER_HTTP_SINK_SEND_CONCURRENCY; + +// flusher runner metrics +extern const std::string METRIC_RUNNER_FLUSHER_IN_ITEM_RAW_SIZE_BYTES; +extern const std::string METRIC_RUNNER_FLUSHER_WAITING_ITEMS_CNT; + +// file server metrics +extern const std::string METRIC_RUNNER_FILE_WATCHED_DIRS_CNT; +extern const std::string METRIC_RUNNER_FILE_ACTIVE_READERS_CNT; +extern const std::string METRIC_RUNNER_FILE_ENABLE_FILE_INCLUDED_BY_MULTI_CONFIGS_FLAG; +extern const std::string METRIC_RUNNER_FILE_POLLING_MODIFY_CACHE_SIZE; +extern const std::string METRIC_RUNNER_FILE_POLLING_DIR_CACHE_SIZE; +extern const std::string METRIC_RUNNER_FILE_POLLING_FILE_CACHE_SIZE; + } // namespace logtail diff --git a/core/monitor/MetricExportor.cpp b/core/monitor/MetricExportor.cpp index f8a1b10298..e8198d8150 100644 --- a/core/monitor/MetricExportor.cpp +++ b/core/monitor/MetricExportor.cpp @@ -90,10 +90,10 @@ void MetricExportor::SendToSLS(std::map& logGr logGroup->set_source(LogFileProfiler::mIpAddr); logGroup->set_topic(METRIC_TOPIC_TYPE); if (METRIC_REGION_DEFAULT == iter->first) { - ProfileSender::GetInstance()->SendToProfileProject(ProfileSender::GetInstance()->GetDefaultProfileRegion(), + GetProfileSender()->SendToProfileProject(GetProfileSender()->GetDefaultProfileRegion(), *logGroup); } else { - ProfileSender::GetInstance()->SendToProfileProject(iter->first, *logGroup); + GetProfileSender()->SendToProfileProject(iter->first, *logGroup); } delete logGroup; } diff --git a/core/monitor/MetricExportor.h b/core/monitor/MetricExportor.h index dfb524814a..09c3908a3f 100644 --- a/core/monitor/MetricExportor.h +++ b/core/monitor/MetricExportor.h @@ -17,7 +17,7 @@ #pragma once #include "Monitor.h" -#include "profile_sender/ProfileSender.h" +#include "provider/Provider.h" namespace logtail { diff --git a/core/monitor/Monitor.cpp b/core/monitor/Monitor.cpp index 54cf59b0fb..580b6e7e1d 100644 --- a/core/monitor/Monitor.cpp +++ b/core/monitor/Monitor.cpp @@ -50,7 +50,7 @@ #include "config/provider/EnterpriseConfigProvider.h" #endif #include "pipeline/PipelineManager.h" -#include "profile_sender/ProfileSender.h" +#include "provider/Provider.h" using namespace std; using namespace sls_logs; @@ -116,8 +116,6 @@ bool LogtailMonitor::Init() { // init metrics mAgentCpuGauge = LoongCollectorMonitor::GetInstance()->GetDoubleGauge(METRIC_AGENT_CPU); mAgentMemoryGauge = LoongCollectorMonitor::GetInstance()->GetIntGauge(METRIC_AGENT_MEMORY); - mAgentUsedSendingConcurrency - = LoongCollectorMonitor::GetInstance()->GetIntGauge(METRIC_AGENT_USED_SENDING_CONCURRENCY); // Initialize monitor thread. mThreadRes = async(launch::async, &LogtailMonitor::Monitor, this); @@ -286,7 +284,6 @@ bool LogtailMonitor::SendStatusProfile(bool suicide) { AddLogContent(logPtr, "projects", FlusherSLS::GetAllProjects()); AddLogContent(logPtr, "instance_id", Application::GetInstance()->GetInstanceId()); AddLogContent(logPtr, "instance_key", id); - AddLogContent(logPtr, "syslog_open", AppConfig::GetInstance()->GetOpenStreamLog()); // Host informations. AddLogContent(logPtr, "ip", LogFileProfiler::mIpAddr); AddLogContent(logPtr, "hostname", LogFileProfiler::mHostname); @@ -299,7 +296,7 @@ bool LogtailMonitor::SendStatusProfile(bool suicide) { AddLogContent(logPtr, "plugin_stats", PipelineManager::GetInstance()->GetPluginStatistics()); // Metrics. vector allProfileRegion; - ProfileSender::GetInstance()->GetAllProfileRegion(allProfileRegion); + GetProfileSender()->GetAllProfileRegion(allProfileRegion); UpdateMetric("region", allProfileRegion); #ifdef __ENTERPRISE__ UpdateMetric("config_update_count", EnterpriseConfigProvider::GetInstance()->GetConfigUpdateTotalCount()); @@ -320,7 +317,6 @@ bool LogtailMonitor::SendStatusProfile(bool suicide) { } int32_t usedSendingConcurrency = FlusherRunner::GetInstance()->GetSendingBufferCount(); UpdateMetric("used_sending_concurrency", usedSendingConcurrency); - mAgentUsedSendingConcurrency->Set(usedSendingConcurrency); AddLogContent(logPtr, "metric_json", MetricToString()); AddLogContent(logPtr, "status", CheckLogtailStatus()); @@ -347,10 +343,10 @@ bool LogtailMonitor::SendStatusProfile(bool suicide) { } if (i == allProfileRegion.size() - 1) { - ProfileSender::GetInstance()->SendToProfileProject(allProfileRegion[i], logGroup); + GetProfileSender()->SendToProfileProject(allProfileRegion[i], logGroup); } else { LogGroup copyLogGroup = logGroup; - ProfileSender::GetInstance()->SendToProfileProject(allProfileRegion[i], copyLogGroup); + GetProfileSender()->SendToProfileProject(allProfileRegion[i], copyLogGroup); } } return true; @@ -731,14 +727,6 @@ void LoongCollectorMonitor::Init() { mIntGauges[METRIC_AGENT_MEMORY_GO] = mMetricsRecordRef.CreateIntGauge(METRIC_AGENT_MEMORY_GO); mIntGauges[METRIC_AGENT_GO_ROUTINES_TOTAL] = mMetricsRecordRef.CreateIntGauge(METRIC_AGENT_GO_ROUTINES_TOTAL); mIntGauges[METRIC_AGENT_OPEN_FD_TOTAL] = mMetricsRecordRef.CreateIntGauge(METRIC_AGENT_OPEN_FD_TOTAL); - mIntGauges[METRIC_AGENT_POLLING_DIR_CACHE_SIZE_TOTAL] - = mMetricsRecordRef.CreateIntGauge(METRIC_AGENT_POLLING_DIR_CACHE_SIZE_TOTAL); - mIntGauges[METRIC_AGENT_POLLING_FILE_CACHE_SIZE_TOTAL] - = mMetricsRecordRef.CreateIntGauge(METRIC_AGENT_POLLING_FILE_CACHE_SIZE_TOTAL); - mIntGauges[METRIC_AGENT_POLLING_MODIFY_SIZE_TOTAL] - = mMetricsRecordRef.CreateIntGauge(METRIC_AGENT_POLLING_MODIFY_SIZE_TOTAL); - mIntGauges[METRIC_AGENT_REGISTER_HANDLER_TOTAL] - = mMetricsRecordRef.CreateIntGauge(METRIC_AGENT_REGISTER_HANDLER_TOTAL); // mIntGauges[METRIC_AGENT_INSTANCE_CONFIG_TOTAL] = // mMetricsRecordRef.CreateIntGauge(METRIC_AGENT_INSTANCE_CONFIG_TOTAL); mIntGauges[METRIC_AGENT_PIPELINE_CONFIG_TOTAL] @@ -750,14 +738,6 @@ void LoongCollectorMonitor::Init() { // mIntGauges[METRIC_AGENT_CONSOLE_PIPELINE_CONFIG_TOTAL] // = mMetricsRecordRef.CreateIntGauge(METRIC_AGENT_CONSOLE_PIPELINE_CONFIG_TOTAL); // mIntGauges[METRIC_AGENT_PLUGIN_TOTAL] = mMetricsRecordRef.CreateIntGauge(METRIC_AGENT_PLUGIN_TOTAL); - mIntGauges[METRIC_AGENT_PROCESS_QUEUE_FULL_TOTAL] - = mMetricsRecordRef.CreateIntGauge(METRIC_AGENT_PROCESS_QUEUE_FULL_TOTAL); - mIntGauges[METRIC_AGENT_PROCESS_QUEUE_TOTAL] = mMetricsRecordRef.CreateIntGauge(METRIC_AGENT_PROCESS_QUEUE_TOTAL); - mIntGauges[METRIC_AGENT_SEND_QUEUE_FULL_TOTAL] - = mMetricsRecordRef.CreateIntGauge(METRIC_AGENT_SEND_QUEUE_FULL_TOTAL); - mIntGauges[METRIC_AGENT_SEND_QUEUE_TOTAL] = mMetricsRecordRef.CreateIntGauge(METRIC_AGENT_SEND_QUEUE_TOTAL); - mIntGauges[METRIC_AGENT_USED_SENDING_CONCURRENCY] - = mMetricsRecordRef.CreateIntGauge(METRIC_AGENT_USED_SENDING_CONCURRENCY); LOG_INFO(sLogger, ("LoongCollectorMonitor", "started")); } diff --git a/core/monitor/Monitor.h b/core/monitor/Monitor.h index d0cac3314e..0a4d557361 100644 --- a/core/monitor/Monitor.h +++ b/core/monitor/Monitor.h @@ -24,6 +24,7 @@ #include "LogtailMetric.h" #include "MetricConstants.h" #include "MetricStore.h" + #if defined(_MSC_VER) #include #endif @@ -166,8 +167,6 @@ class LogtailMonitor : public MetricStore { MemStat mMemStat; IntGaugePtr mAgentMemoryGauge; - IntGaugePtr mAgentUsedSendingConcurrency; - // Current scale up level, updated by CheckScaledCpuUsageUpLimit. float mScaledCpuUsageUpLimit; #if defined(__linux__) diff --git a/core/pipeline/Pipeline.cpp b/core/pipeline/Pipeline.cpp index 511b642b96..1b58a5a044 100644 --- a/core/pipeline/Pipeline.cpp +++ b/core/pipeline/Pipeline.cpp @@ -16,6 +16,7 @@ #include "pipeline/Pipeline.h" +#include #include #include @@ -314,10 +315,20 @@ bool Pipeline::Init(PipelineConfig&& config) { ProcessQueueManager::GetInstance()->SetDownStreamQueues(mContext.GetProcessQueueKey(), std::move(senderQueues)); } + WriteMetrics::GetInstance()->PrepareMetricsRecordRef( + mMetricsRecordRef, {{METRIC_LABEL_PROJECT, mContext.GetProjectName()}, {METRIC_LABEL_CONFIG_NAME, mName}}); + mStartTime = mMetricsRecordRef.CreateIntGauge(METRIC_PIPELINE_START_TIME); + mProcessorsInEventsCnt = mMetricsRecordRef.CreateCounter(METRIC_PIPELINE_PROCESSORS_IN_EVENTS_CNT); + mProcessorsInGroupsCnt = mMetricsRecordRef.CreateCounter(METRIC_PIPELINE_PROCESSORS_IN_EVENT_GROUPS_CNT); + mProcessorsInGroupDataSizeBytes + = mMetricsRecordRef.CreateCounter(METRIC_PIPELINE_PROCESSORS_IN_EVENT_GROUP_SIZE_BYTES); + mProcessorsTotalDelayMs = mMetricsRecordRef.CreateCounter(METRIC_PIPELINE_PROCESSORS_TOTAL_DELAY_MS); + return true; } void Pipeline::Start() { +#ifndef APSARA_UNIT_TEST_MAIN // TODO: 应该保证指定时间内返回，如果无法返回，将配置放入startDisabled里 for (const auto& flusher : mFlushers) { flusher->Start(); @@ -327,7 +338,7 @@ void Pipeline::Start() { // TODO: 加载该Go流水线 } - // TODO: 启用Process中改流水线对应的输入队列 + ProcessQueueManager::GetInstance()->EnablePop(mName); if (!mGoPipelineWithInput.isNull()) { // TODO: 加载该Go流水线 @@ -337,16 +348,27 @@ void Pipeline::Start() { input->Start(); } + mStartTime->Set(chrono::duration_cast(chrono::system_clock::now().time_since_epoch()).count()); +#endif LOG_INFO(sLogger, ("pipeline start", "succeeded")("config", mName)); } void Pipeline::Process(vector& logGroupList, size_t inputIndex) { + for (const auto& logGroup : logGroupList) { + mProcessorsInEventsCnt->Add(logGroup.GetEvents().size()); + mProcessorsInGroupDataSizeBytes->Add(logGroup.DataSize()); + } + mProcessorsInGroupsCnt->Add(logGroupList.size()); + + auto before = chrono::system_clock::now(); for (auto& p : mInputs[inputIndex]->GetInnerProcessors()) { p->Process(logGroupList); } for (auto& p : mProcessorLine) { p->Process(logGroupList); } + mProcessorsTotalDelayMs->Add( + chrono::duration_cast(chrono::system_clock::now() - before).count()); } bool Pipeline::Send(vector&& groupList) { @@ -381,6 +403,7 @@ bool Pipeline::FlushBatch() { } void Pipeline::Stop(bool isRemoving) { +#ifndef APSARA_UNIT_TEST_MAIN // TODO: 应该保证指定时间内返回，如果无法返回，将配置放入stopDisabled里 for (const auto& input : mInputs) { input->Stop(isRemoving); @@ -403,7 +426,7 @@ void Pipeline::Stop(bool isRemoving) { for (const auto& flusher : mFlushers) { flusher->Stop(isRemoving); } - +#endif LOG_INFO(sLogger, ("pipeline stop", "succeeded")("config", mName)); } diff --git a/core/pipeline/Pipeline.h b/core/pipeline/Pipeline.h index 90bc5ada01..e3775f7b48 100644 --- a/core/pipeline/Pipeline.h +++ b/core/pipeline/Pipeline.h @@ -24,14 +24,15 @@ #include #include "config/PipelineConfig.h" -#include "plugin/input/InputContainerStdio.h" -#include "plugin/input/InputFile.h" #include "models/PipelineEventGroup.h" +#include "monitor/LogtailMetric.h" #include "pipeline/PipelineContext.h" #include "pipeline/plugin/instance/FlusherInstance.h" #include "pipeline/plugin/instance/InputInstance.h" #include "pipeline/plugin/instance/ProcessorInstance.h" #include "pipeline/route/Router.h" +#include "plugin/input/InputContainerStdio.h" +#include "plugin/input/InputFile.h" namespace logtail { @@ -84,8 +85,15 @@ class Pipeline { std::unique_ptr mConfig; std::atomic_uint16_t mPluginID; + mutable MetricsRecordRef mMetricsRecordRef; + IntGaugePtr mStartTime; + CounterPtr mProcessorsInEventsCnt; + CounterPtr mProcessorsInGroupsCnt; + CounterPtr mProcessorsInGroupDataSizeBytes; + CounterPtr mProcessorsTotalDelayMs; + #ifdef APSARA_UNIT_TEST_MAIN - friend class PipelineMock; + friend class PipelineMock; friend class PipelineUnittest; friend class InputContainerStdioUnittest; friend class InputFileUnittest; diff --git a/core/pipeline/PipelineManager.cpp b/core/pipeline/PipelineManager.cpp index 9a0a69f60c..f5cac36598 100644 --- a/core/pipeline/PipelineManager.cpp +++ b/core/pipeline/PipelineManager.cpp @@ -28,7 +28,6 @@ #if defined(__ENTERPRISE__) && defined(__linux__) && !defined(__ANDROID__) #include "app_config/AppConfig.h" #include "shennong/ShennongManager.h" -#include "streamlog/StreamLogManager.h" #endif #include "config/feedbacker/ConfigFeedbackReceiver.h" #include "pipeline/queue/ProcessQueueManager.h" @@ -42,40 +41,26 @@ void logtail::PipelineManager::UpdatePipelines(PipelineConfigDiff& diff) { #ifndef APSARA_UNIT_TEST_MAIN // 过渡使用 static bool isFileServerStarted = false, isInputObserverStarted = false; -#if defined(__ENTERPRISE__) && defined(__linux__) && !defined(__ANDROID__) - static bool isInputStreamStarted = false; -#endif - bool isInputObserverChanged = false, isInputFileChanged = false, isInputStreamChanged = false, - isInputContainerStdioChanged = false; + bool isInputObserverChanged = false, isInputFileChanged = false, isInputContainerStdioChanged = false; for (const auto& name : diff.mRemoved) { CheckIfInputUpdated(mPipelineNameEntityMap[name]->GetConfig()["inputs"][0], isInputObserverChanged, isInputFileChanged, - isInputStreamChanged, isInputContainerStdioChanged); } for (const auto& config : diff.mModified) { - CheckIfInputUpdated(*config.mInputs[0], - isInputObserverChanged, - isInputFileChanged, - isInputStreamChanged, - isInputContainerStdioChanged); + CheckIfInputUpdated( + *config.mInputs[0], isInputObserverChanged, isInputFileChanged, isInputContainerStdioChanged); } for (const auto& config : diff.mAdded) { - CheckIfInputUpdated(*config.mInputs[0], - isInputObserverChanged, - isInputFileChanged, - isInputStreamChanged, - isInputContainerStdioChanged); + CheckIfInputUpdated( + *config.mInputs[0], isInputObserverChanged, isInputFileChanged, isInputContainerStdioChanged); } #if defined(__ENTERPRISE__) && defined(__linux__) && !defined(__ANDROID__) if (AppConfig::GetInstance()->ShennongSocketEnabled()) { ShennongManager::GetInstance()->Pause(); } - if (isInputStreamStarted && isInputStreamChanged) { - StreamLogManager::GetInstance()->ShutdownConfigUsage(); - } #endif #if defined(__linux__) && !defined(__ANDROID__) if (isInputObserverStarted && isInputObserverChanged) { @@ -182,16 +167,6 @@ void logtail::PipelineManager::UpdatePipelines(PipelineConfigDiff& diff) { } #endif #if defined(__ENTERPRISE__) && defined(__linux__) && !defined(__ANDROID__) - if (isInputStreamChanged) { - if (isInputStreamStarted) { - StreamLogManager::GetInstance()->StartupConfigUsage(); - } else { - if (AppConfig::GetInstance()->GetOpenStreamLog()) { - StreamLogManager::GetInstance()->Init(); - isInputStreamStarted = true; - } - } - } if (AppConfig::GetInstance()->ShennongSocketEnabled()) { ShennongManager::GetInstance()->Resume(); } @@ -228,11 +203,6 @@ string PipelineManager::GetPluginStatistics() const { void PipelineManager::StopAllPipelines() { LOG_INFO(sLogger, ("stop all pipelines", "starts")); -#if defined(__ENTERPRISE__) && defined(__linux__) && !defined(__ANDROID__) - if (AppConfig::GetInstance()->GetOpenStreamLog()) { - StreamLogManager::GetInstance()->Shutdown(); - } -#endif PrometheusInputRunner::GetInstance()->Stop(); #if defined(__linux__) && !defined(__ANDROID__) ObserverManager::GetInstance()->HoldOn(true); @@ -296,15 +266,12 @@ void PipelineManager::DecreasePluginUsageCnt(const unordered_map> mPipelineNameEntityMap; @@ -71,6 +70,10 @@ class PipelineManager { #ifdef APSARA_UNIT_TEST_MAIN friend class PipelineManagerMock; friend class PipelineManagerUnittest; + friend class ProcessQueueManagerUnittest; + friend class ExactlyOnceQueueManagerUnittest; + friend class BoundedProcessQueueUnittest; + friend class CircularProcessQueueUnittest; friend class CommonConfigProviderUnittest; #endif }; diff --git a/core/pipeline/batch/BatchItem.h b/core/pipeline/batch/BatchItem.h index b27fa0f5e8..9fa956dbef 100644 --- a/core/pipeline/batch/BatchItem.h +++ b/core/pipeline/batch/BatchItem.h @@ -20,11 +20,11 @@ #include #include +#include "models/PipelineEventGroup.h" +#include "models/StringView.h" #include "pipeline/batch/BatchStatus.h" #include "pipeline/batch/BatchedEvents.h" #include "pipeline/batch/FlushStrategy.h" -#include "models/PipelineEventGroup.h" -#include "models/StringView.h" namespace logtail { @@ -32,7 +32,9 @@ class GroupBatchItem { public: GroupBatchItem() { mStatus.Reset(); } - void Add(BatchedEvents&& g) { + void Add(BatchedEvents&& g, int64_t totalEnqueTimeMs) { + mEventsCnt += g.mEvents.size(); + mTotalEnqueTimeMs += totalEnqueTimeMs; mGroups.emplace_back(std::move(g)); mStatus.Update(mGroups.back()); } @@ -56,6 +58,10 @@ class GroupBatchItem { } GroupBatchStatus& GetStatus() { return mStatus; } + size_t GroupSize() const { return mGroups.size(); } + size_t EventSize() const { return mEventsCnt; } + size_t DataSize() const { return mStatus.GetSize(); } + int64_t TotalEnqueTimeMs() const { return mTotalEnqueTimeMs; } bool IsEmpty() { return mGroups.empty(); } @@ -63,11 +69,17 @@ class GroupBatchItem { void Clear() { mGroups.clear(); mStatus.Reset(); + mEventsCnt = 0; + mTotalEnqueTimeMs = 0; } std::vector mGroups; GroupBatchStatus mStatus; + size_t mEventsCnt = 0; + // if more than 10^6 events are contained in the batch, the value may overflow + // however, this is almost impossible in practice + int64_t mTotalEnqueTimeMs = 0; #ifdef APSARA_UNIT_TEST_MAIN friend class EventBatchItemUnittest; @@ -82,6 +94,9 @@ class EventBatchItem { void Add(PipelineEventPtr&& e) { mBatch.mEvents.emplace_back(std::move(e)); mStatus.Update(mBatch.mEvents.back()); + mTotalEnqueTimeMs += std::chrono::time_point_cast(std::chrono::system_clock::now()) + .time_since_epoch() + .count(); } void Flush(GroupBatchItem& res) { @@ -91,7 +106,8 @@ class EventBatchItem { if (mBatch.mExactlyOnceCheckpoint) { UpdateExactlyOnceLogPosition(); } - res.Add(std::move(mBatch)); + mBatch.mSizeBytes = DataSize(); + res.Add(std::move(mBatch), mTotalEnqueTimeMs); Clear(); } @@ -102,6 +118,7 @@ class EventBatchItem { if (mBatch.mExactlyOnceCheckpoint) { UpdateExactlyOnceLogPosition(); } + mBatch.mSizeBytes = DataSize(); res.emplace_back(std::move(mBatch)); Clear(); } @@ -114,6 +131,7 @@ class EventBatchItem { if (mBatch.mExactlyOnceCheckpoint) { UpdateExactlyOnceLogPosition(); } + mBatch.mSizeBytes = DataSize(); res.back().emplace_back(std::move(mBatch)); Clear(); } @@ -141,11 +159,16 @@ class EventBatchItem { bool IsEmpty() { return mBatch.mEvents.empty(); } + size_t DataSize() const { return sizeof(decltype(mBatch.mEvents)) + mStatus.GetSize() + mBatch.mTags.DataSize(); } + size_t EventSize() const { return mBatch.mEvents.size(); } + int64_t TotalEnqueTimeMs() const { return mTotalEnqueTimeMs; } + private: void Clear() { mBatch.Clear(); mSourceBuffers.clear(); mStatus.Reset(); + mTotalEnqueTimeMs = 0; } void UpdateExactlyOnceLogPosition() { @@ -159,6 +182,9 @@ class EventBatchItem { BatchedEvents mBatch; std::unordered_set mSourceBuffers; T mStatus; + // if more than 10^6 events are contained in the batch, the value may overflow + // however, this is almost impossible in practice + int64_t mTotalEnqueTimeMs = 0; #ifdef APSARA_UNIT_TEST_MAIN friend class EventBatchItemUnittest; diff --git a/core/pipeline/batch/BatchStatus.h b/core/pipeline/batch/BatchStatus.h index 18c2b7db6c..224a83dc78 100644 --- a/core/pipeline/batch/BatchStatus.h +++ b/core/pipeline/batch/BatchStatus.h @@ -67,7 +67,7 @@ class GroupBatchStatus { if (mCreateTime == 0) { mCreateTime = time(nullptr); } - mSizeBytes += g.DataSize(); + mSizeBytes += g.mSizeBytes; } uint32_t GetSize() const { return mSizeBytes; } diff --git a/core/pipeline/batch/BatchedEvents.h b/core/pipeline/batch/BatchedEvents.h index 9ec5d0ff37..e16d15be15 100644 --- a/core/pipeline/batch/BatchedEvents.h +++ b/core/pipeline/batch/BatchedEvents.h @@ -28,6 +28,7 @@ struct BatchedEvents { EventsContainer mEvents; SizedMap mTags; std::vector> mSourceBuffers; + size_t mSizeBytes = 0; // only set on completion // for flusher_sls only RangeCheckpointPtr mExactlyOnceCheckpoint; StringView mPackIdPrefix; @@ -45,20 +46,17 @@ struct BatchedEvents { mExactlyOnceCheckpoint(std::move(eoo)), mPackIdPrefix(packIdPrefix) { mSourceBuffers.emplace_back(std::move(sourceBuffer)); - } - - size_t DataSize() const { - size_t eventsSize = sizeof(decltype(mEvents)); + mSizeBytes = sizeof(decltype(mEvents)) + mTags.DataSize(); for (const auto& item : mEvents) { - eventsSize += item->DataSize(); + mSizeBytes += item->DataSize(); } - return eventsSize + mTags.DataSize(); } void Clear() { mEvents.clear(); mTags.Clear(); mSourceBuffers.clear(); + mSizeBytes = 0; mExactlyOnceCheckpoint.reset(); mPackIdPrefix = StringView(); } diff --git a/core/pipeline/batch/Batcher.h b/core/pipeline/batch/Batcher.h index b165bab370..0c09990649 100644 --- a/core/pipeline/batch/Batcher.h +++ b/core/pipeline/batch/Batcher.h @@ -24,14 +24,16 @@ #include #include -#include "pipeline/batch/BatchItem.h" -#include "pipeline/batch/BatchStatus.h" -#include "pipeline/batch/FlushStrategy.h" -#include "pipeline/batch/TimeoutFlushManager.h" #include "common/Flags.h" #include "common/ParamExtractor.h" #include "models/PipelineEventGroup.h" +#include "monitor/LogtailMetric.h" +#include "monitor/MetricConstants.h" #include "pipeline/PipelineContext.h" +#include "pipeline/batch/BatchItem.h" +#include "pipeline/batch/BatchStatus.h" +#include "pipeline/batch/FlushStrategy.h" +#include "pipeline/batch/TimeoutFlushManager.h" namespace logtail { @@ -97,6 +99,26 @@ class Batcher { mFlusher = flusher; + std::vector> labels{ + {METRIC_LABEL_PROJECT, ctx.GetProjectName()}, + {METRIC_LABEL_CONFIG_NAME, ctx.GetConfigName()}, + {METRIC_LABEL_KEY_COMPONENT_NAME, "batcher"}, + {METRIC_LABEL_KEY_FLUSHER_NODE_ID, flusher->GetNodeID()}}; + if (enableGroupBatch) { + labels.emplace_back("enable_group_batch", "true"); + } else { + labels.emplace_back("enable_group_batch", "false"); + } + WriteMetrics::GetInstance()->PrepareMetricsRecordRef(mMetricsRecordRef, std::move(labels)); + mInEventsCnt = mMetricsRecordRef.CreateCounter(METRIC_COMPONENT_IN_EVENTS_CNT); + mInGroupDataSizeBytes = mMetricsRecordRef.CreateCounter(METRIC_COMPONENT_IN_EVENT_GROUP_SIZE_BYTES); + mOutEventsCnt = mMetricsRecordRef.CreateCounter(METRIC_COMPONENT_OUT_EVENTS_CNT); + mTotalDelayMs = mMetricsRecordRef.CreateCounter(METRIC_COMPONENT_TOTAL_DELAY_MS); + mEventBatchItemsCnt = mMetricsRecordRef.CreateIntGauge(METRIC_COMPONENT_BATCHER_EVENT_BATCHES_CNT); + mBufferedGroupsCnt = mMetricsRecordRef.CreateIntGauge(METRIC_COMPONENT_BATCHER_BUFFERED_GROUPS_CNT); + mBufferedEventsCnt = mMetricsRecordRef.CreateIntGauge(METRIC_COMPONENT_BATCHER_BUFFERED_EVENTS_CNT); + mBufferedDataSizeByte = mMetricsRecordRef.CreateIntGauge(METRIC_COMPONENT_BATCHER_BUFFERED_SIZE_BYTES); + return true; } @@ -105,15 +127,20 @@ class Batcher { std::lock_guard lock(mMux); size_t key = g.GetTagsHash(); EventBatchItem& item = mEventQueueMap[key]; + mInEventsCnt->Add(g.GetEvents().size()); + mInGroupDataSizeBytes->Add(g.DataSize()); + mEventBatchItemsCnt->Set(mEventQueueMap.size()); size_t eventsSize = g.GetEvents().size(); for (size_t i = 0; i < eventsSize; ++i) { PipelineEventPtr& e = g.MutableEvents()[i]; if (!item.IsEmpty() && mEventFlushStrategy.NeedFlushByTime(item.GetStatus(), e)) { if (!mGroupQueue) { + UpdateMetricsOnFlushingEventQueue(item); item.Flush(res); } else { if (!mGroupQueue->IsEmpty() && mGroupFlushStrategy->NeedFlushByTime(mGroupQueue->GetStatus())) { + UpdateMetricsOnFlushingGroupQueue(); mGroupQueue->Flush(res); } if (mGroupQueue->IsEmpty()) { @@ -125,6 +152,7 @@ class Batcher { } item.Flush(mGroupQueue.value()); if (mGroupFlushStrategy->NeedFlushBySize(mGroupQueue->GetStatus())) { + UpdateMetricsOnFlushingGroupQueue(); mGroupQueue->Flush(res); } } @@ -136,12 +164,17 @@ class Batcher { g.GetMetadata(EventGroupMetaKey::SOURCE_ID)); TimeoutFlushManager::GetInstance()->UpdateRecord( mFlusher->GetContext().GetConfigName(), 0, key, mEventFlushStrategy.GetTimeoutSecs(), mFlusher); + mBufferedGroupsCnt->Add(1); + mBufferedDataSizeByte->Add(item.DataSize()); } else if (i == 0) { item.AddSourceBuffer(g.GetSourceBuffer()); } + mBufferedEventsCnt->Add(1); + mBufferedDataSizeByte->Add(e->DataSize()); item.Add(std::move(e)); if (mEventFlushStrategy.NeedFlushBySize(item.GetStatus()) || mEventFlushStrategy.NeedFlushByCnt(item.GetStatus())) { + UpdateMetricsOnFlushingEventQueue(item); item.Flush(res); } } @@ -155,6 +188,7 @@ class Batcher { if (!mGroupQueue) { return; } + UpdateMetricsOnFlushingGroupQueue(); return mGroupQueue->Flush(res); } @@ -164,12 +198,15 @@ class Batcher { } if (!mGroupQueue) { + UpdateMetricsOnFlushingEventQueue(iter->second); iter->second.Flush(res); mEventQueueMap.erase(iter); + mEventBatchItemsCnt->Set(mEventQueueMap.size()); return; } if (!mGroupQueue->IsEmpty() && mGroupFlushStrategy->NeedFlushByTime(mGroupQueue->GetStatus())) { + UpdateMetricsOnFlushingGroupQueue(); mGroupQueue->Flush(res); } if (mGroupQueue->IsEmpty()) { @@ -178,7 +215,9 @@ class Batcher { } iter->second.Flush(mGroupQueue.value()); mEventQueueMap.erase(iter); + mEventBatchItemsCnt->Set(mEventQueueMap.size()); if (mGroupFlushStrategy->NeedFlushBySize(mGroupQueue->GetStatus())) { + UpdateMetricsOnFlushingGroupQueue(); mGroupQueue->Flush(res); } } @@ -187,20 +226,25 @@ class Batcher { std::lock_guard lock(mMux); for (auto& item : mEventQueueMap) { if (!mGroupQueue) { + UpdateMetricsOnFlushingEventQueue(item.second); item.second.Flush(res); } else { if (!mGroupQueue->IsEmpty() && mGroupFlushStrategy->NeedFlushByTime(mGroupQueue->GetStatus())) { + UpdateMetricsOnFlushingGroupQueue(); mGroupQueue->Flush(res); } item.second.Flush(mGroupQueue.value()); if (mGroupFlushStrategy->NeedFlushBySize(mGroupQueue->GetStatus())) { + UpdateMetricsOnFlushingGroupQueue(); mGroupQueue->Flush(res); } } } if (mGroupQueue) { + UpdateMetricsOnFlushingGroupQueue(); mGroupQueue->Flush(res); } + mEventBatchItemsCnt->Set(0); mEventQueueMap.clear(); } @@ -210,6 +254,32 @@ class Batcher { #endif private: + void UpdateMetricsOnFlushingEventQueue(const EventBatchItem& item) { + mOutEventsCnt->Add(item.EventSize()); + mTotalDelayMs->Add( + item.EventSize() + * std::chrono::time_point_cast(std::chrono::system_clock::now()) + .time_since_epoch() + .count() + - item.TotalEnqueTimeMs()); + mBufferedGroupsCnt->Sub(1); + mBufferedEventsCnt->Sub(item.EventSize()); + mBufferedDataSizeByte->Sub(item.DataSize()); + } + + void UpdateMetricsOnFlushingGroupQueue() { + mOutEventsCnt->Add(mGroupQueue->EventSize()); + mTotalDelayMs->Add( + mGroupQueue->EventSize() + * std::chrono::time_point_cast(std::chrono::system_clock::now()) + .time_since_epoch() + .count() + - mGroupQueue->TotalEnqueTimeMs()); + mBufferedGroupsCnt->Sub(mGroupQueue->GroupSize()); + mBufferedEventsCnt->Sub(mGroupQueue->EventSize()); + mBufferedDataSizeByte->Sub(mGroupQueue->DataSize()); + } + std::mutex mMux; std::map> mEventQueueMap; EventFlushStrategy mEventFlushStrategy; @@ -219,6 +289,16 @@ class Batcher { Flusher* mFlusher = nullptr; + mutable MetricsRecordRef mMetricsRecordRef; + CounterPtr mInEventsCnt; + CounterPtr mInGroupDataSizeBytes; + CounterPtr mOutEventsCnt; + CounterPtr mTotalDelayMs; + IntGaugePtr mEventBatchItemsCnt; + IntGaugePtr mBufferedGroupsCnt; + IntGaugePtr mBufferedEventsCnt; + IntGaugePtr mBufferedDataSizeByte; + #ifdef APSARA_UNIT_TEST_MAIN friend class BatcherUnittest; #endif diff --git a/core/pipeline/plugin/PluginRegistry.cpp b/core/pipeline/plugin/PluginRegistry.cpp index 5beeb2ab0e..ba04e1eedd 100644 --- a/core/pipeline/plugin/PluginRegistry.cpp +++ b/core/pipeline/plugin/PluginRegistry.cpp @@ -36,9 +36,6 @@ #include "plugin/input/InputNetworkSecurity.h" #include "plugin/input/InputProcessSecurity.h" #include "plugin/input/InputObserverNetwork.h" -#ifdef __ENTERPRISE__ -#include "plugin/input/InputStream.h" -#endif #endif #include "logger/Logger.h" #include "pipeline/plugin/creator/CProcessor.h" @@ -136,9 +133,6 @@ void PluginRegistry::LoadStaticPlugins() { RegisterInputCreator(new StaticInputCreator()); RegisterInputCreator(new StaticInputCreator()); RegisterInputCreator(new StaticInputCreator()); -#ifdef __ENTERPRISE__ - RegisterInputCreator(new StaticInputCreator()); -#endif #endif RegisterProcessorCreator(new StaticProcessorCreator()); diff --git a/core/pipeline/plugin/instance/FlusherInstance.cpp b/core/pipeline/plugin/instance/FlusherInstance.cpp index 9623a96b75..003b279022 100644 --- a/core/pipeline/plugin/instance/FlusherInstance.cpp +++ b/core/pipeline/plugin/instance/FlusherInstance.cpp @@ -25,14 +25,14 @@ bool FlusherInstance::Init(const Json::Value& config, PipelineContext& context, return false; } - mFlusherInRecordsTotal = mPlugin->GetMetricsRecordRef().CreateCounter(METRIC_FLUSHER_IN_RECORDS_TOTAL); - mFlusherInRecordsSizeBytes = mPlugin->GetMetricsRecordRef().CreateCounter(METRIC_FLUSHER_IN_RECORDS_SIZE_BYTES); + mInEventsCnt = mPlugin->GetMetricsRecordRef().CreateCounter(METRIC_PLUGIN_IN_EVENTS_CNT); + mInGroupDataSizeBytes = mPlugin->GetMetricsRecordRef().CreateCounter(METRIC_PLUGIN_IN_EVENT_GROUP_SIZE_BYTES); return true; } bool FlusherInstance::Send(PipelineEventGroup&& g) { - mFlusherInRecordsTotal->Add(g.GetEvents().size()); - mFlusherInRecordsSizeBytes->Add(g.DataSize()); + mInEventsCnt->Add(g.GetEvents().size()); + mInGroupDataSizeBytes->Add(g.DataSize()); return mPlugin->Send(std::move(g)); } diff --git a/core/pipeline/plugin/instance/FlusherInstance.h b/core/pipeline/plugin/instance/FlusherInstance.h index 2900a594df..4abec0d766 100644 --- a/core/pipeline/plugin/instance/FlusherInstance.h +++ b/core/pipeline/plugin/instance/FlusherInstance.h @@ -46,8 +46,8 @@ class FlusherInstance : public PluginInstance { private: std::unique_ptr mPlugin; - CounterPtr mFlusherInRecordsTotal; - CounterPtr mFlusherInRecordsSizeBytes; + CounterPtr mInEventsCnt; + CounterPtr mInGroupDataSizeBytes; }; } // namespace logtail diff --git a/core/pipeline/plugin/instance/ProcessorInstance.cpp b/core/pipeline/plugin/instance/ProcessorInstance.cpp index 8790886cd4..70dbc5a172 100644 --- a/core/pipeline/plugin/instance/ProcessorInstance.cpp +++ b/core/pipeline/plugin/instance/ProcessorInstance.cpp @@ -49,11 +49,9 @@ void ProcessorInstance::Process(vector& logGroupList) { mProcInRecordsTotal->Add(logGroup.GetEvents().size()); } - uint64_t startTime = GetCurrentTimeInMilliSeconds(); + auto before = chrono::system_clock::now(); mPlugin->Process(logGroupList); - uint64_t durationTime = GetCurrentTimeInMilliSeconds() - startTime; - - mProcTimeMS->Add(durationTime); + mProcTimeMS->Add(chrono::duration_cast(chrono::system_clock::now() - before).count()); for (const auto& logGroup : logGroupList) { mProcOutRecordsTotal->Add(logGroup.GetEvents().size()); diff --git a/core/pipeline/plugin/interface/Plugin.h b/core/pipeline/plugin/interface/Plugin.h index e63d098ae3..1ee9464022 100644 --- a/core/pipeline/plugin/interface/Plugin.h +++ b/core/pipeline/plugin/interface/Plugin.h @@ -21,6 +21,7 @@ #include #include "monitor/LogtailMetric.h" +#include "monitor/MetricConstants.h" #include "pipeline/PipelineContext.h" namespace logtail { @@ -39,18 +40,13 @@ class Plugin { const std::string& id, const std::string& nodeID, const std::string& childNodeID) { - std::vector> labels; - WriteMetrics::GetInstance()->PreparePluginCommonLabels(mContext->GetProjectName(), - mContext->GetLogstoreName(), - mContext->GetRegion(), - mContext->GetConfigName(), - name, - id, - nodeID, - childNodeID, - labels); - - WriteMetrics::GetInstance()->PrepareMetricsRecordRef(mMetricsRecordRef, std::move(labels)); + WriteMetrics::GetInstance()->PrepareMetricsRecordRef(mMetricsRecordRef, + {{METRIC_LABEL_PROJECT, mContext->GetProjectName()}, + {METRIC_LABEL_CONFIG_NAME, mContext->GetConfigName()}, + {METRIC_LABEL_PLUGIN_NAME, name}, + {METRIC_LABEL_PLUGIN_ID, id}, + {METRIC_LABEL_NODE_ID, nodeID}, + {METRIC_LABEL_CHILD_NODE_ID, childNodeID}}); } protected: diff --git a/core/pipeline/queue/BoundedProcessQueue.cpp b/core/pipeline/queue/BoundedProcessQueue.cpp index ced855e43e..b577cf79fc 100644 --- a/core/pipeline/queue/BoundedProcessQueue.cpp +++ b/core/pipeline/queue/BoundedProcessQueue.cpp @@ -14,6 +14,8 @@ #include "pipeline/queue/BoundedProcessQueue.h" +#include "pipeline/PipelineManager.h" + using namespace std; namespace logtail { @@ -35,7 +37,7 @@ bool BoundedProcessQueue::Push(unique_ptr&& item) { } item->mEnqueTime = chrono::system_clock::now(); auto size = item->mEventGroup.DataSize(); - mQueue.push(std::move(item)); + mQueue.push_back(std::move(item)); ChangeStateIfNeededAfterPush(); mInItemsCnt->Add(1); @@ -51,7 +53,7 @@ bool BoundedProcessQueue::Pop(unique_ptr& item) { return false; } item = std::move(mQueue.front()); - mQueue.pop(); + mQueue.pop_front(); if (ChangeStateIfNeededAfterPop()) { GiveFeedback(); } @@ -65,6 +67,15 @@ bool BoundedProcessQueue::Pop(unique_ptr& item) { return true; } +void BoundedProcessQueue::SetPipelineForItems(const std::string& name) const { + auto p = PipelineManager::GetInstance()->FindConfigByName(name); + for (auto& item : mQueue) { + if (!item->mPipeline) { + item->mPipeline = p; + } + } +} + void BoundedProcessQueue::SetUpStreamFeedbacks(vector&& feedbacks) { mUpStreamFeedbacks.clear(); for (auto& item : feedbacks) { diff --git a/core/pipeline/queue/BoundedProcessQueue.h b/core/pipeline/queue/BoundedProcessQueue.h index 78c7772fd0..1a4c793e9f 100644 --- a/core/pipeline/queue/BoundedProcessQueue.h +++ b/core/pipeline/queue/BoundedProcessQueue.h @@ -35,6 +35,7 @@ class BoundedProcessQueue : public BoundedQueueInterface&& item) override; bool Pop(std::unique_ptr& item) override; + void SetPipelineForItems(const std::string& name) const override; void SetUpStreamFeedbacks(std::vector&& feedbacks); @@ -43,7 +44,7 @@ class BoundedProcessQueue : public BoundedQueueInterface> mQueue; + std::deque> mQueue; std::vector mUpStreamFeedbacks; #ifdef APSARA_UNIT_TEST_MAIN diff --git a/core/pipeline/queue/BoundedQueueInterface.h b/core/pipeline/queue/BoundedQueueInterface.h index 350e22be37..c8102a5fbd 100644 --- a/core/pipeline/queue/BoundedQueueInterface.h +++ b/core/pipeline/queue/BoundedQueueInterface.h @@ -26,7 +26,7 @@ class BoundedQueueInterface : virtual public QueueInterface { BoundedQueueInterface(QueueKey key, size_t cap, size_t low, size_t high, const PipelineContext& ctx) : QueueInterface(key, cap, ctx), mLowWatermark(low), mHighWatermark(high) { this->mMetricsRecordRef.AddLabels({{METRIC_LABEL_KEY_QUEUE_TYPE, "bounded"}}); - mValidToPushFlag = this->mMetricsRecordRef.CreateIntGauge("valid_to_push"); + mValidToPushFlag = this->mMetricsRecordRef.CreateIntGauge(METRIC_COMPONENT_QUEUE_VALID_TO_PUSH_FLAG); } virtual ~BoundedQueueInterface() = default; diff --git a/core/pipeline/queue/BoundedSenderQueueInterface.cpp b/core/pipeline/queue/BoundedSenderQueueInterface.cpp index 03e26ebac3..3e58d48e49 100644 --- a/core/pipeline/queue/BoundedSenderQueueInterface.cpp +++ b/core/pipeline/queue/BoundedSenderQueueInterface.cpp @@ -26,8 +26,8 @@ BoundedSenderQueueInterface::BoundedSenderQueueInterface( : QueueInterface(key, cap, ctx), BoundedQueueInterface>(key, cap, low, high, ctx) { mMetricsRecordRef.AddLabels({{METRIC_LABEL_KEY_COMPONENT_NAME, "sender_queue"}}); mMetricsRecordRef.AddLabels({{METRIC_LABEL_KEY_FLUSHER_NODE_ID, flusherId}}); - mExtraBufferSize = mMetricsRecordRef.CreateIntGauge("extra_buffer_size"); - mExtraBufferDataSizeBytes = mMetricsRecordRef.CreateIntGauge("extra_buffer_data_size_bytes"); + mExtraBufferSize = mMetricsRecordRef.CreateIntGauge(METRIC_COMPONENT_QUEUE_EXTRA_BUFFER_SIZE); + mExtraBufferDataSizeBytes = mMetricsRecordRef.CreateIntGauge(METRIC_COMPONENT_QUEUE_EXTRA_BUFFER_SIZE_BYTES); } void BoundedSenderQueueInterface::SetFeedback(FeedbackInterface* feedback) { diff --git a/core/pipeline/queue/CircularProcessQueue.cpp b/core/pipeline/queue/CircularProcessQueue.cpp index c6df833e7f..d9b7b551c8 100644 --- a/core/pipeline/queue/CircularProcessQueue.cpp +++ b/core/pipeline/queue/CircularProcessQueue.cpp @@ -15,6 +15,7 @@ #include "pipeline/queue/CircularProcessQueue.h" #include "logger/Logger.h" +#include "pipeline/PipelineManager.h" #include "pipeline/queue/QueueKeyManager.h" using namespace std; @@ -24,7 +25,7 @@ namespace logtail { CircularProcessQueue::CircularProcessQueue(size_t cap, int64_t key, uint32_t priority, const PipelineContext& ctx) : QueueInterface>(key, cap, ctx), ProcessQueueInterface(key, cap, priority, ctx) { mMetricsRecordRef.AddLabels({{METRIC_LABEL_KEY_QUEUE_TYPE, "circular"}}); - mDroppedEventsCnt = mMetricsRecordRef.CreateCounter("dropped_events_cnt"); + mDiscardedEventsCnt = mMetricsRecordRef.CreateCounter(METRIC_COMPONENT_QUEUE_DISCARDED_EVENTS_CNT); WriteMetrics::GetInstance()->CommitMetricsRecordRef(mMetricsRecordRef); } @@ -37,7 +38,7 @@ bool CircularProcessQueue::Push(unique_ptr&& item) { mQueue.pop_front(); mQueueSize->Set(Size()); mQueueDataSizeByte->Sub(size); - mDroppedEventsCnt->Add(cnt); + mDiscardedEventsCnt->Add(cnt); } if (mEventCnt + newCnt > mCapacity) { return false; @@ -71,6 +72,15 @@ bool CircularProcessQueue::Pop(unique_ptr& item) { return true; } +void CircularProcessQueue::SetPipelineForItems(const std::string& name) const { + auto p = PipelineManager::GetInstance()->FindConfigByName(name); + for (auto& item : mQueue) { + if (!item->mPipeline) { + item->mPipeline = p; + } + } +} + void CircularProcessQueue::Reset(size_t cap) { // it seems more reasonable to retain extra items and process them immediately, however this contray to current // framework design so we simply discard extra items, considering that it is a rare case to change capacity diff --git a/core/pipeline/queue/CircularProcessQueue.h b/core/pipeline/queue/CircularProcessQueue.h index 813929c799..8a7c8c1ee9 100644 --- a/core/pipeline/queue/CircularProcessQueue.h +++ b/core/pipeline/queue/CircularProcessQueue.h @@ -33,6 +33,7 @@ class CircularProcessQueue : virtual public QueueInterface&& item) override; bool Pop(std::unique_ptr& item) override; + void SetPipelineForItems(const std::string& name) const override; void Reset(size_t cap); @@ -42,7 +43,7 @@ class CircularProcessQueue : virtual public QueueInterface> mQueue; size_t mEventCnt = 0; - CounterPtr mDroppedEventsCnt; + CounterPtr mDiscardedEventsCnt; #ifdef APSARA_UNIT_TEST_MAIN friend class CircularProcessQueueUnittest; diff --git a/core/pipeline/queue/ExactlyOnceQueueManager.cpp b/core/pipeline/queue/ExactlyOnceQueueManager.cpp index 3c66b636e3..b78aba82d8 100644 --- a/core/pipeline/queue/ExactlyOnceQueueManager.cpp +++ b/core/pipeline/queue/ExactlyOnceQueueManager.cpp @@ -148,20 +148,23 @@ bool ExactlyOnceQueueManager::IsAllProcessQueueEmpty() const { return true; } -void ExactlyOnceQueueManager::InvalidatePopProcessQueue(const string& configName) { +void ExactlyOnceQueueManager::DisablePopProcessQueue(const string& configName, bool isPipelineRemoving) { lock_guard lock(mProcessQueueMux); for (auto& iter : mProcessQueues) { if (iter.second->GetConfigName() == configName) { - iter.second->InvalidatePop(); + iter.second->DisablePop(); + if (!isPipelineRemoving) { + iter.second->SetPipelineForItems(configName); + } } } } -void ExactlyOnceQueueManager::ValidatePopProcessQueue(const string& configName) { +void ExactlyOnceQueueManager::EnablePopProcessQueue(const string& configName) { lock_guard lock(mProcessQueueMux); for (auto& iter : mProcessQueues) { if (iter.second->GetConfigName() == configName) { - iter.second->ValidatePop(); + iter.second->EnablePop(); } } } diff --git a/core/pipeline/queue/ExactlyOnceQueueManager.h b/core/pipeline/queue/ExactlyOnceQueueManager.h index 7f5e8a70ea..7ca468f84d 100644 --- a/core/pipeline/queue/ExactlyOnceQueueManager.h +++ b/core/pipeline/queue/ExactlyOnceQueueManager.h @@ -59,8 +59,8 @@ class ExactlyOnceQueueManager { // 0: success, 1: queue is full, 2: queue not found int PushProcessQueue(QueueKey key, std::unique_ptr&& item); bool IsAllProcessQueueEmpty() const; - void InvalidatePopProcessQueue(const std::string& configName); - void ValidatePopProcessQueue(const std::string& configName); + void DisablePopProcessQueue(const std::string& configName, bool isPipelineRemoving); + void EnablePopProcessQueue(const std::string& configName); // 0: success, 1: queue is full, 2: queue not found int PushSenderQueue(QueueKey key, std::unique_ptr&& item); diff --git a/core/pipeline/queue/ProcessQueueInterface.h b/core/pipeline/queue/ProcessQueueInterface.h index 6a3b8833c8..12baec4029 100644 --- a/core/pipeline/queue/ProcessQueueInterface.h +++ b/core/pipeline/queue/ProcessQueueInterface.h @@ -42,8 +42,10 @@ class ProcessQueueInterface : virtual public QueueInterface&& ques); - void InvalidatePop() { mValidToPop = false; } - void ValidatePop() { mValidToPop = true; } + void DisablePop() { mValidToPop = false; } + void EnablePop() { mValidToPop = true; } + + virtual void SetPipelineForItems(const std::string& name) const = 0; void Reset() { mDownStreamQueues.clear(); } @@ -57,7 +59,7 @@ class ProcessQueueInterface : virtual public QueueInterface mDownStreamQueues; - bool mValidToPop = true; + bool mValidToPop = false; #ifdef APSARA_UNIT_TEST_MAIN friend class BoundedProcessQueueUnittest; diff --git a/core/pipeline/queue/ProcessQueueManager.cpp b/core/pipeline/queue/ProcessQueueManager.cpp index 0b66f6769b..26bc1b8072 100644 --- a/core/pipeline/queue/ProcessQueueManager.cpp +++ b/core/pipeline/queue/ProcessQueueManager.cpp @@ -229,29 +229,32 @@ bool ProcessQueueManager::SetFeedbackInterface(QueueKey key, vectorHasKey(configName)) { auto key = QueueKeyManager::GetInstance()->GetKey(configName); lock_guard lock(mQueueMux); auto iter = mQueues.find(key); if (iter != mQueues.end()) { - (*iter->second.first)->InvalidatePop(); + (*iter->second.first)->DisablePop(); + if (!isPipelineRemoving) { + (*iter->second.first)->SetPipelineForItems(configName); + } } } else { - ExactlyOnceQueueManager::GetInstance()->InvalidatePopProcessQueue(configName); + ExactlyOnceQueueManager::GetInstance()->DisablePopProcessQueue(configName, isPipelineRemoving); } } -void ProcessQueueManager::ValidatePop(const string& configName) { +void ProcessQueueManager::EnablePop(const string& configName) { if (QueueKeyManager::GetInstance()->HasKey(configName)) { auto key = QueueKeyManager::GetInstance()->GetKey(configName); lock_guard lock(mQueueMux); auto iter = mQueues.find(key); if (iter != mQueues.end()) { - (*iter->second.first)->ValidatePop(); + (*iter->second.first)->EnablePop(); } } else { - ExactlyOnceQueueManager::GetInstance()->ValidatePopProcessQueue(configName); + ExactlyOnceQueueManager::GetInstance()->EnablePopProcessQueue(configName); } } diff --git a/core/pipeline/queue/ProcessQueueManager.h b/core/pipeline/queue/ProcessQueueManager.h index dd0198c4b4..ab877a2c7d 100644 --- a/core/pipeline/queue/ProcessQueueManager.h +++ b/core/pipeline/queue/ProcessQueueManager.h @@ -62,8 +62,8 @@ class ProcessQueueManager : public FeedbackInterface { bool IsAllQueueEmpty() const; bool SetDownStreamQueues(QueueKey key, std::vector&& ques); bool SetFeedbackInterface(QueueKey key, std::vector&& feedback); - void InvalidatePop(const std::string& configName); - void ValidatePop(const std::string& configName); + void DisablePop(const std::string& configName, bool isPipelineRemoving); + void EnablePop(const std::string& configName); bool Wait(uint64_t ms); void Trigger(); diff --git a/core/pipeline/queue/QueueInterface.h b/core/pipeline/queue/QueueInterface.h index 5c2e3b82cb..7754190000 100644 --- a/core/pipeline/queue/QueueInterface.h +++ b/core/pipeline/queue/QueueInterface.h @@ -33,12 +33,12 @@ class QueueInterface { {METRIC_LABEL_CONFIG_NAME, ctx.GetConfigName()}, }); - mInItemsCnt = mMetricsRecordRef.CreateCounter("in_items_cnt"); - mInItemDataSizeBytes = mMetricsRecordRef.CreateCounter("in_item_data_size_bytes"); - mOutItemsCnt = mMetricsRecordRef.CreateCounter("out_items_cnt"); - mTotalDelayMs = mMetricsRecordRef.CreateCounter("total_delay_ms"); - mQueueSize = mMetricsRecordRef.CreateIntGauge("queue_size"); - mQueueDataSizeByte = mMetricsRecordRef.CreateIntGauge("queue_data_size_bytes"); + mInItemsCnt = mMetricsRecordRef.CreateCounter(METRIC_COMPONENT_IN_ITEMS_CNT); + mInItemDataSizeBytes = mMetricsRecordRef.CreateCounter(METRIC_COMPONENT_IN_ITEM_SIZE_BYTES); + mOutItemsCnt = mMetricsRecordRef.CreateCounter(METRIC_COMPONENT_OUT_ITEMS_CNT); + mTotalDelayMs = mMetricsRecordRef.CreateCounter(METRIC_COMPONENT_TOTAL_DELAY_MS); + mQueueSize = mMetricsRecordRef.CreateIntGauge(METRIC_COMPONENT_QUEUE_SIZE_CNT); + mQueueDataSizeByte = mMetricsRecordRef.CreateIntGauge(METRIC_COMPONENT_QUEUE_SIZE_BYTES); } virtual ~QueueInterface() = default; diff --git a/core/pipeline/route/Router.cpp b/core/pipeline/route/Router.cpp index e9c4a817cd..a83e1b4225 100644 --- a/core/pipeline/route/Router.cpp +++ b/core/pipeline/route/Router.cpp @@ -15,6 +15,7 @@ #include "pipeline/route/Router.h" #include "common/ParamExtractor.h" +#include "monitor/MetricConstants.h" #include "pipeline/Pipeline.h" #include "pipeline/plugin/interface/Flusher.h" @@ -33,10 +34,20 @@ bool Router::Init(std::vector> configs, const P mAlwaysMatchedFlusherIdx.push_back(item.first); } } + + WriteMetrics::GetInstance()->PrepareMetricsRecordRef(mMetricsRecordRef, + {{METRIC_LABEL_PROJECT, ctx.GetProjectName()}, + {METRIC_LABEL_CONFIG_NAME, ctx.GetConfigName()}, + {METRIC_LABEL_KEY_COMPONENT_NAME, "router"}}); + mInEventsCnt = mMetricsRecordRef.CreateCounter(METRIC_COMPONENT_IN_EVENTS_CNT); + mInGroupDataSizeBytes = mMetricsRecordRef.CreateCounter(METRIC_COMPONENT_IN_EVENT_GROUP_SIZE_BYTES); return true; } vector Router::Route(const PipelineEventGroup& g) const { + mInEventsCnt->Add(g.GetEvents().size()); + mInGroupDataSizeBytes->Add(g.DataSize()); + vector res(mAlwaysMatchedFlusherIdx); for (size_t i = 0; i < mConditions.size(); ++i) { if (mConditions[i].second.Check(g)) { diff --git a/core/pipeline/route/Router.h b/core/pipeline/route/Router.h index c46dedca6e..b036256551 100644 --- a/core/pipeline/route/Router.h +++ b/core/pipeline/route/Router.h @@ -22,6 +22,7 @@ #include #include "models/PipelineEventGroup.h" +#include "monitor/LogtailMetric.h" #include "pipeline/route/Condition.h" namespace logtail { @@ -37,6 +38,10 @@ class Router { std::vector> mConditions; std::vector mAlwaysMatchedFlusherIdx; + mutable MetricsRecordRef mMetricsRecordRef; + CounterPtr mInEventsCnt; + CounterPtr mInGroupDataSizeBytes; + #ifdef APSARA_UNIT_TEST_MAIN friend class RouterUnittest; friend class PipelineUnittest; diff --git a/core/pipeline/serializer/SLSSerializer.cpp b/core/pipeline/serializer/SLSSerializer.cpp index 600e709f68..f24030c100 100644 --- a/core/pipeline/serializer/SLSSerializer.cpp +++ b/core/pipeline/serializer/SLSSerializer.cpp @@ -17,14 +17,14 @@ #include "application/Application.h" #include "common/Flags.h" #include "common/TimeUtil.h" -#include "pipeline/compression/CompressType.h" +#include "common/compression/CompressType.h" #include "plugin/flusher/sls/FlusherSLS.h" DEFINE_FLAG_INT32(max_send_log_group_size, "bytes", 10 * 1024 * 1024); const std::string METRIC_RESERVED_KEY_NAME = "__name__"; -const std::string METRIC_RESERVED_KEY_LABELS = "__labels__"; +const std::string METRIC_RESERVED_KEY_LABELS = "__labels__"; const std::string METRIC_RESERVED_KEY_VALUE = "__value__"; const std::string METRIC_RESERVED_KEY_TIME_NANO = "__time_nano__"; @@ -35,7 +35,31 @@ using namespace std; namespace logtail { +template <> +bool Serializer>::DoSerialize(vector&& p, + std::string& output, + std::string& errorMsg) { + auto inputSize = 0; + for (auto& item : p) { + inputSize += item.mData.size(); + } + mInItemsCnt->Add(1); + mInItemSizeBytes->Add(inputSize); + auto before = std::chrono::system_clock::now(); + auto res = Serialize(std::move(p), output, errorMsg); + mTotalDelayMs->Add( + std::chrono::duration_cast(std::chrono::system_clock::now() - before).count()); + + if (res) { + mOutItemsCnt->Add(1); + mOutItemSizeBytes->Add(output.size()); + } else { + mDiscardedItemsCnt->Add(1); + mDiscardedItemSizeBytes->Add(inputSize); + } + return res; +} bool SLSEventGroupSerializer::Serialize(BatchedEvents&& group, string& res, string& errorMsg) { sls_logs::LogGroup logGroup; @@ -56,7 +80,7 @@ bool SLSEventGroupSerializer::Serialize(BatchedEvents&& group, string& res, stri } else if (e.Is()) { const auto& metricEvent = e.Cast(); if (metricEvent.Is()) { - continue; + continue; } auto log = logGroup.add_logs(); std::ostringstream oss; @@ -78,9 +102,10 @@ bool SLSEventGroupSerializer::Serialize(BatchedEvents&& group, string& res, stri logPtr = log->add_contents(); logPtr->set_key(METRIC_RESERVED_KEY_TIME_NANO); if (metricEvent.GetTimestampNanosecond()) { - logPtr->set_value(std::to_string(metricEvent.GetTimestamp()) + NumberToDigitString(metricEvent.GetTimestampNanosecond().value(), 9)); + logPtr->set_value(std::to_string(metricEvent.GetTimestamp()) + + NumberToDigitString(metricEvent.GetTimestampNanosecond().value(), 9)); } else { - logPtr->set_value(std::to_string(metricEvent.GetTimestamp())); + logPtr->set_value(std::to_string(metricEvent.GetTimestamp())); } // set __value__ if (metricEvent.Is()) { @@ -88,7 +113,7 @@ bool SLSEventGroupSerializer::Serialize(BatchedEvents&& group, string& res, stri logPtr = log->add_contents(); logPtr->set_key(METRIC_RESERVED_KEY_VALUE); logPtr->set_value(std::to_string(value)); - } + } // set __name__ logPtr = log->add_contents(); logPtr->set_key(METRIC_RESERVED_KEY_NAME); @@ -122,9 +147,7 @@ bool SLSEventGroupSerializer::Serialize(BatchedEvents&& group, string& res, stri return true; } -bool SLSEventGroupListSerializer::Serialize(vector&& v, - string& res, - string& errorMsg) { +bool SLSEventGroupListSerializer::Serialize(vector&& v, string& res, string& errorMsg) { sls_logs::SlsLogPackageList logPackageList; for (const auto& item : v) { auto package = logPackageList.add_packages(); diff --git a/core/pipeline/serializer/SLSSerializer.h b/core/pipeline/serializer/SLSSerializer.h index f19d0f8e3a..c9bdb56798 100644 --- a/core/pipeline/serializer/SLSSerializer.h +++ b/core/pipeline/serializer/SLSSerializer.h @@ -27,6 +27,7 @@ class SLSEventGroupSerializer : public Serializer { public: SLSEventGroupSerializer(Flusher* f) : Serializer(f) {} +private: bool Serialize(BatchedEvents&& p, std::string& res, std::string& errorMsg) override; }; @@ -37,10 +38,14 @@ struct CompressedLogGroup { CompressedLogGroup(std::string&& data, size_t rawSize) : mData(std::move(data)), mRawSize(rawSize) {} }; +template<> +bool Serializer>::DoSerialize(std::vector&& p, std::string& output, std::string& errorMsg); + class SLSEventGroupListSerializer : public Serializer> { public: SLSEventGroupListSerializer(Flusher* f) : Serializer>(f) {} +private: bool Serialize(std::vector&& v, std::string& res, std::string& errorMsg) override; }; diff --git a/core/pipeline/serializer/Serializer.h b/core/pipeline/serializer/Serializer.h index 968f882a91..c4e7c674ae 100644 --- a/core/pipeline/serializer/Serializer.h +++ b/core/pipeline/serializer/Serializer.h @@ -16,27 +16,92 @@ #pragma once +#include #include -#include "pipeline/batch/BatchedEvents.h" #include "models/PipelineEventPtr.h" +#include "monitor/MetricConstants.h" +#include "pipeline/batch/BatchedEvents.h" #include "pipeline/plugin/interface/Flusher.h" namespace logtail { +inline size_t GetInputSize(const PipelineEventPtr& p) { + return p->DataSize(); +} + +inline size_t GetInputSize(const BatchedEvents& p) { + return p.mSizeBytes; +} + +inline size_t GetInputSize(const BatchedEventsList& p) { + size_t size = 0; + for (const auto& e : p) { + size += GetInputSize(e); + } + return size; +} + // T: PipelineEventPtr, BatchedEvents, BatchedEventsList template class Serializer { public: - Serializer() = default; - Serializer(Flusher* f) : mFlusher(f) {} + Serializer(Flusher* f) : mFlusher(f) { + WriteMetrics::GetInstance()->PrepareMetricsRecordRef( + mMetricsRecordRef, + {{METRIC_LABEL_PROJECT, f->GetContext().GetProjectName()}, + {METRIC_LABEL_CONFIG_NAME, f->GetContext().GetConfigName()}, + {METRIC_LABEL_KEY_COMPONENT_NAME, "serializer"}, + {METRIC_LABEL_KEY_FLUSHER_NODE_ID, f->GetNodeID()}}); + mInItemsCnt = mMetricsRecordRef.CreateCounter(METRIC_COMPONENT_IN_ITEMS_CNT); + mInItemSizeBytes = mMetricsRecordRef.CreateCounter(METRIC_COMPONENT_IN_ITEM_SIZE_BYTES); + mOutItemsCnt = mMetricsRecordRef.CreateCounter(METRIC_COMPONENT_OUT_ITEMS_CNT); + mOutItemSizeBytes = mMetricsRecordRef.CreateCounter(METRIC_COMPONENT_OUT_ITEM_SIZE_BYTES); + mTotalDelayMs = mMetricsRecordRef.CreateCounter(METRIC_COMPONENT_TOTAL_DELAY_MS); + mDiscardedItemsCnt = mMetricsRecordRef.CreateCounter(METRIC_COMPONENT_DISCARDED_ITEMS_CNT); + mDiscardedItemSizeBytes = mMetricsRecordRef.CreateCounter(METRIC_COMPONENT_DISCARDED_ITEMS_SIZE_BYTES); + } virtual ~Serializer() = default; - virtual bool Serialize(T&& p, std::string& res, std::string& errorMsg) = 0; + bool DoSerialize(T&& p, std::string& output, std::string& errorMsg) { + auto inputSize = GetInputSize(p); + mInItemsCnt->Add(1); + mInItemSizeBytes->Add(inputSize); + + auto before = std::chrono::system_clock::now(); + auto res = Serialize(std::move(p), output, errorMsg); + mTotalDelayMs->Add( + std::chrono::duration_cast(std::chrono::system_clock::now() - before).count()); + + if (res) { + mOutItemsCnt->Add(1); + mOutItemSizeBytes->Add(output.size()); + } else { + mDiscardedItemsCnt->Add(1); + mDiscardedItemSizeBytes->Add(inputSize); + } + return res; + } protected: // if serialized output contains output related info, it can be obtained via this member const Flusher* mFlusher = nullptr; + + mutable MetricsRecordRef mMetricsRecordRef; + CounterPtr mInItemsCnt; + CounterPtr mInItemSizeBytes; + CounterPtr mOutItemsCnt; + CounterPtr mOutItemSizeBytes; + CounterPtr mDiscardedItemsCnt; + CounterPtr mDiscardedItemSizeBytes; + CounterPtr mTotalDelayMs; + +private: + virtual bool Serialize(T&& p, std::string& res, std::string& errorMsg) = 0; + +#ifdef APSARA_UNIT_TEST_MAIN + friend class SerializerUnittest; +#endif }; using EventSerializer = Serializer; diff --git a/core/plugin/flusher/sls/DiskBufferWriter.cpp b/core/plugin/flusher/sls/DiskBufferWriter.cpp index 49bb2789d0..94471e1e54 100644 --- a/core/plugin/flusher/sls/DiskBufferWriter.cpp +++ b/core/plugin/flusher/sls/DiskBufferWriter.cpp @@ -27,7 +27,7 @@ #include "protobuf/sls/sls_logs.pb.h" #include "logger/Logger.h" #include "monitor/LogtailAlarm.h" -#include "profile_sender/ProfileSender.h" +#include "provider/Provider.h" #include "pipeline/queue/QueueKeyManager.h" #include "pipeline/queue/SLSSenderQueueItem.h" #include "sdk/Exception.h" @@ -765,7 +765,7 @@ SendResult DiskBufferWriter::SendToNetSync(sdk::Client* sendClient, bufferMeta.logstore(), ""); // no region - if (!ProfileSender::GetInstance()->IsProfileData("", bufferMeta.project(), bufferMeta.logstore())) + if (!GetProfileSender()->IsProfileData("", bufferMeta.project(), bufferMeta.logstore())) LOG_ERROR(sLogger, ("send data to SLS fail, error_code", errorCode)("error_message", ex.GetMessage())( "endpoint", sendClient->GetRawSlsHost())("projectName", bufferMeta.project())( diff --git a/core/plugin/flusher/sls/FlusherSLS.cpp b/core/plugin/flusher/sls/FlusherSLS.cpp index 5df36854c9..1650d9fb79 100644 --- a/core/plugin/flusher/sls/FlusherSLS.cpp +++ b/core/plugin/flusher/sls/FlusherSLS.cpp @@ -23,9 +23,9 @@ #include "common/LogtailCommonFlags.h" #include "common/ParamExtractor.h" #include "common/TimeUtil.h" +#include "common/compression/CompressorFactory.h" #include "pipeline/Pipeline.h" #include "pipeline/batch/FlushStrategy.h" -#include "pipeline/compression/CompressorFactory.h" #include "pipeline/queue/QueueKeyManager.h" #include "pipeline/queue/SLSSenderQueueItem.h" #include "pipeline/queue/SenderQueueManager.h" @@ -33,7 +33,7 @@ #include "plugin/flusher/sls/SLSClientManager.h" #include "plugin/flusher/sls/SLSResponse.h" #include "plugin/flusher/sls/SendResult.h" -#include "profile_sender/ProfileSender.h" +#include "provider/Provider.h" #include "runner/FlusherRunner.h" #include "sdk/Common.h" #include "sls_control/SLSControl.h" @@ -445,7 +445,7 @@ bool FlusherSLS::Init(const Json::Value& config, Json::Value& optionalGoPipeline // CompressType if (BOOL_FLAG(sls_client_send_compress)) { - mCompressor = CompressorFactory::GetInstance()->Create(config, *mContext, sName, CompressType::LZ4); + mCompressor = CompressorFactory::GetInstance()->Create(config, *mContext, sName, mNodeID, CompressType::LZ4); } mGroupSerializer = make_unique(this); @@ -610,7 +610,7 @@ void FlusherSLS::OnSendDone(const HttpResponse& response, SenderQueueItem* item) auto data = static_cast(item); string configName = HasContext() ? GetContext().GetConfigName() : ""; - bool isProfileData = ProfileSender::GetInstance()->IsProfileData(mRegion, mProject, data->mLogstore); + bool isProfileData = GetProfileSender()->IsProfileData(mRegion, mProject, data->mLogstore); int32_t curTime = time(NULL); auto curSystemTime = chrono::system_clock::now(); if (slsResponse.mStatusCode == 200) { @@ -809,7 +809,7 @@ bool FlusherSLS::Send(string&& data, const string& shardHashKey, const string& l string compressedData; if (mCompressor) { string errorMsg; - if (!mCompressor->Compress(data, compressedData, errorMsg)) { + if (!mCompressor->DoCompress(data, compressedData, errorMsg)) { LOG_WARNING(mContext->GetLogger(), ("failed to compress data", errorMsg)("action", "discard data")("plugin", sName)("config", mContext->GetConfigName())); @@ -866,7 +866,7 @@ bool FlusherSLS::SerializeAndPush(PipelineEventGroup&& group) { std::move(group.GetExactlyOnceCheckpoint())); AddPackId(g); string errorMsg; - if (!mGroupSerializer->Serialize(std::move(g), serializedData, errorMsg)) { + if (!mGroupSerializer->DoSerialize(std::move(g), serializedData, errorMsg)) { LOG_WARNING(mContext->GetLogger(), ("failed to serialize event group", errorMsg)("action", "discard data")("plugin", sName)("config", mContext->GetConfigName())); @@ -880,7 +880,7 @@ bool FlusherSLS::SerializeAndPush(PipelineEventGroup&& group) { return false; } if (mCompressor) { - if (!mCompressor->Compress(serializedData, compressedData, errorMsg)) { + if (!mCompressor->DoCompress(serializedData, compressedData, errorMsg)) { LOG_WARNING(mContext->GetLogger(), ("failed to compress event group", errorMsg)("action", "discard data")("plugin", sName)("config", mContext->GetConfigName())); @@ -926,7 +926,7 @@ bool FlusherSLS::SerializeAndPush(BatchedEventsList&& groupList) { } AddPackId(group); string errorMsg; - if (!mGroupSerializer->Serialize(std::move(group), serializedData, errorMsg)) { + if (!mGroupSerializer->DoSerialize(std::move(group), serializedData, errorMsg)) { LOG_WARNING(mContext->GetLogger(), ("failed to serialize event group", errorMsg)("action", "discard data")("plugin", sName)("config", mContext->GetConfigName())); @@ -941,7 +941,7 @@ bool FlusherSLS::SerializeAndPush(BatchedEventsList&& groupList) { continue; } if (mCompressor) { - if (!mCompressor->Compress(serializedData, compressedData, errorMsg)) { + if (!mCompressor->DoCompress(serializedData, compressedData, errorMsg)) { LOG_WARNING(mContext->GetLogger(), ("failed to compress event group", errorMsg)("action", "discard data")("plugin", sName)("config", mContext->GetConfigName())); @@ -991,7 +991,7 @@ bool FlusherSLS::SerializeAndPush(BatchedEventsList&& groupList) { } if (enablePackageList) { string errorMsg; - mGroupListSerializer->Serialize(std::move(compressedLogGroups), serializedData, errorMsg); + mGroupListSerializer->DoSerialize(std::move(compressedLogGroups), serializedData, errorMsg); allSucceeded = Flusher::PushToQueue(make_unique( std::move(serializedData), packageSize, this, mQueueKey, mLogstore, RawDataType::EVENT_GROUP_LIST)) diff --git a/core/plugin/flusher/sls/FlusherSLS.h b/core/plugin/flusher/sls/FlusherSLS.h index 2840b94fe4..7e3d4658a1 100644 --- a/core/plugin/flusher/sls/FlusherSLS.h +++ b/core/plugin/flusher/sls/FlusherSLS.h @@ -24,12 +24,12 @@ #include #include +#include "common/compression/Compressor.h" +#include "models/PipelineEventGroup.h" #include "pipeline/batch/BatchStatus.h" #include "pipeline/batch/Batcher.h" -#include "pipeline/compression/Compressor.h" -#include "models/PipelineEventGroup.h" -#include "pipeline/plugin/interface/HttpFlusher.h" #include "pipeline/limiter/ConcurrencyLimiter.h" +#include "pipeline/plugin/interface/HttpFlusher.h" #include "pipeline/serializer/SLSSerializer.h" namespace logtail { diff --git a/core/plugin/input/input.cmake b/core/plugin/input/input.cmake index eb213b46b5..b986267eca 100644 --- a/core/plugin/input/input.cmake +++ b/core/plugin/input/input.cmake @@ -23,17 +23,10 @@ list(APPEND THIS_SOURCE_FILES_LIST ${THIS_SOURCE_FILES}) if(MSVC) # remove observer related files in input list(REMOVE_ITEM THIS_SOURCE_FILES_LIST ${CMAKE_SOURCE_DIR}/plugin/input/InputObserverNetwork.cpp ${CMAKE_SOURCE_DIR}/plugin/input/InputObserverNetwork.h) - if (ENABLE_ENTERPRISE) - list(REMOVE_ITEM THIS_SOURCE_FILES_LIST ${CMAKE_SOURCE_DIR}/plugin/input/InputStream.cpp ${CMAKE_SOURCE_DIR}/plugin/input/InputStream.h) - endif () elseif(UNIX) if (NOT LINUX) # remove observer related files in input list(REMOVE_ITEM THIS_SOURCE_FILES_LIST ${CMAKE_SOURCE_DIR}/plugin/input/InputObserverNetwork.cpp ${CMAKE_SOURCE_DIR}/plugin/input/InputObserverNetwork.h) - # remove inputStream in input - if (ENABLE_ENTERPRISE) - list(REMOVE_ITEM THIS_SOURCE_FILES_LIST ${CMAKE_SOURCE_DIR}/plugin/input/InputStream.cpp ${CMAKE_SOURCE_DIR}/plugin/input/InputStream.h) - endif () endif() endif() diff --git a/core/profile_sender/ProfileSender.cpp b/core/profile_sender/ProfileSender.cpp index 1121e91216..7098f31ce1 100644 --- a/core/profile_sender/ProfileSender.cpp +++ b/core/profile_sender/ProfileSender.cpp @@ -28,7 +28,7 @@ #include "sdk/Exception.h" #include "sls_control/SLSControl.h" // TODO: temporarily used -#include "pipeline/compression/CompressorFactory.h" +#include "common/compression/CompressorFactory.h" using namespace std; @@ -101,8 +101,8 @@ void ProfileSender::SetProfileProjectName(const string& region, const string& pr flusher.mAliuid = STRING_FLAG(logtail_profile_aliuid); // logstore is given at send time // TODO: temporarily used - flusher.mCompressor - = CompressorFactory::GetInstance()->Create(Json::Value(), PipelineContext(), "flusher_sls", CompressType::ZSTD); + flusher.mCompressor = CompressorFactory::GetInstance()->Create( + Json::Value(), PipelineContext(), "flusher_sls", "", CompressType::ZSTD); } FlusherSLS* ProfileSender::GetFlusher(const string& region) { diff --git a/core/prometheus/Constants.h b/core/prometheus/Constants.h index fbd33597ae..6a60e3a161 100644 --- a/core/prometheus/Constants.h +++ b/core/prometheus/Constants.h @@ -50,6 +50,7 @@ const char* const SCHEME = "scheme"; const char* const METRICS_PATH = "metrics_path"; const char* const SCRAPE_INTERVAL = "scrape_interval"; const char* const SCRAPE_TIMEOUT = "scrape_timeout"; +const char* const SCRAPE_PROTOCOLS = "scrape_protocols"; const char* const HEADERS = "headers"; const char* const PARAMS = "params"; const char* const QUERY_STRING = "query_string"; @@ -71,6 +72,14 @@ const char* const PASSWORD = "password"; const char* const PASSWORD_FILE = "password_file"; const char* const BASIC_PREFIX = "Basic "; +// scrape protocols, from https://prometheus.io/docs/prometheus/latest/configuration/configuration/#scrape_config +// text/plain, application/openmetrics-text will be used +// version of openmetrics is 1.0.0 or 0.0.1, from +// https://github.com/OpenObservability/OpenMetrics/blob/main/specification/OpenMetrics.md#extensions-and-improvements +const char* const PrometheusProto = "PrometheusProto"; +const char* const PrometheusText0_0_4 = "PrometheusText0.0.4"; +const char* const OpenMetricsText0_0_1 = "OpenMetricsText0.0.1"; +const char* const OpenMetricsText1_0_0 = "OpenMetricsText1.0.0"; // metric labels const char* const JOB = "job"; @@ -94,4 +103,10 @@ const char* const UP = "up"; const char* const SCRAPE_TIMESTAMP_MILLISEC = "scrape_timestamp_millisec"; +// scrape config compression +const char* const ENABLE_COMPRESSION = "enable_compression"; +const char* const ACCEPT_ENCODING = "Accept-Encoding"; +const char* const GZIP = "gzip"; +const char* const IDENTITY = "identity"; + } // namespace logtail::prometheus diff --git a/core/prometheus/schedulers/ScrapeConfig.cpp b/core/prometheus/schedulers/ScrapeConfig.cpp index 177c9a7e27..ff6ee77b15 100644 --- a/core/prometheus/schedulers/ScrapeConfig.cpp +++ b/core/prometheus/schedulers/ScrapeConfig.cpp @@ -35,6 +35,7 @@ bool ScrapeConfig::Init(const Json::Value& scrapeConfig) { return false; } + if (scrapeConfig.isMember(prometheus::SCRAPE_INTERVAL) && scrapeConfig[prometheus::SCRAPE_INTERVAL].isString()) { string tmpScrapeIntervalString = scrapeConfig[prometheus::SCRAPE_INTERVAL].asString(); mScrapeIntervalSeconds = DurationToSecond(tmpScrapeIntervalString); @@ -43,6 +44,23 @@ bool ScrapeConfig::Init(const Json::Value& scrapeConfig) { string tmpScrapeTimeoutString = scrapeConfig[prometheus::SCRAPE_TIMEOUT].asString(); mScrapeTimeoutSeconds = DurationToSecond(tmpScrapeTimeoutString); } + if (scrapeConfig.isMember(prometheus::SCRAPE_PROTOCOLS) && scrapeConfig[prometheus::SCRAPE_PROTOCOLS].isArray()) { + if (!InitScrapeProtocols(scrapeConfig[prometheus::SCRAPE_PROTOCOLS])) { + LOG_ERROR(sLogger, ("scrape protocol config error", scrapeConfig[prometheus::SCRAPE_PROTOCOLS])); + return false; + } + } else { + Json::Value nullJson; + InitScrapeProtocols(nullJson); + } + + if (scrapeConfig.isMember(prometheus::ENABLE_COMPRESSION) + && scrapeConfig[prometheus::ENABLE_COMPRESSION].isBool()) { + InitEnableCompression(scrapeConfig[prometheus::ENABLE_COMPRESSION].asBool()); + } else { + InitEnableCompression(true); + } + if (scrapeConfig.isMember(prometheus::METRICS_PATH) && scrapeConfig[prometheus::METRICS_PATH].isString()) { mMetricsPath = scrapeConfig[prometheus::METRICS_PATH].asString(); } @@ -185,7 +203,7 @@ bool ScrapeConfig::InitBasicAuth(const Json::Value& basicAuth) { auto token = username + ":" + password; auto token64 = sdk::Base64Enconde(token); - mAuthHeaders[prometheus::A_UTHORIZATION] = prometheus::BASIC_PREFIX + token64; + mRequestHeaders[prometheus::A_UTHORIZATION] = prometheus::BASIC_PREFIX + token64; return true; } @@ -219,8 +237,98 @@ bool ScrapeConfig::InitAuthorization(const Json::Value& authorization) { return false; } - mAuthHeaders[prometheus::A_UTHORIZATION] = type + " " + credentials; + mRequestHeaders[prometheus::A_UTHORIZATION] = type + " " + credentials; + return true; +} + +bool ScrapeConfig::InitScrapeProtocols(const Json::Value& scrapeProtocols) { + static auto sScrapeProtocolsHeaders = std::map{ + {prometheus::PrometheusProto, + "application/vnd.google.protobuf;proto=io.prometheus.client.MetricFamily;encoding=delimited"}, + {prometheus::PrometheusText0_0_4, "text/plain;version=0.0.4"}, + {prometheus::OpenMetricsText0_0_1, "application/openmetrics-text;version=0.0.1"}, + {prometheus::OpenMetricsText1_0_0, "application/openmetrics-text;version=1.0.0"}, + }; + static auto sDefaultScrapeProtocols = vector{ + prometheus::PrometheusText0_0_4, + prometheus::PrometheusProto, + prometheus::OpenMetricsText0_0_1, + prometheus::OpenMetricsText1_0_0, + }; + + auto join = [](const vector& strs, const string& sep) { + string result; + for (const auto& str : strs) { + if (!result.empty()) { + result += sep; + } + result += str; + } + return result; + }; + + auto getScrapeProtocols = [](const Json::Value& scrapeProtocols, vector& res) { + for (const auto& scrapeProtocol : scrapeProtocols) { + if (scrapeProtocol.isString()) { + res.push_back(scrapeProtocol.asString()); + } else { + LOG_ERROR(sLogger, ("scrape_protocols config error", "")); + return false; + } + } + return true; + }; + + auto validateScrapeProtocols = [](const vector& scrapeProtocols) { + set dups; + for (const auto& scrapeProtocol : scrapeProtocols) { + if (!sScrapeProtocolsHeaders.count(scrapeProtocol)) { + LOG_ERROR(sLogger, + ("unknown scrape protocol prometheusproto", scrapeProtocol)( + "supported", + "[OpenMetricsText0.0.1 OpenMetricsText1.0.0 PrometheusProto PrometheusText0.0.4]")); + return false; + } + if (dups.count(scrapeProtocol)) { + LOG_ERROR(sLogger, ("duplicated protocol in scrape_protocols", scrapeProtocol)); + return false; + } + dups.insert(scrapeProtocol); + } + return true; + }; + + vector tmpScrapeProtocols; + + if (!getScrapeProtocols(scrapeProtocols, tmpScrapeProtocols)) { + return false; + } + + // if scrape_protocols is empty, use default protocols + if (tmpScrapeProtocols.empty()) { + tmpScrapeProtocols = sDefaultScrapeProtocols; + } + if (!validateScrapeProtocols(tmpScrapeProtocols)) { + return false; + } + + auto weight = tmpScrapeProtocols.size() + 1; + for (auto& tmpScrapeProtocol : tmpScrapeProtocols) { + auto val = sScrapeProtocolsHeaders[tmpScrapeProtocol]; + val += ";q=0." + std::to_string(weight--); + tmpScrapeProtocol = val; + } + tmpScrapeProtocols.push_back("*/*;q=0." + ToString(weight)); + mRequestHeaders[prometheus::ACCEPT] = join(tmpScrapeProtocols, ","); return true; } +void ScrapeConfig::InitEnableCompression(bool enableCompression) { + if (enableCompression) { + mRequestHeaders[prometheus::ACCEPT_ENCODING] = prometheus::GZIP; + } else { + mRequestHeaders[prometheus::ACCEPT_ENCODING] = prometheus::IDENTITY; + } +} + } // namespace logtail \ No newline at end of file diff --git a/core/prometheus/schedulers/ScrapeConfig.h b/core/prometheus/schedulers/ScrapeConfig.h index 4f8e866cbf..eac6d9615a 100644 --- a/core/prometheus/schedulers/ScrapeConfig.h +++ b/core/prometheus/schedulers/ScrapeConfig.h @@ -12,6 +12,7 @@ namespace logtail { + class ScrapeConfig { public: std::string mJobName; @@ -20,7 +21,10 @@ class ScrapeConfig { std::string mMetricsPath; std::string mScheme; - std::map mAuthHeaders; + // auth header + // scrape_protocols Accept header: PrometheusProto, OpenMetricsText0.0.1, OpenMetricsText1.0.0, PrometheusText0.0.4 + // enable_compression Accept-Encoding header: gzip, identity + std::map mRequestHeaders; int64_t mMaxScrapeSizeBytes; int64_t mSampleLimit; @@ -37,10 +41,13 @@ class ScrapeConfig { private: bool InitBasicAuth(const Json::Value& basicAuth); bool InitAuthorization(const Json::Value& authorization); + bool InitScrapeProtocols(const Json::Value& scrapeProtocols); + void InitEnableCompression(bool enableCompression); #ifdef APSARA_UNIT_TEST_MAIN friend class ScrapeConfigUnittest; #endif }; + } // namespace logtail \ No newline at end of file diff --git a/core/prometheus/schedulers/ScrapeScheduler.cpp b/core/prometheus/schedulers/ScrapeScheduler.cpp index f0494c60ae..f5013eebea 100644 --- a/core/prometheus/schedulers/ScrapeScheduler.cpp +++ b/core/prometheus/schedulers/ScrapeScheduler.cpp @@ -67,7 +67,7 @@ void ScrapeScheduler::OnMetricResult(const HttpResponse& response, uint64_t time if (response.mStatusCode != 200) { mScrapeResponseSizeBytes = 0; string headerStr; - for (const auto& [k, v] : mScrapeConfigPtr->mAuthHeaders) { + for (const auto& [k, v] : mScrapeConfigPtr->mRequestHeaders) { headerStr.append(k).append(":").append(v).append(";"); } LOG_WARNING(sLogger, @@ -144,7 +144,7 @@ std::unique_ptr ScrapeScheduler::BuildScrapeTimerEvent(std::chrono:: mPort, mScrapeConfigPtr->mMetricsPath, mScrapeConfigPtr->mQueryString, - mScrapeConfigPtr->mAuthHeaders, + mScrapeConfigPtr->mRequestHeaders, "", mScrapeConfigPtr->mScrapeTimeoutSeconds, mScrapeConfigPtr->mScrapeIntervalSeconds diff --git a/core/prometheus/schedulers/TargetSubscriberScheduler.cpp b/core/prometheus/schedulers/TargetSubscriberScheduler.cpp index 8a77c200fd..c8bc2d501e 100644 --- a/core/prometheus/schedulers/TargetSubscriberScheduler.cpp +++ b/core/prometheus/schedulers/TargetSubscriberScheduler.cpp @@ -217,7 +217,7 @@ TargetSubscriberScheduler::BuildScrapeSchedulerSet(std::vector& targetGr scrapeScheduler->SetTimer(mTimer); auto randSleepMilliSec = GetRandSleepMilliSec( - scrapeScheduler->GetId(), prometheus::RefeshIntervalSeconds, GetCurrentTimeInMilliSeconds()); + scrapeScheduler->GetId(), mScrapeConfigPtr->mScrapeIntervalSeconds, GetCurrentTimeInMilliSeconds()); auto firstExecTime = std::chrono::steady_clock::now() + std::chrono::milliseconds(randSleepMilliSec); scrapeScheduler->SetFirstExecTime(firstExecTime); diff --git a/core/provider/CMakeLists.txt b/core/provider/CMakeLists.txt new file mode 100644 index 0000000000..b0e0feb6c3 --- /dev/null +++ b/core/provider/CMakeLists.txt @@ -0,0 +1,39 @@ +# Copyright 2024 iLogtail Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +cmake_minimum_required(VERSION 3.22) +project(provider) + +file(GLOB LIB_SOURCE_FILES *.cpp *.h) + +set(PROVIDER_SUB_DIRECTORIES_LIST + profile_sender config/feedbacker config/provider config/common_provider protobuf/config_server/v1 protobuf/config_server/v2 +) + +foreach(DIR_NAME IN LISTS PROVIDER_SUB_DIRECTORIES_LIST) + file(GLOB TEMP_SOURCE_FILES + "${CMAKE_CURRENT_SOURCE_DIR}/../${DIR_NAME}/*.cpp" + "${CMAKE_CURRENT_SOURCE_DIR}/../${DIR_NAME}/*.cc" + "${CMAKE_CURRENT_SOURCE_DIR}/../${DIR_NAME}/*.c" + ) + list(APPEND LIB_SOURCE_FILES ${TEMP_SOURCE_FILES}) +endforeach() + +add_library(${PROJECT_NAME} STATIC ${LIB_SOURCE_FILES}) + +# message(STATUS "PROVIDER SOURCE FILES: ${LIB_SOURCE_FILES}") + +foreach(DIR_NAME IN LISTS PROVIDER_SUB_DIRECTORIES_LIST) + target_include_directories(provider PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/../${DIR_NAME}) +endforeach() \ No newline at end of file diff --git a/core/provider/Provider.cpp b/core/provider/Provider.cpp new file mode 100644 index 0000000000..52523b256c --- /dev/null +++ b/core/provider/Provider.cpp @@ -0,0 +1,41 @@ +/* + * Copyright 2024 iLogtail Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "Provider.h" + +#include "config/common_provider/CommonConfigProvider.h" +#include "config/common_provider/LegacyCommonConfigProvider.h" + +namespace logtail { + + +std::vector GetRemoteConfigProviders() { + std::vector providers; + providers.push_back(LegacyCommonConfigProvider::GetInstance()); + providers.push_back(CommonConfigProvider::GetInstance()); + return providers; +} + +void InitRemoteConfigProviders() { + CommonConfigProvider::GetInstance()->Init("common_v2"); + LegacyCommonConfigProvider::GetInstance()->Init("common"); +} + +ProfileSender* GetProfileSender() { + return ProfileSender::GetInstance(); +} + +} // namespace logtail \ No newline at end of file diff --git a/core/provider/Provider.h b/core/provider/Provider.h new file mode 100644 index 0000000000..999aaa8a62 --- /dev/null +++ b/core/provider/Provider.h @@ -0,0 +1,37 @@ +/* + * Copyright 2024 iLogtail Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include "config/provider/ConfigProvider.h" +#include "profile_sender/ProfileSender.h" + +namespace logtail { +/* + providers modules are used to replace the default implementation of ilogtail. +*/ + +// GetRemoteConfigProviders returns a vector of pairs for remote config providers. +// It currently returns two providers: LegacyCommonConfigProvider and CommonConfigProvider. +std::vector GetRemoteConfigProviders(); + +// InitRemoteConfigProviders initializes the remote config providers. +// It currently initializes the LegacyCommonConfigProvider and CommonConfigProvider. +void InitRemoteConfigProviders(); + +// GetProfileSender returns the ProfileSender instance. +ProfileSender* GetProfileSender(); +} // namespace logtail \ No newline at end of file diff --git a/core/runner/FlusherRunner.cpp b/core/runner/FlusherRunner.cpp index 16af0a51df..3d176b75ef 100644 --- a/core/runner/FlusherRunner.cpp +++ b/core/runner/FlusherRunner.cpp @@ -41,6 +41,16 @@ namespace logtail { bool FlusherRunner::Init() { srand(time(nullptr)); + WriteMetrics::GetInstance()->PrepareMetricsRecordRef(mMetricsRecordRef, + {{METRIC_LABEL_KEY_RUNNER_NAME, "flusher_runner"}}); + mInItemsCnt = mMetricsRecordRef.CreateCounter(METRIC_RUNNER_IN_ITEMS_CNT); + mInItemDataSizeBytes = mMetricsRecordRef.CreateCounter(METRIC_RUNNER_IN_ITEM_SIZE_BYTES); + mOutItemsCnt = mMetricsRecordRef.CreateCounter(METRIC_RUNNER_OUT_ITEMS_CNT); + mTotalDelayMs = mMetricsRecordRef.CreateCounter(METRIC_RUNNER_TOTAL_DELAY_MS); + mLastRunTime = mMetricsRecordRef.CreateIntGauge(METRIC_RUNNER_LAST_RUN_TIME); + mInItemRawDataSizeBytes = mMetricsRecordRef.CreateCounter(METRIC_RUNNER_FLUSHER_IN_ITEM_RAW_SIZE_BYTES); + mWaitingItemsCnt = mMetricsRecordRef.CreateIntGauge(METRIC_RUNNER_FLUSHER_WAITING_ITEMS_CNT); + mThreadRes = async(launch::async, &FlusherRunner::Run, this); mLastCheckSendClientTime = time(nullptr); return true; @@ -99,12 +109,20 @@ void FlusherRunner::Run() { LOG_INFO(sLogger, ("flusher runner", "started")); while (true) { auto curTime = chrono::system_clock::now(); + mLastRunTime->Set(chrono::duration_cast(curTime.time_since_epoch()).count()); vector items; SenderQueueManager::GetInstance()->GetAllAvailableItems(items, !Application::GetInstance()->IsExiting()); if (items.empty()) { SenderQueueManager::GetInstance()->Wait(1000); } else { + for (auto itr = items.begin(); itr != items.end(); ++itr) { + mInItemDataSizeBytes->Add((*itr)->mData.size()); + mInItemRawDataSizeBytes->Add((*itr)->mRawSize); + } + mInItemsCnt->Add(items.size()); + mWaitingItemsCnt->Add(items.size()); + // smoothing send tps, walk around webserver load burst uint32_t bufferPackageCount = items.size(); if (!Application::GetInstance()->IsExiting() && AppConfig::GetInstance()->IsSendRandomSleep()) { @@ -132,6 +150,10 @@ void FlusherRunner::Run() { } Dispatch(*itr); + mWaitingItemsCnt->Sub(1); + mOutItemsCnt->Add(1); + mTotalDelayMs->Add( + chrono::duration_cast(chrono::system_clock::now() - curTime).count()); } // TODO: move the following logic to scheduler diff --git a/core/runner/FlusherRunner.h b/core/runner/FlusherRunner.h index fb7347cddb..5681c4b263 100644 --- a/core/runner/FlusherRunner.h +++ b/core/runner/FlusherRunner.h @@ -20,6 +20,7 @@ #include #include +#include "monitor/LogtailMetric.h" #include "pipeline/plugin/interface/Flusher.h" #include "pipeline/queue/SenderQueueItem.h" #include "runner/sink/SinkType.h" @@ -63,6 +64,15 @@ class FlusherRunner { int64_t mSendLastTime = 0; int32_t mSendLastByte = 0; + mutable MetricsRecordRef mMetricsRecordRef; + CounterPtr mInItemsCnt; + CounterPtr mInItemDataSizeBytes; + CounterPtr mInItemRawDataSizeBytes; + CounterPtr mOutItemsCnt; + CounterPtr mTotalDelayMs; + IntGaugePtr mWaitingItemsCnt; + IntGaugePtr mLastRunTime; + #ifdef APSARA_UNIT_TEST_MAIN friend class PluginRegistryUnittest; friend class FlusherRunnerUnittest; diff --git a/core/runner/LogProcess.cpp b/core/runner/LogProcess.cpp index cd65998408..dc3cb84ad1 100644 --- a/core/runner/LogProcess.cpp +++ b/core/runner/LogProcess.cpp @@ -15,12 +15,13 @@ #include "runner/LogProcess.h" #include "app_config/AppConfig.h" -#include "pipeline/batch/TimeoutFlushManager.h" #include "common/Flags.h" #include "go_pipeline/LogtailPlugin.h" #include "monitor/LogFileProfiler.h" #include "monitor/LogtailAlarm.h" +#include "monitor/MetricConstants.h" #include "pipeline/PipelineManager.h" +#include "pipeline/batch/TimeoutFlushManager.h" #include "pipeline/queue/ExactlyOnceQueueManager.h" #include "pipeline/queue/ProcessQueueManager.h" #include "pipeline/queue/QueueKeyManager.h" @@ -39,6 +40,12 @@ DEFINE_FLAG_INT32(default_flush_merged_buffer_interval, "default flush merged bu namespace logtail { +thread_local MetricsRecordRef LogProcess::sMetricsRecordRef; +thread_local CounterPtr LogProcess::sInGroupsCnt; +thread_local CounterPtr LogProcess::sInEventsCnt; +thread_local CounterPtr LogProcess::sInGroupDataSizeBytes; +thread_local IntGaugePtr LogProcess::sLastRunTime; + LogProcess::LogProcess() : mAccessProcessThreadRWL(ReadWriteLock::PREFER_WRITER) { } @@ -56,8 +63,6 @@ LogProcess::~LogProcess() { void LogProcess::Start() { if (mInitialized) return; - mAgentProcessQueueFullTotal = LoongCollectorMonitor::GetInstance()->GetIntGauge(METRIC_AGENT_PROCESS_QUEUE_FULL_TOTAL); - mAgentProcessQueueTotal = LoongCollectorMonitor::GetInstance()->GetIntGauge(METRIC_AGENT_PROCESS_QUEUE_TOTAL); mInitialized = true; mThreadCount = AppConfig::GetInstance()->GetProcessThreadCount(); @@ -137,13 +142,16 @@ bool LogProcess::FlushOut(int32_t waitMs) { } void* LogProcess::ProcessLoop(int32_t threadNo) { - LOG_DEBUG(sLogger, ("runner/LogProcess.hread", "Start")("threadNo", threadNo)); + LOG_DEBUG(sLogger, ("LogProcess", "Start")("threadNo", threadNo)); + // thread local metrics should be initialized in each thread + WriteMetrics::GetInstance()->PrepareMetricsRecordRef( + sMetricsRecordRef, {{METRIC_LABEL_KEY_RUNNER_NAME, "processor_runner"}, {"thread_no", ToString(threadNo)}}); + sInGroupsCnt = sMetricsRecordRef.CreateCounter(METRIC_RUNNER_IN_EVENT_GROUPS_CNT); + sInEventsCnt = sMetricsRecordRef.CreateCounter(METRIC_RUNNER_IN_EVENTS_CNT); + sInGroupDataSizeBytes = sMetricsRecordRef.CreateCounter(METRIC_RUNNER_IN_EVENT_GROUP_SIZE_BYTES); + sLastRunTime = sMetricsRecordRef.CreateIntGauge(METRIC_RUNNER_LAST_RUN_TIME); + static int32_t lastMergeTime = 0; - static atomic_int s_processCount{0}; - static atomic_long s_processBytes{0}; - static atomic_int s_processLines{0}; - // only thread 0 update metric - int32_t lastUpdateMetricTime = time(NULL); while (true) { mThreadFlags[threadNo] = false; @@ -153,35 +161,9 @@ void* LogProcess::ProcessLoop(int32_t threadNo) { lastMergeTime = curTime; } - if (threadNo == 0 && curTime - lastUpdateMetricTime >= 40) { - static auto sMonitor = LogtailMonitor::GetInstance(); - - // atomic counter will be negative if process speed is too fast. - sMonitor->UpdateMetric("process_tps", 1.0 * s_processCount / (curTime - lastUpdateMetricTime)); - sMonitor->UpdateMetric("process_bytes_ps", 1.0 * s_processBytes / (curTime - lastUpdateMetricTime)); - sMonitor->UpdateMetric("process_lines_ps", 1.0 * s_processLines / (curTime - lastUpdateMetricTime)); - lastUpdateMetricTime = curTime; - s_processCount = 0; - s_processBytes = 0; - s_processLines = 0; - - // update process queue status - uint32_t InvalidProcessQueueTotal = ProcessQueueManager::GetInstance()->GetInvalidCnt(); - sMonitor->UpdateMetric("process_queue_full", InvalidProcessQueueTotal); - mAgentProcessQueueFullTotal->Set(InvalidProcessQueueTotal); - uint32_t ProcessQueueTotal = ProcessQueueManager::GetInstance()->GetCnt(); - sMonitor->UpdateMetric("process_queue_total", ProcessQueueTotal); - mAgentProcessQueueTotal->Set(ProcessQueueTotal); - if (ExactlyOnceQueueManager::GetInstance()->GetProcessQueueCnt() > 0) { - sMonitor->UpdateMetric("eo_process_queue_full", - ExactlyOnceQueueManager::GetInstance()->GetInvalidProcessQueueCnt()); - sMonitor->UpdateMetric("eo_process_queue_total", - ExactlyOnceQueueManager::GetInstance()->GetProcessQueueCnt()); - } - } - { ReadLock lock(mAccessProcessThreadRWL); + sLastRunTime->Set(curTime); unique_ptr item; string configName; @@ -189,6 +171,9 @@ void* LogProcess::ProcessLoop(int32_t threadNo) { ProcessQueueManager::GetInstance()->Wait(100); continue; } + sInEventsCnt->Add(item->mEventGroup.GetEvents().size()); + sInGroupsCnt->Add(1); + sInGroupDataSizeBytes->Add(item->mEventGroup.DataSize()); mThreadFlags[threadNo] = true; auto pipeline = PipelineManager::GetInstance()->FindConfigByName(configName); @@ -227,12 +212,6 @@ void* LogProcess::ProcessLoop(int32_t threadNo) { pipeline->GetContext().GetRegion()); } - s_processCount++; - if (isLog) { - s_processBytes += profile.readBytes; - s_processLines += profile.splitLines; - } - if (eventGroupList.empty()) { continue; } @@ -296,7 +275,7 @@ void* LogProcess::ProcessLoop(int32_t threadNo) { } } } - LOG_WARNING(sLogger, ("runner/LogProcess.hread", "Exit")("threadNo", threadNo)); + LOG_WARNING(sLogger, ("LogProcess", "Exit")("threadNo", threadNo)); return NULL; } diff --git a/core/runner/LogProcess.h b/core/runner/LogProcess.h index c0bd77e9ad..624bb09be8 100644 --- a/core/runner/LogProcess.h +++ b/core/runner/LogProcess.h @@ -23,6 +23,7 @@ #include "common/LogRunnable.h" #include "common/Thread.h" #include "models/PipelineEventGroup.h" +#include "monitor/LogtailMetric.h" #include "monitor/Monitor.h" #include "pipeline/queue/QueueKey.h" @@ -60,8 +61,11 @@ class LogProcess : public LogRunnable { std::atomic_bool* mThreadFlags; ReadWriteLock mAccessProcessThreadRWL; - IntGaugePtr mAgentProcessQueueFullTotal; - IntGaugePtr mAgentProcessQueueTotal; + thread_local static MetricsRecordRef sMetricsRecordRef; + thread_local static CounterPtr sInGroupsCnt; + thread_local static CounterPtr sInEventsCnt; + thread_local static CounterPtr sInGroupDataSizeBytes; + thread_local static IntGaugePtr sLastRunTime; }; } // namespace logtail \ No newline at end of file diff --git a/core/runner/sink/http/HttpSink.cpp b/core/runner/sink/http/HttpSink.cpp index 0a5e7de3d6..68e276397f 100644 --- a/core/runner/sink/http/HttpSink.cpp +++ b/core/runner/sink/http/HttpSink.cpp @@ -18,6 +18,7 @@ #include "common/StringTools.h" #include "common/http/Curl.h" #include "logger/Logger.h" +#include "monitor/MetricConstants.h" #include "pipeline/plugin/interface/HttpFlusher.h" #include "pipeline/queue/QueueKeyManager.h" #include "pipeline/queue/SenderQueueItem.h" @@ -33,6 +34,19 @@ bool HttpSink::Init() { LOG_ERROR(sLogger, ("failed to init http sink", "failed to init curl multi client")); return false; } + + WriteMetrics::GetInstance()->PrepareMetricsRecordRef(mMetricsRecordRef, + {{METRIC_LABEL_KEY_RUNNER_NAME, "http sink"}}); + mInItemsCnt = mMetricsRecordRef.CreateCounter(METRIC_RUNNER_IN_ITEMS_CNT); + mLastRunTime = mMetricsRecordRef.CreateIntGauge(METRIC_RUNNER_LAST_RUN_TIME); + mOutSuccessfulItemsCnt = mMetricsRecordRef.CreateCounter(METRIC_RUNNER_HTTP_SINK_OUT_SUCCESSFUL_ITEMS_CNT); + mOutFailedItemsCnt = mMetricsRecordRef.CreateCounter(METRIC_RUNNER_HTTP_SINK_OUT_FAILED_ITEMS_CNT); + mSendingItemsCnt = mMetricsRecordRef.CreateIntGauge(METRIC_RUNNER_HTTP_SINK_SENDING_ITEMS_CNT); + mSendConcurrency = mMetricsRecordRef.CreateIntGauge(METRIC_RUNNER_HTTP_SINK_SEND_CONCURRENCY); + + // TODO: should be dynamic + mSendConcurrency->Set(AppConfig::GetInstance()->GetSendRequestConcurrency()); + mThreadRes = async(launch::async, &HttpSink::Run, this); return true; } @@ -48,9 +62,13 @@ void HttpSink::Stop() { } void HttpSink::Run() { + LOG_INFO(sLogger, ("http sink", "started")); while (true) { + mLastRunTime->Set( + chrono::duration_cast(chrono::system_clock::now().time_since_epoch()).count()); unique_ptr request; if (mQueue.WaitAndPop(request, 500)) { + mInItemsCnt->Add(1); LOG_DEBUG( sLogger, ("got item from flusher runner, item address", request->mItem)( @@ -59,6 +77,7 @@ void HttpSink::Run() { if (!AddRequestToClient(std::move(request))) { continue; } + mSendingItemsCnt->Add(1); } else if (mIsFlush && mQueue.Empty()) { break; } else { @@ -72,7 +91,7 @@ void HttpSink::Run() { } } -bool HttpSink::AddRequestToClient(std::unique_ptr&& request) { +bool HttpSink::AddRequestToClient(unique_ptr&& request) { curl_slist* headers = nullptr; CURL* curl = CreateCurlHandler(request->mMethod, request->mHTTPSFlag, @@ -90,6 +109,7 @@ bool HttpSink::AddRequestToClient(std::unique_ptr&& request) { if (curl == nullptr) { request->mItem->mStatus = SendingStatus::IDLE; FlusherRunner::GetInstance()->DecreaseHttpSendingCnt(); + mOutFailedItemsCnt->Add(1); LOG_ERROR(sLogger, ("failed to send request", "failed to init curl handler")( "action", "put sender queue item back to sender queue")("item address", request->mItem)( @@ -99,13 +119,14 @@ bool HttpSink::AddRequestToClient(std::unique_ptr&& request) { request->mPrivateData = headers; curl_easy_setopt(curl, CURLOPT_PRIVATE, request.get()); - request->mLastSendTime = std::chrono::system_clock::now(); + request->mLastSendTime = chrono::system_clock::now(); auto res = curl_multi_add_handle(mClient, curl); if (res != CURLM_OK) { request->mItem->mStatus = SendingStatus::IDLE; FlusherRunner::GetInstance()->DecreaseHttpSendingCnt(); curl_easy_cleanup(curl); + mOutFailedItemsCnt->Add(1); LOG_ERROR(sLogger, ("failed to send request", "failed to add the easy curl handle to multi_handle")("errMsg", curl_multi_strerror(res))( @@ -122,6 +143,8 @@ void HttpSink::DoRun() { CURLMcode mc; int runningHandlers = 1; while (runningHandlers) { + auto curTime = chrono::system_clock::now(); + mLastRunTime->Set(chrono::duration_cast(curTime.time_since_epoch()).count()); if ((mc = curl_multi_perform(mClient, &runningHandlers)) != CURLM_OK) { LOG_ERROR( sLogger, @@ -133,6 +156,7 @@ void HttpSink::DoRun() { unique_ptr request; if (mQueue.TryPop(request)) { + mInItemsCnt->Add(1); LOG_DEBUG( sLogger, ("got item from flusher runner, item address", request->mItem)( @@ -140,6 +164,7 @@ void HttpSink::DoRun() { "wait time", ToString(time(nullptr) - request->mEnqueTime))("try cnt", ToString(request->mTryCnt))); if (AddRequestToClient(std::move(request))) { ++runningHandlers; + mSendingItemsCnt->Add(1); } } @@ -191,10 +216,13 @@ void HttpSink::HandleCompletedRequests() { HttpSinkRequest* request = nullptr; curl_easy_getinfo(handler, CURLINFO_PRIVATE, &request); LOG_DEBUG(sLogger, - ("send http request completed, item address", request->mItem) - ("config-flusher-dst", QueueKeyManager::GetInstance()->GetName(request->mItem->mQueueKey)) - ("response time",ToString(chrono::duration_cast(chrono::system_clock::now()- request->mLastSendTime).count()) + "ms") - ("try cnt", ToString(request->mTryCnt))); + ("send http request completed, item address", request->mItem)( + "config-flusher-dst", QueueKeyManager::GetInstance()->GetName(request->mItem->mQueueKey))( + "response time", + ToString(chrono::duration_cast(chrono::system_clock::now() + - request->mLastSendTime) + .count()) + + "ms")("try cnt", ToString(request->mTryCnt))); switch (msg->data.result) { case CURLE_OK: { long statusCode = 0; @@ -202,6 +230,8 @@ void HttpSink::HandleCompletedRequests() { request->mResponse.mStatusCode = (int32_t)statusCode; static_cast(request->mItem->mFlusher)->OnSendDone(request->mResponse, request->mItem); FlusherRunner::GetInstance()->DecreaseHttpSendingCnt(); + mOutSuccessfulItemsCnt->Add(1); + mSendingItemsCnt->Sub(1); break; } default: @@ -225,6 +255,8 @@ void HttpSink::HandleCompletedRequests() { ->OnSendDone(request->mResponse, request->mItem); FlusherRunner::GetInstance()->DecreaseHttpSendingCnt(); } + mOutFailedItemsCnt->Add(1); + mSendingItemsCnt->Sub(1); break; } curl_multi_remove_handle(mClient, handler); diff --git a/core/runner/sink/http/HttpSink.h b/core/runner/sink/http/HttpSink.h index 6e11b84855..56d9e01302 100644 --- a/core/runner/sink/http/HttpSink.h +++ b/core/runner/sink/http/HttpSink.h @@ -25,6 +25,7 @@ #include "runner/sink/Sink.h" #include "runner/sink/http/HttpSinkRequest.h" +#include "monitor/LogtailMetric.h" namespace logtail { @@ -55,6 +56,15 @@ class HttpSink : public Sink { std::future mThreadRes; std::atomic_bool mIsFlush = false; + mutable MetricsRecordRef mMetricsRecordRef; + CounterPtr mInItemsCnt; + CounterPtr mOutSuccessfulItemsCnt; + CounterPtr mOutFailedItemsCnt; + // CounterPtr mTotalDelayMs; // TODO: should record distribution instead of average + IntGaugePtr mSendingItemsCnt; + IntGaugePtr mSendConcurrency; + IntGaugePtr mLastRunTime; + #ifdef APSARA_UNIT_TEST_MAIN friend class FlusherRunnerUnittest; #endif diff --git a/core/unittest/CMakeLists.txt b/core/unittest/CMakeLists.txt index 111bff640b..ba42e5bf98 100644 --- a/core/unittest/CMakeLists.txt +++ b/core/unittest/CMakeLists.txt @@ -44,6 +44,7 @@ macro(add_core_subdir) add_subdirectory(plugin) add_subdirectory(polling) add_subdirectory(processor) + add_subdirectory(provider) add_subdirectory(queue) add_subdirectory(reader) add_subdirectory(sdk) @@ -68,6 +69,7 @@ macro(ut_link ut_link_withspl) flusher_link(${UT_BASE_TARGET}) all_link(${UT_BASE_TARGET}) common_link(${UT_BASE_TARGET}) + target_link_libraries(${UT_BASE_TARGET} provider) endmacro() set(SOURCE_FILES_CORE ${FRAMEWORK_SOURCE_FILES} ${PLUGIN_SOURCE_FILES_CORE}) diff --git a/core/unittest/batch/BatchItemUnittest.cpp b/core/unittest/batch/BatchItemUnittest.cpp index 71d2d7fa59..0aac5d30b3 100644 --- a/core/unittest/batch/BatchItemUnittest.cpp +++ b/core/unittest/batch/BatchItemUnittest.cpp @@ -92,6 +92,7 @@ void EventBatchItemUnittest::TestAdd() { APSARA_TEST_EQUAL(1U, mItem.mBatch.mEvents.size()); APSARA_TEST_EQUAL(1U, mItem.GetStatus().GetCnt()); APSARA_TEST_EQUAL(size, mItem.GetStatus().GetSize()); + APSARA_TEST_NOT_EQUAL(0, mItem.mTotalEnqueTimeMs); } void EventBatchItemUnittest::TestFlushEmpty() { @@ -116,10 +117,12 @@ void EventBatchItemUnittest::TestFlushGroupBatchItem() { sEventGroup->AddLogEvent(); PipelineEventPtr& e = sEventGroup->MutableEvents().back(); mItem.Add(std::move(e)); + auto size = mItem.DataSize(); GroupBatchItem res; mItem.Flush(res); APSARA_TEST_EQUAL(1U, res.mGroups.size()); + APSARA_TEST_EQUAL(size, res.mGroups[0].mSizeBytes); APSARA_TEST_TRUE(mItem.IsEmpty()); APSARA_TEST_TRUE(mItem.mBatch.mTags.mInner.empty()); @@ -130,12 +133,14 @@ void EventBatchItemUnittest::TestFlushGroupBatchItem() { APSARA_TEST_EQUAL(0U, mItem.GetStatus().GetCnt()); APSARA_TEST_EQUAL(0U, mItem.GetStatus().GetSize()); APSARA_TEST_EQUAL(0, mItem.GetStatus().GetCreateTime()); + APSARA_TEST_EQUAL(0, mItem.mTotalEnqueTimeMs); } void EventBatchItemUnittest::TestFlushBatchedEvensList() { sEventGroup->AddLogEvent(); PipelineEventPtr& e = sEventGroup->MutableEvents().back(); mItem.Add(std::move(e)); + auto size = mItem.DataSize(); BatchedEventsList res; mItem.Flush(res); @@ -145,6 +150,7 @@ void EventBatchItemUnittest::TestFlushBatchedEvensList() { APSARA_TEST_NOT_EQUAL(nullptr, res[0].mExactlyOnceCheckpoint); APSARA_TEST_STREQ("pack_id", res[0].mPackIdPrefix.data()); APSARA_TEST_EQUAL(1U, res[0].mSourceBuffers.size()); + APSARA_TEST_EQUAL(size, res[0].mSizeBytes); APSARA_TEST_TRUE(mItem.IsEmpty()); APSARA_TEST_TRUE(mItem.mBatch.mTags.mInner.empty()); @@ -155,12 +161,14 @@ void EventBatchItemUnittest::TestFlushBatchedEvensList() { APSARA_TEST_EQUAL(0U, mItem.GetStatus().GetCnt()); APSARA_TEST_EQUAL(0U, mItem.GetStatus().GetSize()); APSARA_TEST_EQUAL(0, mItem.GetStatus().GetCreateTime()); + APSARA_TEST_EQUAL(0, mItem.mTotalEnqueTimeMs); } void EventBatchItemUnittest::TestFlushBatchedEvensLists() { sEventGroup->AddLogEvent(); PipelineEventPtr& e = sEventGroup->MutableEvents().back(); mItem.Add(std::move(e)); + auto size = mItem.DataSize(); vector res; mItem.Flush(res); @@ -171,6 +179,7 @@ void EventBatchItemUnittest::TestFlushBatchedEvensLists() { APSARA_TEST_NOT_EQUAL(nullptr, res[0][0].mExactlyOnceCheckpoint); APSARA_TEST_STREQ("pack_id", res[0][0].mPackIdPrefix.data()); APSARA_TEST_EQUAL(1U, res[0][0].mSourceBuffers.size()); + APSARA_TEST_EQUAL(size, res[0][0].mSizeBytes); APSARA_TEST_TRUE(mItem.IsEmpty()); APSARA_TEST_TRUE(mItem.mBatch.mTags.mInner.empty()); @@ -181,6 +190,7 @@ void EventBatchItemUnittest::TestFlushBatchedEvensLists() { APSARA_TEST_EQUAL(0U, mItem.GetStatus().GetCnt()); APSARA_TEST_EQUAL(0U, mItem.GetStatus().GetSize()); APSARA_TEST_EQUAL(0, mItem.GetStatus().GetCreateTime()); + APSARA_TEST_EQUAL(0, mItem.mTotalEnqueTimeMs); } void EventBatchItemUnittest::TestExactlyOnce() { @@ -223,6 +233,7 @@ class GroupBatchItemUnittest : public ::testing::Test { mBatch.mEvents = std::move(eventGroup.MutableEvents()); mBatch.mSourceBuffers.emplace_back(std::move(eventGroup.GetSourceBuffer())); mBatch.mTags = std::move(eventGroup.GetSizedTags()); + mBatch.mSizeBytes = 100; } void TearDown() override { @@ -236,11 +247,15 @@ class GroupBatchItemUnittest : public ::testing::Test { }; void GroupBatchItemUnittest::TestAdd() { - size_t size = mBatch.DataSize(); - mItem.Add(std::move(mBatch)); + size_t size = mBatch.mSizeBytes; + mItem.Add(std::move(mBatch), 1234567890000); APSARA_TEST_EQUAL(1U, mItem.mGroups.size()); APSARA_TEST_EQUAL(size, mItem.GetStatus().GetSize()); + APSARA_TEST_EQUAL(1234567890000, mItem.TotalEnqueTimeMs()); + APSARA_TEST_EQUAL(1U, mItem.EventSize()); + APSARA_TEST_EQUAL(1U, mItem.GroupSize()); + APSARA_TEST_EQUAL(100U, mItem.DataSize()); } void GroupBatchItemUnittest::TestFlushEmpty() { @@ -257,7 +272,7 @@ void GroupBatchItemUnittest::TestFlushEmpty() { } void GroupBatchItemUnittest::TestFlushBatchedEvensList() { - mItem.Add(std::move(mBatch)); + mItem.Add(std::move(mBatch), 1234567890000); BatchedEventsList res; mItem.Flush(res); @@ -266,10 +281,14 @@ void GroupBatchItemUnittest::TestFlushBatchedEvensList() { APSARA_TEST_TRUE(mItem.IsEmpty()); APSARA_TEST_EQUAL(0U, mItem.GetStatus().GetSize()); APSARA_TEST_EQUAL(0, mItem.GetStatus().GetCreateTime()); + APSARA_TEST_EQUAL(0, mItem.TotalEnqueTimeMs()); + APSARA_TEST_EQUAL(0U, mItem.EventSize()); + APSARA_TEST_EQUAL(0U, mItem.GroupSize()); + APSARA_TEST_EQUAL(0U, mItem.DataSize()); } void GroupBatchItemUnittest::TestFlushBatchedEvensLists() { - mItem.Add(std::move(mBatch)); + mItem.Add(std::move(mBatch), 1234567890000); vector res; mItem.Flush(res); @@ -279,6 +298,10 @@ void GroupBatchItemUnittest::TestFlushBatchedEvensLists() { APSARA_TEST_TRUE(mItem.IsEmpty()); APSARA_TEST_EQUAL(0U, mItem.GetStatus().GetSize()); APSARA_TEST_EQUAL(0, mItem.GetStatus().GetCreateTime()); + APSARA_TEST_EQUAL(0, mItem.TotalEnqueTimeMs()); + APSARA_TEST_EQUAL(0U, mItem.EventSize()); + APSARA_TEST_EQUAL(0U, mItem.GroupSize()); + APSARA_TEST_EQUAL(0U, mItem.DataSize()); } UNIT_TEST_CASE(GroupBatchItemUnittest, TestAdd) diff --git a/core/unittest/batch/BatchStatusUnittest.cpp b/core/unittest/batch/BatchStatusUnittest.cpp index 23932bef46..094333fd3d 100644 --- a/core/unittest/batch/BatchStatusUnittest.cpp +++ b/core/unittest/batch/BatchStatusUnittest.cpp @@ -124,6 +124,7 @@ class GroupBatchStatusUnittest : public ::testing::Test { void TestUpdate(); protected: + static void SetUpTestCase() { sBatch.mSizeBytes = 100; } void SetUp() override { mStatus.Reset(); } private: @@ -142,10 +143,10 @@ void GroupBatchStatusUnittest::TestReset() { void GroupBatchStatusUnittest::TestUpdate() { mStatus.Update(sBatch); time_t createTime = mStatus.GetCreateTime(); - APSARA_TEST_EQUAL(sBatch.DataSize(), mStatus.GetSize()); + APSARA_TEST_EQUAL(sBatch.mSizeBytes, mStatus.GetSize()); mStatus.Update(sBatch); - APSARA_TEST_EQUAL(2 * sBatch.DataSize(), mStatus.GetSize()); + APSARA_TEST_EQUAL(2 * sBatch.mSizeBytes, mStatus.GetSize()); APSARA_TEST_EQUAL(createTime, mStatus.GetCreateTime()); } diff --git a/core/unittest/batch/BatcherUnittest.cpp b/core/unittest/batch/BatcherUnittest.cpp index 8d40721565..799153692d 100644 --- a/core/unittest/batch/BatcherUnittest.cpp +++ b/core/unittest/batch/BatcherUnittest.cpp @@ -12,9 +12,8 @@ // See the License for the specific language governing permissions and // limitations under the License. - -#include "pipeline/batch/Batcher.h" #include "common/JsonUtil.h" +#include "pipeline/batch/Batcher.h" #include "unittest/Unittest.h" #include "unittest/plugin/PluginMock.h" @@ -34,6 +33,7 @@ class BatcherUnittest : public ::testing::Test { void TestFlushGroupQueue(); void TestFlushAllWithoutGroupBatch(); void TestFlushAllWithGroupBatch(); + void TestMetric(); protected: static void SetUpTestCase() { sFlusher = make_unique(); } @@ -42,6 +42,7 @@ class BatcherUnittest : public ::testing::Test { mCtx.SetConfigName("test_config"); sFlusher->SetContext(mCtx); sFlusher->SetMetricsRecordRef(FlusherMock::sName, "1", "1", "1"); + sFlusher->SetNodeID("1"); } void TearDown() override { TimeoutFlushManager::GetInstance()->mTimeoutRecords.clear(); } @@ -559,6 +560,63 @@ void BatcherUnittest::TestFlushAllWithGroupBatch() { APSARA_TEST_STREQ("pack_id", res[1][0].mPackIdPrefix.data()); } +void BatcherUnittest::TestMetric() { + { + DefaultFlushStrategyOptions strategy; + strategy.mMaxCnt = 2; + strategy.mMaxSizeBytes = 1000; + strategy.mTimeoutSecs = 3; + + Batcher<> batch; + batch.Init(Json::Value(), sFlusher.get(), strategy, false); + + PipelineEventGroup g = CreateEventGroup(3); + auto groupSize = g.DataSize(); + auto batchSize = groupSize - 2 * g.GetEvents()[0]->DataSize(); + + vector res; + batch.Add(std::move(g), res); + APSARA_TEST_EQUAL(5U, batch.mMetricsRecordRef->GetLabels()->size()); + APSARA_TEST_TRUE(batch.mMetricsRecordRef.HasLabel(METRIC_LABEL_PROJECT, "")); + APSARA_TEST_TRUE(batch.mMetricsRecordRef.HasLabel(METRIC_LABEL_CONFIG_NAME, "test_config")); + APSARA_TEST_TRUE(batch.mMetricsRecordRef.HasLabel(METRIC_LABEL_KEY_COMPONENT_NAME, "batcher")); + APSARA_TEST_TRUE(batch.mMetricsRecordRef.HasLabel(METRIC_LABEL_KEY_FLUSHER_NODE_ID, "1")); + APSARA_TEST_TRUE(batch.mMetricsRecordRef.HasLabel("enable_group_batch", "false")); + APSARA_TEST_EQUAL(3U, batch.mInEventsCnt->GetValue()); + APSARA_TEST_EQUAL(groupSize, batch.mInGroupDataSizeBytes->GetValue()); + APSARA_TEST_EQUAL(2U, batch.mOutEventsCnt->GetValue()); + APSARA_TEST_EQUAL(1U, batch.mEventBatchItemsCnt->GetValue()); + APSARA_TEST_EQUAL(1U, batch.mBufferedGroupsCnt->GetValue()); + APSARA_TEST_EQUAL(1U, batch.mBufferedEventsCnt->GetValue()); + APSARA_TEST_EQUAL(batchSize, batch.mBufferedDataSizeByte->GetValue()); + } + { + DefaultFlushStrategyOptions strategy; + strategy.mMaxCnt = 2; + strategy.mMaxSizeBytes = 1000; + strategy.mTimeoutSecs = 3; + + Batcher<> batch; + batch.Init(Json::Value(), sFlusher.get(), strategy, true); + + PipelineEventGroup g = CreateEventGroup(3); + auto groupSize = g.DataSize(); + auto batchSize = groupSize - 2 * g.GetEvents()[0]->DataSize(); + + vector res; + batch.Add(std::move(g), res); + batch.FlushQueue(0, res[0]); + APSARA_TEST_TRUE(batch.mMetricsRecordRef.HasLabel("enable_group_batch", "true")); + APSARA_TEST_EQUAL(3U, batch.mInEventsCnt->GetValue()); + APSARA_TEST_EQUAL(groupSize, batch.mInGroupDataSizeBytes->GetValue()); + APSARA_TEST_EQUAL(2U, batch.mOutEventsCnt->GetValue()); + APSARA_TEST_EQUAL(1U, batch.mEventBatchItemsCnt->GetValue()); + APSARA_TEST_EQUAL(1U, batch.mBufferedGroupsCnt->GetValue()); + APSARA_TEST_EQUAL(1U, batch.mBufferedEventsCnt->GetValue()); + APSARA_TEST_EQUAL(batchSize, batch.mBufferedDataSizeByte->GetValue()); + } +} + PipelineEventGroup BatcherUnittest::CreateEventGroup(size_t cnt) { PipelineEventGroup group(make_shared()); group.SetTag(string("key"), string("val")); @@ -581,6 +639,7 @@ UNIT_TEST_CASE(BatcherUnittest, TestFlushEventQueueWithGroupBatch) UNIT_TEST_CASE(BatcherUnittest, TestFlushGroupQueue) UNIT_TEST_CASE(BatcherUnittest, TestFlushAllWithoutGroupBatch) UNIT_TEST_CASE(BatcherUnittest, TestFlushAllWithGroupBatch) +UNIT_TEST_CASE(BatcherUnittest, TestMetric) } // namespace logtail diff --git a/core/unittest/compression/CMakeLists.txt b/core/unittest/compression/CMakeLists.txt index 442a8db893..60f1655e5d 100644 --- a/core/unittest/compression/CMakeLists.txt +++ b/core/unittest/compression/CMakeLists.txt @@ -18,6 +18,9 @@ project(compression_unittest) add_executable(compressor_factory_unittest CompressorFactoryUnittest.cpp) target_link_libraries(compressor_factory_unittest ${UT_BASE_TARGET}) +add_executable(compressor_unittest CompressorUnittest.cpp) +target_link_libraries(compressor_unittest ${UT_BASE_TARGET}) + add_executable(lz4_compressor_unittest LZ4CompressorUnittest.cpp) target_link_libraries(lz4_compressor_unittest ${UT_BASE_TARGET}) @@ -26,5 +29,6 @@ target_link_libraries(zstd_compressor_unittest ${UT_BASE_TARGET}) include(GoogleTest) gtest_discover_tests(compressor_factory_unittest) +gtest_discover_tests(compressor_unittest) gtest_discover_tests(lz4_compressor_unittest) gtest_discover_tests(zstd_compressor_unittest) diff --git a/core/unittest/compression/CompressorFactoryUnittest.cpp b/core/unittest/compression/CompressorFactoryUnittest.cpp index 7093a37ff3..84e159ee34 100644 --- a/core/unittest/compression/CompressorFactoryUnittest.cpp +++ b/core/unittest/compression/CompressorFactoryUnittest.cpp @@ -12,7 +12,8 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include "pipeline/compression/CompressorFactory.h" +#include "monitor/MetricConstants.h" +#include "common/compression/CompressorFactory.h" #include "unittest/Unittest.h" using namespace std; @@ -23,56 +24,64 @@ class CompressorFactoryUnittest : public ::testing::Test { public: void TestCreate(); void TestCompressTypeToString(); + void TestMetric(); protected: - void SetUp() { mCtx.SetConfigName("test_config"); } + void SetUp() { + mCtx.SetConfigName("test_config"); + mFlusherId = "1"; + } private: PipelineContext mCtx; + string mFlusherId; }; void CompressorFactoryUnittest::TestCreate() { { // use default - auto compressor - = CompressorFactory::GetInstance()->Create(Json::Value(), mCtx, "test_plugin", CompressType::LZ4); + auto compressor = CompressorFactory::GetInstance()->Create( + Json::Value(), mCtx, "test_plugin", mFlusherId, CompressType::LZ4); APSARA_TEST_EQUAL(CompressType::LZ4, compressor->GetCompressType()); } { // lz4 Json::Value config; config["CompressType"] = "lz4"; - auto compressor = CompressorFactory::GetInstance()->Create(config, mCtx, "test_plugin", CompressType::ZSTD); + auto compressor + = CompressorFactory::GetInstance()->Create(config, mCtx, "test_plugin", mFlusherId, CompressType::ZSTD); APSARA_TEST_EQUAL(CompressType::LZ4, compressor->GetCompressType()); } { // zstd Json::Value config; config["CompressType"] = "zstd"; - auto compressor = CompressorFactory::GetInstance()->Create(config, mCtx, "test_plugin", CompressType::LZ4); + auto compressor + = CompressorFactory::GetInstance()->Create(config, mCtx, "test_plugin", mFlusherId, CompressType::LZ4); APSARA_TEST_EQUAL(CompressType::ZSTD, compressor->GetCompressType()); } { // none Json::Value config; config["CompressType"] = "none"; - auto compressor = CompressorFactory::GetInstance()->Create(config, mCtx, "test_plugin", CompressType::LZ4); + auto compressor + = CompressorFactory::GetInstance()->Create(config, mCtx, "test_plugin", mFlusherId, CompressType::LZ4); APSARA_TEST_EQUAL(nullptr, compressor); } { // unknown Json::Value config; config["CompressType"] = "unknown"; - auto compressor - = CompressorFactory::GetInstance()->Create(Json::Value(), mCtx, "test_plugin", CompressType::LZ4); + auto compressor = CompressorFactory::GetInstance()->Create( + Json::Value(), mCtx, "test_plugin", mFlusherId, CompressType::LZ4); APSARA_TEST_EQUAL(CompressType::LZ4, compressor->GetCompressType()); } { // invalid Json::Value config; config["CompressType"] = 123; - auto compressor - = CompressorFactory::GetInstance()->Create(Json::Value(), mCtx, "test_plugin", CompressType::LZ4); + auto compressor = CompressorFactory::GetInstance()->Create( + Json::Value(), mCtx, "test_plugin", mFlusherId, CompressType::LZ4); APSARA_TEST_EQUAL(CompressType::LZ4, compressor->GetCompressType()); } } @@ -83,8 +92,19 @@ void CompressorFactoryUnittest::TestCompressTypeToString() { APSARA_TEST_STREQ("none", CompressTypeToString(CompressType::NONE).data()); } +void CompressorFactoryUnittest::TestMetric() { + auto compressor + = CompressorFactory::GetInstance()->Create(Json::Value(), mCtx, "test_plugin", mFlusherId, CompressType::LZ4); + APSARA_TEST_EQUAL(4U, compressor->mMetricsRecordRef->GetLabels()->size()); + APSARA_TEST_TRUE(compressor->mMetricsRecordRef.HasLabel(METRIC_LABEL_PROJECT, "")); + APSARA_TEST_TRUE(compressor->mMetricsRecordRef.HasLabel(METRIC_LABEL_CONFIG_NAME, "test_config")); + APSARA_TEST_TRUE(compressor->mMetricsRecordRef.HasLabel(METRIC_LABEL_KEY_COMPONENT_NAME, "compressor")); + APSARA_TEST_TRUE(compressor->mMetricsRecordRef.HasLabel(METRIC_LABEL_KEY_FLUSHER_NODE_ID, mFlusherId)); +} + UNIT_TEST_CASE(CompressorFactoryUnittest, TestCreate) UNIT_TEST_CASE(CompressorFactoryUnittest, TestCompressTypeToString) +UNIT_TEST_CASE(CompressorFactoryUnittest, TestMetric) } // namespace logtail diff --git a/core/unittest/compression/CompressorUnittest.cpp b/core/unittest/compression/CompressorUnittest.cpp new file mode 100644 index 0000000000..54576c63a4 --- /dev/null +++ b/core/unittest/compression/CompressorUnittest.cpp @@ -0,0 +1,79 @@ +// Copyright 2024 iLogtail Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "common/compression/LZ4Compressor.h" +#include "monitor/MetricConstants.h" +#include "unittest/Unittest.h" + +using namespace std; + +namespace logtail { + +class CompressorMock : public Compressor { +public: + CompressorMock(CompressType type) : Compressor(type) {}; + + bool UnCompress(const std::string& input, std::string& output, std::string& errorMsg) override { return true; } + +private: + bool Compress(const std::string& input, std::string& output, std::string& errorMsg) override { + if (input == "failed") { + return false; + } + output = input.substr(0, input.size() / 2); + return true; + } +}; + +class CompressorUnittest : public ::testing::Test { +public: + void TestMetric(); +}; + +void CompressorUnittest::TestMetric() { + { + CompressorMock compressor(CompressType::MOCK); + compressor.SetMetricRecordRef({}); + string input = "hello world"; + string output; + string errorMsg; + compressor.DoCompress(input, output, errorMsg); + APSARA_TEST_EQUAL(1U, compressor.mInItemsCnt->GetValue()); + APSARA_TEST_EQUAL(input.size(), compressor.mInItemSizeBytes->GetValue()); + APSARA_TEST_EQUAL(1U, compressor.mOutItemsCnt->GetValue()); + APSARA_TEST_EQUAL(output.size(), compressor.mOutItemSizeBytes->GetValue()); + APSARA_TEST_EQUAL(0U, compressor.mDiscardedItemsCnt->GetValue()); + APSARA_TEST_EQUAL(0U, compressor.mDiscardedItemSizeBytes->GetValue()); + } + { + CompressorMock compressor(CompressType::MOCK); + compressor.SetMetricRecordRef({}); + string input = "failed"; + string output; + string errorMsg; + compressor.DoCompress(input, output, errorMsg); + APSARA_TEST_EQUAL(1U, compressor.mInItemsCnt->GetValue()); + APSARA_TEST_EQUAL(input.size(), compressor.mInItemSizeBytes->GetValue()); + APSARA_TEST_EQUAL(0U, compressor.mOutItemsCnt->GetValue()); + APSARA_TEST_EQUAL(0U, compressor.mOutItemSizeBytes->GetValue()); + APSARA_TEST_EQUAL(1U, compressor.mDiscardedItemsCnt->GetValue()); + APSARA_TEST_EQUAL(input.size(), compressor.mDiscardedItemSizeBytes->GetValue()); + } +} + +UNIT_TEST_CASE(CompressorUnittest, TestMetric) + +} // namespace logtail + +UNIT_TEST_MAIN diff --git a/core/unittest/compression/LZ4CompressorUnittest.cpp b/core/unittest/compression/LZ4CompressorUnittest.cpp index 7bd62300f7..e852c063cf 100644 --- a/core/unittest/compression/LZ4CompressorUnittest.cpp +++ b/core/unittest/compression/LZ4CompressorUnittest.cpp @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include "pipeline/compression/LZ4Compressor.h" +#include "common/compression/LZ4Compressor.h" #include "unittest/Unittest.h" using namespace std; @@ -29,7 +29,7 @@ void LZ4CompressorUnittest::TestCompress() { string input = "hello world"; string output; string errorMsg; - APSARA_TEST_TRUE(compressor.Compress(input, output, errorMsg)); + APSARA_TEST_TRUE(compressor.DoCompress(input, output, errorMsg)); string decompressed; decompressed.resize(input.size()); APSARA_TEST_TRUE(compressor.UnCompress(output, decompressed, errorMsg)); diff --git a/core/unittest/compression/ZstdCompressorUnittest.cpp b/core/unittest/compression/ZstdCompressorUnittest.cpp index 6362907173..0e165b1987 100644 --- a/core/unittest/compression/ZstdCompressorUnittest.cpp +++ b/core/unittest/compression/ZstdCompressorUnittest.cpp @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include "pipeline/compression/ZstdCompressor.h" +#include "common/compression/ZstdCompressor.h" #include "unittest/Unittest.h" using namespace std; @@ -29,7 +29,7 @@ void ZstdCompressorUnittest::TestCompress() { string input = "hello world"; string output; string errorMsg; - APSARA_TEST_TRUE(compressor.Compress(input, output, errorMsg)); + APSARA_TEST_TRUE(compressor.DoCompress(input, output, errorMsg)); string decompressed; decompressed.resize(input.size()); APSARA_TEST_TRUE(compressor.UnCompress(output, decompressed, errorMsg)); diff --git a/core/unittest/config/CommonConfigProviderUnittest.cpp b/core/unittest/config/CommonConfigProviderUnittest.cpp index d68f049327..e434210b7c 100644 --- a/core/unittest/config/CommonConfigProviderUnittest.cpp +++ b/core/unittest/config/CommonConfigProviderUnittest.cpp @@ -19,7 +19,7 @@ #include "common/FileSystemUtil.h" #include "common/version.h" #include "config/ConfigDiff.h" -#include "config/provider/CommonConfigProvider.h" +#include "config/common_provider/CommonConfigProvider.h" #include "config/watcher/ConfigWatcher.h" #include "gmock/gmock.h" #include "pipeline/PipelineManager.h" diff --git a/core/unittest/flusher/FlusherSLSUnittest.cpp b/core/unittest/flusher/FlusherSLSUnittest.cpp index 52a3d2f510..71e13cbcc6 100644 --- a/core/unittest/flusher/FlusherSLSUnittest.cpp +++ b/core/unittest/flusher/FlusherSLSUnittest.cpp @@ -22,10 +22,7 @@ #ifdef __ENTERPRISE__ #include "config/provider/EnterpriseConfigProvider.h" #endif -#include "pipeline/compression/CompressorFactory.h" -#include "plugin/flusher/sls/FlusherSLS.h" -#include "plugin/flusher/sls/PackIdManager.h" -#include "plugin/flusher/sls/SLSClientManager.h" +#include "common/compression/CompressorFactory.h" #include "pipeline/Pipeline.h" #include "pipeline/PipelineContext.h" #include "pipeline/queue/ExactlyOnceQueueManager.h" @@ -33,6 +30,9 @@ #include "pipeline/queue/QueueKeyManager.h" #include "pipeline/queue/SLSSenderQueueItem.h" #include "pipeline/queue/SenderQueueManager.h" +#include "plugin/flusher/sls/FlusherSLS.h" +#include "plugin/flusher/sls/PackIdManager.h" +#include "plugin/flusher/sls/SLSClientManager.h" #include "unittest/Unittest.h" DECLARE_FLAG_INT32(batch_send_interval); @@ -617,7 +617,7 @@ void FlusherSLSUnittest::TestSend() { APSARA_TEST_EQUAL(cpt, item->mExactlyOnceCheckpoint); auto compressor - = CompressorFactory::GetInstance()->Create(Json::Value(), ctx, "flusher_sls", CompressType::LZ4); + = CompressorFactory::GetInstance()->Create(Json::Value(), ctx, "flusher_sls", "1", CompressType::LZ4); string output, errorMsg; output.resize(item->mRawSize); APSARA_TEST_TRUE(compressor->UnCompress(item->mData, output, errorMsg)); @@ -671,7 +671,7 @@ void FlusherSLSUnittest::TestSend() { APSARA_TEST_EQUAL(checkpoints[0], item->mExactlyOnceCheckpoint); auto compressor - = CompressorFactory::GetInstance()->Create(Json::Value(), ctx, "flusher_sls", CompressType::LZ4); + = CompressorFactory::GetInstance()->Create(Json::Value(), ctx, "flusher_sls", "1", CompressType::LZ4); string output, errorMsg; output.resize(item->mRawSize); APSARA_TEST_TRUE(compressor->UnCompress(item->mData, output, errorMsg)); @@ -750,7 +750,7 @@ void FlusherSLSUnittest::TestSend() { APSARA_TEST_EQUAL(flusher.mLogstore, item->mLogstore); auto compressor - = CompressorFactory::GetInstance()->Create(Json::Value(), ctx, "flusher_sls", CompressType::LZ4); + = CompressorFactory::GetInstance()->Create(Json::Value(), ctx, "flusher_sls", "1", CompressType::LZ4); string output, errorMsg; output.resize(item->mRawSize); APSARA_TEST_TRUE(compressor->UnCompress(item->mData, output, errorMsg)); @@ -844,7 +844,7 @@ void FlusherSLSUnittest::TestSend() { APSARA_TEST_EQUAL(flusher.mLogstore, item->mLogstore); auto compressor - = CompressorFactory::GetInstance()->Create(Json::Value(), ctx, "flusher_sls", CompressType::LZ4); + = CompressorFactory::GetInstance()->Create(Json::Value(), ctx, "flusher_sls", "1", CompressType::LZ4); sls_logs::SlsLogPackageList packageList; APSARA_TEST_TRUE(packageList.ParseFromString(item->mData)); @@ -1017,7 +1017,7 @@ void FlusherSLSUnittest::OnGoPipelineSend() { APSARA_TEST_EQUAL("other_logstore", item->mLogstore); auto compressor - = CompressorFactory::GetInstance()->Create(Json::Value(), ctx, "flusher_sls", CompressType::LZ4); + = CompressorFactory::GetInstance()->Create(Json::Value(), ctx, "flusher_sls", "1", CompressType::LZ4); string output; output.resize(item->mRawSize); APSARA_TEST_TRUE(compressor->UnCompress(item->mData, output, errorMsg)); @@ -1040,7 +1040,7 @@ void FlusherSLSUnittest::OnGoPipelineSend() { flusher.mProject = "test_project"; flusher.mLogstore = "test_logstore"; flusher.mCompressor = CompressorFactory::GetInstance()->Create( - Json::Value(), PipelineContext(), "flusher_sls", CompressType::LZ4); + Json::Value(), PipelineContext(), "flusher_sls", "1", CompressType::LZ4); APSARA_TEST_TRUE(flusher.Send("content", "")); @@ -1060,7 +1060,7 @@ void FlusherSLSUnittest::OnGoPipelineSend() { APSARA_TEST_EQUAL("test_logstore", item->mLogstore); auto compressor - = CompressorFactory::GetInstance()->Create(Json::Value(), ctx, "flusher_sls", CompressType::LZ4); + = CompressorFactory::GetInstance()->Create(Json::Value(), ctx, "flusher_sls", "1", CompressType::LZ4); string output; output.resize(item->mRawSize); string errorMsg; diff --git a/core/unittest/pipeline/PipelineUnittest.cpp b/core/unittest/pipeline/PipelineUnittest.cpp index 98b41a5ff6..4c061b255d 100644 --- a/core/unittest/pipeline/PipelineUnittest.cpp +++ b/core/unittest/pipeline/PipelineUnittest.cpp @@ -18,17 +18,17 @@ #include #include "app_config/AppConfig.h" -#include "pipeline/batch/TimeoutFlushManager.h" #include "common/JsonUtil.h" #include "config/PipelineConfig.h" -#include "plugin/input/InputFeedbackInterfaceRegistry.h" #include "pipeline/Pipeline.h" +#include "pipeline/batch/TimeoutFlushManager.h" #include "pipeline/plugin/PluginRegistry.h" -#include "plugin/processor/inner/ProcessorSplitLogStringNative.h" -#include "plugin/processor/inner/ProcessorSplitMultilineLogStringNative.h" #include "pipeline/queue/BoundedProcessQueue.h" #include "pipeline/queue/ProcessQueueManager.h" #include "pipeline/queue/QueueKeyManager.h" +#include "plugin/input/InputFeedbackInterfaceRegistry.h" +#include "plugin/processor/inner/ProcessorSplitLogStringNative.h" +#include "plugin/processor/inner/ProcessorSplitMultilineLogStringNative.h" #include "unittest/Unittest.h" #include "unittest/plugin/PluginMock.h" @@ -112,6 +112,9 @@ void PipelineUnittest::OnSuccessfulInit() const { APSARA_TEST_EQUAL("test_region", pipeline->GetContext().GetRegion()); APSARA_TEST_EQUAL(QueueKeyManager::GetInstance()->GetKey("test_config-flusher_sls-test_project#test_logstore"), pipeline->GetContext().GetLogstoreKey()); + APSARA_TEST_EQUAL(2U, pipeline->mMetricsRecordRef->GetLabels()->size()); + APSARA_TEST_TRUE(pipeline->mMetricsRecordRef.HasLabel(METRIC_LABEL_CONFIG_NAME, configName)); + APSARA_TEST_TRUE(pipeline->mMetricsRecordRef.HasLabel(METRIC_LABEL_PROJECT, "test_project")); // without sls flusher configStr = R"( @@ -2684,12 +2687,24 @@ void PipelineUnittest::TestProcess() const { processor->Init(Json::Value(), ctx); pipeline.mProcessorLine.emplace_back(std::move(processor)); - vector group; - group.emplace_back(make_shared()); - pipeline.Process(group, 0); + WriteMetrics::GetInstance()->PrepareMetricsRecordRef(pipeline.mMetricsRecordRef, {}); + pipeline.mProcessorsInEventsCnt = pipeline.mMetricsRecordRef.CreateCounter("pipeline_processors_in_events_total"); + pipeline.mProcessorsInGroupsCnt = pipeline.mMetricsRecordRef.CreateCounter("pipeline_processors_in_event_groups_total"); + pipeline.mProcessorsInGroupDataSizeBytes + = pipeline.mMetricsRecordRef.CreateCounter("processors_in_event_group_size_bytes"); + pipeline.mProcessorsTotalDelayMs = pipeline.mMetricsRecordRef.CreateCounter("processors_total_delay_ms"); + + vector groups; + groups.emplace_back(make_shared()); + groups.back().AddLogEvent(); + auto size = groups.back().DataSize(); + pipeline.Process(groups, 0); APSARA_TEST_EQUAL( 1U, static_cast(pipeline.mInputs[0]->GetInnerProcessors()[0]->mPlugin.get())->mCnt); APSARA_TEST_EQUAL(1U, static_cast(pipeline.mProcessorLine[0]->mPlugin.get())->mCnt); + APSARA_TEST_EQUAL(1U, pipeline.mProcessorsInEventsCnt->GetValue()); + APSARA_TEST_EQUAL(1U, pipeline.mProcessorsInGroupsCnt->GetValue()); + APSARA_TEST_EQUAL(size, pipeline.mProcessorsInGroupDataSizeBytes->GetValue()); } void PipelineUnittest::TestSend() const { diff --git a/core/unittest/prometheus/ScrapeConfigUnittest.cpp b/core/unittest/prometheus/ScrapeConfigUnittest.cpp index 8560892071..65e7aeb8da 100644 --- a/core/unittest/prometheus/ScrapeConfigUnittest.cpp +++ b/core/unittest/prometheus/ScrapeConfigUnittest.cpp @@ -17,6 +17,8 @@ class ScrapeConfigUnittest : public testing::Test { void TestAuth(); void TestBasicAuth(); void TestAuthorization(); + void TestScrapeProtocols(); + void TestEnableCompression(); private: void SetUp() override; @@ -55,6 +57,12 @@ void ScrapeConfigUnittest::TestInit() { "scrape_interval": "30s", "scrape_timeout": "30s", "metrics_path": "/metrics", + "scrape_protocols": [ + "PrometheusText0.0.4", + "PrometheusProto", + "OpenMetricsText0.0.1" + ], + "enable_compression": false, "scheme": "http", "basic_auth": { "username": "test_user", @@ -92,8 +100,17 @@ void ScrapeConfigUnittest::TestInit() { APSARA_TEST_EQUAL(scrapeConfig.mMetricsPath, "/metrics"); APSARA_TEST_EQUAL(scrapeConfig.mScheme, "http"); + // scrape protocols + APSARA_TEST_EQUAL(scrapeConfig.mRequestHeaders["Accept"], + "text/plain;version=0.0.4;q=0.4,application/" + "vnd.google.protobuf;proto=io.prometheus.client.MetricFamily;encoding=delimited;q=0.3," + "application/openmetrics-text;version=0.0.1;q=0.2,*/*;q=0.1"); + + // disable compression + APSARA_TEST_EQUAL(scrapeConfig.mRequestHeaders["Accept-Encoding"], "identity"); + // basic auth - APSARA_TEST_EQUAL(scrapeConfig.mAuthHeaders["Authorization"], "Basic dGVzdF91c2VyOnRlc3RfcGFzc3dvcmQ="); + APSARA_TEST_EQUAL(scrapeConfig.mRequestHeaders["Authorization"], "Basic dGVzdF91c2VyOnRlc3RfcGFzc3dvcmQ="); APSARA_TEST_EQUAL(scrapeConfig.mMaxScrapeSizeBytes, 1024 * 1024 * 1024); APSARA_TEST_EQUAL(scrapeConfig.mSampleLimit, 10000); @@ -150,9 +167,9 @@ void ScrapeConfigUnittest::TestBasicAuth() { APSARA_TEST_TRUE(ParseJsonTable(configStr, config, errorMsg)); APSARA_TEST_TRUE(scrapeConfig.Init(config)); - APSARA_TEST_EQUAL(scrapeConfig.mAuthHeaders["Authorization"], "Basic dGVzdF91c2VyOnRlc3RfcGFzc3dvcmQ="); + APSARA_TEST_EQUAL(scrapeConfig.mRequestHeaders["Authorization"], "Basic dGVzdF91c2VyOnRlc3RfcGFzc3dvcmQ="); - scrapeConfig.mAuthHeaders.clear(); + scrapeConfig.mRequestHeaders.clear(); configStr = R"JSON({ "job_name": "test_job", "scrape_interval": "30s", @@ -167,10 +184,10 @@ void ScrapeConfigUnittest::TestBasicAuth() { APSARA_TEST_TRUE(ParseJsonTable(configStr, config, errorMsg)); APSARA_TEST_TRUE(scrapeConfig.Init(config)); - APSARA_TEST_EQUAL(scrapeConfig.mAuthHeaders["Authorization"], "Basic dGVzdF91c2VyOnRlc3RfcGFzc3dvcmQuZmlsZQ=="); + APSARA_TEST_EQUAL(scrapeConfig.mRequestHeaders["Authorization"], "Basic dGVzdF91c2VyOnRlc3RfcGFzc3dvcmQuZmlsZQ=="); // error - scrapeConfig.mAuthHeaders.clear(); + scrapeConfig.mRequestHeaders.clear(); configStr = R"JSON({ "job_name": "test_job", "scrape_interval": "30s", @@ -206,12 +223,12 @@ void ScrapeConfigUnittest::TestAuthorization() { })JSON"; APSARA_TEST_TRUE(ParseJsonTable(configStr, config, errorMsg)); - scrapeConfig.mAuthHeaders.clear(); + scrapeConfig.mRequestHeaders.clear(); APSARA_TEST_TRUE(scrapeConfig.Init(config)); // bearer auth - APSARA_TEST_EQUAL(scrapeConfig.mAuthHeaders["Authorization"], "Bearer test_token"); + APSARA_TEST_EQUAL(scrapeConfig.mRequestHeaders["Authorization"], "Bearer test_token"); - scrapeConfig.mAuthHeaders.clear(); + scrapeConfig.mRequestHeaders.clear(); // default Bearer auth configStr = R"JSON({ @@ -226,15 +243,179 @@ void ScrapeConfigUnittest::TestAuthorization() { })JSON"; APSARA_TEST_TRUE(ParseJsonTable(configStr, config, errorMsg)); - scrapeConfig.mAuthHeaders.clear(); + scrapeConfig.mRequestHeaders.clear(); + APSARA_TEST_TRUE(scrapeConfig.Init(config)); + APSARA_TEST_EQUAL(scrapeConfig.mRequestHeaders["Authorization"], "Bearer " + mKey); +} + +void ScrapeConfigUnittest::TestScrapeProtocols() { + Json::Value config; + ScrapeConfig scrapeConfig; + string errorMsg; + string configStr; + + // default + configStr = R"JSON({ + "job_name": "test_job", + "scrape_interval": "30s", + "scrape_timeout": "30s", + "metrics_path": "/metrics", + "scheme": "http" + })JSON"; + APSARA_TEST_TRUE(ParseJsonTable(configStr, config, errorMsg)); + scrapeConfig.mRequestHeaders.clear(); + APSARA_TEST_TRUE(scrapeConfig.Init(config)); + APSARA_TEST_EQUAL( + "text/plain;version=0.0.4;q=0.5,application/" + "vnd.google.protobuf;proto=io.prometheus.client.MetricFamily;encoding=delimited;q=0.4,application/" + "openmetrics-text;version=0.0.1;q=0.3,application/openmetrics-text;version=1.0.0;q=0.2,*/*;q=0.1", + scrapeConfig.mRequestHeaders["Accept"]); + + // custom quality protocols + configStr = R"JSON({ + "job_name": "test_job", + "scrape_interval": "30s", + "scrape_timeout": "30s", + "metrics_path": "/metrics", + "scheme": "http", + "scrape_protocols": ["PrometheusProto", "OpenMetricsText1.0.0", "PrometheusText0.0.4", "OpenMetricsText0.0.1"] + })JSON"; + APSARA_TEST_TRUE(ParseJsonTable(configStr, config, errorMsg)); + scrapeConfig.mRequestHeaders.clear(); + APSARA_TEST_TRUE(scrapeConfig.Init(config)); + APSARA_TEST_EQUAL( + "application/vnd.google.protobuf;proto=io.prometheus.client.MetricFamily;encoding=delimited;q=0.5," + "application/openmetrics-text;version=1.0.0;q=0.4," + "text/plain;version=0.0.4;q=0.3,application/openmetrics-text;version=0.0.1;q=0.2,*/*;q=0.1", + scrapeConfig.mRequestHeaders["Accept"]); + + // only prometheus0.0.4 protocols + configStr = R"JSON({ + "job_name": "test_job", + "scrape_interval": "30s", + "scrape_timeout": "30s", + "metrics_path": "/metrics", + "scheme": "http", + "scrape_protocols": ["PrometheusText0.0.4"] + })JSON"; + APSARA_TEST_TRUE(ParseJsonTable(configStr, config, errorMsg)); + scrapeConfig.mRequestHeaders.clear(); + APSARA_TEST_TRUE(scrapeConfig.Init(config)); + APSARA_TEST_EQUAL( + "text/plain;version=0.0.4;q=0.2,*/*;q=0.1", + scrapeConfig.mRequestHeaders["Accept"]); + + // Capital error + configStr = R"JSON({ + "job_name": "test_job", + "scrape_interval": "30s", + "scrape_timeout": "30s", + "metrics_path": "/metrics", + "scheme": "http", + "scrape_protocols": ["prometheusproto"] + })JSON"; + + APSARA_TEST_TRUE(ParseJsonTable(configStr, config, errorMsg)); + scrapeConfig.mRequestHeaders.clear(); + APSARA_TEST_FALSE(scrapeConfig.Init(config)); + + // OpenMetricsText1.0.0 duplication error + configStr = R"JSON({ + "job_name": "test_job", + "scrape_interval": "30s", + "scrape_timeout": "30s", + "metrics_path": "/metrics", + "scheme": "http", + "scrape_protocols": ["OpenMetricsText1.0.0", "PrometheusProto", "OpenMetricsText1.0.0"] + })JSON"; + + APSARA_TEST_TRUE(ParseJsonTable(configStr, config, errorMsg)); + scrapeConfig.mRequestHeaders.clear(); + APSARA_TEST_FALSE(scrapeConfig.Init(config)); + + // protocols invalid + configStr = R"JSON({ + "job_name": "test_job", + "scrape_interval": "30s", + "scrape_timeout": "30s", + "metrics_path": "/metrics", + "scheme": "http", + "scrape_protocols": ["OpenMetricsText1.0.0", "PrometheusProto", 999] + })JSON"; + + APSARA_TEST_TRUE(ParseJsonTable(configStr, config, errorMsg)); + scrapeConfig.mRequestHeaders.clear(); + APSARA_TEST_FALSE(scrapeConfig.Init(config)); + + // unknown protocol + configStr = R"JSON({ + "job_name": "test_job", + "scrape_interval": "30s", + "scrape_timeout": "30s", + "metrics_path": "/metrics", + "scheme": "http", + "scrape_protocols": ["OpenMetricsText"] + })JSON"; + + APSARA_TEST_TRUE(ParseJsonTable(configStr, config, errorMsg)); + scrapeConfig.mRequestHeaders.clear(); + APSARA_TEST_FALSE(scrapeConfig.Init(config)); +} + +void ScrapeConfigUnittest::TestEnableCompression() { + Json::Value config; + ScrapeConfig scrapeConfig; + string errorMsg; + string configStr; + + // default + configStr = R"JSON({ + "job_name": "test_job", + "scrape_interval": "30s", + "scrape_timeout": "30s", + "metrics_path": "/metrics", + "scheme": "http" + })JSON"; + APSARA_TEST_TRUE(ParseJsonTable(configStr, config, errorMsg)); + scrapeConfig.mRequestHeaders.clear(); + APSARA_TEST_TRUE(scrapeConfig.Init(config)); + APSARA_TEST_EQUAL("gzip", scrapeConfig.mRequestHeaders["Accept-Encoding"]); + + // disable + configStr = R"JSON({ + "job_name": "test_job", + "scrape_interval": "30s", + "scrape_timeout": "30s", + "metrics_path": "/metrics", + "scheme": "http", + "enable_compression": false + })JSON"; + APSARA_TEST_TRUE(ParseJsonTable(configStr, config, errorMsg)); + scrapeConfig.mRequestHeaders.clear(); + APSARA_TEST_TRUE(scrapeConfig.Init(config)); + APSARA_TEST_EQUAL("identity", scrapeConfig.mRequestHeaders["Accept-Encoding"]); + + // enable + configStr = R"JSON({ + "job_name": "test_job", + "scrape_interval": "30s", + "scrape_timeout": "30s", + "metrics_path": "/metrics", + "scheme": "http", + "enable_compression": true + })JSON"; + APSARA_TEST_TRUE(ParseJsonTable(configStr, config, errorMsg)); + scrapeConfig.mRequestHeaders.clear(); APSARA_TEST_TRUE(scrapeConfig.Init(config)); - APSARA_TEST_EQUAL(scrapeConfig.mAuthHeaders["Authorization"], "Bearer " + mKey); + APSARA_TEST_EQUAL("gzip", scrapeConfig.mRequestHeaders["Accept-Encoding"]); } UNIT_TEST_CASE(ScrapeConfigUnittest, TestInit); UNIT_TEST_CASE(ScrapeConfigUnittest, TestAuth); UNIT_TEST_CASE(ScrapeConfigUnittest, TestBasicAuth); UNIT_TEST_CASE(ScrapeConfigUnittest, TestAuthorization); +UNIT_TEST_CASE(ScrapeConfigUnittest, TestScrapeProtocols); +UNIT_TEST_CASE(ScrapeConfigUnittest, TestEnableCompression); } // namespace logtail diff --git a/core/unittest/prometheus/ScrapeSchedulerUnittest.cpp b/core/unittest/prometheus/ScrapeSchedulerUnittest.cpp index 951507b97b..fd3d540a92 100644 --- a/core/unittest/prometheus/ScrapeSchedulerUnittest.cpp +++ b/core/unittest/prometheus/ScrapeSchedulerUnittest.cpp @@ -56,7 +56,7 @@ class ScrapeSchedulerUnittest : public testing::Test { mScrapeConfig->mScrapeIntervalSeconds = 10; mScrapeConfig->mScrapeTimeoutSeconds = 10; mScrapeConfig->mMetricsPath = "/metrics"; - mScrapeConfig->mAuthHeaders = {{"Authorization", "Bearer xxxxx"}}; + mScrapeConfig->mRequestHeaders = {{"Authorization", "Bearer xxxxx"}}; mHttpResponse.mBody = "# HELP go_gc_duration_seconds A summary of the pause duration of garbage collection cycles.\n" diff --git a/core/unittest/provider/CMakeLists.txt b/core/unittest/provider/CMakeLists.txt new file mode 100644 index 0000000000..e3f7f7abaf --- /dev/null +++ b/core/unittest/provider/CMakeLists.txt @@ -0,0 +1,22 @@ +# Copyright 2024 iLogtail Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +cmake_minimum_required(VERSION 3.22) +project(provider_unittest) + +add_executable(provider_unittest ProviderUnittest.cpp) +target_link_libraries(provider_unittest ${UT_BASE_TARGET}) + +include(GoogleTest) +gtest_discover_tests(provider_unittest) diff --git a/core/unittest/provider/ProviderUnittest.cpp b/core/unittest/provider/ProviderUnittest.cpp new file mode 100644 index 0000000000..7fb455699b --- /dev/null +++ b/core/unittest/provider/ProviderUnittest.cpp @@ -0,0 +1,42 @@ +// Copyright 2023 iLogtail Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "unittest/Unittest.h" +#include "provider/Provider.h" + + +namespace logtail { + +class ProviderUnittest : public testing::Test { +public: + void TestGetRemoteConfigProvider(); + void TestGetProfileSender(); + +}; +void ProviderUnittest::TestGetRemoteConfigProvider() { + auto remoteConfigProviders = GetRemoteConfigProviders(); + APSARA_TEST_GT(remoteConfigProviders.size(), 0); +} + +void ProviderUnittest::TestGetProfileSender() { + auto profileSender = GetProfileSender(); + APSARA_TEST_NOT_EQUAL(nullptr, profileSender); +} + + +UNIT_TEST_CASE(ProviderUnittest, TestGetRemoteConfigProvider) +UNIT_TEST_CASE(ProviderUnittest, TestGetProfileSender) +} // namespace logtail + +UNIT_TEST_MAIN \ No newline at end of file diff --git a/core/unittest/queue/BoundedProcessQueueUnittest.cpp b/core/unittest/queue/BoundedProcessQueueUnittest.cpp index e2228108d5..bd68ccb470 100644 --- a/core/unittest/queue/BoundedProcessQueueUnittest.cpp +++ b/core/unittest/queue/BoundedProcessQueueUnittest.cpp @@ -16,6 +16,7 @@ #include "common/FeedbackInterface.h" #include "models/PipelineEventGroup.h" +#include "pipeline/PipelineManager.h" #include "pipeline/queue/BoundedProcessQueue.h" #include "pipeline/queue/SenderQueue.h" #include "unittest/Unittest.h" @@ -30,6 +31,7 @@ class BoundedProcessQueueUnittest : public testing::Test { void TestPush(); void TestPop(); void TestMetric(); + void TestSetPipeline(); protected: static void SetUpTestCase() { sCtx.SetConfigName("test_config"); } @@ -44,6 +46,7 @@ class BoundedProcessQueueUnittest : public testing::Test { mFeedback1.reset(new FeedbackInterfaceMock); mFeedback2.reset(new FeedbackInterfaceMock); mQueue->SetUpStreamFeedbacks(vector{mFeedback1.get(), mFeedback2.get()}); + mQueue->EnablePop(); } private: @@ -91,9 +94,9 @@ void BoundedProcessQueueUnittest::TestPop() { mQueue->Push(GenerateItem()); // invalidate pop - mQueue->InvalidatePop(); + mQueue->DisablePop(); APSARA_TEST_EQUAL(0, mQueue->Pop(item)); - mQueue->ValidatePop(); + mQueue->EnablePop(); // downstream queues are not valid to push mSenderQueue1->mValidToPush = false; @@ -140,9 +143,30 @@ void BoundedProcessQueueUnittest::TestMetric() { APSARA_TEST_EQUAL(1U, mQueue->mValidToPushFlag->GetValue()); } +void BoundedProcessQueueUnittest::TestSetPipeline() { + auto pipeline = make_shared(); + PipelineManager::GetInstance()->mPipelineNameEntityMap["test_config"] = pipeline; + + auto item1 = GenerateItem(); + auto p1 = item1.get(); + auto pipelineTmp = make_shared(); + item1->mPipeline = pipelineTmp; + + auto item2 = GenerateItem(); + auto p2 = item2.get(); + + mQueue->Push(std::move(item1)); + mQueue->Push(std::move(item2)); + mQueue->SetPipelineForItems("test_config"); + + APSARA_TEST_EQUAL(pipelineTmp, p1->mPipeline); + APSARA_TEST_EQUAL(pipeline, p2->mPipeline); +} + UNIT_TEST_CASE(BoundedProcessQueueUnittest, TestPush) UNIT_TEST_CASE(BoundedProcessQueueUnittest, TestPop) UNIT_TEST_CASE(BoundedProcessQueueUnittest, TestMetric) +UNIT_TEST_CASE(BoundedProcessQueueUnittest, TestSetPipeline) } // namespace logtail diff --git a/core/unittest/queue/CircularProcessQueueUnittest.cpp b/core/unittest/queue/CircularProcessQueueUnittest.cpp index 7b5ef9d0d3..2e6e1411f5 100644 --- a/core/unittest/queue/CircularProcessQueueUnittest.cpp +++ b/core/unittest/queue/CircularProcessQueueUnittest.cpp @@ -15,6 +15,7 @@ #include #include "models/PipelineEventGroup.h" +#include "pipeline/PipelineManager.h" #include "pipeline/queue/CircularProcessQueue.h" #include "pipeline/queue/SenderQueue.h" #include "unittest/Unittest.h" @@ -28,6 +29,7 @@ class CircularProcessQueueUnittest : public testing::Test { void TestPop(); void TestReset(); void TestMetric(); + void TestSetPipeline(); protected: static void SetUpTestCase() { sCtx.SetConfigName("test_config"); } @@ -38,6 +40,7 @@ class CircularProcessQueueUnittest : public testing::Test { mSenderQueue1.reset(new SenderQueue(10, 0, 10, 0, "", sCtx)); mSenderQueue2.reset(new SenderQueue(10, 0, 10, 0, "", sCtx)); mQueue->SetDownStreamQueues(vector{mSenderQueue1.get(), mSenderQueue2.get()}); + mQueue->EnablePop(); } private: @@ -97,9 +100,9 @@ void CircularProcessQueueUnittest::TestPop() { mQueue->Push(GenerateItem(1)); // invalidate pop - mQueue->InvalidatePop(); + mQueue->DisablePop(); APSARA_TEST_FALSE(mQueue->Pop(item)); - mQueue->ValidatePop(); + mQueue->EnablePop(); // downstream queues are not valid to push mSenderQueue1->mValidToPush = false; @@ -169,7 +172,7 @@ void CircularProcessQueueUnittest::TestMetric() { APSARA_TEST_EQUAL(dataSize1 + dataSize2, mQueue->mInItemDataSizeBytes->GetValue()); APSARA_TEST_EQUAL(1U, mQueue->mQueueSize->GetValue()); APSARA_TEST_EQUAL(dataSize2, mQueue->mQueueDataSizeByte->GetValue()); - APSARA_TEST_EQUAL(2U, mQueue->mDroppedEventsCnt->GetValue()); + APSARA_TEST_EQUAL(2U, mQueue->mDiscardedEventsCnt->GetValue()); mQueue->Pop(item); APSARA_TEST_EQUAL(1U, mQueue->mOutItemsCnt->GetValue()); @@ -177,10 +180,31 @@ void CircularProcessQueueUnittest::TestMetric() { APSARA_TEST_EQUAL(0U, mQueue->mQueueDataSizeByte->GetValue()); } +void CircularProcessQueueUnittest::TestSetPipeline() { + auto pipeline = make_shared(); + PipelineManager::GetInstance()->mPipelineNameEntityMap["test_config"] = pipeline; + + auto item1 = GenerateItem(1); + auto p1 = item1.get(); + auto pipelineTmp = make_shared(); + item1->mPipeline = pipelineTmp; + + auto item2 = GenerateItem(1); + auto p2 = item2.get(); + + mQueue->Push(std::move(item1)); + mQueue->Push(std::move(item2)); + mQueue->SetPipelineForItems("test_config"); + + APSARA_TEST_EQUAL(pipelineTmp, p1->mPipeline); + APSARA_TEST_EQUAL(pipeline, p2->mPipeline); +} + UNIT_TEST_CASE(CircularProcessQueueUnittest, TestPush) UNIT_TEST_CASE(CircularProcessQueueUnittest, TestPop) UNIT_TEST_CASE(CircularProcessQueueUnittest, TestReset) UNIT_TEST_CASE(CircularProcessQueueUnittest, TestMetric) +UNIT_TEST_CASE(CircularProcessQueueUnittest, TestSetPipeline) } // namespace logtail diff --git a/core/unittest/queue/ExactlyOnceQueueManagerUnittest.cpp b/core/unittest/queue/ExactlyOnceQueueManagerUnittest.cpp index 5e929eb477..06f1f58b10 100644 --- a/core/unittest/queue/ExactlyOnceQueueManagerUnittest.cpp +++ b/core/unittest/queue/ExactlyOnceQueueManagerUnittest.cpp @@ -15,6 +15,7 @@ #include #include "models/PipelineEventGroup.h" +#include "pipeline/PipelineManager.h" #include "pipeline/queue/ExactlyOnceQueueManager.h" #include "pipeline/queue/QueueKeyManager.h" #include "pipeline/queue/SLSSenderQueueItem.h" @@ -278,16 +279,66 @@ void ExactlyOnceQueueManagerUnittest::TestIsAllSenderQueueEmpty() { void ExactlyOnceQueueManagerUnittest::OnPipelineUpdate() { PipelineContext ctx; ctx.SetConfigName("test_config"); - sManager->CreateOrUpdateQueue(0, 0, ctx, sCheckpoints); sManager->CreateOrUpdateQueue(1, 0, ctx, sCheckpoints); + sManager->CreateOrUpdateQueue(2, 0, ctx, sCheckpoints); - sManager->InvalidatePopProcessQueue("test_config"); - APSARA_TEST_FALSE(sManager->mProcessQueues[0]->mValidToPop); + auto pipeline1 = make_shared(); + PipelineManager::GetInstance()->mPipelineNameEntityMap["test_config"] = pipeline1; + + auto item1 = GenerateProcessItem(); + auto p1 = item1.get(); + sManager->PushProcessQueue(1, std::move(item1)); + + auto item2 = GenerateProcessItem(); + auto p2 = item2.get(); + sManager->PushProcessQueue(2, std::move(item2)); + + sManager->DisablePopProcessQueue("test_config", false); + APSARA_TEST_FALSE(ExactlyOnceQueueManager::GetInstance()->mProcessQueues[1]->mValidToPop); + APSARA_TEST_FALSE(ExactlyOnceQueueManager::GetInstance()->mProcessQueues[2]->mValidToPop); + APSARA_TEST_EQUAL(pipeline1, p1->mPipeline); + APSARA_TEST_EQUAL(pipeline1, p2->mPipeline); + + auto item3 = GenerateProcessItem(); + auto p3 = item3.get(); + sManager->PushProcessQueue(1, std::move(item3)); + + auto item4 = GenerateProcessItem(); + auto p4 = item4.get(); + sManager->PushProcessQueue(2, std::move(item4)); + + auto pipeline2 = make_shared(); + PipelineManager::GetInstance()->mPipelineNameEntityMap["test_config"] = pipeline2; + + sManager->DisablePopProcessQueue("test_config", false); APSARA_TEST_FALSE(sManager->mProcessQueues[1]->mValidToPop); + APSARA_TEST_FALSE(sManager->mProcessQueues[2]->mValidToPop); + APSARA_TEST_EQUAL(pipeline1, p1->mPipeline); + APSARA_TEST_EQUAL(pipeline1, p2->mPipeline); + APSARA_TEST_EQUAL(pipeline2, p3->mPipeline); + APSARA_TEST_EQUAL(pipeline2, p4->mPipeline); + + auto item5 = GenerateProcessItem(); + auto p5 = item5.get(); + sManager->PushProcessQueue(1, std::move(item5)); - sManager->ValidatePopProcessQueue("test_config"); - APSARA_TEST_TRUE(sManager->mProcessQueues[0]->mValidToPop); + auto item6 = GenerateProcessItem(); + auto p6 = item6.get(); + sManager->PushProcessQueue(2, std::move(item6)); + + sManager->DisablePopProcessQueue("test_config", true); + APSARA_TEST_FALSE(sManager->mProcessQueues[1]->mValidToPop); + APSARA_TEST_FALSE(sManager->mProcessQueues[2]->mValidToPop); + APSARA_TEST_EQUAL(pipeline1, p1->mPipeline); + APSARA_TEST_EQUAL(pipeline1, p2->mPipeline); + APSARA_TEST_EQUAL(pipeline2, p3->mPipeline); + APSARA_TEST_EQUAL(pipeline2, p4->mPipeline); + APSARA_TEST_EQUAL(nullptr, p5->mPipeline); + APSARA_TEST_EQUAL(nullptr, p6->mPipeline); + + sManager->EnablePopProcessQueue("test_config"); APSARA_TEST_TRUE(sManager->mProcessQueues[1]->mValidToPop); + APSARA_TEST_TRUE(sManager->mProcessQueues[2]->mValidToPop); } unique_ptr ExactlyOnceQueueManagerUnittest::GenerateProcessItem() { diff --git a/core/unittest/queue/ProcessQueueManagerUnittest.cpp b/core/unittest/queue/ProcessQueueManagerUnittest.cpp index d87bf88e96..21226fe19b 100644 --- a/core/unittest/queue/ProcessQueueManagerUnittest.cpp +++ b/core/unittest/queue/ProcessQueueManagerUnittest.cpp @@ -15,6 +15,7 @@ #include #include "models/PipelineEventGroup.h" +#include "pipeline/PipelineManager.h" #include "pipeline/queue/ExactlyOnceQueueManager.h" #include "pipeline/queue/ProcessQueueManager.h" #include "pipeline/queue/QueueKeyManager.h" @@ -267,17 +268,22 @@ void ProcessQueueManagerUnittest::TestPopItem() { ctx.SetConfigName("test_config_1"); QueueKey key1 = QueueKeyManager::GetInstance()->GetKey("test_config_1"); sProcessQueueManager->CreateOrUpdateBoundedQueue(key1, 0, ctx); + sProcessQueueManager->EnablePop("test_config_1"); ctx.SetConfigName("test_config_2"); QueueKey key2 = QueueKeyManager::GetInstance()->GetKey("test_config_2"); sProcessQueueManager->CreateOrUpdateBoundedQueue(key2, 1, ctx); + sProcessQueueManager->EnablePop("test_config_2"); ctx.SetConfigName("test_config_3"); QueueKey key3 = QueueKeyManager::GetInstance()->GetKey("test_config_3"); sProcessQueueManager->CreateOrUpdateBoundedQueue(key3, 1, ctx); + sProcessQueueManager->EnablePop("test_config_3"); ctx.SetConfigName("test_config_4"); QueueKey key4 = QueueKeyManager::GetInstance()->GetKey("test_config_4"); sProcessQueueManager->CreateOrUpdateBoundedQueue(key4, 1, ctx); + sProcessQueueManager->EnablePop("test_config_4"); ctx.SetConfigName("test_config_5"); ExactlyOnceQueueManager::GetInstance()->CreateOrUpdateQueue(5, 0, ctx, vector(5)); + ExactlyOnceQueueManager::GetInstance()->EnablePopProcessQueue("test_config_5"); sProcessQueueManager->PushQueue(key2, GenerateItem()); sProcessQueueManager->PushQueue(key3, GenerateItem()); @@ -318,10 +324,26 @@ void ProcessQueueManagerUnittest::TestPopItem() { } void ProcessQueueManagerUnittest::TestIsAllQueueEmpty() { - sProcessQueueManager->CreateOrUpdateBoundedQueue(0, 0, sCtx); - sProcessQueueManager->CreateOrUpdateBoundedQueue(1, 1, sCtx); - ExactlyOnceQueueManager::GetInstance()->CreateOrUpdateQueue(2, 0, sCtx, vector(5)); - ExactlyOnceQueueManager::GetInstance()->CreateOrUpdateQueue(3, 2, sCtx, vector(5)); + PipelineContext ctx; + ctx.SetConfigName("test_config_1"); + QueueKey key1 = QueueKeyManager::GetInstance()->GetKey("test_config_1"); + sProcessQueueManager->CreateOrUpdateBoundedQueue(key1, 0, ctx); + sProcessQueueManager->EnablePop("test_config_1"); + + ctx.SetConfigName("test_config_2"); + QueueKey key2 = QueueKeyManager::GetInstance()->GetKey("test_config_2"); + sProcessQueueManager->CreateOrUpdateBoundedQueue(key2, 1, ctx); + sProcessQueueManager->EnablePop("test_config_2"); + + ctx.SetConfigName("test_config_3"); + QueueKey key3 = QueueKeyManager::GetInstance()->GetKey("test_config_3"); + ExactlyOnceQueueManager::GetInstance()->CreateOrUpdateQueue(key3, 0, ctx, vector(5)); + ExactlyOnceQueueManager::GetInstance()->EnablePopProcessQueue("test_config_3"); + + ctx.SetConfigName("test_config_4"); + QueueKey key4 = QueueKeyManager::GetInstance()->GetKey("test_config_4"); + ExactlyOnceQueueManager::GetInstance()->CreateOrUpdateQueue(key4, 2, ctx, vector(5)); + ExactlyOnceQueueManager::GetInstance()->EnablePopProcessQueue("test_config_4"); APSARA_TEST_TRUE(sProcessQueueManager->IsAllQueueEmpty()); // non empty normal process queue @@ -334,7 +356,7 @@ void ProcessQueueManagerUnittest::TestIsAllQueueEmpty() { APSARA_TEST_TRUE(sProcessQueueManager->IsAllQueueEmpty()); // non empty exactly once process queue - sProcessQueueManager->PushQueue(2, GenerateItem()); + sProcessQueueManager->PushQueue(key3, GenerateItem()); APSARA_TEST_FALSE(sProcessQueueManager->IsAllQueueEmpty()); sProcessQueueManager->PopItem(0, item, configName); @@ -342,27 +364,109 @@ void ProcessQueueManagerUnittest::TestIsAllQueueEmpty() { } void ProcessQueueManagerUnittest::OnPipelineUpdate() { - PipelineContext ctx; - ctx.SetConfigName("test_config_1"); + PipelineContext ctx1, ctx2; + ctx1.SetConfigName("test_config_1"); + ctx2.SetConfigName("test_config_2"); QueueKey key = QueueKeyManager::GetInstance()->GetKey("test_config_1"); - sProcessQueueManager->CreateOrUpdateBoundedQueue(key, 0, ctx); - ctx.SetConfigName("test_config_2"); - ExactlyOnceQueueManager::GetInstance()->CreateOrUpdateQueue(1, 0, ctx, vector(5)); - ExactlyOnceQueueManager::GetInstance()->CreateOrUpdateQueue(2, 0, ctx, vector(5)); - - sProcessQueueManager->InvalidatePop("test_config_1"); - APSARA_TEST_FALSE((*sProcessQueueManager->mQueues[key].first)->mValidToPop); - - sProcessQueueManager->InvalidatePop("test_config_2"); - APSARA_TEST_FALSE(ExactlyOnceQueueManager::GetInstance()->mProcessQueues[1]->mValidToPop); - APSARA_TEST_FALSE(ExactlyOnceQueueManager::GetInstance()->mProcessQueues[2]->mValidToPop); - - sProcessQueueManager->ValidatePop("test_config_1"); - APSARA_TEST_TRUE((*sProcessQueueManager->mQueues[key].first)->mValidToPop); - - sProcessQueueManager->ValidatePop("test_config_2"); - APSARA_TEST_TRUE(ExactlyOnceQueueManager::GetInstance()->mProcessQueues[1]->mValidToPop); - APSARA_TEST_TRUE(ExactlyOnceQueueManager::GetInstance()->mProcessQueues[2]->mValidToPop); + sProcessQueueManager->CreateOrUpdateBoundedQueue(key, 0, ctx1); + ExactlyOnceQueueManager::GetInstance()->CreateOrUpdateQueue(1, 0, ctx2, vector(5)); + ExactlyOnceQueueManager::GetInstance()->CreateOrUpdateQueue(2, 0, ctx2, vector(5)); + + auto pipeline1 = make_shared(); + auto pipeline2 = make_shared(); + PipelineManager::GetInstance()->mPipelineNameEntityMap["test_config_1"] = pipeline1; + PipelineManager::GetInstance()->mPipelineNameEntityMap["test_config_2"] = pipeline2; + + { + auto item1 = GenerateItem(); + auto p1 = item1.get(); + sProcessQueueManager->PushQueue(key, std::move(item1)); + + sProcessQueueManager->DisablePop("test_config_1", false); + APSARA_TEST_FALSE((*sProcessQueueManager->mQueues[key].first)->mValidToPop); + APSARA_TEST_EQUAL(pipeline1, p1->mPipeline); + + auto item2 = GenerateItem(); + auto p2 = item2.get(); + sProcessQueueManager->PushQueue(key, std::move(item2)); + + auto pipeline3 = make_shared(); + PipelineManager::GetInstance()->mPipelineNameEntityMap["test_config_1"] = pipeline3; + + sProcessQueueManager->DisablePop("test_config_1", false); + APSARA_TEST_FALSE((*sProcessQueueManager->mQueues[key].first)->mValidToPop); + APSARA_TEST_EQUAL(pipeline1, p1->mPipeline); + APSARA_TEST_EQUAL(pipeline3, p2->mPipeline); + + auto item3 = GenerateItem(); + auto p3 = item3.get(); + sProcessQueueManager->PushQueue(key, std::move(item3)); + + sProcessQueueManager->DisablePop("test_config_1", true); + APSARA_TEST_FALSE((*sProcessQueueManager->mQueues[key].first)->mValidToPop); + APSARA_TEST_EQUAL(pipeline1, p1->mPipeline); + APSARA_TEST_EQUAL(pipeline3, p2->mPipeline); + APSARA_TEST_EQUAL(nullptr, p3->mPipeline); + + sProcessQueueManager->EnablePop("test_config_1"); + APSARA_TEST_TRUE((*sProcessQueueManager->mQueues[key].first)->mValidToPop); + } + { + auto item1 = GenerateItem(); + auto p1 = item1.get(); + sProcessQueueManager->PushQueue(1, std::move(item1)); + + auto item2 = GenerateItem(); + auto p2 = item2.get(); + sProcessQueueManager->PushQueue(2, std::move(item2)); + + sProcessQueueManager->DisablePop("test_config_2", false); + APSARA_TEST_FALSE(ExactlyOnceQueueManager::GetInstance()->mProcessQueues[1]->mValidToPop); + APSARA_TEST_FALSE(ExactlyOnceQueueManager::GetInstance()->mProcessQueues[2]->mValidToPop); + APSARA_TEST_EQUAL(pipeline2, p1->mPipeline); + APSARA_TEST_EQUAL(pipeline2, p2->mPipeline); + + auto item3 = GenerateItem(); + auto p3 = item3.get(); + sProcessQueueManager->PushQueue(1, std::move(item3)); + + auto item4 = GenerateItem(); + auto p4 = item4.get(); + sProcessQueueManager->PushQueue(2, std::move(item4)); + + auto pipeline3 = make_shared(); + PipelineManager::GetInstance()->mPipelineNameEntityMap["test_config_2"] = pipeline3; + + sProcessQueueManager->DisablePop("test_config_2", false); + APSARA_TEST_FALSE(ExactlyOnceQueueManager::GetInstance()->mProcessQueues[1]->mValidToPop); + APSARA_TEST_FALSE(ExactlyOnceQueueManager::GetInstance()->mProcessQueues[2]->mValidToPop); + APSARA_TEST_EQUAL(pipeline2, p1->mPipeline); + APSARA_TEST_EQUAL(pipeline2, p2->mPipeline); + APSARA_TEST_EQUAL(pipeline3, p3->mPipeline); + APSARA_TEST_EQUAL(pipeline3, p4->mPipeline); + + auto item5 = GenerateItem(); + auto p5 = item5.get(); + sProcessQueueManager->PushQueue(1, std::move(item5)); + + auto item6 = GenerateItem(); + auto p6 = item6.get(); + sProcessQueueManager->PushQueue(2, std::move(item6)); + + sProcessQueueManager->DisablePop("test_config_2", true); + APSARA_TEST_FALSE(ExactlyOnceQueueManager::GetInstance()->mProcessQueues[1]->mValidToPop); + APSARA_TEST_FALSE(ExactlyOnceQueueManager::GetInstance()->mProcessQueues[2]->mValidToPop); + APSARA_TEST_EQUAL(pipeline2, p1->mPipeline); + APSARA_TEST_EQUAL(pipeline2, p2->mPipeline); + APSARA_TEST_EQUAL(pipeline3, p3->mPipeline); + APSARA_TEST_EQUAL(pipeline3, p4->mPipeline); + APSARA_TEST_EQUAL(nullptr, p5->mPipeline); + APSARA_TEST_EQUAL(nullptr, p6->mPipeline); + + sProcessQueueManager->EnablePop("test_config_2"); + APSARA_TEST_TRUE(ExactlyOnceQueueManager::GetInstance()->mProcessQueues[1]->mValidToPop); + APSARA_TEST_TRUE(ExactlyOnceQueueManager::GetInstance()->mProcessQueues[2]->mValidToPop); + } } UNIT_TEST_CASE(ProcessQueueManagerUnittest, TestUpdateSameTypeQueue) diff --git a/core/unittest/reader/ForceReadUnittest.cpp b/core/unittest/reader/ForceReadUnittest.cpp index aac74460f1..fa99c10eec 100644 --- a/core/unittest/reader/ForceReadUnittest.cpp +++ b/core/unittest/reader/ForceReadUnittest.cpp @@ -119,6 +119,7 @@ class ForceReadUnittest : public testing::Test { FileServer::GetInstance()->AddFileReaderConfig(mConfigName, &readerOpts, &ctx); FileServer::GetInstance()->AddMultilineConfig(mConfigName, &multilineOpts, &ctx); ProcessQueueManager::GetInstance()->CreateOrUpdateBoundedQueue(0, 0, ctx); + ProcessQueueManager::GetInstance()->EnablePop(mConfigName); } void TearDown() override { remove(utf8File.c_str()); } diff --git a/core/unittest/route/RouterUnittest.cpp b/core/unittest/route/RouterUnittest.cpp index bee7bb1803..3e42ff722a 100644 --- a/core/unittest/route/RouterUnittest.cpp +++ b/core/unittest/route/RouterUnittest.cpp @@ -13,6 +13,7 @@ // limitations under the License. #include "common/JsonUtil.h" +#include "monitor/MetricConstants.h" #include "pipeline/Pipeline.h" #include "pipeline/route/Router.h" #include "unittest/Unittest.h" @@ -25,6 +26,10 @@ class RouterUnittest : public testing::Test { public: void TestInit(); void TestRoute(); + void TestMetric(); + +protected: + void SetUp() override { ctx.SetConfigName("test_config"); } private: PipelineContext ctx; @@ -124,8 +129,44 @@ void RouterUnittest::TestRoute() { } } +void RouterUnittest::TestMetric() { + Json::Value configJson; + string errorMsg; + string configStr = R"( + [ + { + "Type": "event_type", + "Value": "log" + } + ] + )"; + APSARA_TEST_TRUE(ParseJsonTable(configStr, configJson, errorMsg)); + vector> configs; + for (Json::Value::ArrayIndex i = 0; i < configJson.size(); ++i) { + configs.emplace_back(i, &configJson[i]); + } + configs.emplace_back(configJson.size(), nullptr); + + Router router; + router.Init(configs, ctx); + + APSARA_TEST_EQUAL(3U, router.mMetricsRecordRef->GetLabels()->size()); + APSARA_TEST_TRUE(router.mMetricsRecordRef.HasLabel(METRIC_LABEL_PROJECT, "")); + APSARA_TEST_TRUE(router.mMetricsRecordRef.HasLabel(METRIC_LABEL_CONFIG_NAME, "test_config")); + APSARA_TEST_TRUE(router.mMetricsRecordRef.HasLabel(METRIC_LABEL_KEY_COMPONENT_NAME, "router")); + + PipelineEventGroup g(make_shared()); + g.AddLogEvent(); + auto size = g.DataSize(); + router.Route(g); + + APSARA_TEST_EQUAL(1U, router.mInEventsCnt->GetValue()); + APSARA_TEST_EQUAL(size, router.mInGroupDataSizeBytes->GetValue()); +} + UNIT_TEST_CASE(RouterUnittest, TestInit) UNIT_TEST_CASE(RouterUnittest, TestRoute) +UNIT_TEST_CASE(RouterUnittest, TestMetric) } // namespace logtail diff --git a/core/unittest/serializer/CMakeLists.txt b/core/unittest/serializer/CMakeLists.txt index 11c1436ee5..b471e3b81a 100644 --- a/core/unittest/serializer/CMakeLists.txt +++ b/core/unittest/serializer/CMakeLists.txt @@ -15,8 +15,12 @@ cmake_minimum_required(VERSION 3.22) project(serializer_unittest) +add_executable(serializer_unittest SerializerUnittest.cpp) +target_link_libraries(serializer_unittest ${UT_BASE_TARGET}) + add_executable(sls_serializer_unittest SLSSerializerUnittest.cpp) target_link_libraries(sls_serializer_unittest ${UT_BASE_TARGET}) include(GoogleTest) +gtest_discover_tests(serializer_unittest) gtest_discover_tests(sls_serializer_unittest) diff --git a/core/unittest/serializer/SLSSerializerUnittest.cpp b/core/unittest/serializer/SLSSerializerUnittest.cpp index a3695df4d1..7d57cef290 100644 --- a/core/unittest/serializer/SLSSerializerUnittest.cpp +++ b/core/unittest/serializer/SLSSerializerUnittest.cpp @@ -52,7 +52,7 @@ void SLSSerializerUnittest::TestSerializeEventGroup() { { // nano second disabled, and set string res, errorMsg; - APSARA_TEST_TRUE(serializer.Serialize(CreateBatchedEvents(false), res, errorMsg)); + APSARA_TEST_TRUE(serializer.DoSerialize(CreateBatchedEvents(false), res, errorMsg)); sls_logs::LogGroup logGroup; APSARA_TEST_TRUE(logGroup.ParseFromString(res)); APSARA_TEST_EQUAL(1, logGroup.logs_size()); @@ -72,7 +72,7 @@ void SLSSerializerUnittest::TestSerializeEventGroup() { // nano second enabled, and set const_cast(mCtx.GetGlobalConfig()).mEnableTimestampNanosecond = true; string res, errorMsg; - APSARA_TEST_TRUE(serializer.Serialize(CreateBatchedEvents(true), res, errorMsg)); + APSARA_TEST_TRUE(serializer.DoSerialize(CreateBatchedEvents(true), res, errorMsg)); sls_logs::LogGroup logGroup; APSARA_TEST_TRUE(logGroup.ParseFromString(res)); APSARA_TEST_EQUAL(1234567890U, logGroup.logs(0).time()); @@ -83,7 +83,7 @@ void SLSSerializerUnittest::TestSerializeEventGroup() { // nano second enabled, not set const_cast(mCtx.GetGlobalConfig()).mEnableTimestampNanosecond = true; string res, errorMsg; - APSARA_TEST_TRUE(serializer.Serialize(CreateBatchedEvents(false), res, errorMsg)); + APSARA_TEST_TRUE(serializer.DoSerialize(CreateBatchedEvents(false), res, errorMsg)); sls_logs::LogGroup logGroup; APSARA_TEST_TRUE(logGroup.ParseFromString(res)); APSARA_TEST_EQUAL(1234567890U, logGroup.logs(0).time()); @@ -94,13 +94,13 @@ void SLSSerializerUnittest::TestSerializeEventGroup() { // log group exceed size limit INT32_FLAG(max_send_log_group_size) = 0; string res, errorMsg; - APSARA_TEST_FALSE(serializer.Serialize(CreateBatchedEvents(true), res, errorMsg)); + APSARA_TEST_FALSE(serializer.DoSerialize(CreateBatchedEvents(true), res, errorMsg)); INT32_FLAG(max_send_log_group_size) = 10 * 1024 * 1024; } { // metric event string res, errorMsg; - APSARA_TEST_TRUE(serializer.Serialize(CreateBatchedMetricEvents(false, 0, false), res, errorMsg)); + APSARA_TEST_TRUE(serializer.DoSerialize(CreateBatchedMetricEvents(false, 0, false), res, errorMsg)); sls_logs::LogGroup logGroup; APSARA_TEST_TRUE(logGroup.ParseFromString(res)); @@ -125,7 +125,7 @@ void SLSSerializerUnittest::TestSerializeEventGroup() { const_cast(mCtx.GetGlobalConfig()).mEnableTimestampNanosecond = true; string res, errorMsg; - APSARA_TEST_TRUE(serializer.Serialize(CreateBatchedMetricEvents(true, 1, false), res, errorMsg)); + APSARA_TEST_TRUE(serializer.DoSerialize(CreateBatchedMetricEvents(true, 1, false), res, errorMsg)); sls_logs::LogGroup logGroup; APSARA_TEST_TRUE(logGroup.ParseFromString(res)); @@ -151,7 +151,7 @@ void SLSSerializerUnittest::TestSerializeEventGroup() { const_cast(mCtx.GetGlobalConfig()).mEnableTimestampNanosecond = true; string res, errorMsg; - APSARA_TEST_TRUE(serializer.Serialize(CreateBatchedMetricEvents(true, 1999999999, false), res, errorMsg)); + APSARA_TEST_TRUE(serializer.DoSerialize(CreateBatchedMetricEvents(true, 1999999999, false), res, errorMsg)); sls_logs::LogGroup logGroup; APSARA_TEST_TRUE(logGroup.ParseFromString(res)); @@ -178,7 +178,7 @@ void SLSSerializerUnittest::TestSerializeEventGroup() { const_cast(mCtx.GetGlobalConfig()).mEnableTimestampNanosecond = true; string res, errorMsg; - APSARA_TEST_TRUE(serializer.Serialize(CreateBatchedMetricEvents(false, 0, true), res, errorMsg)); + APSARA_TEST_TRUE(serializer.DoSerialize(CreateBatchedMetricEvents(false, 0, true), res, errorMsg)); sls_logs::LogGroup logGroup; APSARA_TEST_TRUE(logGroup.ParseFromString(res)); @@ -192,7 +192,7 @@ void SLSSerializerUnittest::TestSerializeEventGroupList() { SLSEventGroupListSerializer serializer(sFlusher.get()); string res, errorMsg; - APSARA_TEST_TRUE(serializer.Serialize(std::move(v), res, errorMsg)); + APSARA_TEST_TRUE(serializer.DoSerialize(std::move(v), res, errorMsg)); sls_logs::SlsLogPackageList logPackageList; APSARA_TEST_TRUE(logPackageList.ParseFromString(res)); APSARA_TEST_EQUAL(1, logPackageList.packages_size()); diff --git a/core/unittest/serializer/SerializerUnittest.cpp b/core/unittest/serializer/SerializerUnittest.cpp new file mode 100644 index 0000000000..ed632e02ae --- /dev/null +++ b/core/unittest/serializer/SerializerUnittest.cpp @@ -0,0 +1,114 @@ +// Copyright 2024 iLogtail Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "monitor/MetricConstants.h" +#include "pipeline/plugin/interface/Flusher.h" +#include "pipeline/serializer/Serializer.h" +#include "unittest/Unittest.h" +#include "unittest/plugin/PluginMock.h" + +using namespace std; + +namespace logtail { + +class SerializerMock : public Serializer { +public: + SerializerMock(Flusher* f) : Serializer(f) {}; + +private: + bool Serialize(BatchedEvents&& p, std::string& res, std::string& errorMsg) override { + if (p.mEvents.empty()) { + return false; + } + res = "result"; + return true; + } +}; + +class SerializerUnittest : public ::testing::Test { +public: + void TestMetric(); + +protected: + static void SetUpTestCase() { sFlusher = make_unique(); } + + void SetUp() override { + mCtx.SetConfigName("test_config"); + sFlusher->SetContext(mCtx); + sFlusher->SetMetricsRecordRef(FlusherMock::sName, "1", "1", "1"); + } + +private: + static unique_ptr sFlusher; + + BatchedEvents CreateBatchedMetricEvents(bool withEvents = true); + + PipelineContext mCtx; +}; + +unique_ptr SerializerUnittest::sFlusher; + +void SerializerUnittest::TestMetric() { + { + SerializerMock serializer(sFlusher.get()); + auto input = CreateBatchedMetricEvents(); + auto inputSize = input.mSizeBytes; + string output; + string errorMsg; + serializer.DoSerialize(std::move(input), output, errorMsg); + APSARA_TEST_EQUAL(1U, serializer.mInItemsCnt->GetValue()); + APSARA_TEST_EQUAL(inputSize, serializer.mInItemSizeBytes->GetValue()); + APSARA_TEST_EQUAL(1U, serializer.mOutItemsCnt->GetValue()); + APSARA_TEST_EQUAL(output.size(), serializer.mOutItemSizeBytes->GetValue()); + APSARA_TEST_EQUAL(0U, serializer.mDiscardedItemsCnt->GetValue()); + APSARA_TEST_EQUAL(0U, serializer.mDiscardedItemSizeBytes->GetValue()); + } + { + SerializerMock serializer(sFlusher.get()); + auto input = CreateBatchedMetricEvents(false); + auto inputSize = input.mSizeBytes; + string output; + string errorMsg; + serializer.DoSerialize(std::move(input), output, errorMsg); + APSARA_TEST_EQUAL(1U, serializer.mInItemsCnt->GetValue()); + APSARA_TEST_EQUAL(inputSize, serializer.mInItemSizeBytes->GetValue()); + APSARA_TEST_EQUAL(0U, serializer.mOutItemsCnt->GetValue()); + APSARA_TEST_EQUAL(0U, serializer.mOutItemSizeBytes->GetValue()); + APSARA_TEST_EQUAL(1U, serializer.mDiscardedItemsCnt->GetValue()); + APSARA_TEST_EQUAL(inputSize, serializer.mDiscardedItemSizeBytes->GetValue()); + } +} + +BatchedEvents SerializerUnittest::CreateBatchedMetricEvents(bool withEvents) { + PipelineEventGroup group(make_shared()); + group.SetTag(string("key"), string("value")); + StringBuffer b = group.GetSourceBuffer()->CopyString(string("pack_id")); + group.SetMetadataNoCopy(EventGroupMetaKey::SOURCE_ID, StringView(b.data, b.size)); + group.SetExactlyOnceCheckpoint(RangeCheckpointPtr(new RangeCheckpoint)); + if (withEvents) { + group.AddLogEvent(); + } + BatchedEvents batch(std::move(group.MutableEvents()), + std::move(group.GetSizedTags()), + std::move(group.GetSourceBuffer()), + group.GetMetadata(EventGroupMetaKey::SOURCE_ID), + std::move(group.GetExactlyOnceCheckpoint())); + return batch; +} + +UNIT_TEST_CASE(SerializerUnittest, TestMetric) + +} // namespace logtail + +UNIT_TEST_MAIN diff --git a/docs/cn/installation/sources/build.md b/docs/cn/installation/sources/build.md index 8eca9dded5..60703e3c0a 100644 --- a/docs/cn/installation/sources/build.md +++ b/docs/cn/installation/sources/build.md @@ -119,3 +119,13 @@ cd 到ilogtail/scripts目录下，执行windows64_build.bat(windows32_build.bat) - PluginAdapter.dll (插件接口) - PluginBase.dll (插件lib) - PluginBase.h + + +## 编译时替换外部模块 +iLogtail通过 Provider 模块暴露出一些拓展点，这些拓展点可以由用户自行实现，并通过编译时CMAKE DPROVIDER_PATH选项替换掉默认的实现。 + +示例： + +```bash +cmake -DPROVIDER_PATH=../../../core_extensions/provider .. +``` \ No newline at end of file diff --git a/plugins/input/canal/input_canal.go b/plugins/input/canal/input_canal.go index 3f0b4d7b3b..0f35d9d52a 100644 --- a/plugins/input/canal/input_canal.go +++ b/plugins/input/canal/input_canal.go @@ -759,9 +759,13 @@ func (sc *ServiceCanal) Start(c pipeline.Collector) error { startPos.Pos = sc.checkpoint.Offset } if nil == gtid && 0 == len(startPos.Name) && !sc.StartFromBegining { - gtid, err = sc.getLatestGTID() - if err != nil { - logger.Warning(sc.context.GetRuntimeContext(), "CANAL_START_ALARM", "Call getLatestGTID failed, error", err) + if sc.isGTIDEnabled { + gtid, err = sc.getLatestGTID() + if err != nil { + logger.Warning(sc.context.GetRuntimeContext(), "CANAL_START_ALARM", "Call getLatestGTID failed, error", err) + } + } + if gtid == nil { startPos = sc.GetBinlogLatestPos() } logger.Infof(sc.context.GetRuntimeContext(), "Get latest checkpoint GTID: %v Position: %v", gtid, startPos)