From 997ec7463716e68873273e488eb3b6aa0fe8d593 Mon Sep 17 00:00:00 2001 From: Gavin Halliday Date: Fri, 15 Dec 2023 12:52:42 +0000 Subject: [PATCH 1/3] HPCC-31025 Add support for multiple trace exporters Signed-off-by: Gavin Halliday --- helm/examples/tracing/README.md | 37 ++--- helm/hpcc/values.schema.json | 14 +- system/jlib/jtrace.cpp | 271 +++++++++++++++++--------------- 3 files changed, 163 insertions(+), 159 deletions(-) diff --git a/helm/examples/tracing/README.md b/helm/examples/tracing/README.md index 0c9676891f0..a284c51bf6e 100644 --- a/helm/examples/tracing/README.md +++ b/helm/examples/tracing/README.md @@ -13,24 +13,23 @@ All configuration options detailed here are part of the HPCC Systems Helm chart, - optAlwaysCreateTraceIds - If true components generate trace/span ids if none are provided by the remote caller. - exporter - Defines The type of exporter in charge of forwarding span data to target back-end - type - (default: JLOG) "OTLP-HTTP" | "OTLP-GRPC" | "OS" | "JLOG" | "NONE" - - JLOG - - logSpanDetails - Log span details such as description, status, kind - - logParentInfo - Log the span's parent info such as ParentSpanId, and TraceState - - logAttributes - Log the span's attributes - - logEvents - Log the span's events - - logLinks - Log the span's links - - logResources - Log the span's resources such as telemetry.sdk version, name, language - - OTLP-HTTP - - endpoint - (default localhost:4318) Specifies the target OTLP-HTTP backend - - timeOutSecs - (default 10secs) - - consoleDebug - (default false) - - OTLP-GRPC - - endpoint: (default localhost:4317) The endpoint to export to. By default the OpenTelemetry Collector's default endpoint. - - useSslCredentials - By default when false, uses grpc::InsecureChannelCredentials; If true uses sslCredentialsCACertPath - - sslCredentialsCACertPath - Path to .pem file to be used for SSL encryption. - - timeOutSeconds - (default 10secs) Timeout for grpc deadline -- processor - Controls span processing style. One by one as available, or in batches. - - type - (default: simple) "simple" | "batch" + - JLOG + - logSpanDetails - Log span details such as description, status, kind + - logParentInfo - Log the span's parent info such as ParentSpanId, and TraceState + - logAttributes - Log the span's attributes + - logEvents - Log the span's events + - logLinks - Log the span's links + - logResources - Log the span's resources such as telemetry.sdk version, name, language + - OTLP-HTTP + - endpoint - (default localhost:4318) Specifies the target OTLP-HTTP backend + - timeOutSecs - (default 10secs) + - consoleDebug - (default false) + - OTLP-GRPC + - endpoint: (default localhost:4317) The endpoint to export to. By default the OpenTelemetry Collector's default endpoint. + - useSslCredentials - By default when false, uses grpc::InsecureChannelCredentials; If true uses sslCredentialsCACertPath + - sslCredentialsCACertPath - Path to .pem file to be used for SSL encryption. + - timeOutSeconds - (default 10secs) Timeout for grpc deadline + - batch - If true, trace data is processed in a batch, if false when it is available (simple) ### Sample configuration Below is a sample helm values block directing the HPCC tracing framework to process span information serially, and export the data over OTLP/HTTP protocol to localhost:4318 and output export debug information to console: @@ -41,8 +40,6 @@ global: exporter: type: OTLP-HTTP consoleDebug: true - processor: - type: simple ``` ### Sample configuration command diff --git a/helm/hpcc/values.schema.json b/helm/hpcc/values.schema.json index 71a1c7ff05d..8e640007c67 100644 --- a/helm/hpcc/values.schema.json +++ b/helm/hpcc/values.schema.json @@ -1123,16 +1123,10 @@ "type": "string", "enum": ["OTLP-HTTP", "OTLP-GRPC", "OS", "JLOG", "NONE"], "description": "The type of exporter in charge of forwarding span data to target back-end" - } - } - }, - "processor": { - "type": "object", - "properties": { - "type": { - "type": "string", - "enum": ["batch", "simple"], - "description": "Defines the manner in which trace data is processed - in batches, or simple as available" + }, + "batch": { + "type": "boolean", + "description": "If true, trace data is processed in a batch, if false when it is available" } } } diff --git a/system/jlib/jtrace.cpp b/system/jlib/jtrace.cpp index 714ec9e5f85..bcc65dfec68 100644 --- a/system/jlib/jtrace.cpp +++ b/system/jlib/jtrace.cpp @@ -452,6 +452,8 @@ class CTraceManager : implements ITraceManager, public CInterface void initTracerProviderAndGlobalInternals(const IPropertyTree * traceConfig); void initTracer(const IPropertyTree * traceConfig); void cleanupTracer(); + std::unique_ptr createExporter(const IPropertyTree * exportConfig); + std::unique_ptr createProcessor(const IPropertyTree * exportConfig); public: CTraceManager(const char * componentName, const IPropertyTree * componentConfig, const IPropertyTree * globalConfig); @@ -1059,161 +1061,172 @@ IProperties * getSpanContext(const ISpan * span) //--------------------------------------------------------------------------------------------------------------------- -void CTraceManager::initTracerProviderAndGlobalInternals(const IPropertyTree * traceConfig) +std::unique_ptr CTraceManager::createExporter(const IPropertyTree * exportConfig) { - //Trace to JLog by default. - std::unique_ptr exporter = JLogSpanExporterFactory::Create(DEFAULT_SPAN_LOG_FLAGS); + assertex(exportConfig); + + StringBuffer exportType; + exportConfig->getProp("@type", exportType); - //Administrators can choose to export trace data to a different backend by specifying the exporter type - if (traceConfig && traceConfig->hasProp("exporter")) + LOG(MCoperatorInfo, "Exporter type: %s", exportType.str()); + if (!exportType.isEmpty()) { - Owned exportConfig = traceConfig->getPropTree("exporter"); - if (exportConfig) + if (stricmp(exportType.str(), "OS")==0) //To stdout/err { - StringBuffer exportType; - exportConfig->getProp("@type", exportType); - LOG(MCoperatorInfo, "Exporter type: %s", exportType.str()); - - if (!exportType.isEmpty()) - { - if (stricmp(exportType.str(), "OS")==0) //To stdout/err - { - exporter = opentelemetry::exporter::trace::OStreamSpanExporterFactory::Create(); - LOG(MCoperatorInfo, "Tracing exporter set OS"); - } - else if (stricmp(exportType.str(), "OTLP")==0 || stricmp(exportType.str(), "OTLP-HTTP")==0) - { - opentelemetry::exporter::otlp::OtlpHttpExporterOptions trace_opts; - const char * endPoint = exportConfig->queryProp("@endpoint"); - if (endPoint) - trace_opts.url = endPoint; + LOG(MCoperatorInfo, "Tracing exporter set OS"); + return opentelemetry::exporter::trace::OStreamSpanExporterFactory::Create(); + } + else if (stricmp(exportType.str(), "OTLP")==0 || stricmp(exportType.str(), "OTLP-HTTP")==0) + { + opentelemetry::exporter::otlp::OtlpHttpExporterOptions trace_opts; + const char * endPoint = exportConfig->queryProp("@endpoint"); + if (endPoint) + trace_opts.url = endPoint; - if (exportConfig->hasProp("@timeOutSecs")) //not sure exactly what this value actually affects - trace_opts.timeout = std::chrono::seconds(exportConfig->getPropInt("@timeOutSecs")); + if (exportConfig->hasProp("@timeOutSecs")) //not sure exactly what this value actually affects + trace_opts.timeout = std::chrono::seconds(exportConfig->getPropInt("@timeOutSecs")); - // Whether to print the status of the exporter in the console - trace_opts.console_debug = exportConfig->getPropBool("@consoleDebug", false); + // Whether to print the status of the exporter in the console + trace_opts.console_debug = exportConfig->getPropBool("@consoleDebug", false); - exporter = opentelemetry::exporter::otlp::OtlpHttpExporterFactory::Create(trace_opts); - LOG(MCoperatorInfo,"Tracing exporter set to OTLP/HTTP to: (%s)", trace_opts.url.c_str()); - } - else if (stricmp(exportType.str(), "OTLP-GRPC")==0) - { - namespace otlp = opentelemetry::exporter::otlp; + LOG(MCoperatorInfo,"Tracing exporter set to OTLP/HTTP to: (%s)", trace_opts.url.c_str()); + return opentelemetry::exporter::otlp::OtlpHttpExporterFactory::Create(trace_opts); + } + else if (stricmp(exportType.str(), "OTLP-GRPC")==0) + { + namespace otlp = opentelemetry::exporter::otlp; - otlp::OtlpGrpcExporterOptions opts; + otlp::OtlpGrpcExporterOptions opts; - const char * endPoint = exportConfig->queryProp("@endpoint"); - if (endPoint) - opts.endpoint = endPoint; + const char * endPoint = exportConfig->queryProp("@endpoint"); + if (endPoint) + opts.endpoint = endPoint; - opts.use_ssl_credentials = exportConfig->getPropBool("@useSslCredentials", false); + opts.use_ssl_credentials = exportConfig->getPropBool("@useSslCredentials", false); - if (opts.use_ssl_credentials) - { - StringBuffer sslCACertPath; - exportConfig->getProp("@sslCredentialsCACertPath", sslCACertPath); - opts.ssl_credentials_cacert_path = sslCACertPath.str(); - } + if (opts.use_ssl_credentials) + { + StringBuffer sslCACertPath; + exportConfig->getProp("@sslCredentialsCACertPath", sslCACertPath); + opts.ssl_credentials_cacert_path = sslCACertPath.str(); + } - if (exportConfig->hasProp("@timeOutSecs")) //grpc deadline timeout in seconds - opts.timeout = std::chrono::seconds(exportConfig->getPropInt("@timeOutSecs")); + if (exportConfig->hasProp("@timeOutSecs")) //grpc deadline timeout in seconds + opts.timeout = std::chrono::seconds(exportConfig->getPropInt("@timeOutSecs")); - exporter = otlp::OtlpGrpcExporterFactory::Create(opts); - LOG(MCoperatorInfo, "Tracing exporter set to OTLP/GRPC to: (%s)", opts.endpoint.c_str()); - } - else if (stricmp(exportType.str(), "JLOG")==0) - { - StringBuffer logFlagsStr; - SpanLogFlags logFlags = SpanLogFlags::LogNone; - - if (exportConfig->getPropBool("@logSpanDetails", false)) - { - logFlags |= SpanLogFlags::LogSpanDetails; - logFlagsStr.append(" LogDetails "); - } - if (exportConfig->getPropBool("@logParentInfo", false)) - { - logFlags |= SpanLogFlags::LogParentInfo; - logFlagsStr.append(" LogParentInfo "); - } - if (exportConfig->getPropBool("@logAttributes", false)) - { - logFlags |= SpanLogFlags::LogAttributes; - logFlagsStr.append(" LogAttributes "); - } - if (exportConfig->getPropBool("@logEvents", false)) - { - logFlags |= SpanLogFlags::LogEvents; - logFlagsStr.append(" LogEvents "); - } - if (exportConfig->getPropBool("@logLinks", false)) - { - logFlags |= SpanLogFlags::LogLinks; - logFlagsStr.append(" LogLinks "); - } - if (exportConfig->getPropBool("@logResources", false)) - { - logFlags |= SpanLogFlags::LogResources; - logFlagsStr.append(" LogLinks "); - } + LOG(MCoperatorInfo, "Tracing exporter set to OTLP/GRPC to: (%s)", opts.endpoint.c_str()); + return otlp::OtlpGrpcExporterFactory::Create(opts); + } + else if (stricmp(exportType.str(), "JLOG")==0) + { + StringBuffer logFlagsStr; + SpanLogFlags logFlags = SpanLogFlags::LogNone; - //if no log feature flags provided, use default - if (logFlags == SpanLogFlags::LogNone) - logFlags = DEFAULT_SPAN_LOG_FLAGS; + if (exportConfig->getPropBool("@logSpanDetails", false)) + { + logFlags |= SpanLogFlags::LogSpanDetails; + logFlagsStr.append(" LogDetails "); + } + if (exportConfig->getPropBool("@logParentInfo", false)) + { + logFlags |= SpanLogFlags::LogParentInfo; + logFlagsStr.append(" LogParentInfo "); + } + if (exportConfig->getPropBool("@logAttributes", false)) + { + logFlags |= SpanLogFlags::LogAttributes; + logFlagsStr.append(" LogAttributes "); + } + if (exportConfig->getPropBool("@logEvents", false)) + { + logFlags |= SpanLogFlags::LogEvents; + logFlagsStr.append(" LogEvents "); + } + if (exportConfig->getPropBool("@logLinks", false)) + { + logFlags |= SpanLogFlags::LogLinks; + logFlagsStr.append(" LogLinks "); + } + if (exportConfig->getPropBool("@logResources", false)) + { + logFlags |= SpanLogFlags::LogResources; + logFlagsStr.append(" LogLinks "); + } - exporter = JLogSpanExporterFactory::Create(logFlags); + //if no log feature flags provided, use default + if (logFlags == SpanLogFlags::LogNone) + logFlags = DEFAULT_SPAN_LOG_FLAGS; - LOG(MCoperatorInfo, "Tracing exporter set to JLog: logFlags( LogAttributes LogParentInfo %s)", logFlagsStr.str()); - } - else if (stricmp(exportType.str(), "Prometheus")==0) - LOG(MCoperatorInfo, "Tracing to Prometheus currently not supported"); - else if (stricmp(exportType.str(), "NONE")==0) - { - exporter = NoopSpanExporterFactory::Create(); - LOG(MCoperatorInfo, "Tracing exporter set to 'NONE', no trace exporting will be performed"); - } - else - LOG(MCoperatorInfo, "Tracing exporter type not supported: '%s', JLog trace exporting will be performed", exportType.str()); - } - else - LOG(MCoperatorInfo, "Tracing exporter type not specified"); + LOG(MCoperatorInfo, "Tracing exporter set to JLog: logFlags( LogAttributes LogParentInfo %s)", logFlagsStr.str()); + return JLogSpanExporterFactory::Create(logFlags); } + else if (stricmp(exportType.str(), "Prometheus")==0) + LOG(MCoperatorInfo, "Tracing to Prometheus currently not supported"); + else if (stricmp(exportType.str(), "NONE")==0) + { + LOG(MCoperatorInfo, "Tracing exporter set to 'NONE', no trace exporting will be performed"); + return NoopSpanExporterFactory::Create(); + } + else + LOG(MCoperatorInfo, "Tracing exporter type not supported: '%s', JLog trace exporting will be performed", exportType.str()); } + else + LOG(MCoperatorInfo, "Tracing exporter type not specified"); + return nullptr; +} - //Administrator can choose to process spans in batches or one at a time - //Default: SimpleSpanProcesser sends spans one by one to an exporter. - std::unique_ptr processor = opentelemetry::sdk::trace::SimpleSpanProcessorFactory::Create(std::move(exporter)); - if (traceConfig && traceConfig->hasProp("processor/@type")) - { - StringBuffer processorType; - bool foundProcessorType = traceConfig->getProp("processor/@type", processorType); +std::unique_ptr CTraceManager::createProcessor(const IPropertyTree * exportConfig) +{ + auto exporter = createExporter(exportConfig); + if (!exporter) + return nullptr; + + if (exportConfig->getPropBool("@batch", false)) + { + //Groups several spans together, before sending them to an exporter. + //MORE: These options should be configurable + opentelemetry::v1::sdk::trace::BatchSpanProcessorOptions options; //size_t max_queue_size = 2048; + //The time interval between two consecutive exports + //std::chrono::milliseconds(5000); + //The maximum batch size of every export. It must be smaller or + //equal to max_queue_size. + //size_t max_export_batch_size = 512 + return opentelemetry::sdk::trace::BatchSpanProcessorFactory::Create(std::move(exporter), options); + } - if (foundProcessorType && strcmp("batch", processorType.str())==0) - { - //Groups several spans together, before sending them to an exporter. - //These options should be configurable - opentelemetry::v1::sdk::trace::BatchSpanProcessorOptions options; //size_t max_queue_size = 2048; - //The time interval between two consecutive exports - //std::chrono::milliseconds(5000); - //The maximum batch size of every export. It must be smaller or - //equal to max_queue_size. - //size_t max_export_batch_size = 512 - processor = opentelemetry::sdk::trace::BatchSpanProcessorFactory::Create(std::move(exporter), options); - LOG(MCoperatorInfo, "OpenTel tracing using batch Span Processor"); - } - else if (foundProcessorType && strcmp("simple", processorType.str())==0) + return opentelemetry::sdk::trace::SimpleSpanProcessorFactory::Create(std::move(exporter)); +} + +void CTraceManager::initTracerProviderAndGlobalInternals(const IPropertyTree * traceConfig) +{ + std::vector> processors; + if (traceConfig) + { + //Administrators can choose to export trace data to a different backend by specifying the exporter type + IPropertyTree * exportConfig = traceConfig->queryPropTree("exporter"); + if (exportConfig) { - LOG(MCoperatorInfo, "OpenTel tracing using batch simple Processor"); + std::unique_ptr processor = createProcessor(exportConfig); + if (processor) + processors.push_back(std::move(processor)); } - else + + Owned iter = traceConfig->getElements("exporters"); + ForEach(*iter) { - LOG(MCoperatorInfo, "OpenTel tracing detected invalid processor type: '%s'", processorType.str()); + IPropertyTree & curExporter = iter->query(); + std::unique_ptr processor = createProcessor(&curExporter); + if (processor) + processors.push_back(std::move(processor)); } } - std::vector> processors; - processors.push_back(std::move(processor)); + if (processors.empty()) + { + //Default to tracing to the log file if no exporters are specified + std::unique_ptr exporter = JLogSpanExporterFactory::Create(DEFAULT_SPAN_LOG_FLAGS); + processors.push_back(opentelemetry::sdk::trace::SimpleSpanProcessorFactory::Create(std::move(exporter))); + } // Default is an always-on sampler. std::shared_ptr context = From 5c36a2578335ed59e7ddd37dc1ec3c48dd97ba03 Mon Sep 17 00:00:00 2001 From: Gavin Halliday Date: Tue, 19 Dec 2023 09:56:33 +0000 Subject: [PATCH 2/3] Changes following review Signed-off-by: Gavin Halliday --- helm/examples/tracing/README.md | 14 ++++++----- helm/hpcc/values.schema.json | 42 +++++++++++++++++++++++---------- system/jlib/jtrace.cpp | 41 +++++++++++++------------------- system/jlib/jutil.hpp | 7 ++++++ testing/helm/tests/tracing.yaml | 14 +++++++++++ 5 files changed, 75 insertions(+), 43 deletions(-) create mode 100644 testing/helm/tests/tracing.yaml diff --git a/helm/examples/tracing/README.md b/helm/examples/tracing/README.md index a284c51bf6e..f2beff33d89 100644 --- a/helm/examples/tracing/README.md +++ b/helm/examples/tracing/README.md @@ -11,25 +11,27 @@ All configuration options detailed here are part of the HPCC Systems Helm chart, - disabled - (default: false) disables tracking and reporting of internal traces and spans - alwaysCreateGlobalIds - If true, assign newly created global ID to any requests that do not supply one. - optAlwaysCreateTraceIds - If true components generate trace/span ids if none are provided by the remote caller. -- exporter - Defines The type of exporter in charge of forwarding span data to target back-end - - type - (default: JLOG) "OTLP-HTTP" | "OTLP-GRPC" | "OS" | "JLOG" | "NONE" - - JLOG +- createDefaultLogExporter - If true, creates a trace exporter outputting to the log using the default options +- exporters: - Defines a list of exporters in charge of forwarding span data to target back-end + - type - "OTLP-HTTP" | "OTLP-GRPC" | "OS" | "JLOG" + - "JLOG" - logSpanDetails - Log span details such as description, status, kind - logParentInfo - Log the span's parent info such as ParentSpanId, and TraceState - logAttributes - Log the span's attributes - logEvents - Log the span's events - logLinks - Log the span's links - logResources - Log the span's resources such as telemetry.sdk version, name, language - - OTLP-HTTP + - "OTLP-HTTP" - endpoint - (default localhost:4318) Specifies the target OTLP-HTTP backend - timeOutSecs - (default 10secs) - consoleDebug - (default false) - - OTLP-GRPC + - "OTLP-GRPC" - endpoint: (default localhost:4317) The endpoint to export to. By default the OpenTelemetry Collector's default endpoint. - useSslCredentials - By default when false, uses grpc::InsecureChannelCredentials; If true uses sslCredentialsCACertPath - sslCredentialsCACertPath - Path to .pem file to be used for SSL encryption. - timeOutSeconds - (default 10secs) Timeout for grpc deadline - - batch - If true, trace data is processed in a batch, if false when it is available (simple) + - batch: + - enabled - If true, trace data is processed in a batch, if false when it is available (simple) ### Sample configuration Below is a sample helm values block directing the HPCC tracing framework to process span information serially, and export the data over OTLP/HTTP protocol to localhost:4318 and output export debug information to console: diff --git a/helm/hpcc/values.schema.json b/helm/hpcc/values.schema.json index 8e640007c67..45a4a083cec 100644 --- a/helm/hpcc/values.schema.json +++ b/helm/hpcc/values.schema.json @@ -1116,23 +1116,41 @@ "type": "boolean", "description": "If true, components generate trace/span ids if none are provided by the remote caller" }, - "exporter": { - "type": "object", - "properties": { - "type": { - "type": "string", - "enum": ["OTLP-HTTP", "OTLP-GRPC", "OS", "JLOG", "NONE"], - "description": "The type of exporter in charge of forwarding span data to target back-end" - }, - "batch": { - "type": "boolean", - "description": "If true, trace data is processed in a batch, if false when it is available" - } + "createDefaultLogExporter": { + "type": "boolean", + "description": "If true, creates a trace exporter outputting to the log using the default options" + }, + "exporters": { + "type": "array", + "description": "List of trace exporters", + "items": { + "$ref": "#/definitions/traceExporter" } } }, "additionalProperties": { "type": ["integer", "string", "boolean"] } }, + "traceExporter": { + "type": "object", + "properties": { + "type": { + "type": "string", + "enum": ["OTLP-HTTP", "OTLP-GRPC", "OS", "JLOG"], + "description": "The type of exporter in charge of forwarding span data to target back-end" + }, + "batch": { + "type": "object", + "properties": { + "enabled": { + "type": "boolean", + "description": "If true, trace data is processed in a batch, if false, as soon as it is available" + } + }, + "additionalProperties": { "type": ["integer", "string", "boolean"] } + }, + "additionalProperties": { "type": ["integer", "string", "boolean"] } + } + }, "compileOption": { "type": "object", "properties": { diff --git a/system/jlib/jtrace.cpp b/system/jlib/jtrace.cpp index bcc65dfec68..47ed0f5b9c8 100644 --- a/system/jlib/jtrace.cpp +++ b/system/jlib/jtrace.cpp @@ -1160,18 +1160,11 @@ std::unique_ptr CTraceManager::createEx LOG(MCoperatorInfo, "Tracing exporter set to JLog: logFlags( LogAttributes LogParentInfo %s)", logFlagsStr.str()); return JLogSpanExporterFactory::Create(logFlags); } - else if (stricmp(exportType.str(), "Prometheus")==0) - LOG(MCoperatorInfo, "Tracing to Prometheus currently not supported"); - else if (stricmp(exportType.str(), "NONE")==0) - { - LOG(MCoperatorInfo, "Tracing exporter set to 'NONE', no trace exporting will be performed"); - return NoopSpanExporterFactory::Create(); - } else - LOG(MCoperatorInfo, "Tracing exporter type not supported: '%s', JLog trace exporting will be performed", exportType.str()); + LOG(MCoperatorWarning, "Tracing exporter type not supported: '%s'", exportType.str()); } else - LOG(MCoperatorInfo, "Tracing exporter type not specified"); + LOG(MCoperatorWarning, "Tracing exporter type not specified"); return nullptr; } @@ -1181,10 +1174,10 @@ std::unique_ptr CTraceManager::createP if (!exporter) return nullptr; - if (exportConfig->getPropBool("@batch", false)) + if (exportConfig->getPropBool("batch/@enabled", false)) { //Groups several spans together, before sending them to an exporter. - //MORE: These options should be configurable + //MORE: These options should be configurable from batch/@option opentelemetry::v1::sdk::trace::BatchSpanProcessorOptions options; //size_t max_queue_size = 2048; //The time interval between two consecutive exports //std::chrono::milliseconds(5000); @@ -1200,17 +1193,13 @@ std::unique_ptr CTraceManager::createP void CTraceManager::initTracerProviderAndGlobalInternals(const IPropertyTree * traceConfig) { std::vector> processors; + + //By default trace spans to the logs in debug builds - so that developers get used to seeing them. + //Default off for release builds to avoid flooding the logs, and because they are likely to use OTLP + bool createDefaultLogExporter = isDebugBuild(); if (traceConfig) { //Administrators can choose to export trace data to a different backend by specifying the exporter type - IPropertyTree * exportConfig = traceConfig->queryPropTree("exporter"); - if (exportConfig) - { - std::unique_ptr processor = createProcessor(exportConfig); - if (processor) - processors.push_back(std::move(processor)); - } - Owned iter = traceConfig->getElements("exporters"); ForEach(*iter) { @@ -1219,11 +1208,13 @@ void CTraceManager::initTracerProviderAndGlobalInternals(const IPropertyTree * t if (processor) processors.push_back(std::move(processor)); } + + createDefaultLogExporter = traceConfig->getPropBool("createDefaultLogExporter", createDefaultLogExporter); } - if (processors.empty()) + if (createDefaultLogExporter) { - //Default to tracing to the log file if no exporters are specified + //Simple option to create logging to the log file - primarily to aid developers. std::unique_ptr exporter = JLogSpanExporterFactory::Create(DEFAULT_SPAN_LOG_FLAGS); processors.push_back(opentelemetry::sdk::trace::SimpleSpanProcessorFactory::Create(std::move(exporter))); } @@ -1245,13 +1236,13 @@ Expected Configuration format: disabled: true #optional - disable OTel tracing alwaysCreateGlobalIds : false #optional - should global ids always be created? alwaysCreateTraceIds #optional - should trace ids always be created? - exporter: #optional - Controls how trace data is exported/reported - type: OTLP #OS|OTLP|Prometheus|JLOG (default: JLOG) + exporters: #optional - Controls how trace data is exported/reported + - type: OTLP #OS|OTLP|Prometheus|JLOG endpoint: "localhost:4317" #exporter specific key/value pairs useSslCredentials: true sslCredentialsCACcert: "ssl-certificate" - processor: #optional - Controls span processing style - type: batch #simple|batch (default: simple) + batch: #optional - Controls span processing style + enabled #is batched processing enabled? */ void CTraceManager::initTracer(const IPropertyTree * traceConfig) { diff --git a/system/jlib/jutil.hpp b/system/jlib/jutil.hpp index a6fa94f2a46..c0fffad2c63 100644 --- a/system/jlib/jutil.hpp +++ b/system/jlib/jutil.hpp @@ -315,6 +315,13 @@ inline constexpr bool isContainerized() { return true; } inline constexpr bool isContainerized() { return false; } #endif +//Same for isDebugBuild() rather than requiring #ifdef _DEBUG +#ifdef _DEBUG +inline constexpr bool isDebugBuild() { return true; } +#else +inline constexpr bool isDebugBuild() { return false; } +#endif + #ifndef arraysize #define arraysize(T) (sizeof(T)/sizeof(*T)) #endif diff --git a/testing/helm/tests/tracing.yaml b/testing/helm/tests/tracing.yaml new file mode 100644 index 00000000000..fd23a5fb912 --- /dev/null +++ b/testing/helm/tests/tracing.yaml @@ -0,0 +1,14 @@ +global: + tracing: + createDefaultLogExporter: true + exporters: + - type: OTLP-HTTP + consoleDebug: true + batch: + enabled: true + - type: JLOG + logSpanDetails: true + logParentInfo: true + logAttributes: true + logEvents: true + logLinks: true From 87f516846ab306942a174f0c95074f7e248b9b85 Mon Sep 17 00:00:00 2001 From: Gavin Halliday Date: Wed, 3 Jan 2024 14:37:51 +0000 Subject: [PATCH 3/3] Further changes following review Signed-off-by: Gavin Halliday --- helm/examples/tracing/README.md | 8 ++++---- helm/examples/tracing/jlog-collector-fulloutput.yaml | 5 +++-- helm/examples/tracing/otlp-grpc-collector-default.yaml | 4 ++-- helm/examples/tracing/otlp-http-collector-default.yaml | 4 ++-- helm/hpcc/values.schema.json | 4 ++-- system/jlib/jtrace.cpp | 6 +++--- testing/helm/tests/tracing.yaml | 2 +- 7 files changed, 17 insertions(+), 16 deletions(-) diff --git a/helm/examples/tracing/README.md b/helm/examples/tracing/README.md index f2beff33d89..672d27e0dba 100644 --- a/helm/examples/tracing/README.md +++ b/helm/examples/tracing/README.md @@ -11,7 +11,7 @@ All configuration options detailed here are part of the HPCC Systems Helm chart, - disabled - (default: false) disables tracking and reporting of internal traces and spans - alwaysCreateGlobalIds - If true, assign newly created global ID to any requests that do not supply one. - optAlwaysCreateTraceIds - If true components generate trace/span ids if none are provided by the remote caller. -- createDefaultLogExporter - If true, creates a trace exporter outputting to the log using the default options +- enableDefaultLogExporter - If true, creates a trace exporter outputting to the log using the default options - exporters: - Defines a list of exporters in charge of forwarding span data to target back-end - type - "OTLP-HTTP" | "OTLP-GRPC" | "OS" | "JLOG" - "JLOG" @@ -31,7 +31,7 @@ All configuration options detailed here are part of the HPCC Systems Helm chart, - sslCredentialsCACertPath - Path to .pem file to be used for SSL encryption. - timeOutSeconds - (default 10secs) Timeout for grpc deadline - batch: - - enabled - If true, trace data is processed in a batch, if false when it is available (simple) + - enabled - If true, trace data is processed in a batch, if false, trace data is processed immediately ### Sample configuration Below is a sample helm values block directing the HPCC tracing framework to process span information serially, and export the data over OTLP/HTTP protocol to localhost:4318 and output export debug information to console: @@ -39,8 +39,8 @@ Below is a sample helm values block directing the HPCC tracing framework to proc ```console global: tracing: - exporter: - type: OTLP-HTTP + exporters: + - type: OTLP-HTTP consoleDebug: true ``` ### Sample configuration command diff --git a/helm/examples/tracing/jlog-collector-fulloutput.yaml b/helm/examples/tracing/jlog-collector-fulloutput.yaml index c3ce5704e49..e251f758884 100644 --- a/helm/examples/tracing/jlog-collector-fulloutput.yaml +++ b/helm/examples/tracing/jlog-collector-fulloutput.yaml @@ -1,7 +1,8 @@ global: tracing: - exporter: - type: JLog + enableDefaultLogExporter: false + exporters: + - type: JLOG logSpanDetails: true logParentInfo: true logAttributes: true diff --git a/helm/examples/tracing/otlp-grpc-collector-default.yaml b/helm/examples/tracing/otlp-grpc-collector-default.yaml index 35ea30dfc32..90ca78a56b0 100644 --- a/helm/examples/tracing/otlp-grpc-collector-default.yaml +++ b/helm/examples/tracing/otlp-grpc-collector-default.yaml @@ -1,6 +1,6 @@ global: tracing: - exporter: - type: OTLP-GRPC + exporters: + - type: OTLP-GRPC endpoint: "localhost:4317" useSslCredentials: false diff --git a/helm/examples/tracing/otlp-http-collector-default.yaml b/helm/examples/tracing/otlp-http-collector-default.yaml index 717c0984298..5c442e8552d 100644 --- a/helm/examples/tracing/otlp-http-collector-default.yaml +++ b/helm/examples/tracing/otlp-http-collector-default.yaml @@ -1,6 +1,6 @@ global: tracing: - exporter: - type: OTLP-HTTP + exporters: + - type: OTLP-HTTP endpoint: "localhost:4318" consoleDebug: true \ No newline at end of file diff --git a/helm/hpcc/values.schema.json b/helm/hpcc/values.schema.json index 45a4a083cec..5627adae912 100644 --- a/helm/hpcc/values.schema.json +++ b/helm/hpcc/values.schema.json @@ -1116,7 +1116,7 @@ "type": "boolean", "description": "If true, components generate trace/span ids if none are provided by the remote caller" }, - "createDefaultLogExporter": { + "enableDefaultLogExporter": { "type": "boolean", "description": "If true, creates a trace exporter outputting to the log using the default options" }, @@ -1143,7 +1143,7 @@ "properties": { "enabled": { "type": "boolean", - "description": "If true, trace data is processed in a batch, if false, as soon as it is available" + "description": "If true, trace data is processed in a batch, if false, trace data is processed immediately" } }, "additionalProperties": { "type": ["integer", "string", "boolean"] } diff --git a/system/jlib/jtrace.cpp b/system/jlib/jtrace.cpp index 47ed0f5b9c8..e500b11f02b 100644 --- a/system/jlib/jtrace.cpp +++ b/system/jlib/jtrace.cpp @@ -1196,7 +1196,7 @@ void CTraceManager::initTracerProviderAndGlobalInternals(const IPropertyTree * t //By default trace spans to the logs in debug builds - so that developers get used to seeing them. //Default off for release builds to avoid flooding the logs, and because they are likely to use OTLP - bool createDefaultLogExporter = isDebugBuild(); + bool enableDefaultLogExporter = isDebugBuild(); if (traceConfig) { //Administrators can choose to export trace data to a different backend by specifying the exporter type @@ -1209,10 +1209,10 @@ void CTraceManager::initTracerProviderAndGlobalInternals(const IPropertyTree * t processors.push_back(std::move(processor)); } - createDefaultLogExporter = traceConfig->getPropBool("createDefaultLogExporter", createDefaultLogExporter); + enableDefaultLogExporter = traceConfig->getPropBool("enableDefaultLogExporter", enableDefaultLogExporter); } - if (createDefaultLogExporter) + if (enableDefaultLogExporter) { //Simple option to create logging to the log file - primarily to aid developers. std::unique_ptr exporter = JLogSpanExporterFactory::Create(DEFAULT_SPAN_LOG_FLAGS); diff --git a/testing/helm/tests/tracing.yaml b/testing/helm/tests/tracing.yaml index fd23a5fb912..801c420aee9 100644 --- a/testing/helm/tests/tracing.yaml +++ b/testing/helm/tests/tracing.yaml @@ -1,6 +1,6 @@ global: tracing: - createDefaultLogExporter: true + enableDefaultLogExporter: true exporters: - type: OTLP-HTTP consoleDebug: true