From 316a1b3ed7ca2b08b8e0e4b76836f16153bdfb1a Mon Sep 17 00:00:00 2001 From: Gavin Halliday Date: Fri, 15 Dec 2023 12:52:42 +0000 Subject: [PATCH] HPCC-30125 Add support for multiple trace exporters Signed-off-by: Gavin Halliday --- helm/examples/tracing/README.md | 37 ++--- helm/hpcc/values.schema.json | 14 +- system/jlib/jtrace.cpp | 271 +++++++++++++++++--------------- 3 files changed, 163 insertions(+), 159 deletions(-) diff --git a/helm/examples/tracing/README.md b/helm/examples/tracing/README.md index bf5cbcd1003..dd1981988fb 100644 --- a/helm/examples/tracing/README.md +++ b/helm/examples/tracing/README.md @@ -13,24 +13,23 @@ All configuration options detailed here are part of the HPCC Systems Helm chart, - optAlwaysCreateTraceIds - If true components generate trace/span ids if none are provided by the remote caller. - exporter - Defines The type of exporter in charge of forwarding span data to target back-end - type - (default: JLOG) "OTLP-HTTP" | "OTLP-GRCP" | "OS" | "JLOG" | "NONE" - - JLOG - - logSpanDetails - Log span details such as description, status, kind - - logParentInfo - Log the span's parent info such as ParentSpanId, and TraceState - - logAttributes - Log the span's attributes - - logEvents - Log the span's events - - logLinks - Log the span's links - - logResources - Log the span's resources such as telemetry.sdk version, name, language - - OTLP-HTTP - - endpoint - (default localhost:4318) Specifies the target OTLP-HTTP backend - - timeOutSecs - (default 10secs) - - consoleDebug - (default false) - - OTLP-GRCP - - endpoint: (default localhost:4317) The endpoint to export to. By default the OpenTelemetry Collector's default endpoint. - - useSslCredentials - By default when false, uses grpc::InsecureChannelCredentials; If true uses sslCredentialsCACertPath - - sslCredentialsCACertPath - Path to .pem file to be used for SSL encryption. - - timeOutSeconds - (default 10secs) Timeout for grpc deadline -- processor - Controls span processing style. One by one as available, or in batches. - - type - (default: simple) "simple" | "batch" + - JLOG + - logSpanDetails - Log span details such as description, status, kind + - logParentInfo - Log the span's parent info such as ParentSpanId, and TraceState + - logAttributes - Log the span's attributes + - logEvents - Log the span's events + - logLinks - Log the span's links + - logResources - Log the span's resources such as telemetry.sdk version, name, language + - OTLP-HTTP + - endpoint - (default localhost:4318) Specifies the target OTLP-HTTP backend + - timeOutSecs - (default 10secs) + - consoleDebug - (default false) + - OTLP-GRCP + - endpoint: (default localhost:4317) The endpoint to export to. By default the OpenTelemetry Collector's default endpoint. + - useSslCredentials - By default when false, uses grpc::InsecureChannelCredentials; If true uses sslCredentialsCACertPath + - sslCredentialsCACertPath - Path to .pem file to be used for SSL encryption. + - timeOutSeconds - (default 10secs) Timeout for grpc deadline + - batch - If true, trace data is processed in a batch, if false when it is available (simple) ### Sample configuration Below is a sample helm values block directing the HPCC tracing framework to process span information serially, and export the data over OTLP/HTTP protocol to localhost:4318 and output export debug information to console: @@ -41,8 +40,6 @@ global: exporter: type: OTLP-HTTP consoleDebug: true - processor: - type: simple ``` ### Sample configuration command diff --git a/helm/hpcc/values.schema.json b/helm/hpcc/values.schema.json index 0b07509ec32..96b2ca41dc4 100644 --- a/helm/hpcc/values.schema.json +++ b/helm/hpcc/values.schema.json @@ -1123,16 +1123,10 @@ "type": "string", "enum": ["OTLP-HTTP", "OTLP-GRCP", "OS", "JLOG", "NONE"], "description": "The type of exporter in charge of forwarding span data to target back-end" - } - } - }, - "processor": { - "type": "object", - "properties": { - "type": { - "type": "string", - "enum": ["batch", "simple"], - "description": "Defines the manner in which trace data is processed - in batches, or simple as available" + }, + "batch": { + "type": "boolean", + "description": "If true, trace data is processed in a batch, if false when it is available" } } } diff --git a/system/jlib/jtrace.cpp b/system/jlib/jtrace.cpp index c27b33b1b9a..e868b741292 100644 --- a/system/jlib/jtrace.cpp +++ b/system/jlib/jtrace.cpp @@ -452,6 +452,8 @@ class CTraceManager : implements ITraceManager, public CInterface void initTracerProviderAndGlobalInternals(const IPropertyTree * traceConfig); void initTracer(const IPropertyTree * traceConfig); void cleanupTracer(); + std::unique_ptr createExporter(const IPropertyTree * exportConfig); + std::unique_ptr createProcessor(const IPropertyTree * exportConfig); public: CTraceManager(const char * componentName, const IPropertyTree * componentConfig, const IPropertyTree * globalConfig); @@ -1052,161 +1054,172 @@ IProperties * getSpanContext(const ISpan * span) //--------------------------------------------------------------------------------------------------------------------- -void CTraceManager::initTracerProviderAndGlobalInternals(const IPropertyTree * traceConfig) +std::unique_ptr CTraceManager::createExporter(const IPropertyTree * exportConfig) { - //Trace to JLog by default. - std::unique_ptr exporter = JLogSpanExporterFactory::Create(DEFAULT_SPAN_LOG_FLAGS); + assertex(exportConfig); + + StringBuffer exportType; + exportConfig->getProp("@type", exportType); - //Administrators can choose to export trace data to a different backend by specifying the exporter type - if (traceConfig && traceConfig->hasProp("exporter")) + LOG(MCoperatorInfo, "Exporter type: %s", exportType.str()); + if (!exportType.isEmpty()) { - Owned exportConfig = traceConfig->getPropTree("exporter"); - if (exportConfig) + if (stricmp(exportType.str(), "OS")==0) //To stdout/err { - StringBuffer exportType; - exportConfig->getProp("@type", exportType); - LOG(MCoperatorInfo, "Exporter type: %s", exportType.str()); - - if (!exportType.isEmpty()) - { - if (stricmp(exportType.str(), "OS")==0) //To stdout/err - { - exporter = opentelemetry::exporter::trace::OStreamSpanExporterFactory::Create(); - LOG(MCoperatorInfo, "Tracing exporter set OS"); - } - else if (stricmp(exportType.str(), "OTLP")==0 || stricmp(exportType.str(), "OTLP-HTTP")==0) - { - opentelemetry::exporter::otlp::OtlpHttpExporterOptions trace_opts; - const char * endPoint = exportConfig->queryProp("@endpoint"); - if (endPoint) - trace_opts.url = endPoint; + LOG(MCoperatorInfo, "Tracing exporter set OS"); + return opentelemetry::exporter::trace::OStreamSpanExporterFactory::Create(); + } + else if (stricmp(exportType.str(), "OTLP")==0 || stricmp(exportType.str(), "OTLP-HTTP")==0) + { + opentelemetry::exporter::otlp::OtlpHttpExporterOptions trace_opts; + const char * endPoint = exportConfig->queryProp("@endpoint"); + if (endPoint) + trace_opts.url = endPoint; - if (exportConfig->hasProp("@timeOutSecs")) //not sure exactly what this value actually affects - trace_opts.timeout = std::chrono::seconds(exportConfig->getPropInt("@timeOutSecs")); + if (exportConfig->hasProp("@timeOutSecs")) //not sure exactly what this value actually affects + trace_opts.timeout = std::chrono::seconds(exportConfig->getPropInt("@timeOutSecs")); - // Whether to print the status of the exporter in the console - trace_opts.console_debug = exportConfig->getPropBool("@consoleDebug", false); + // Whether to print the status of the exporter in the console + trace_opts.console_debug = exportConfig->getPropBool("@consoleDebug", false); - exporter = opentelemetry::exporter::otlp::OtlpHttpExporterFactory::Create(trace_opts); - LOG(MCoperatorInfo,"Tracing exporter set to OTLP/HTTP to: (%s)", trace_opts.url.c_str()); - } - else if (stricmp(exportType.str(), "OTLP-GRPC")==0) - { - namespace otlp = opentelemetry::exporter::otlp; + LOG(MCoperatorInfo,"Tracing exporter set to OTLP/HTTP to: (%s)", trace_opts.url.c_str()); + return opentelemetry::exporter::otlp::OtlpHttpExporterFactory::Create(trace_opts); + } + else if (stricmp(exportType.str(), "OTLP-GRPC")==0) + { + namespace otlp = opentelemetry::exporter::otlp; - otlp::OtlpGrpcExporterOptions opts; + otlp::OtlpGrpcExporterOptions opts; - const char * endPoint = exportConfig->queryProp("@endpoint"); - if (endPoint) - opts.endpoint = endPoint; + const char * endPoint = exportConfig->queryProp("@endpoint"); + if (endPoint) + opts.endpoint = endPoint; - opts.use_ssl_credentials = exportConfig->getPropBool("@useSslCredentials", false); + opts.use_ssl_credentials = exportConfig->getPropBool("@useSslCredentials", false); - if (opts.use_ssl_credentials) - { - StringBuffer sslCACertPath; - exportConfig->getProp("@sslCredentialsCACertPath", sslCACertPath); - opts.ssl_credentials_cacert_path = sslCACertPath.str(); - } + if (opts.use_ssl_credentials) + { + StringBuffer sslCACertPath; + exportConfig->getProp("@sslCredentialsCACertPath", sslCACertPath); + opts.ssl_credentials_cacert_path = sslCACertPath.str(); + } - if (exportConfig->hasProp("@timeOutSecs")) //grpc deadline timeout in seconds - opts.timeout = std::chrono::seconds(exportConfig->getPropInt("@timeOutSecs")); + if (exportConfig->hasProp("@timeOutSecs")) //grpc deadline timeout in seconds + opts.timeout = std::chrono::seconds(exportConfig->getPropInt("@timeOutSecs")); - exporter = otlp::OtlpGrpcExporterFactory::Create(opts); - LOG(MCoperatorInfo, "Tracing exporter set to OTLP/GRPC to: (%s)", opts.endpoint.c_str()); - } - else if (stricmp(exportType.str(), "JLOG")==0) - { - StringBuffer logFlagsStr; - SpanLogFlags logFlags = SpanLogFlags::LogNone; - - if (exportConfig->getPropBool("@logSpanDetails", false)) - { - logFlags |= SpanLogFlags::LogSpanDetails; - logFlagsStr.append(" LogDetails "); - } - if (exportConfig->getPropBool("@logParentInfo", false)) - { - logFlags |= SpanLogFlags::LogParentInfo; - logFlagsStr.append(" LogParentInfo "); - } - if (exportConfig->getPropBool("@logAttributes", false)) - { - logFlags |= SpanLogFlags::LogAttributes; - logFlagsStr.append(" LogAttributes "); - } - if (exportConfig->getPropBool("@logEvents", false)) - { - logFlags |= SpanLogFlags::LogEvents; - logFlagsStr.append(" LogEvents "); - } - if (exportConfig->getPropBool("@logLinks", false)) - { - logFlags |= SpanLogFlags::LogLinks; - logFlagsStr.append(" LogLinks "); - } - if (exportConfig->getPropBool("@logResources", false)) - { - logFlags |= SpanLogFlags::LogResources; - logFlagsStr.append(" LogLinks "); - } + LOG(MCoperatorInfo, "Tracing exporter set to OTLP/GRPC to: (%s)", opts.endpoint.c_str()); + return otlp::OtlpGrpcExporterFactory::Create(opts); + } + else if (stricmp(exportType.str(), "JLOG")==0) + { + StringBuffer logFlagsStr; + SpanLogFlags logFlags = SpanLogFlags::LogNone; - //if no log feature flags provided, use default - if (logFlags == SpanLogFlags::LogNone) - logFlags = DEFAULT_SPAN_LOG_FLAGS; + if (exportConfig->getPropBool("@logSpanDetails", false)) + { + logFlags |= SpanLogFlags::LogSpanDetails; + logFlagsStr.append(" LogDetails "); + } + if (exportConfig->getPropBool("@logParentInfo", false)) + { + logFlags |= SpanLogFlags::LogParentInfo; + logFlagsStr.append(" LogParentInfo "); + } + if (exportConfig->getPropBool("@logAttributes", false)) + { + logFlags |= SpanLogFlags::LogAttributes; + logFlagsStr.append(" LogAttributes "); + } + if (exportConfig->getPropBool("@logEvents", false)) + { + logFlags |= SpanLogFlags::LogEvents; + logFlagsStr.append(" LogEvents "); + } + if (exportConfig->getPropBool("@logLinks", false)) + { + logFlags |= SpanLogFlags::LogLinks; + logFlagsStr.append(" LogLinks "); + } + if (exportConfig->getPropBool("@logResources", false)) + { + logFlags |= SpanLogFlags::LogResources; + logFlagsStr.append(" LogLinks "); + } - exporter = JLogSpanExporterFactory::Create(logFlags); + //if no log feature flags provided, use default + if (logFlags == SpanLogFlags::LogNone) + logFlags = DEFAULT_SPAN_LOG_FLAGS; - LOG(MCoperatorInfo, "Tracing exporter set to JLog: logFlags( LogAttributes LogParentInfo %s)", logFlagsStr.str()); - } - else if (stricmp(exportType.str(), "Prometheus")==0) - LOG(MCoperatorInfo, "Tracing to Prometheus currently not supported"); - else if (stricmp(exportType.str(), "NONE")==0) - { - exporter = NoopSpanExporterFactory::Create(); - LOG(MCoperatorInfo, "Tracing exporter set to 'NONE', no trace exporting will be performed"); - } - else - LOG(MCoperatorInfo, "Tracing exporter type not supported: '%s', JLog trace exporting will be performed", exportType.str()); - } - else - LOG(MCoperatorInfo, "Tracing exporter type not specified"); + LOG(MCoperatorInfo, "Tracing exporter set to JLog: logFlags( LogAttributes LogParentInfo %s)", logFlagsStr.str()); + return JLogSpanExporterFactory::Create(logFlags); } + else if (stricmp(exportType.str(), "Prometheus")==0) + LOG(MCoperatorInfo, "Tracing to Prometheus currently not supported"); + else if (stricmp(exportType.str(), "NONE")==0) + { + LOG(MCoperatorInfo, "Tracing exporter set to 'NONE', no trace exporting will be performed"); + return NoopSpanExporterFactory::Create(); + } + else + LOG(MCoperatorInfo, "Tracing exporter type not supported: '%s', JLog trace exporting will be performed", exportType.str()); } + else + LOG(MCoperatorInfo, "Tracing exporter type not specified"); + return nullptr; +} - //Administrator can choose to process spans in batches or one at a time - //Default: SimpleSpanProcesser sends spans one by one to an exporter. - std::unique_ptr processor = opentelemetry::sdk::trace::SimpleSpanProcessorFactory::Create(std::move(exporter)); - if (traceConfig && traceConfig->hasProp("processor/@type")) - { - StringBuffer processorType; - bool foundProcessorType = traceConfig->getProp("processor/@type", processorType); +std::unique_ptr CTraceManager::createProcessor(const IPropertyTree * exportConfig) +{ + auto exporter = createExporter(exportConfig); + if (!exporter) + return nullptr; + + if (exportConfig->getPropBool("@batch", false)) + { + //Groups several spans together, before sending them to an exporter. + //MORE: These options should be configurable + opentelemetry::v1::sdk::trace::BatchSpanProcessorOptions options; //size_t max_queue_size = 2048; + //The time interval between two consecutive exports + //std::chrono::milliseconds(5000); + //The maximum batch size of every export. It must be smaller or + //equal to max_queue_size. + //size_t max_export_batch_size = 512 + return opentelemetry::sdk::trace::BatchSpanProcessorFactory::Create(std::move(exporter), options); + } - if (foundProcessorType && strcmp("batch", processorType.str())==0) - { - //Groups several spans together, before sending them to an exporter. - //These options should be configurable - opentelemetry::v1::sdk::trace::BatchSpanProcessorOptions options; //size_t max_queue_size = 2048; - //The time interval between two consecutive exports - //std::chrono::milliseconds(5000); - //The maximum batch size of every export. It must be smaller or - //equal to max_queue_size. - //size_t max_export_batch_size = 512 - processor = opentelemetry::sdk::trace::BatchSpanProcessorFactory::Create(std::move(exporter), options); - LOG(MCoperatorInfo, "OpenTel tracing using batch Span Processor"); - } - else if (foundProcessorType && strcmp("simple", processorType.str())==0) + return opentelemetry::sdk::trace::SimpleSpanProcessorFactory::Create(std::move(exporter)); +} + +void CTraceManager::initTracerProviderAndGlobalInternals(const IPropertyTree * traceConfig) +{ + std::vector> processors; + if (traceConfig) + { + //Administrators can choose to export trace data to a different backend by specifying the exporter type + IPropertyTree * exportConfig = traceConfig->queryPropTree("exporter"); + if (exportConfig) { - LOG(MCoperatorInfo, "OpenTel tracing using batch simple Processor"); + std::unique_ptr processor = createProcessor(exportConfig); + if (processor) + processors.push_back(std::move(processor)); } - else + + Owned iter = traceConfig->getElements("exporters"); + ForEach(*iter) { - LOG(MCoperatorInfo, "OpenTel tracing detected invalid processor type: '%s'", processorType.str()); + IPropertyTree & curExporter = iter->query(); + std::unique_ptr processor = createProcessor(&curExporter); + if (processor) + processors.push_back(std::move(processor)); } } - std::vector> processors; - processors.push_back(std::move(processor)); + if (processors.empty()) + { + //Default to tracing to the log file if no exporters are specified + std::unique_ptr exporter = JLogSpanExporterFactory::Create(DEFAULT_SPAN_LOG_FLAGS); + processors.push_back(opentelemetry::sdk::trace::SimpleSpanProcessorFactory::Create(std::move(exporter))); + } // Default is an always-on sampler. std::shared_ptr context =