From 42050e822249e2ce74a42ddc28d26974241d87cf Mon Sep 17 00:00:00 2001 From: Gavin Halliday Date: Mon, 2 Oct 2023 17:01:42 +0100 Subject: [PATCH] HPCC-30376 Allow the open telemetry tracing to be configured Signed-off-by: Gavin Halliday --- helm/hpcc/values.schema.json | 40 +++++++++++++++++++ .../configxml/RoxieTopology.xsl | 8 ++++ .../componentfiles/configxml/agentexec.xsl | 8 ++++ initfiles/componentfiles/configxml/dali.xsl | 8 ++++ .../componentfiles/configxml/dfuserver.xsl | 8 ++++ .../componentfiles/configxml/eclccserver.xsl | 8 ++++ .../componentfiles/configxml/eclscheduler.xsl | 8 ++++ initfiles/componentfiles/configxml/esp.xsl | 8 ++++ initfiles/componentfiles/configxml/sasha.xsl | 8 ++++ initfiles/componentfiles/configxml/thor.xsl | 8 ++++ .../componentfiles/configxml/toposerver.xsl | 8 ++++ system/jlib/jptree.cpp | 2 +- system/jlib/jtrace.cpp | 16 +++++--- system/jlib/jtrace.hpp | 2 +- testing/unittests/jlibtests.cpp | 8 ++-- 15 files changed, 136 insertions(+), 12 deletions(-) diff --git a/helm/hpcc/values.schema.json b/helm/hpcc/values.schema.json index e0c4e424796..569f7ad715b 100644 --- a/helm/hpcc/values.schema.json +++ b/helm/hpcc/values.schema.json @@ -291,6 +291,9 @@ "logging": { "$ref": "#/definitions/logging" }, + "tracing": { + "$ref": "#/definitions/tracing" + }, "logAccess": { "$ref": "#/definitions/logAccess" }, @@ -1093,6 +1096,16 @@ }, "additionalProperties": { "type": ["integer", "string", "boolean"] } }, + "tracing": { + "type": "object", + "properties": { + "alwaysCreateGlobalIds": { + "type": "boolean", + "description": "If true, allocate global ids to any requests that do not supply one" + } + }, + "additionalProperties": { "type": ["integer", "string", "boolean"] } + }, "compileOption": { "type": "object", "properties": { @@ -1243,6 +1256,9 @@ "logging": { "$ref": "#/definitions/logging" }, + "tracing": { + "$ref": "#/definitions/tracing" + }, "auth": { "$ref": "#/definitions/auth" }, @@ -1323,6 +1339,9 @@ "logging": { "$ref": "#/definitions/logging" }, + "tracing": { + "$ref": "#/definitions/tracing" + }, "annotations": { "type": "object", "additionalProperties": { "type": "string" } @@ -1379,6 +1398,9 @@ "logging": { "$ref": "#/definitions/logging" }, + "tracing": { + "$ref": "#/definitions/tracing" + }, "annotations": { "type": "object", "additionalProperties": { "type": "string" } @@ -1413,6 +1435,9 @@ "logging": { "$ref": "#/definitions/logging" }, + "tracing": { + "$ref": "#/definitions/tracing" + }, "resources": { "$ref": "#/definitions/resources" }, @@ -1500,6 +1525,9 @@ "logging": { "$ref": "#/definitions/logging" }, + "tracing": { + "$ref": "#/definitions/tracing" + }, "dataPlane": { "description": "The default storage plane to write data files to", "type": "string" @@ -1585,6 +1613,9 @@ "logging": { "$ref": "#/definitions/logging" }, + "tracing": { + "$ref": "#/definitions/tracing" + }, "preload": { "description": "Preloaded plugins", "type": "array", @@ -2275,6 +2306,9 @@ "logging": { "$ref": "#/definitions/logging" }, + "tracing": { + "$ref": "#/definitions/tracing" + }, "annotations": { "type": "object", "additionalProperties": { "type": "string" } @@ -2428,6 +2462,9 @@ "logging": { "$ref": "#/definitions/logging" }, + "tracing": { + "$ref": "#/definitions/tracing" + }, "dataPlane": { "description": "The default storage plane to write data files to", "type": "string" @@ -2805,6 +2842,9 @@ "logging": { "$ref": "#/definitions/logging" }, + "tracing": { + "$ref": "#/definitions/tracing" + }, "resources": { "$ref": "#/definitions/resources" }, diff --git a/initfiles/componentfiles/configxml/RoxieTopology.xsl b/initfiles/componentfiles/configxml/RoxieTopology.xsl index fc2b92a7ff9..96e584b6b0e 100644 --- a/initfiles/componentfiles/configxml/RoxieTopology.xsl +++ b/initfiles/componentfiles/configxml/RoxieTopology.xsl @@ -123,6 +123,14 @@ + + + + + + + + diff --git a/initfiles/componentfiles/configxml/agentexec.xsl b/initfiles/componentfiles/configxml/agentexec.xsl index b2e323b9ebc..182e503e021 100644 --- a/initfiles/componentfiles/configxml/agentexec.xsl +++ b/initfiles/componentfiles/configxml/agentexec.xsl @@ -123,6 +123,14 @@ + + + + + + + + diff --git a/initfiles/componentfiles/configxml/dali.xsl b/initfiles/componentfiles/configxml/dali.xsl index a7a4516a181..ffafa3199cc 100644 --- a/initfiles/componentfiles/configxml/dali.xsl +++ b/initfiles/componentfiles/configxml/dali.xsl @@ -104,6 +104,14 @@ + + + + + + + + diff --git a/initfiles/componentfiles/configxml/dfuserver.xsl b/initfiles/componentfiles/configxml/dfuserver.xsl index 042aa3d7dff..fb5c607731a 100644 --- a/initfiles/componentfiles/configxml/dfuserver.xsl +++ b/initfiles/componentfiles/configxml/dfuserver.xsl @@ -85,6 +85,14 @@ + + + + + + + + diff --git a/initfiles/componentfiles/configxml/eclscheduler.xsl b/initfiles/componentfiles/configxml/eclscheduler.xsl index 5da39d74e41..9d7d84facf5 100644 --- a/initfiles/componentfiles/configxml/eclscheduler.xsl +++ b/initfiles/componentfiles/configxml/eclscheduler.xsl @@ -70,6 +70,14 @@ + + + + + + + + diff --git a/initfiles/componentfiles/configxml/esp.xsl b/initfiles/componentfiles/configxml/esp.xsl index c699adb0c02..369112c0b81 100644 --- a/initfiles/componentfiles/configxml/esp.xsl +++ b/initfiles/componentfiles/configxml/esp.xsl @@ -70,6 +70,14 @@ + + + + + + + + diff --git a/initfiles/componentfiles/configxml/sasha.xsl b/initfiles/componentfiles/configxml/sasha.xsl index 6cf201d2975..f739bce0e4e 100644 --- a/initfiles/componentfiles/configxml/sasha.xsl +++ b/initfiles/componentfiles/configxml/sasha.xsl @@ -81,6 +81,14 @@ + + + + + + + + diff --git a/initfiles/componentfiles/configxml/toposerver.xsl b/initfiles/componentfiles/configxml/toposerver.xsl index d28b7a916e5..1b49bbc51a9 100644 --- a/initfiles/componentfiles/configxml/toposerver.xsl +++ b/initfiles/componentfiles/configxml/toposerver.xsl @@ -70,6 +70,14 @@ + + + + + + + + diff --git a/system/jlib/jptree.cpp b/system/jlib/jptree.cpp index 4b314e3f5b3..340cad9cbd7 100644 --- a/system/jlib/jptree.cpp +++ b/system/jlib/jptree.cpp @@ -9068,7 +9068,7 @@ jlib_decl IPropertyTree * loadConfiguration(IPropertyTree *componentDefault, con if (monitor) configFileUpdater->startMonitoring(); - initTraceManager(componentTag, componentConfiguration.get()); + initTraceManager(componentTag, componentConfiguration.get(), globalConfiguration.get()); return componentConfiguration.getLink(); } diff --git a/system/jlib/jtrace.cpp b/system/jlib/jtrace.cpp index b45b67aed8e..52a8d33dbed 100644 --- a/system/jlib/jtrace.cpp +++ b/system/jlib/jtrace.cpp @@ -659,7 +659,7 @@ class CTraceManager : implements ITraceManager, public CInterface //and an exporter which is responsible for exporting the processed spans. //By default, an InMemorySpan exporter is used in the absence of a proper noop exporter. //Also, a SimpleSpanProcessor is used in the absence of a configuration directive to process spans in batches - void initTracerProviderAndGlobalInternals(IPropertyTree * traceConfig) + void initTracerProviderAndGlobalInternals(const IPropertyTree * traceConfig) { //Currently using InMemorySpanExporter as default, until a noop exporter is available std::shared_ptr data; @@ -768,13 +768,13 @@ class CTraceManager : implements ITraceManager, public CInterface processor: #optional - Controls span processing style type: batch #simple|batch (default: simple) */ - void initTracer(IPropertyTree * traceConfig) + void initTracer(const IPropertyTree * traceConfig) { try { #ifdef TRACECONFIGDEBUG Owned testTree; - if (!traceConfig || !traceConfig->hasProp("tracing")) + if (!traceConfig) { const char * simulatedGlobalYaml = R"!!(global: tracing: @@ -843,9 +843,13 @@ class CTraceManager : implements ITraceManager, public CInterface public: IMPLEMENT_IINTERFACE; - CTraceManager(const char * componentName, IPropertyTree * traceConfig) + CTraceManager(const char * componentName, const IPropertyTree * componentConfig, const IPropertyTree * globalConfig) { + assertex(componentConfig); moduleName.set(componentName); + const IPropertyTree * traceConfig = componentConfig->queryPropTree("tracing"); + if (!traceConfig && globalConfig) + traceConfig = globalConfig->queryPropTree("tracing"); initTracer(traceConfig); auto provider = opentelemetry::trace::Provider::GetTracerProvider(); @@ -896,9 +900,9 @@ MODULE_EXIT() theTraceManager.destroy(); } -void initTraceManager(const char * componentName, IPropertyTree * config) +void initTraceManager(const char * componentName, const IPropertyTree * componentConfig, const IPropertyTree * globalConfig) { - theTraceManager.query([=] () { return new CTraceManager(componentName, config); }); + theTraceManager.query([=] () { return new CTraceManager(componentName, componentConfig, globalConfig); }); } ITraceManager & queryTraceManager() diff --git a/system/jlib/jtrace.hpp b/system/jlib/jtrace.hpp index 34ab6100f19..0c852c5f54b 100644 --- a/system/jlib/jtrace.hpp +++ b/system/jlib/jtrace.hpp @@ -65,7 +65,7 @@ interface ITraceManager : extends IInterface virtual const char * getTracedComponentName() const = 0; }; -extern jlib_decl void initTraceManager(const char * componentName, IPropertyTree * traceConfig); +extern jlib_decl void initTraceManager(const char * componentName, const IPropertyTree * componentConfig, const IPropertyTree * globalConfig); extern jlib_decl ITraceManager & queryTraceManager(); /* diff --git a/testing/unittests/jlibtests.cpp b/testing/unittests/jlibtests.cpp index ff59ee2c637..0f3a99b29fe 100644 --- a/testing/unittests/jlibtests.cpp +++ b/testing/unittests/jlibtests.cpp @@ -101,10 +101,10 @@ class JlibTraceTest : public CppUnit::TestFixture void testTraceDisableConfig() { Owned testTree = createPTreeFromYAMLString(disableTracingYaml, ipt_none, ptr_ignoreWhiteSpace, nullptr); - Owned traceConfig = testTree->getPropTree("global/tracing"); + Owned traceConfig = testTree->getPropTree("global"); //Not valid, due to tracemanager initialized at component load time - initTraceManager("somecomponent", traceConfig); + initTraceManager("somecomponent", traceConfig, nullptr); } void testIDPropegation() @@ -148,9 +148,9 @@ class JlibTraceTest : public CppUnit::TestFixture void testTraceConfig() { Owned testTree = createPTreeFromYAMLString(simulatedGlobalYaml, ipt_none, ptr_ignoreWhiteSpace, nullptr); - Owned traceConfig = testTree->getPropTree("global/tracing"); + Owned traceConfig = testTree->getPropTree("global"); - initTraceManager("somecomponent", traceConfig); + initTraceManager("somecomponent", traceConfig, nullptr); } void testClientSpan()