Hpcc 30401 j trace optional traceid support

esp/scm/ws_logaccess.ecm

@@ -76,13 +76,14 @@ ESPenum LogColumnValueType : string
     enum("enum")
 };
 
-ESPStruct LogColumn
+ESPStruct [nil_remove] LogColumn
 {
     string Name;
     ESPenum LogColumnType LogType;
     ESParray<string> EnumeratedValues;
     LogSelectColumnMode ColumnMode;
     ESPenum LogColumnValueType ColumnType;
+    [min_ver("1.06")] string Alias;
 };
 
 ESPResponse GetLogAccessInfoResponse
@@ -219,7 +220,7 @@ ESPResponse GetLogsResponse
     [min_ver("1.02")] unsigned int TotalLogLinesAvailable;
 };
 
-ESPservice [auth_feature("WsLogAccess:READ"), version("1.05"), default_client_version("1.05"), exceptions_inline("xslt/exceptions.xslt")] ws_logaccess
+ESPservice [auth_feature("WsLogAccess:READ"), version("1.06"), default_client_version("1.06"), exceptions_inline("xslt/exceptions.xslt")] ws_logaccess
 {
     ESPmethod GetLogAccessInfo(GetLogAccessInfoRequest, GetLogAccessInfoResponse);
     ESPmethod GetLogs(GetLogsRequest, GetLogsResponse);

esp/services/ws_logaccess/WsLogAccessService.cpp

@@ -71,6 +71,26 @@ bool Cws_logaccessEx::onGetLogAccessInfo(IEspContext &context, IEspGetLogAccessI
                                 WARNLOG("Invalid col type found in logaccess logmap config");
                             }
                         }
+                        if (clientVer >= 1.06)
+                        {
+                            if (column.hasProp("@alias"))
+                            {
+                                try
+                                {
+                                    espLogColumn->setAlias(column.queryProp("@alias"));
+                                }
+                                catch (IException *e)
+                                {
+                                    VStringBuffer msg("Invalid alias found in logaccess logmap config for '%s'", csearchColumn);
+                                    EXCLOG(e, msg.str());
+                                    e->Release();
+                                }
+                                catch(...)
+                                {
+                                    WARNLOG("Invalid alias found in logaccess logmap config");
+                                }
+                            }
+                        }
                         logColumns.append(*espLogColumn.getClear());
                     }
                     else

helm/examples/azure/log-analytics/loganalytics-hpcc-logaccess.yaml

@@ -59,7 +59,7 @@ global:
     - type: "instance"
       storeName: "ContainerInventory"
       searchColumn: "Name"
-      projectName: "hpcc_log_component"
+      alias: "hpcc_log_component"
       columnMode: "ALL"
       columnType: "string"
     - type: "node"

helm/hpcc/templates/_helpers.tpl

@@ -288,6 +288,18 @@ logging:
 {{- end -}}
 {{- end -}}
 
+{{/*
+Generate local tracing info, merged with global
+Pass in dict with root and me
+*/}}
+{{- define "hpcc.generateTracingConfig" -}}
+{{- $tracing := deepCopy (.me.tracing | default dict) | mergeOverwrite dict (.root.Values.global.tracing | default dict) -}}
+{{- if not (empty $tracing) }}
+tracing:
+{{ toYaml $tracing | indent 2 }}
+{{- end -}}
+{{- end -}}
+
 {{/*
 Generate local metrics configuration, merged with global
 Pass in dict with root and me
@@ -1286,8 +1298,9 @@ data:
   {{ $configMapName }}.yaml:
     version: 1.0
     sasha:
-{{ toYaml (omit .me "logging") | indent 6 }}
+{{ toYaml (omit .me "logging" "tracing") | indent 6 }}
 {{- include "hpcc.generateLoggingConfig" . | indent 6 }}
+{{- include "hpcc.generateTracingConfig" . | indent 6 }}
 {{ include "hpcc.generateVaultConfig" . | indent 6 }}
 {{- if hasKey .me "plane" }}
  {{- $sashaStoragePlane := .me.plane | default (include "hpcc.getFirstPlaneForCategory" (dict "root" .root "category" "sasha")) }}

helm/hpcc/templates/dafilesrv.yaml

@@ -10,8 +10,9 @@ data:
   {{ .me.name }}.yaml:
     version: 1.0
     dafilesrv:
-{{ toYaml (omit .me "logging" "env") | indent 6 }}
+{{ toYaml (omit .me "logging" "tracing" "env") | indent 6 }}
 {{- include "hpcc.generateLoggingConfig" . | indent 6 }}
+{{- include "hpcc.generateTracingConfig" . | indent 6 }}
     global:
 {{ include "hpcc.generateGlobalConfigMap" .root | indent 6 }}
 {{- end -}}

helm/hpcc/templates/dali.yaml

@@ -171,9 +171,10 @@ data:
   {{ $dali.name }}.yaml: |
     version: 1.0
     dali:
-{{ toYaml (omit $dali "logging" "services" "env") | indent 6 }}
+{{ toYaml (omit $dali "logging" "tracing" "services" "env") | indent 6 }}
       dataPath: {{ include "hpcc.getPlanePrefix" (dict "root" $ "planeName" $daliStoragePlane) }}
 {{- include "hpcc.generateLoggingConfig" $commonCtx | indent 6 }}
+{{- include "hpcc.generateTracingConfig" $commonCtx | indent 6 }}
 {{- include "hpcc.generateMetricsConfig" $commonCtx | indent 6 }}
 {{ include "hpcc.generateVaultConfig" (dict "root" $ "secretsCategories" $daliSecretsCategories ) | indent 6 }}
       foreignAccess: {{ include "hpcc.isForeignAccessConfigured" (dict "root" $ "me" .) | default "false" }}

helm/hpcc/templates/dfuserver.yaml

@@ -29,8 +29,9 @@ data:
   {{ .me.name }}.yaml:
     version: 1.0
     dfuserver:
-{{ toYaml (omit .me "logging" "env") | indent 6 }}
+{{ toYaml (omit .me "logging" "tracing" "env") | indent 6 }}
 {{- include "hpcc.generateLoggingConfig" . | indent 6 }}
+{{- include "hpcc.generateTracingConfig" . | indent 6 }}
     global:
 {{ include "hpcc.generateGlobalConfigMap" .root | indent 6 }}
 {{- end -}}

helm/hpcc/templates/eclagent.yaml

@@ -35,11 +35,13 @@ data:
   {{ .me.name }}.yaml:
     version: 1.0
     eclagent:
-{{ toYaml (omit .me "logging" "env") | indent 6 }}
+{{ toYaml (omit .me "logging" "tracing" "env") | indent 6 }}
 {{- include "hpcc.generateLoggingConfig" . | indent 6 }}
+{{- include "hpcc.generateTracingConfig" . | indent 6 }}
     {{ $apptype }}:
-{{ toYaml (omit .me "logging" "env") | indent 6 }}
+{{ toYaml (omit .me "logging" "tracing" "env") | indent 6 }}
 {{- include "hpcc.generateLoggingConfig" . | indent 6 }}
+{{- include "hpcc.generateTracingConfig" . | indent 6 }}
 {{ include "hpcc.generateVaultConfig" . | indent 6 }}
     global:
 {{ include "hpcc.generateGlobalConfigMap" .root | indent 6 }}

helm/hpcc/templates/eclccserver.yaml

@@ -34,8 +34,9 @@ data:
   {{ .me.name }}.yaml:
     version: 1.0
     eclccserver:
-{{ toYaml (omit .me "logging") | indent 6 }}
+{{ toYaml (omit .me "logging" "tracing") | indent 6 }}
 {{- include "hpcc.generateLoggingConfig" . | indent 6 }}
+{{- include "hpcc.generateTracingConfig" . | indent 6 }}
       queues:
 {{ include "hpcc.generateConfigMapQueues" .root | indent 6 }}
 {{ include "hpcc.generateVaultConfig" . | indent 6 }}

helm/hpcc/templates/eclscheduler.yaml

@@ -33,8 +33,9 @@ data:
   {{ .me.name }}.yaml:
     version: 1.0
     eclscheduler:
-{{ toYaml (omit .me "logging" "env") | indent 6 }}
+{{ toYaml (omit .me "logging" "tracing" "env") | indent 6 }}
 {{- include "hpcc.generateLoggingConfig" . | indent 6 }}
+{{- include "hpcc.generateTracingConfig" . | indent 6 }}
       queues:
 {{ include "hpcc.generateConfigMapQueues" .root | indent 6 }}
 {{ include "hpcc.generateVaultConfig" . | indent 6 }}

helm/hpcc/templates/esp.yaml

@@ -33,8 +33,9 @@ data:
   {{ .me.name }}.yaml:
     version: 1.0
     esp:
-{{ toYaml (omit .me "logging" "metrics" "env") | indent 6 }}
+{{ toYaml (omit .me "logging" "tracing" "metrics" "env") | indent 6 }}
 {{- include "hpcc.generateLoggingConfig" . | indent 6 }}
+{{- include "hpcc.generateTracingConfig" . | indent 6 }}
 {{- include "hpcc.generateMetricsConfig" . | indent 6 }}
 {{- if and .root.Values.certificates .root.Values.certificates.enabled }}
  {{- $externalCert := (ne (include "hpcc.isVisibilityPublic" (dict "root" .root "visibility" .me.service.visibility)) "") -}}

helm/hpcc/templates/localroxie.yaml

@@ -33,8 +33,9 @@ data:
   {{ .me.name }}.yaml:
     version: 1.0
     roxie:
-{{ toYaml (omit .me "logging" "env") | indent 6 }}
+{{ toYaml (omit .me "logging" "tracing" "env") | indent 6 }}
 {{- include "hpcc.generateLoggingConfig" . | indent 6 }}
+{{- include "hpcc.generateTracingConfig" . | indent 6 }}
 {{ include "hpcc.generateVaultConfig" . | indent 6 }}
     global:
 {{ include "hpcc.generateGlobalConfigMap" .root | indent 6 }}

helm/hpcc/templates/roxie.yaml

@@ -44,7 +44,7 @@ data:
      {{- include "hpcc.addTLSServiceEntries" (dict "root" $root "service" $service "component" $component "visibility" $service.visibility "remoteClients" $service.remoteClients "trustClients" $service.trustClients "includeTrustedPeers" true "incluedRoxieAndEspServices" true) | indent 6 }}
    {{- end }}
   {{- end }}
-{{ toYaml ( omit .me "logging" "topoServer" "encryptInTransit" "env" "services") | indent 6 }}
+{{ toYaml ( omit .me "logging" "tracing" "topoServer" "encryptInTransit" "env" "services") | indent 6 }}
       numChannels: {{ .numChannels }}
       topologyServers: "{{ .toponame }}:{{ .topoport }}"
       heartbeatInterval: {{ .heartbeatInterval }}
@@ -60,6 +60,7 @@ data:
       encryptInTransit: {{ $mtlsEnabled }}
 {{ end -}}
 {{- include "hpcc.generateLoggingConfig" (dict "root" .root "me" .me) | indent 6 }}
+{{- include "hpcc.generateTracingConfig" (dict "root" .root "me" .me) | indent 6 }}
 {{ include "hpcc.generateVaultConfig" . | indent 6 }}
     global:
 {{ include "hpcc.generateGlobalConfigMap" .root | indent 6 }}
@@ -73,8 +74,9 @@ data:
   {{ .toponame }}.yaml:
     version: 1.0
     toposerver:
-{{ toYaml ( omit .toposerver "logging" "env") | indent 6 }}
+{{ toYaml ( omit .toposerver "logging" "tracing" "env") | indent 6 }}
 {{- include "hpcc.generateLoggingConfig" (dict "root" .root "me" .toposerver) | indent 6 }}
+{{- include "hpcc.generateTracingConfig" (dict "root" .root "me" .toposerver) | indent 6 }}
     global:
 {{ include "hpcc.generateGlobalConfigMap" .root | indent 6 }}
 {{- end -}}

helm/hpcc/templates/thor.yaml

@@ -32,7 +32,7 @@ Pass in dict with root and me
 {{- $thorAgentScope := dict "name" .thorAgentName "replicas" .thorAgentReplicas "maxActive" .me.maxGraphs | merge (pick .me "keepJobs") }}
 {{- $eclAgentResources := .me.eclAgentResources | default dict -}}
 {{- $hthorScope := dict "name" $hthorName "jobMemorySectionName" "eclAgentMemory" | merge (pick .me "multiJobLinger" "maxGraphStartupTime") | merge (dict "resources" (deepCopy $eclAgentResources)) }}
-{{- $thorScope := omit .me "eclagent" "thoragent" "hthor" "logging" "env" "eclAgentResources" "eclAgentUseChildProcesses" "eclAgentReplicas" "thorAgentReplicas" "eclAgentType" }}
+{{- $thorScope := omit .me "eclagent" "thoragent" "hthor" "logging" "Tracing" "env" "eclAgentResources" "eclAgentUseChildProcesses" "eclAgentReplicas" "thorAgentReplicas" "eclAgentType" }}
 {{- $misc := .root.Values.global.misc | default dict }}
 {{- $postJobCommand := $misc.postJobCommand | default "" }}
 {{- $eclAgentJobName := printf "%s-job-_HPCC_JOBNAME_" $eclAgentType }}
@@ -51,20 +51,24 @@ data:
     thor:
 {{ toYaml $thorScope | indent 6 }}
 {{- include "hpcc.generateLoggingConfig" (dict "root" .root "me" $thorScope) | indent 6 }}
+{{- include "hpcc.generateTracingConfig" (dict "root" .root "me" $thorScope) | indent 6 }}
 {{ include "hpcc.generateVaultConfig" . | indent 6 }}
     {{ $eclAgentType }}: # hthor or roxie
 {{ toYaml $hthorScope | indent 6 }}
       platform:
         type: "thor"
         width: {{ mul ($thorScope.numWorkers | default 1) ( $thorScope.channelsPerWorker | default 1) }}
 {{- include "hpcc.generateLoggingConfig" (dict "root" .root "me" $hthorScope ) | indent 6 }}
+{{- include "hpcc.generateTracingConfig" (dict "root" .root "me" $hthorScope ) | indent 6 }}
 {{ include "hpcc.generateVaultConfig" . | indent 6 }}
     eclagent: # main agent Q handler
 {{ toYaml $eclAgentScope | indent 6 }}
 {{- include "hpcc.generateLoggingConfig" (dict "root" .root "me" $eclAgentScope) | indent 6 }}
+{{- include "hpcc.generateTracingConfig" (dict "root" .root "me" $eclAgentScope) | indent 6 }}
     thoragent: # Thor graph handler
 {{ toYaml $thorAgentScope | indent 6 }}
 {{- include "hpcc.generateLoggingConfig" (dict "root" .root "me" $thorAgentScope) | indent 6 }}
+{{- include "hpcc.generateTracingConfig" (dict "root" .root "me" $thorAgentScope) | indent 6 }}
       type: thor
     global:
 {{ include "hpcc.generateGlobalConfigMap" .root| indent 6 }}

helm/hpcc/values.schema.json

@@ -1102,6 +1102,10 @@
         "alwaysCreateGlobalIds": {
           "type": "boolean",
           "description": "If true, allocate global ids to any requests that do not supply one"
+        },
+        "alwaysCreateTraceIds": {
+          "type": "boolean",
+          "description": "If true, components generate trace/span ids to aid in unit of worktracing"
         }
       },
       "additionalProperties": { "type": ["integer", "string", "boolean"] }
@@ -3003,7 +3007,7 @@
         "type": {
           "type": "string",
           "description" : "The searchable HPCC log column to be mapped - 'global' applies to all known fields",
-          "enum": [ "global", "workunits", "components", "audience", "class", "instance", "host", "node", "message", "logid", "processid", "threadid", "timestamp"]
+          "enum": [ "global", "workunits", "components", "audience", "class", "instance", "host", "node", "message", "logid", "processid", "threadid", "timestamp", "alias"]
         },
         "timeStampColumn": {
           "description" : "Name of timestamp column related to mapped field (only requried for 'global' mapping)",

helm/hpcc/values.yaml

@@ -24,7 +24,8 @@ global:
   # logging sets the default logging information for all components. Can be overridden locally
   logging:
     detail: 80
-
+  tracing:
+    alwaysCreateTraceIds: true
   ## resource settings for stub components
   #stubInstanceResources:
   #  memory: "200Mi"

system/jlib/jtrace.cpp

@@ -16,7 +16,7 @@
 ############################################################################## */
 
 
-
+#include "opentelemetry/sdk/trace/random_id_generator_factory.h"
 #include "opentelemetry/trace/semantic_conventions.h" //known span defines
 #include "opentelemetry/context/propagation/global_propagator.h" // context::propagation::GlobalTextMapPropagator::GetGlobalPropagator
 #include "opentelemetry/sdk/trace/tracer_provider_factory.h" //opentelemetry::sdk::trace::TracerProviderFactory::Create(context)
@@ -602,11 +602,6 @@ class CServerSpan : public CSpan
     {
         if (httpHeaders)
         {
-            // perform any key mapping needed...
-            //Instrumented http client/server Capitalizes the first letter of the header name
-            //if (key == opentel_trace::propagation::kTraceParent || key == opentel_trace::propagation::kTraceState )
-            //    theKey[0] = toupper(theKey[0]);
-
             if (httpHeaders->hasProp(kGlobalIdHttpHeaderName))
                 hpccGlobalId.set(httpHeaders->queryProp(kGlobalIdHttpHeaderName));
             else if (httpHeaders->hasProp(kLegacyGlobalIdHttpHeaderName))
@@ -637,6 +632,35 @@ class CServerSpan : public CSpan
                 opts.parent = remoteParentSpanCtx;
             }
         }
+
+       if ((!httpHeaders ||  !httpHeaders->hasProp("traceparent")) && queryTraceManager().alwaysCreateTraceIds())
+        {
+                //Generate random trace and span IDs
+
+                //Don't get the generator every time
+                auto randomIDGenerator = opentelemetry::sdk::trace::RandomIdGeneratorFactory::Create();
+
+                auto randomTraceID = randomIDGenerator->GenerateTraceId();
+                auto randomSpanID = randomIDGenerator->GenerateSpanId();
+
+                remoteParentSpanCtx = opentelemetry::trace::SpanContext(randomTraceID, randomSpanID, opentelemetry::trace::TraceFlags(), true);
+                opts.parent = remoteParentSpanCtx;
+
+
+//this is only for debugging:
+                char trace_id[32] = {0};
+
+                randomTraceID.ToLowerBase16(trace_id);
+                StringAttr traceID;
+                traceID.set(trace_id, 32);
+
+                char span_id[16] = {0};
+                randomSpanID.ToLowerBase16(span_id);
+                StringAttr spanID;
+                spanID.set(span_id, 16);
+
+                DBGLOG("!!No remoteParent received, generating remoteParent IDs: Trace '%s' span '%s' ", traceID.get(), spanID.get());
+        }
     }
 
     bool getSpanContext(IProperties * ctxProps, bool otelFormatted) const override
@@ -710,6 +734,7 @@ class CTraceManager : implements ITraceManager, public CInterface
 private:
     bool enabled = true;
     bool optAlwaysCreateGlobalIds = false;
+    bool optAlwaysCreateTraceIds = true;
     StringAttr moduleName;
 
     //Initializes the global trace provider which is required for all Otel based tracing operations.
@@ -816,6 +841,7 @@ class CTraceManager : implements ITraceManager, public CInterface
         tracing:                            #optional - tracing enabled by default
             disable: true                   #optional - disable OTel tracing
             alwaysCreateGlobalIds : false   #optional - should global ids always be created?
+            alwaysCreateTraceIds            #optional - should trace ids always be created?
             exporter:                       #optional - Controls how trace data is exported/reported
               type: OTLP                    #OS|OTLP|Prometheus|HPCC (default: no export, jlog entry)
               endpoint: "localhost:4317"    #exporter specific key/value pairs
@@ -870,6 +896,7 @@ class CTraceManager : implements ITraceManager, public CInterface
             if (traceConfig)
             {
                 optAlwaysCreateGlobalIds = traceConfig->getPropBool("@alwaysCreateGlobalIds", optAlwaysCreateGlobalIds);
+                optAlwaysCreateTraceIds = traceConfig->getPropBool("@alwaysCreateTraceIds", optAlwaysCreateTraceIds);
             }
 
             // The global propagator should be set regardless of whether tracing is enabled or not.
@@ -942,6 +969,11 @@ class CTraceManager : implements ITraceManager, public CInterface
     {
         return optAlwaysCreateGlobalIds;
     }
+
+        virtual bool alwaysCreateTraceIds() const
+    {
+        return optAlwaysCreateTraceIds;
+    }
 };
 
 static Singleton<CTraceManager> theTraceManager;

system/jlib/jtrace.hpp

@@ -62,6 +62,7 @@ interface ITraceManager : extends IInterface
     virtual ISpan * createServerSpan(const char * name, const IProperties * httpHeaders, SpanFlags flags = SpanFlags::None) = 0;
     virtual bool isTracingEnabled() const = 0;
     virtual bool alwaysCreateGlobalIds() const = 0;
+    virtual bool alwaysCreateTraceIds() const = 0;
     virtual const char * getTracedComponentName() const = 0;
  };