HPCC-32465 Add ESP support for trace level

- Define an ESP process configuration node that supports specification of global
  TraceFlags values for each ESP.
- Reserve traceDetail to request a specific trace level (0: none, 1: standard,
  2: detailed, 3: max). This replaces acting on the last observed occurrence
  of either traceNone, traceStandard, traceDetailed, and traceMax.
- Override default flag settings when not configured and a debug build.

Signed-off-by: Tim Klemm <[email protected]>

esp/platform/application_config.cpp

@@ -27,6 +27,7 @@
 #include "espcfg.ipp"
 #include "esplog.hpp"
 #include "espcontext.hpp"
+#include "esptrace.h"
 
 enum class LdapType { LegacyAD, AzureAD };
 
@@ -436,6 +437,15 @@ void setLDAPSecurityInWSAccess(IPropertyTree *legacyEsp, IPropertyTree *legacyLd
     }
 }
 
+// Copy trace flags from appEsp to legacyEsp. The source is expected to be an application's `esp`
+// configuration object. The destination is expected to be the `EspProcess` element.
+inline static void addTraceFlags(IPropertyTree *legacyEsp, IPropertyTree *appEsp)
+{
+    IPropertyTree *traceFlags = appEsp->queryPropTree(propTraceFlags);
+    if (traceFlags)
+        legacyEsp->setPropTree(propTraceFlags, traceFlags);
+}
+
 IPropertyTree *buildApplicationLegacyConfig(const char *application, const char* argv[])
 {
     Owned<IPropertyTree> appEspConfig = loadApplicationConfig(application, argv);
@@ -468,5 +478,7 @@ IPropertyTree *buildApplicationLegacyConfig(const char *application, const char*
     IPropertyTree *legacyDirectories = legacy->queryPropTree("Software/Directories");
     IPropertyTree *appDirectories = appEspConfig->queryPropTree("directories");
     copyDirectories(legacyDirectories, appDirectories);
+
+    addTraceFlags(legacyEsp, appEspConfig);
     return legacy.getClear();
 }

esp/platform/espp.cpp

@@ -49,6 +49,7 @@
 
 #include "jmetrics.hpp"
 #include "workunit.hpp"
+#include "esptrace.h"
 
 using namespace hpccMetrics;
 
@@ -354,6 +355,33 @@ static void usage()
 
 IPropertyTree *buildApplicationLegacyConfig(const char *application, const char* argv[]);
 
+//
+// Initialize trace settings
+void initializeTraceFlags(CEspConfig* config)
+{
+    IPropertyTree* pEspTree = config->queryConfigPTree();
+    Owned<IPropertyTree> pTraceTree = pEspTree->getPropTree(propTraceFlags);
+    if (!pTraceTree)
+    {
+        pTraceTree.setown(getComponentConfigSP()->getPropTree(propTraceFlags));
+    }
+#ifdef _DEBUG
+    if (!pTraceTree)
+    {
+        static const char * defaultTraceFlagsYaml = R"!!(
+traceDetail: max
+)!!";
+        pTraceTree.setown(createPTreeFromYAMLString(defaultTraceFlagsYaml));
+    }
+#endif
+
+    if (pTraceTree)
+    {
+        TraceFlags defaults = loadTraceFlags(pTraceTree, mapTraceOptions(pEspTree), queryDefaultTraceFlags());
+        updateTraceFlags(defaults, true);
+    }
+}
+
 //
 // Initialize metrics
 void initializeMetrics(CEspConfig* config)
@@ -577,6 +605,7 @@ int init_main(int argc, const char* argv[])
             config->bindServer(*server.get(), *server.get());
             config->checkESPCache(*server.get());
 
+            initializeTraceFlags(config);
             initializeMetrics(config);        
             initializeStorageGroups(daliClientActive());
         }

esp/platform/esptrace.h

@@ -0,0 +1,58 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2024 HPCC Systems®.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+#pragma once
+
+#include "jtrace.hpp"
+#include <initializer_list>
+
+// Refer to helm/examples/esp/README.md for more information on the trace options.
+
+constexpr const char* propTraceFlags = "traceFlags";
+
+// Trace option list fragment for jtrace-defined options used by ESPs
+#define PLATFORM_OPTIONS_FRAGMENT
+
+// Trace option list fragment for options used by most ESPs
+#define ESP_OPTIONS_FRAGMENT \
+    PLATFORM_OPTIONS_FRAGMENT \
+    TRACEOPT(traceDetail),
+
+// Trace option initializer list for ESPs that do not define their own options.
+constexpr std::initializer_list<TraceOption> espTraceOptions
+{
+    ESP_OPTIONS_FRAGMENT
+};
+
+/**
+ * @brief Returns the trace options appropriate for the ESP process being initialized.
+ *
+ * Most ESPs will simply return espTraceOptions. Any ESP that defines options not applicable to
+ * other ESPs would return a different list. The determination of which list to return is
+ * expected to be based on the configuration's `application` property.
+ *
+ * If options for all ESPs are defined with no value collisions, there may be no need to define
+ * separate option lists for individual ESPs. However, if value collisions cannot be avoided,
+ * separate lists will be needed.
+ *
+ * Consider ESDL Script, and the applications that use it. The potential for a significant number
+ * of options is high, increasing the likelihood of collisions with applications that don't use it.
+ */
+inline const std::initializer_list<TraceOption>& mapTraceOptions(const IPTree* config)
+{
+    return espTraceOptions;
+}

helm/examples/esp/README.md

@@ -0,0 +1,62 @@
+# ESP Trace Flags
+
+Each ESP process may specify its own set of flags for controlling trace behavior. The specific flags may be shared with other platform components, shared amongst ESPs, or unique to an ESP.
+
+## Accepted Flags
+
+Detailed description of flags used by any ESP.
+
+### Shared Platform Flags
+
+Flags defined in `system/jlib/jtrace.hpp` and used by multiple platform processes.
+
+Flags will be added to this list as tracing logic is updated in ESP code. For example, the shared platform flag `traceHttp` is expected to be used, as are a number of ESP-specific options.
+
+### Shared ESP Flags
+
+Flags defined in `esp/platform/esptrace.h` and applicable to most, if not all, ESP configurations.
+
+#### traceDetail
+
+Set the default trace level in the process. Accepted case-insensitive values are:
+- `1`, `standard`: most important output
+- `2`, `detailed`: average verbosity output
+- `3`, `max`: highest verbosity output
+- `default`: use existing level
+  - Release builds default to `standard`
+  - Debug builds default to `max`
+- `0`, `none`, *all other values*: no trace output
+
+## Process Configuration
+
+### Containerized
+
+#### esp.yaml
+
+Each ESP application's configuration object may embed one instance of a `traceFlags` object. Within this object, at most one property per [accepted flag](#accepted-flags) is expected. Properties not described here are ignored.
+
+For example, the `eclwatch` process might be configured to use detailed reporting like this:
+
+```yml
+esp:
+- name: eclwatch
+  traceFlags:
+    traceDetail: 2
+```
+
+## Cluster Overrides
+
+A values file may be used with the `helm install` command to override the settings of all ESPs. The `--set` option may be used to target the settings of a specific ESP in the configured array.
+
+### Bare-Metal
+
+No support for defining trace flags is provided by the `configmgr` application. Within a stand-alone `esp.xml` file, however, a `traceFlags` child of the `EspProcess` element may be defined.
+
+The previous YAML example may be reproduced in XML with the following:
+
+```xml
+<EspProcess ...>
+  <traceFlags traceDetail="2" />
+  ...
+<EspProcess>
+```

system/jlib/jlog.cpp

@@ -3306,27 +3306,46 @@ void JobNameScope::set(const char * name)
 
 TraceFlags loadTraceFlags(const IPropertyTree *ptree, const std::initializer_list<TraceOption> &optNames, TraceFlags dft)
 {
-    for (auto &o: optNames)
+    for (const TraceOption& option: optNames)
     {
-        VStringBuffer attrName("@%s", o.name);
-        if (!ptree->hasProp(attrName))
-            attrName.clear().appendf("_%s", o.name);
-        if (ptree->hasProp(attrName))
+        VStringBuffer attrName("@%s", option.name);
+        const char* value = ptree->queryProp(attrName);
+        if (!value)
         {
-            if (o.value <= TraceFlags::LevelMask)
-            {
-                dft &= ~TraceFlags::LevelMask;
-                dft |= o.value;
-            }
+            attrName.setCharAt(0, '_');
+            value = ptree->queryProp(attrName);
+            if (!value)
+                continue;
+        }
+        if (strieq(value, "default")) // allow a configuration to explicitly request a default value
+            continue;
+        if (option.value == traceDetail) // non-Boolean traceDetail
+        {
+            dft &= ~TraceFlags::LevelMask;
+            if (strieq(value, "standard"))
+                dft |= traceStandard;
+            else if (strieq(value, "detailed"))
+                dft |= traceDetailed;
+            else if (strieq(value, "max"))
+                dft |= traceMax;
             else
             {
-                if (ptree->getPropBool(attrName, false))
-                    dft |= o.value;
-                else
-                    dft &= ~o.value;
+                char* endptr = nullptr;
+                unsigned tmp = strtoul(value, &endptr, 10);
+                if (endptr && !*endptr && TraceFlags(tmp) <= TraceFlags::LevelMask)
+                    dft |= TraceFlags(tmp);
             }
         }
-
+        else if (option.value <= TraceFlags::LevelMask) // block individual trace level names
+            continue;
+        else // Boolean trace options
+        {
+            bool flag = strToBool(value);
+            if (flag)
+                dft |= option.value;
+            else
+                dft &= ~option.value;
+        }
     }
     return dft;
 }

system/jlib/jtrace.hpp

@@ -325,6 +325,7 @@ constexpr TraceFlags traceNone = TraceFlags::None;
 constexpr TraceFlags traceStandard = TraceFlags::Standard;
 constexpr TraceFlags traceDetailed = TraceFlags::Detailed;
 constexpr TraceFlags traceMax = TraceFlags::Max;
+constexpr TraceFlags traceDetail = TraceFlags(0xFFFFFFFF);   // reserved term for one of traceNone, traceStandard, traceDetailed, or traceMax values
 constexpr TraceFlags traceAll = (TraceFlags)(~TraceFlags::LevelMask);   // i.e. all feature flags except for the detail level
 
 // Common to several engines