HPCC-29674 LogAccess Pod data support

- Introduces PodName log column type - Adds filter support per PodName - Includes PodName in ALA default column requests - Adds Pod logMap to ALAv2 values file - Adds WsLogAccess byPod definitions Signed-off-by: Rodrigo Pastrana <[email protected]>
hpcc-systems · Oct 18, 2023 · af5aa8f · af5aa8f
1 parent bc1db47
commit af5aa8f
Show file tree

Hide file tree

Showing 8 changed files with 58 additions and 4 deletions.
diff --git a/esp/scm/ws_logaccess.ecm b/esp/scm/ws_logaccess.ecm
@@ -24,7 +24,8 @@ ESPenum LogAccessType : int
     ByTargetAudience(4, "ByTargetAudience"),
     BySourceInstance(5, "BySourceInstance"),
     BySourceNode(6, "BySourceNode"),
-    ByFieldName(7, "ByFieldName")
+    ByFieldName(7, "ByFieldName"),
+    ByPod(8, "ByPod")
 };
 
 ESPenum LogAccessLogFormat : int
@@ -65,7 +66,8 @@ ESPenum LogColumnType : string
     logid("logid"),
     processid("processid"),
     threadid("threadid"),
-    timestamp("timestamp")
+    timestamp("timestamp"),
+    pod("pod")
 };
 
 ESPenum LogColumnValueType : string
@@ -188,7 +190,8 @@ ESPenum SortColumType : int
     ByTargetAudience(4, "ByTargetAudience"),
     BySourceInstance(5, "BySourceInstance"),
     BySourceNode(6, "BySourceNode"),
-    ByFieldName(7, "ByFieldName")
+    ByFieldName(7, "ByFieldName"),
+    ByPod(8, "ByPod")
 };
 
 ESPStruct SortCondition

diff --git a/esp/services/ws_logaccess/WsLogAccessService.cpp b/esp/services/ws_logaccess/WsLogAccessService.cpp
@@ -139,6 +139,8 @@ ILogAccessFilter * buildLogFilterByFields(CLogAccessType searchByCategory, const
             return getJobIDLogAccessFilter(searchByValue);
         case CLogAccessType_ByComponent:
             return getComponentLogAccessFilter(searchByValue);
+        case CLogAccessType_ByPod:
+            return getPodLogAccessFilter(searchByValue);
         case CLogAccessType_ByLogType:
         {
             LogMsgClass logType = LogMsgClassFromAbbrev(searchByValue);

diff --git a/helm/examples/azure/log-analytics/loganalytics-hpcc-logaccessV2.yaml b/helm/examples/azure/log-analytics/loganalytics-hpcc-logaccessV2.yaml
@@ -78,6 +78,10 @@ global:
       searchColumn: "hpcc_log_timestamp"
       columnMode: "MIN"
       columnType: "datetime"
+    - type: "pod"
+      searchColumn: "PodName"
+      columnMode: "DEFAULT"
+      columnType: "string"
 secrets:
   esp:
     azure-logaccess: "azure-logaccess"

diff --git a/helm/hpcc/values.schema.json b/helm/hpcc/values.schema.json
@@ -3003,7 +3003,7 @@
         "type": {
           "type": "string",
           "description" : "The searchable HPCC log column to be mapped - 'global' applies to all known fields",
-          "enum": [ "global", "workunits", "components", "audience", "class", "instance", "host", "node", "message", "logid", "processid", "threadid", "timestamp"]
+          "enum": [ "global", "workunits", "components", "audience", "class", "instance", "host", "node", "message", "logid", "processid", "threadid", "timestamp", "pod"]
         },
         "timeStampColumn": {
           "description" : "Name of timestamp column related to mapped field (only requried for 'global' mapping)",

diff --git a/system/jlib/jlog.cpp b/system/jlib/jlog.cpp
@@ -3191,6 +3191,11 @@ ILogAccessFilter * getComponentLogAccessFilter(const char * component)
     return new FieldLogAccessFilter(component, LOGACCESS_FILTER_component);
 }
 
+ILogAccessFilter * getPodLogAccessFilter(const char * podName)
+{
+    return new FieldLogAccessFilter(podName, LOGACCESS_FILTER_pod);
+}
+
 ILogAccessFilter * getAudienceLogAccessFilter(MessageAudience audience)
 {
     return new FieldLogAccessFilter(LogMsgAudienceToFixString(audience), LOGACCESS_FILTER_audience);
@@ -3240,6 +3245,12 @@ bool fetchComponentLog(LogQueryResultDetails & resultDetails, StringBuffer & ret
     return fetchLog(resultDetails, returnbuf, logAccess, getComponentLogAccessFilter(component), timeRange, cols, format);
 }
 
+// Fetches log entries based on provided pod name, via provided IRemoteLogAccess instance
+bool fetchPodLog(LogQueryResultDetails & resultDetails, StringBuffer & returnbuf, IRemoteLogAccess & logAccess, const char * podName, LogAccessTimeRange timeRange, StringArray & cols, LogAccessLogFormat format = LOGACCESS_LOGFORMAT_json)
+{
+    return fetchLog(resultDetails, returnbuf, logAccess, getPodLogAccessFilter(podName), timeRange, cols, format);
+}
+
 // Fetches log entries based on provided audience, via provided IRemoteLogAccess instance
 bool fetchLogByAudience(LogQueryResultDetails & resultDetails, StringBuffer & returnbuf, IRemoteLogAccess & logAccess, MessageAudience audience, LogAccessTimeRange timeRange, StringArray & cols, LogAccessLogFormat format = LOGACCESS_LOGFORMAT_json)
 {

diff --git a/system/jlib/jlog.hpp b/system/jlib/jlog.hpp
@@ -1373,6 +1373,7 @@ typedef enum
     LOGACCESS_FILTER_instance,
     LOGACCESS_FILTER_host,
     LOGACCESS_FILTER_column,
+    LOGACCESS_FILTER_pod,
     LOGACCESS_FILTER_unknown
 } LogAccessFilterType;
 
@@ -1388,6 +1389,8 @@ inline const char * logAccessFilterTypeToString(LogAccessFilterType field)
         return "audience";
     case LOGACCESS_FILTER_component:
         return "component";
+    case LOGACCESS_FILTER_pod:
+        return "pod";
     case LOGACCESS_FILTER_instance:
         return "instance";
     case LOGACCESS_FILTER_host:
@@ -1416,6 +1419,8 @@ inline unsigned logAccessFilterTypeFromName(char const * name)
         return LOGACCESS_FILTER_audience;
     if(strieq(name, "component"))
         return LOGACCESS_FILTER_component;
+    if(strieq(name, "pod"))
+        return LOGACCESS_FILTER_pod;
     if(strieq(name, "instance"))
         return LOGACCESS_FILTER_instance;
     if(strieq(name, "host"))
@@ -1466,6 +1471,7 @@ enum LogAccessMappedField
     LOGACCESS_MAPPEDFIELD_class,
     LOGACCESS_MAPPEDFIELD_audience,
     LOGACCESS_MAPPEDFIELD_instance,
+    LOGACCESS_MAPPEDFIELD_pod,
     LOGACCESS_MAPPEDFIELD_host,
     LOGACCESS_MAPPEDFIELD_unmapped
 };
@@ -1665,6 +1671,7 @@ extern jlib_decl ILogAccessFilter * getInstanceLogAccessFilter(const char * inst
 extern jlib_decl ILogAccessFilter * getHostLogAccessFilter(const char * host);
 extern jlib_decl ILogAccessFilter * getJobIDLogAccessFilter(const char * jobId);
 extern jlib_decl ILogAccessFilter * getComponentLogAccessFilter(const char * component);
+extern jlib_decl ILogAccessFilter * getPodLogAccessFilter(const char * podName);
 extern jlib_decl ILogAccessFilter * getAudienceLogAccessFilter(MessageAudience audience);
 extern jlib_decl ILogAccessFilter * getClassLogAccessFilter(LogMsgClass logclass);
 extern jlib_decl ILogAccessFilter * getBinaryLogAccessFilter(ILogAccessFilter * arg1, ILogAccessFilter * arg2, LogAccessFilterType type);

diff --git a/system/logaccess/Azure/LogAnalytics/CurlClient/AzureLogAnalyticsCurlClient.cpp b/system/logaccess/Azure/LogAnalytics/CurlClient/AzureLogAnalyticsCurlClient.cpp
@@ -39,6 +39,7 @@ static constexpr const char * defaultHPCCLogComponentCol   = "hpcc_log_component
 static constexpr const char * defaultHPCCLogTypeCol        = "hpcc_log_class";
 static constexpr const char * defaultHPCCLogAudCol         = "hpcc_log_audience";
 static constexpr const char * defaultHPCCLogComponentTSCol = "TimeGenerated";
+static constexpr const char * defaultPodHPCCLogCol         = "PodName";
 
 static constexpr const char * logMapIndexPatternAtt = "@storeName";
 static constexpr const char * logMapSearchColAtt = "@searchColumn";
@@ -426,6 +427,13 @@ AzureLogAnalyticsCurlClient::AzureLogAnalyticsCurlClient(IPropertyTree & logAcce
             else
                 OERRLOG("%s: Possible LogMap collision detected, 'host' and 'node' refer to same log column!", COMPONENT_NAME); 
         }
+        else if (streq(logMapType, "pod"))
+        {
+            if (logMap.hasProp(logMapIndexPatternAtt))
+                m_podIndexSearchPattern = logMap.queryProp(logMapIndexPatternAtt);
+            if (logMap.hasProp(logMapSearchColAtt))
+                m_podSearchColName = logMap.queryProp(logMapSearchColAtt);
+        }
         else
         {
             ERRLOG("Encountered invalid LogAccess field map type: '%s'", logMapType);
@@ -479,6 +487,9 @@ void AzureLogAnalyticsCurlClient::getDefaultReturnColumns(StringBuffer & columns
         columns.append(", ");
     }
 
+    if (!isEmptyString(m_podSearchColName))
+        columns.appendf("%s, ", m_podSearchColName.str());
+
     columns.appendf("%s, %s, %s, %s, %s, %s, %s",
     m_globalIndexTimestampField.str(), defaultHPCCLogMessageCol, m_classSearchColName.str(),
     m_audienceSearchColName.str(), m_workunitSearchColName.str(), defaultHPCCLogSeqCol, defaultHPCCLogThreadIDCol);
@@ -725,6 +736,19 @@ void AzureLogAnalyticsCurlClient::populateKQLQueryString(StringBuffer & queryStr
         queryString.append(" ) ");
         return; // queryString populated, need to break out
     }
+    case LOGACCESS_FILTER_pod:
+    {
+        queryField = m_podSearchColName.str();
+
+        if (!m_podIndexSearchPattern.isEmpty())
+        {
+            throwIfMultiIndexDetected(queryIndex.str(),  m_podIndexSearchPattern.str());
+            queryIndex = m_podIndexSearchPattern.str();
+        }
+
+        DBGLOG("%s: Searching log entries by Pod: '%s'", COMPONENT_NAME, queryValue.str() );
+        break;
+    }
     case LOGACCESS_FILTER_column:
         if (filter->getFieldName() == nullptr)
             throw makeStringExceptionV(-1, "%s: empty field name detected in filter by column!", COMPONENT_NAME);

diff --git a/system/logaccess/Azure/LogAnalytics/CurlClient/AzureLogAnalyticsCurlClient.hpp b/system/logaccess/Azure/LogAnalytics/CurlClient/AzureLogAnalyticsCurlClient.hpp
@@ -59,6 +59,9 @@ class AzureLogAnalyticsCurlClient : public CInterfaceOf<IRemoteLogAccess>
     StringBuffer m_hostSearchColName;
     StringBuffer m_hostIndexSearchPattern;
 
+    StringBuffer m_podIndexSearchPattern;
+    StringBuffer m_podSearchColName;
+
     StringBuffer m_logAnalyticsWorkspaceID;
     StringBuffer m_aadTenantID;
     StringBuffer m_aadClientID;