From af5aa8fa6c1f0b731b4230af0c3770f486588464 Mon Sep 17 00:00:00 2001 From: Rodrigo Pastrana Date: Wed, 18 Oct 2023 08:54:40 -0400 Subject: [PATCH] HPCC-29674 LogAccess Pod data support - Introduces PodName log column type - Adds filter support per PodName - Includes PodName in ALA default column requests - Adds Pod logMap to ALAv2 values file - Adds WsLogAccess byPod definitions Signed-off-by: Rodrigo Pastrana --- esp/scm/ws_logaccess.ecm | 9 ++++--- .../ws_logaccess/WsLogAccessService.cpp | 2 ++ .../loganalytics-hpcc-logaccessV2.yaml | 4 ++++ helm/hpcc/values.schema.json | 2 +- system/jlib/jlog.cpp | 11 +++++++++ system/jlib/jlog.hpp | 7 ++++++ .../AzureLogAnalyticsCurlClient.cpp | 24 +++++++++++++++++++ .../AzureLogAnalyticsCurlClient.hpp | 3 +++ 8 files changed, 58 insertions(+), 4 deletions(-) diff --git a/esp/scm/ws_logaccess.ecm b/esp/scm/ws_logaccess.ecm index d89dc4d4a30..8fa42f07e96 100644 --- a/esp/scm/ws_logaccess.ecm +++ b/esp/scm/ws_logaccess.ecm @@ -24,7 +24,8 @@ ESPenum LogAccessType : int ByTargetAudience(4, "ByTargetAudience"), BySourceInstance(5, "BySourceInstance"), BySourceNode(6, "BySourceNode"), - ByFieldName(7, "ByFieldName") + ByFieldName(7, "ByFieldName"), + ByPod(8, "ByPod") }; ESPenum LogAccessLogFormat : int @@ -65,7 +66,8 @@ ESPenum LogColumnType : string logid("logid"), processid("processid"), threadid("threadid"), - timestamp("timestamp") + timestamp("timestamp"), + pod("pod") }; ESPenum LogColumnValueType : string @@ -188,7 +190,8 @@ ESPenum SortColumType : int ByTargetAudience(4, "ByTargetAudience"), BySourceInstance(5, "BySourceInstance"), BySourceNode(6, "BySourceNode"), - ByFieldName(7, "ByFieldName") + ByFieldName(7, "ByFieldName"), + ByPod(8, "ByPod") }; ESPStruct SortCondition diff --git a/esp/services/ws_logaccess/WsLogAccessService.cpp b/esp/services/ws_logaccess/WsLogAccessService.cpp index b0f055e26f9..97c035f6ab0 100644 --- a/esp/services/ws_logaccess/WsLogAccessService.cpp +++ b/esp/services/ws_logaccess/WsLogAccessService.cpp @@ -139,6 +139,8 @@ ILogAccessFilter * buildLogFilterByFields(CLogAccessType searchByCategory, const return getJobIDLogAccessFilter(searchByValue); case CLogAccessType_ByComponent: return getComponentLogAccessFilter(searchByValue); + case CLogAccessType_ByPod: + return getPodLogAccessFilter(searchByValue); case CLogAccessType_ByLogType: { LogMsgClass logType = LogMsgClassFromAbbrev(searchByValue); diff --git a/helm/examples/azure/log-analytics/loganalytics-hpcc-logaccessV2.yaml b/helm/examples/azure/log-analytics/loganalytics-hpcc-logaccessV2.yaml index f680737b823..ca249e5f262 100644 --- a/helm/examples/azure/log-analytics/loganalytics-hpcc-logaccessV2.yaml +++ b/helm/examples/azure/log-analytics/loganalytics-hpcc-logaccessV2.yaml @@ -78,6 +78,10 @@ global: searchColumn: "hpcc_log_timestamp" columnMode: "MIN" columnType: "datetime" + - type: "pod" + searchColumn: "PodName" + columnMode: "DEFAULT" + columnType: "string" secrets: esp: azure-logaccess: "azure-logaccess" diff --git a/helm/hpcc/values.schema.json b/helm/hpcc/values.schema.json index 569f7ad715b..7095bea25d3 100644 --- a/helm/hpcc/values.schema.json +++ b/helm/hpcc/values.schema.json @@ -3003,7 +3003,7 @@ "type": { "type": "string", "description" : "The searchable HPCC log column to be mapped - 'global' applies to all known fields", - "enum": [ "global", "workunits", "components", "audience", "class", "instance", "host", "node", "message", "logid", "processid", "threadid", "timestamp"] + "enum": [ "global", "workunits", "components", "audience", "class", "instance", "host", "node", "message", "logid", "processid", "threadid", "timestamp", "pod"] }, "timeStampColumn": { "description" : "Name of timestamp column related to mapped field (only requried for 'global' mapping)", diff --git a/system/jlib/jlog.cpp b/system/jlib/jlog.cpp index 70efd4d3f84..d7485432d06 100644 --- a/system/jlib/jlog.cpp +++ b/system/jlib/jlog.cpp @@ -3191,6 +3191,11 @@ ILogAccessFilter * getComponentLogAccessFilter(const char * component) return new FieldLogAccessFilter(component, LOGACCESS_FILTER_component); } +ILogAccessFilter * getPodLogAccessFilter(const char * podName) +{ + return new FieldLogAccessFilter(podName, LOGACCESS_FILTER_pod); +} + ILogAccessFilter * getAudienceLogAccessFilter(MessageAudience audience) { return new FieldLogAccessFilter(LogMsgAudienceToFixString(audience), LOGACCESS_FILTER_audience); @@ -3240,6 +3245,12 @@ bool fetchComponentLog(LogQueryResultDetails & resultDetails, StringBuffer & ret return fetchLog(resultDetails, returnbuf, logAccess, getComponentLogAccessFilter(component), timeRange, cols, format); } +// Fetches log entries based on provided pod name, via provided IRemoteLogAccess instance +bool fetchPodLog(LogQueryResultDetails & resultDetails, StringBuffer & returnbuf, IRemoteLogAccess & logAccess, const char * podName, LogAccessTimeRange timeRange, StringArray & cols, LogAccessLogFormat format = LOGACCESS_LOGFORMAT_json) +{ + return fetchLog(resultDetails, returnbuf, logAccess, getPodLogAccessFilter(podName), timeRange, cols, format); +} + // Fetches log entries based on provided audience, via provided IRemoteLogAccess instance bool fetchLogByAudience(LogQueryResultDetails & resultDetails, StringBuffer & returnbuf, IRemoteLogAccess & logAccess, MessageAudience audience, LogAccessTimeRange timeRange, StringArray & cols, LogAccessLogFormat format = LOGACCESS_LOGFORMAT_json) { diff --git a/system/jlib/jlog.hpp b/system/jlib/jlog.hpp index 9686e21feca..39723c6741e 100644 --- a/system/jlib/jlog.hpp +++ b/system/jlib/jlog.hpp @@ -1373,6 +1373,7 @@ typedef enum LOGACCESS_FILTER_instance, LOGACCESS_FILTER_host, LOGACCESS_FILTER_column, + LOGACCESS_FILTER_pod, LOGACCESS_FILTER_unknown } LogAccessFilterType; @@ -1388,6 +1389,8 @@ inline const char * logAccessFilterTypeToString(LogAccessFilterType field) return "audience"; case LOGACCESS_FILTER_component: return "component"; + case LOGACCESS_FILTER_pod: + return "pod"; case LOGACCESS_FILTER_instance: return "instance"; case LOGACCESS_FILTER_host: @@ -1416,6 +1419,8 @@ inline unsigned logAccessFilterTypeFromName(char const * name) return LOGACCESS_FILTER_audience; if(strieq(name, "component")) return LOGACCESS_FILTER_component; + if(strieq(name, "pod")) + return LOGACCESS_FILTER_pod; if(strieq(name, "instance")) return LOGACCESS_FILTER_instance; if(strieq(name, "host")) @@ -1466,6 +1471,7 @@ enum LogAccessMappedField LOGACCESS_MAPPEDFIELD_class, LOGACCESS_MAPPEDFIELD_audience, LOGACCESS_MAPPEDFIELD_instance, + LOGACCESS_MAPPEDFIELD_pod, LOGACCESS_MAPPEDFIELD_host, LOGACCESS_MAPPEDFIELD_unmapped }; @@ -1665,6 +1671,7 @@ extern jlib_decl ILogAccessFilter * getInstanceLogAccessFilter(const char * inst extern jlib_decl ILogAccessFilter * getHostLogAccessFilter(const char * host); extern jlib_decl ILogAccessFilter * getJobIDLogAccessFilter(const char * jobId); extern jlib_decl ILogAccessFilter * getComponentLogAccessFilter(const char * component); +extern jlib_decl ILogAccessFilter * getPodLogAccessFilter(const char * podName); extern jlib_decl ILogAccessFilter * getAudienceLogAccessFilter(MessageAudience audience); extern jlib_decl ILogAccessFilter * getClassLogAccessFilter(LogMsgClass logclass); extern jlib_decl ILogAccessFilter * getBinaryLogAccessFilter(ILogAccessFilter * arg1, ILogAccessFilter * arg2, LogAccessFilterType type); diff --git a/system/logaccess/Azure/LogAnalytics/CurlClient/AzureLogAnalyticsCurlClient.cpp b/system/logaccess/Azure/LogAnalytics/CurlClient/AzureLogAnalyticsCurlClient.cpp index 6cd4c6bf59a..0ed01dd2c78 100644 --- a/system/logaccess/Azure/LogAnalytics/CurlClient/AzureLogAnalyticsCurlClient.cpp +++ b/system/logaccess/Azure/LogAnalytics/CurlClient/AzureLogAnalyticsCurlClient.cpp @@ -39,6 +39,7 @@ static constexpr const char * defaultHPCCLogComponentCol = "hpcc_log_component static constexpr const char * defaultHPCCLogTypeCol = "hpcc_log_class"; static constexpr const char * defaultHPCCLogAudCol = "hpcc_log_audience"; static constexpr const char * defaultHPCCLogComponentTSCol = "TimeGenerated"; +static constexpr const char * defaultPodHPCCLogCol = "PodName"; static constexpr const char * logMapIndexPatternAtt = "@storeName"; static constexpr const char * logMapSearchColAtt = "@searchColumn"; @@ -426,6 +427,13 @@ AzureLogAnalyticsCurlClient::AzureLogAnalyticsCurlClient(IPropertyTree & logAcce else OERRLOG("%s: Possible LogMap collision detected, 'host' and 'node' refer to same log column!", COMPONENT_NAME); } + else if (streq(logMapType, "pod")) + { + if (logMap.hasProp(logMapIndexPatternAtt)) + m_podIndexSearchPattern = logMap.queryProp(logMapIndexPatternAtt); + if (logMap.hasProp(logMapSearchColAtt)) + m_podSearchColName = logMap.queryProp(logMapSearchColAtt); + } else { ERRLOG("Encountered invalid LogAccess field map type: '%s'", logMapType); @@ -479,6 +487,9 @@ void AzureLogAnalyticsCurlClient::getDefaultReturnColumns(StringBuffer & columns columns.append(", "); } + if (!isEmptyString(m_podSearchColName)) + columns.appendf("%s, ", m_podSearchColName.str()); + columns.appendf("%s, %s, %s, %s, %s, %s, %s", m_globalIndexTimestampField.str(), defaultHPCCLogMessageCol, m_classSearchColName.str(), m_audienceSearchColName.str(), m_workunitSearchColName.str(), defaultHPCCLogSeqCol, defaultHPCCLogThreadIDCol); @@ -725,6 +736,19 @@ void AzureLogAnalyticsCurlClient::populateKQLQueryString(StringBuffer & queryStr queryString.append(" ) "); return; // queryString populated, need to break out } + case LOGACCESS_FILTER_pod: + { + queryField = m_podSearchColName.str(); + + if (!m_podIndexSearchPattern.isEmpty()) + { + throwIfMultiIndexDetected(queryIndex.str(), m_podIndexSearchPattern.str()); + queryIndex = m_podIndexSearchPattern.str(); + } + + DBGLOG("%s: Searching log entries by Pod: '%s'", COMPONENT_NAME, queryValue.str() ); + break; + } case LOGACCESS_FILTER_column: if (filter->getFieldName() == nullptr) throw makeStringExceptionV(-1, "%s: empty field name detected in filter by column!", COMPONENT_NAME); diff --git a/system/logaccess/Azure/LogAnalytics/CurlClient/AzureLogAnalyticsCurlClient.hpp b/system/logaccess/Azure/LogAnalytics/CurlClient/AzureLogAnalyticsCurlClient.hpp index b622db4d749..bedcff652a1 100644 --- a/system/logaccess/Azure/LogAnalytics/CurlClient/AzureLogAnalyticsCurlClient.hpp +++ b/system/logaccess/Azure/LogAnalytics/CurlClient/AzureLogAnalyticsCurlClient.hpp @@ -59,6 +59,9 @@ class AzureLogAnalyticsCurlClient : public CInterfaceOf StringBuffer m_hostSearchColName; StringBuffer m_hostIndexSearchPattern; + StringBuffer m_podIndexSearchPattern; + StringBuffer m_podSearchColName; + StringBuffer m_logAnalyticsWorkspaceID; StringBuffer m_aadTenantID; StringBuffer m_aadClientID;