diff --git a/esp/scm/ws_logaccess.ecm b/esp/scm/ws_logaccess.ecm index 455561e7f7f..f889495f140 100644 --- a/esp/scm/ws_logaccess.ecm +++ b/esp/scm/ws_logaccess.ecm @@ -24,7 +24,8 @@ ESPenum LogAccessType : int ByTargetAudience(4, "ByTargetAudience"), BySourceInstance(5, "BySourceInstance"), BySourceNode(6, "BySourceNode"), - ByFieldName(7, "ByFieldName") + ByFieldName(7, "ByFieldName"), + ByPod(8, "ByPod") }; ESPenum LogAccessLogFormat : int @@ -65,7 +66,8 @@ ESPenum LogColumnType : string logid("logid"), processid("processid"), threadid("threadid"), - timestamp("timestamp") + timestamp("timestamp"), + pod("pod") }; ESPenum LogColumnValueType : string @@ -185,7 +187,8 @@ ESPenum SortColumType : int ByTargetAudience(4, "ByTargetAudience"), BySourceInstance(5, "BySourceInstance"), BySourceNode(6, "BySourceNode"), - ByFieldName(7, "ByFieldName") + ByFieldName(7, "ByFieldName"), + ByPod(8, "ByPod") }; ESPStruct SortCondition diff --git a/esp/services/ws_logaccess/WsLogAccessService.cpp b/esp/services/ws_logaccess/WsLogAccessService.cpp index b0f055e26f9..97c035f6ab0 100644 --- a/esp/services/ws_logaccess/WsLogAccessService.cpp +++ b/esp/services/ws_logaccess/WsLogAccessService.cpp @@ -139,6 +139,8 @@ ILogAccessFilter * buildLogFilterByFields(CLogAccessType searchByCategory, const return getJobIDLogAccessFilter(searchByValue); case CLogAccessType_ByComponent: return getComponentLogAccessFilter(searchByValue); + case CLogAccessType_ByPod: + return getPodLogAccessFilter(searchByValue); case CLogAccessType_ByLogType: { LogMsgClass logType = LogMsgClassFromAbbrev(searchByValue); diff --git a/helm/examples/azure/log-analytics/loganalytics-hpcc-logaccessV2.yaml b/helm/examples/azure/log-analytics/loganalytics-hpcc-logaccessV2.yaml index f680737b823..ca249e5f262 100644 --- a/helm/examples/azure/log-analytics/loganalytics-hpcc-logaccessV2.yaml +++ b/helm/examples/azure/log-analytics/loganalytics-hpcc-logaccessV2.yaml @@ -78,6 +78,10 @@ global: searchColumn: "hpcc_log_timestamp" columnMode: "MIN" columnType: "datetime" + - type: "pod" + searchColumn: "PodName" + columnMode: "DEFAULT" + columnType: "string" secrets: esp: azure-logaccess: "azure-logaccess" diff --git a/helm/hpcc/values.schema.json b/helm/hpcc/values.schema.json index 5525f1916c2..5239201ecc8 100644 --- a/helm/hpcc/values.schema.json +++ b/helm/hpcc/values.schema.json @@ -2964,7 +2964,7 @@ "type": { "type": "string", "description" : "The searchable HPCC log column to be mapped - 'global' applies to all known fields", - "enum": [ "global", "workunits", "components", "audience", "class", "instance", "host", "node", "message", "logid", "processid", "threadid", "timestamp"] + "enum": [ "global", "workunits", "components", "audience", "class", "instance", "host", "node", "message", "logid", "processid", "threadid", "timestamp", "pod"] }, "timeStampColumn": { "description" : "Name of timestamp column related to mapped field (only requried for 'global' mapping)", diff --git a/system/jlib/jlog.cpp b/system/jlib/jlog.cpp index 6d813f878e5..89af8be1aec 100644 --- a/system/jlib/jlog.cpp +++ b/system/jlib/jlog.cpp @@ -3186,6 +3186,11 @@ ILogAccessFilter * getComponentLogAccessFilter(const char * component) return new FieldLogAccessFilter(component, LOGACCESS_FILTER_component); } +ILogAccessFilter * getPodLogAccessFilter(const char * podName) +{ + return new FieldLogAccessFilter(podName, LOGACCESS_FILTER_pod); +} + ILogAccessFilter * getAudienceLogAccessFilter(MessageAudience audience) { return new FieldLogAccessFilter(LogMsgAudienceToFixString(audience), LOGACCESS_FILTER_audience); @@ -3235,6 +3240,12 @@ bool fetchComponentLog(LogQueryResultDetails & resultDetails, StringBuffer & ret return fetchLog(resultDetails, returnbuf, logAccess, getComponentLogAccessFilter(component), timeRange, cols, format); } +// Fetches log entries based on provided pod name, via provided IRemoteLogAccess instance +bool fetchPodLog(LogQueryResultDetails & resultDetails, StringBuffer & returnbuf, IRemoteLogAccess & logAccess, const char * podName, LogAccessTimeRange timeRange, StringArray & cols, LogAccessLogFormat format = LOGACCESS_LOGFORMAT_json) +{ + return fetchLog(resultDetails, returnbuf, logAccess, getPodLogAccessFilter(podName), timeRange, cols, format); +} + // Fetches log entries based on provided audience, via provided IRemoteLogAccess instance bool fetchLogByAudience(LogQueryResultDetails & resultDetails, StringBuffer & returnbuf, IRemoteLogAccess & logAccess, MessageAudience audience, LogAccessTimeRange timeRange, StringArray & cols, LogAccessLogFormat format = LOGACCESS_LOGFORMAT_json) { diff --git a/system/jlib/jlog.hpp b/system/jlib/jlog.hpp index cfb7692af02..99ce2c89c8e 100644 --- a/system/jlib/jlog.hpp +++ b/system/jlib/jlog.hpp @@ -1361,6 +1361,7 @@ typedef enum LOGACCESS_FILTER_instance, LOGACCESS_FILTER_host, LOGACCESS_FILTER_column, + LOGACCESS_FILTER_pod, LOGACCESS_FILTER_unknown } LogAccessFilterType; @@ -1376,6 +1377,8 @@ inline const char * logAccessFilterTypeToString(LogAccessFilterType field) return "audience"; case LOGACCESS_FILTER_component: return "component"; + case LOGACCESS_FILTER_pod: + return "pod"; case LOGACCESS_FILTER_instance: return "instance"; case LOGACCESS_FILTER_host: @@ -1404,6 +1407,8 @@ inline unsigned logAccessFilterTypeFromName(char const * name) return LOGACCESS_FILTER_audience; if(strieq(name, "component")) return LOGACCESS_FILTER_component; + if(strieq(name, "pod")) + return LOGACCESS_FILTER_pod; if(strieq(name, "instance")) return LOGACCESS_FILTER_instance; if(strieq(name, "host")) @@ -1454,6 +1459,7 @@ enum LogAccessMappedField LOGACCESS_MAPPEDFIELD_class, LOGACCESS_MAPPEDFIELD_audience, LOGACCESS_MAPPEDFIELD_instance, + LOGACCESS_MAPPEDFIELD_pod, LOGACCESS_MAPPEDFIELD_host, LOGACCESS_MAPPEDFIELD_unmapped }; @@ -1653,6 +1659,7 @@ extern jlib_decl ILogAccessFilter * getInstanceLogAccessFilter(const char * inst extern jlib_decl ILogAccessFilter * getHostLogAccessFilter(const char * host); extern jlib_decl ILogAccessFilter * getJobIDLogAccessFilter(const char * jobId); extern jlib_decl ILogAccessFilter * getComponentLogAccessFilter(const char * component); +extern jlib_decl ILogAccessFilter * getPodLogAccessFilter(const char * podName); extern jlib_decl ILogAccessFilter * getAudienceLogAccessFilter(MessageAudience audience); extern jlib_decl ILogAccessFilter * getClassLogAccessFilter(LogMsgClass logclass); extern jlib_decl ILogAccessFilter * getBinaryLogAccessFilter(ILogAccessFilter * arg1, ILogAccessFilter * arg2, LogAccessFilterType type); diff --git a/system/logaccess/Azure/LogAnalytics/CurlClient/AzureLogAnalyticsCurlClient.cpp b/system/logaccess/Azure/LogAnalytics/CurlClient/AzureLogAnalyticsCurlClient.cpp index 6cd4c6bf59a..0ed01dd2c78 100644 --- a/system/logaccess/Azure/LogAnalytics/CurlClient/AzureLogAnalyticsCurlClient.cpp +++ b/system/logaccess/Azure/LogAnalytics/CurlClient/AzureLogAnalyticsCurlClient.cpp @@ -39,6 +39,7 @@ static constexpr const char * defaultHPCCLogComponentCol = "hpcc_log_component static constexpr const char * defaultHPCCLogTypeCol = "hpcc_log_class"; static constexpr const char * defaultHPCCLogAudCol = "hpcc_log_audience"; static constexpr const char * defaultHPCCLogComponentTSCol = "TimeGenerated"; +static constexpr const char * defaultPodHPCCLogCol = "PodName"; static constexpr const char * logMapIndexPatternAtt = "@storeName"; static constexpr const char * logMapSearchColAtt = "@searchColumn"; @@ -426,6 +427,13 @@ AzureLogAnalyticsCurlClient::AzureLogAnalyticsCurlClient(IPropertyTree & logAcce else OERRLOG("%s: Possible LogMap collision detected, 'host' and 'node' refer to same log column!", COMPONENT_NAME); } + else if (streq(logMapType, "pod")) + { + if (logMap.hasProp(logMapIndexPatternAtt)) + m_podIndexSearchPattern = logMap.queryProp(logMapIndexPatternAtt); + if (logMap.hasProp(logMapSearchColAtt)) + m_podSearchColName = logMap.queryProp(logMapSearchColAtt); + } else { ERRLOG("Encountered invalid LogAccess field map type: '%s'", logMapType); @@ -479,6 +487,9 @@ void AzureLogAnalyticsCurlClient::getDefaultReturnColumns(StringBuffer & columns columns.append(", "); } + if (!isEmptyString(m_podSearchColName)) + columns.appendf("%s, ", m_podSearchColName.str()); + columns.appendf("%s, %s, %s, %s, %s, %s, %s", m_globalIndexTimestampField.str(), defaultHPCCLogMessageCol, m_classSearchColName.str(), m_audienceSearchColName.str(), m_workunitSearchColName.str(), defaultHPCCLogSeqCol, defaultHPCCLogThreadIDCol); @@ -725,6 +736,19 @@ void AzureLogAnalyticsCurlClient::populateKQLQueryString(StringBuffer & queryStr queryString.append(" ) "); return; // queryString populated, need to break out } + case LOGACCESS_FILTER_pod: + { + queryField = m_podSearchColName.str(); + + if (!m_podIndexSearchPattern.isEmpty()) + { + throwIfMultiIndexDetected(queryIndex.str(), m_podIndexSearchPattern.str()); + queryIndex = m_podIndexSearchPattern.str(); + } + + DBGLOG("%s: Searching log entries by Pod: '%s'", COMPONENT_NAME, queryValue.str() ); + break; + } case LOGACCESS_FILTER_column: if (filter->getFieldName() == nullptr) throw makeStringExceptionV(-1, "%s: empty field name detected in filter by column!", COMPONENT_NAME); diff --git a/system/logaccess/Azure/LogAnalytics/CurlClient/AzureLogAnalyticsCurlClient.hpp b/system/logaccess/Azure/LogAnalytics/CurlClient/AzureLogAnalyticsCurlClient.hpp index b622db4d749..bedcff652a1 100644 --- a/system/logaccess/Azure/LogAnalytics/CurlClient/AzureLogAnalyticsCurlClient.hpp +++ b/system/logaccess/Azure/LogAnalytics/CurlClient/AzureLogAnalyticsCurlClient.hpp @@ -59,6 +59,9 @@ class AzureLogAnalyticsCurlClient : public CInterfaceOf StringBuffer m_hostSearchColName; StringBuffer m_hostIndexSearchPattern; + StringBuffer m_podIndexSearchPattern; + StringBuffer m_podSearchColName; + StringBuffer m_logAnalyticsWorkspaceID; StringBuffer m_aadTenantID; StringBuffer m_aadClientID;