diff --git a/helm/examples/azure/log-analytics/loganalytics-hpcc-logaccessV2.yaml b/helm/examples/azure/log-analytics/loganalytics-hpcc-logaccessV2.yaml index 844e21c7b8b..bdfff71c9ab 100644 --- a/helm/examples/azure/log-analytics/loganalytics-hpcc-logaccessV2.yaml +++ b/helm/examples/azure/log-analytics/loganalytics-hpcc-logaccessV2.yaml @@ -22,17 +22,16 @@ global: columnType: "dynamic" columnMode: "ALL" - type: "workunits" - storeName: "ContainerLogV2" searchColumn: "hpcc_log_jobid" columnMode: "DEFAULT" columnType: "string" - type: "components" storeName: "ContainerLogV2" - searchColumn: "ContainerName" + searchColumn: "ContainerName" # Container name happens to coincide with component name + keyColumn: "ContainerName" columnMode: "DEFAULT" columnType: "string" disableJoins: true - projectName: "hpcc_log_component" - type: "audience" searchColumn: "hpcc_log_audience" enumValues: @@ -54,9 +53,8 @@ global: columnMode: "DEFAULT" columnType: "enum" - type: "instance" - storeName: "ContainerLogV2" searchColumn: "PodName" - projectName: "hpcc_log_pod" + keyColumn: "PodName" columnMode: "DEFAULT" columnType: "string" - type: "node" diff --git a/system/logaccess/Azure/LogAnalytics/CurlClient/AzureLogAnalyticsCurlClient.cpp b/system/logaccess/Azure/LogAnalytics/CurlClient/AzureLogAnalyticsCurlClient.cpp index d69c8b31293..bed8f547e9d 100644 --- a/system/logaccess/Azure/LogAnalytics/CurlClient/AzureLogAnalyticsCurlClient.cpp +++ b/system/logaccess/Azure/LogAnalytics/CurlClient/AzureLogAnalyticsCurlClient.cpp @@ -374,7 +374,7 @@ AzureLogAnalyticsCurlClient::AzureLogAnalyticsCurlClient(IPropertyTree & logAcce m_disableComponentNameJoins = logMap.getPropBool(logMapDisableJoinsAtt, false); if (targetIsContainerLogV2) - m_disableComponentNameJoins = true; + m_disableComponentNameJoins = true; //Don't attempt a join on ContainerLogV2 else { if (strcmp("ContainerLogV2", m_componentsIndexSearchPattern)==0) @@ -403,6 +403,8 @@ AzureLogAnalyticsCurlClient::AzureLogAnalyticsCurlClient(IPropertyTree & logAcce m_instanceIndexSearchPattern = logMap.queryProp(logMapIndexPatternAtt); if (logMap.hasProp(logMapSearchColAtt)) m_instanceSearchColName = logMap.queryProp(logMapSearchColAtt); + if (logMap.hasProp(logMapKeyColAtt)) + m_instanceLookupKeyColumn = logMap.queryProp(logMapKeyColAtt); } else if (streq(logMapType, "node")) { @@ -431,34 +433,51 @@ AzureLogAnalyticsCurlClient::AzureLogAnalyticsCurlClient(IPropertyTree & logAcce } } -void AzureLogAnalyticsCurlClient::getMinReturnColumns(StringBuffer & columns, bool & includeComponentName) +void AzureLogAnalyticsCurlClient::getMinReturnColumns(StringBuffer & columns, const bool includeComponentName) { columns.append("\n| project "); if (includeComponentName) { - if (m_componentsSearchColName.length() > 0) + if (targetIsContainerLogV2 && m_componentsSearchColName.length() > 0) { + columns.append(m_componentsSearchColName.str()); if (m_componentsLookupKeyColumn.length() > 0 && !strsame(m_componentsSearchColName.str(), m_componentsLookupKeyColumn.str())) - columns.appendf("%s=%s, ", m_componentsSearchColName.str(), m_componentsLookupKeyColumn.str()); + columns.appendf("=%s", m_componentsLookupKeyColumn.str()); } else - columns.appendf("%s, ", defaultHPCCLogComponentCol); + columns.append(defaultHPCCLogComponentCol); + columns.append(", "); } columns.appendf("%s, %s", m_globalIndexTimestampField.str(), defaultHPCCLogMessageCol); } -void AzureLogAnalyticsCurlClient::getDefaultReturnColumns(StringBuffer & columns, bool & includeComponentName) +void AzureLogAnalyticsCurlClient::getDefaultReturnColumns(StringBuffer & columns, const bool includeComponentName) { columns.append("\n| project "); + if (includeComponentName) { - if (m_componentsSearchColName.length() > 0) + if (targetIsContainerLogV2 && m_componentsSearchColName.length() > 0) { + columns.append(m_componentsSearchColName.str()); if (m_componentsLookupKeyColumn.length() > 0 && !strsame(m_componentsSearchColName.str(), m_componentsLookupKeyColumn.str())) - columns.appendf("%s=%s, ", m_componentsSearchColName.str(), m_componentsLookupKeyColumn.str()); + columns.appendf("=%s", m_componentsLookupKeyColumn.str()); } else - columns.appendf("%s, ", defaultHPCCLogComponentCol); + { + columns.append(defaultHPCCLogComponentCol); + } + columns.append(", "); + } + + if (targetIsContainerLogV2) + { + columns.appendf("%s", m_instanceSearchColName.str()); + + if (m_instanceLookupKeyColumn.length()>0 && !strsame(m_instanceLookupKeyColumn.str(),m_instanceSearchColName.str())) + columns.appendf("=%s, ", m_instanceLookupKeyColumn.str()); + else + columns.append(", "); } columns.appendf("%s, %s, %s, %s, %s, %s, %s", @@ -466,7 +485,7 @@ void AzureLogAnalyticsCurlClient::getDefaultReturnColumns(StringBuffer & columns m_audienceSearchColName.str(), m_workunitSearchColName.str(), defaultHPCCLogSeqCol, defaultHPCCLogThreadIDCol); } -bool generateHPCCLogColumnstAllColumns(StringBuffer & kql, const char * colName) +bool generateHPCCLogColumnstAllColumns(StringBuffer & kql, const char * colName, bool targetsV2) { if (isEmptyString(colName)) { @@ -480,7 +499,6 @@ bool generateHPCCLogColumnstAllColumns(StringBuffer & kql, const char * colName) else sourceCol.append(colName); - //kql.appendf("\n| extend hpcclogfields = extract_all(@\'^([0-9A-Fa-f]+)\\s+(OPR|USR|PRG|AUD|UNK)\\s+(DIS|ERR|WRN|INF|PRO|MET|UNK)\\s+(\\d{4}-\\d{2}-\\d{2}\\s\\d{2}:\\d{2}:\\d{2}\\.\\d+)\\s+(\\d+)\\s+(\\d+)\\s+(UNK|[A-Z]\\d{8}-\\d{6}(?:-\\d+)?)\\s+\\\"(.*)\\\"$', %s)[0]", colName); kql.appendf("\n| extend hpcclogfields = extract_all(@\'^([0-9A-Fa-f]+)\\s+(OPR|USR|PRG|AUD|UNK)\\s+(DIS|ERR|WRN|INF|PRO|MET|UNK)\\s+(\\d{4}-\\d{2}-\\d{2}\\s\\d{2}:\\d{2}:\\d{2}\\.\\d+)\\s+(\\d+)\\s+(\\d+)\\s+(UNK|[A-Z]\\d{8}-\\d{6}(?:-\\d+)?)\\s+\\\"(.*)\\\"$', %s)[0]", sourceCol.str()); kql.appendf("\n| extend %s = tostring(hpcclogfields.[0])", defaultHPCCLogSeqCol); kql.appendf("\n| extend %s = tostring(hpcclogfields.[1])", defaultHPCCLogAudCol); @@ -490,6 +508,13 @@ bool generateHPCCLogColumnstAllColumns(StringBuffer & kql, const char * colName) kql.appendf("\n| extend %s = toint(hpcclogfields.[5])", defaultHPCCLogThreadIDCol); kql.appendf("\n| extend %s = tostring(hpcclogfields.[6])", defaultHPCCLogJobIDCol); kql.appendf("\n| extend %s = tostring(hpcclogfields.[7])", defaultHPCCLogMessageCol); + kql.appendf("\n| project-away hpcclogfields, Type, TenantId, _ResourceId, %s, ", colName); + + if (targetsV2) + kql.append("LogSource, SourceSystem"); + else + kql.append("LogEntrySource, TimeOfCommand, SourceSystem"); + return true; } @@ -664,7 +689,10 @@ void AzureLogAnalyticsCurlClient::populateKQLQueryString(StringBuffer & queryStr if (m_instanceSearchColName.isEmpty()) throw makeStringExceptionV(-1, "%s: 'Instance' log entry field not configured", COMPONENT_NAME); - queryField = m_instanceSearchColName.str(); + if (m_instanceLookupKeyColumn.length()>0 && !strsame(m_instanceLookupKeyColumn.str(),m_instanceSearchColName.str())) + queryField = m_instanceLookupKeyColumn.str(); + else + queryField = m_instanceSearchColName.str(); if (!m_instanceIndexSearchPattern.isEmpty()) { @@ -742,15 +770,13 @@ void AzureLogAnalyticsCurlClient::populateKQLQueryString(StringBuffer & queryStr queryIndex.set(m_globalIndexSearchPattern.str()); StringBuffer searchColumns; - //bool includeComponentName = !m_disableComponentNameJoins; bool includeComponentName = !m_disableComponentNameJoins || targetIsContainerLogV2; searchMetaData(searchColumns, options.getReturnColsMode(), options.getLogFieldNames(), includeComponentName, options.getLimit(), options.getStartFrom()); - //if (includeComponentName) if (!m_disableComponentNameJoins && !targetIsContainerLogV2) declareContainerIndexJoinTable(queryString, options); queryString.append(queryIndex); - generateHPCCLogColumnstAllColumns(queryString, m_globalSearchColName.str()); + generateHPCCLogColumnstAllColumns(queryString, m_globalSearchColName.str(), targetIsContainerLogV2); if (options.queryFilter() == nullptr || options.queryFilter()->filterType() == LOGACCESS_FILTER_wildcard) // No filter { diff --git a/system/logaccess/Azure/LogAnalytics/CurlClient/AzureLogAnalyticsCurlClient.hpp b/system/logaccess/Azure/LogAnalytics/CurlClient/AzureLogAnalyticsCurlClient.hpp index 7ddbb1d39ba..b622db4d749 100644 --- a/system/logaccess/Azure/LogAnalytics/CurlClient/AzureLogAnalyticsCurlClient.hpp +++ b/system/logaccess/Azure/LogAnalytics/CurlClient/AzureLogAnalyticsCurlClient.hpp @@ -65,13 +65,14 @@ class AzureLogAnalyticsCurlClient : public CInterfaceOf StringBuffer m_aadClientSecret; StringBuffer m_componentsLookupKeyColumn; + StringBuffer m_instanceLookupKeyColumn; bool targetIsContainerLogV2 = false; public: AzureLogAnalyticsCurlClient(IPropertyTree & logAccessPluginConfig); - void getMinReturnColumns(StringBuffer & columns, bool & includeComponentName); - void getDefaultReturnColumns(StringBuffer & columns, bool & includeComponentName); + void getMinReturnColumns(StringBuffer & columns, const bool includeComponentName); + void getDefaultReturnColumns(StringBuffer & columns, const bool includeComponentName); void searchMetaData(StringBuffer & search, const LogAccessReturnColsMode retcolmode, const StringArray & selectcols, bool & includeComponentName, unsigned size = defaultEntryLimit, offset_t from = defaultEntryStart); void populateKQLQueryString(StringBuffer & queryString, StringBuffer& queryIndex, const LogAccessConditions & options); void populateKQLQueryString(StringBuffer & queryString, StringBuffer& queryIndex, const ILogAccessFilter * filter);