Skip to content

Commit

Permalink
HPCC-30295 Add ALA ContainerLogV2 Support
Browse files Browse the repository at this point in the history 
- Updates sample values file
- Projects-away superfluous columns
- Honors lookup key config for instance type
- Fixes getXReturnColumns signature

Signed-off-by: Rodrigo Pastrana <[email protected]>
  • Loading branch information
@rpastrana
rpastrana committed Oct 2, 2023
1 parent bc375f5 commit 2664af5
Show file tree
Hide file tree
Showing 3 changed files with 47 additions and 22 deletions.
8 changes: 3 additions & 5 deletions helm/examples/azure/log-analytics/loganalytics-hpcc-logaccessV2.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,17 +22,16 @@ global:
columnType: "dynamic"
columnMode: "ALL"
- type: "workunits"
storeName: "ContainerLogV2"
searchColumn: "hpcc_log_jobid"
columnMode: "DEFAULT"
columnType: "string"
- type: "components"
storeName: "ContainerLogV2"
searchColumn: "ContainerName"
searchColumn: "ContainerName" # Container name happens to coincide with component name
keyColumn: "ContainerName"
columnMode: "DEFAULT"
columnType: "string"
disableJoins: true
projectName: "hpcc_log_component"
- type: "audience"
searchColumn: "hpcc_log_audience"
enumValues:
Expand All @@ -54,9 +53,8 @@ global:
columnMode: "DEFAULT"
columnType: "enum"
- type: "instance"
storeName: "ContainerLogV2"
searchColumn: "PodName"
projectName: "hpcc_log_pod"
keyColumn: "PodName"
columnMode: "DEFAULT"
columnType: "string"
- type: "node"
Expand Down
56 changes: 41 additions & 15 deletions system/logaccess/Azure/LogAnalytics/CurlClient/AzureLogAnalyticsCurlClient.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -374,7 +374,7 @@ AzureLogAnalyticsCurlClient::AzureLogAnalyticsCurlClient(IPropertyTree & logAcce

m_disableComponentNameJoins = logMap.getPropBool(logMapDisableJoinsAtt, false);
if (targetIsContainerLogV2)
m_disableComponentNameJoins = true;
m_disableComponentNameJoins = true; //Don't attempt a join on ContainerLogV2
else
{
if (strcmp("ContainerLogV2", m_componentsIndexSearchPattern)==0)
Expand Down Expand Up @@ -403,6 +403,8 @@ AzureLogAnalyticsCurlClient::AzureLogAnalyticsCurlClient(IPropertyTree & logAcce
m_instanceIndexSearchPattern = logMap.queryProp(logMapIndexPatternAtt);
if (logMap.hasProp(logMapSearchColAtt))
m_instanceSearchColName = logMap.queryProp(logMapSearchColAtt);
if (logMap.hasProp(logMapKeyColAtt))
m_instanceLookupKeyColumn = logMap.queryProp(logMapKeyColAtt);
}
else if (streq(logMapType, "node"))
{
Expand Down Expand Up @@ -431,42 +433,59 @@ AzureLogAnalyticsCurlClient::AzureLogAnalyticsCurlClient(IPropertyTree & logAcce
}
}

void AzureLogAnalyticsCurlClient::getMinReturnColumns(StringBuffer & columns, bool & includeComponentName)
void AzureLogAnalyticsCurlClient::getMinReturnColumns(StringBuffer & columns, const bool includeComponentName)
{
columns.append("\n| project ");
if (includeComponentName)
{
if (m_componentsSearchColName.length() > 0)
if (targetIsContainerLogV2 && m_componentsSearchColName.length() > 0)
{
columns.append(m_componentsSearchColName.str());
if (m_componentsLookupKeyColumn.length() > 0 && !strsame(m_componentsSearchColName.str(), m_componentsLookupKeyColumn.str()))
columns.appendf("%s=%s, ", m_componentsSearchColName.str(), m_componentsLookupKeyColumn.str());
columns.appendf("=%s", m_componentsLookupKeyColumn.str());
}
else
columns.appendf("%s, ", defaultHPCCLogComponentCol);
columns.append(defaultHPCCLogComponentCol);
columns.append(", ");
}
columns.appendf("%s, %s", m_globalIndexTimestampField.str(), defaultHPCCLogMessageCol);
}

void AzureLogAnalyticsCurlClient::getDefaultReturnColumns(StringBuffer & columns, bool & includeComponentName)
void AzureLogAnalyticsCurlClient::getDefaultReturnColumns(StringBuffer & columns, const bool includeComponentName)
{
columns.append("\n| project ");

if (includeComponentName)
{
if (m_componentsSearchColName.length() > 0)
if (targetIsContainerLogV2 && m_componentsSearchColName.length() > 0)
{
columns.append(m_componentsSearchColName.str());
if (m_componentsLookupKeyColumn.length() > 0 && !strsame(m_componentsSearchColName.str(), m_componentsLookupKeyColumn.str()))
columns.appendf("%s=%s, ", m_componentsSearchColName.str(), m_componentsLookupKeyColumn.str());
columns.appendf("=%s", m_componentsLookupKeyColumn.str());
}
else
columns.appendf("%s, ", defaultHPCCLogComponentCol);
{
columns.append(defaultHPCCLogComponentCol);
}
columns.append(", ");
}

if (targetIsContainerLogV2)
{
columns.appendf("%s", m_instanceSearchColName.str());

if (m_instanceLookupKeyColumn.length()>0 && !strsame(m_instanceLookupKeyColumn.str(),m_instanceSearchColName.str()))
columns.appendf("=%s, ", m_instanceLookupKeyColumn.str());
else
columns.append(", ");
}

columns.appendf("%s, %s, %s, %s, %s, %s, %s",
m_globalIndexTimestampField.str(), defaultHPCCLogMessageCol, m_classSearchColName.str(),
m_audienceSearchColName.str(), m_workunitSearchColName.str(), defaultHPCCLogSeqCol, defaultHPCCLogThreadIDCol);
}

bool generateHPCCLogColumnstAllColumns(StringBuffer & kql, const char * colName)
bool generateHPCCLogColumnstAllColumns(StringBuffer & kql, const char * colName, bool targetsV2)
{
if (isEmptyString(colName))
{
Expand All @@ -480,7 +499,6 @@ bool generateHPCCLogColumnstAllColumns(StringBuffer & kql, const char * colName)
else
sourceCol.append(colName);

//kql.appendf("\n| extend hpcclogfields = extract_all(@\'^([0-9A-Fa-f]+)\\s+(OPR|USR|PRG|AUD|UNK)\\s+(DIS|ERR|WRN|INF|PRO|MET|UNK)\\s+(\\d{4}-\\d{2}-\\d{2}\\s\\d{2}:\\d{2}:\\d{2}\\.\\d+)\\s+(\\d+)\\s+(\\d+)\\s+(UNK|[A-Z]\\d{8}-\\d{6}(?:-\\d+)?)\\s+\\\"(.*)\\\"$', %s)[0]", colName);
kql.appendf("\n| extend hpcclogfields = extract_all(@\'^([0-9A-Fa-f]+)\\s+(OPR|USR|PRG|AUD|UNK)\\s+(DIS|ERR|WRN|INF|PRO|MET|UNK)\\s+(\\d{4}-\\d{2}-\\d{2}\\s\\d{2}:\\d{2}:\\d{2}\\.\\d+)\\s+(\\d+)\\s+(\\d+)\\s+(UNK|[A-Z]\\d{8}-\\d{6}(?:-\\d+)?)\\s+\\\"(.*)\\\"$', %s)[0]", sourceCol.str());
kql.appendf("\n| extend %s = tostring(hpcclogfields.[0])", defaultHPCCLogSeqCol);
kql.appendf("\n| extend %s = tostring(hpcclogfields.[1])", defaultHPCCLogAudCol);
Expand All @@ -490,6 +508,13 @@ bool generateHPCCLogColumnstAllColumns(StringBuffer & kql, const char * colName)
kql.appendf("\n| extend %s = toint(hpcclogfields.[5])", defaultHPCCLogThreadIDCol);
kql.appendf("\n| extend %s = tostring(hpcclogfields.[6])", defaultHPCCLogJobIDCol);
kql.appendf("\n| extend %s = tostring(hpcclogfields.[7])", defaultHPCCLogMessageCol);
kql.appendf("\n| project-away hpcclogfields, Type, TenantId, _ResourceId, %s, ", colName);

if (targetsV2)
kql.append("LogSource, SourceSystem");
else
kql.append("LogEntrySource, TimeOfCommand, SourceSystem");


return true;
}
Expand Down Expand Up @@ -664,7 +689,10 @@ void AzureLogAnalyticsCurlClient::populateKQLQueryString(StringBuffer & queryStr
if (m_instanceSearchColName.isEmpty())
throw makeStringExceptionV(-1, "%s: 'Instance' log entry field not configured", COMPONENT_NAME);

queryField = m_instanceSearchColName.str();
if (m_instanceLookupKeyColumn.length()>0 && !strsame(m_instanceLookupKeyColumn.str(),m_instanceSearchColName.str()))
queryField = m_instanceLookupKeyColumn.str();
else
queryField = m_instanceSearchColName.str();

if (!m_instanceIndexSearchPattern.isEmpty())
{
Expand Down Expand Up @@ -742,15 +770,13 @@ void AzureLogAnalyticsCurlClient::populateKQLQueryString(StringBuffer & queryStr
queryIndex.set(m_globalIndexSearchPattern.str());

StringBuffer searchColumns;
//bool includeComponentName = !m_disableComponentNameJoins;
bool includeComponentName = !m_disableComponentNameJoins || targetIsContainerLogV2;
searchMetaData(searchColumns, options.getReturnColsMode(), options.getLogFieldNames(), includeComponentName, options.getLimit(), options.getStartFrom());
//if (includeComponentName)
if (!m_disableComponentNameJoins && !targetIsContainerLogV2)
declareContainerIndexJoinTable(queryString, options);

queryString.append(queryIndex);
generateHPCCLogColumnstAllColumns(queryString, m_globalSearchColName.str());
generateHPCCLogColumnstAllColumns(queryString, m_globalSearchColName.str(), targetIsContainerLogV2);

if (options.queryFilter() == nullptr || options.queryFilter()->filterType() == LOGACCESS_FILTER_wildcard) // No filter
{
Expand Down
5 changes: 3 additions & 2 deletions system/logaccess/Azure/LogAnalytics/CurlClient/AzureLogAnalyticsCurlClient.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -65,13 +65,14 @@ class AzureLogAnalyticsCurlClient : public CInterfaceOf<IRemoteLogAccess>
StringBuffer m_aadClientSecret;

StringBuffer m_componentsLookupKeyColumn;
StringBuffer m_instanceLookupKeyColumn;
bool targetIsContainerLogV2 = false;

public:
AzureLogAnalyticsCurlClient(IPropertyTree & logAccessPluginConfig);

void getMinReturnColumns(StringBuffer & columns, bool & includeComponentName);
void getDefaultReturnColumns(StringBuffer & columns, bool & includeComponentName);
void getMinReturnColumns(StringBuffer & columns, const bool includeComponentName);
void getDefaultReturnColumns(StringBuffer & columns, const bool includeComponentName);
void searchMetaData(StringBuffer & search, const LogAccessReturnColsMode retcolmode, const StringArray & selectcols, bool & includeComponentName, unsigned size = defaultEntryLimit, offset_t from = defaultEntryStart);
void populateKQLQueryString(StringBuffer & queryString, StringBuffer& queryIndex, const LogAccessConditions & options);
void populateKQLQueryString(StringBuffer & queryString, StringBuffer& queryIndex, const ILogAccessFilter * filter);
Expand Down

0 comments on commit 2664af5

Please sign in to comment.