From ffa9cca16f231138068a03da8db7adb80cfc182b Mon Sep 17 00:00:00 2001 From: Rodrigo Pastrana Date: Fri, 20 Dec 2024 11:37:24 -0500 Subject: [PATCH] HPCC-32874 Code review 1 - Adds ESP respons structure - Introduces top level gree/yellow/red status concept - Introduces jlog structs to help status reporting - Utilizes JSON appending/encoding functionality - Various other minor changes Signed-off-by: Rodrigo Pastrana --- esp/scm/ws_logaccess.ecm | 46 ++++- .../ws_logaccess/WsLogAccessService.cpp | 48 +++-- .../loki-stack/grafana-hpcc-logaccess.yaml | 2 +- system/jlib/jlog.hpp | 115 ++++------- .../AzureLogAnalyticsCurlClient.cpp | 195 ++++++++++++------ .../AzureLogAnalyticsCurlClient.hpp | 2 +- .../ElasticStack/ElasticStackLogAccess.cpp | 74 ++++--- .../ElasticStack/ElasticStackLogAccess.hpp | 2 +- .../Grafana/CurlClient/GrafanaCurlClient.cpp | 17 +- .../Grafana/CurlClient/GrafanaCurlClient.hpp | 4 +- 10 files changed, 303 insertions(+), 202 deletions(-) diff --git a/esp/scm/ws_logaccess.ecm b/esp/scm/ws_logaccess.ecm index a91e20490b3..5d82e8ee9f6 100644 --- a/esp/scm/ws_logaccess.ecm +++ b/esp/scm/ws_logaccess.ecm @@ -231,14 +231,50 @@ ESPResponse GetLogsResponse ESPRequest GetHealthReportRequest { - bool IncludeServerInternals(true); - bool IncludePluginInternals(true); - bool IncludeSampleQuery(true); + bool IncludeConfiguration(false); + bool IncludeDebugReport(false); + bool IncludeSampleQuery(false); }; -ESPResponse GetHealthReportResponse +ESPenum LogAccessStatusCode : int { - string Report; + Unknown(0, "Unknown"), + Green(1, "Green"), + Yellow(2, "Yellow"), + Red(3, "Red") +}; + +/* +ESPenum LogAccessStatusCode : int +{ + NotConfigured(0, "NotConfigured"), + Misconfigured(1, "Misconfigured"), + FailedToConnect(2, "FailedToConnect"), + FailedToAuth(3, "FailedToAuth"), + FailedToQuery(4, "FailedToQuery"), + EmptyQueryResult(5, "EmptyQueryResult"), + Warned(6, "Warned"), + Failed(7, "Failed") +};*/ + +ESPStruct LogAccessStatus +{ + LogAccessStatusCode Code; + LogAccessStatusCode Messages; +}; + +ESPStruct LogAccessDebugReport +{ + string SampleQueryReport; + string PluginDebugReport; + string ServerDebugReport; +}; + +ESPResponse [exceptions_inline] GetHealthReportResponse +{ + ESPStruct LogAccessStatus Status; + ESPStruct LogAccessDebugReport DebugReport; + string Configuration; }; ESPservice [auth_feature("WsLogAccess:READ"), version("1.07"), default_client_version("1.07"), exceptions_inline("xslt/exceptions.xslt")] ws_logaccess diff --git a/esp/services/ws_logaccess/WsLogAccessService.cpp b/esp/services/ws_logaccess/WsLogAccessService.cpp index 092e5ffab22..3dde28f79a1 100644 --- a/esp/services/ws_logaccess/WsLogAccessService.cpp +++ b/esp/services/ws_logaccess/WsLogAccessService.cpp @@ -388,27 +388,49 @@ bool Cws_logaccessEx::onGetLogs(IEspContext &context, IEspGetLogsRequest &req, I bool Cws_logaccessEx::onGetHealthReport(IEspContext &context, IEspGetHealthReportRequest &req, IEspGetHealthReportResponse &resp) { + IEspLogAccessStatus * status = createLogAccessStatus("",""); + StringBuffer report; - //LogAccessHealthReportDetails reportDetails; + LogAccessHealthReportDetails reportDetails; LogAccessHealthReportOptions options; - options.IncludeServerInternals = req.getIncludeServerInternals(); - options.IncludePluginInternals = req.getIncludePluginInternals(); + options.IncludeConfiguration = req.getIncludeConfiguration(); + options.IncludeDebugReport = req.getIncludeDebugReport(); options.IncludeSampleQuery = req.getIncludeSampleQuery(); - report.set("{ "); - bool success = true; if (!queryRemoteLogAccessor()) { - report.append("\"Error\": \"LogAccess plugin not available, review logAccess configuration!\""); - success = false; + status->setCode("Red"); + status->setMessages("Configuration Error - LogAccess plugin not available, review logAccess configuration!"); } else { - //queryRemoteLogAccessor()->healthReport(report, reportDetails); - queryRemoteLogAccessor()->healthReport(report, options); + IEspLogAccessDebugReport * debugReport = createLogAccessDebugReport(); + queryRemoteLogAccessor()->healthReport(options, reportDetails); + status->setCode(LogAccessHealthStatusToString(reportDetails.status.code)); + VStringBuffer encapsulatedMessages("{%s}", reportDetails.status.message.str()); + status->setMessages(encapsulatedMessages.str()); + + if (options.IncludeConfiguration) + { + resp.setConfiguration(reportDetails.Configuration.str()); + DBGLOG("WsLogAccessHealth: configuration: %s", reportDetails.Configuration.str()); + } + + if (options.IncludeSampleQuery) + { + debugReport->setSampleQueryReport(reportDetails.DebugReport.SampleQueryReport.str()); + } + + if (options.IncludeDebugReport) + { + debugReport->setPluginDebugReport(reportDetails.DebugReport.PluginDebugReport.str()); + debugReport->setServerDebugReport(reportDetails.DebugReport.ServerDebugReport.str()); + } + + resp.setDebugReport(*debugReport); } - report.append(" }"); - resp.setReport(report.str()); - return success; -} \ No newline at end of file + resp.setStatus(*status); + + return true; +} diff --git a/helm/managed/logging/loki-stack/grafana-hpcc-logaccess.yaml b/helm/managed/logging/loki-stack/grafana-hpcc-logaccess.yaml index 03c0bc66dae..1b2375f09c8 100644 --- a/helm/managed/logging/loki-stack/grafana-hpcc-logaccess.yaml +++ b/helm/managed/logging/loki-stack/grafana-hpcc-logaccess.yaml @@ -11,7 +11,7 @@ global: id: "1" name: "Loki" namespace: - name: "default" + name: "hpcc" logFormat: type: "json" logMaps: diff --git a/system/jlib/jlog.hpp b/system/jlib/jlog.hpp index f3bb6d26f81..425d5656b0f 100644 --- a/system/jlib/jlog.hpp +++ b/system/jlib/jlog.hpp @@ -1710,104 +1710,66 @@ struct LogQueryResultDetails unsigned int totalReceived; unsigned int totalAvailable; }; -/* + + typedef enum { - LOGACCESS_STATUS_unknown, - LOGACCESS_STATUS_ok, - LOGACCESS_STATUS_fail -} LogAccessHealthStatus; + LOGACCESS_STATUS_unknown = 0, + LOGACCESS_STATUS_green = 1, + LOGACCESS_STATUS_yellow = 2, + LOGACCESS_STATUS_red = 3 +} LogAccessHealthStatusCode; -struct LogAccessConnectionDetails +struct LogAccessHealthStatus { - StringAttr connectionString; - StringAttr connectionInfo; - StringAttr connectionStatus; - StringAttr sampleQueryStatus; - - void toJSON(StringBuffer & out) - { - } -}; + LogAccessHealthStatusCode code; + StringBuffer message; -struct LogAccessConfigLogMap -{ - LogAccessMappedField logFieldType; - StringAttr fieldName; - StringAttr sourceName; - LogAccessConfigLogMap(LogAccessMappedField type, const char * name, const char * source) + LogAccessHealthStatus(LogAccessHealthStatusCode code_) { - logFieldType = type; - fieldName.set(name); - sourceName.set(source); + code = code_; } - void toJSON(StringBuffer & out) + void appendMessage(const char * message_) { - out.appendf("\"%s\": { \"ColName\": \"%s\", \"Source\": \"%s\" }", MappedFieldTypeToString(logFieldType), fieldName.str(), sourceName.str()); + message.append(message_); } }; -struct LogAccessConfigDetails -{ - StringAttr connectionInfo; - StringArray logMaps; - //IArrayOf logMaps; - void appendLogMap(LogAccessConfigLogMap logMap) +inline const char * LogAccessHealthStatusToString(LogAccessHealthStatusCode statusCode) +{ + switch(statusCode) { - StringBuffer logMapJson; - logMap.toJSON(logMapJson); - logMaps.append(logMapJson.str()); - //logMaps.append(logMap); + case LOGACCESS_STATUS_green: + return "Green"; + case LOGACCESS_STATUS_yellow: + return "Yellow"; + case LOGACCESS_STATUS_red: + return "Red"; + default: + return "Unknown"; } +}; - void toJSON(StringBuffer & out) - { - out.appendf("\"ConfigInfo\": { \"LogMaps\": {"); - - ForEachItemIn(i, logMaps) - //logMaps.item(i).toJSON(out); - out.append(logMaps.item(i)); - - //close out the logmaps - out.append(" }"); - //close out the ConfigInfo - out.append(" }"); - } +struct LogAccessDebugReport +{ + StringBuffer SampleQueryReport; + StringBuffer PluginDebugReport; + StringBuffer ServerDebugReport; }; struct LogAccessHealthReportDetails { - LogAccessConnectionDetails connectionInfo; - LogAccessConfigDetails configInfo; - StringBuffer JsonMessages; - - void appendLogMap(LogAccessConfigLogMap logMap) - { - configInfo.appendLogMap(logMap); - } - - void toJSON(StringBuffer & out) - { - StringBuffer scratch; - - out.append("{ \"Connection\": "); - connectionInfo.toJSON(scratch); - out.append(scratch.str()); - - out.append(", \"ConfigInfo\": "); - configInfo.toJSON(scratch.clear()); - out.append(scratch.str()); - - out.appendf(", \"Messages\": \"%s\"", JsonMessages.str()); - } + LogAccessHealthStatus status = LOGACCESS_STATUS_unknown; + LogAccessDebugReport DebugReport; + StringAttr Configuration; }; -*/ + struct LogAccessHealthReportOptions { - bool IncludeServerInternals = true; - bool IncludePluginInternals = true; + bool IncludeConfiguration = true; + bool IncludeDebugReport = true; bool IncludeSampleQuery = true; }; @@ -1825,8 +1787,7 @@ interface IRemoteLogAccess : extends IInterface virtual IPropertyTree * queryLogMap() const = 0; virtual const char * fetchConnectionStr() const = 0; virtual bool supportsResultPaging() const = 0; - //virtual bool healthReport(StringBuffer & messages, LogAccessHealthReportDetails & report) = 0; - virtual bool healthReport(StringBuffer & report, LogAccessHealthReportOptions options) = 0; + virtual void healthReport(LogAccessHealthReportOptions options, LogAccessHealthReportDetails & report) = 0; }; // Helper functions to construct log access filters diff --git a/system/logaccess/Azure/LogAnalytics/CurlClient/AzureLogAnalyticsCurlClient.cpp b/system/logaccess/Azure/LogAnalytics/CurlClient/AzureLogAnalyticsCurlClient.cpp index ab65c76d829..6656186c5e3 100644 --- a/system/logaccess/Azure/LogAnalytics/CurlClient/AzureLogAnalyticsCurlClient.cpp +++ b/system/logaccess/Azure/LogAnalytics/CurlClient/AzureLogAnalyticsCurlClient.cpp @@ -1076,96 +1076,150 @@ bool AzureLogAnalyticsCurlClient::processSearchJsonResp(LogQueryResultDetails & if (!tree) throw makeStringExceptionV(-1, "%s: Could not parse query response", COMPONENT_NAME); + /*IPropertyTreeIterator * statsiter = tree->getElements("statistics"); + if (!statsiter) + DBGLOG("^^^^^^^^^^^^^^No stats found"); + + StringArray header; + ForEach(*statsiter) + { + Owned names = statsiter->query().getElements("."); + }*/ + resultDetails.totalReceived = processHitsJsonResp(tree->getElements("tables/rows"), tree->getElements("tables/columns"), returnbuf, format, true, reportHeader); resultDetails.totalAvailable = 0; return true; } -bool AzureLogAnalyticsCurlClient::healthReport(StringBuffer & report, LogAccessHealthReportOptions options) +void AzureLogAnalyticsCurlClient::healthReport(LogAccessHealthReportOptions options, LogAccessHealthReportDetails & report) { + LogAccessHealthStatus status = LOGACCESS_STATUS_unknown; try { - report.appendf("\"ConnectionInfo\": { \"TargetALAWorkspaceID\": \"%s\" ", m_logAnalyticsWorkspaceID.str()); - report.appendf(", \"TargetALATenantID\": \"%s\"", m_aadTenantID.str()); - report.appendf(", \"TargetALAClientID\": \"%s\"", m_aadClientID.str()); - report.appendf(", \"TargetALASecret\": \"%sempty\"", m_aadClientSecret.length()==0 ? "" : "not "); - report.appendf(", \"TargetsContainerLogV2\": \"%s\"", targetIsContainerLogV2 ? "true" : "false"); - report.appendf(", \"ComponentsQueryJoins\": \"%sabled\"", m_disableComponentNameJoins ? "dis" : "en"); - report.appendf(", \"BlobModeUnstructuredLogData\": \"%sabled\"", m_blobMode ? "en" : "dis"); - report.append( "}"); //close conninfo - - report.append(", \"ConfigurationInfo\": { "); - - if (m_pluginCfg) - { - StringBuffer configJSON; - toJSON(m_pluginCfg, configJSON, 0); - report.appendf("\"ConfigurationTree\": %s", configJSON.str()); //json encode - } - else + if (options.IncludeConfiguration) { - report.append("\"Error\": \"Configuration tree is empty!!!\""); - } - report.append(" }"); // close config info + StringBuffer configuration; + configuration.set("{"); - report.append(", \"Internals\": { "); - if (options.IncludeServerInternals) - { - report.appendf("\"Plugin\": { \"LogMaps\": {"); - report.appendf("\"Global\": { \"ColName\": \"%s\", \"Source\": \"%s\", \"TimeStampCol\": \"%s\"}", m_globalSearchColName.str(), m_globalIndexSearchPattern.str(), m_globalIndexTimestampField.str()); - report.appendf(", \"Workunits\": { \"ColName\": \"%s\", \"Source\": \"%s\"}", m_workunitSearchColName.str(), m_workunitIndexSearchPattern.str()); - report.appendf(", \"Components\": { \"ColName\": \"%s\", \"Source\": \"%s\", \"LookupKey\": \"%s\", \"TimeStampCol\": \"%s\"}", m_componentsSearchColName.str(), m_componentsIndexSearchPattern.str(), m_componentsLookupKeyColumn.str(), m_componentsTimestampField.str()); - report.appendf(", \"Audience\": { \"ColName\": \"%s\", \"Source\": \"%s\" }", m_audienceSearchColName.str(), m_audienceIndexSearchPattern.str()); - report.appendf(", \"Class\": { \"ColName\": \"%s\", \"Source\": \"%s\"}", m_classSearchColName.str(), m_classIndexSearchPattern.str()); - report.appendf(", \"Instance\": { \"ColName\": \"%s\", \"Source\": \"%s\", \"LookupKey\": \"%s\"}", m_instanceSearchColName.str(), m_instanceIndexSearchPattern.str(), m_instanceLookupKeyColumn.str()); - report.appendf(", \"Pod\": { \"ColName\": \"%s\", \"Source\": \"%s\"}", m_podSearchColName.str(), m_podIndexSearchPattern.str()); - report.appendf(", \"TraceID\": { \"ColName\": \"%s\", \"Source\": \"%s\"}", m_traceSearchColName.str(), m_traceIndexSearchPattern.str()); - report.appendf(", \"SpanID\": { \"ColName\": \"%s\", \"Source\": \"%s\"}", m_spanSearchColName.str(), m_spanIndexSearchPattern.str()); - report.appendf(", \"Host\": { \"ColName\": \"%s\", \"Source\": \"%s\"}", m_hostSearchColName.str(), m_hostIndexSearchPattern.str()); - report.append(" }"); //close logmaps - report.append(" }"); //close plugin + if (m_pluginCfg) + { + StringBuffer configJSON; + toJSON(m_pluginCfg, configJSON, 0, JSON_Format); + appendJSONStringValue(configuration, "ConfigurationTree", configJSON.str(), false); + } + else + { + status = LOGACCESS_STATUS_red; + appendJSONStringValue(status.message, "Message", "ALA Pluging Configuration tree is empty!!!", false); + } + + configuration.append(" }"); // close config info + report.Configuration.set(configuration.str()); } - if (options.IncludeServerInternals) + if (options.IncludeDebugReport) { - report.append(", \"Server\": { }"); + StringBuffer debugReport; + debugReport.set("{"); + debugReport.append("\"ConnectionInfo\": {"); + appendJSONStringValue(debugReport, "TargetALAWorkspaceID", m_logAnalyticsWorkspaceID.str(), true); + appendJSONStringValue(debugReport, "TargetALATenantID", m_aadTenantID.str(), true); + appendJSONStringValue(debugReport, "TargetALAClientID", m_aadClientID.str(), true); + debugReport.appendf(", \"TargetALASecret\": \"%sempty\"", m_aadClientSecret.length()==0 ? "" : "not "); + appendJSONValue(debugReport, "TargetsContainerLogV2", targetIsContainerLogV2 ? true : false); + appendJSONValue(debugReport, "ComponentsJoinedQueryEnabled", m_disableComponentNameJoins ? false : true); + appendJSONValue(debugReport, "BlobModeEnabled", m_blobMode ? true : false); + debugReport.append( "}"); //close conninfo + + debugReport.appendf(", { \"LogMaps\": {"); + debugReport.appendf("\"Global\": { "); + appendJSONStringValue(debugReport, "ColName", m_globalSearchColName.str(), true); + appendJSONStringValue(debugReport, "Source", m_globalIndexSearchPattern.str(), true); + appendJSONStringValue(debugReport, "TimeStampCol", m_globalIndexTimestampField.str(), true); + debugReport.append(" }"); // end Global + debugReport.appendf("\"Components\": { "); + appendJSONStringValue(debugReport, "ColName", m_componentsSearchColName.str(), true); + appendJSONStringValue(debugReport, "Source", m_componentsIndexSearchPattern.str(), true); + appendJSONStringValue(debugReport, "LookupKey", m_componentsLookupKeyColumn.str(), true); + appendJSONStringValue(debugReport, "TimeStampCol", m_globalIndexTimestampField.str(), true); + debugReport.appendf(" }"); // end Components + debugReport.appendf("\"Workunits\": { "); + appendJSONStringValue(debugReport, "ColName", m_workunitSearchColName.str(), true); + appendJSONStringValue(debugReport, "Source", m_workunitIndexSearchPattern.str(), true); + appendJSONStringValue(debugReport, "TimeStampCol", m_globalIndexTimestampField.str(), true); + debugReport.append(" }"); // end Workunits + debugReport.appendf("\"Audience\": { "); + appendJSONStringValue(debugReport, "ColName", m_audienceSearchColName.str(), true); + appendJSONStringValue(debugReport, "Source", m_audienceIndexSearchPattern.str(), true); + debugReport.appendf(" }"); // end Audience + debugReport.appendf("\"Class\": { "); + appendJSONStringValue(debugReport, "ColName", m_classSearchColName.str(), true); + appendJSONStringValue(debugReport, "Source", m_classIndexSearchPattern.str(), true); + debugReport.appendf(" }"); // end Class + debugReport.appendf("\"Instance\": { "); + appendJSONStringValue(debugReport, "ColName", m_instanceSearchColName.str(), true); + appendJSONStringValue(debugReport, "Source", m_instanceIndexSearchPattern.str(), true); + appendJSONStringValue(debugReport, "LookupKey", m_instanceLookupKeyColumn.str(), true); + debugReport.appendf(" }"); // end Instance + debugReport.appendf("\"Pod\": { "); + appendJSONStringValue(debugReport, "ColName", m_podSearchColName.str(), true); + appendJSONStringValue(debugReport, "Source", m_podIndexSearchPattern.str(), true); + debugReport.appendf(" }"); // end Pod + debugReport.appendf("\"TraceID\": { "); + appendJSONStringValue(debugReport, "ColName", m_traceSearchColName.str(), true); + appendJSONStringValue(debugReport, "Source", m_traceIndexSearchPattern.str(), true); + debugReport.appendf(" }"); // end TraceID + debugReport.appendf("\"SpanID\": { "); + appendJSONStringValue(debugReport, "ColName", m_spanSearchColName.str(), true); + appendJSONStringValue(debugReport, "Source", m_spanIndexSearchPattern.str(), true); + debugReport.appendf(" }"); // end SpanID + debugReport.appendf("\"Host\": { "); + appendJSONStringValue(debugReport, "ColName", m_hostSearchColName.str(), true); + appendJSONStringValue(debugReport, "Source", m_hostIndexSearchPattern.str(), true); + debugReport.appendf(" }"); // end Host + debugReport.append(" }"); //close logmaps + + debugReport.append(" }"); //close debugreport + report.DebugReport.PluginDebugReport.set(debugReport); + report.DebugReport.ServerDebugReport.set("{}"); + + appendJSONStringValue(status.message, "Message", "ALA Debug report succeeded", false); } - report.append(" }"); //close internals if (options.IncludeSampleQuery) { - report.append(", \"SampleTokenRequest\": { "); + StringBuffer sampleQueryReport; + sampleQueryReport.append("{ \"SampleTokenRequest\": { "); try { - StringBuffer token; requestLogAnalyticsAccessToken(token, m_aadClientID, m_aadClientSecret, m_aadTenantID); //throws if issues encountered - if (token.isEmpty()) - report.append("\"Error\": \"Empty token received\""); - + appendJSONStringValue(sampleQueryReport, "Result", token.isEmpty() ? "Error - Empty token received" : "Success", true); } catch(IException * e) { StringBuffer description; e->errorMessage(description); - report.appendf("\"Error\": \"Exception while requesting token (%d) - %s\"", e->errorCode(), description.str()); + status = LOGACCESS_STATUS_red; + appendJSONStringValue(status.message, "Result", "Exception while requesting sample token (%d) - %s", e->errorCode(), description.str()); e->Release(); } catch(...) { - report.append("\"Error\": \"Unknown exception while requesting token\""); + appendJSONStringValue(status.message, "Message", "Unknown exception while requesting sample token", false); + status = LOGACCESS_STATUS_red; } - report.append(" }"); //close sample token request + sampleQueryReport.append(" }"); //close sample token request - report.append(", \"SampleQuery\": { "); + sampleQueryReport.append(", \"SampleQuery\": { "); try { - report.appendf("\"Query\": { \"LogFormat\": \"JSON\","); + sampleQueryReport.appendf("\"Query\": { \"LogFormat\": \"JSON\","); LogAccessLogFormat outputFormat = LOGACCESS_LOGFORMAT_json; LogAccessConditions queryOptions; - report.appendf("\"Filter\": {\"type\": \"byWildcard\", \"value\": \"*\" },"); + sampleQueryReport.appendf("\"Filter\": {\"type\": \"byWildcard\", \"value\": \"*\" },"); queryOptions.setFilter(getWildCardLogAccessFilter("*")); struct LogAccessTimeRange range; @@ -1182,48 +1236,55 @@ bool AzureLogAnalyticsCurlClient::healthReport(StringBuffer & report, LogAccessH StringBuffer startstr; startt.getString(startstr); - report.appendf("\"TimeRange\": {\"Start\": \"%s\", \"End\": \"%s\" },", startstr.str(), endstr.str()); + sampleQueryReport.appendf("\"TimeRange\": {\"Start\": \"%s\", \"End\": \"%s\" },", startstr.str(), endstr.str()); queryOptions.setTimeRange(range); queryOptions.setLimit(5); - report.appendf("\"Limit\": \"5\" }, "); + sampleQueryReport.appendf("\"Limit\": \"5\" }, "); StringBuffer queryString, queryIndex; populateKQLQueryString(queryString, queryIndex, queryOptions); - - StringBuffer encodedValue; - encodeJSON(encodedValue, queryString.str()); - report.appendf("\"KQLQuery\": \"%s\", ", encodedValue.str()); - report.appendf("\"QueryIndex\": \"%s\", ", queryIndex.str()); + appendJSONStringValue(sampleQueryReport, "KQLQuery", queryString.str(), true); + appendJSONStringValue(sampleQueryReport, "QueryIndex", queryIndex.str(), true); StringBuffer logs; LogQueryResultDetails resultDetails; fetchLog(resultDetails, queryOptions, logs, outputFormat); - report.appendf("\"ResultCount\": \"%d\", ", resultDetails.totalReceived); - report.appendf("\"Results\": %s", logs.str()); + appendJSONValue(sampleQueryReport, "ResultCount", resultDetails.totalReceived); + if (resultDetails.totalReceived==0) + { + status = LOGACCESS_STATUS_yellow; + appendJSONStringValue(status.message, "Message", "Query succeeded but returned 0 log entries", false); + } + + appendJSONStringValue(sampleQueryReport, "Results", logs.str(), true); } catch(IException * e) { StringBuffer description; e->errorMessage(description); - report.appendf("\"Error\": \"Exception while executing sample ALA query (%d) - %s\"", e->errorCode(), description.str()); + status = LOGACCESS_STATUS_red; + status.message.appendf("%s{\"Message\": \"Exception while executing sample ALA query (%d) - %s\"", status.message.length() == 0 ? "" : ", ", e->errorCode(), description.str()); e->Release(); } catch(...) { - report.append("\"Error\": \"Unknown exception while executing sample ALA query\""); + appendJSONStringValue(status.message, "Message", "Unknown exception while executing sample ALA query", false); + status = LOGACCESS_STATUS_red; } - report.append(" }"); //close sample query + sampleQueryReport.append(" }"); //close sample query + + report.DebugReport.SampleQueryReport.set(sampleQueryReport); } } catch(...) { - report.append("\"Error\": \"Encountered unexpected exception during health report\""); - return false; + status = LOGACCESS_STATUS_red; + appendJSONStringValue(status.message, "Message", "Encountered unexpected exception during health report", false); } - return true; + report.status = status; } bool AzureLogAnalyticsCurlClient::fetchLog(LogQueryResultDetails & resultDetails, const LogAccessConditions & options, StringBuffer & returnbuf, LogAccessLogFormat format) diff --git a/system/logaccess/Azure/LogAnalytics/CurlClient/AzureLogAnalyticsCurlClient.hpp b/system/logaccess/Azure/LogAnalytics/CurlClient/AzureLogAnalyticsCurlClient.hpp index a1b8eb2606e..928ae2f0be7 100644 --- a/system/logaccess/Azure/LogAnalytics/CurlClient/AzureLogAnalyticsCurlClient.hpp +++ b/system/logaccess/Azure/LogAnalytics/CurlClient/AzureLogAnalyticsCurlClient.hpp @@ -101,5 +101,5 @@ class AzureLogAnalyticsCurlClient : public CInterfaceOf virtual IRemoteLogAccessStream * getLogReader(const LogAccessConditions & options, LogAccessLogFormat format) override; virtual IRemoteLogAccessStream * getLogReader(const LogAccessConditions & options, LogAccessLogFormat format, unsigned int pageSize) override; virtual bool supportsResultPaging() const override { return false;} - virtual bool healthReport(StringBuffer & report, LogAccessHealthReportOptions options) override; + virtual void healthReport(LogAccessHealthReportOptions options, LogAccessHealthReportDetails & report) override; }; diff --git a/system/logaccess/ElasticStack/ElasticStackLogAccess.cpp b/system/logaccess/ElasticStack/ElasticStackLogAccess.cpp index 2624b9d00ee..5775526e34f 100644 --- a/system/logaccess/ElasticStack/ElasticStackLogAccess.cpp +++ b/system/logaccess/ElasticStack/ElasticStackLogAccess.cpp @@ -263,9 +263,10 @@ const IPropertyTree * ElasticStackLogAccess::getESStatus() return performAndLogESRequest(Client::HTTPMethod::GET, "_cluster/health", "", "Target cluster health"); } - bool ElasticStackLogAccess::healthReport(StringBuffer & report, LogAccessHealthReportOptions options) + void ElasticStackLogAccess::healthReport(LogAccessHealthReportOptions options, LogAccessHealthReportDetails & report) { - try + LogAccessHealthStatus status = LOGACCESS_STATUS_green; + /*try { report.appendf("\"ConnectionInfo\": { \"ConnectionString\": \"%s\" }", m_esConnectionStr.str()); @@ -279,11 +280,12 @@ const IPropertyTree * ElasticStackLogAccess::getESStatus() else { report.append("\"Error\": \"Configuration tree is empty!!!\""); + status = LOGACCESS_STATUS_red; } report.append(" }"); // close config info report.append(", \"Internals\": { "); - if (options.IncludeServerInternals) + //if (options.IncludeServerInternals) { report.appendf("\"Plugin\": { \"LogMaps\": {"); report.appendf("\"Global\": { \"ColName\": \"%s\", \"Source\": \"%s\", \"TimeStampCol\": \"%s\" }", m_globalSearchColName.str(), m_globalIndexSearchPattern.str(), m_globalIndexTimestampField.str()); @@ -300,82 +302,91 @@ const IPropertyTree * ElasticStackLogAccess::getESStatus() report.append(" }"); //close plugin } - if (options.IncludeServerInternals) + //if (options.IncludeServerInternals) { - report.append(", \"Server\": { \"ESStatus\": "); + report.append(", \"Server\": {"); + report.append("\"AvailableIndices\": "); try { - StringBuffer out; - const IPropertyTree * status = getESStatus(); - if (status) + const IPropertyTree * is = getIndexSearchStatus(m_globalIndexSearchPattern); + if (is) { - toJSON(status, report); + toJSON(is, report); } else { - report.append("\"Could not populate ES Status\""); + report.appendf("\"Could not populate Available Indices for Index %s\"", m_globalIndexSearchPattern.str()); + status = LOGACCESS_STATUS_yellow; } } catch(IException * e) { StringBuffer description; e->errorMessage(description); - report.appendf("\"Exception fetching ES Status (%d) - %s\"", e->errorCode(), description.str()); + report.appendf("\"Exception fetching available ES indices (%d) - %s\"", e->errorCode(), description.str()); e->Release(); + status = LOGACCESS_STATUS_yellow; } catch(...) { - report.append("\"Unknown exception while fetching ES Status\""); + report.append("\"Unknown exception while fetching available ES indices\""); + status = LOGACCESS_STATUS_yellow; } - report.append(", \"AvailableIndices\": "); + report.append(", \"TimestampField\": "); try { - const IPropertyTree * is = getIndexSearchStatus(m_globalIndexSearchPattern); - if (is) + const IPropertyTree * ts = getTimestampTypeFormat(m_globalIndexSearchPattern, m_globalIndexTimestampField); + if (ts) { - toJSON(is, report); + toJSON(ts, report); } else { - report.appendf("\"Could not populate Available Indices for Index %s\"", m_globalIndexSearchPattern.str()); + report.appendf("\"Could not populate AES timestamp format for IndexPattern %s\"", m_globalIndexSearchPattern.str()); } } catch(IException * e) { StringBuffer description; e->errorMessage(description); - report.appendf("\"Exception fetching available ES indices (%d) - %s\"", e->errorCode(), description.str()); + report.appendf("\"Exception fetching target ES timestamp format (%d) - %s\"", e->errorCode(), description.str()); e->Release(); + status = LOGACCESS_STATUS_red; } catch(...) { - report.append("\"Unknown exception while fetching available ES indices\""); + report.append("\"Unknown exception while fetching target ES timestamp format\""); + status = LOGACCESS_STATUS_red; } - report.append(", \"TimestampField\": "); + report.append(", \"ESStatus\": "); try { - const IPropertyTree * ts = getTimestampTypeFormat(m_globalIndexSearchPattern, m_globalIndexTimestampField); - if (ts) + StringBuffer out; + const IPropertyTree * esStatus = getESStatus(); + if (esStatus) { - toJSON(ts, report); + toJSON(esStatus, report); //extract esstatus info to set status green/yellow/red? } else { - report.appendf("\"Could not populate AES timestamp format for IndexPattern %s\"", m_globalIndexSearchPattern.str()); + report.append("\"Could not populate ES Status\""); + status = LOGACCESS_STATUS_yellow; } } catch(IException * e) { StringBuffer description; e->errorMessage(description); - report.appendf("\"Exception fetching target ES timestamp format (%d) - %s\"", e->errorCode(), description.str()); + report.appendf("\"Exception fetching ES Status (%d) - %s\"", e->errorCode(), description.str()); e->Release(); + status = LOGACCESS_STATUS_red; } catch(...) { - report.append("\"Unknown exception while fetching target ES timestamp format\""); + report.append("\"Unknown exception while fetching ES Status\""); + status = LOGACCESS_STATUS_red; } report.append(" } "); //close Server } @@ -415,6 +426,9 @@ const IPropertyTree * ElasticStackLogAccess::getESStatus() LogQueryResultDetails resultDetails; fetchLog(resultDetails, queryOptions, logs, outputFormat); report.appendf("\"ResultCount\": \"%d\", ", resultDetails.totalReceived); + if (resultDetails.totalReceived == 0) + status = LOGACCESS_STATUS_yellow; + report.appendf("\"Results\": %s", logs.str()); } catch(IException * e) @@ -423,10 +437,12 @@ const IPropertyTree * ElasticStackLogAccess::getESStatus() e->errorMessage(description); report.appendf("\"Error\": \"Exception while executing sample query (%d) - %s\"", e->errorCode(), description.str()); e->Release(); + status = LOGACCESS_STATUS_red; } catch(...) { report.append("\"Error\": \"Unknown exception while executing samplequery\""); + status = LOGACCESS_STATUS_red; } report.append(" }"); //close sample query } @@ -434,10 +450,10 @@ const IPropertyTree * ElasticStackLogAccess::getESStatus() catch(...) { report.append("\"Error\": \"Encountered unexpected exception during health report\""); - return false; + status = LOGACCESS_STATUS_red; } - - return true; +*/ + //return status; } /* diff --git a/system/logaccess/ElasticStack/ElasticStackLogAccess.hpp b/system/logaccess/ElasticStack/ElasticStackLogAccess.hpp index 26f9026d96c..c6a34e43116 100644 --- a/system/logaccess/ElasticStack/ElasticStackLogAccess.hpp +++ b/system/logaccess/ElasticStack/ElasticStackLogAccess.hpp @@ -103,5 +103,5 @@ class ElasticStackLogAccess : public CInterfaceOf virtual IRemoteLogAccessStream * getLogReader(const LogAccessConditions & options, LogAccessLogFormat format) override; virtual IRemoteLogAccessStream * getLogReader(const LogAccessConditions & options, LogAccessLogFormat format, unsigned int pageSize) override; virtual bool supportsResultPaging() const override { return true;} - virtual bool healthReport(StringBuffer & report, LogAccessHealthReportOptions options) override; + virtual void healthReport(LogAccessHealthReportOptions options, LogAccessHealthReportDetails & report) override; }; diff --git a/system/logaccess/Grafana/CurlClient/GrafanaCurlClient.cpp b/system/logaccess/Grafana/CurlClient/GrafanaCurlClient.cpp index 2e57725d8fb..181be8a616b 100644 --- a/system/logaccess/Grafana/CurlClient/GrafanaCurlClient.cpp +++ b/system/logaccess/Grafana/CurlClient/GrafanaCurlClient.cpp @@ -760,14 +760,15 @@ void processLogMapConfig(const IPropertyTree * logMapConfig, LogField * targetFi targetField->name = logMapConfig->queryProp(logMapSearchColAtt); } -bool GrafanaLogAccessCurlClient::healthReport(StringBuffer & report, LogAccessHealthReportOptions options) +void GrafanaLogAccessCurlClient::healthReport(LogAccessHealthReportOptions options, LogAccessHealthReportDetails & report) { - try + LogAccessHealthStatus status = LOGACCESS_STATUS_green; + /*try { report.append("\"ConnectionInfo\": { "); report.appendf("\"ConnectionString\": \"%s\"", m_grafanaConnectionStr.str()); report.appendf(", \"UserName\": \"%s\"", m_grafanaUserName.str()); - report.appendf(", \"PasswordProvided\": %s", isEmptyString(m_grafanaPassword.str()) ? "true" : "false"); + report.appendf(", \"PasswordProvided\": %s", !isEmptyString(m_grafanaPassword.str()) ? "true" : "false"); report.appendf(", \"TargetDatasourceID\": \"%s\"", m_targetDataSource.id.str()); report.appendf(", \"TargetDatasourceName\": \"%s\"", m_targetDataSource.name.str()); report.appendf(", \"TargetLogsNamespace\": \"%s\"", m_targetNamespace.str()); @@ -933,6 +934,9 @@ bool GrafanaLogAccessCurlClient::healthReport(StringBuffer & report, LogAccessHe LogQueryResultDetails resultDetails; fetchLog(resultDetails, queryOptions, logs, outputFormat); report.appendf("\"ResultCount\": \"%d\", ", resultDetails.totalReceived); + if (resultDetails.totalReceived == 0) + status = LOGACCESS_STATUS_yellow; + report.appendf("\"Results\": %s", logs.str()); } catch(IException * e) @@ -941,6 +945,7 @@ bool GrafanaLogAccessCurlClient::healthReport(StringBuffer & report, LogAccessHe e->errorMessage(description); report.appendf("\"Error\": \"Exception while executing sample Grafana/Loki query (%d) - %s\"", e->errorCode(), description.str()); e->Release(); + status = LOGACCESS_STATUS_red; } catch(...) { @@ -951,10 +956,10 @@ bool GrafanaLogAccessCurlClient::healthReport(StringBuffer & report, LogAccessHe catch(...) { report.append("\"Error\": \"Encountered unexpected exception during health report\""); - return false; + status = LOGACCESS_STATUS_red; } - - return true; +*/ + //return status; } GrafanaLogAccessCurlClient::GrafanaLogAccessCurlClient(IPropertyTree & logAccessPluginConfig) diff --git a/system/logaccess/Grafana/CurlClient/GrafanaCurlClient.hpp b/system/logaccess/Grafana/CurlClient/GrafanaCurlClient.hpp index 6eab0c84cd4..20a214e64d4 100644 --- a/system/logaccess/Grafana/CurlClient/GrafanaCurlClient.hpp +++ b/system/logaccess/Grafana/CurlClient/GrafanaCurlClient.hpp @@ -109,5 +109,5 @@ class GrafanaLogAccessCurlClient : public CInterfaceOf virtual IRemoteLogAccessStream * getLogReader(const LogAccessConditions & options, LogAccessLogFormat format) override; virtual IRemoteLogAccessStream * getLogReader(const LogAccessConditions & options, LogAccessLogFormat format, unsigned int pageSize) override; virtual bool supportsResultPaging() const override { return false;} - virtual bool healthReport(StringBuffer & report, LogAccessHealthReportOptions options) override; -}; \ No newline at end of file + virtual void healthReport(LogAccessHealthReportOptions options, LogAccessHealthReportDetails & report) override; +};