Merge pull request #18200 from wangkx/h30641

HPCC-30641 Fetch pod logs for ZAP report Reviewed-by: Jake Smith <[email protected]> Merged-by: Gavin Halliday <[email protected]>
hpcc-systems · Jan 19, 2024 · 8c62d30 · 8c62d30
2 parents fc85d2a + 9d3a822
commit 8c62d30
Show file tree

Hide file tree

Showing 2 changed files with 29 additions and 10 deletions.
diff --git a/esp/services/ws_workunits/ws_workunitsHelpers.cpp b/esp/services/ws_workunits/ws_workunitsHelpers.cpp
@@ -276,9 +276,6 @@ void streamFilteredLogsToFile(const char* outFile, LogAccessConditions & logFetc
 
 void WsWuInfo::readWorkunitComponentLogs(const char* outFile, CWsWuZAPInfoReq& zapLogFilterOptions)
 {
-    if (!queryRemoteLogAccessor())
-        throw makeStringException(ECLWATCH_LOGACCESS_UNAVAILABLE, "WsWuInfo: Remote Log Access plug-in not available!");
-
     if (isEmptyString(outFile))
         throw makeStringException(ECLWATCH_INVALID_FILE_NAME, "WsWuInfo: Target filename not provided!");
 
@@ -4201,13 +4198,15 @@ void CWsWuFileHelper::createZAPECLQueryArchiveFiles(IConstWorkUnit* cwu, const c
     }
 }
 
-void CWsWuFileHelper::createWULogFile(IConstWorkUnit *cwu, WsWuInfo &winfo, const char *path, CWsWuZAPInfoReq & zapLogFilterOptions)
+void CWsWuFileHelper::readWULogToFiles(IConstWorkUnit *cwu, WsWuInfo &winfo, const char *path, CWsWuZAPInfoReq &zapLogFilterOptions)
 {
     if (cwu->getWuidVersion() == 0)
         return;
 
-    LogAccessLogFormat logFormat = zapLogFilterOptions.logFilter.logDataFormat;
+    if (!queryRemoteLogAccessor())
+        throw makeStringException(ECLWATCH_LOGACCESS_UNAVAILABLE, "CWsWuFileHelper: Remote Log Access plug-in not available!");
 
+    LogAccessLogFormat logFormat = zapLogFilterOptions.logFilter.logDataFormat;
     StringBuffer logfileextension;
     if (logFormat == LOGACCESS_LOGFORMAT_csv)
         logfileextension.set("csv");
@@ -4218,18 +4217,37 @@ void CWsWuFileHelper::createWULogFile(IConstWorkUnit *cwu, WsWuInfo &winfo, cons
     else
         logfileextension.set("log");
 
-    VStringBuffer fileName("%s%c%s-log.%s", path, PATHSEPCHAR, cwu->queryWuid(), logfileextension.str());
+    const char *wuid = cwu->queryWuid();
+    ILogAccessFilter *logFetchFilter = getJobIDLogAccessFilter(wuid);
+    zapLogFilterOptions.logFilter.logFetchOptions.setFilter(logFetchFilter);
+
+    VStringBuffer componentLog("%s%c%s-log.%s", path, PATHSEPCHAR, wuid, logfileextension.str());
+    readWULogToFile(componentLog, winfo, zapLogFilterOptions);
 
+    Owned<IPropertyTreeIterator> iter = cwu->getProcesses("*", nullptr);
+    ForEach(*iter)
+    {
+        const char *processName = iter->query().queryProp("@podName");
+        ILogAccessFilter *processLogFetchFilter = getBinaryLogAccessFilter(logFetchFilter, getPodLogAccessFilter(processName), LOGACCESS_FILTER_and);
+        zapLogFilterOptions.logFilter.logFetchOptions.setFilter(processLogFetchFilter);
+
+        VStringBuffer processLog("%s%c%s-%s-log.%s", path, PATHSEPCHAR, wuid, processName, logfileextension.str());
+        readWULogToFile(processLog, winfo, zapLogFilterOptions);
+    }
+}
+
+void CWsWuFileHelper::readWULogToFile(const char *logFileName, WsWuInfo &winfo, CWsWuZAPInfoReq &zapLogFilterOptions)
+{
     try
     {
-        winfo.readWorkunitComponentLogs(fileName.str(), zapLogFilterOptions);
+        winfo.readWorkunitComponentLogs(logFileName, zapLogFilterOptions);
     }
     catch(IException* e)
     {
         StringBuffer s;
         e->errorMessage(s);
         IERRLOG(e, "Error accessing WU logs");
-        writeToFile(fileName.str(), s.length(), s.str());
+        writeToFile(logFileName, s.length(), s.str());
         e->Release();
     }
 }
@@ -4480,7 +4498,7 @@ void CWsWuFileHelper::createWUZAPFile(IEspContext& context, IConstWorkUnit* cwu,
     if (request.includeThorSlaveLog.isEmpty() || strieq(request.includeThorSlaveLog.str(), "on"))
         createThorSlaveLogfile(cwu, winfo, tempDirName);
 #else
-    createWULogFile(cwu, winfo, tempDirName, request);
+    readWULogToFiles(cwu, winfo, tempDirName, request);
 #endif
 
     //Write out to ZIP file

diff --git a/esp/services/ws_workunits/ws_workunitsHelpers.hpp b/esp/services/ws_workunits/ws_workunitsHelpers.hpp
@@ -897,7 +897,8 @@ class CWsWuFileHelper
     void createThorSlaveLogfile(IConstWorkUnit *cwu, WsWuInfo &winfo, const char *path);
 #endif
     LogAccessLogFormat getComponentLogFormatFromLogName(const char *log);
-    void createWULogFile(IConstWorkUnit *cwu, WsWuInfo &winfo, const char *path, CWsWuZAPInfoReq & zapLogFilterOptions);
+    void readWULogToFiles(IConstWorkUnit *cwu, WsWuInfo &winfo, const char *path, CWsWuZAPInfoReq &zapLogFilterOptions);
+    void readWULogToFile(const char *logFileName, WsWuInfo &winfo, CWsWuZAPInfoReq &zapLogFilterOptions);
     void writeZAPWUInfoToIOStream(IFileIOStream *outFile, const char *name, SCMStringBuffer &value);
     void writeZAPWUInfoToIOStream(IFileIOStream *outFile, const char *name, const char *value);
     void readWUIDRequest(CHttpRequest *request, StringBuffer &wuid);