Merge pull request #18688 from ghalliday/issue31848

HPCC-31848 Run workunit analyser after each graph and on failure Reviewed-By: Shamser Ahmed <[email protected]> Merged-by: Gavin Halliday <[email protected]>
hpcc-systems · May 31, 2024 · 8eb67ed · 8eb67ed
2 parents c9ca733 + c74f3ed
commit 8eb67ed
Show file tree

Hide file tree

Showing 5 changed files with 64 additions and 18 deletions.
diff --git a/common/wuanalysis/anawu.cpp b/common/wuanalysis/anawu.cpp
@@ -155,7 +155,7 @@ class WorkunitAnalyserBase
 public:
     WorkunitAnalyserBase();
 
-    void analyse(IConstWorkUnit * wu);
+    void analyse(IConstWorkUnit * wu, const char * optGraph);
     WuScope * getRootScope() { return LINK(root); }
 
 protected:
@@ -1282,10 +1282,17 @@ WorkunitAnalyserBase::WorkunitAnalyserBase() : root(new WuScope("", nullptr))
 {
 }
 
-void WorkunitAnalyserBase::analyse(IConstWorkUnit * wu)
+void WorkunitAnalyserBase::analyse(IConstWorkUnit * wu, const char * optGraph)
 {
     WuScopeFilter filter;
     filter.addOutputProperties(PTstatistics).addOutputProperties(PTattributes);
+    if (optGraph)
+    {
+        //Only include the specified graph, and include everything that matches below that graph
+        filter.addScopeType(SSTgraph);
+        filter.addId(optGraph);
+        filter.setIncludeNesting((unsigned)-1);
+    }
     filter.finishedFilter();
     collateWorkunitStats(wu, filter);
     root->connectActivities();
@@ -2079,11 +2086,11 @@ void WorkunitStatsAnalyser::traceDependencies()
 
 //---------------------------------------------------------------------------------------------------------------------
 
-void WUANALYSIS_API analyseWorkunit(IWorkUnit * wu, IPropertyTree *options, double costPerMs)
+void WUANALYSIS_API analyseWorkunit(IWorkUnit * wu, const char *optGraph, IPropertyTree *options, double costPerMs)
 {
     WorkunitRuleAnalyser analyser;
     analyser.applyConfig(options, wu);
-    analyser.analyse(wu);
+    analyser.analyse(wu, optGraph);
     analyser.applyRules();
     analyser.update(wu, costPerMs);
 }
@@ -2092,7 +2099,7 @@ void WUANALYSIS_API analyseAndPrintIssues(IConstWorkUnit * wu, double costRate,
 {
     WorkunitRuleAnalyser analyser;
     analyser.applyConfig(nullptr, wu);
-    analyser.analyse(wu);
+    analyser.analyse(wu, nullptr);
     analyser.applyRules();
     analyser.print();
     if (updatewu)
@@ -2114,7 +2121,7 @@ void analyseActivity(IConstWorkUnit * wu, IPropertyTree * cfg, const StringArray
 {
     WorkunitStatsAnalyser analyser;
     analyser.applyOptions(cfg);
-    analyser.analyse(wu);
+    analyser.analyse(wu, nullptr);
     analyser.adjustTimestamps();
     analyser.reportActivity(args);
 }
@@ -2123,7 +2130,7 @@ void analyseDependencies(IConstWorkUnit * wu, IPropertyTree * cfg, const StringA
 {
     WorkunitStatsAnalyser analyser;
     analyser.applyOptions(cfg);
-    analyser.analyse(wu);
+    analyser.analyse(wu, nullptr);
     analyser.adjustTimestamps();
     analyser.calcDependencies();
     analyser.spotCommonPath(args);
@@ -2137,7 +2144,7 @@ void analyseOutputDependencyGraph(IConstWorkUnit * wu, IPropertyTree * cfg)
 {
     WorkunitStatsAnalyser analyser;
     analyser.applyOptions(cfg);
-    analyser.analyse(wu);
+    analyser.analyse(wu, nullptr);
     analyser.adjustTimestamps();
     analyser.calcDependencies();
     analyser.traceDependencies();
@@ -2147,7 +2154,7 @@ void analyseCriticalPath(IConstWorkUnit * wu, IPropertyTree * cfg, const StringA
 {
     WorkunitStatsAnalyser analyser;
     analyser.applyOptions(cfg);
-    analyser.analyse(wu);
+    analyser.analyse(wu, nullptr);
     analyser.adjustTimestamps();
     analyser.calcDependencies();
     analyser.traceCriticalPaths(args);
@@ -2157,7 +2164,7 @@ void analyseHotspots(IConstWorkUnit * wu, IPropertyTree * cfg, const StringArray
 {
     WorkunitStatsAnalyser analyser;
     analyser.applyOptions(cfg);
-    analyser.analyse(wu);
+    analyser.analyse(wu, nullptr);
 
     const char * rootScope = nullptr;
     if (args.ordinality())
@@ -2177,7 +2184,7 @@ void analyseHotspots(WuHotspotResults & results, IConstWorkUnit * wu, IPropertyT
 {
     WorkunitStatsAnalyser analyser;
     analyser.applyOptions(cfg);
-    analyser.analyse(wu);
+    analyser.analyse(wu, nullptr);
 
     analyser.findHotspots(cfg->queryProp("@rootScope"), results.totalTime, results.hotspots);
     results.root.setown(analyser.getRootScope());

diff --git a/common/wuanalysis/anawu.hpp b/common/wuanalysis/anawu.hpp
@@ -26,7 +26,7 @@
 
 #include "anacommon.hpp"
 
-void WUANALYSIS_API analyseWorkunit(IWorkUnit * wu, IPropertyTree *options, double costPerMs);
+void WUANALYSIS_API analyseWorkunit(IWorkUnit * wu, const char *optGraph, IPropertyTree *options, double costPerMs);
 void WUANALYSIS_API analyseAndPrintIssues(IConstWorkUnit * wu, double costPerMs, bool updatewu);
 
 //---------------------------------------------------------------------------------------------------------------------

diff --git a/ecl/eclagent/eclagent.cpp b/ecl/eclagent/eclagent.cpp
@@ -1856,6 +1856,27 @@ void EclAgent::setRetcode(int code)
     retcode = code;
 }
 
+
+void EclAgent::runWorkunitAnalyser(IWorkUnit * w, const char * optGraph)
+{
+    if (w->getDebugValueBool("analyzeWorkunit", agentTopology->getPropBool("@analyzeWorkunit", true)))
+    {
+        double costPerMs = calculateThorCost(1, getNodes());
+        IPropertyTree *analyzerOptions = agentTopology->queryPropTree("analyzerOptions");
+        analyseWorkunit(w, optGraph, analyzerOptions, costPerMs);
+    }
+}
+
+static constexpr bool defaultAnalyzeWhenComplete = true;
+void EclAgent::runWorkunitAnalyserAfterGraph(const char * graph)
+{
+    if (!wuRead->getDebugValueBool("analyzeWhenComplete", agentTopology->getPropBool("@analyzeWhenComplete", defaultAnalyzeWhenComplete)))
+    {
+        Owned<IWorkUnit> wu(updateWorkUnit());
+        runWorkunitAnalyser(wu, graph);
+    }
+}
+
 void EclAgent::doProcess()
 {
 #ifdef _DEBUG
@@ -2026,15 +2047,22 @@ void EclAgent::doProcess()
             break;
         }
 
-        if (w->getState() == WUStateCompleted && getClusterType(clusterType)==ThorLCRCluster)
+
+        if (getClusterType(clusterType)==ThorLCRCluster)
         {
-            if (w->getDebugValueBool("analyzeWorkunit", agentTopology->getPropBool("@analyzeWorkunit", true)))
+            if (w->getDebugValueBool("analyzeWhenComplete", agentTopology->getPropBool("@analyzeWhenComplete", defaultAnalyzeWhenComplete)))
             {
-                double costPerMs = calculateThorCost(1, getNodes());
-                IPropertyTree *analyzerOptions = agentTopology->queryPropTree("analyzerOptions");
-                analyseWorkunit(w.get(), analyzerOptions, costPerMs);
+                switch (w->getState())
+                {
+                case WUStateFailed:
+                case WUStateAborted:
+                case WUStateCompleted:
+                    runWorkunitAnalyser(w, nullptr);
+                    break;
+                }
             }
         }
+
         if(w->queryEventScheduledCount() > 0)
             switch(w->getState())
             {

diff --git a/ecl/eclagent/eclagent.ipp b/ecl/eclagent/eclagent.ipp
@@ -422,6 +422,8 @@ private:
     EclAgentQueryLibrary * loadEclLibrary(const char * libraryName, unsigned expectedInterfaceHash, const char * embeddedGraphName);
     virtual bool getWorkunitResultFilename(StringBuffer & diskFilename, const char * wuid, const char * name, int seq);
     virtual IDebuggableContext *queryDebugContext() const { return debugContext; };
+    void runWorkunitAnalyser(IWorkUnit * w, const char * optGraph);
+    void runWorkunitAnalyserAfterGraph(const char * optGraph);
 
     //protected by critical section
     EclGraph * addGraph(const char * graphName);

diff --git a/ecl/eclagent/eclgraph.cpp b/ecl/eclagent/eclgraph.cpp
@@ -1598,7 +1598,16 @@ void EclAgent::executeGraph(const char * graphName, bool realThor, size32_t pare
     {
         if (isStandAloneExe)
             throw MakeStringException(0, "Cannot execute Thor Graph in standalone mode");
-        executeThorGraph(graphName, *wuRead, *agentTopology);
+        try
+        {
+            executeThorGraph(graphName, *wuRead, *agentTopology);
+            runWorkunitAnalyserAfterGraph(graphName);
+        }
+        catch (...)
+        {
+            runWorkunitAnalyserAfterGraph(graphName);
+            throw;
+        }
     }
     else
     {