From c74f3ed692a3b3a88e6e45ef972ce8d83a5c6f90 Mon Sep 17 00:00:00 2001 From: Gavin Halliday Date: Mon, 20 May 2024 17:35:19 +0100 Subject: [PATCH] HPCC-31848 Run workunit analyser after each graph and on failure Signed-off-by: Gavin Halliday --- common/wuanalysis/anawu.cpp | 29 +++++++++++++++++----------- common/wuanalysis/anawu.hpp | 2 +- ecl/eclagent/eclagent.cpp | 38 ++++++++++++++++++++++++++++++++----- ecl/eclagent/eclagent.ipp | 2 ++ ecl/eclagent/eclgraph.cpp | 11 ++++++++++- 5 files changed, 64 insertions(+), 18 deletions(-) diff --git a/common/wuanalysis/anawu.cpp b/common/wuanalysis/anawu.cpp index a8b7aac1c96..79aa20c9495 100644 --- a/common/wuanalysis/anawu.cpp +++ b/common/wuanalysis/anawu.cpp @@ -155,7 +155,7 @@ class WorkunitAnalyserBase public: WorkunitAnalyserBase(); - void analyse(IConstWorkUnit * wu); + void analyse(IConstWorkUnit * wu, const char * optGraph); WuScope * getRootScope() { return LINK(root); } protected: @@ -1282,10 +1282,17 @@ WorkunitAnalyserBase::WorkunitAnalyserBase() : root(new WuScope("", nullptr)) { } -void WorkunitAnalyserBase::analyse(IConstWorkUnit * wu) +void WorkunitAnalyserBase::analyse(IConstWorkUnit * wu, const char * optGraph) { WuScopeFilter filter; filter.addOutputProperties(PTstatistics).addOutputProperties(PTattributes); + if (optGraph) + { + //Only include the specified graph, and include everything that matches below that graph + filter.addScopeType(SSTgraph); + filter.addId(optGraph); + filter.setIncludeNesting((unsigned)-1); + } filter.finishedFilter(); collateWorkunitStats(wu, filter); root->connectActivities(); @@ -2079,11 +2086,11 @@ void WorkunitStatsAnalyser::traceDependencies() //--------------------------------------------------------------------------------------------------------------------- -void WUANALYSIS_API analyseWorkunit(IWorkUnit * wu, IPropertyTree *options, double costPerMs) +void WUANALYSIS_API analyseWorkunit(IWorkUnit * wu, const char *optGraph, IPropertyTree *options, double costPerMs) { WorkunitRuleAnalyser analyser; analyser.applyConfig(options, wu); - analyser.analyse(wu); + analyser.analyse(wu, optGraph); analyser.applyRules(); analyser.update(wu, costPerMs); } @@ -2092,7 +2099,7 @@ void WUANALYSIS_API analyseAndPrintIssues(IConstWorkUnit * wu, double costRate, { WorkunitRuleAnalyser analyser; analyser.applyConfig(nullptr, wu); - analyser.analyse(wu); + analyser.analyse(wu, nullptr); analyser.applyRules(); analyser.print(); if (updatewu) @@ -2114,7 +2121,7 @@ void analyseActivity(IConstWorkUnit * wu, IPropertyTree * cfg, const StringArray { WorkunitStatsAnalyser analyser; analyser.applyOptions(cfg); - analyser.analyse(wu); + analyser.analyse(wu, nullptr); analyser.adjustTimestamps(); analyser.reportActivity(args); } @@ -2123,7 +2130,7 @@ void analyseDependencies(IConstWorkUnit * wu, IPropertyTree * cfg, const StringA { WorkunitStatsAnalyser analyser; analyser.applyOptions(cfg); - analyser.analyse(wu); + analyser.analyse(wu, nullptr); analyser.adjustTimestamps(); analyser.calcDependencies(); analyser.spotCommonPath(args); @@ -2137,7 +2144,7 @@ void analyseOutputDependencyGraph(IConstWorkUnit * wu, IPropertyTree * cfg) { WorkunitStatsAnalyser analyser; analyser.applyOptions(cfg); - analyser.analyse(wu); + analyser.analyse(wu, nullptr); analyser.adjustTimestamps(); analyser.calcDependencies(); analyser.traceDependencies(); @@ -2147,7 +2154,7 @@ void analyseCriticalPath(IConstWorkUnit * wu, IPropertyTree * cfg, const StringA { WorkunitStatsAnalyser analyser; analyser.applyOptions(cfg); - analyser.analyse(wu); + analyser.analyse(wu, nullptr); analyser.adjustTimestamps(); analyser.calcDependencies(); analyser.traceCriticalPaths(args); @@ -2157,7 +2164,7 @@ void analyseHotspots(IConstWorkUnit * wu, IPropertyTree * cfg, const StringArray { WorkunitStatsAnalyser analyser; analyser.applyOptions(cfg); - analyser.analyse(wu); + analyser.analyse(wu, nullptr); const char * rootScope = nullptr; if (args.ordinality()) @@ -2177,7 +2184,7 @@ void analyseHotspots(WuHotspotResults & results, IConstWorkUnit * wu, IPropertyT { WorkunitStatsAnalyser analyser; analyser.applyOptions(cfg); - analyser.analyse(wu); + analyser.analyse(wu, nullptr); analyser.findHotspots(cfg->queryProp("@rootScope"), results.totalTime, results.hotspots); results.root.setown(analyser.getRootScope()); diff --git a/common/wuanalysis/anawu.hpp b/common/wuanalysis/anawu.hpp index 8db7d227426..db2f004bb46 100644 --- a/common/wuanalysis/anawu.hpp +++ b/common/wuanalysis/anawu.hpp @@ -26,7 +26,7 @@ #include "anacommon.hpp" -void WUANALYSIS_API analyseWorkunit(IWorkUnit * wu, IPropertyTree *options, double costPerMs); +void WUANALYSIS_API analyseWorkunit(IWorkUnit * wu, const char *optGraph, IPropertyTree *options, double costPerMs); void WUANALYSIS_API analyseAndPrintIssues(IConstWorkUnit * wu, double costPerMs, bool updatewu); //--------------------------------------------------------------------------------------------------------------------- diff --git a/ecl/eclagent/eclagent.cpp b/ecl/eclagent/eclagent.cpp index 8071d605cb4..55ffbca51e4 100644 --- a/ecl/eclagent/eclagent.cpp +++ b/ecl/eclagent/eclagent.cpp @@ -1856,6 +1856,27 @@ void EclAgent::setRetcode(int code) retcode = code; } + +void EclAgent::runWorkunitAnalyser(IWorkUnit * w, const char * optGraph) +{ + if (w->getDebugValueBool("analyzeWorkunit", agentTopology->getPropBool("@analyzeWorkunit", true))) + { + double costPerMs = calculateThorCost(1, getNodes()); + IPropertyTree *analyzerOptions = agentTopology->queryPropTree("analyzerOptions"); + analyseWorkunit(w, optGraph, analyzerOptions, costPerMs); + } +} + +static constexpr bool defaultAnalyzeWhenComplete = true; +void EclAgent::runWorkunitAnalyserAfterGraph(const char * graph) +{ + if (!wuRead->getDebugValueBool("analyzeWhenComplete", agentTopology->getPropBool("@analyzeWhenComplete", defaultAnalyzeWhenComplete))) + { + Owned wu(updateWorkUnit()); + runWorkunitAnalyser(wu, graph); + } +} + void EclAgent::doProcess() { #ifdef _DEBUG @@ -2026,15 +2047,22 @@ void EclAgent::doProcess() break; } - if (w->getState() == WUStateCompleted && getClusterType(clusterType)==ThorLCRCluster) + + if (getClusterType(clusterType)==ThorLCRCluster) { - if (w->getDebugValueBool("analyzeWorkunit", agentTopology->getPropBool("@analyzeWorkunit", true))) + if (w->getDebugValueBool("analyzeWhenComplete", agentTopology->getPropBool("@analyzeWhenComplete", defaultAnalyzeWhenComplete))) { - double costPerMs = calculateThorCost(1, getNodes()); - IPropertyTree *analyzerOptions = agentTopology->queryPropTree("analyzerOptions"); - analyseWorkunit(w.get(), analyzerOptions, costPerMs); + switch (w->getState()) + { + case WUStateFailed: + case WUStateAborted: + case WUStateCompleted: + runWorkunitAnalyser(w, nullptr); + break; + } } } + if(w->queryEventScheduledCount() > 0) switch(w->getState()) { diff --git a/ecl/eclagent/eclagent.ipp b/ecl/eclagent/eclagent.ipp index 10c86cc6ebb..e16a082755b 100644 --- a/ecl/eclagent/eclagent.ipp +++ b/ecl/eclagent/eclagent.ipp @@ -422,6 +422,8 @@ private: EclAgentQueryLibrary * loadEclLibrary(const char * libraryName, unsigned expectedInterfaceHash, const char * embeddedGraphName); virtual bool getWorkunitResultFilename(StringBuffer & diskFilename, const char * wuid, const char * name, int seq); virtual IDebuggableContext *queryDebugContext() const { return debugContext; }; + void runWorkunitAnalyser(IWorkUnit * w, const char * optGraph); + void runWorkunitAnalyserAfterGraph(const char * optGraph); //protected by critical section EclGraph * addGraph(const char * graphName); diff --git a/ecl/eclagent/eclgraph.cpp b/ecl/eclagent/eclgraph.cpp index d4ffdd99c42..95f533d8c5e 100644 --- a/ecl/eclagent/eclgraph.cpp +++ b/ecl/eclagent/eclgraph.cpp @@ -1598,7 +1598,16 @@ void EclAgent::executeGraph(const char * graphName, bool realThor, size32_t pare { if (isStandAloneExe) throw MakeStringException(0, "Cannot execute Thor Graph in standalone mode"); - executeThorGraph(graphName, *wuRead, *agentTopology); + try + { + executeThorGraph(graphName, *wuRead, *agentTopology); + runWorkunitAnalyserAfterGraph(graphName); + } + catch (...) + { + runWorkunitAnalyserAfterGraph(graphName); + throw; + } } else {