From bc9998d64aebca47da4af3b863475e621afc4cf7 Mon Sep 17 00:00:00 2001
From: Shamser Ahmed <shamser.ahmed@lexisnexis.com>
Date: Wed, 3 Jan 2024 12:42:11 +0000
Subject: [PATCH] HPCC-31062 Include dfu file op cost in workunit aggregates

The StCostFileAccess from dfu scopes are included in the aggregation
of the StCostFileAccess totals.

Note that an aggregate may be written by one component and then later
updated by another component.  CLocalWorkUnit::setStatistic has been
modified to ignore the creator when looking up the existing value for
a statistic.

Signed-off-by: Shamser Ahmed <shamser.ahmed@lexisnexis.com>

 #	dockerfiles/t4.yaml
---
 common/workunit/workunit.cpp | 30 +++++++++++++++++++++++++-----
 1 file changed, 25 insertions(+), 5 deletions(-)

diff --git a/common/workunit/workunit.cpp b/common/workunit/workunit.cpp
index 3d3f16ddb04..e42373425f3 100644
--- a/common/workunit/workunit.cpp
+++ b/common/workunit/workunit.cpp
@@ -2681,13 +2681,12 @@ void StatisticsAggregator::loadExistingAggregates(const IConstWorkUnit &workunit
     };
 
     WuScopeFilter filter;
-    filter.addScopeType(SSTglobal).addScopeType(SSTworkflow).addScopeType(SSTgraph);
+    filter.addScopeType(SSTglobal).addScopeType(SSTworkflow).addScopeType(SSTgraph).addScopeType(SSToperation);
     const unsigned numStats = mapping.numStatistics();
     for (unsigned i=0; i<numStats; ++i)
         filter.addOutputStatistic(mapping.getKind(i));
-    filter.setDepth(1,3); // 1=global, 2=workflow, 3=graph
+    filter.setDepth(0,3); // 0=global, 1=workflow, (2=graph, 3=subgraph | 2=>dfu)
     filter.setSources(SSFsearchGlobalStats);
-    filter.setIncludeNesting(0);
     filter.finishedFilter();
 
     StatsCollectionAggregatesLoader aggregatesLoader(statsCollection);
@@ -8941,7 +8940,7 @@ void CLocalWorkUnit::setStatistic(StatisticCreatorType creatorType, const char *
     if (mergeAction != StatsMergeAppend)
     {
         StringBuffer xpath;
-        xpath.append("Statistic[@creator='").append(creator).append("'][@scope='").append(scope).append("'][@kind='").append(kindName).append("']");
+        xpath.append("Statistic[@scope='").append(scope).append("'][@kind='").append(kindName).append("']");
         statTree = stats->queryPropTree(xpath.str());
     }
 
@@ -8969,6 +8968,7 @@ void CLocalWorkUnit::setStatistic(StatisticCreatorType creatorType, const char *
         mergeAction = StatsMergeAppend;
     }
 
+    unsigned __int64 deltaValue = 0;
     if (mergeAction != StatsMergeAppend) // RKC->GH Is this right??
     {
         unsigned __int64 oldValue = statTree->getPropInt64("@value", 0);
@@ -8977,14 +8977,18 @@ void CLocalWorkUnit::setStatistic(StatisticCreatorType creatorType, const char *
         if (oldMax < oldValue)
             oldMax = oldValue;
 
-        statTree->setPropInt64("@value", mergeStatisticValue(oldValue, value, mergeAction));
+        unsigned __int64 newValue = mergeStatisticValue(oldValue, value, mergeAction);
+        statTree->setPropInt64("@value", newValue);
         statTree->setPropInt64("@count", count + oldCount);
         if (maxValue > oldMax)
             statTree->setPropInt64("@max", maxValue);
+        deltaValue = newValue - oldValue;
     }
     else
     {
         statTree->setPropInt64("@value", value);
+        deltaValue = value;
+
         statTree->setPropInt64("@count", count);
         if (maxValue)
             statTree->setPropInt64("@max", maxValue);
@@ -9016,6 +9020,22 @@ void CLocalWorkUnit::setStatistic(StatisticCreatorType creatorType, const char *
     }
     if (kind == StCostCompile)
         p->setPropInt64("@costCompile", value);
+
+    // Special case - update aggregates for dfu FileAccessCost.  This is needed because although
+    // fileservices can update dfu cost in the workunit, it does not have a mechanism to update
+    // the aggregates for this stat.
+    if (scopeType == SSTdfuworkunit && kind == StCostFileAccess && deltaValue)
+    {
+        StringBuffer currentScope(scope), parent;
+        while (getParentScope(parent.clear(), currentScope.str()))
+        {
+            currentScope.set(parent);
+            StatisticScopeType sst = getScopeType(queryScopeTail(currentScope.str()));
+            if (sst!=SSTnone)
+                setStatistic(creatorType, creator, sst, currentScope.str(), StCostFileAccess, "", deltaValue, 1, 0, StatsMergeSum);
+        }
+        setStatistic(creatorType, creator, SSTglobal, "", StCostFileAccess, "", deltaValue, 1, 0, StatsMergeSum);
+    }
 }
 
 void CLocalWorkUnit::_loadStatistics() const