HPCC-32138 Capture all temp file stats (including read stats). Implem…

…ent a generic mergeStats function that will remap to the stat names before setting them Signed-off-by: Shamser Ahmed <[email protected]>
hpcc-systems · Jun 27, 2024 · c4d2ad8 · c4d2ad8
1 parent 29ad0a3
commit c4d2ad8
Show file tree

Hide file tree

Showing 4 changed files with 40 additions and 18 deletions.
diff --git a/system/jlib/jstats.h b/system/jlib/jstats.h
@@ -23,6 +23,7 @@
 #include "jmutex.hpp"
 #include <vector>
 #include <initializer_list>
+#include <map>
 
 #include "jstatcodes.h"
 
@@ -874,6 +875,34 @@ void setStat(CRuntimeStatisticCollection & stats, INTERFACE * source, StatisticK
 template <class INTERFACE>
 void setStat(CRuntimeStatisticCollection & stats, const Shared<INTERFACE> & source, StatisticKind kind) { setStat(stats, source.get(), kind); }
 
+template <class INTERFACE>
+void mergeRemappedStats(CRuntimeStatisticCollection & stats, INTERFACE * source, const StatisticsMapping & mapping, const std::map<StatisticKind, StatisticKind> & remap)
+{
+    if (!source)
+        return;
+    unsigned max = mapping.numStatistics();
+    for (unsigned i=0; i < max; i++)
+    {
+        StatisticKind kind = mapping.getKind(i);
+        if (auto matched=remap.find(kind);  matched != remap.end())
+            stats.mergeStatistic(matched->second, source->getStatistic(kind));
+        else
+            stats.mergeStatistic(kind, source->getStatistic(kind));
+    }
+}
+
+template <class INTERFACE>
+void mergeRemappedStats(CRuntimeStatisticCollection & stats, INTERFACE * source, const std::map<StatisticKind, StatisticKind> & remap)
+{
+    mergeRemappedStats(stats, source, stats.queryMapping(), remap);
+}
+
+template <class INTERFACE>
+void mergeRemappedStats(CRuntimeStatisticCollection & stats, const Shared<INTERFACE> & source, const std::map<StatisticKind, StatisticKind> & remap)
+{
+    mergeRemappedStats(stats, source.get(), stats.queryMapping(), remap);
+}
+
 //---------------------------------------------------------------------------------------------------------------------
 
 //A class for minimizing the overhead of collecting timestamps.

diff --git a/thorlcr/activities/hashdistrib/thhashdistribslave.cpp b/thorlcr/activities/hashdistrib/thhashdistribslave.cpp
@@ -2760,26 +2760,11 @@ class CSpill : implements IRowWriter, public CSimpleInterface
         ::Release(writer);
         writer = NULL;
         spillFileIO->flush();
-        mergeStats(stats, this);
-        spillFile->noteSize(getStatistic(StSizeSpillFile));
+        mergeRemappedStats(stats, spillFileIO, diskToSpillStatsMap);
+        stats.setStatistic(StNumSpills, 1);
+        spillFile->noteSize(spillFileIO->getStatistic(StSizeDiskWrite));
         spillFileIO.clear();
     }
-    inline __int64 getStatistic(StatisticKind kind) const
-    {
-        switch (kind)
-        {
-        case StSizeSpillFile:
-            return spillFileIO->getStatistic(StSizeDiskWrite);
-        case StTimeSortElapsed:
-            return spillFileIO->getStatistic(StTimeDiskWriteIO);
-        case StSizeDiskWrite:
-            return 0; // Return file size as StSizeSpillFile kind. To avoid confusion, StSizeDiskWrite will not be returned
-        case StNumSpills:
-            return 1;
-        default:
-            return spillFileIO->getStatistic(kind);
-        }
-    }
 // IRowWriter
     virtual void putRow(const void *row) override
     {

diff --git a/thorlcr/thorutil/thormisc.cpp b/thorlcr/thorutil/thormisc.cpp
@@ -95,6 +95,11 @@ const StatisticsMapping soapcallActivityStatistics({}, basicActivityStatistics,
 const StatisticsMapping hashDedupActivityStatistics({StNumSpills, StSizeSpillFile, StTimeSortElapsed, StSizePeakTempDisk}, diskWriteRemoteStatistics, basicActivityStatistics);
 const StatisticsMapping hashDistribActivityStatistics({StNumLocalRows, StNumRemoteRows, StSizeRemoteWrite}, basicActivityStatistics);
 
+const std::map<StatisticKind, StatisticKind> diskToSpillStatsMap
+={ {StSizeDiskWrite, StSizeSpillFile},
+   {StTimeDiskWriteIO, StTimeSortElapsed}
+ };
+
 MODULE_INIT(INIT_PRIORITY_STANDARD)
 {
     ClusterMPAllocator.setown(createMPtagRangeAllocator(MPTAG_THORGLOBAL_BASE,MPTAG_THORGLOBAL_COUNT));

diff --git a/thorlcr/thorutil/thormisc.hpp b/thorlcr/thorutil/thormisc.hpp
@@ -162,6 +162,9 @@ extern graph_decl const StatisticsMapping indexReadFileStatistics;
 extern graph_decl const StatisticsMapping hashDedupActivityStatistics;
 extern graph_decl const StatisticsMapping hashDistribActivityStatistics;
 
+// Maps disk related stats to spill stats
+extern graph_decl const std::map<StatisticKind, StatisticKind> diskToSpillStatsMap;
+
 class BooleanOnOff
 {
     bool &tf;