Merge pull request #17885 from richardkchapman/callLessSIze

HPCC-30475 Roxie disk read calls size() too often Reviewed-by: Gavin Halliday <[email protected]> Merged-by: Gavin Halliday <[email protected]>
hpcc-systems · Oct 12, 2023 · bae0f58 · bae0f58
2 parents 9f4aa1e + 8d9f77c
commit bae0f58
Show file tree

Hide file tree

Showing 3 changed files with 24 additions and 6 deletions.
diff --git a/roxie/ccd/ccdfile.cpp b/roxie/ccd/ccdfile.cpp
@@ -2353,6 +2353,7 @@ class CFileIOArray : implements IFileIOArray, public CInterface
     UnsignedArray subfiles;
     StringArray filenames;
     Int64Array bases;
+    mutable Int64Array sizes;     // Lazy-evaluated and cached
     int actualCrc = 0;
     unsigned valid = 0;
     bool multipleFormatsSeen = false;
@@ -2418,6 +2419,7 @@ class CFileIOArray : implements IFileIOArray, public CInterface
             valid++;
         files.append(f);
         bases.append(base);
+        sizes.append(-1);    // Calculated if/when needed
         if (_actualCrc)
         {
             if (actualCrc && actualCrc != _actualCrc)
@@ -2468,14 +2470,28 @@ class CFileIOArray : implements IFileIOArray, public CInterface
             totalSize = 0;
             ForEachItemIn(idx, files)
             {
-                IFileIO *file = files.item(idx);
-                if (file)
-                    totalSize += file->size();
+                totalSize += partSize(idx);
             }
         }
         return totalSize;
     }
 
+    virtual unsigned __int64 partSize(unsigned idx) const override
+    {
+        CriticalBlock b(crit);
+        __int64 fsize = sizes.item(idx);
+        if (fsize == -1)
+        {
+            IFileIO *file = files.item(idx);
+            if (file)
+                fsize = file->size();
+            else
+                fsize = 0;
+            sizes.replace(fsize, idx);
+        }
+        return (unsigned __int64) fsize;
+    }
+
     virtual StringBuffer &getId(StringBuffer &ret) const override
     {
         CriticalBlock b(crit);

diff --git a/roxie/ccd/ccdkey.cpp b/roxie/ccd/ccdkey.cpp
@@ -691,10 +691,11 @@ class BufferedDirectReader : public CDirectReaderBase
         while (!_partNo && !thisPart && ++thisPartIdx < maxParts) // MORE
         {
             thisPart.setown(f->getFilePart(thisPartIdx, thisFileStartPos));
-            if (thisPart && _startPos >= thisPart->size())
+            offset_t thisPartSize = f->partSize(thisPartIdx);
+            if (thisPart && _startPos >= thisPartSize)
             {
-                _startPos -= thisPart->size();
-                completedStreamsSize += thisPart->size();
+                _startPos -= thisPartSize;
+                completedStreamsSize += thisPartSize;
                 thisPart.clear();
             }
         }

diff --git a/roxie/ccd/ccdstate.hpp b/roxie/ccd/ccdstate.hpp
@@ -93,6 +93,7 @@ interface IFileIOArray : extends IInterface
     virtual unsigned numValid() const = 0;
     virtual bool isValid(unsigned partNo) const = 0;
     virtual unsigned __int64 size() const = 0;
+    virtual unsigned __int64 partSize(unsigned partNo) const = 0;
     virtual StringBuffer &getId(StringBuffer &) const = 0;
     virtual const char *queryLogicalFilename(unsigned partNo) const = 0;
     virtual int queryActualFormatCrc() const = 0;    // Actual format on disk