Merge pull request #19354 from jakesmith/HPCC-33116-suppress-filepart…

…-check HPCC-33116 Suppress file check if invalid @compressedSize(0) Reviewed-by: Gavin Halliday <[email protected]> Merged-by: Gavin Halliday <[email protected]>
hpcc-systems · Dec 12, 2024 · 6b80b53 · 6b80b53
2 parents 797e82d + b91afa5
commit 6b80b53
Show file tree

Hide file tree

Showing 4 changed files with 49 additions and 14 deletions.
diff --git a/dali/base/dadfs.cpp b/dali/base/dadfs.cpp
@@ -13807,6 +13807,45 @@ void configurePreferredPlanes()
     }
 }
 
+static bool doesPhysicalMatchMeta(IPropertyTree &partProps, IFile &iFile, offset_t expectedSize, offset_t &actualSize)
+{
+    // NB: temporary workaround for 'narrow' files publishing extra empty parts with the wrong @compressedSize(0)
+    // causing a new check introduced in HPCC-33064 to be hit (fixed in HPCC-33113, but will continue to affect exiting files)
+    unsigned __int64 size = partProps.getPropInt64("@size", unknownFileSize);
+    unsigned __int64 compressedSize = partProps.getPropInt64("@compressedSize", unknownFileSize);
+    if ((0 == size) && (0 == compressedSize))
+    {
+        actualSize = unknownFileSize;
+        return true;
+    }
+
+    if (expectedSize != unknownFileSize)
+    {
+        actualSize = iFile.size();
+        if (actualSize != expectedSize)
+            return false;
+    }
+    else
+        actualSize = unknownFileSize;
+
+    return true;
+}
+
+bool doesPhysicalMatchMeta(IPartDescriptor &partDesc, IFile &iFile, offset_t &expectedSize, offset_t &actualSize)
+{
+    IPropertyTree &partProps = partDesc.queryProperties();
+    expectedSize = partDesc.getDiskSize(false, false);
+    return doesPhysicalMatchMeta(partProps, iFile, expectedSize, actualSize);
+}
+
+bool doesPhysicalMatchMeta(IDistributedFilePart &part, IFile &iFile, offset_t &expectedSize, offset_t &actualSize)
+{
+    IPropertyTree &partProps = part.queryAttributes();
+    expectedSize = part.getDiskSize(false, false);
+    return doesPhysicalMatchMeta(partProps, iFile, expectedSize, actualSize);
+}
+
+
 #ifdef _USE_CPPUNIT
 /*
  * This method removes files only logically. removeEntry() used to do that, but the only

diff --git a/dali/base/dadfs.hpp b/dali/base/dadfs.hpp
@@ -925,4 +925,8 @@ inline cost_type getLegacyWriteCost(const IPropertyTree & fileAttr, Source sourc
     else
         return 0;
 }
+
+extern da_decl bool doesPhysicalMatchMeta(IPartDescriptor &partDesc, IFile &iFile, offset_t &expectedSize, offset_t &actualSize);
+extern da_decl bool doesPhysicalMatchMeta(IDistributedFilePart &partDesc, IFile &iFile, offset_t &expectedSize, offset_t &actualSize);
+
 #endif
diff --git a/ecl/hthor/hthor.cpp b/ecl/hthor/hthor.cpp
@@ -8737,13 +8737,9 @@ bool CHThorDiskReadBaseActivity::openNext()
 
                         if (curPart)
                         {
-                            offset_t expectedSize = curPart->getDiskSize(false, false);
-                            if (expectedSize != unknownFileSize)
-                            {
-                                offset_t actualSize = inputfile->size();
-                                if(actualSize != expectedSize)
-                                    throw MakeStringException(0, "File size mismatch: file %s was supposed to be %" I64F "d bytes but appears to be %" I64F "d bytes", inputfile->queryFilename(), expectedSize, actualSize);
-                            }
+                            offset_t expectedSize, actualSize;
+                            if (!doesPhysicalMatchMeta(*curPart, *inputfile, expectedSize, actualSize))
+                                throw makeStringExceptionV(0, "File size mismatch: file %s was supposed to be %" I64F "d bytes but appears to be %" I64F "d bytes", inputfile->queryFilename(), expectedSize, actualSize);
                         }
 
                         if (compressed)

diff --git a/thorlcr/activities/diskread/thdiskreadslave.cpp b/thorlcr/activities/diskread/thdiskreadslave.cpp
@@ -366,13 +366,9 @@ void CDiskRecordPartHandler::open()
 
         rwFlags |= DEFAULT_RWFLAGS;
 
-        offset_t expectedSize = partDesc->getDiskSize(false, false);
-        if (expectedSize != unknownFileSize)
-        {
-            offset_t actualSize = iFile->size();
-            if(actualSize != expectedSize)
-                throw MakeStringException(0, "File size mismatch: file %s was supposed to be %" I64F "d bytes but appears to be %" I64F "d bytes", iFile->queryFilename(), expectedSize, actualSize);
-        }
+        offset_t expectedSize, actualSize;
+        if (!doesPhysicalMatchMeta(*partDesc, *iFile, expectedSize, actualSize))
+            throw MakeActivityException(&activity, 0, "File size mismatch: file %s was supposed to be %" I64F "d bytes but appears to be %" I64F "d bytes", iFile->queryFilename(), expectedSize, actualSize);
 
         if (compressed)
         {