From b6aaaa12c14ec284d61bcc9c33c7f9e674f72f26 Mon Sep 17 00:00:00 2001 From: Jake Smith Date: Wed, 11 Dec 2024 16:06:15 +0000 Subject: [PATCH] HPCC-33113 Ensure empty compressed part size is correct When writing a logical file that is wider than Thor with, Thor pads the file at publish time with empty parts. If compressed it creates an empty compressed file, which will typically be non-zero in size (e.g. contains compressed header). The logical file part meta data did not relect this correct physical size. Consequently, as a result of the check added in HPCC-33064, reading these files was provoking this error check. Signed-off-by: Jake Smith --- thorlcr/activities/thdiskbase.cpp | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/thorlcr/activities/thdiskbase.cpp b/thorlcr/activities/thdiskbase.cpp index dae7d5d870f..ba5f23b0498 100644 --- a/thorlcr/activities/thdiskbase.cpp +++ b/thorlcr/activities/thdiskbase.cpp @@ -293,6 +293,9 @@ void CWriteMasterBase::publish() compMethod = COMPRESS_METHOD_LZ4HC; bool blockCompressed; bool compressed = fileDesc->isCompressed(&blockCompressed); + + // NB: it would be far preferable to avoid this and have the file reference a group with the correct number of parts + // Possibly could use subgroup syntax: 'data[1..n]' for (unsigned clusterIdx=0; clusterIdxnumClusters(); clusterIdx++) { StringBuffer clusterName; @@ -305,6 +308,7 @@ void CWriteMasterBase::publish() p += queryJob().querySlaves(); IPartDescriptor *partDesc = fileDesc->queryPart(p); CDateTime createTime, modifiedTime; + offset_t compSize = 0; for (unsigned c=0; cnumCopies(); c++) { RemoteFilename rfn; @@ -316,7 +320,7 @@ void CWriteMasterBase::publish() ensureDirectoryForFile(path.str()); OwnedIFile iFile = createIFile(path.str()); Owned iFileIO; - if (compressed) + if (compressed) // NB: this would not be necessary if all builds have the changes in HPCC-32651 iFileIO.setown(createCompressedFileWriter(iFile, recordSize, false, true, NULL, compMethod)); else iFileIO.setown(iFile->open(IFOcreate)); @@ -324,7 +328,10 @@ void CWriteMasterBase::publish() iFileIO.clear(); // ensure copies have matching datestamps, as they would do normally (backupnode expects it) if (0 == c) + { iFile->getTime(&createTime, &modifiedTime, NULL); + compSize = iFile->size(); + } else iFile->setTime(&createTime, &modifiedTime, NULL); } @@ -345,7 +352,7 @@ void CWriteMasterBase::publish() props.setPropInt64("@recordCount", 0); props.setPropInt64("@size", 0); if (compressed) - props.setPropInt64("@compressedSize", 0); + props.setPropInt64("@compressedSize", compSize); p++; } }