From a1078252ee5f316d36526c72443f27d674b3fb32 Mon Sep 17 00:00:00 2001 From: Jake Smith Date: Fri, 22 Sep 2023 14:37:51 +0100 Subject: [PATCH] HPCC-30285 remove multi-plane consistency check Signed-off-by: Jake Smith --- ecl/hthor/hthor.cpp | 13 +++---------- roxie/ccd/ccdserver.cpp | 15 ++++++++------- thorlcr/activities/thdiskbase.cpp | 17 +++++++++-------- 3 files changed, 20 insertions(+), 25 deletions(-) diff --git a/ecl/hthor/hthor.cpp b/ecl/hthor/hthor.cpp index 9c33337a66d..dd419667028 100644 --- a/ecl/hthor/hthor.cpp +++ b/ecl/hthor/hthor.cpp @@ -362,7 +362,6 @@ ClusterWriteHandler *createClusterWriteHandler(IAgentContext &agent, IHThorIndex getDefaultStoragePlane(defaultCluster); Owned clusterHandler; unsigned clusterIdx = 0; - bool outputPlaneCompressed = false; while(true) { @@ -382,14 +381,6 @@ ClusterWriteHandler *createClusterWriteHandler(IAgentContext &agent, IHThorIndex clusterHandler.setown(new CHThorClusterWriteHandler(lfn, "OUTPUT", agent)); } clusterHandler->addCluster(cluster); - - // This is checking that all output planes agree on compression being used or not - Owned plane = getStoragePlane(cluster); - bool thisPlaneCompressed = plane ? plane->getPropBool("@compressLogicalFiles") : false; - if (1 == clusterIdx) - outputPlaneCompressed = thisPlaneCompressed; - else if (outputPlaneCompressed != thisPlaneCompressed) - throw makeStringExceptionV(ENGINEERR_MIXED_COMPRESSED_WRITE, "Writing to planes with mixed compression is not supported %s", lfn); } if(clusterHandler) { @@ -561,8 +552,10 @@ void CHThorDiskWriteActivity::resolve() else getDefaultStoragePlane(planeName); bool outputCompressionDefault = agent.queryWorkUnit()->getDebugValueBool("compressAllOutputs", isContainerized()); + outputPlaneCompressed = outputCompressionDefault; Owned plane = getStoragePlane(planeName); - outputPlaneCompressed = plane ? plane->getPropBool("@compressLogicalFiles", outputCompressionDefault) : outputCompressionDefault; + if (plane) + outputPlaneCompressed = plane->getPropBool("@compressLogicalFiles", outputCompressionDefault); } } else diff --git a/roxie/ccd/ccdserver.cpp b/roxie/ccd/ccdserver.cpp index 49a3ee94855..56eaf6e69e3 100644 --- a/roxie/ccd/ccdserver.cpp +++ b/roxie/ccd/ccdserver.cpp @@ -11970,13 +11970,13 @@ class CRoxieServerDiskWriteActivity : public CRoxieServerInternalSinkActivity, i clusters.append(cluster); clusterIdx++; - // This is checking that all output planes agree on compression being used or not - Owned plane = getStoragePlane(cluster); - bool thisPlaneCompressed = plane ? plane->getPropBool("@compressLogicalFiles", outputCompressionDefault) : outputCompressionDefault; if (1 == clusterIdx) - outputPlaneCompressed = thisPlaneCompressed; - else if (outputPlaneCompressed != thisPlaneCompressed) - throw makeStringExceptionV(ENGINEERR_MIXED_COMPRESSED_WRITE, "Writing to planes with mixed compression is not supported %s", rawLogicalName.get()); + { + // establish default compression from 1st plane, but ECL compression attributes take precedence + Owned plane = getStoragePlane(cluster); + if (plane) + outputPlaneCompressed = plane->getPropBool("@compressLogicalFiles", outputCompressionDefault); + } } if (clusters.length()) { @@ -11990,7 +11990,8 @@ class CRoxieServerDiskWriteActivity : public CRoxieServerInternalSinkActivity, i { clusters.append(defaultCluster); Owned plane = getStoragePlane(defaultCluster); - outputPlaneCompressed = plane ? plane->getPropBool("@compressLogicalFiles", outputCompressionDefault) : outputCompressionDefault; + if (plane) + outputPlaneCompressed = plane->getPropBool("@compressLogicalFiles", outputCompressionDefault); } else if (roxieName.length()) clusters.append(roxieName.str()); diff --git a/thorlcr/activities/thdiskbase.cpp b/thorlcr/activities/thdiskbase.cpp index fc3e6d63798..269d55be407 100644 --- a/thorlcr/activities/thdiskbase.cpp +++ b/thorlcr/activities/thdiskbase.cpp @@ -163,8 +163,8 @@ void CWriteMasterBase::init() } if (dlfn.isExternal()) mpTag = container.queryJob().allocateMPTag(); // used - bool outputPlaneCompressed = false; bool outputCompressionDefault = getOptBool(THOROPT_COMPRESS_ALLFILES, isContainerized()); + bool outputPlaneCompressed = outputCompressionDefault; if (NULL == fileDesc.get()) { bool overwriteok = 0!=(TDWoverwrite & diskHelperBase->getFlags()); @@ -178,13 +178,13 @@ void CWriteMasterBase::init() clusters.append(cluster); idx++; - // This is checking that all output planes agree on compression being used or not - Owned plane = getStoragePlane(cluster); - bool thisPlaneCompressed = plane ? plane->getPropBool("@compressLogicalFiles", outputCompressionDefault) : outputCompressionDefault; if (1 == idx) - outputPlaneCompressed = thisPlaneCompressed; - else if (outputPlaneCompressed != thisPlaneCompressed) - throw makeStringExceptionV(ENGINEERR_MIXED_COMPRESSED_WRITE, "Writing to planes with mixed compression is not supported %s", fileName.get()); + { + // establish default compression from 1st plane, but ECL compression attributes take precedence + Owned plane = getStoragePlane(cluster); + if (plane) + outputPlaneCompressed = plane->getPropBool("@compressLogicalFiles", outputCompressionDefault); + } } IArrayOf groups; @@ -209,7 +209,8 @@ void CWriteMasterBase::init() { clusters.append(defaultCluster); Owned plane = getStoragePlane(defaultCluster); - outputPlaneCompressed = plane ? plane->getPropBool("@compressLogicalFiles", outputCompressionDefault) : outputCompressionDefault; + if (plane) + outputPlaneCompressed = plane->getPropBool("@compressLogicalFiles", outputCompressionDefault); } } }