From 5122080ff8d43f584ee420a217747bee628fef7f Mon Sep 17 00:00:00 2001 From: Jack Del Vecchio Date: Mon, 25 Mar 2024 16:27:20 +0000 Subject: [PATCH] HPCC-31715 Change the default field translation mode --- helm/hpcc/values.schema.json | 4 ++-- initfiles/componentfiles/configschema/xsd/eclagent.xsd | 5 +++-- initfiles/componentfiles/configschema/xsd/roxie.xsd | 5 +++-- initfiles/componentfiles/configschema/xsd/thor.xsd | 5 +++-- initfiles/componentfiles/configxml/eclagent_config.xsd.in | 3 ++- initfiles/componentfiles/configxml/roxie.xsd.in | 3 ++- initfiles/componentfiles/configxml/thor.xsd.in | 3 ++- initfiles/etc/DIR_NAME/environment.xml.in | 2 +- roxie/ccd/ccdmain.cpp | 4 ++-- roxie/ccd/ccdstate.cpp | 2 +- rtl/eclrtl/rtldynfield.cpp | 4 +++- testing/regress/ecl/layouttrans.ecl | 2 +- testing/regress/ecl/sqtrans.ecl | 2 ++ testing/regress/ecl/translatedisk.ecl | 2 +- testing/regress/environment.xml.in | 2 +- 15 files changed, 29 insertions(+), 19 deletions(-) diff --git a/helm/hpcc/values.schema.json b/helm/hpcc/values.schema.json index b7d14db9d44..21b7a74173b 100644 --- a/helm/hpcc/values.schema.json +++ b/helm/hpcc/values.schema.json @@ -1860,8 +1860,8 @@ "description": "Should the index node memory allocation flush the cache and retry if memory allocation fails" }, "fieldTranslationEnabled": { - "default": "payload", - "enum": ["false", "true", "payload"], + "default": "payloadRemoveOnly", + "enum": ["false", "true", "payload", "payloadRemoveOnly"], "description": "Enables translation (where possible) of mismatched file layouts on-the-fly. Specify 'payload' to attempt to translate payload fields only" }, "highTimeout": { diff --git a/initfiles/componentfiles/configschema/xsd/eclagent.xsd b/initfiles/componentfiles/configschema/xsd/eclagent.xsd index 9278367bc88..a1ca70f7f1c 100644 --- a/initfiles/componentfiles/configschema/xsd/eclagent.xsd +++ b/initfiles/componentfiles/configschema/xsd/eclagent.xsd @@ -57,13 +57,14 @@ hpcc:presetValue="false" hpcc:tooltip="eclAgent analyzes workunit post-execution"/> - + + diff --git a/initfiles/componentfiles/configschema/xsd/roxie.xsd b/initfiles/componentfiles/configschema/xsd/roxie.xsd index 14da617e884..65eef7ddea2 100644 --- a/initfiles/componentfiles/configschema/xsd/roxie.xsd +++ b/initfiles/componentfiles/configschema/xsd/roxie.xsd @@ -110,13 +110,14 @@ hpcc:displayName="flushJHtreeCacheOnOOM" hpcc:presetValue="true" hpcc:tooltip="Should the index node memory allocation flush the cache and retry if memory allocation fails"/> - + + diff --git a/initfiles/componentfiles/configschema/xsd/thor.xsd b/initfiles/componentfiles/configschema/xsd/thor.xsd index ae0c8579ec4..c9183448b86 100644 --- a/initfiles/componentfiles/configschema/xsd/thor.xsd +++ b/initfiles/componentfiles/configschema/xsd/thor.xsd @@ -224,13 +224,14 @@ - + + diff --git a/initfiles/componentfiles/configxml/eclagent_config.xsd.in b/initfiles/componentfiles/configxml/eclagent_config.xsd.in index d4c6e0b1153..447c347192f 100644 --- a/initfiles/componentfiles/configxml/eclagent_config.xsd.in +++ b/initfiles/componentfiles/configxml/eclagent_config.xsd.in @@ -190,7 +190,7 @@ - + Enables translation (where possible) of mismatched file layouts on-the-fly. Specify 'payload' to attempt to translate payload fields only @@ -201,6 +201,7 @@ + diff --git a/initfiles/componentfiles/configxml/roxie.xsd.in b/initfiles/componentfiles/configxml/roxie.xsd.in index 3175574f6d7..dcb5a533a7c 100644 --- a/initfiles/componentfiles/configxml/roxie.xsd.in +++ b/initfiles/componentfiles/configxml/roxie.xsd.in @@ -638,7 +638,7 @@ - + Enables translation (where possible) of mismatched file layouts on-the-fly. Specify 'payload' to attempt to translate payload fields only @@ -649,6 +649,7 @@ + diff --git a/initfiles/componentfiles/configxml/thor.xsd.in b/initfiles/componentfiles/configxml/thor.xsd.in index b3556aa3324..696e0985b5f 100644 --- a/initfiles/componentfiles/configxml/thor.xsd.in +++ b/initfiles/componentfiles/configxml/thor.xsd.in @@ -677,7 +677,7 @@ - + Enables translation (where possible) of mismatched file layouts on-the-fly. Specify 'payload' to attempt to translate payload fields only @@ -688,6 +688,7 @@ + diff --git a/initfiles/etc/DIR_NAME/environment.xml.in b/initfiles/etc/DIR_NAME/environment.xml.in index 14a6ada1e06..afbe84bc65d 100644 --- a/initfiles/etc/DIR_NAME/environment.xml.in +++ b/initfiles/etc/DIR_NAME/environment.xml.in @@ -68,7 +68,7 @@ enableKeyDiff="true" enableSysLog="true" fastLaneQueue="true" - fieldTranslationEnabled="payload" + fieldTranslationEnabled="payloadRemoveOnly" flushJHtreeCacheOnOOM="true" forceStdLog="false" heapRetainMemory="false" diff --git a/roxie/ccd/ccdmain.cpp b/roxie/ccd/ccdmain.cpp index cb12d56df0e..82685985033 100644 --- a/roxie/ccd/ccdmain.cpp +++ b/roxie/ccd/ccdmain.cpp @@ -103,7 +103,7 @@ unsigned defaultTraceLimit = 10; unsigned watchActivityId = 0; unsigned testAgentFailure = 0; RelaxedAtomic restarts; -RecordTranslationMode fieldTranslationEnabled = RecordTranslationMode::Payload; +RecordTranslationMode fieldTranslationEnabled = RecordTranslationMode::PayloadRemoveOnly; bool mergeAgentStatistics = true; PTreeReaderOptions defaultXmlReadFlags = ptr_ignoreWhiteSpace; bool runOnce = false; @@ -1181,7 +1181,7 @@ int CCD_API roxie_main(int argc, const char *argv[], const char * defaultYaml) agentQueryReleaseDelaySeconds = topology->getPropInt("@agentQueryReleaseDelaySeconds", topology->getPropInt("@slaveQueryReleaseDelaySeconds", 60)); // legacy name coresPerQuery = topology->getPropInt("@coresPerQuery", 0); - fieldTranslationEnabled = RecordTranslationMode::Payload; + fieldTranslationEnabled = RecordTranslationMode::PayloadRemoveOnly; const char *val = topology->queryProp("@fieldTranslationEnabled"); if (val) fieldTranslationEnabled = getTranslationMode(val, false); diff --git a/roxie/ccd/ccdstate.cpp b/roxie/ccd/ccdstate.cpp index c09c603e6ed..1b239971d9c 100644 --- a/roxie/ccd/ccdstate.cpp +++ b/roxie/ccd/ccdstate.cpp @@ -2463,7 +2463,7 @@ class CRoxiePackageSetManager : implements IRoxieQueryPackageManagerSet, impleme if (val) fieldTranslationEnabled = getTranslationMode(val, false); else - fieldTranslationEnabled = RecordTranslationMode::Payload; + fieldTranslationEnabled = RecordTranslationMode::PayloadRemoveOnly; val = getTranslationModeText(fieldTranslationEnabled); topology->setProp("@fieldTranslationEnabled", val); } diff --git a/rtl/eclrtl/rtldynfield.cpp b/rtl/eclrtl/rtldynfield.cpp index 31d8ad9f02c..c4517176910 100644 --- a/rtl/eclrtl/rtldynfield.cpp +++ b/rtl/eclrtl/rtldynfield.cpp @@ -36,7 +36,9 @@ extern ECLRTL_API RecordTranslationMode getTranslationMode(const char *val, bool isLocal) { - if (isEmptyString(val) || strToBool(val) || strieq(val, "payload")) + if (isEmptyString(val) || strToBool(val) || strieq(val, "payloadRemoveOnly")) + return RecordTranslationMode::PayloadRemoveOnly; + else if (strieq(val, "payload")) return RecordTranslationMode::Payload; else if (strieq(val, "payloadRemoveOnly")) return RecordTranslationMode::PayloadRemoveOnly; diff --git a/testing/regress/ecl/layouttrans.ecl b/testing/regress/ecl/layouttrans.ecl index fb4535c9e8d..b60f3a68304 100644 --- a/testing/regress/ecl/layouttrans.ecl +++ b/testing/regress/ecl/layouttrans.ecl @@ -26,7 +26,7 @@ version := #IFDEFINED(root.version, 1); //--- end of version configuration --- -#option ('layoutTranslation', true); +#option ('layoutTranslation', 'payload'); #onwarning (4515, ignore); #onwarning (4522, ignore); #onwarning (4523, ignore); diff --git a/testing/regress/ecl/sqtrans.ecl b/testing/regress/ecl/sqtrans.ecl index b342b7e817e..a712e067766 100644 --- a/testing/regress/ecl/sqtrans.ecl +++ b/testing/regress/ecl/sqtrans.ecl @@ -22,6 +22,8 @@ multiPart := #IFDEFINED(root.multiPart, false); //--- end of version configuration --- +#option('layoutTranslation', 'payload'); + //check that compound disk read activities workcorrectly when the input file is translated //include keyed filters to ensure they are also translated correctly diff --git a/testing/regress/ecl/translatedisk.ecl b/testing/regress/ecl/translatedisk.ecl index 619e627b9ad..5cb92428242 100644 --- a/testing/regress/ecl/translatedisk.ecl +++ b/testing/regress/ecl/translatedisk.ecl @@ -27,7 +27,7 @@ optRemoteRead := #IFDEFINED(root.optRemoteRead, true); #onwarning(2036, ignore); #onwarning(4522, ignore); -#option ('layoutTranslation', true); +#option ('layoutTranslation', 'payload'); #option('forceRemoteRead', optRemoteRead); import $.Setup; diff --git a/testing/regress/environment.xml.in b/testing/regress/environment.xml.in index 46eb65517d2..709423ee436 100644 --- a/testing/regress/environment.xml.in +++ b/testing/regress/environment.xml.in @@ -65,7 +65,7 @@ enableKeyDiff="true" enableSysLog="true" fastLaneQueue="true" - fieldTranslationEnabled="payload" + fieldTranslationEnabled="payloadRemoveOnly" flushJHtreeCacheOnOOM="true" forceStdLog="false" heapRetainMemory="false"