HPCC-31715 Change the default field translation mode

hpcc-systems · May 7, 2024 · 5122080 · 5122080
1 parent ef381ba
commit 5122080
Show file tree

Hide file tree

Showing 15 changed files with 29 additions and 19 deletions.
diff --git a/helm/hpcc/values.schema.json b/helm/hpcc/values.schema.json
@@ -1860,8 +1860,8 @@
           "description": "Should the index node memory allocation flush the cache and retry if memory allocation fails"
         },
         "fieldTranslationEnabled": {
-          "default": "payload",
-          "enum": ["false", "true", "payload"],
+          "default": "payloadRemoveOnly",
+          "enum": ["false", "true", "payload", "payloadRemoveOnly"],
           "description": "Enables translation (where possible) of mismatched file layouts on-the-fly. Specify 'payload' to attempt to translate payload fields only"
         },
         "highTimeout": { 

diff --git a/initfiles/componentfiles/configschema/xsd/eclagent.xsd b/initfiles/componentfiles/configschema/xsd/eclagent.xsd
@@ -57,13 +57,14 @@
                                   hpcc:presetValue="false"
                                   hpcc:tooltip="eclAgent analyzes workunit post-execution"/>
                     <xs:attribute name="fieldTranslationEnabled" hpcc:displayName="Enable Field Translation"
-                                  hpcc:presetValue="payload"
+                                  hpcc:presetValue="payloadRemoveOnly"
                                   hpcc:tooltip="Enables translation (where possible) of mismatched index layouts on-the-fly. Specify 'payload' to attempt to translate payload fields only">
                         <xs:simpleType>
                             <xs:restriction base="xs:string">
                                 <xs:enumeration value="false" hpcc:displayName="False" hpcc:desciption=""/>
-                                <xs:enumeration value="true" hpcc:displayName="true" hpcc:description=""/>
+                                <xs:enumeration value="true" hpcc:displayName="True" hpcc:description=""/>
                                 <xs:enumeration value="payload" hpcc:displayName="Payload" hpcc:description=""/>
+                                <xs:enumeration value="payloadRemoveOnly" hpcc:displayName="PayloadRemoveOnly" hpcc:description=""/>
                             </xs:restriction>
                         </xs:simpleType>
                     </xs:attribute>

diff --git a/initfiles/componentfiles/configschema/xsd/roxie.xsd b/initfiles/componentfiles/configschema/xsd/roxie.xsd
@@ -110,13 +110,14 @@
                                   hpcc:displayName="flushJHtreeCacheOnOOM" hpcc:presetValue="true"
                                   hpcc:tooltip="Should the index node memory allocation flush the cache and retry if memory allocation fails"/>
                     <xs:attribute name="fieldTranslationEnabled" hpcc:displayName="Enable Field Translation"
-                                  hpcc:presetValue="payload"
+                                  hpcc:presetValue="payloadRemoveOnly"
                                   hpcc:tooltip="Enables translation (where possible) of mismatched index layouts on-the-fly. Specify 'payload' to attempt to translate payload fields only">
                         <xs:simpleType>
                             <xs:restriction base="xs:string">
                                 <xs:enumeration value="false" hpcc:displayName="False" hpcc:desciption=""/>
-                                <xs:enumeration value="true" hpcc:displayName="true" hpcc:description=""/>
+                                <xs:enumeration value="true" hpcc:displayName="True" hpcc:description=""/>
                                 <xs:enumeration value="payload" hpcc:displayName="Payload" hpcc:description=""/>
+                                <xs:enumeration value="payloadRemoveOnly" hpcc:displayName="PayloadRemoveOnly" hpcc:description=""/>
                             </xs:restriction>
                         </xs:simpleType>
                     </xs:attribute>

diff --git a/initfiles/componentfiles/configschema/xsd/thor.xsd b/initfiles/componentfiles/configschema/xsd/thor.xsd
@@ -224,13 +224,14 @@
                 <xs:attribute name="httpGlobalIdHeader" type="xs:string" hpcc:displayName="HTTP GlobalId Header" hpcc:presetValue="HPCC-Global-Id"
                               hpcc:tooltip="HTTP Header field to use for sending and receiving GlobalId"/>
                 <xs:attribute name="fieldTranslationEnabled" hpcc:displayName="Enable Field Translation"
-                              hpcc:presetValue="payload"
+                              hpcc:presetValue="payloadRemoveOnly"
                               hpcc:tooltip="Enables translation (where possible) of mismatched index layouts on-the-fly. Specify 'payload' to attempt to translate payload fields only">
                     <xs:simpleType>
                         <xs:restriction base="xs:string">
                             <xs:enumeration value="false" hpcc:displayName="False" hpcc:desciption=""/>
-                            <xs:enumeration value="true" hpcc:displayName="true" hpcc:description=""/>
+                            <xs:enumeration value="true" hpcc:displayName="True" hpcc:description=""/>
                             <xs:enumeration value="payload" hpcc:displayName="Payload" hpcc:description=""/>
+                            <xs:enumeration value="payloadRemoveOnly" hpcc:displayName="PayloadRemoveOnly" hpcc:description=""/>
                         </xs:restriction>
                     </xs:simpleType>
                 </xs:attribute>

diff --git a/initfiles/componentfiles/configxml/eclagent_config.xsd.in b/initfiles/componentfiles/configxml/eclagent_config.xsd.in
@@ -190,7 +190,7 @@
       </xs:annotation>
     </xs:attribute>
 
-    <xs:attribute name="fieldTranslationEnabled" use="optional" default="payload">
+    <xs:attribute name="fieldTranslationEnabled" use="optional" default="payloadRemoveOnly">
       <xs:annotation>
         <xs:appinfo>
           <tooltip>Enables translation (where possible) of mismatched file layouts on-the-fly. Specify 'payload' to attempt to translate payload fields only</tooltip>
@@ -201,6 +201,7 @@
           <xs:enumeration value="false"/>
           <xs:enumeration value="true"/>
           <xs:enumeration value="payload"/>
+          <xs:enumeration value="payloadRemoveOnly"/>
         </xs:restriction>
       </xs:simpleType>
     </xs:attribute>

diff --git a/initfiles/componentfiles/configxml/roxie.xsd.in b/initfiles/componentfiles/configxml/roxie.xsd.in
@@ -638,7 +638,7 @@
         </xs:appinfo>
       </xs:annotation>
     </xs:attribute>
-    <xs:attribute name="fieldTranslationEnabled" use="optional" default="payload">
+    <xs:attribute name="fieldTranslationEnabled" use="optional" default="payloadRemoveOnly">
       <xs:annotation>
         <xs:appinfo>
           <tooltip>Enables translation (where possible) of mismatched file layouts on-the-fly. Specify 'payload' to attempt to translate payload fields only</tooltip>
@@ -649,6 +649,7 @@
           <xs:enumeration value="false"/>
           <xs:enumeration value="true"/>
           <xs:enumeration value="payload"/>
+          <xs:enumeration value="payloadRemoveOnly"/>
         </xs:restriction>
       </xs:simpleType>
     </xs:attribute>

diff --git a/initfiles/componentfiles/configxml/thor.xsd.in b/initfiles/componentfiles/configxml/thor.xsd.in
@@ -677,7 +677,7 @@
         </xs:appinfo>
       </xs:annotation>
     </xs:attribute>
-    <xs:attribute name="fieldTranslationEnabled" use="optional" default="payload">
+    <xs:attribute name="fieldTranslationEnabled" use="optional" default="payloadRemoveOnly">
       <xs:annotation>
         <xs:appinfo>
           <tooltip>Enables translation (where possible) of mismatched file layouts on-the-fly. Specify 'payload' to attempt to translate payload fields only</tooltip>
@@ -688,6 +688,7 @@
           <xs:enumeration value="false"/>
           <xs:enumeration value="true"/>
           <xs:enumeration value="payload"/>
+          <xs:enumeration value="payloadRemoveOnly"/>
         </xs:restriction>
       </xs:simpleType>
     </xs:attribute>

diff --git a/initfiles/etc/DIR_NAME/environment.xml.in b/initfiles/etc/DIR_NAME/environment.xml.in
@@ -68,7 +68,7 @@
                 enableKeyDiff="true"
                 enableSysLog="true"
                 fastLaneQueue="true"
-                fieldTranslationEnabled="payload"
+                fieldTranslationEnabled="payloadRemoveOnly"
                 flushJHtreeCacheOnOOM="true"
                 forceStdLog="false"
                 heapRetainMemory="false"

diff --git a/roxie/ccd/ccdmain.cpp b/roxie/ccd/ccdmain.cpp
@@ -103,7 +103,7 @@ unsigned defaultTraceLimit = 10;
 unsigned watchActivityId = 0;
 unsigned testAgentFailure = 0;
 RelaxedAtomic<unsigned> restarts;
-RecordTranslationMode fieldTranslationEnabled = RecordTranslationMode::Payload;
+RecordTranslationMode fieldTranslationEnabled = RecordTranslationMode::PayloadRemoveOnly;
 bool mergeAgentStatistics = true;
 PTreeReaderOptions defaultXmlReadFlags = ptr_ignoreWhiteSpace;
 bool runOnce = false;
@@ -1181,7 +1181,7 @@ int CCD_API roxie_main(int argc, const char *argv[], const char * defaultYaml)
         agentQueryReleaseDelaySeconds = topology->getPropInt("@agentQueryReleaseDelaySeconds", topology->getPropInt("@slaveQueryReleaseDelaySeconds", 60));  // legacy name
         coresPerQuery = topology->getPropInt("@coresPerQuery", 0);
 
-        fieldTranslationEnabled = RecordTranslationMode::Payload;
+        fieldTranslationEnabled = RecordTranslationMode::PayloadRemoveOnly;
         const char *val = topology->queryProp("@fieldTranslationEnabled");
         if (val)
             fieldTranslationEnabled = getTranslationMode(val, false);

diff --git a/roxie/ccd/ccdstate.cpp b/roxie/ccd/ccdstate.cpp
@@ -2463,7 +2463,7 @@ class CRoxiePackageSetManager : implements IRoxieQueryPackageManagerSet, impleme
                 if (val)
                     fieldTranslationEnabled = getTranslationMode(val, false);
                 else
-                    fieldTranslationEnabled = RecordTranslationMode::Payload;
+                    fieldTranslationEnabled = RecordTranslationMode::PayloadRemoveOnly;
                 val = getTranslationModeText(fieldTranslationEnabled);
                 topology->setProp("@fieldTranslationEnabled", val);
             }

diff --git a/rtl/eclrtl/rtldynfield.cpp b/rtl/eclrtl/rtldynfield.cpp
@@ -36,7 +36,9 @@
 
 extern ECLRTL_API RecordTranslationMode getTranslationMode(const char *val, bool isLocal)
 {
-    if (isEmptyString(val) || strToBool(val) || strieq(val, "payload"))
+    if (isEmptyString(val) || strToBool(val) || strieq(val, "payloadRemoveOnly"))
+        return RecordTranslationMode::PayloadRemoveOnly;
+    else if (strieq(val, "payload"))
         return RecordTranslationMode::Payload;
     else if (strieq(val, "payloadRemoveOnly"))
         return RecordTranslationMode::PayloadRemoveOnly;

diff --git a/testing/regress/ecl/layouttrans.ecl b/testing/regress/ecl/layouttrans.ecl
@@ -26,7 +26,7 @@ version := #IFDEFINED(root.version, 1);
 
 //--- end of version configuration ---
 
-#option ('layoutTranslation', true);
+#option ('layoutTranslation', 'payload');
 #onwarning (4515, ignore);
 #onwarning (4522, ignore);
 #onwarning (4523, ignore);

diff --git a/testing/regress/ecl/sqtrans.ecl b/testing/regress/ecl/sqtrans.ecl
@@ -22,6 +22,8 @@ multiPart := #IFDEFINED(root.multiPart, false);
 
 //--- end of version configuration ---
 
+#option('layoutTranslation', 'payload');
+
 //check that compound disk read activities workcorrectly when the input file is translated
 //include keyed filters to ensure they are also translated correctly
 

diff --git a/testing/regress/ecl/translatedisk.ecl b/testing/regress/ecl/translatedisk.ecl
@@ -27,7 +27,7 @@ optRemoteRead := #IFDEFINED(root.optRemoteRead, true);
 
 #onwarning(2036, ignore);
 #onwarning(4522, ignore);
-#option ('layoutTranslation', true);
+#option ('layoutTranslation', 'payload');
 #option('forceRemoteRead', optRemoteRead);
 import $.Setup;
 

diff --git a/testing/regress/environment.xml.in b/testing/regress/environment.xml.in
@@ -65,7 +65,7 @@
                 enableKeyDiff="true"
                 enableSysLog="true"
                 fastLaneQueue="true"
-                fieldTranslationEnabled="payload"
+                fieldTranslationEnabled="payloadRemoveOnly"
                 flushJHtreeCacheOnOOM="true"
                 forceStdLog="false"
                 heapRetainMemory="false"