HPCC-31715 Change the default field translation mode

common/thorhelper/thorcommon.cpp

@@ -2148,6 +2148,9 @@ static bool getTranslators(Owned<const IDynamicTransform> &translator, Owned<con
             if (!translator->canTranslate())
                 throw MakeStringException(0, "Untranslatable record layout mismatch detected for file %s", tracing);
 
+            if (mode == RecordTranslationMode::PayloadRemoveOnly && translator->hasNewFields())
+                throw MakeStringException(0, "Translatable file layout mismatch reading file %s but translation disabled when expected fields are missing from source.", tracing);
+
             if (translator->needsTranslate())
             {
                 if (keyedTranslator && (sourceFormat != expectedFormat))

common/thorhelper/thorread.cpp

@@ -176,6 +176,9 @@ void DiskReadMapping::ensureTranslators() const
         if (!translator->canTranslate())
             throw MakeStringException(0, "Untranslatable record layout mismatch detected for file %s", filename);
 
+        if (mode == RecordTranslationMode::PayloadRemoveOnly && translator->hasNewFields())
+            throw MakeStringException(0, "Translatable file layout mismatch reading file %s but translation disabled when expected fields are missing from source.", filename);
+
         if (translator->needsTranslate())
         {
             if (sourceMeta != expectedMeta)

ecl/hthor/hthorkey.cpp

@@ -700,6 +700,8 @@ const IDynamicTransform * CHThorIndexReadActivityBase::getLayoutTranslator(IDist
     Owned<const IDynamicTransform> payloadTranslator =  createRecordTranslator(projectedFormat->queryRecordAccessor(true), actualFormat->queryRecordAccessor(true));
     if (!payloadTranslator->canTranslate())
         throw MakeStringException(0, "Untranslatable key layout mismatch reading index %s", f->queryLogicalName());
+    if (getLayoutTranslationMode() == RecordTranslationMode::PayloadRemoveOnly && payloadTranslator->hasNewFields())
+        throw MakeStringException(0, "Translatable file layout mismatch reading file %s but translation disabled when expected fields are missing from source.", f->queryLogicalName());
     if (payloadTranslator->needsTranslate())
         return payloadTranslator.getClear();
     return nullptr;
@@ -715,6 +717,8 @@ void CHThorIndexReadActivityBase::verifyIndex(IKeyIndex * idx)
         {
             if (!layoutTrans->canTranslate())
                 throw MakeStringException(0, "Untranslatable key layout mismatch reading index %s", df->queryLogicalName());
+            if (getLayoutTranslationMode() == RecordTranslationMode::PayloadRemoveOnly && layoutTrans->hasNewFields())
+                throw MakeStringException(0, "Translatable file layout mismatch reading file %s but translation disabled when expected fields are missing from source.", df->queryLogicalName());
         }
         else
         {
@@ -2462,6 +2466,8 @@ class CHThorFlatFetchActivity : public CHThorFetchActivityBase, public IFlatFetc
                     translator->describe();
                     if (translator->canTranslate())
                     {
+                        if (getLayoutTranslationMode()==RecordTranslationMode::PayloadRemoveOnly && translator->hasNewFields())
+                            throw MakeStringException(0, "Translatable file layout mismatch reading file %s but translation disabled when expected fields are missing from source.", f->queryLogicalName());
                         if (getLayoutTranslationMode()==RecordTranslationMode::None)
                             throw MakeStringException(0, "Translatable file layout mismatch reading file %s but translation disabled", f->queryLogicalName());
                         VStringBuffer msg("Record layout translation required for %s", f->queryLogicalName());
@@ -4102,6 +4108,8 @@ class CHThorKeyedJoinActivity  : public CHThorThreadedActivityBase, implements I
                 throw MakeStringException(0, "Untranslatable key layout mismatch reading index %s", f->queryLogicalName());
             if (payloadTranslator->keyedTranslated())
                 throw MakeStringException(0, "Untranslatable key layout mismatch reading index %s - keyed fields do not match", f->queryLogicalName());
+            if (getLayoutTranslationMode()==RecordTranslationMode::PayloadRemoveOnly && payloadTranslator->hasNewFields())
+                throw MakeStringException(0, "Translatable file layout mismatch reading file %s but translation disabled when expected fields are missing from source.", f->queryLogicalName());
             if (getLayoutTranslationMode()==RecordTranslationMode::None)
                 throw MakeStringException(0, "Translatable file layout mismatch reading file %s but translation disabled", f->queryLogicalName());
             VStringBuffer msg("Record layout translation required for %s", f->queryLogicalName());
@@ -4119,6 +4127,8 @@ class CHThorKeyedJoinActivity  : public CHThorThreadedActivityBase, implements I
             {
                 if (!trans->canTranslate())
                     throw MakeStringException(0, "Untranslatable key layout mismatch reading index %s", f->queryLogicalName());
+                if (getLayoutTranslationMode() == RecordTranslationMode::PayloadRemoveOnly && trans->hasNewFields())
+                    throw MakeStringException(0, "Translatable file layout mismatch reading file %s but translation disabled when expected fields are missing from source.", f->queryLogicalName());
             }
             else
             {
@@ -4149,6 +4159,8 @@ class CHThorKeyedJoinActivity  : public CHThorThreadedActivityBase, implements I
                     translator.setown(createRecordTranslator(helper.queryProjectedDiskRecordSize()->queryRecordAccessor(true), actualDiskMeta->queryRecordAccessor(true)));
                     if (translator->canTranslate())
                     {
+                        if (getLayoutTranslationMode()==RecordTranslationMode::PayloadRemoveOnly && translator->hasNewFields())
+                            throw MakeStringException(0, "Translatable file layout mismatch reading file %s but translation disabled when expected fields are missing from source.", f->queryLogicalName());
                         if (getLayoutTranslationMode()==RecordTranslationMode::None)
                             throw MakeStringException(0, "Translatable file layout mismatch reading file %s but translation disabled", f->queryLogicalName());
                         VStringBuffer msg("Record layout translation required for %s", f->queryLogicalName());

helm/hpcc/values.schema.json

@@ -1825,8 +1825,8 @@
           "description": "Should the index node memory allocation flush the cache and retry if memory allocation fails"
         },
         "fieldTranslationEnabled": {
-          "default": "payload",
-          "enum": ["false", "true", "payload"],
+          "default": "payloadRemoveOnly",
+          "enum": ["false", "true", "payload", "payloadRemoveOnly"],
           "description": "Enables translation (where possible) of mismatched file layouts on-the-fly. Specify 'payload' to attempt to translate payload fields only"
         },
         "highTimeout": { 

initfiles/componentfiles/configschema/xsd/eclagent.xsd

@@ -57,13 +57,14 @@
                                   hpcc:presetValue="false"
                                   hpcc:tooltip="eclAgent analyzes workunit post-execution"/>
                     <xs:attribute name="fieldTranslationEnabled" hpcc:displayName="Enable Field Translation"
-                                  hpcc:presetValue="payload"
+                                  hpcc:presetValue="payloadRemoveOnly"
                                   hpcc:tooltip="Enables translation (where possible) of mismatched index layouts on-the-fly. Specify 'payload' to attempt to translate payload fields only">
                         <xs:simpleType>
                             <xs:restriction base="xs:string">
                                 <xs:enumeration value="false" hpcc:displayName="False" hpcc:desciption=""/>
-                                <xs:enumeration value="true" hpcc:displayName="true" hpcc:description=""/>
+                                <xs:enumeration value="true" hpcc:displayName="True" hpcc:description=""/>
                                 <xs:enumeration value="payload" hpcc:displayName="Payload" hpcc:description=""/>
+                                <xs:enumeration value="payloadRemoveOnly" hpcc:displayName="PayloadRemoveOnly" hpcc:description=""/>
                             </xs:restriction>
                         </xs:simpleType>
                     </xs:attribute>

initfiles/componentfiles/configschema/xsd/roxie.xsd

@@ -110,13 +110,14 @@
                                   hpcc:displayName="flushJHtreeCacheOnOOM" hpcc:presetValue="true"
                                   hpcc:tooltip="Should the index node memory allocation flush the cache and retry if memory allocation fails"/>
                     <xs:attribute name="fieldTranslationEnabled" hpcc:displayName="Enable Field Translation"
-                                  hpcc:presetValue="payload"
+                                  hpcc:presetValue="payloadRemoveOnly"
                                   hpcc:tooltip="Enables translation (where possible) of mismatched index layouts on-the-fly. Specify 'payload' to attempt to translate payload fields only">
                         <xs:simpleType>
                             <xs:restriction base="xs:string">
                                 <xs:enumeration value="false" hpcc:displayName="False" hpcc:desciption=""/>
-                                <xs:enumeration value="true" hpcc:displayName="true" hpcc:description=""/>
+                                <xs:enumeration value="true" hpcc:displayName="True" hpcc:description=""/>
                                 <xs:enumeration value="payload" hpcc:displayName="Payload" hpcc:description=""/>
+                                <xs:enumeration value="payloadRemoveOnly" hpcc:displayName="PayloadRemoveOnly" hpcc:description=""/>
                             </xs:restriction>
                         </xs:simpleType>
                     </xs:attribute>

initfiles/componentfiles/configschema/xsd/thor.xsd

@@ -224,13 +224,14 @@
                 <xs:attribute name="httpGlobalIdHeader" type="xs:string" hpcc:displayName="HTTP GlobalId Header" hpcc:presetValue="HPCC-Global-Id"
                               hpcc:tooltip="HTTP Header field to use for sending and receiving GlobalId"/>
                 <xs:attribute name="fieldTranslationEnabled" hpcc:displayName="Enable Field Translation"
-                              hpcc:presetValue="payload"
+                              hpcc:presetValue="payloadRemoveOnly"
                               hpcc:tooltip="Enables translation (where possible) of mismatched index layouts on-the-fly. Specify 'payload' to attempt to translate payload fields only">
                     <xs:simpleType>
                         <xs:restriction base="xs:string">
                             <xs:enumeration value="false" hpcc:displayName="False" hpcc:desciption=""/>
-                            <xs:enumeration value="true" hpcc:displayName="true" hpcc:description=""/>
+                            <xs:enumeration value="true" hpcc:displayName="True" hpcc:description=""/>
                             <xs:enumeration value="payload" hpcc:displayName="Payload" hpcc:description=""/>
+                            <xs:enumeration value="payloadRemoveOnly" hpcc:displayName="PayloadRemoveOnly" hpcc:description=""/>
                         </xs:restriction>
                     </xs:simpleType>
                 </xs:attribute>

initfiles/componentfiles/configxml/eclagent_config.xsd.in

@@ -190,7 +190,7 @@
       </xs:annotation>
     </xs:attribute>
 
-    <xs:attribute name="fieldTranslationEnabled" use="optional" default="payload">
+    <xs:attribute name="fieldTranslationEnabled" use="optional" default="payloadRemoveOnly">
       <xs:annotation>
         <xs:appinfo>
           <tooltip>Enables translation (where possible) of mismatched file layouts on-the-fly. Specify 'payload' to attempt to translate payload fields only</tooltip>
@@ -201,6 +201,7 @@
           <xs:enumeration value="false"/>
           <xs:enumeration value="true"/>
           <xs:enumeration value="payload"/>
+          <xs:enumeration value="payloadRemoveOnly"/>
         </xs:restriction>
       </xs:simpleType>
     </xs:attribute>

initfiles/componentfiles/configxml/roxie.xsd.in

@@ -638,7 +638,7 @@
         </xs:appinfo>
       </xs:annotation>
     </xs:attribute>
-    <xs:attribute name="fieldTranslationEnabled" use="optional" default="payload">
+    <xs:attribute name="fieldTranslationEnabled" use="optional" default="payloadRemoveOnly">
       <xs:annotation>
         <xs:appinfo>
           <tooltip>Enables translation (where possible) of mismatched file layouts on-the-fly. Specify 'payload' to attempt to translate payload fields only</tooltip>
@@ -649,6 +649,7 @@
           <xs:enumeration value="false"/>
           <xs:enumeration value="true"/>
           <xs:enumeration value="payload"/>
+          <xs:enumeration value="payloadRemoveOnly"/>
         </xs:restriction>
       </xs:simpleType>
     </xs:attribute>

initfiles/componentfiles/configxml/thor.xsd.in

@@ -677,7 +677,7 @@
         </xs:appinfo>
       </xs:annotation>
     </xs:attribute>
-    <xs:attribute name="fieldTranslationEnabled" use="optional" default="payload">
+    <xs:attribute name="fieldTranslationEnabled" use="optional" default="payloadRemoveOnly">
       <xs:annotation>
         <xs:appinfo>
           <tooltip>Enables translation (where possible) of mismatched file layouts on-the-fly. Specify 'payload' to attempt to translate payload fields only</tooltip>
@@ -688,6 +688,7 @@
           <xs:enumeration value="false"/>
           <xs:enumeration value="true"/>
           <xs:enumeration value="payload"/>
+          <xs:enumeration value="payloadRemoveOnly"/>
         </xs:restriction>
       </xs:simpleType>
     </xs:attribute>

initfiles/etc/DIR_NAME/environment.xml.in

@@ -68,7 +68,7 @@
                 enableKeyDiff="true"
                 enableSysLog="true"
                 fastLaneQueue="true"
-                fieldTranslationEnabled="payload"
+                fieldTranslationEnabled="payloadRemoveOnly"
                 flushJHtreeCacheOnOOM="true"
                 forceStdLog="false"
                 heapRetainMemory="false"

roxie/ccd/ccdfile.cpp

@@ -3013,6 +3013,8 @@ class CResolvedFile : implements IResolvedFileCreator, implements ISafeSDSSubscr
                             }
                             if (!translator || !translator->canTranslate())
                                 throw MakeStringException(ROXIE_MISMATCH, "Untranslatable record layout mismatch detected for file %s", subname);
+                            else if (mode == RecordTranslationMode::PayloadRemoveOnly && translator->hasNewFields())
+                                throw MakeStringException(0, "Translatable file layout mismatch reading file %s but translation disabled when expected fields are missing from source.", subname);
                             else if (translator->needsTranslate())
                             {
                                 if (fileMode==FileFormatMode::index && translator->keyedTranslated())

roxie/ccd/ccdmain.cpp

@@ -103,7 +103,7 @@ unsigned defaultTraceLimit = 10;
 unsigned watchActivityId = 0;
 unsigned testAgentFailure = 0;
 RelaxedAtomic<unsigned> restarts;
-RecordTranslationMode fieldTranslationEnabled = RecordTranslationMode::Payload;
+RecordTranslationMode fieldTranslationEnabled = RecordTranslationMode::PayloadRemoveOnly;
 bool mergeAgentStatistics = true;
 PTreeReaderOptions defaultXmlReadFlags = ptr_ignoreWhiteSpace;
 bool runOnce = false;
@@ -1181,7 +1181,7 @@ int CCD_API roxie_main(int argc, const char *argv[], const char * defaultYaml)
         agentQueryReleaseDelaySeconds = topology->getPropInt("@agentQueryReleaseDelaySeconds", topology->getPropInt("@slaveQueryReleaseDelaySeconds", 60));  // legacy name
         coresPerQuery = topology->getPropInt("@coresPerQuery", 0);
 
-        fieldTranslationEnabled = RecordTranslationMode::Payload;
+        fieldTranslationEnabled = RecordTranslationMode::PayloadRemoveOnly;
         const char *val = topology->queryProp("@fieldTranslationEnabled");
         if (val)
             fieldTranslationEnabled = getTranslationMode(val, false);

roxie/ccd/ccdstate.cpp

@@ -2463,7 +2463,7 @@ class CRoxiePackageSetManager : implements IRoxieQueryPackageManagerSet, impleme
                 if (val)
                     fieldTranslationEnabled = getTranslationMode(val, false);
                 else
-                    fieldTranslationEnabled = RecordTranslationMode::Payload;
+                    fieldTranslationEnabled = RecordTranslationMode::PayloadRemoveOnly;
                 val = getTranslationModeText(fieldTranslationEnabled);
                 topology->setProp("@fieldTranslationEnabled", val);
             }

rtl/eclrtl/rtldynfield.cpp

@@ -36,7 +36,9 @@
 
 extern ECLRTL_API RecordTranslationMode getTranslationMode(const char *val, bool isLocal)
 {
-    if (isEmptyString(val) || strToBool(val) || strieq(val, "payload"))
+    if (isEmptyString(val) || strToBool(val) || strieq(val, "payloadRemoveOnly"))
+        return RecordTranslationMode::PayloadRemoveOnly;
+    else if (strieq(val, "payload"))
         return RecordTranslationMode::Payload;
     else if (strieq(val, "alwaysDisk") || strieq(val, "disk"))
     {
@@ -60,6 +62,7 @@ extern ECLRTL_API const char *getTranslationModeText(RecordTranslationMode val)
     case RecordTranslationMode::AlwaysDisk: return "alwaysDisk";
     case RecordTranslationMode::AlwaysECL: return "alwaysECL";
     case RecordTranslationMode::Payload: return "payload";
+    case RecordTranslationMode::PayloadRemoveOnly: return "payloadRemoveOnly";
     case RecordTranslationMode::None: return "off";
     }
     throwUnexpected();
@@ -1085,6 +1088,10 @@ class GeneralRecordTranslator : public CInterfaceOf<IDynamicTransform>
     {
         return (matchFlags & match_keychange) != 0;
     }
+    virtual bool hasNewFields() const override
+    {
+        return (matchFlags & match_none) != 0;
+    }
 private:
     void doDescribe(unsigned indent) const
     {
@@ -1948,6 +1955,10 @@ class CloneVirtualRecordTranslator : public CInterfaceOf<IDynamicTransform>
     {
         return false;
     }
+    virtual bool hasNewFields() const override
+    {
+        return false;
+    }
 private:
     void doDescribe(unsigned indent) const
     {

rtl/eclrtl/rtldynfield.hpp

@@ -110,7 +110,8 @@ enum class RecordTranslationMode : byte
     Payload = 2,    // Translate all fields in datasets, and only payload fields in indexes
     AlwaysDisk = 3, // Always translate - even if wouldn't normally (e.g. csv/xml source read as binary), or crcs happen to match
     AlwaysECL = 4,  // Ignore the published format - can make sense to force no translation e.g. when field names have changed
-    Unspecified = 5
+    PayloadRemoveOnly = 5, // Allow fields to be removed from the incoming dataset, but not allow fields to be missing
+    Unspecified = 6
 };  // AlwaysDisk and AlwaysECL are for testing purposes only, and can only be set per file (not globally)
 
 extern ECLRTL_API RecordTranslationMode getTranslationMode(const char *modeStr, bool isLocal);
@@ -139,6 +140,7 @@ interface IDynamicTransform : public IInterface
     virtual bool needsTranslate() const = 0;
     virtual bool keyedTranslated() const = 0;
     virtual bool needsNonVirtualTranslate() const = 0;
+    virtual bool hasNewFields() const = 0;
 };
 
 interface IKeyTranslator : public IInterface

testing/regress/ecl/layouttrans.ecl

@@ -26,7 +26,7 @@ version := #IFDEFINED(root.version, 1);
 
 //--- end of version configuration ---
 
-#option ('layoutTranslation', true);
+#option ('layoutTranslation', 'payload');
 #onwarning (4515, ignore);
 #onwarning (4522, ignore);
 #onwarning (4523, ignore);

testing/regress/ecl/sqtrans.ecl

@@ -22,6 +22,8 @@ multiPart := #IFDEFINED(root.multiPart, false);
 
 //--- end of version configuration ---
 
+#option('layoutTranslation', 'payload');
+
 //check that compound disk read activities workcorrectly when the input file is translated
 //include keyed filters to ensure they are also translated correctly
 

testing/regress/ecl/translatedisk.ecl

@@ -27,7 +27,7 @@ optRemoteRead := #IFDEFINED(root.optRemoteRead, true);
 
 #onwarning(2036, ignore);
 #onwarning(4522, ignore);
-#option ('layoutTranslation', true);
+#option ('layoutTranslation', 'payload');
 #option('forceRemoteRead', optRemoteRead);
 import $.Setup;
 

testing/regress/environment.xml.in

@@ -65,7 +65,7 @@
                 enableKeyDiff="true"
                 enableSysLog="true"
                 fastLaneQueue="true"
-                fieldTranslationEnabled="payload"
+                fieldTranslationEnabled="payloadRemoveOnly"
                 flushJHtreeCacheOnOOM="true"
                 forceStdLog="false"
                 heapRetainMemory="false"