Merge remote-tracking branch 'origin/candidate-9.6.x'

Signed-off-by: Jake Smith <[email protected]>
hpcc-systems · May 2, 2024 · 38a368a · 38a368a
2 parents 73bfe4c + 4be19cb
commit 38a368a
Show file tree

Hide file tree

Showing 18 changed files with 99 additions and 57 deletions.
diff --git a/common/pkgfiles/referencedfilelist.cpp b/common/pkgfiles/referencedfilelist.cpp
@@ -228,7 +228,7 @@ class ReferencedFileList : implements IReferencedFileList, public CInterface
     ReferencedFileList(const char *username, const char *pw, bool allowForeignFiles, bool allowFileSizeCalc, const char *_jobname)
         : jobName(_jobname), allowForeign(allowForeignFiles), allowSizeCalc(allowFileSizeCalc)
     {
-        if (username && pw)
+        if (username)
         {
             user.setown(createUserDescriptor());
             user->set(username, pw);

diff --git a/common/thorhelper/thorcommon.cpp b/common/thorhelper/thorcommon.cpp
@@ -2148,6 +2148,9 @@ static bool getTranslators(Owned<const IDynamicTransform> &translator, Owned<con
             if (!translator->canTranslate())
                 throw MakeStringException(0, "Untranslatable record layout mismatch detected for file %s", tracing);
 
+            if (mode == RecordTranslationMode::PayloadRemoveOnly && translator->hasNewFields())
+                throw MakeStringException(0, "Translatable file layout mismatch reading file %s but translation disabled when expected fields are missing from source.", tracing);
+
             if (translator->needsTranslate())
             {
                 if (keyedTranslator && (sourceFormat != expectedFormat))

diff --git a/common/thorhelper/thorread.cpp b/common/thorhelper/thorread.cpp
@@ -176,6 +176,9 @@ void DiskReadMapping::ensureTranslators() const
         if (!translator->canTranslate())
             throw MakeStringException(0, "Untranslatable record layout mismatch detected for file %s", filename);
 
+        if (mode == RecordTranslationMode::PayloadRemoveOnly && translator->hasNewFields())
+            throw MakeStringException(0, "Translatable file layout mismatch reading file %s but translation disabled when expected fields are missing from source.", filename);
+
         if (translator->needsTranslate())
         {
             if (sourceMeta != expectedMeta)

diff --git a/ecl/hthor/hthorkey.cpp b/ecl/hthor/hthorkey.cpp
@@ -700,6 +700,8 @@ const IDynamicTransform * CHThorIndexReadActivityBase::getLayoutTranslator(IDist
     Owned<const IDynamicTransform> payloadTranslator =  createRecordTranslator(projectedFormat->queryRecordAccessor(true), actualFormat->queryRecordAccessor(true));
     if (!payloadTranslator->canTranslate())
         throw MakeStringException(0, "Untranslatable key layout mismatch reading index %s", f->queryLogicalName());
+    if (getLayoutTranslationMode() == RecordTranslationMode::PayloadRemoveOnly && payloadTranslator->hasNewFields())
+        throw MakeStringException(0, "Translatable file layout mismatch reading file %s but translation disabled when expected fields are missing from source.", f->queryLogicalName());
     if (payloadTranslator->needsTranslate())
         return payloadTranslator.getClear();
     return nullptr;
@@ -715,6 +717,8 @@ void CHThorIndexReadActivityBase::verifyIndex(IKeyIndex * idx)
         {
             if (!layoutTrans->canTranslate())
                 throw MakeStringException(0, "Untranslatable key layout mismatch reading index %s", df->queryLogicalName());
+            if (getLayoutTranslationMode() == RecordTranslationMode::PayloadRemoveOnly && layoutTrans->hasNewFields())
+                throw MakeStringException(0, "Translatable file layout mismatch reading file %s but translation disabled when expected fields are missing from source.", df->queryLogicalName());
         }
         else
         {
@@ -2462,6 +2466,8 @@ class CHThorFlatFetchActivity : public CHThorFetchActivityBase, public IFlatFetc
                     translator->describe();
                     if (translator->canTranslate())
                     {
+                        if (getLayoutTranslationMode()==RecordTranslationMode::PayloadRemoveOnly && translator->hasNewFields())
+                            throw MakeStringException(0, "Translatable file layout mismatch reading file %s but translation disabled when expected fields are missing from source.", f->queryLogicalName());
                         if (getLayoutTranslationMode()==RecordTranslationMode::None)
                             throw MakeStringException(0, "Translatable file layout mismatch reading file %s but translation disabled", f->queryLogicalName());
                         VStringBuffer msg("Record layout translation required for %s", f->queryLogicalName());
@@ -4102,6 +4108,8 @@ class CHThorKeyedJoinActivity  : public CHThorThreadedActivityBase, implements I
                 throw MakeStringException(0, "Untranslatable key layout mismatch reading index %s", f->queryLogicalName());
             if (payloadTranslator->keyedTranslated())
                 throw MakeStringException(0, "Untranslatable key layout mismatch reading index %s - keyed fields do not match", f->queryLogicalName());
+            if (getLayoutTranslationMode()==RecordTranslationMode::PayloadRemoveOnly && payloadTranslator->hasNewFields())
+                throw MakeStringException(0, "Translatable file layout mismatch reading file %s but translation disabled when expected fields are missing from source.", f->queryLogicalName());
             if (getLayoutTranslationMode()==RecordTranslationMode::None)
                 throw MakeStringException(0, "Translatable file layout mismatch reading file %s but translation disabled", f->queryLogicalName());
             VStringBuffer msg("Record layout translation required for %s", f->queryLogicalName());
@@ -4119,6 +4127,8 @@ class CHThorKeyedJoinActivity  : public CHThorThreadedActivityBase, implements I
             {
                 if (!trans->canTranslate())
                     throw MakeStringException(0, "Untranslatable key layout mismatch reading index %s", f->queryLogicalName());
+                if (getLayoutTranslationMode() == RecordTranslationMode::PayloadRemoveOnly && trans->hasNewFields())
+                    throw MakeStringException(0, "Translatable file layout mismatch reading file %s but translation disabled when expected fields are missing from source.", f->queryLogicalName());
             }
             else
             {
@@ -4149,6 +4159,8 @@ class CHThorKeyedJoinActivity  : public CHThorThreadedActivityBase, implements I
                     translator.setown(createRecordTranslator(helper.queryProjectedDiskRecordSize()->queryRecordAccessor(true), actualDiskMeta->queryRecordAccessor(true)));
                     if (translator->canTranslate())
                     {
+                        if (getLayoutTranslationMode()==RecordTranslationMode::PayloadRemoveOnly && translator->hasNewFields())
+                            throw MakeStringException(0, "Translatable file layout mismatch reading file %s but translation disabled when expected fields are missing from source.", f->queryLogicalName());
                         if (getLayoutTranslationMode()==RecordTranslationMode::None)
                             throw MakeStringException(0, "Translatable file layout mismatch reading file %s but translation disabled", f->queryLogicalName());
                         VStringBuffer msg("Record layout translation required for %s", f->queryLogicalName());

diff --git a/ecllibrary/std/DataPatterns/Profile.ecl b/ecllibrary/std/DataPatterns/Profile.ecl
@@ -617,7 +617,7 @@ EXPORT Profile(inFile,
         #UNIQUENAME(_MapUpperCharUni);
         LOCAL %_MapUpperCharUni%(UNICODE s) := REGEXREPLACE(u'\\p{Lu}', s, u'A');
         #UNIQUENAME(_MapLowerCharUni);
-        LOCAL %_MapLowerCharUni%(UNICODE s) := REGEXREPLACE(u'[[\\p{Ll}][\\p{Lt}][\\p{Lm}][\\p{Lo}]]', s, u'a');
+        LOCAL %_MapLowerCharUni%(UNICODE s) := REGEXREPLACE(u'[\\p{Ll}\\p{Lt}\\p{Lm}\\p{Lo}]', s, u'a');
         #UNIQUENAME(_MapDigitUni);
         LOCAL %_MapDigitUni%(UNICODE s) := REGEXREPLACE(u'[1-9]', s, u'9'); // Leave '0' as-is and replace with '9' later
         #UNIQUENAME(_MapAllUni);

diff --git a/esp/services/ws_dali/ws_daliservice.cpp b/esp/services/ws_dali/ws_daliservice.cpp
@@ -718,7 +718,7 @@ bool CWSDaliEx::onSetTraceSlowTransactions(IEspContext& context, IEspSetTraceSlo
 
         StringAttr result;
         mb.read(result);
-        resp.setResult(result.isEmpty() ? "SetTraceSlowTransactions called." : result);
+        resp.setResult(result.isEmpty() ? "SetTraceSlowTransactions called." : result.get());
     }
     catch(IException* e)
     {

diff --git a/esp/services/ws_packageprocess/ws_packageprocessService.cpp b/esp/services/ws_packageprocess/ws_packageprocessService.cpp
@@ -319,7 +319,7 @@ class PackageMapUpdater
     }
     void setUser(const char *user, const char *password, IEspContext *context)
     {
-        if (user && *user && password && *password)
+        if (user && *user)
         {
             userdesc.setown(createUserDescriptor());
             userdesc->set(user, password);

diff --git a/esp/src/eclwatch/_Widget.js b/esp/src/eclwatch/_Widget.js
@@ -130,22 +130,22 @@ define([
             return retVal;
         },
 
-        getURL: function () {
+        getURL: function (componentFilterParams) {
             var baseUrl = document.URL.split("#")[0].replace("index.html", "stub.htm");
             var baseUrlParts = baseUrl.split("?");
             baseUrl = baseUrlParts[0];
             var args = baseUrlParts[1];
             delete this.params.__filter;
-            var filterParams = this.getFilterParams();
+            var filterParams = componentFilterParams ? componentFilterParams : this.getFilterParams();
             if (filterParams) {
                 this.params.__filter = ioQuery.objectToQuery(filterParams);
             }
             var paramsString = ioQuery.objectToQuery(this.params);
             return baseUrl + "?" + paramsString;
         },
 
-        _onNewPage: function (event) {
-            var win = window.open(this.getURL(), "_blank");
+        _onNewPage: function (_event, componentFilterParams) {
+            var win = window.open(this.getURL(componentFilterParams), "_blank");
             if (win) {
                 win.focus();
             }

diff --git a/esp/src/package-lock.json b/esp/src/package-lock.json
diff --git a/esp/src/src-react/components/Security.tsx b/esp/src/src-react/components/Security.tsx
@@ -28,6 +28,20 @@ export const Security: React.FunctionComponent<SecurityProps> = ({
 
     const [, { opsCategory }] = useBuildInfo();
 
+    const [permissionTabTitle, setPermissionTabTitle] = React.useState(nlsHPCC.Permissions);
+
+    React.useEffect(() => {
+        setPermissionTabTitle(nlsHPCC.Permissions);
+        if (name === "_") {
+            if (baseDn === "File Scopes") setPermissionTabTitle(nlsHPCC.FileScopeDefaultPermissions);
+            else if (baseDn === "Workunit Scopes") setPermissionTabTitle(nlsHPCC.WorkUnitScopeDefaultPermissions);
+        } else if (name === "file") {
+            if (baseDn === "File Scopes") setPermissionTabTitle(nlsHPCC.PhysicalFiles);
+        } else if (name) {
+            setPermissionTabTitle(name);
+        }
+    }, [name, baseDn]);
+
     return <>
         <SizeMe monitorHeight>{({ size }) =>
             <Pivot
@@ -40,7 +54,7 @@ export const Security: React.FunctionComponent<SecurityProps> = ({
                 <PivotItem headerText={nlsHPCC.Groups} itemKey="groups" style={pivotItemStyle(size)}>
                     <Groups page={page} />
                 </PivotItem>
-                <PivotItem headerText={nlsHPCC.Permissions} itemKey="permissions" style={pivotItemStyle(size)}>
+                <PivotItem headerText={permissionTabTitle} itemKey="permissions" style={pivotItemStyle(size)}>
                     {!name && !baseDn &&
                         <Permissions />
                     }

diff --git a/esp/src/src-react/components/forms/ZAPDialog.tsx b/esp/src/src-react/components/forms/ZAPDialog.tsx
@@ -376,30 +376,34 @@ export const ZAPDialog: React.FunctionComponent<ZAPDialogProps> = ({
                 />}
         />
         <div style={{ padding: "15px 0 7px 0" }}>
-            <div>
-                <Controller
-                    control={control} name="IncludeThorSlaveLog"
-                    render={({
-                        field: { onChange, name: fieldName, value }
-                    }) => <Checkbox name={fieldName} checked={value} onChange={onChange} label={nlsHPCC.IncludeSlaveLogs} />}
-                />
-            </div>
-            <div style={{ marginTop: 2, marginBottom: 2 }}>
-                <Controller
-                    control={control} name="IncludeRelatedLogs"
-                    render={({
-                        field: { onChange, name: fieldName, value }
-                    }) => <Checkbox name={fieldName} checked={value} onChange={onChange} label={nlsHPCC.IncludeRelatedLogs} />}
-                />
-            </div>
-            <div>
-                <Controller
-                    control={control} name="IncludePerComponentLogs"
-                    render={({
-                        field: { onChange, name: fieldName, value }
-                    }) => <Checkbox name={fieldName} checked={value} onChange={onChange} label={nlsHPCC.IncludePerComponentLogs} />}
-                />
-            </div>
+            {!isContainer
+                ? <div>
+                    <Controller
+                        control={control} name="IncludeThorSlaveLog"
+                        render={({
+                            field: { onChange, name: fieldName, value }
+                        }) => <Checkbox name={fieldName} checked={value} onChange={onChange} label={nlsHPCC.IncludeSlaveLogs} />}
+                    />
+                </div>
+                : <div>
+                    <div style={{ marginBottom: 4 }}>
+                        <Controller
+                            control={control} name="IncludeRelatedLogs"
+                            render={({
+                                field: { onChange, name: fieldName, value }
+                            }) => <Checkbox name={fieldName} checked={value} onChange={onChange} label={nlsHPCC.IncludeRelatedLogs} />}
+                        />
+                    </div>
+                    <div>
+                        <Controller
+                            control={control} name="IncludePerComponentLogs"
+                            render={({
+                                field: { onChange, name: fieldName, value }
+                            }) => <Checkbox name={fieldName} checked={value} onChange={onChange} label={nlsHPCC.IncludePerComponentLogs} />}
+                        />
+                    </div>
+                </div>
+            }
             <div style={{ paddingTop: "10px" }}>
                 <Controller
                     control={control} name="SendEmail"

diff --git a/esp/src/src/react/recentFilters.tsx b/esp/src/src/react/recentFilters.tsx
@@ -37,7 +37,7 @@ export const RecentFilters: React.FunctionComponent<RecentFilterProps> = ({
 
     const handleClick = (e) => {
         const tempObj = JSON.parse(decodeURIComponent(e.currentTarget.value));
-        widget.NewPage.onClick(tempObj);
+        widget.NewPage.onClick(e, tempObj);
     };
 
     const shimmerElements = React.useMemo(() => [

diff --git a/helm/hpcc/templates/_helpers.tpl b/helm/hpcc/templates/_helpers.tpl
@@ -1339,7 +1339,7 @@ Pass in dict with root, me and instances defined
 {{- $stubInstanceResources := .stubResources | default .root.Values.global.stubInstanceResources | default dict }}
 {{- $milliCPUText := $stubInstanceResources.cpu | default "200m" }}
 {{- $milliCPUs := int (include "hpcc.k8sCPUStringToMilliCPU" $milliCPUText) }}
-{{- $memoryText := $stubInstanceResources.memory | default "50Mi" }}
+{{- $memoryText := $stubInstanceResources.memory | default "400Mi" }}
 {{- $memory := int64 (include "hpcc.k8sMemoryStringToBytes" $memoryText) }}
 resources:
   limits:

diff --git a/plugins/parquet/parquetembed.cpp b/plugins/parquet/parquetembed.cpp
@@ -761,24 +761,13 @@ arrow::Status ParquetWriter::fieldToNode(const std::string &name, const RtlField
     case type_real:
         arrowFields.push_back(std::make_shared<arrow::Field>(name, arrow::float64()));
         break;
-    case type_string:
-        arrowFields.push_back(std::make_shared<arrow::Field>(name, arrow::utf8()));
-        break;
     case type_char:
-        arrowFields.push_back(std::make_shared<arrow::Field>(name, arrow::utf8()));
-        break;
+    case type_string:
     case type_varstring:
-        arrowFields.push_back(std::make_shared<arrow::Field>(name, arrow::utf8()));
-        break;
     case type_qstring:
-        arrowFields.push_back(std::make_shared<arrow::Field>(name, arrow::utf8()));
-        break;
-    case type_unicode:
-        arrowFields.push_back(std::make_shared<arrow::Field>(name, arrow::utf8()));
-        break;
     case type_utf8:
-        arrowFields.push_back(std::make_shared<arrow::Field>(name, arrow::utf8()));
-        break;
+    case type_unicode:
+    case type_varunicode:
     case type_decimal:
         arrowFields.push_back(std::make_shared<arrow::Field>(name, arrow::utf8())); //TODO add decimal encoding
         break;

diff --git a/roxie/ccd/ccdfile.cpp b/roxie/ccd/ccdfile.cpp
@@ -3013,6 +3013,8 @@ class CResolvedFile : implements IResolvedFileCreator, implements ISafeSDSSubscr
                             }
                             if (!translator || !translator->canTranslate())
                                 throw MakeStringException(ROXIE_MISMATCH, "Untranslatable record layout mismatch detected for file %s", subname);
+                            else if (mode == RecordTranslationMode::PayloadRemoveOnly && translator->hasNewFields())
+                                throw MakeStringException(0, "Translatable file layout mismatch reading file %s but translation disabled when expected fields are missing from source.", subname);
                             else if (translator->needsTranslate())
                             {
                                 if (fileMode==FileFormatMode::index && translator->keyedTranslated())

diff --git a/rtl/eclrtl/rtldynfield.cpp b/rtl/eclrtl/rtldynfield.cpp
@@ -38,6 +38,8 @@ extern ECLRTL_API RecordTranslationMode getTranslationMode(const char *val, bool
 {
     if (isEmptyString(val) || strToBool(val) || strieq(val, "payload"))
         return RecordTranslationMode::Payload;
+    else if (strieq(val, "payloadRemoveOnly"))
+        return RecordTranslationMode::PayloadRemoveOnly;
     else if (strieq(val, "alwaysDisk") || strieq(val, "disk"))
     {
         if (!isLocal)
@@ -60,6 +62,7 @@ extern ECLRTL_API const char *getTranslationModeText(RecordTranslationMode val)
     case RecordTranslationMode::AlwaysDisk: return "alwaysDisk";
     case RecordTranslationMode::AlwaysECL: return "alwaysECL";
     case RecordTranslationMode::Payload: return "payload";
+    case RecordTranslationMode::PayloadRemoveOnly: return "payloadRemoveOnly";
     case RecordTranslationMode::None: return "off";
     }
     throwUnexpected();
@@ -1085,6 +1088,10 @@ class GeneralRecordTranslator : public CInterfaceOf<IDynamicTransform>
     {
         return (matchFlags & match_keychange) != 0;
     }
+    virtual bool hasNewFields() const override
+    {
+        return (matchFlags & match_none) != 0;
+    }
 private:
     void doDescribe(unsigned indent) const
     {
@@ -1948,6 +1955,10 @@ class CloneVirtualRecordTranslator : public CInterfaceOf<IDynamicTransform>
     {
         return false;
     }
+    virtual bool hasNewFields() const override
+    {
+        return false;
+    }
 private:
     void doDescribe(unsigned indent) const
     {

diff --git a/rtl/eclrtl/rtldynfield.hpp b/rtl/eclrtl/rtldynfield.hpp
@@ -110,7 +110,8 @@ enum class RecordTranslationMode : byte
     Payload = 2,    // Translate all fields in datasets, and only payload fields in indexes
     AlwaysDisk = 3, // Always translate - even if wouldn't normally (e.g. csv/xml source read as binary), or crcs happen to match
     AlwaysECL = 4,  // Ignore the published format - can make sense to force no translation e.g. when field names have changed
-    Unspecified = 5
+    PayloadRemoveOnly = 5, // Allow fields to be removed from the incoming dataset, but not allow fields to be missing
+    Unspecified = 6
 };  // AlwaysDisk and AlwaysECL are for testing purposes only, and can only be set per file (not globally)
 
 extern ECLRTL_API RecordTranslationMode getTranslationMode(const char *modeStr, bool isLocal);
@@ -139,6 +140,7 @@ interface IDynamicTransform : public IInterface
     virtual bool needsTranslate() const = 0;
     virtual bool keyedTranslated() const = 0;
     virtual bool needsNonVirtualTranslate() const = 0;
+    virtual bool hasNewFields() const = 0;
 };
 
 interface IKeyTranslator : public IInterface