diff --git a/common/pkgfiles/referencedfilelist.cpp b/common/pkgfiles/referencedfilelist.cpp index f7c1b0884bd..837a1ee1e4f 100644 --- a/common/pkgfiles/referencedfilelist.cpp +++ b/common/pkgfiles/referencedfilelist.cpp @@ -228,7 +228,7 @@ class ReferencedFileList : implements IReferencedFileList, public CInterface ReferencedFileList(const char *username, const char *pw, bool allowForeignFiles, bool allowFileSizeCalc, const char *_jobname) : jobName(_jobname), allowForeign(allowForeignFiles), allowSizeCalc(allowFileSizeCalc) { - if (username && pw) + if (username) { user.setown(createUserDescriptor()); user->set(username, pw); diff --git a/common/thorhelper/thorcommon.cpp b/common/thorhelper/thorcommon.cpp index 6c9668cfcb8..38f29328615 100644 --- a/common/thorhelper/thorcommon.cpp +++ b/common/thorhelper/thorcommon.cpp @@ -2148,6 +2148,9 @@ static bool getTranslators(Owned &translator, OwnedcanTranslate()) throw MakeStringException(0, "Untranslatable record layout mismatch detected for file %s", tracing); + if (mode == RecordTranslationMode::PayloadRemoveOnly && translator->hasNewFields()) + throw MakeStringException(0, "Translatable file layout mismatch reading file %s but translation disabled when expected fields are missing from source.", tracing); + if (translator->needsTranslate()) { if (keyedTranslator && (sourceFormat != expectedFormat)) diff --git a/common/thorhelper/thorread.cpp b/common/thorhelper/thorread.cpp index 591d5d553cc..cc8386ffd21 100644 --- a/common/thorhelper/thorread.cpp +++ b/common/thorhelper/thorread.cpp @@ -176,6 +176,9 @@ void DiskReadMapping::ensureTranslators() const if (!translator->canTranslate()) throw MakeStringException(0, "Untranslatable record layout mismatch detected for file %s", filename); + if (mode == RecordTranslationMode::PayloadRemoveOnly && translator->hasNewFields()) + throw MakeStringException(0, "Translatable file layout mismatch reading file %s but translation disabled when expected fields are missing from source.", filename); + if (translator->needsTranslate()) { if (sourceMeta != expectedMeta) diff --git a/ecl/hthor/hthorkey.cpp b/ecl/hthor/hthorkey.cpp index 7fb6c7952ef..1c9094a3a63 100644 --- a/ecl/hthor/hthorkey.cpp +++ b/ecl/hthor/hthorkey.cpp @@ -700,6 +700,8 @@ const IDynamicTransform * CHThorIndexReadActivityBase::getLayoutTranslator(IDist Owned payloadTranslator = createRecordTranslator(projectedFormat->queryRecordAccessor(true), actualFormat->queryRecordAccessor(true)); if (!payloadTranslator->canTranslate()) throw MakeStringException(0, "Untranslatable key layout mismatch reading index %s", f->queryLogicalName()); + if (getLayoutTranslationMode() == RecordTranslationMode::PayloadRemoveOnly && payloadTranslator->hasNewFields()) + throw MakeStringException(0, "Translatable file layout mismatch reading file %s but translation disabled when expected fields are missing from source.", f->queryLogicalName()); if (payloadTranslator->needsTranslate()) return payloadTranslator.getClear(); return nullptr; @@ -715,6 +717,8 @@ void CHThorIndexReadActivityBase::verifyIndex(IKeyIndex * idx) { if (!layoutTrans->canTranslate()) throw MakeStringException(0, "Untranslatable key layout mismatch reading index %s", df->queryLogicalName()); + if (getLayoutTranslationMode() == RecordTranslationMode::PayloadRemoveOnly && layoutTrans->hasNewFields()) + throw MakeStringException(0, "Translatable file layout mismatch reading file %s but translation disabled when expected fields are missing from source.", df->queryLogicalName()); } else { @@ -2462,6 +2466,8 @@ class CHThorFlatFetchActivity : public CHThorFetchActivityBase, public IFlatFetc translator->describe(); if (translator->canTranslate()) { + if (getLayoutTranslationMode()==RecordTranslationMode::PayloadRemoveOnly && translator->hasNewFields()) + throw MakeStringException(0, "Translatable file layout mismatch reading file %s but translation disabled when expected fields are missing from source.", f->queryLogicalName()); if (getLayoutTranslationMode()==RecordTranslationMode::None) throw MakeStringException(0, "Translatable file layout mismatch reading file %s but translation disabled", f->queryLogicalName()); VStringBuffer msg("Record layout translation required for %s", f->queryLogicalName()); @@ -4102,6 +4108,8 @@ class CHThorKeyedJoinActivity : public CHThorThreadedActivityBase, implements I throw MakeStringException(0, "Untranslatable key layout mismatch reading index %s", f->queryLogicalName()); if (payloadTranslator->keyedTranslated()) throw MakeStringException(0, "Untranslatable key layout mismatch reading index %s - keyed fields do not match", f->queryLogicalName()); + if (getLayoutTranslationMode()==RecordTranslationMode::PayloadRemoveOnly && payloadTranslator->hasNewFields()) + throw MakeStringException(0, "Translatable file layout mismatch reading file %s but translation disabled when expected fields are missing from source.", f->queryLogicalName()); if (getLayoutTranslationMode()==RecordTranslationMode::None) throw MakeStringException(0, "Translatable file layout mismatch reading file %s but translation disabled", f->queryLogicalName()); VStringBuffer msg("Record layout translation required for %s", f->queryLogicalName()); @@ -4119,6 +4127,8 @@ class CHThorKeyedJoinActivity : public CHThorThreadedActivityBase, implements I { if (!trans->canTranslate()) throw MakeStringException(0, "Untranslatable key layout mismatch reading index %s", f->queryLogicalName()); + if (getLayoutTranslationMode() == RecordTranslationMode::PayloadRemoveOnly && trans->hasNewFields()) + throw MakeStringException(0, "Translatable file layout mismatch reading file %s but translation disabled when expected fields are missing from source.", f->queryLogicalName()); } else { @@ -4149,6 +4159,8 @@ class CHThorKeyedJoinActivity : public CHThorThreadedActivityBase, implements I translator.setown(createRecordTranslator(helper.queryProjectedDiskRecordSize()->queryRecordAccessor(true), actualDiskMeta->queryRecordAccessor(true))); if (translator->canTranslate()) { + if (getLayoutTranslationMode()==RecordTranslationMode::PayloadRemoveOnly && translator->hasNewFields()) + throw MakeStringException(0, "Translatable file layout mismatch reading file %s but translation disabled when expected fields are missing from source.", f->queryLogicalName()); if (getLayoutTranslationMode()==RecordTranslationMode::None) throw MakeStringException(0, "Translatable file layout mismatch reading file %s but translation disabled", f->queryLogicalName()); VStringBuffer msg("Record layout translation required for %s", f->queryLogicalName()); diff --git a/ecllibrary/std/DataPatterns/Profile.ecl b/ecllibrary/std/DataPatterns/Profile.ecl index 65643ed6317..877d1e085af 100644 --- a/ecllibrary/std/DataPatterns/Profile.ecl +++ b/ecllibrary/std/DataPatterns/Profile.ecl @@ -617,7 +617,7 @@ EXPORT Profile(inFile, #UNIQUENAME(_MapUpperCharUni); LOCAL %_MapUpperCharUni%(UNICODE s) := REGEXREPLACE(u'\\p{Lu}', s, u'A'); #UNIQUENAME(_MapLowerCharUni); - LOCAL %_MapLowerCharUni%(UNICODE s) := REGEXREPLACE(u'[[\\p{Ll}][\\p{Lt}][\\p{Lm}][\\p{Lo}]]', s, u'a'); + LOCAL %_MapLowerCharUni%(UNICODE s) := REGEXREPLACE(u'[\\p{Ll}\\p{Lt}\\p{Lm}\\p{Lo}]', s, u'a'); #UNIQUENAME(_MapDigitUni); LOCAL %_MapDigitUni%(UNICODE s) := REGEXREPLACE(u'[1-9]', s, u'9'); // Leave '0' as-is and replace with '9' later #UNIQUENAME(_MapAllUni); diff --git a/esp/services/ws_dali/ws_daliservice.cpp b/esp/services/ws_dali/ws_daliservice.cpp index 5382963ea1f..22bf64346bc 100644 --- a/esp/services/ws_dali/ws_daliservice.cpp +++ b/esp/services/ws_dali/ws_daliservice.cpp @@ -718,7 +718,7 @@ bool CWSDaliEx::onSetTraceSlowTransactions(IEspContext& context, IEspSetTraceSlo StringAttr result; mb.read(result); - resp.setResult(result.isEmpty() ? "SetTraceSlowTransactions called." : result); + resp.setResult(result.isEmpty() ? "SetTraceSlowTransactions called." : result.get()); } catch(IException* e) { diff --git a/esp/services/ws_packageprocess/ws_packageprocessService.cpp b/esp/services/ws_packageprocess/ws_packageprocessService.cpp index 442896cd7a0..3fc6110b5d9 100644 --- a/esp/services/ws_packageprocess/ws_packageprocessService.cpp +++ b/esp/services/ws_packageprocess/ws_packageprocessService.cpp @@ -319,7 +319,7 @@ class PackageMapUpdater } void setUser(const char *user, const char *password, IEspContext *context) { - if (user && *user && password && *password) + if (user && *user) { userdesc.setown(createUserDescriptor()); userdesc->set(user, password); diff --git a/esp/src/eclwatch/_Widget.js b/esp/src/eclwatch/_Widget.js index cdcb068b575..6b807662b8a 100644 --- a/esp/src/eclwatch/_Widget.js +++ b/esp/src/eclwatch/_Widget.js @@ -130,13 +130,13 @@ define([ return retVal; }, - getURL: function () { + getURL: function (componentFilterParams) { var baseUrl = document.URL.split("#")[0].replace("index.html", "stub.htm"); var baseUrlParts = baseUrl.split("?"); baseUrl = baseUrlParts[0]; var args = baseUrlParts[1]; delete this.params.__filter; - var filterParams = this.getFilterParams(); + var filterParams = componentFilterParams ? componentFilterParams : this.getFilterParams(); if (filterParams) { this.params.__filter = ioQuery.objectToQuery(filterParams); } @@ -144,8 +144,8 @@ define([ return baseUrl + "?" + paramsString; }, - _onNewPage: function (event) { - var win = window.open(this.getURL(), "_blank"); + _onNewPage: function (_event, componentFilterParams) { + var win = window.open(this.getURL(componentFilterParams), "_blank"); if (win) { win.focus(); } diff --git a/esp/src/package-lock.json b/esp/src/package-lock.json index 83e594ec3dc..84c45e73947 100644 --- a/esp/src/package-lock.json +++ b/esp/src/package-lock.json @@ -1928,7 +1928,7 @@ "dependencies": { "@hpcc-js/common": "^2.71.16", "@hpcc-js/preact-shim": "^2.16.10", - "@hpcc-js/util": "^2.50.6" + "@hpcc-js/util": "^2.51.0" } }, "node_modules/@hpcc-js/eclwatch": { @@ -2058,12 +2058,12 @@ "resolved": "https://registry.npmjs.org/@hpcc-js/timeline/-/timeline-2.51.24.tgz", "integrity": "sha512-QNgXhJ6/hQHfP2Lge2zL1X5ERI813KKpFN+DNFqufhWoZIT/7x3kr1If8r1mC74hYt4xqkFAdoveEepFT+lYhQ==", "dependencies": { - "@hpcc-js/api": "^2.12.15", - "@hpcc-js/chart": "^2.83.1", - "@hpcc-js/common": "^2.71.15", - "@hpcc-js/html": "^2.42.18", - "@hpcc-js/layout": "^2.49.20", - "@hpcc-js/react": "^2.53.14" + "@hpcc-js/api": "^2.12.16", + "@hpcc-js/chart": "^2.83.2", + "@hpcc-js/common": "^2.71.16", + "@hpcc-js/html": "^2.42.19", + "@hpcc-js/layout": "^2.49.21", + "@hpcc-js/react": "^2.53.15" } }, "node_modules/@hpcc-js/tree": { diff --git a/esp/src/src-react/components/Security.tsx b/esp/src/src-react/components/Security.tsx index debe5089863..fac4c8ed7a6 100644 --- a/esp/src/src-react/components/Security.tsx +++ b/esp/src/src-react/components/Security.tsx @@ -28,6 +28,20 @@ export const Security: React.FunctionComponent = ({ const [, { opsCategory }] = useBuildInfo(); + const [permissionTabTitle, setPermissionTabTitle] = React.useState(nlsHPCC.Permissions); + + React.useEffect(() => { + setPermissionTabTitle(nlsHPCC.Permissions); + if (name === "_") { + if (baseDn === "File Scopes") setPermissionTabTitle(nlsHPCC.FileScopeDefaultPermissions); + else if (baseDn === "Workunit Scopes") setPermissionTabTitle(nlsHPCC.WorkUnitScopeDefaultPermissions); + } else if (name === "file") { + if (baseDn === "File Scopes") setPermissionTabTitle(nlsHPCC.PhysicalFiles); + } else if (name) { + setPermissionTabTitle(name); + } + }, [name, baseDn]); + return <> {({ size }) => = ({ - + {!name && !baseDn && } diff --git a/esp/src/src-react/components/forms/ZAPDialog.tsx b/esp/src/src-react/components/forms/ZAPDialog.tsx index 2ee5225dd60..74574bdf4a2 100644 --- a/esp/src/src-react/components/forms/ZAPDialog.tsx +++ b/esp/src/src-react/components/forms/ZAPDialog.tsx @@ -376,30 +376,34 @@ export const ZAPDialog: React.FunctionComponent = ({ />} />
-
- } - /> -
-
- } - /> -
-
- } - /> -
+ {!isContainer + ?
+ } + /> +
+ :
+
+ } + /> +
+
+ } + /> +
+
+ }
= ({ const handleClick = (e) => { const tempObj = JSON.parse(decodeURIComponent(e.currentTarget.value)); - widget.NewPage.onClick(tempObj); + widget.NewPage.onClick(e, tempObj); }; const shimmerElements = React.useMemo(() => [ diff --git a/helm/hpcc/templates/_helpers.tpl b/helm/hpcc/templates/_helpers.tpl index be24dd5a427..efa18d914eb 100644 --- a/helm/hpcc/templates/_helpers.tpl +++ b/helm/hpcc/templates/_helpers.tpl @@ -1339,7 +1339,7 @@ Pass in dict with root, me and instances defined {{- $stubInstanceResources := .stubResources | default .root.Values.global.stubInstanceResources | default dict }} {{- $milliCPUText := $stubInstanceResources.cpu | default "200m" }} {{- $milliCPUs := int (include "hpcc.k8sCPUStringToMilliCPU" $milliCPUText) }} -{{- $memoryText := $stubInstanceResources.memory | default "50Mi" }} +{{- $memoryText := $stubInstanceResources.memory | default "400Mi" }} {{- $memory := int64 (include "hpcc.k8sMemoryStringToBytes" $memoryText) }} resources: limits: diff --git a/plugins/parquet/parquetembed.cpp b/plugins/parquet/parquetembed.cpp index ee6a9e53377..f9ada13e93f 100644 --- a/plugins/parquet/parquetembed.cpp +++ b/plugins/parquet/parquetembed.cpp @@ -761,24 +761,13 @@ arrow::Status ParquetWriter::fieldToNode(const std::string &name, const RtlField case type_real: arrowFields.push_back(std::make_shared(name, arrow::float64())); break; - case type_string: - arrowFields.push_back(std::make_shared(name, arrow::utf8())); - break; case type_char: - arrowFields.push_back(std::make_shared(name, arrow::utf8())); - break; + case type_string: case type_varstring: - arrowFields.push_back(std::make_shared(name, arrow::utf8())); - break; case type_qstring: - arrowFields.push_back(std::make_shared(name, arrow::utf8())); - break; - case type_unicode: - arrowFields.push_back(std::make_shared(name, arrow::utf8())); - break; case type_utf8: - arrowFields.push_back(std::make_shared(name, arrow::utf8())); - break; + case type_unicode: + case type_varunicode: case type_decimal: arrowFields.push_back(std::make_shared(name, arrow::utf8())); //TODO add decimal encoding break; diff --git a/roxie/ccd/ccdfile.cpp b/roxie/ccd/ccdfile.cpp index e1e7cb3f1c5..a08ee799ceb 100644 --- a/roxie/ccd/ccdfile.cpp +++ b/roxie/ccd/ccdfile.cpp @@ -3013,6 +3013,8 @@ class CResolvedFile : implements IResolvedFileCreator, implements ISafeSDSSubscr } if (!translator || !translator->canTranslate()) throw MakeStringException(ROXIE_MISMATCH, "Untranslatable record layout mismatch detected for file %s", subname); + else if (mode == RecordTranslationMode::PayloadRemoveOnly && translator->hasNewFields()) + throw MakeStringException(0, "Translatable file layout mismatch reading file %s but translation disabled when expected fields are missing from source.", subname); else if (translator->needsTranslate()) { if (fileMode==FileFormatMode::index && translator->keyedTranslated()) diff --git a/rtl/eclrtl/rtldynfield.cpp b/rtl/eclrtl/rtldynfield.cpp index 14f60d4ca22..31d8ad9f02c 100644 --- a/rtl/eclrtl/rtldynfield.cpp +++ b/rtl/eclrtl/rtldynfield.cpp @@ -38,6 +38,8 @@ extern ECLRTL_API RecordTranslationMode getTranslationMode(const char *val, bool { if (isEmptyString(val) || strToBool(val) || strieq(val, "payload")) return RecordTranslationMode::Payload; + else if (strieq(val, "payloadRemoveOnly")) + return RecordTranslationMode::PayloadRemoveOnly; else if (strieq(val, "alwaysDisk") || strieq(val, "disk")) { if (!isLocal) @@ -60,6 +62,7 @@ extern ECLRTL_API const char *getTranslationModeText(RecordTranslationMode val) case RecordTranslationMode::AlwaysDisk: return "alwaysDisk"; case RecordTranslationMode::AlwaysECL: return "alwaysECL"; case RecordTranslationMode::Payload: return "payload"; + case RecordTranslationMode::PayloadRemoveOnly: return "payloadRemoveOnly"; case RecordTranslationMode::None: return "off"; } throwUnexpected(); @@ -1085,6 +1088,10 @@ class GeneralRecordTranslator : public CInterfaceOf { return (matchFlags & match_keychange) != 0; } + virtual bool hasNewFields() const override + { + return (matchFlags & match_none) != 0; + } private: void doDescribe(unsigned indent) const { @@ -1948,6 +1955,10 @@ class CloneVirtualRecordTranslator : public CInterfaceOf { return false; } + virtual bool hasNewFields() const override + { + return false; + } private: void doDescribe(unsigned indent) const { diff --git a/rtl/eclrtl/rtldynfield.hpp b/rtl/eclrtl/rtldynfield.hpp index 2e7ae6f1df6..d9ddae13068 100644 --- a/rtl/eclrtl/rtldynfield.hpp +++ b/rtl/eclrtl/rtldynfield.hpp @@ -110,7 +110,8 @@ enum class RecordTranslationMode : byte Payload = 2, // Translate all fields in datasets, and only payload fields in indexes AlwaysDisk = 3, // Always translate - even if wouldn't normally (e.g. csv/xml source read as binary), or crcs happen to match AlwaysECL = 4, // Ignore the published format - can make sense to force no translation e.g. when field names have changed - Unspecified = 5 + PayloadRemoveOnly = 5, // Allow fields to be removed from the incoming dataset, but not allow fields to be missing + Unspecified = 6 }; // AlwaysDisk and AlwaysECL are for testing purposes only, and can only be set per file (not globally) extern ECLRTL_API RecordTranslationMode getTranslationMode(const char *modeStr, bool isLocal); @@ -139,6 +140,7 @@ interface IDynamicTransform : public IInterface virtual bool needsTranslate() const = 0; virtual bool keyedTranslated() const = 0; virtual bool needsNonVirtualTranslate() const = 0; + virtual bool hasNewFields() const = 0; }; interface IKeyTranslator : public IInterface diff --git a/thorlcr/slave/slavmain.cpp b/thorlcr/slave/slavmain.cpp index 4e3943e13a1..037d5e8fc11 100644 --- a/thorlcr/slave/slavmain.cpp +++ b/thorlcr/slave/slavmain.cpp @@ -353,6 +353,8 @@ class CKJService : public CSimpleInterfaceOf, implements IThreaded, translator.setown(createRecordTranslator(projectedFormat->queryRecordAccessor(true), publishedFormat->queryRecordAccessor(true))); if (!translator->canTranslate()) throw MakeStringException(0, "Untranslatable record layout mismatch detected for: %s", tracing); + if (RecordTranslationMode::PayloadRemoveOnly == translationMode && translator->hasNewFields()) + throw MakeStringException(0, "Translatable file layout mismatch reading file %s but translation disabled when expected fields are missing from source.", tracing); } DBGLOG("Record layout translator created for %s", tracing); translator->describe();