From 16be60c473ae6e3986a99ef4815aa094c35fcd01 Mon Sep 17 00:00:00 2001 From: Gavin Halliday Date: Thu, 25 Jul 2024 09:48:36 +0100 Subject: [PATCH 1/7] HPCC-32299 Improve coverage of helm regression tests Signed-off-by: Gavin Halliday --- helm/hpcc/templates/dafilesrv.yaml | 2 +- helm/hpcc/templates/roxie.yaml | 2 +- helm/hpcc/templates/thor.yaml | 2 +- testing/helm/tests/furthercoverage.yaml | 31 +++++++++++++++++++++++++ 4 files changed, 34 insertions(+), 3 deletions(-) create mode 100644 testing/helm/tests/furthercoverage.yaml diff --git a/helm/hpcc/templates/dafilesrv.yaml b/helm/hpcc/templates/dafilesrv.yaml index aee2fa519d6..75a631f2b6f 100644 --- a/helm/hpcc/templates/dafilesrv.yaml +++ b/helm/hpcc/templates/dafilesrv.yaml @@ -133,7 +133,7 @@ Generate network policies for spray services to allow ingress from dfuservers {{- if hasKey $networkPolicyCtx.dfuServerMap .name -}} {{- $_ := set $commonCtx "allowedDfuServers" (concat $commonCtx.allowedDfuServers (get $networkPolicyCtx.dfuServerMap .name)) -}} {{- end -}} - {{- if $commonCtx.allowedDfuServers -}} + {{- if $commonCtx.allowedDfuServers }} apiVersion: networking.k8s.io/v1 kind: NetworkPolicy metadata: diff --git a/helm/hpcc/templates/roxie.yaml b/helm/hpcc/templates/roxie.yaml index a8c12ba0dbf..dd040746eec 100644 --- a/helm/hpcc/templates/roxie.yaml +++ b/helm/hpcc/templates/roxie.yaml @@ -222,7 +222,7 @@ kind: ConfigMap {{- $_ := set $commonCtx "instanceNames" list -}} {{ if $roxie.serverReplicas -}} -{{ $_ := set $commonCtx "instanceNames" (list $servername) -}} +{{ $_ := set $commonCtx "instanceNames" (list $servername) }} apiVersion: apps/v1 kind: Deployment metadata: diff --git a/helm/hpcc/templates/thor.yaml b/helm/hpcc/templates/thor.yaml index a2fff9e6611..8781ca10736 100644 --- a/helm/hpcc/templates/thor.yaml +++ b/helm/hpcc/templates/thor.yaml @@ -83,7 +83,7 @@ data: accessDali: "yes" accessEsp: "yes" {{- include "hpcc.generateHelmVersion" . | nindent 12 }} - {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "eclagent" "name" "thor" "instance" $eclAgentJobName "instanceOf" (printf "%s-job" .eclAgentName)) | indent 8 }} + {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "eclagent" "name" "thor" "instance" $eclAgentJobName "instanceOf" (printf "%s-job" .eclAgentName)) | indent 12 }} {{- if hasKey .me "labels" }} {{ toYaml .me.labels | indent 12 }} {{- end }} diff --git a/testing/helm/tests/furthercoverage.yaml b/testing/helm/tests/furthercoverage.yaml new file mode 100644 index 00000000000..7396612ba53 --- /dev/null +++ b/testing/helm/tests/furthercoverage.yaml @@ -0,0 +1,31 @@ +# This values file is here to exrcise all of the different paths in the helm chart. +# It is a copy of values.yaml, all changes are marked with #CHANGE. +# +# - roxie: Enabling server replicas in roxie +# - thor: eclAgentUseChildProcesses=false + +roxie: +- name: roxie + disabled: false + prefix: roxie + services: + - name: roxie + servicePort: 9876 + listenQueue: 200 + numThreads: 30 + visibility: local + replicas: 2 + numChannels: 2 + singleNode: false + traceLevel: 1 + serverReplicas: 4 #CHANGE + topoServer: + replicas: 1 + +thor: +- name: thor + prefix: thor + numWorkers: 2 + maxJobs: 4 + maxGraphs: 2 + eclAgentUseChildProcesses: false #CHANGE From 91ceb070ff83c48f44178c083dd209034e8edf01 Mon Sep 17 00:00:00 2001 From: Jeremy Clements <79224539+jeclrsg@users.noreply.github.com> Date: Mon, 29 Jul 2024 11:38:15 -0400 Subject: [PATCH 2/7] HPCC-32319 ECL Watch update WU list after SetToFailed fix an issue where the WU list did not properly update to reflect the changes of having set Workunit(s) to failed Signed-off-by: Jeremy Clements <79224539+jeclrsg@users.noreply.github.com> --- esp/src/eclwatch/WUQueryWidget.js | 8 ++++---- esp/src/src-react/components/Workunits.tsx | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/esp/src/eclwatch/WUQueryWidget.js b/esp/src/eclwatch/WUQueryWidget.js index 8936500ec04..6cffa29a6f3 100644 --- a/esp/src/eclwatch/WUQueryWidget.js +++ b/esp/src/eclwatch/WUQueryWidget.js @@ -158,19 +158,19 @@ define([ }, _onSetToFailed: function (event) { - WsWorkunits.WUAction(this.workunitsGrid.getSelected(), "SetToFailed"); + WsWorkunits.WUAction(this.workunitsGrid.getSelected(), "SetToFailed").then(() => this.refreshGrid()); }, _onAbort: function (event) { - WsWorkunits.WUAction(this.workunitsGrid.getSelected(), "Abort"); + WsWorkunits.WUAction(this.workunitsGrid.getSelected(), "Abort").then(() => this.refreshGrid()); }, _onProtect: function (event) { - WsWorkunits.WUAction(this.workunitsGrid.getSelected(), "Protect"); + WsWorkunits.WUAction(this.workunitsGrid.getSelected(), "Protect").then(() => this.refreshGrid()); }, _onUnprotect: function (event) { - WsWorkunits.WUAction(this.workunitsGrid.getSelected(), "Unprotect"); + WsWorkunits.WUAction(this.workunitsGrid.getSelected(), "Unprotect").then(() => this.refreshGrid()); }, _onReschedule: function (event) { diff --git a/esp/src/src-react/components/Workunits.tsx b/esp/src/src-react/components/Workunits.tsx index b2047850ee7..b4ff6a3a291 100644 --- a/esp/src/src-react/components/Workunits.tsx +++ b/esp/src/src-react/components/Workunits.tsx @@ -213,7 +213,7 @@ export const Workunits: React.FunctionComponent = ({ { key: "divider_2", itemType: ContextualMenuItemType.Divider, onRender: () => }, { key: "setFailed", text: nlsHPCC.SetToFailed, disabled: !uiState.hasNotProtected, - onClick: () => { WsWorkunits.WUAction(selection, "SetToFailed"); } + onClick: () => { WsWorkunits.WUAction(selection, "SetToFailed").then(() => refreshTable.call()); } }, { key: "abort", text: nlsHPCC.Abort, disabled: !uiState.hasNotCompleted, From 8cc466872fa008713c5633e59aca321cf17d6c45 Mon Sep 17 00:00:00 2001 From: Jeremy Clements <79224539+jeclrsg@users.noreply.github.com> Date: Wed, 24 Jul 2024 14:29:03 -0400 Subject: [PATCH 3/7] HPCC-30048 ECL Watch disable password change when unsupported use the "CanUpdatePassword" boolean returned by /ws_account/MyAccount to enable/disable the change password fields in ECL Watch Signed-off-by: Jeremy Clements <79224539+jeclrsg@users.noreply.github.com> --- esp/src/eclwatch/CurrentUserDetailsWidget.js | 173 +++++++++--------- esp/src/package-lock.json | 8 +- esp/src/package.json | 2 +- esp/src/src-react/components/MyAccount.tsx | 24 ++- esp/src/src-react/components/Title.tsx | 6 +- esp/src/src-react/components/forms/Fields.tsx | 1 + esp/src/src-react/hooks/user.ts | 6 + esp/src/src/nls/hpcc.ts | 1 + 8 files changed, 122 insertions(+), 99 deletions(-) diff --git a/esp/src/eclwatch/CurrentUserDetailsWidget.js b/esp/src/eclwatch/CurrentUserDetailsWidget.js index b96e014a669..eebf772193a 100644 --- a/esp/src/eclwatch/CurrentUserDetailsWidget.js +++ b/esp/src/eclwatch/CurrentUserDetailsWidget.js @@ -26,89 +26,96 @@ define([ ], function (declare, lang, nlsHPCCMod, dom, domForm, arrayUtil, - registry, - _Widget, WsAccount, - template) { - - var nlsHPCC = nlsHPCCMod.default; - return declare("CurrentUserDetailsWidget", [_Widget], { - templateString: template, - baseClass: "CurrentUserDetailsWidget", - i18n: nlsHPCC, - - user: null, - - getTitle: function () { - return this.i18n.UserDetails; - }, - - postCreate: function (args) { - this.inherited(arguments); - this.userForm = registry.byId(this.id + "UserForm"); - }, - - resize: function (args) { - this.inherited(arguments); - this.widget.BorderContainer.resize(); - }, - - // Hitched actions --- - _onSave: function (event) { - var context = this; - var dialog = this.params.Widget; - - if (this.userForm.validate()) { - var formInfo = domForm.toObject(this.id + "UserForm"); - WsAccount.UpdateUser({ - showOkMsg: true, - request: { - username: this.user, - oldpass: formInfo.oldPassword, - newpass1: formInfo.newPassword, - newpass2: formInfo.newPassword - } - }).then(function (response) { - if (lang.exists("UpdateUserResponse", response)) { - arrayUtil.forEach(context.userForm.getDescendants(), function (item, idx) { - item.set("value", ""); - }); - } - }); - dialog.hide(); - } - }, - - // Implementation --- - init: function (params) { - if (this.inherited(arguments)) - return; - - this.user = params.Username; - this.refresh(); - }, - - refresh: function () { - if (this.user) { - this.updateInput("User", null, this.user); - this.updateInput("Username", null, this.user); - + registry, + _Widget, WsAccount, + template) { + + var nlsHPCC = nlsHPCCMod.default; + return declare("CurrentUserDetailsWidget", [_Widget], { + templateString: template, + baseClass: "CurrentUserDetailsWidget", + i18n: nlsHPCC, + + user: null, + canUpdatePassword: false, + + getTitle: function () { + return this.i18n.UserDetails; + }, + + postCreate: function (args) { + this.inherited(arguments); + this.userForm = registry.byId(this.id + "UserForm"); + }, + + resize: function (args) { + this.inherited(arguments); + this.widget.BorderContainer.resize(); + }, + + // Hitched actions --- + _onSave: function (event) { var context = this; - WsAccount.MyAccount({ - }).then(function (response) { - if (lang.exists("MyAccountResponse.firstName", response)) { - context.updateInput("FirstName", null, response.MyAccountResponse.firstName); - } - if (lang.exists("MyAccountResponse.employeeID", response)) { - context.updateInput("EmployeeID", null, response.MyAccountResponse.employeeID); - } - if (lang.exists("MyAccountResponse.lastName", response)) { - context.updateInput("LastName", null, response.MyAccountResponse.lastName); - } - if (lang.exists("MyAccountResponse.passwordExpiration", response)) { - context.updateInput("PasswordExpiration", null, response.MyAccountResponse.passwordExpiration); - } - }); + var dialog = this.params.Widget; + + if (this.canUpdatePassword && this.userForm.validate()) { + var formInfo = domForm.toObject(this.id + "UserForm"); + WsAccount.UpdateUser({ + showOkMsg: true, + request: { + username: this.user, + oldpass: formInfo.oldPassword, + newpass1: formInfo.newPassword, + newpass2: formInfo.newPassword + } + }).then(function (response) { + if (lang.exists("UpdateUserResponse", response)) { + arrayUtil.forEach(context.userForm.getDescendants(), function (item, idx) { + item.set("value", ""); + }); + } + }); + dialog.hide(); + } + }, + + // Implementation --- + init: function (params) { + if (this.inherited(arguments)) + return; + + this.user = params.Username; + this.refresh(); + }, + + refresh: function () { + if (this.user) { + this.updateInput("User", null, this.user); + this.updateInput("Username", null, this.user); + + var context = this; + WsAccount.MyAccount({ + }).then(function (response) { + if (lang.exists("MyAccountResponse.firstName", response)) { + context.updateInput("FirstName", null, response.MyAccountResponse.firstName); + } + if (lang.exists("MyAccountResponse.employeeID", response)) { + context.updateInput("EmployeeID", null, response.MyAccountResponse.employeeID); + } + if (lang.exists("MyAccountResponse.lastName", response)) { + context.updateInput("LastName", null, response.MyAccountResponse.lastName); + } + if (lang.exists("MyAccountResponse.passwordExpiration", response)) { + context.updateInput("PasswordExpiration", null, response.MyAccountResponse.passwordExpiration); + } + if (!response?.MyAccountResponse?.CanUpdatePassword) { + context.setDisabled("dijit_form_ValidationTextBox_0", true); + context.setDisabled("dojox_form__NewPWBox_0", true); + context.setDisabled("dojox_form__VerifyPWBox_0", true); + } + context.canUpdatePassword = response?.MyAccountResponse?.CanUpdatePassword; + }); + } } - } + }); }); -}); diff --git a/esp/src/package-lock.json b/esp/src/package-lock.json index bb858aa4279..28aef7baf94 100644 --- a/esp/src/package-lock.json +++ b/esp/src/package-lock.json @@ -18,7 +18,7 @@ "@hpcc-js/chart": "2.83.4", "@hpcc-js/codemirror": "2.62.1", "@hpcc-js/common": "2.71.18", - "@hpcc-js/comms": "2.94.0", + "@hpcc-js/comms": "2.94.1", "@hpcc-js/dataflow": "8.1.7", "@hpcc-js/eclwatch": "2.74.8", "@hpcc-js/graph": "2.85.16", @@ -2079,9 +2079,9 @@ } }, "node_modules/@hpcc-js/comms": { - "version": "2.94.0", - "resolved": "https://registry.npmjs.org/@hpcc-js/comms/-/comms-2.94.0.tgz", - "integrity": "sha512-+AfJsqj648638hTUeLYd0Thvu1QMHX9zLflrep2xVtz7Wo1OmOiI/mrjClMqK8A8drMa3AduKuQS1R2rL15wZw==", + "version": "2.94.1", + "resolved": "https://registry.npmjs.org/@hpcc-js/comms/-/comms-2.94.1.tgz", + "integrity": "sha512-ROCHmHogsZ5/G9LsRPHRIx25rpVoR9d5TakbSkXeSugRkhPV+16+3C/Lfd4d8ZzNLR99Jmnvjd0NXZAMgpAkGQ==", "dependencies": { "@hpcc-js/ddl-shim": "^2.21.0", "@hpcc-js/util": "^2.52.0", diff --git a/esp/src/package.json b/esp/src/package.json index a30728c2cf2..7a4508f1cf3 100644 --- a/esp/src/package.json +++ b/esp/src/package.json @@ -44,7 +44,7 @@ "@hpcc-js/chart": "2.83.4", "@hpcc-js/codemirror": "2.62.1", "@hpcc-js/common": "2.71.18", - "@hpcc-js/comms": "2.94.0", + "@hpcc-js/comms": "2.94.1", "@hpcc-js/dataflow": "8.1.7", "@hpcc-js/eclwatch": "2.74.8", "@hpcc-js/graph": "2.85.16", diff --git a/esp/src/src-react/components/MyAccount.tsx b/esp/src/src-react/components/MyAccount.tsx index 51130178fca..09fb51bc9d2 100644 --- a/esp/src/src-react/components/MyAccount.tsx +++ b/esp/src/src-react/components/MyAccount.tsx @@ -1,10 +1,11 @@ import * as React from "react"; import { DefaultButton, Dialog, DialogFooter, DialogType, MessageBar, MessageBarType, PrimaryButton } from "@fluentui/react"; +import { useConst } from "@fluentui/react-hooks"; import { AccountService, WsAccount } from "@hpcc-js/comms"; import { scopedLogger } from "@hpcc-js/util"; +import { PasswordStatus } from "../hooks/user"; import nlsHPCC from "src/nlsHPCC"; import { TableGroup } from "./forms/Groups"; -import { useConst } from "@fluentui/react-hooks"; const logger = scopedLogger("src-react/components/MyAccount.tsx"); @@ -36,8 +37,14 @@ export const MyAccount: React.FunctionComponent = ({ }; }, [currentUser]); + const resetForm = React.useCallback(() => { + setOldPassword(""); + setNewPassword1(""); + setNewPassword2(""); + }, []); + const saveUser = React.useCallback(() => { - if (oldPassword !== "" && newPassword1 !== "") { + if (currentUser?.CanUpdatePassword && oldPassword !== "" && newPassword1 !== "") { service.UpdateUser({ username: currentUser.username, oldpass: oldPassword, @@ -51,13 +58,14 @@ export const MyAccount: React.FunctionComponent = ({ } else { setShowError(false); setErrorMessage(""); + resetForm(); onClose(); } }) .catch(err => logger.error(err)) ; } - }, [currentUser, newPassword1, newPassword2, oldPassword, onClose, service]); + }, [currentUser, newPassword1, newPassword2, oldPassword, onClose, resetForm, service]); return ; }; diff --git a/esp/src/src-react/components/Title.tsx b/esp/src/src-react/components/Title.tsx index f42b2e6ad77..5257ba34747 100644 --- a/esp/src/src-react/components/Title.tsx +++ b/esp/src/src-react/components/Title.tsx @@ -15,7 +15,7 @@ import { replaceUrl } from "../util/history"; import { useECLWatchLogger } from "../hooks/logging"; import { useBuildInfo, useModernMode, useCheckFeatures } from "../hooks/platform"; import { useGlobalStore } from "../hooks/store"; -import { useMyAccount, useUserSession } from "../hooks/user"; +import { PasswordStatus, useMyAccount, useUserSession } from "../hooks/user"; import { TitlebarConfig } from "./forms/TitlebarConfig"; import { switchTechPreview } from "./controls/ComingSoon"; @@ -246,10 +246,10 @@ export const DevTitle: React.FunctionComponent = ({ // cookie expires option expects whole number of days, use a decimal < 1 for hours cookie("PasswordExpiredCheck", "true", { expires: 0.5, path: "/" }); switch (currentUser.passwordDaysRemaining) { - case -1: // password has expired + case PasswordStatus.Expired: setPasswordExpiredConfirm(true); break; - case -2: // password never expires + case PasswordStatus.NeverExpires: case null: break; default: diff --git a/esp/src/src-react/components/forms/Fields.tsx b/esp/src/src-react/components/forms/Fields.tsx index cb97c9079aa..f7c435b1129 100644 --- a/esp/src/src-react/components/forms/Fields.tsx +++ b/esp/src/src-react/components/forms/Fields.tsx @@ -863,6 +863,7 @@ export function createInputs(fields: Fields, onChange?: (id: string, newValue: a onChange={(evt, newValue) => onChange(fieldID, newValue)} borderless={field.readonly && !field.multiline} readOnly={field.readonly} + disabled={field.disabled(field) ? true : false} required={field.required} multiline={field.multiline} errorMessage={field.errorMessage ?? ""} diff --git a/esp/src/src-react/hooks/user.ts b/esp/src/src-react/hooks/user.ts index 005ffc4ce5e..a1b49dab49e 100644 --- a/esp/src/src-react/hooks/user.ts +++ b/esp/src/src-react/hooks/user.ts @@ -12,6 +12,12 @@ const defaults = { const userSession = { ...defaults }; +export enum PasswordStatus { + NeverExpires = -2, + Expired = -1, + Unexpired = 0, +} + export interface UserSession { ESPSessionTimeout: number; ESPAuthenticated: boolean; diff --git a/esp/src/src/nls/hpcc.ts b/esp/src/src/nls/hpcc.ts index 0c79a9bcf03..a46da16dfbe 100644 --- a/esp/src/src/nls/hpcc.ts +++ b/esp/src/src/nls/hpcc.ts @@ -660,6 +660,7 @@ export = { PasswordExpired: "Your password has expired. Please change now.", PasswordExpirePrefix: "Your password will expire in ", PasswordExpirePostfix: " day(s). Do you want to change it now?", + PasswordNeverExpires: "Password never expires", Path: "Path", PathAndNameOnly: "Path and name only?", PathMask: "Path Mask", From 5f066c5e928dfe05ad5db3647479156b6377a767 Mon Sep 17 00:00:00 2001 From: Jake Smith Date: Mon, 29 Jul 2024 18:34:31 +0100 Subject: [PATCH 4/7] HPCC-30534 Prevent spurious workunit failed states The thor agent (managing the instance queue), was spuriously setting workunits to failed, if the Thor instance died, which included when the Thor instance span down when idle (linger). The thoragent manages the instances that jobs target, that in a default configuration (multiLingerJob) will spin up when a wuid needs an instance, and spin down when idle. If whilst and instance was being recycles it threw a k8s exception (e.g. 'Job has reached the specified backoff limit'), it would spuriously cause the original workunit that span up the instance to be marked failed. The workunit should only be updated at this point, if it is still marked as having a running state. Normally the workunit workflow instance should manage the final state. Signed-off-by: Jake Smith --- ecl/agentexec/agentexec.cpp | 27 +++++++++++++++++++++------ 1 file changed, 21 insertions(+), 6 deletions(-) diff --git a/ecl/agentexec/agentexec.cpp b/ecl/agentexec/agentexec.cpp index 3cb3364ebb4..58ab247d984 100644 --- a/ecl/agentexec/agentexec.cpp +++ b/ecl/agentexec/agentexec.cpp @@ -261,6 +261,7 @@ class WaitThread : public CInterfaceOf virtual void threadmain() override { Owned exception; + bool sharedK8sJob = false; try { StringAttr jobSpecName(apptype); @@ -276,6 +277,7 @@ class WaitThread : public CInterfaceOf bool useChildProcesses = compConfig->getPropBool("@useChildProcesses"); if (isContainerized() && !useChildProcesses) { + sharedK8sJob = true; constexpr unsigned queueWaitingTimeoutMs = 10000; constexpr unsigned queueWaitingCheckPeriodMs = 1000; if (!owner.lingerQueue || !queueJobIfQueueWaiting(owner.lingerQueue, item, queueWaitingCheckPeriodMs, queueWaitingCheckPeriodMs)) @@ -337,13 +339,26 @@ class WaitThread : public CInterfaceOf { EXCLOG(exception); Owned factory = getWorkUnitFactory(); - Owned workunit = factory->updateWorkUnit(wuid); - if (workunit) + Owned cw = factory->openWorkUnit(wuid); + if (cw) { - workunit->setState(WUStateFailed); - StringBuffer eStr; - addExceptionToWorkunit(workunit, SeverityError, "agentexec", exception->errorCode(), exception->errorMessage(eStr).str(), nullptr, 0, 0, 0); - workunit->commit(); + // if either a) NOT a thoragent with useChildProcesses=false (default in k8s config) or b) is still in an executing state + if (!sharedK8sJob || (cw->getState() == WUStateRunning) || (cw->getState() == WUStateBlocked) || (cw->getState() == WUStateWait)) + { + // For a shared k8s job, i.e. where this agent is thoragent launching shared (multiJobLinger) k8s jobs + // the job agent should handle the job state. + // In that scenario, this is a fallback that should only come into effect if the job workflow instance has failed to handle the exception + // e.g. because it abruptly disappeared. + Owned workunit = &cw->lock(); + // recheck now locked + if ((workunit->getState() == WUStateRunning) || (workunit->getState() == WUStateBlocked) || (workunit->getState() == WUStateWait)) + { + workunit->setState(WUStateFailed); + StringBuffer eStr; + addExceptionToWorkunit(workunit, SeverityError, "agentexec", exception->errorCode(), exception->errorMessage(eStr).str(), nullptr, 0, 0, 0); + workunit->commit(); + } + } } } } From 4e5dd5a62d61c9fb8c338efad83c16f5e67614d9 Mon Sep 17 00:00:00 2001 From: Jake Smith Date: Thu, 1 Aug 2024 18:37:09 +0100 Subject: [PATCH 5/7] HPCC-32368 Rename some compression #options to prevent legacy use Rename "compressInternalSpills", "spillCompressorType" "hdCompressorType" and "hdCompressorOptions", so that old ECL that include them, no longer have any [detrimental] effect. Signed-off-by: Jake Smith --- thorlcr/thorutil/thormisc.hpp | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/thorlcr/thorutil/thormisc.hpp b/thorlcr/thorutil/thormisc.hpp index 6764a98599b..4d6ffabe197 100644 --- a/thorlcr/thorutil/thormisc.hpp +++ b/thorlcr/thorutil/thormisc.hpp @@ -43,17 +43,17 @@ #endif /// Thor options, that can be hints, workunit options, or global settings -#define THOROPT_COMPRESS_SPILLS "compressInternalSpills" // Compress internal spills, e.g. spills created by lookahead or sort gathering (default = true) -#define THOROPT_COMPRESS_SPILL_TYPE "spillCompressorType" // Compress spill type, e.g. FLZ, LZ4 (or other to get previous) (default = LZ4) -#define THOROPT_HDIST_SPILL "hdistSpill" // Allow distribute receiver to spill to disk, rather than blocking (default = true) -#define THOROPT_HDIST_WRITE_POOL_SIZE "hdistSendPoolSize" // Distribute send thread pool size (default = 16) -#define THOROPT_HDIST_BUCKET_SIZE "hdOutBufferSize" // Distribute target bucket send size (default = 1MB) -#define THOROPT_HDIST_BUFFER_SIZE "hdInBufferSize" // Distribute send buffer size (for all targets) (default = 32MB) -#define THOROPT_HDIST_PULLBUFFER_SIZE "hdPullBufferSize" // Distribute pull buffer size (receiver side limit, before spilling) -#define THOROPT_HDIST_CANDIDATELIMIT "hdCandidateLimit" // Limits # of buckets to push to the writers when send buffer is full (default = is 50% largest) -#define THOROPT_HDIST_TARGETWRITELIMIT "hdTargetLimit" // Limit # of writer threads working on a single target (default = unbound, but picks round-robin) -#define THOROPT_HDIST_COMP "hdCompressorType" // Distribute compressor to use (default = "LZ4") -#define THOROPT_HDIST_COMPOPTIONS "hdCompressorOptions" // Distribute compressor options, e.g. AES key (default = "") +#define THOROPT_COMPRESS_SPILLS "v9_4_compressInternalSpills" // Compress internal spills, e.g. spills created by lookahead or sort gathering (default = true) +#define THOROPT_COMPRESS_SPILL_TYPE "v9_4_spillCompressorType" // Compress spill type, e.g. FLZ, LZ4 (or other to get previous) (default = LZ4) +#define THOROPT_HDIST_SPILL "hdistSpill" // Allow distribute receiver to spill to disk, rather than blocking (default = true) +#define THOROPT_HDIST_WRITE_POOL_SIZE "hdistSendPoolSize" // Distribute send thread pool size (default = 16) +#define THOROPT_HDIST_BUCKET_SIZE "hdOutBufferSize" // Distribute target bucket send size (default = 1MB) +#define THOROPT_HDIST_BUFFER_SIZE "hdInBufferSize" // Distribute send buffer size (for all targets) (default = 32MB) +#define THOROPT_HDIST_PULLBUFFER_SIZE "hdPullBufferSize" // Distribute pull buffer size (receiver side limit, before spilling) +#define THOROPT_HDIST_CANDIDATELIMIT "hdCandidateLimit" // Limits # of buckets to push to the writers when send buffer is full (default = is 50% largest) +#define THOROPT_HDIST_TARGETWRITELIMIT "hdTargetLimit" // Limit # of writer threads working on a single target (default = unbound, but picks round-robin) +#define THOROPT_HDIST_COMP "v9_4_hdCompressorType" // Distribute compressor to use (default = "LZ4") +#define THOROPT_HDIST_COMPOPTIONS "v9_4_hdCompressorOptions" // Distribute compressor options, e.g. AES key (default = "") #define THOROPT_SPLITTER_SPILL "splitterSpill" // Force splitters to spill or not, default is to adhere to helper setting (default = -1) #define THOROPT_LOOP_MAX_EMPTY "loopMaxEmpty" // Max # of iterations that LOOP can cycle through with 0 results before errors (default = 1000) #define THOROPT_SMALLSORT "smallSortThreshold" // Use minisort approach, if estimate size of data to sort is below this setting (default = 0) From d002ae83680379169d00b8035d847d0f00223fc6 Mon Sep 17 00:00:00 2001 From: Jeremy Clements <79224539+jeclrsg@users.noreply.github.com> Date: Thu, 1 Aug 2024 14:41:03 -0400 Subject: [PATCH 6/7] HPCC-32367 ECL Watch fix JS exception when viewing archived WUs fixes an issue where there was an uncaught JS exception on the Workunit list page when viewing archived workunits. TotalClusterTime on those is undefined. Signed-off-by: Jeremy Clements <79224539+jeclrsg@users.noreply.github.com> --- esp/src/src/ESPWorkunit.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/esp/src/src/ESPWorkunit.ts b/esp/src/src/ESPWorkunit.ts index 77b406e53fd..0b7713df67f 100644 --- a/esp/src/src/ESPWorkunit.ts +++ b/esp/src/src/ESPWorkunit.ts @@ -1096,7 +1096,7 @@ export function CreateWUQueryStore(): BaseStore Date: Thu, 1 Aug 2024 17:29:03 -0400 Subject: [PATCH 7/7] HPCC-32285 ECL Watch v9 file contents displays empty when loading workaround for an issue in @hpcc-js/dgrid(-shim?), where ECL Watch will display "...empty..." while the contents of a file are being loaded in the dgrid component, instead of the expected "loading..." message this change circumvents the dgrid's messaging by displaying a Fluent UI Spinner until the call to result.fetchXMLScheme(...) resolves Signed-off-by: Jeremy Clements <79224539+jeclrsg@users.noreply.github.com> --- esp/src/src-react/components/Result.tsx | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/esp/src/src-react/components/Result.tsx b/esp/src/src-react/components/Result.tsx index a9d47294417..3490153b800 100644 --- a/esp/src/src-react/components/Result.tsx +++ b/esp/src/src-react/components/Result.tsx @@ -1,6 +1,6 @@ import * as React from "react"; import * as ReactDOM from "react-dom"; -import { Checkbox, CommandBar, ContextualMenuItemType, DefaultButton, Dialog, DialogFooter, DialogType, ICommandBarItemProps, PrimaryButton, SpinButton, Stack } from "@fluentui/react"; +import { Checkbox, CommandBar, ContextualMenuItemType, DefaultButton, Dialog, DialogFooter, DialogType, ICommandBarItemProps, PrimaryButton, SpinButton, Spinner, Stack } from "@fluentui/react"; import { useConst } from "@fluentui/react-hooks"; import { Result as CommsResult, XSDXMLNode } from "@hpcc-js/comms"; import { scopedLogger } from "@hpcc-js/util"; @@ -247,6 +247,7 @@ export const Result: React.FunctionComponent = ({ const [wu] = useWorkunit(wuid); const [result, setResult] = React.useState(resultTable.calcResult()); const [FilterFields, setFilterFields] = React.useState({}); + const [loading, setLoading] = React.useState(true); const [showFilter, setShowFilter] = React.useState(false); React.useEffect(() => { @@ -276,6 +277,7 @@ export const Result: React.FunctionComponent = ({ }; }); setFilterFields(filterFields); + setLoading(false); }).catch(err => { logger.error(err); if (err.message.indexOf("Cannot open the workunit result") > -1) { @@ -370,7 +372,10 @@ export const Result: React.FunctionComponent = ({ header={} main={ <> - + {loading ? + : + + }