diff --git a/ecl/agentexec/agentexec.cpp b/ecl/agentexec/agentexec.cpp index d001bd8eeba..5582af12177 100644 --- a/ecl/agentexec/agentexec.cpp +++ b/ecl/agentexec/agentexec.cpp @@ -261,6 +261,7 @@ class WaitThread : public CInterfaceOf virtual void threadmain() override { Owned exception; + bool sharedK8sJob = false; try { StringAttr jobSpecName(apptype); @@ -276,6 +277,7 @@ class WaitThread : public CInterfaceOf bool useChildProcesses = compConfig->getPropBool("@useChildProcesses"); if (isContainerized() && !useChildProcesses) { + sharedK8sJob = true; constexpr unsigned queueWaitingTimeoutMs = 10000; constexpr unsigned queueWaitingCheckPeriodMs = 1000; if (!owner.lingerQueue || !queueJobIfQueueWaiting(owner.lingerQueue, item, queueWaitingCheckPeriodMs, queueWaitingCheckPeriodMs)) @@ -337,13 +339,26 @@ class WaitThread : public CInterfaceOf { EXCLOG(exception); Owned factory = getWorkUnitFactory(); - Owned workunit = factory->updateWorkUnit(wuid); - if (workunit) + Owned cw = factory->openWorkUnit(wuid); + if (cw) { - workunit->setState(WUStateFailed); - StringBuffer eStr; - addExceptionToWorkunit(workunit, SeverityError, "agentexec", exception->errorCode(), exception->errorMessage(eStr).str(), nullptr, 0, 0, 0); - workunit->commit(); + // if either a) NOT a thoragent with useChildProcesses=false (default in k8s config) or b) is still in an executing state + if (!sharedK8sJob || (cw->getState() == WUStateRunning) || (cw->getState() == WUStateBlocked) || (cw->getState() == WUStateWait)) + { + // For a shared k8s job, i.e. where this agent is thoragent launching shared (multiJobLinger) k8s jobs + // the job agent should handle the job state. + // In that scenario, this is a fallback that should only come into effect if the job workflow instance has failed to handle the exception + // e.g. because it abruptly disappeared. + Owned workunit = &cw->lock(); + // recheck now locked + if ((workunit->getState() == WUStateRunning) || (workunit->getState() == WUStateBlocked) || (workunit->getState() == WUStateWait)) + { + workunit->setState(WUStateFailed); + StringBuffer eStr; + addExceptionToWorkunit(workunit, SeverityError, "agentexec", exception->errorCode(), exception->errorMessage(eStr).str(), nullptr, 0, 0, 0); + workunit->commit(); + } + } } } } diff --git a/esp/src/eclwatch/CurrentUserDetailsWidget.js b/esp/src/eclwatch/CurrentUserDetailsWidget.js index b96e014a669..eebf772193a 100644 --- a/esp/src/eclwatch/CurrentUserDetailsWidget.js +++ b/esp/src/eclwatch/CurrentUserDetailsWidget.js @@ -26,89 +26,96 @@ define([ ], function (declare, lang, nlsHPCCMod, dom, domForm, arrayUtil, - registry, - _Widget, WsAccount, - template) { - - var nlsHPCC = nlsHPCCMod.default; - return declare("CurrentUserDetailsWidget", [_Widget], { - templateString: template, - baseClass: "CurrentUserDetailsWidget", - i18n: nlsHPCC, - - user: null, - - getTitle: function () { - return this.i18n.UserDetails; - }, - - postCreate: function (args) { - this.inherited(arguments); - this.userForm = registry.byId(this.id + "UserForm"); - }, - - resize: function (args) { - this.inherited(arguments); - this.widget.BorderContainer.resize(); - }, - - // Hitched actions --- - _onSave: function (event) { - var context = this; - var dialog = this.params.Widget; - - if (this.userForm.validate()) { - var formInfo = domForm.toObject(this.id + "UserForm"); - WsAccount.UpdateUser({ - showOkMsg: true, - request: { - username: this.user, - oldpass: formInfo.oldPassword, - newpass1: formInfo.newPassword, - newpass2: formInfo.newPassword - } - }).then(function (response) { - if (lang.exists("UpdateUserResponse", response)) { - arrayUtil.forEach(context.userForm.getDescendants(), function (item, idx) { - item.set("value", ""); - }); - } - }); - dialog.hide(); - } - }, - - // Implementation --- - init: function (params) { - if (this.inherited(arguments)) - return; - - this.user = params.Username; - this.refresh(); - }, - - refresh: function () { - if (this.user) { - this.updateInput("User", null, this.user); - this.updateInput("Username", null, this.user); - + registry, + _Widget, WsAccount, + template) { + + var nlsHPCC = nlsHPCCMod.default; + return declare("CurrentUserDetailsWidget", [_Widget], { + templateString: template, + baseClass: "CurrentUserDetailsWidget", + i18n: nlsHPCC, + + user: null, + canUpdatePassword: false, + + getTitle: function () { + return this.i18n.UserDetails; + }, + + postCreate: function (args) { + this.inherited(arguments); + this.userForm = registry.byId(this.id + "UserForm"); + }, + + resize: function (args) { + this.inherited(arguments); + this.widget.BorderContainer.resize(); + }, + + // Hitched actions --- + _onSave: function (event) { var context = this; - WsAccount.MyAccount({ - }).then(function (response) { - if (lang.exists("MyAccountResponse.firstName", response)) { - context.updateInput("FirstName", null, response.MyAccountResponse.firstName); - } - if (lang.exists("MyAccountResponse.employeeID", response)) { - context.updateInput("EmployeeID", null, response.MyAccountResponse.employeeID); - } - if (lang.exists("MyAccountResponse.lastName", response)) { - context.updateInput("LastName", null, response.MyAccountResponse.lastName); - } - if (lang.exists("MyAccountResponse.passwordExpiration", response)) { - context.updateInput("PasswordExpiration", null, response.MyAccountResponse.passwordExpiration); - } - }); + var dialog = this.params.Widget; + + if (this.canUpdatePassword && this.userForm.validate()) { + var formInfo = domForm.toObject(this.id + "UserForm"); + WsAccount.UpdateUser({ + showOkMsg: true, + request: { + username: this.user, + oldpass: formInfo.oldPassword, + newpass1: formInfo.newPassword, + newpass2: formInfo.newPassword + } + }).then(function (response) { + if (lang.exists("UpdateUserResponse", response)) { + arrayUtil.forEach(context.userForm.getDescendants(), function (item, idx) { + item.set("value", ""); + }); + } + }); + dialog.hide(); + } + }, + + // Implementation --- + init: function (params) { + if (this.inherited(arguments)) + return; + + this.user = params.Username; + this.refresh(); + }, + + refresh: function () { + if (this.user) { + this.updateInput("User", null, this.user); + this.updateInput("Username", null, this.user); + + var context = this; + WsAccount.MyAccount({ + }).then(function (response) { + if (lang.exists("MyAccountResponse.firstName", response)) { + context.updateInput("FirstName", null, response.MyAccountResponse.firstName); + } + if (lang.exists("MyAccountResponse.employeeID", response)) { + context.updateInput("EmployeeID", null, response.MyAccountResponse.employeeID); + } + if (lang.exists("MyAccountResponse.lastName", response)) { + context.updateInput("LastName", null, response.MyAccountResponse.lastName); + } + if (lang.exists("MyAccountResponse.passwordExpiration", response)) { + context.updateInput("PasswordExpiration", null, response.MyAccountResponse.passwordExpiration); + } + if (!response?.MyAccountResponse?.CanUpdatePassword) { + context.setDisabled("dijit_form_ValidationTextBox_0", true); + context.setDisabled("dojox_form__NewPWBox_0", true); + context.setDisabled("dojox_form__VerifyPWBox_0", true); + } + context.canUpdatePassword = response?.MyAccountResponse?.CanUpdatePassword; + }); + } } - } + }); }); -}); diff --git a/esp/src/eclwatch/WUQueryWidget.js b/esp/src/eclwatch/WUQueryWidget.js index 8936500ec04..6cffa29a6f3 100644 --- a/esp/src/eclwatch/WUQueryWidget.js +++ b/esp/src/eclwatch/WUQueryWidget.js @@ -158,19 +158,19 @@ define([ }, _onSetToFailed: function (event) { - WsWorkunits.WUAction(this.workunitsGrid.getSelected(), "SetToFailed"); + WsWorkunits.WUAction(this.workunitsGrid.getSelected(), "SetToFailed").then(() => this.refreshGrid()); }, _onAbort: function (event) { - WsWorkunits.WUAction(this.workunitsGrid.getSelected(), "Abort"); + WsWorkunits.WUAction(this.workunitsGrid.getSelected(), "Abort").then(() => this.refreshGrid()); }, _onProtect: function (event) { - WsWorkunits.WUAction(this.workunitsGrid.getSelected(), "Protect"); + WsWorkunits.WUAction(this.workunitsGrid.getSelected(), "Protect").then(() => this.refreshGrid()); }, _onUnprotect: function (event) { - WsWorkunits.WUAction(this.workunitsGrid.getSelected(), "Unprotect"); + WsWorkunits.WUAction(this.workunitsGrid.getSelected(), "Unprotect").then(() => this.refreshGrid()); }, _onReschedule: function (event) { diff --git a/esp/src/package-lock.json b/esp/src/package-lock.json index bb858aa4279..28aef7baf94 100644 --- a/esp/src/package-lock.json +++ b/esp/src/package-lock.json @@ -18,7 +18,7 @@ "@hpcc-js/chart": "2.83.4", "@hpcc-js/codemirror": "2.62.1", "@hpcc-js/common": "2.71.18", - "@hpcc-js/comms": "2.94.0", + "@hpcc-js/comms": "2.94.1", "@hpcc-js/dataflow": "8.1.7", "@hpcc-js/eclwatch": "2.74.8", "@hpcc-js/graph": "2.85.16", @@ -2079,9 +2079,9 @@ } }, "node_modules/@hpcc-js/comms": { - "version": "2.94.0", - "resolved": "https://registry.npmjs.org/@hpcc-js/comms/-/comms-2.94.0.tgz", - "integrity": "sha512-+AfJsqj648638hTUeLYd0Thvu1QMHX9zLflrep2xVtz7Wo1OmOiI/mrjClMqK8A8drMa3AduKuQS1R2rL15wZw==", + "version": "2.94.1", + "resolved": "https://registry.npmjs.org/@hpcc-js/comms/-/comms-2.94.1.tgz", + "integrity": "sha512-ROCHmHogsZ5/G9LsRPHRIx25rpVoR9d5TakbSkXeSugRkhPV+16+3C/Lfd4d8ZzNLR99Jmnvjd0NXZAMgpAkGQ==", "dependencies": { "@hpcc-js/ddl-shim": "^2.21.0", "@hpcc-js/util": "^2.52.0", diff --git a/esp/src/package.json b/esp/src/package.json index a30728c2cf2..7a4508f1cf3 100644 --- a/esp/src/package.json +++ b/esp/src/package.json @@ -44,7 +44,7 @@ "@hpcc-js/chart": "2.83.4", "@hpcc-js/codemirror": "2.62.1", "@hpcc-js/common": "2.71.18", - "@hpcc-js/comms": "2.94.0", + "@hpcc-js/comms": "2.94.1", "@hpcc-js/dataflow": "8.1.7", "@hpcc-js/eclwatch": "2.74.8", "@hpcc-js/graph": "2.85.16", diff --git a/esp/src/src-react/components/MyAccount.tsx b/esp/src/src-react/components/MyAccount.tsx index 51130178fca..09fb51bc9d2 100644 --- a/esp/src/src-react/components/MyAccount.tsx +++ b/esp/src/src-react/components/MyAccount.tsx @@ -1,10 +1,11 @@ import * as React from "react"; import { DefaultButton, Dialog, DialogFooter, DialogType, MessageBar, MessageBarType, PrimaryButton } from "@fluentui/react"; +import { useConst } from "@fluentui/react-hooks"; import { AccountService, WsAccount } from "@hpcc-js/comms"; import { scopedLogger } from "@hpcc-js/util"; +import { PasswordStatus } from "../hooks/user"; import nlsHPCC from "src/nlsHPCC"; import { TableGroup } from "./forms/Groups"; -import { useConst } from "@fluentui/react-hooks"; const logger = scopedLogger("src-react/components/MyAccount.tsx"); @@ -36,8 +37,14 @@ export const MyAccount: React.FunctionComponent = ({ }; }, [currentUser]); + const resetForm = React.useCallback(() => { + setOldPassword(""); + setNewPassword1(""); + setNewPassword2(""); + }, []); + const saveUser = React.useCallback(() => { - if (oldPassword !== "" && newPassword1 !== "") { + if (currentUser?.CanUpdatePassword && oldPassword !== "" && newPassword1 !== "") { service.UpdateUser({ username: currentUser.username, oldpass: oldPassword, @@ -51,13 +58,14 @@ export const MyAccount: React.FunctionComponent = ({ } else { setShowError(false); setErrorMessage(""); + resetForm(); onClose(); } }) .catch(err => logger.error(err)) ; } - }, [currentUser, newPassword1, newPassword2, oldPassword, onClose, service]); + }, [currentUser, newPassword1, newPassword2, oldPassword, onClose, resetForm, service]); return ; }; diff --git a/esp/src/src-react/components/Result.tsx b/esp/src/src-react/components/Result.tsx index a9d47294417..3490153b800 100644 --- a/esp/src/src-react/components/Result.tsx +++ b/esp/src/src-react/components/Result.tsx @@ -1,6 +1,6 @@ import * as React from "react"; import * as ReactDOM from "react-dom"; -import { Checkbox, CommandBar, ContextualMenuItemType, DefaultButton, Dialog, DialogFooter, DialogType, ICommandBarItemProps, PrimaryButton, SpinButton, Stack } from "@fluentui/react"; +import { Checkbox, CommandBar, ContextualMenuItemType, DefaultButton, Dialog, DialogFooter, DialogType, ICommandBarItemProps, PrimaryButton, SpinButton, Spinner, Stack } from "@fluentui/react"; import { useConst } from "@fluentui/react-hooks"; import { Result as CommsResult, XSDXMLNode } from "@hpcc-js/comms"; import { scopedLogger } from "@hpcc-js/util"; @@ -247,6 +247,7 @@ export const Result: React.FunctionComponent = ({ const [wu] = useWorkunit(wuid); const [result, setResult] = React.useState(resultTable.calcResult()); const [FilterFields, setFilterFields] = React.useState({}); + const [loading, setLoading] = React.useState(true); const [showFilter, setShowFilter] = React.useState(false); React.useEffect(() => { @@ -276,6 +277,7 @@ export const Result: React.FunctionComponent = ({ }; }); setFilterFields(filterFields); + setLoading(false); }).catch(err => { logger.error(err); if (err.message.indexOf("Cannot open the workunit result") > -1) { @@ -370,7 +372,10 @@ export const Result: React.FunctionComponent = ({ header={} main={ <> - + {loading ? + : + + } diff --git a/esp/src/src-react/components/Title.tsx b/esp/src/src-react/components/Title.tsx index f42b2e6ad77..5257ba34747 100644 --- a/esp/src/src-react/components/Title.tsx +++ b/esp/src/src-react/components/Title.tsx @@ -15,7 +15,7 @@ import { replaceUrl } from "../util/history"; import { useECLWatchLogger } from "../hooks/logging"; import { useBuildInfo, useModernMode, useCheckFeatures } from "../hooks/platform"; import { useGlobalStore } from "../hooks/store"; -import { useMyAccount, useUserSession } from "../hooks/user"; +import { PasswordStatus, useMyAccount, useUserSession } from "../hooks/user"; import { TitlebarConfig } from "./forms/TitlebarConfig"; import { switchTechPreview } from "./controls/ComingSoon"; @@ -246,10 +246,10 @@ export const DevTitle: React.FunctionComponent = ({ // cookie expires option expects whole number of days, use a decimal < 1 for hours cookie("PasswordExpiredCheck", "true", { expires: 0.5, path: "/" }); switch (currentUser.passwordDaysRemaining) { - case -1: // password has expired + case PasswordStatus.Expired: setPasswordExpiredConfirm(true); break; - case -2: // password never expires + case PasswordStatus.NeverExpires: case null: break; default: diff --git a/esp/src/src-react/components/Workunits.tsx b/esp/src/src-react/components/Workunits.tsx index b2047850ee7..b4ff6a3a291 100644 --- a/esp/src/src-react/components/Workunits.tsx +++ b/esp/src/src-react/components/Workunits.tsx @@ -213,7 +213,7 @@ export const Workunits: React.FunctionComponent = ({ { key: "divider_2", itemType: ContextualMenuItemType.Divider, onRender: () => }, { key: "setFailed", text: nlsHPCC.SetToFailed, disabled: !uiState.hasNotProtected, - onClick: () => { WsWorkunits.WUAction(selection, "SetToFailed"); } + onClick: () => { WsWorkunits.WUAction(selection, "SetToFailed").then(() => refreshTable.call()); } }, { key: "abort", text: nlsHPCC.Abort, disabled: !uiState.hasNotCompleted, diff --git a/esp/src/src-react/components/forms/Fields.tsx b/esp/src/src-react/components/forms/Fields.tsx index 6ec76f804a4..5ae9c59e80e 100644 --- a/esp/src/src-react/components/forms/Fields.tsx +++ b/esp/src/src-react/components/forms/Fields.tsx @@ -890,6 +890,7 @@ export function createInputs(fields: Fields, onChange?: (id: string, newValue: a onChange={(evt, newValue) => onChange(fieldID, newValue)} borderless={field.readonly && !field.multiline} readOnly={field.readonly} + disabled={field.disabled(field) ? true : false} required={field.required} multiline={field.multiline} errorMessage={field.errorMessage ?? ""} diff --git a/esp/src/src-react/hooks/user.ts b/esp/src/src-react/hooks/user.ts index 005ffc4ce5e..a1b49dab49e 100644 --- a/esp/src/src-react/hooks/user.ts +++ b/esp/src/src-react/hooks/user.ts @@ -12,6 +12,12 @@ const defaults = { const userSession = { ...defaults }; +export enum PasswordStatus { + NeverExpires = -2, + Expired = -1, + Unexpired = 0, +} + export interface UserSession { ESPSessionTimeout: number; ESPAuthenticated: boolean; diff --git a/esp/src/src/ESPWorkunit.ts b/esp/src/src/ESPWorkunit.ts index 77b406e53fd..0b7713df67f 100644 --- a/esp/src/src/ESPWorkunit.ts +++ b/esp/src/src/ESPWorkunit.ts @@ -1096,7 +1096,7 @@ export function CreateWUQueryStore(): BaseStore #include "thactivityutil.ipp" #include "thcompressutil.hpp" #include "thexception.hpp" @@ -155,6 +156,8 @@ class CBroadcaster : public CSimpleInterface CThreadedPersistent threaded; SimpleInterThreadQueueOf broadcastQueue; Owned exception; + unsigned myNode; + unsigned nodes; bool aborted; void clearQueue() { @@ -169,6 +172,12 @@ class CBroadcaster : public CSimpleInterface CSend(CBroadcaster &_broadcaster) : threaded("CBroadcaster::CSend", this), broadcaster(_broadcaster) { aborted = false; + myNode = broadcaster.activity.queryJob().queryMyNodeRank()-1; // 0 based + nodes = broadcaster.activity.queryJob().queryNodes(); + + // in theory each worker could be sending log(n) packets, with the broadcaster on each blocking waiting for acks + unsigned limit = nodes * std::ceil(std::log2(nodes)); + broadcastQueue.setLimit(limit); } ~CSend() { @@ -182,7 +191,22 @@ class CBroadcaster : public CSimpleInterface sendItem->Release(); throw exception.getClear(); } - broadcastQueue.enqueue(sendItem); // will block if queue full + // check if anywhere else to send to + if (sendItem) + { + unsigned origin = sendItem->queryNode(); + unsigned pseudoNode = (myNode=nodes) + { + sendItem->Release(); + return; + } + } + while (!broadcastQueue.enqueue(sendItem, 5000)) // will block if queue full + { + DBGLOG("CSend::addBlock() - broadcastQueue full, waiting for space"); + } } void start() { @@ -272,6 +296,8 @@ class CBroadcaster : public CSimpleInterface mptag_t rt = ::createReplyTag(); unsigned origin = sendItem->queryNode(); unsigned pseudoNode = (myNodequerySlave()+1, sendLen, (unsigned)sendItem->queryCode()); #endif @@ -330,6 +360,11 @@ class CBroadcaster : public CSimpleInterface } return false; } + // recieve loop, receives CSendItem packets, adds them to broadcaster thread ('sender'), and processes the packet via 'bCastReceive'. + // bcast_sendStopping are regular row packets that inform us that the sender is stopping (something upstream has asked it to stop()) + // - If all workers have signalled stopping, 'allRequestStop' will be set and will curtail the broadcast of more packets. + // - Or, if the broadcaster has explicitly been stopped (occurs via failover to local lookup), this will also curtail the broadcast of more packets. + // bcast_stop contains no row data, it signals that the sender has finished sending data. void recvLoop() { // my sender is implicitly stopped (never sends to self) @@ -349,16 +384,20 @@ class CBroadcaster : public CSimpleInterface break; } mptag_t replyTag = msg.getReplyTag(); - CMessageBuffer ackMsg; Owned sendItem = new CSendItem(msg); #ifdef _TRACEBROADCAST ActPrintLog(&activity, "Broadcast node %d received from node %d, origin node %d, origin slave %d, size %d, code=%d", myNode+1, (unsigned)sendRank, sendItem->queryNode()+1, sendItem->querySlave()+1, sendItem->length(), (unsigned)sendItem->queryCode()); #endif + CMessageBuffer ackMsg; + bool stopping = isStopping(); // this is effectively a shortcut to inform sender asap. bcast_sendStopping/bcast_stop will be queued soon + ackMsg.append(stopping); comm.send(ackMsg, sendRank, replyTag); // send ack #ifdef _TRACEBROADCAST ActPrintLog(&activity, "Broadcast node %d, sent ack to node %d, replyTag=%d", myNode+1, (unsigned)sendRank, (unsigned)replyTag); #endif - sender.addBlock(sendItem.getLink()); + // if all stopping, then suppress broadcasting (except stop packets) + if (!allRequestStop || (bcast_stop == sendItem->queryCode())) + sender.addBlock(sendItem.getLink()); assertex(myNode != sendItem->queryNode()); switch (sendItem->queryCode()) { diff --git a/thorlcr/thorutil/thormisc.hpp b/thorlcr/thorutil/thormisc.hpp index 6c667ebd8e9..3159d58ad48 100644 --- a/thorlcr/thorutil/thormisc.hpp +++ b/thorlcr/thorutil/thormisc.hpp @@ -43,17 +43,17 @@ #endif /// Thor options, that can be hints, workunit options, or global settings -#define THOROPT_COMPRESS_SPILLS "compressInternalSpills" // Compress internal spills, e.g. spills created by lookahead or sort gathering (default = true) -#define THOROPT_COMPRESS_SPILL_TYPE "spillCompressorType" // Compress spill type, e.g. FLZ, LZ4 (or other to get previous) (default = LZ4) -#define THOROPT_HDIST_SPILL "hdistSpill" // Allow distribute receiver to spill to disk, rather than blocking (default = true) -#define THOROPT_HDIST_WRITE_POOL_SIZE "hdistSendPoolSize" // Distribute send thread pool size (default = 16) -#define THOROPT_HDIST_BUCKET_SIZE "hdOutBufferSize" // Distribute target bucket send size (default = 1MB) -#define THOROPT_HDIST_BUFFER_SIZE "hdInBufferSize" // Distribute send buffer size (for all targets) (default = 32MB) -#define THOROPT_HDIST_PULLBUFFER_SIZE "hdPullBufferSize" // Distribute pull buffer size (receiver side limit, before spilling) -#define THOROPT_HDIST_CANDIDATELIMIT "hdCandidateLimit" // Limits # of buckets to push to the writers when send buffer is full (default = is 50% largest) -#define THOROPT_HDIST_TARGETWRITELIMIT "hdTargetLimit" // Limit # of writer threads working on a single target (default = unbound, but picks round-robin) -#define THOROPT_HDIST_COMP "hdCompressorType" // Distribute compressor to use (default = "LZ4") -#define THOROPT_HDIST_COMPOPTIONS "hdCompressorOptions" // Distribute compressor options, e.g. AES key (default = "") +#define THOROPT_COMPRESS_SPILLS "v9_4_compressInternalSpills" // Compress internal spills, e.g. spills created by lookahead or sort gathering (default = true) +#define THOROPT_COMPRESS_SPILL_TYPE "v9_4_spillCompressorType" // Compress spill type, e.g. FLZ, LZ4 (or other to get previous) (default = LZ4) +#define THOROPT_HDIST_SPILL "hdistSpill" // Allow distribute receiver to spill to disk, rather than blocking (default = true) +#define THOROPT_HDIST_WRITE_POOL_SIZE "hdistSendPoolSize" // Distribute send thread pool size (default = 16) +#define THOROPT_HDIST_BUCKET_SIZE "hdOutBufferSize" // Distribute target bucket send size (default = 1MB) +#define THOROPT_HDIST_BUFFER_SIZE "hdInBufferSize" // Distribute send buffer size (for all targets) (default = 32MB) +#define THOROPT_HDIST_PULLBUFFER_SIZE "hdPullBufferSize" // Distribute pull buffer size (receiver side limit, before spilling) +#define THOROPT_HDIST_CANDIDATELIMIT "hdCandidateLimit" // Limits # of buckets to push to the writers when send buffer is full (default = is 50% largest) +#define THOROPT_HDIST_TARGETWRITELIMIT "hdTargetLimit" // Limit # of writer threads working on a single target (default = unbound, but picks round-robin) +#define THOROPT_HDIST_COMP "v9_4_hdCompressorType" // Distribute compressor to use (default = "LZ4") +#define THOROPT_HDIST_COMPOPTIONS "v9_4_hdCompressorOptions" // Distribute compressor options, e.g. AES key (default = "") #define THOROPT_SPLITTER_SPILL "splitterSpill" // Force splitters to spill or not, default is to adhere to helper setting (default = -1) #define THOROPT_SPLITTER_MAXROWMEMK "splitterRowMemK" // Splitter max memory (K) to use before spilling (default = 2MB) #define THOROPT_SPLITTER_READAHEADGRANULARITYK "inMemReadAheadGranularityK" // Splitter in memory read ahead granularity (K) (default = 128K)