Merge remote-tracking branch 'origin/candidate-9.2.x' into candidate-…

…9.4.x Signed-off-by: Jake Smith <[email protected]> # Conflicts: # helm/hpcc/Chart.yaml # helm/hpcc/templates/_helpers.tpl # helm/hpcc/templates/dafilesrv.yaml # helm/hpcc/templates/dali.yaml # helm/hpcc/templates/dfuserver.yaml # helm/hpcc/templates/eclagent.yaml # helm/hpcc/templates/eclccserver.yaml # helm/hpcc/templates/eclscheduler.yaml # helm/hpcc/templates/esp.yaml # helm/hpcc/templates/localroxie.yaml # helm/hpcc/templates/roxie.yaml # helm/hpcc/templates/sasha.yaml # helm/hpcc/templates/thor.yaml # version.cmake
hpcc-systems · Feb 29, 2024 · 4f4b672 · 4f4b672
2 parents b6e09dc + 20b30eb
commit 4f4b672
Show file tree

Hide file tree

Showing 20 changed files with 140 additions and 27 deletions.
diff --git a/common/remote/hooks/git/gitfile.cpp b/common/remote/hooks/git/gitfile.cpp
@@ -236,9 +236,14 @@ class GitRepositoryFileIO : implements CSimpleInterfaceOf<IFileIO>
 
         //If fetching from git and the username is specified then use the script file to provide the username/password
         //Only support retrieving the password as a secret - not as a filename
+        //NB: This code should be kept in sync with runGitCommand() in hqlrepository.cpp (and ideally combined)
         if (!isEmptyString(gitUser))
         {
+            env.emplace_back("HPCC_GIT_USERNAME", gitUser);
+
+            // If gituser is specified never prompt for credentials, otherwise the server can hang.
             env.emplace_back("GIT_TERMINAL_PROMPT", "0");
+
             StringBuffer scriptPath;
             getPackageFolder(scriptPath);
             addPathSepChar(scriptPath).append("bin/hpccaskpass.sh");

diff --git a/dali/dfu/dfurun.cpp b/dali/dfu/dfurun.cpp
@@ -1429,15 +1429,19 @@ class CDFUengine: public CInterface, implements IDFUengine
                     // keys default wrap for copy
                     if (destination->getWrap()||(iskey&&(cmd==DFUcmd_copy)))
                     {
-                        if (destination->getNumPartsOverride())
-                            throw makeStringExceptionV(-1, "DestinationNumPartOverride is provided but %s", destination->getWrap()?"getWrap is true":"is copying a key");
-                        dst->setNumPartsOverride(srcFile->numParts());
+                        unsigned numOverrideParts = destination->getNumPartsOverride();
+                        if (numOverrideParts)
+                        {
+                            if (srcFile->numParts() != numOverrideParts)
+                                throw makeStringExceptionV(-1, "Destination NumPartsOverride is provided but %s", (iskey&&(cmd==DFUcmd_copy))?"not supported when copying a key":"getWrap is true");
+                        }
+                        dst->setNumParts(srcFile->numParts());
                     }
                     else if (plane)
                     {
                         // use destination defaultSprayParts if requestor doesn't provide num parts
                         if (plane->hasProp("@defaultSprayParts") && destination->getNumPartsOverride()==0)
-                            dst->setNumPartsOverride(plane->getPropInt("@defaultSprayParts"));
+                            dst->setNumParts(plane->getPropInt("@defaultSprayParts"));
                     }
                 }
                 break;

diff --git a/dali/dfu/dfuwu.cpp b/dali/dfu/dfuwu.cpp
@@ -1378,7 +1378,7 @@ class CDFUfileSpec: public CLinkedDFUWUchild, implements IDFUfileSpec
     {
         queryRoot()->setProp("@partmask",val);
     }
-    void setNumParts(unsigned val)
+    virtual void setNumParts(unsigned val) override
     {
         queryRoot()->setPropInt("@numparts",val);
 

diff --git a/dali/dfu/dfuwu.hpp b/dali/dfu/dfuwu.hpp
@@ -281,6 +281,7 @@ interface IDFUfileSpec: extends IConstDFUfileSpec
     virtual void setFromXML(const char *xml) = 0;
     virtual void setCompressed(bool set) = 0;
     virtual void setWrap(bool val) = 0;
+    virtual void setNumParts(unsigned val) = 0;
     virtual void setNumPartsOverride(unsigned num) = 0;
     virtual void setReplicateOffset(int val) = 0;           // sets for all clusters
     virtual void setDiffKey(const char *keyname) = 0;

diff --git a/ecl/hql/hqlrepository.cpp b/ecl/hql/hqlrepository.cpp
@@ -954,6 +954,7 @@ unsigned EclRepositoryManager::runGitCommand(StringBuffer * output, const char *
     Owned<IFile> extractedKey;
     EnvironmentVector env;
     //If fetching from git and the username is specified then use the script file to provide the username/password
+    //NB: This code should be kept in sync with readLfsContents() in gitfile.cpp (and ideally combined)
     if (needCredentials)
     {
         //If the username is supplied, then get the secret and write it to a temporary location.

diff --git a/esp/src/eclwatch/LockDialogWidget.js b/esp/src/eclwatch/LockDialogWidget.js
@@ -153,7 +153,7 @@ define([
                 cookie("Status", "Locked");
                 context.storage.setItem("Status", "Locked");
             } else if (cookie("Status") === "Unlocked") {
-                xhr("esp/lock", {
+                xhr("/esp/lock", {
                     method: "post"
                 }).then(function (response) {
                     if (response) {

diff --git a/esp/src/eclwatch/WUDetailsWidget.js b/esp/src/eclwatch/WUDetailsWidget.js
@@ -237,6 +237,7 @@ define([
                 this.wu.publish(
                     dom.byId(this.id + "Jobname2").value,
                     dom.byId(this.id + "RemoteDali").value,
+                    dom.byId(this.id + "RemoteStorage").value,
                     dom.byId(this.id + "SourceProcess").value,
                     registry.byId(this.id + "Priority").value,
                     dom.byId(this.id + "Comment").value,

diff --git a/esp/src/eclwatch/templates/WUDetailsWidget.html b/esp/src/eclwatch/templates/WUDetailsWidget.html
@@ -30,6 +30,7 @@
                                 <div class="dijitDialogPaneContentArea" data-dojo-props="cols:2" data-dojo-type="hpcc.TableContainer">
                                     <input id="${id}Jobname2" title="${i18n.JobName}:" colspan="2" style="width:100%" required="true" data-dojo-props="trim: true" data-dojo-type="dijit.form.ValidationTextBox" />
                                     <input id="${id}RemoteDali" title="${i18n.RemoteDali}:" colspan="2" style="width:100%" required="false" data-dojo-props="trim: true" data-dojo-type="dijit.form.ValidationTextBox" />
+                                    <input id="${id}RemoteStorage" title="${i18n.RemoteStorage}:" colspan="2" style="width:100%" required="false" data-dojo-props="trim: true" data-dojo-type="dijit.form.ValidationTextBox" />
                                     <input id="${id}SourceProcess" title="${i18n.SourceProcess}:" colspan="2" style="width:100%" required="false" data-dojo-props="trim: true" data-dojo-type="dijit.form.ValidationTextBox" />
                                     <input id="${id}Comment" title="${i18n.Comment}:" colspan="2" style="width:100%" required="false" data-dojo-props="trim: true" data-dojo-type="dijit.form.ValidationTextBox" />
                                     <select id="${id}Priority" title="${i18n.Priority}:" colspan="2" data-dojo-type="dijit.form.Select">

diff --git a/esp/src/package-lock.json b/esp/src/package-lock.json
diff --git a/esp/src/package.json b/esp/src/package.json
@@ -44,7 +44,7 @@
     "@hpcc-js/chart": "2.81.10",
     "@hpcc-js/codemirror": "2.61.2",
     "@hpcc-js/common": "2.71.15",
-    "@hpcc-js/comms": "2.90.0",
+    "@hpcc-js/comms": "2.91.0",
     "@hpcc-js/dataflow": "8.1.6",
     "@hpcc-js/eclwatch": "2.73.37",
     "@hpcc-js/graph": "2.85.11",

diff --git a/esp/src/src-react/components/Files.tsx b/esp/src/src-react/components/Files.tsx
@@ -9,6 +9,7 @@ import { QuerySortItem } from "src/store/Store";
 import nlsHPCC from "src/nlsHPCC";
 import { useConfirm } from "../hooks/confirm";
 import { useMyAccount } from "../hooks/user";
+import { useHasFocus, useIsMounted } from "../hooks/util";
 import { HolyGrail } from "../layouts/HolyGrail";
 import { pushParams } from "../util/history";
 import { FluentPagedGrid, FluentPagedFooter, useCopyButtons, useFluentStoreState, FluentColumns } from "./controls/Grid";
@@ -116,6 +117,16 @@ export const Files: React.FunctionComponent<FilesProps> = ({
         total, setTotal,
         refreshTable } = useFluentStoreState({ page });
 
+    //  Refresh on focus  ---
+    const isMounted = useIsMounted();
+    const hasFocus = useHasFocus();
+    React.useEffect(() => {
+        if (isMounted && hasFocus) {
+            refreshTable.call();
+        }
+        // eslint-disable-next-line react-hooks/exhaustive-deps
+    }, [hasFocus]);
+
     //  Grid ---
     const gridStore = React.useMemo(() => {
         return store ? store : CreateDFUQueryStore();

diff --git a/esp/src/src-react/components/Queries.tsx b/esp/src/src-react/components/Queries.tsx
@@ -6,6 +6,7 @@ import nlsHPCC from "src/nlsHPCC";
 import { QuerySortItem } from "src/store/Store";
 import { useConfirm } from "../hooks/confirm";
 import { useMyAccount } from "../hooks/user";
+import { useHasFocus, useIsMounted } from "../hooks/util";
 import { HolyGrail } from "../layouts/HolyGrail";
 import { pushParams } from "../util/history";
 import { FluentPagedGrid, FluentPagedFooter, useCopyButtons, useFluentStoreState, FluentColumns } from "./controls/Grid";
@@ -76,6 +77,16 @@ export const Queries: React.FunctionComponent<QueriesProps> = ({
 
     const hasFilter = React.useMemo(() => Object.keys(filter).length > 0, [filter]);
 
+    //  Refresh on focus  ---
+    const isMounted = useIsMounted();
+    const hasFocus = useHasFocus();
+    React.useEffect(() => {
+        if (isMounted && hasFocus) {
+            refreshTable.call();
+        }
+        // eslint-disable-next-line react-hooks/exhaustive-deps
+    }, [hasFocus]);
+
     //  Grid ---
     const gridStore = React.useMemo(() => {
         return store || ESPQuery.CreateQueryStore({});

diff --git a/esp/src/src-react/components/Title.tsx b/esp/src/src-react/components/Title.tsx
@@ -67,7 +67,7 @@ export const DevTitle: React.FunctionComponent<DevTitleProps> = ({
 
     const titlebarColorSet = React.useMemo(() => {
         return titlebarColor && titlebarColor !== theme.palette.themeLight;
-    }, [theme.palette, titlebarColor]);    
+    }, [theme.palette, titlebarColor]);
 
     const personaProps: IPersonaSharedProps = React.useMemo(() => {
         return {
@@ -139,7 +139,7 @@ export const DevTitle: React.FunctionComponent<DevTitleProps> = ({
                 { key: "divider_2", itemType: ContextualMenuItemType.Divider },
                 {
                     key: "lock", text: nlsHPCC.Lock, disabled: !currentUser?.username, onClick: () => {
-                        fetch("esp/lock", {
+                        fetch("/esp/lock", {
                             method: "post"
                         }).then(() => {
                             setUserSession({ ...userSession, Status: "Locked" });
@@ -287,7 +287,7 @@ export const DevTitle: React.FunctionComponent<DevTitleProps> = ({
                         </DefaultButton>
                     </Stack.Item>
                     <Stack.Item align="center">
-                        <IconButton title={nlsHPCC.Advanced} iconProps={collapseMenuIcon} menuProps={advMenuProps} style={{color: titlebarColorSet ? Utility.textColor(titlebarColor) : theme.palette.themeDarker }}/>
+                        <IconButton title={nlsHPCC.Advanced} iconProps={collapseMenuIcon} menuProps={advMenuProps} style={{ color: titlebarColorSet ? Utility.textColor(titlebarColor) : theme.palette.themeDarker }} />
                     </Stack.Item>
                 </Stack>
                 <Toaster position="top-right" gutter={8 - (90 - toasterScale(90))} containerStyle={{

diff --git a/esp/src/src-react/components/Workunits.tsx b/esp/src/src-react/components/Workunits.tsx
@@ -8,8 +8,9 @@ import { formatCost } from "src/Session";
 import nlsHPCC from "src/nlsHPCC";
 import { useConfirm } from "../hooks/confirm";
 import { useMyAccount } from "../hooks/user";
-import { HolyGrail } from "../layouts/HolyGrail";
 import { pushParams } from "../util/history";
+import { useHasFocus, useIsMounted } from "../hooks/util";
+import { HolyGrail } from "../layouts/HolyGrail";
 import { FluentPagedGrid, FluentPagedFooter, useCopyButtons, useFluentStoreState, FluentColumns } from "./controls/Grid";
 import { Fields } from "./forms/Fields";
 import { Filter } from "./forms/Filter";
@@ -103,6 +104,16 @@ export const Workunits: React.FunctionComponent<WorkunitsProps> = ({
         total, setTotal,
         refreshTable } = useFluentStoreState({ page });
 
+    //  Refresh on focus  ---
+    const isMounted = useIsMounted();
+    const hasFocus = useHasFocus();
+    React.useEffect(() => {
+        if (isMounted && hasFocus) {
+            refreshTable.call();
+        }
+        // eslint-disable-next-line react-hooks/exhaustive-deps
+    }, [hasFocus]);
+
     //  Grid ---
     const query = React.useMemo(() => {
         return formatQuery(filter);

diff --git a/esp/src/src-react/components/controls/Grid.tsx b/esp/src/src-react/components/controls/Grid.tsx
@@ -158,13 +158,13 @@ export interface FluentStoreStateResponse {
     setPageSize: (pageSize: number) => void;
     total: number;
     setTotal: (total: number) => void;
-    refreshTable: RefreshTable
+    refreshTable: RefreshTable;
 }
 
 export function useFluentStoreState({ page }: FluentStoreStateProps): FluentStoreStateResponse {
     const [selection, setSelection] = React.useState([]);
     const [pageNum, setPageNum] = React.useState(page);
-    const [pageSize, setPageSize] = React.useState(25);
+    const [pageSize, setPageSize] = React.useState<number>();
     const [total, setTotal] = React.useState(0);
     const refreshTable = useRefreshTable();
 

diff --git a/esp/src/src-react/components/forms/PublishQuery.tsx b/esp/src/src-react/components/forms/PublishQuery.tsx
@@ -11,6 +11,7 @@ const logger = scopedLogger("../components/forms/PublishQuery.tsx");
 interface PublishFormValues {
     jobName: string;
     remoteDali: string;
+    remoteStorage: string;
     sourceProcess: string;
     comment: string;
     priority: string;
@@ -21,6 +22,7 @@ interface PublishFormValues {
 const defaultValues: PublishFormValues = {
     jobName: "",
     remoteDali: "",
+    remoteStorage: "",
     sourceProcess: "",
     comment: "",
     priority: "",
@@ -52,7 +54,20 @@ export const PublishQueryForm: React.FunctionComponent<PublishFormProps> = ({
     const onSubmit = React.useCallback(() => {
         handleSubmit(
             (data, evt) => {
-                workunit.publish(data.jobName).then(() => {
+                const request = {
+                    Wuid: workunit?.Wuid,
+                    Cluster: workunit?.Cluster,
+
+                    JobName: data.jobName,
+                    RemoteDali: data?.remoteDali ?? "",
+                    RemoteStorage: data?.remoteStorage ?? "",
+                    Comment: data?.comment ?? "",
+                    SourceProcess: data?.sourceProcess ?? "",
+                    Priority: data?.priority,
+                    AllowForeignFiles: data?.allowForeignFiles,
+                    UpdateSuperFiles: data?.updateSuperFiles
+                };
+                workunit.publishEx(request).then(() => {
                     return workunit.update({ Jobname: data.jobName });
                 }).then(() => {
                     closeForm();
@@ -104,6 +119,19 @@ export const PublishQueryForm: React.FunctionComponent<PublishFormProps> = ({
                     errorMessage={error && error?.message}
                 />}
         />
+        <Controller
+            control={control} name="remoteStorage"
+            render={({
+                field: { onChange, name: fieldName, value },
+                fieldState: { error }
+            }) => <TextField
+                    name={fieldName}
+                    onChange={onChange}
+                    label={nlsHPCC.RemoteStorage}
+                    value={value}
+                    errorMessage={error && error?.message}
+                />}
+        />
         <Controller
             control={control} name="sourceProcess"
             render={({

diff --git a/esp/src/src-react/hooks/store.ts b/esp/src/src-react/hooks/store.ts
@@ -40,7 +40,7 @@ function fromString<T>(value: string, defaultValue: T): T {
 
 function useStore<T>(store: IKeyValStore, key: string, defaultValue: T, monitor: boolean = false): [value: T, setValue: (value: T) => void, reset: () => void] {
 
-    const [value, setValue] = React.useState<T>(defaultValue);
+    const [value, setValue] = React.useState<T>();
 
     React.useEffect(() => {
         if (!store) return;

diff --git a/esp/src/src-react/hooks/util.ts b/esp/src/src-react/hooks/util.ts
@@ -7,3 +7,40 @@ export function useCounter(): [number, () => void] {
     return [counter, () => setCounter(counter + 1)];
 }
 
+export function useIsMounted(): boolean {
+    const isMountedRef = React.useRef(false);
+
+    React.useEffect(() => {
+        isMountedRef.current = true;
+        return () => {
+            isMountedRef.current = false;
+        };
+    }, []);
+
+    return isMountedRef.current;
+}
+
+export function useHasFocus(): boolean {
+
+    const [hasFocus, setHasFocus] = React.useState(true);
+
+    const onFocus = () => {
+        setHasFocus(true);
+    };
+
+    const onBlur = () => {
+        setHasFocus(false);
+    };
+
+    React.useEffect(() => {
+        window.addEventListener("focus", onFocus);
+        window.addEventListener("blur", onBlur);
+
+        return () => {
+            window.removeEventListener("focus", onFocus);
+            window.removeEventListener("blur", onBlur);
+        };
+    }, []);
+
+    return hasFocus;
+}