diff --git a/common/workunit/workunit.cpp b/common/workunit/workunit.cpp index 14d4a1c319f..3214bcca214 100644 --- a/common/workunit/workunit.cpp +++ b/common/workunit/workunit.cpp @@ -8759,14 +8759,26 @@ void CLocalWorkUnit::setDebugValue(const char *propname, const char *value, bool lower.append(propname); if (!strchr(lower, ':')) lower.toLowerCase(); - CriticalBlock block(crit); - StringBuffer prop("Debug/"); - prop.append(lower); - if (overwrite || !p->hasProp(prop.str())) + try { - // MORE - not sure this line should be needed.... - p->setProp("Debug", ""); - p->setProp(prop.str(), value); + CriticalBlock block(crit); + StringBuffer prop("Debug/"); + prop.append(lower); + if (overwrite || !p->hasProp(prop.str())) + { + // MORE - not sure this line should be needed.... + p->setProp("Debug", ""); + p->setProp(prop.str(), value); + } + } + catch (IException * e) + { + if (!validateXMLTag(propname)) + { + e->Release(); + throw makeStringExceptionV(WUERR_InvalidDebugValueName, "Attempt to set debug value with invalid name: %s", propname); + } + throw; } } @@ -8844,14 +8856,26 @@ void CLocalWorkUnit::setDebugValueInt(const char *propname, int value, bool over lower.append(propname); if (!strchr(lower, ':')) lower.toLowerCase(); - CriticalBlock block(crit); - StringBuffer prop("Debug/"); - prop.append(lower); - if (overwrite || !p->hasProp(prop.str())) + try { - // MORE - not sure this line should be needed.... - p->setProp("Debug", ""); - p->setPropInt(prop.str(), value); + CriticalBlock block(crit); + StringBuffer prop("Debug/"); + prop.append(lower); + if (overwrite || !p->hasProp(prop.str())) + { + // MORE - not sure this line should be needed.... + p->setProp("Debug", ""); + p->setPropInt(prop.str(), value); + } + } + catch (IException * e) + { + if (!validateXMLTag(propname)) + { + e->Release(); + throw makeStringExceptionV(WUERR_InvalidDebugValueName, "Attempt to set debug value with invalid name: %s", propname); + } + throw; } } diff --git a/common/workunit/workunit.hpp b/common/workunit/workunit.hpp index ae94e98d447..743672a82f3 100644 --- a/common/workunit/workunit.hpp +++ b/common/workunit/workunit.hpp @@ -59,6 +59,7 @@ enum : unsigned { WUERR_ModifyFilterAfterFinalize = WORKUNIT_ERROR_START, WUERR_FinalizeAfterFinalize, + WUERR_InvalidDebugValueName, }; // error codes diff --git a/dali/base/dacoven.cpp b/dali/base/dacoven.cpp index 8aa60877944..fc5661219ed 100644 --- a/dali/base/dacoven.cpp +++ b/dali/base/dacoven.cpp @@ -1067,33 +1067,40 @@ DALI_UID getGlobalUniqueIds(unsigned num,SocketEndpoint *_foreignnode) { if (num==0) return 0; - if (coven) - return coven->getUniqueIds(num,_foreignnode); - if (!_foreignnode||_foreignnode->isNull()) - throw MakeStringException(99,"getUniqueIds: Not connected to dali"); - SocketEndpoint foreignnode; - foreignnode.set(*_foreignnode); - if (foreignnode.port==0) - foreignnode.port=DALI_SERVER_PORT; - CDaliUidAllocator &uidAllocator = CDaliUidAllocator::find(CCovenBase::foreginUidallocators,foreignnode); - DALI_UID uid; - CriticalBlock block(uidAllocator.crit); - while (!uidAllocator.allocUIDs(uid,num)) { - unsigned n = uidAllocator.getBankSize(); - if (n foreign; - Owned group = createIGroup(1,&foreignnode); - foreign.setown(createCommunicator(group)); - foreign->sendRecv(mb,RANK_RANDOM,MPTAG_DALI_COVEN_REQUEST); - mb.read(next); - if ((next==0)&&mb.remaining()) // server exception - throw deserializeException(mb); - uidAllocator.addUIDs((__uint64)next,n); - } - return uid; + try + { + if (coven) + return coven->getUniqueIds(num,_foreignnode); + if (!_foreignnode||_foreignnode->isNull()) + throw MakeStringException(99,"getUniqueIds: Not connected to dali"); + SocketEndpoint foreignnode; + foreignnode.set(*_foreignnode); + if (foreignnode.port==0) + foreignnode.port=DALI_SERVER_PORT; + CDaliUidAllocator &uidAllocator = CDaliUidAllocator::find(CCovenBase::foreginUidallocators,foreignnode); + DALI_UID uid; + CriticalBlock block(uidAllocator.crit); + while (!uidAllocator.allocUIDs(uid,num)) { + unsigned n = uidAllocator.getBankSize(); + if (n foreign; + Owned group = createIGroup(1,&foreignnode); + foreign.setown(createCommunicator(group)); + foreign->sendRecv(mb,RANK_RANDOM,MPTAG_DALI_COVEN_REQUEST); + mb.read(next); + if ((next==0)&&mb.remaining()) // server exception + throw deserializeException(mb); + uidAllocator.addUIDs((__uint64)next,n); + } + return uid; + } + catch (IException *e) + { + throw makeWrappedException(e, e->errorCode(), "getGlobalUniqueIds"); + } } diff --git a/esp/services/ws_workunits/ws_workunitsHelpers.cpp b/esp/services/ws_workunits/ws_workunitsHelpers.cpp index 62000fcf450..6532f42acdc 100644 --- a/esp/services/ws_workunits/ws_workunitsHelpers.cpp +++ b/esp/services/ws_workunits/ws_workunitsHelpers.cpp @@ -3741,81 +3741,93 @@ void WsWuHelpers::submitWsWorkunit(IEspContext& context, IConstWorkUnit* cw, con if(!wu.get()) throw MakeStringException(ECLWATCH_CANNOT_UPDATE_WORKUNIT, "Cannot update workunit %s.", wuid.str()); - wu->clearExceptions(); - if(notEmpty(cluster)) - wu->setClusterName(cluster); - if(notEmpty(snapshot)) - wu->setSnapshot(snapshot); - wu->setState(WUStateSubmitted); - if (maxruntime) - wu->setDebugValueInt("maxRunTime",maxruntime,true); - if (maxcost) - wu->setDebugValueInt("maxCost", maxcost, true); - if (debugs && debugs->length()) - { - ForEachItemIn(i, *debugs) + try + { + wu->clearExceptions(); + if(notEmpty(cluster)) + wu->setClusterName(cluster); + if(notEmpty(snapshot)) + wu->setSnapshot(snapshot); + wu->setState(WUStateSubmitted); + if (maxruntime) + wu->setDebugValueInt("maxRunTime",maxruntime,true); + if (maxcost) + wu->setDebugValueInt("maxCost", maxcost, true); + if (debugs && debugs->length()) { - IConstNamedValue &item = debugs->item(i); - const char *name = item.getName(); - const char *value = item.getValue(); - if (!name || !*name) - continue; - StringBuffer expanded; - if (*name=='-') - name=expanded.append("eclcc").append(name).str(); - if (!value) + ForEachItemIn(i, *debugs) { - size_t len = strlen(name); - char last = name[len-1]; - if (last == '-' || last == '+') + IConstNamedValue &item = debugs->item(i); + const char *name = item.getName(); + const char *value = item.getValue(); + if (!name || !*name) + continue; + StringBuffer expanded; + if (*name=='-') + name=expanded.append("eclcc").append(name).str(); + if (!value) { - StringAttr s(name, len-1); - wu->setDebugValueInt(s.get(), last == '+' ? 1 : 0, true); + size_t len = strlen(name); + char last = name[len-1]; + if (last == '-' || last == '+') + { + StringAttr s(name, len-1); + wu->setDebugValueInt(s.get(), last == '+' ? 1 : 0, true); + } + else + wu->setDebugValueInt(name, 1, true); + continue; } - else - wu->setDebugValueInt(name, 1, true); - continue; + wu->setDebugValue(name, value, true); } - wu->setDebugValue(name, value, true); } - } - if (applications) - { - ForEachItemIn(ii, *applications) + if (applications) { - IConstApplicationValue& item = applications->item(ii); - if(notEmpty(item.getApplication()) && notEmpty(item.getName())) - wu->setApplicationValue(item.getApplication(), item.getName(), item.getValue(), true); + ForEachItemIn(ii, *applications) + { + IConstApplicationValue& item = applications->item(ii); + if(notEmpty(item.getApplication()) && notEmpty(item.getName())) + wu->setApplicationValue(item.getApplication(), item.getName(), item.getValue(), true); + } } - } - ISpan * activeSpan = context.queryActiveSpan(); - OwnedSpanScope clientSpan(activeSpan->createClientSpan("run_workunit")); - Owned httpHeaders = ::getClientHeaders(clientSpan); - recordTraceDebugOptions(wu, httpHeaders); + ISpan * activeSpan = context.queryActiveSpan(); + OwnedSpanScope clientSpan(activeSpan->createClientSpan("run_workunit")); + Owned httpHeaders = ::getClientHeaders(clientSpan); + recordTraceDebugOptions(wu, httpHeaders); - if (resetWorkflow) - wu->resetWorkflow(); - if (!compile) - wu->schedule(); + if (resetWorkflow) + wu->resetWorkflow(); + if (!compile) + wu->schedule(); - if (resetVariables) - { - SCMStringBuffer varname; - Owned vars = &wu->getVariables(); - ForEach (*vars) + if (resetVariables) { - vars->query().getResultName(varname); - Owned v = wu->updateVariableByName(varname.str()); - if (v) - v->setResultStatus(ResultStatusUndefined); + SCMStringBuffer varname; + Owned vars = &wu->getVariables(); + ForEach (*vars) + { + vars->query().getResultName(varname); + Owned v = wu->updateVariableByName(varname.str()); + if (v) + v->setResultStatus(ResultStatusUndefined); + } } - } - setXmlParameters(wu, paramXml, variables, (wu->getAction()==WUActionExecuteExisting)); + setXmlParameters(wu, paramXml, variables, (wu->getAction()==WUActionExecuteExisting)); + wu->commit(); + } + catch (IException * e) + { + //An exception occurred when setting up the workunit e.g. an invalid debug value name. + wu->setState(WUStateFailed); + StringBuffer msg; + addExceptionToWorkunit(wu, SeverityError, "esp", e->errorCode(), e->errorMessage(msg).str(), nullptr, 0, 0, 0); + wu->commit(); + throw; + } - wu->commit(); wu.clear(); if (!compile) diff --git a/esp/src/eclwatch/templates/GetDFUWorkunitsWidget.html b/esp/src/eclwatch/templates/GetDFUWorkunitsWidget.html index da361ca3d54..f2d3eb9ec81 100644 --- a/esp/src/eclwatch/templates/GetDFUWorkunitsWidget.html +++ b/esp/src/eclwatch/templates/GetDFUWorkunitsWidget.html @@ -1,7 +1,7 @@
-
+
${i18n.Refresh}
diff --git a/esp/src/eclwatch/templates/HPCCPlatformFilesWidget.html b/esp/src/eclwatch/templates/HPCCPlatformFilesWidget.html index 6a6795ff9be..d7bf3127f4f 100644 --- a/esp/src/eclwatch/templates/HPCCPlatformFilesWidget.html +++ b/esp/src/eclwatch/templates/HPCCPlatformFilesWidget.html @@ -8,7 +8,7 @@
-
+
diff --git a/esp/src/src-react/components/InfoGrid.tsx b/esp/src/src-react/components/InfoGrid.tsx index 160a0dc0271..da3a212ba8b 100644 --- a/esp/src/src-react/components/InfoGrid.tsx +++ b/esp/src/src-react/components/InfoGrid.tsx @@ -5,7 +5,6 @@ import { formatCost, formatTwoDigits } from "src/Session"; import nlsHPCC from "src/nlsHPCC"; import { useWorkunitExceptions } from "../hooks/workunit"; import { FluentGrid, useCopyButtons, useFluentStoreState, FluentColumns } from "./controls/Grid"; -import { pivotItemStyle } from "../layouts/pivot"; function extractGraphInfo(msg) { const regex = /^([a-zA-Z0-9 :]+: )(graph graph(\d+)\[(\d+)\], )(([a-zA-Z]+)\[(\d+)\]: )?(.*)$/gmi; @@ -237,23 +236,19 @@ export const InfoGrid: React.FunctionComponent = ({ } }, [data.length]); - return
+ return
{({ size }) => -
-
- { }} - setTotal={setTotal} - refresh={refreshTable} - height={`${size.height - (44 + 8 + 45 + 12)}px`} - selectionMode={SelectionMode.none} - > -
-
+ { }} + setTotal={setTotal} + refresh={refreshTable} + height={`${size.height - (44 + 8 + 45 + 12)}px`} + selectionMode={SelectionMode.none} + > }
; }; diff --git a/esp/src/src-react/components/LogViewer.tsx b/esp/src/src-react/components/LogViewer.tsx index 82e027a55b4..2e44c369051 100644 --- a/esp/src/src-react/components/LogViewer.tsx +++ b/esp/src/src-react/components/LogViewer.tsx @@ -102,13 +102,16 @@ export const LogViewer: React.FunctionComponent = ({ return } main={ - } +
+ +
+ } />; }; diff --git a/esp/src/src-react/components/Menu.tsx b/esp/src/src-react/components/Menu.tsx index 7f291c40944..ea6b9601d00 100644 --- a/esp/src/src-react/components/Menu.tsx +++ b/esp/src/src-react/components/Menu.tsx @@ -4,6 +4,7 @@ import { useConst } from "@fluentui/react-hooks"; import nlsHPCC from "src/nlsHPCC"; import { hasLogAccess } from "src/ESPLog"; import { containerized, bare_metal } from "src/BuildInfo"; +import { navCategory } from "../util/history"; import { MainNav, routes } from "../routes"; import { useFavorite, useFavorites, useHistory } from "../hooks/favorite"; import { useSessionStore } from "../hooks/store"; @@ -98,7 +99,7 @@ routes.forEach((route: any) => { }); function navSelectedKey(hashPath) { - const rootPath = navIdx(`/${hashPath?.split("/")[1]}`); + const rootPath = navIdx(`/${navCategory(hashPath)?.split("/")[1]}`); if (rootPath?.length) { return rootPath[0]; } @@ -164,7 +165,7 @@ const subMenuItems: SubMenuItems = { "files": [ { headerText: nlsHPCC.LogicalFiles, itemKey: "/files" }, { headerText: nlsHPCC.LandingZones, itemKey: "/landingzone" }, - { headerText: nlsHPCC.Workunits, itemKey: "/dfuworkunits" }, + { headerText: nlsHPCC.title_GetDFUWorkunits, itemKey: "/dfuworkunits" }, { headerText: nlsHPCC.XRef + " (L)", itemKey: "/xref" }, ], "queries": [ @@ -215,8 +216,8 @@ for (const key in subMenuItems) { } function subNavSelectedKey(hashPath) { - const hashCategory = hashPath.split("/").slice(0, 3).join("/"); - return subNavIdx(hashCategory).length ? hashCategory : null; + const category = navCategory(hashPath); + return subNavIdx(category).length ? category : null; } interface SubNavigationProps { diff --git a/esp/src/src-react/components/SourceFiles.tsx b/esp/src/src-react/components/SourceFiles.tsx index ed1e083edfb..cda748e8845 100644 --- a/esp/src/src-react/components/SourceFiles.tsx +++ b/esp/src/src-react/components/SourceFiles.tsx @@ -25,7 +25,7 @@ interface SourceFilesProps { } const emptyFilter: { [id: string]: any } = {}; -const defaultSort = { attribute: "Name", descending: false }; +const defaultSort = { attribute: undefined, descending: false }; export const SourceFiles: React.FunctionComponent = ({ wuid, @@ -52,7 +52,7 @@ export const SourceFiles: React.FunctionComponent = ({ selectorType: "checkbox" }, Name: { - label: "Name", sortable: true, + label: "Name", width: 400, sortable: true, formatter: (Name, row) => { let fileUrl = `#/files/${Name}`; if (row?.FileCluster) { @@ -65,11 +65,8 @@ export const SourceFiles: React.FunctionComponent = ({ ; } }, - FileCluster: { label: nlsHPCC.FileCluster, width: 300, sortable: false }, - Count: { - label: nlsHPCC.Usage, width: 72, sortable: true, - justify: "right" - } + FileCluster: { label: nlsHPCC.FileCluster, width: 200, sortable: false }, + Count: { label: nlsHPCC.Usage, width: 72, sortable: true, justify: "right" } }; }, []); diff --git a/esp/src/src-react/components/controls/Grid.tsx b/esp/src/src-react/components/controls/Grid.tsx index 770b5b759ba..129271d47b3 100644 --- a/esp/src/src-react/components/controls/Grid.tsx +++ b/esp/src/src-react/components/controls/Grid.tsx @@ -1,5 +1,5 @@ import * as React from "react"; -import { DetailsList, DetailsListLayoutMode, Dropdown, IColumn as _IColumn, ICommandBarItemProps, IDetailsHeaderProps, IDetailsListStyles, mergeStyleSets, Selection, Stack, TooltipHost, TooltipOverflowMode, IRenderFunction, IDetailsRowProps, SelectionMode, ConstrainMode, ISelection } from "@fluentui/react"; +import { DetailsList, DetailsListLayoutMode, Dropdown, IColumn as _IColumn, ICommandBarItemProps, IDetailsHeaderProps, IDetailsListStyles, mergeStyleSets, Selection, Stack, TooltipHost, TooltipOverflowMode, IRenderFunction, IDetailsRowProps, SelectionMode, ConstrainMode, ISelection, ScrollablePane, Sticky } from "@fluentui/react"; import { Pagination } from "@fluentui/react-experiments/lib/Pagination"; import { useConst } from "@fluentui/react-hooks"; import { BaseStore, Memory, QueryRequest, QuerySortItem } from "src/store/Memory"; @@ -293,45 +293,22 @@ const FluentStoreGrid: React.FunctionComponent = ({ }, [memoizedColumns]); const renderDetailsHeader = React.useCallback((props: IDetailsHeaderProps, defaultRender?: any) => { - return defaultRender({ - ...props, - onRenderColumnHeaderTooltip: (tooltipHostProps) => { - return ; - }, - styles: { root: { paddingTop: 1 } } - }); + return + {defaultRender({ + ...props, + onRenderColumnHeaderTooltip: (tooltipHostProps) => { + return ; + }, + styles: { root: { paddingTop: 1 } } + })} + ; }, []); const columnResize = React.useCallback((column: IColumn, newWidth: number, columnIndex?: number) => { columnWidths.set(column.key, newWidth); }, [columnWidths]); - /* Monitor Scroll Events (hack) - - Essentially we are setting the scrollElement of the DetailsList to the div that contains the DetailsList (rather than a scrollable pane host). - See: https://github.com/microsoft/fluentui/blob/55d3a31042e8972ea373841bef616c68e6ab69f9/packages/react/src/components/List/List.tsx#L355-L369 - - Note: Not sure if `_onScroll` call is needed, but excluding for now as it seems to work without it and is more performant. - */ - // const id = useId("fluent-store-grid-"); - // const detailListScrollComponent = React.useRef(null); - // const detailListComponent = React.useRef(null); - // const [detailListElement, setDetailListElement] = React.useState(null); - // useMount(() => { - // const detailListElement = document.querySelector(`#${id} .ms-DetailsList`); - // setDetailListElement(detailListElement); - // if (detailListComponent.current?._list?.current) { - // detailListComponent.current._list.current._scrollElement = detailListElement; - // } - // }); - // useOnEvent(detailListScrollComponent, "scroll", () => { - // detailListComponent.current?._list?.current?._onScroll(); - // }); - // useOnEvent(detailListScrollComponent, "scroll", () => { - // detailListComponent.current?._list?.current?._onAsyncScrollDebounced(); - // }); - - return
+ return = ({ styles={gridStyles(height)} selectionMode={selectionMode} /> -
; + ; }; interface FluentGridProps { diff --git a/esp/src/src-react/components/forms/AddBinding.tsx b/esp/src/src-react/components/forms/AddBinding.tsx index 33df4fa0127..abc26d8d4c9 100644 --- a/esp/src/src-react/components/forms/AddBinding.tsx +++ b/esp/src/src-react/components/forms/AddBinding.tsx @@ -1,5 +1,5 @@ import * as React from "react"; -import { DefaultButton, IDropdownOption, PrimaryButton, TextField, } from "@fluentui/react"; +import { DefaultButton, IDropdownOption, PrimaryButton, Spinner, TextField, } from "@fluentui/react"; import { scopedLogger } from "@hpcc-js/util"; import { useForm, Controller } from "react-hook-form"; import { EsdlDefinitionsTextField, EsdlEspProcessesTextField } from "./Fields"; @@ -39,6 +39,8 @@ export const AddBindingForm: React.FunctionComponent = ({ }) => { const { handleSubmit, control, reset } = useForm({ defaultValues }); + const [submitDisabled, setSubmitDisabled] = React.useState(false); + const [spinnerHidden, setSpinnerHidden] = React.useState(true); const closeForm = React.useCallback(() => { setShowForm(false); @@ -47,11 +49,16 @@ export const AddBindingForm: React.FunctionComponent = ({ const onSubmit = React.useCallback(() => { handleSubmit( (data, evt) => { + setSubmitDisabled(true); + setSpinnerHidden(false); + const request: any = data; request.Overwrite = true; WsESDLConfig.PublishESDLBinding({ request: request }) .then(() => { + setSubmitDisabled(false); + setSpinnerHidden(true); closeForm(); reset(defaultValues); if (refreshGrid) refreshGrid(); @@ -66,7 +73,8 @@ export const AddBindingForm: React.FunctionComponent = ({ return - + + closeForm()} /> }> = ({ }) => { const { handleSubmit, control, reset } = useForm({ defaultValues }); + const [submitDisabled, setSubmitDisabled] = React.useState(false); + const [spinnerHidden, setSpinnerHidden] = React.useState(true); const closeForm = React.useCallback(() => { setShowForm(false); @@ -41,10 +43,14 @@ export const AddGroupForm: React.FunctionComponent = ({ const onSubmit = React.useCallback(() => { handleSubmit( (data, evt) => { + setSubmitDisabled(true); + setSpinnerHidden(false); const request: any = data; WsAccess.GroupAdd({ request: request }) .then(() => { + setSubmitDisabled(false); + setSpinnerHidden(true); closeForm(); reset(defaultValues); if (refreshGrid) refreshGrid(); @@ -58,7 +64,8 @@ export const AddGroupForm: React.FunctionComponent = ({ return - + + { reset(defaultValues); closeForm(); }} /> }> { const { handleSubmit, control, reset } = useForm({ defaultValues }); + const [submitDisabled, setSubmitDisabled] = React.useState(false); + const [spinnerHidden, setSpinnerHidden] = React.useState(true); const closeForm = React.useCallback(() => { setShowForm(false); @@ -51,6 +53,8 @@ export const AddGroupResourceForm: React.FunctionComponent { handleSubmit( (data, evt) => { + setSubmitDisabled(true); + setSpinnerHidden(false); const request: any = data; request["action"] = "update"; @@ -60,6 +64,8 @@ export const AddGroupResourceForm: React.FunctionComponent { + setSubmitDisabled(false); + setSpinnerHidden(true); closeForm(); reset(defaultValues); if (refreshGrid) refreshGrid(); @@ -73,7 +79,8 @@ export const AddGroupResourceForm: React.FunctionComponent - + + { reset(defaultValues); closeForm(); }} /> }> = ({ }) => { const { handleSubmit, control, reset } = useForm({ defaultValues }); + const [submitDisabled, setSubmitDisabled] = React.useState(false); + const [spinnerHidden, setSpinnerHidden] = React.useState(true); const [remoteTargets, setRemoteTargets] = React.useState([]); @@ -67,15 +69,21 @@ export const AddPackageMap: React.FunctionComponent = ({ const onSubmit = React.useCallback(() => { handleSubmit( (data, evt) => { + setSubmitDisabled(true); + setSpinnerHidden(false); WsPackageMaps.AddPackage({ request: data }) .then(({ AddPackageResponse, Exceptions }) => { if (AddPackageResponse?.status?.Code === 0) { + setSubmitDisabled(false); + setSpinnerHidden(true); closeForm(); refreshData(true); reset(defaultValues); } else if (Exceptions) { + setSubmitDisabled(false); + setSpinnerHidden(true); closeForm(); logger.error(Exceptions.Exception[0].Message); } @@ -99,7 +107,8 @@ export const AddPackageMap: React.FunctionComponent = ({ return - + + closeForm()} /> }> diff --git a/esp/src/src-react/components/forms/AddPackageMapPart.tsx b/esp/src/src-react/components/forms/AddPackageMapPart.tsx index cae4d80a7c1..23193c02259 100644 --- a/esp/src/src-react/components/forms/AddPackageMapPart.tsx +++ b/esp/src/src-react/components/forms/AddPackageMapPart.tsx @@ -1,5 +1,5 @@ import * as React from "react"; -import { Checkbox, DefaultButton, PrimaryButton, Stack, TextField, } from "@fluentui/react"; +import { Checkbox, DefaultButton, PrimaryButton, Spinner, Stack, TextField, } from "@fluentui/react"; import { useForm, Controller } from "react-hook-form"; import { scopedLogger } from "@hpcc-js/util"; import * as WsPackageMaps from "src/WsPackageMaps"; @@ -50,6 +50,8 @@ export const AddPackageMapPart: React.FunctionComponent refreshData, }) => { const { handleSubmit, control, reset } = useForm({ defaultValues }); + const [submitDisabled, setSubmitDisabled] = React.useState(false); + const [spinnerHidden, setSpinnerHidden] = React.useState(true); const closeForm = React.useCallback(() => { setShowForm(false); @@ -58,15 +60,21 @@ export const AddPackageMapPart: React.FunctionComponent const onSubmit = React.useCallback(() => { handleSubmit( (data, evt) => { + setSubmitDisabled(true); + setSpinnerHidden(false); WsPackageMaps.AddPartToPackageMap({ request: { ...data, Target: target, PackageMap: packageMap } }) .then(({ AddPartToPackageMapResponse, Exceptions }) => { if (AddPartToPackageMapResponse?.status?.Code === 0) { + setSubmitDisabled(false); + setSpinnerHidden(true); closeForm(); if (refreshData) refreshData(); reset(defaultValues); } else if (Exceptions) { + setSubmitDisabled(false); + setSpinnerHidden(true); closeForm(); logger.error(Exceptions.Exception[0].Message); } @@ -82,7 +90,8 @@ export const AddPackageMapPart: React.FunctionComponent return - + + closeForm()} /> }> diff --git a/esp/src/src-react/components/forms/AddPermission.tsx b/esp/src/src-react/components/forms/AddPermission.tsx index 5b44b6eb7e2..40a31d8a604 100644 --- a/esp/src/src-react/components/forms/AddPermission.tsx +++ b/esp/src/src-react/components/forms/AddPermission.tsx @@ -1,5 +1,5 @@ import * as React from "react"; -import { DefaultButton, IDropdownOption, MessageBar, MessageBarType, PrimaryButton, TextField, } from "@fluentui/react"; +import { DefaultButton, IDropdownOption, MessageBar, MessageBarType, PrimaryButton, Spinner, TextField, } from "@fluentui/react"; import { scopedLogger } from "@hpcc-js/util"; import { useForm, Controller } from "react-hook-form"; import nlsHPCC from "src/nlsHPCC"; @@ -34,6 +34,8 @@ export const AddPermissionForm: React.FunctionComponent }) => { const { handleSubmit, control, reset } = useForm({ defaultValues }); + const [submitDisabled, setSubmitDisabled] = React.useState(false); + const [spinnerHidden, setSpinnerHidden] = React.useState(true); const [showError, setShowError] = React.useState(false); const [errorMessage, setErrorMessage] = React.useState(""); @@ -45,6 +47,8 @@ export const AddPermissionForm: React.FunctionComponent const onSubmit = React.useCallback(() => { handleSubmit( (data, evt) => { + setSubmitDisabled(true); + setSpinnerHidden(false); const request: any = data; WsAccess.ResourceAdd({ request: request }) @@ -52,9 +56,13 @@ export const AddPermissionForm: React.FunctionComponent if (ResourceAddResponse?.retcode < 0) { //log exception from API setShowError(true); + setSubmitDisabled(false); + setSpinnerHidden(true); setErrorMessage(ResourceAddResponse?.retmsg); logger.error(ResourceAddResponse?.retmsg); } else { + setSubmitDisabled(false); + setSpinnerHidden(true); closeForm(); reset(defaultValues); if (refreshGrid) refreshGrid(); @@ -69,7 +77,8 @@ export const AddPermissionForm: React.FunctionComponent return - + + { reset(defaultValues); closeForm(); }} /> }> = ({ }) => { const { handleSubmit, control, reset } = useForm({ defaultValues }); + const [submitDisabled, setSubmitDisabled] = React.useState(false); + const [spinnerHidden, setSpinnerHidden] = React.useState(true); const closeForm = React.useCallback(() => { setShowForm(false); @@ -56,8 +58,12 @@ export const AddToSuperfile: React.FunctionComponent = ({ const onSubmit = React.useCallback(() => { handleSubmit( (data, evt) => { + setSubmitDisabled(true); + setSpinnerHidden(false); WsDfu.AddtoSuperfile(data.names, data.superFile, data.existingFile) .then(response => { + setSubmitDisabled(false); + setSpinnerHidden(true); closeForm(); if (refreshGrid) refreshGrid(true); }) @@ -83,7 +89,8 @@ export const AddToSuperfile: React.FunctionComponent = ({ return - + + closeForm()} /> }> diff --git a/esp/src/src-react/components/forms/AddUser.tsx b/esp/src/src-react/components/forms/AddUser.tsx index b74977e1403..377025fdabd 100644 --- a/esp/src/src-react/components/forms/AddUser.tsx +++ b/esp/src/src-react/components/forms/AddUser.tsx @@ -1,5 +1,5 @@ import * as React from "react"; -import { DefaultButton, MessageBar, MessageBarType, PrimaryButton, TextField, } from "@fluentui/react"; +import { DefaultButton, MessageBar, MessageBarType, PrimaryButton, Spinner, TextField, } from "@fluentui/react"; import { scopedLogger } from "@hpcc-js/util"; import { useForm, Controller } from "react-hook-form"; import nlsHPCC from "src/nlsHPCC"; @@ -41,6 +41,8 @@ export const AddUserForm: React.FunctionComponent = ({ }) => { const { handleSubmit, control, reset, watch } = useForm({ defaultValues }); + const [submitDisabled, setSubmitDisabled] = React.useState(false); + const [spinnerHidden, setSpinnerHidden] = React.useState(true); const pwd1 = watch("password1"); @@ -54,6 +56,8 @@ export const AddUserForm: React.FunctionComponent = ({ const onSubmit = React.useCallback(() => { handleSubmit( (data, evt) => { + setSubmitDisabled(true); + setSpinnerHidden(false); const request: any = data; WsAccess.AddUser({ request: request }) @@ -61,9 +65,13 @@ export const AddUserForm: React.FunctionComponent = ({ if (AddUserResponse?.retcode < 0) { //log exception from API setShowError(true); + setSubmitDisabled(false); + setSpinnerHidden(true); setErrorMessage(AddUserResponse?.retmsg); logger.error(AddUserResponse?.retmsg); } else { + setSubmitDisabled(false); + setSpinnerHidden(true); closeForm(); reset(defaultValues); if (refreshGrid) refreshGrid(); @@ -78,7 +86,8 @@ export const AddUserForm: React.FunctionComponent = ({ return - + + { reset(defaultValues); closeForm(); }} /> }> = ({ }) => { const { handleSubmit, control, reset } = useForm({ defaultValues }); + const [submitDisabled, setSubmitDisabled] = React.useState(false); + const [spinnerHidden, setSpinnerHidden] = React.useState(true); const closeForm = React.useCallback(() => { setShowForm(false); @@ -64,10 +66,14 @@ export const CopyFile: React.FunctionComponent = ({ const onSubmit = React.useCallback(() => { handleSubmit( (data, evt) => { + setSubmitDisabled(true); + setSpinnerHidden(false); if (logicalFiles.length > 0) { if (logicalFiles.length === 1) { const request = { ...data, sourceLogicalName: logicalFiles[0] }; FileSpray.Copy({ request: request }).then(response => { + setSubmitDisabled(false); + setSpinnerHidden(true); closeForm(); pushUrl(`/dfuworkunits/${response.CopyResponse.result}`); }); @@ -77,6 +83,8 @@ export const CopyFile: React.FunctionComponent = ({ const requests = []; requests.push(FileSpray.Copy({ request: request })); Promise.all(requests).then(_ => { + setSubmitDisabled(false); + setSpinnerHidden(true); closeForm(); if (refreshGrid) refreshGrid(); }); @@ -115,7 +123,8 @@ export const CopyFile: React.FunctionComponent = ({ return - + + closeForm()} /> }> diff --git a/esp/src/src-react/components/forms/DesprayFile.tsx b/esp/src/src-react/components/forms/DesprayFile.tsx index b57a171d0d1..ee70122c138 100644 --- a/esp/src/src-react/components/forms/DesprayFile.tsx +++ b/esp/src/src-react/components/forms/DesprayFile.tsx @@ -1,5 +1,5 @@ import * as React from "react"; -import { Checkbox, DefaultButton, IDropdownOption, mergeStyleSets, PrimaryButton, Stack, TextField, } from "@fluentui/react"; +import { Checkbox, DefaultButton, IDropdownOption, mergeStyleSets, PrimaryButton, Spinner, Stack, TextField, } from "@fluentui/react"; import { useForm, Controller } from "react-hook-form"; import { FileSpray, FileSprayService } from "@hpcc-js/comms"; import { scopedLogger } from "@hpcc-js/util"; @@ -67,6 +67,8 @@ export const DesprayFile: React.FunctionComponent = ({ const [os, setOs] = React.useState(); const { handleSubmit, control, reset } = useForm({ defaultValues }); + const [submitDisabled, setSubmitDisabled] = React.useState(false); + const [spinnerHidden, setSpinnerHidden] = React.useState(true); const closeForm = React.useCallback(() => { setShowForm(false); @@ -75,6 +77,8 @@ export const DesprayFile: React.FunctionComponent = ({ const onSubmit = React.useCallback(() => { handleSubmit( (data, evt) => { + setSubmitDisabled(true); + setSpinnerHidden(false); if (logicalFiles.length > 0) { if (logicalFiles.length === 1) { const request = { @@ -84,6 +88,8 @@ export const DesprayFile: React.FunctionComponent = ({ sourceLogicalName: logicalFiles[0] } as FileSpray.Despray; myFileSprayService.Despray(request).then(response => { + setSubmitDisabled(false); + setSpinnerHidden(true); closeForm(); reset(defaultValues); if (refreshGrid) refreshGrid(true); @@ -99,6 +105,8 @@ export const DesprayFile: React.FunctionComponent = ({ requests.push(myFileSprayService.Despray(request)); }); Promise.all(requests).then(_ => { + setSubmitDisabled(false); + setSpinnerHidden(true); closeForm(); if (refreshGrid) refreshGrid(true); }).catch(err => logger.error(err)); @@ -136,7 +144,8 @@ export const DesprayFile: React.FunctionComponent = ({ return - + + closeForm()} /> }> diff --git a/esp/src/src-react/components/forms/GroupAddUser.tsx b/esp/src/src-react/components/forms/GroupAddUser.tsx index c8f1cef78b9..518231f1d17 100644 --- a/esp/src/src-react/components/forms/GroupAddUser.tsx +++ b/esp/src/src-react/components/forms/GroupAddUser.tsx @@ -1,5 +1,5 @@ import * as React from "react"; -import { DefaultButton, IDropdownOption, MessageBar, MessageBarType, PrimaryButton, } from "@fluentui/react"; +import { DefaultButton, IDropdownOption, MessageBar, MessageBarType, PrimaryButton, Spinner, } from "@fluentui/react"; import { scopedLogger } from "@hpcc-js/util"; import { useForm, Controller } from "react-hook-form"; import nlsHPCC from "src/nlsHPCC"; @@ -32,6 +32,8 @@ export const GroupAddUserForm: React.FunctionComponent = ({ }) => { const { handleSubmit, control, reset } = useForm({ defaultValues }); + const [submitDisabled, setSubmitDisabled] = React.useState(false); + const [spinnerHidden, setSpinnerHidden] = React.useState(true); const [showError, setShowError] = React.useState(false); const [errorMessage, setErrorMessage] = React.useState(""); @@ -43,6 +45,8 @@ export const GroupAddUserForm: React.FunctionComponent = ({ const onSubmit = React.useCallback(() => { handleSubmit( (data, evt) => { + setSubmitDisabled(true); + setSpinnerHidden(false); const request: any = data; request.groupname = groupname; request.action = "add"; @@ -52,9 +56,13 @@ export const GroupAddUserForm: React.FunctionComponent = ({ if (GroupMemberEditResponse?.retcode < 0) { //log exception from API setShowError(true); + setSubmitDisabled(false); + setSpinnerHidden(true); setErrorMessage(GroupMemberEditResponse?.retmsg); logger.error(GroupMemberEditResponse?.retmsg); } else { + setSubmitDisabled(false); + setSpinnerHidden(true); closeForm(); reset(defaultValues); if (refreshGrid) refreshGrid(); @@ -69,7 +77,8 @@ export const GroupAddUserForm: React.FunctionComponent = ({ return - + + { reset(defaultValues); closeForm(); }} /> }> = ({ const [workunit] = useWorkunit(wuid); const { handleSubmit, control, reset } = useForm({ defaultValues }); + const [submitDisabled, setSubmitDisabled] = React.useState(false); + const [spinnerHidden, setSpinnerHidden] = React.useState(true); const closeForm = React.useCallback(() => { setShowForm(false); @@ -54,6 +56,8 @@ export const PublishQueryForm: React.FunctionComponent = ({ const onSubmit = React.useCallback(() => { handleSubmit( (data, evt) => { + setSubmitDisabled(true); + setSpinnerHidden(false); const request = { Wuid: workunit?.Wuid, Cluster: workunit?.Cluster, @@ -70,6 +74,8 @@ export const PublishQueryForm: React.FunctionComponent = ({ workunit.publishEx(request).then(() => { return workunit.update({ Jobname: data.jobName }); }).then(() => { + setSubmitDisabled(false); + setSpinnerHidden(true); closeForm(); reset(defaultValues); }).catch(err => logger.error(err)); @@ -86,7 +92,8 @@ export const PublishQueryForm: React.FunctionComponent = ({ return - + + closeForm()} /> }> = ({ }) => { const { handleSubmit, control, reset } = useForm({ defaultValues }); + const [submitDisabled, setSubmitDisabled] = React.useState(false); + const [spinnerHidden, setSpinnerHidden] = React.useState(true); const [showError, setShowError] = React.useState(false); const [errorMessage, setErrorMessage] = React.useState(""); @@ -40,6 +42,8 @@ export const PushEventForm: React.FunctionComponent = ({ const onSubmit = React.useCallback(() => { handleSubmit( (data, evt) => { + setSubmitDisabled(true); + setSpinnerHidden(false); const request: any = data; WsWorkunits.WUPushEvent({ request: request }) @@ -47,9 +51,13 @@ export const PushEventForm: React.FunctionComponent = ({ if (WUPushEventResponse?.retcode < 0) { //log exception from API setShowError(true); + setSubmitDisabled(false); + setSpinnerHidden(true); setErrorMessage(WUPushEventResponse?.retmsg); logger.error(WUPushEventResponse?.retmsg); } else { + setSubmitDisabled(false); + setSpinnerHidden(true); closeForm(); reset(defaultValues); } @@ -62,7 +70,8 @@ export const PushEventForm: React.FunctionComponent = ({ return - + + { reset(defaultValues); closeForm(); }} /> }> = ({ const [showError, setShowError] = React.useState(false); const [errorMessage, setErrorMessage] = React.useState(""); - + const [submitDisabled, setSubmitDisabled] = React.useState(false); + const [spinnerHidden, setSpinnerHidden] = React.useState(true); const [selectedDestGroup, setSelectedDestGroup] = React.useState(""); const [replicateDisabled, setReplicateDisabled] = React.useState(true); const onSubmit = React.useCallback(() => { handleSubmit( (data, evt) => { + setSubmitDisabled(true); + setSpinnerHidden(false); + FileSpray.Copy({ request: data }).then(({ CopyResponse, Exceptions }) => { if (Exceptions?.Exception) { + setSubmitDisabled(false); + setSpinnerHidden(true); setShowError(true); setErrorMessage(Exceptions?.Exception[0]?.Message); } else { + setSubmitDisabled(false); + setSpinnerHidden(true); setShowForm(false); reset(defaultValues); if (refreshGrid) refreshGrid(true); @@ -113,7 +121,8 @@ export const RemoteCopy: React.FunctionComponent = ({ return - + + setShowForm(false)} /> }> {showError && diff --git a/esp/src/src-react/components/forms/RenameFile.tsx b/esp/src/src-react/components/forms/RenameFile.tsx index eeedfba61a3..b003f1cc280 100644 --- a/esp/src/src-react/components/forms/RenameFile.tsx +++ b/esp/src/src-react/components/forms/RenameFile.tsx @@ -1,5 +1,5 @@ import * as React from "react"; -import { Checkbox, DefaultButton, mergeStyleSets, PrimaryButton, Stack, TextField, } from "@fluentui/react"; +import { Checkbox, DefaultButton, mergeStyleSets, PrimaryButton, Spinner, Stack, TextField, } from "@fluentui/react"; import { useConst } from "@fluentui/react-hooks"; import { useForm, Controller } from "react-hook-form"; import { FileSprayService, FileSprayStates } from "@hpcc-js/comms"; @@ -39,6 +39,8 @@ export const RenameFile: React.FunctionComponent = ({ }) => { const { handleSubmit, control, reset } = useForm({ defaultValues }); + const [submitDisabled, setSubmitDisabled] = React.useState(false); + const [spinnerHidden, setSpinnerHidden] = React.useState(true); const service = useConst(() => new FileSprayService({ baseUrl: "" })); @@ -49,6 +51,8 @@ export const RenameFile: React.FunctionComponent = ({ const onSubmit = React.useCallback(() => { handleSubmit( async (data, evt) => { + setSubmitDisabled(true); + setSpinnerHidden(false); const renameRequests = []; const getDfuWuRequests = []; @@ -87,6 +91,8 @@ export const RenameFile: React.FunctionComponent = ({ } } }); + setSubmitDisabled(false); + setSpinnerHidden(true); closeForm(); if (refreshGrid) refreshGrid(true); }, @@ -116,7 +122,8 @@ export const RenameFile: React.FunctionComponent = ({ return - + + closeForm()} /> }> diff --git a/esp/src/src-react/components/forms/ReplicateFile.tsx b/esp/src/src-react/components/forms/ReplicateFile.tsx index 4a8b61034f9..ed504ec3b23 100644 --- a/esp/src/src-react/components/forms/ReplicateFile.tsx +++ b/esp/src/src-react/components/forms/ReplicateFile.tsx @@ -1,5 +1,5 @@ import * as React from "react"; -import { DefaultButton, IDropdownOption, PrimaryButton, Stack, TextField, } from "@fluentui/react"; +import { DefaultButton, IDropdownOption, PrimaryButton, Spinner, Stack, TextField, } from "@fluentui/react"; import { useForm, Controller } from "react-hook-form"; import nlsHPCC from "src/nlsHPCC"; import * as FileSpray from "src/FileSpray"; @@ -37,6 +37,8 @@ export const ReplicateFile: React.FunctionComponent = ({ const [file] = useFile(cluster, logicalFile); const { handleSubmit, control, reset } = useForm({ defaultValues }); + const [submitDisabled, setSubmitDisabled] = React.useState(false); + const [spinnerHidden, setSpinnerHidden] = React.useState(true); const closeForm = React.useCallback(() => { setShowForm(false); @@ -45,8 +47,12 @@ export const ReplicateFile: React.FunctionComponent = ({ const onSubmit = React.useCallback(() => { handleSubmit( (data, evt) => { + setSubmitDisabled(true); + setSpinnerHidden(false); const request = { ...data, srcname: logicalFile }; FileSpray.Replicate({ request: request }).then(response => { + setSubmitDisabled(false); + setSpinnerHidden(true); closeForm(); pushUrl(`/dfuworkunits/${response?.ReplicateResponse?.wuid}`); }); @@ -64,7 +70,8 @@ export const ReplicateFile: React.FunctionComponent = ({ return - + + closeForm()} /> }> diff --git a/esp/src/src-react/components/forms/TitlebarConfig.tsx b/esp/src/src-react/components/forms/TitlebarConfig.tsx index b0682975468..59d49193871 100644 --- a/esp/src/src-react/components/forms/TitlebarConfig.tsx +++ b/esp/src/src-react/components/forms/TitlebarConfig.tsx @@ -1,5 +1,5 @@ import * as React from "react"; -import { Checkbox, ColorPicker, DefaultButton, getColorFromString, IColor, Label, PrimaryButton, TextField, TooltipHost } from "@fluentui/react"; +import { Checkbox, ColorPicker, DefaultButton, getColorFromString, IColor, Label, PrimaryButton, Spinner, TextField, TooltipHost } from "@fluentui/react"; import { useForm, Controller } from "react-hook-form"; import { MessageBox } from "../../layouts/MessageBox"; import { useGlobalStore } from "../../hooks/store"; @@ -32,6 +32,8 @@ export const TitlebarConfig: React.FunctionComponent = ({ setShowForm }) => { const { handleSubmit, control, reset } = useForm({ defaultValues }); + const [submitDisabled, setSubmitDisabled] = React.useState(false); + const [spinnerHidden, setSpinnerHidden] = React.useState(true); const [color, setColor] = React.useState(white); const updateColor = React.useCallback((evt: any, colorObj: IColor) => setColor(colorObj), []); const [showEnvironmentTitle, setShowEnvironmentTitle] = useGlobalStore("HPCCPlatformWidget_Toolbar_Active", toolbarThemeDefaults.active, true); @@ -45,6 +47,8 @@ export const TitlebarConfig: React.FunctionComponent = ({ const onSubmit = React.useCallback(() => { handleSubmit( (data, evt) => { + setSubmitDisabled(true); + setSpinnerHidden(false); const request: any = data; request.titlebarColor = color.str; @@ -52,6 +56,8 @@ export const TitlebarConfig: React.FunctionComponent = ({ setEnvironmentTitle(request?.environmentTitle); setTitlebarColor(request.titlebarColor); + setSubmitDisabled(false); + setSpinnerHidden(true); closeForm(); }, )(); @@ -78,7 +84,8 @@ export const TitlebarConfig: React.FunctionComponent = ({ return - + + { reset(defaultValues); closeForm(); }} /> { onReset(); }} /> }> diff --git a/esp/src/src-react/components/forms/UserAddGroup.tsx b/esp/src/src-react/components/forms/UserAddGroup.tsx index b09393c30f1..467271fdeb9 100644 --- a/esp/src/src-react/components/forms/UserAddGroup.tsx +++ b/esp/src/src-react/components/forms/UserAddGroup.tsx @@ -1,5 +1,5 @@ import * as React from "react"; -import { DefaultButton, IDropdownOption, MessageBar, MessageBarType, PrimaryButton, } from "@fluentui/react"; +import { DefaultButton, IDropdownOption, MessageBar, MessageBarType, PrimaryButton, Spinner, } from "@fluentui/react"; import { scopedLogger } from "@hpcc-js/util"; import { useForm, Controller } from "react-hook-form"; import nlsHPCC from "src/nlsHPCC"; @@ -32,6 +32,8 @@ export const UserAddGroupForm: React.FunctionComponent = ({ }) => { const { handleSubmit, control, reset } = useForm({ defaultValues }); + const [submitDisabled, setSubmitDisabled] = React.useState(false); + const [spinnerHidden, setSpinnerHidden] = React.useState(true); const [showError, setShowError] = React.useState(false); const [errorMessage, setErrorMessage] = React.useState(""); @@ -43,6 +45,8 @@ export const UserAddGroupForm: React.FunctionComponent = ({ const onSubmit = React.useCallback(() => { handleSubmit( (data, evt) => { + setSubmitDisabled(true); + setSpinnerHidden(false); const request: any = data; request.username = username; request.action = "add"; @@ -52,9 +56,13 @@ export const UserAddGroupForm: React.FunctionComponent = ({ if (UserGroupEditResponse?.retcode < 0) { //log exception from API setShowError(true); + setSubmitDisabled(false); + setSpinnerHidden(true); setErrorMessage(UserGroupEditResponse?.retmsg); logger.error(UserGroupEditResponse?.retmsg); } else { + setSubmitDisabled(false); + setSpinnerHidden(true); closeForm(); reset(defaultValues); if (refreshGrid) refreshGrid(); @@ -69,7 +77,8 @@ export const UserAddGroupForm: React.FunctionComponent = ({ return - + + { reset(defaultValues); closeForm(); }} /> }> = ({ }) => { const { handleSubmit, control, reset } = useForm({ defaultValues }); + const [submitDisabled, setSubmitDisabled] = React.useState(false); + const [spinnerHidden, setSpinnerHidden] = React.useState(true); const closeForm = React.useCallback(() => { setShowForm(false); @@ -39,6 +41,8 @@ export const AddFileForm: React.FunctionComponent = ({ const onSubmit = React.useCallback(() => { handleSubmit( (data, evt) => { + setSubmitDisabled(true); + setSpinnerHidden(false); const dropZone = { ...store.get(data.NetAddress), NetAddress: data.NetAddress @@ -57,6 +61,8 @@ export const AddFileForm: React.FunctionComponent = ({ }; store.addUserFile(file); refreshGrid(); + setSubmitDisabled(false); + setSpinnerHidden(true); closeForm(); reset(defaultValues); }, @@ -68,7 +74,8 @@ export const AddFileForm: React.FunctionComponent = ({ return - + + closeForm()} /> }> = ({ const [, { isContainer }] = useBuildInfo(); const { handleSubmit, control, reset } = useForm({ defaultValues }); + const [submitDisabled, setSubmitDisabled] = React.useState(false); + const [spinnerHidden, setSpinnerHidden] = React.useState(true); const closeForm = React.useCallback(() => { setShowForm(false); @@ -74,6 +76,8 @@ export const BlobImportForm: React.FunctionComponent = ({ const onSubmit = React.useCallback(() => { handleSubmit( (data, evt) => { + setSubmitDisabled(true); + setSpinnerHidden(false); let request = {}; const files = data.selectedFiles; @@ -115,6 +119,8 @@ export const BlobImportForm: React.FunctionComponent = ({ } }); if (errors.length === 0) { + setSubmitDisabled(false); + setSpinnerHidden(true); closeForm(); } } @@ -153,7 +159,8 @@ export const BlobImportForm: React.FunctionComponent = ({ return - + + closeForm()} /> }> diff --git a/esp/src/src-react/components/forms/landing-zone/DelimitedImportForm.tsx b/esp/src/src-react/components/forms/landing-zone/DelimitedImportForm.tsx index 93510cf8191..ceb386cf2a4 100644 --- a/esp/src/src-react/components/forms/landing-zone/DelimitedImportForm.tsx +++ b/esp/src/src-react/components/forms/landing-zone/DelimitedImportForm.tsx @@ -1,5 +1,5 @@ import * as React from "react"; -import { Checkbox, DefaultButton, Dropdown, IDropdownOption, mergeStyleSets, PrimaryButton, Stack, TextField } from "@fluentui/react"; +import { Checkbox, DefaultButton, Dropdown, IDropdownOption, mergeStyleSets, PrimaryButton, Spinner, Stack, TextField } from "@fluentui/react"; import { scopedLogger } from "@hpcc-js/util"; import { useForm, Controller } from "react-hook-form"; import * as FileSpray from "src/FileSpray"; @@ -82,6 +82,8 @@ export const DelimitedImportForm: React.FunctionComponent({ defaultValues }); + const [submitDisabled, setSubmitDisabled] = React.useState(false); + const [spinnerHidden, setSpinnerHidden] = React.useState(true); const closeForm = React.useCallback(() => { setShowForm(false); @@ -91,6 +93,8 @@ export const DelimitedImportForm: React.FunctionComponent { handleSubmit( (data, evt) => { + setSubmitDisabled(true); + setSpinnerHidden(false); let request = {}; const files = data.selectedFiles; @@ -136,6 +140,8 @@ export const DelimitedImportForm: React.FunctionComponent - + + closeForm()} /> }> diff --git a/esp/src/src-react/components/forms/landing-zone/FixedImportForm.tsx b/esp/src/src-react/components/forms/landing-zone/FixedImportForm.tsx index 6b8a5cbaa02..34846ef4bef 100644 --- a/esp/src/src-react/components/forms/landing-zone/FixedImportForm.tsx +++ b/esp/src/src-react/components/forms/landing-zone/FixedImportForm.tsx @@ -1,5 +1,5 @@ import * as React from "react"; -import { Checkbox, DefaultButton, IDropdownOption, mergeStyleSets, PrimaryButton, Stack, TextField } from "@fluentui/react"; +import { Checkbox, DefaultButton, IDropdownOption, mergeStyleSets, PrimaryButton, Spinner, Stack, TextField } from "@fluentui/react"; import { scopedLogger } from "@hpcc-js/util"; import { useForm, Controller } from "react-hook-form"; import * as FileSpray from "src/FileSpray"; @@ -65,6 +65,8 @@ export const FixedImportForm: React.FunctionComponent = ({ const [, { isContainer }] = useBuildInfo(); const { handleSubmit, control, reset } = useForm({ defaultValues }); + const [submitDisabled, setSubmitDisabled] = React.useState(false); + const [spinnerHidden, setSpinnerHidden] = React.useState(true); const closeForm = React.useCallback(() => { setShowForm(false); @@ -74,6 +76,8 @@ export const FixedImportForm: React.FunctionComponent = ({ const onSubmit = React.useCallback(() => { handleSubmit( (data, evt) => { + setSubmitDisabled(true); + setSpinnerHidden(false); let request = {}; const files = data.selectedFiles; @@ -120,6 +124,8 @@ export const FixedImportForm: React.FunctionComponent = ({ } }); if (errors.length === 0) { + setSubmitDisabled(false); + setSpinnerHidden(true); closeForm(); } } @@ -160,7 +166,8 @@ export const FixedImportForm: React.FunctionComponent = ({ return - + + closeForm()} /> }> diff --git a/esp/src/src-react/components/forms/landing-zone/JsonImportForm.tsx b/esp/src/src-react/components/forms/landing-zone/JsonImportForm.tsx index 7132df1e0fa..590f26e9aba 100644 --- a/esp/src/src-react/components/forms/landing-zone/JsonImportForm.tsx +++ b/esp/src/src-react/components/forms/landing-zone/JsonImportForm.tsx @@ -1,5 +1,5 @@ import * as React from "react"; -import { Checkbox, DefaultButton, Dropdown, IDropdownOption, mergeStyleSets, PrimaryButton, Stack, TextField } from "@fluentui/react"; +import { Checkbox, DefaultButton, Dropdown, IDropdownOption, mergeStyleSets, PrimaryButton, Spinner, Stack, TextField } from "@fluentui/react"; import { scopedLogger } from "@hpcc-js/util"; import { useForm, Controller } from "react-hook-form"; import * as FileSpray from "src/FileSpray"; @@ -69,6 +69,8 @@ export const JsonImportForm: React.FunctionComponent = ({ const [, { isContainer }] = useBuildInfo(); const { handleSubmit, control, reset } = useForm({ defaultValues }); + const [submitDisabled, setSubmitDisabled] = React.useState(false); + const [spinnerHidden, setSpinnerHidden] = React.useState(true); const closeForm = React.useCallback(() => { setShowForm(false); @@ -77,6 +79,8 @@ export const JsonImportForm: React.FunctionComponent = ({ const onSubmit = React.useCallback(() => { handleSubmit( (data, evt) => { + setSubmitDisabled(true); + setSpinnerHidden(false); let request = {}; const files = data.selectedFiles; @@ -125,6 +129,8 @@ export const JsonImportForm: React.FunctionComponent = ({ } }); if (errors.length === 0) { + setSubmitDisabled(false); + setSpinnerHidden(true); closeForm(); } } @@ -165,7 +171,8 @@ export const JsonImportForm: React.FunctionComponent = ({ return - + + closeForm()} /> }> diff --git a/esp/src/src-react/components/forms/landing-zone/VariableImportForm.tsx b/esp/src/src-react/components/forms/landing-zone/VariableImportForm.tsx index 64fadc72493..6ae0bbf3c1c 100644 --- a/esp/src/src-react/components/forms/landing-zone/VariableImportForm.tsx +++ b/esp/src/src-react/components/forms/landing-zone/VariableImportForm.tsx @@ -1,5 +1,5 @@ import * as React from "react"; -import { Checkbox, DefaultButton, Dropdown, IDropdownOption, mergeStyleSets, PrimaryButton, Stack, TextField } from "@fluentui/react"; +import { Checkbox, DefaultButton, Dropdown, IDropdownOption, mergeStyleSets, PrimaryButton, Spinner, Stack, TextField } from "@fluentui/react"; import { scopedLogger } from "@hpcc-js/util"; import { useForm, Controller } from "react-hook-form"; import * as FileSpray from "src/FileSpray"; @@ -65,6 +65,8 @@ export const VariableImportForm: React.FunctionComponent({ defaultValues }); + const [submitDisabled, setSubmitDisabled] = React.useState(false); + const [spinnerHidden, setSpinnerHidden] = React.useState(true); const closeForm = React.useCallback(() => { setShowForm(false); @@ -73,6 +75,8 @@ export const VariableImportForm: React.FunctionComponent { handleSubmit( (data, evt) => { + setSubmitDisabled(true); + setSpinnerHidden(false); let request = {}; const files = data.selectedFiles; @@ -117,6 +121,8 @@ export const VariableImportForm: React.FunctionComponent - + + closeForm()} /> }> diff --git a/esp/src/src-react/components/forms/landing-zone/XmlImportForm.tsx b/esp/src/src-react/components/forms/landing-zone/XmlImportForm.tsx index e5f01992294..f40fd9aca23 100644 --- a/esp/src/src-react/components/forms/landing-zone/XmlImportForm.tsx +++ b/esp/src/src-react/components/forms/landing-zone/XmlImportForm.tsx @@ -1,5 +1,5 @@ import * as React from "react"; -import { Checkbox, DefaultButton, Dropdown, IDropdownOption, mergeStyleSets, PrimaryButton, Stack, TextField } from "@fluentui/react"; +import { Checkbox, DefaultButton, Dropdown, IDropdownOption, mergeStyleSets, PrimaryButton, Spinner, Stack, TextField } from "@fluentui/react"; import { scopedLogger } from "@hpcc-js/util"; import { useForm, Controller } from "react-hook-form"; import * as FileSpray from "src/FileSpray"; @@ -69,6 +69,8 @@ export const XmlImportForm: React.FunctionComponent = ({ const [, { isContainer }] = useBuildInfo(); const { handleSubmit, control, reset } = useForm({ defaultValues }); + const [submitDisabled, setSubmitDisabled] = React.useState(false); + const [spinnerHidden, setSpinnerHidden] = React.useState(true); const closeForm = React.useCallback(() => { setShowForm(false); @@ -77,6 +79,8 @@ export const XmlImportForm: React.FunctionComponent = ({ const onSubmit = React.useCallback(() => { handleSubmit( (data, evt) => { + setSubmitDisabled(true); + setSpinnerHidden(false); let request = {}; const files = data.selectedFiles; @@ -123,6 +127,8 @@ export const XmlImportForm: React.FunctionComponent = ({ } }); if (errors.length === 0) { + setSubmitDisabled(false); + setSpinnerHidden(true); closeForm(); } } @@ -163,7 +169,8 @@ export const XmlImportForm: React.FunctionComponent = ({ return - + + closeForm()} /> }> diff --git a/esp/src/src-react/hooks/platform.ts b/esp/src/src-react/hooks/platform.ts index 8e90b4ab363..66dac5d9487 100644 --- a/esp/src/src-react/hooks/platform.ts +++ b/esp/src/src-react/hooks/platform.ts @@ -137,8 +137,14 @@ export function useCheckFeatures(): Features { timestamp }; } - -const fetchReleases = () => { +interface OctokitRelease { + id: number; + draft: boolean; + prerelease: boolean; + tag_name: string; + html_url: string; +} +const fetchReleases = (): Promise<{ data: OctokitRelease[] }> => { const octokit = new Octokit({}); return octokit.request("GET /repos/{owner}/{repo}/releases", { owner: "hpcc-systems", @@ -149,16 +155,12 @@ const fetchReleases = () => { } }); }; -type ReleasesPromise = ReturnType; -type ReleasesResponse = Awaited; -type Releases = ReleasesResponse["data"]; -type Release = Releases[number]; -const _fetchLatestReleases = (): Promise => { +const _fetchLatestReleases = (): Promise => { return fetchReleases().then(response => { - const latest: { [id: string]: Release } = response.data + const latest: { [releaseID: string]: OctokitRelease } = response.data .filter(release => !release.draft || !release.prerelease) - .reduce((prev, curr: Release) => { + .reduce((prev, curr: OctokitRelease) => { const versionParts = curr.tag_name.split("."); versionParts.length = 2; const partialVersion = versionParts.join("."); @@ -167,14 +169,14 @@ const _fetchLatestReleases = (): Promise => { } return prev; }, {}); - return Object.values(latest) as Releases; + return Object.values(latest); }).catch(err => { logger.error(err); - return [] as Releases; + return []; }); }; -let releasesPromise: Promise | undefined; -export const fetchLatestReleases = (): Promise => { +let releasesPromise: Promise | undefined; +export const fetchLatestReleases = (): Promise => { if (!releasesPromise) { releasesPromise = _fetchLatestReleases(); } diff --git a/esp/src/src-react/hooks/workunit.ts b/esp/src/src-react/hooks/workunit.ts index 662b44dccdb..3bc1c22314a 100644 --- a/esp/src/src-react/hooks/workunit.ts +++ b/esp/src/src-react/hooks/workunit.ts @@ -132,6 +132,21 @@ export function useWorkunitSourceFiles(wuid: string): [SourceFile[], Workunit, W const [sourceFiles, setSourceFiles] = React.useState([]); const [count, inc] = useCounter(); + // sorts the WU source files alphabetically by parent name, then name + // with children immediately following parents + const sortFiles = React.useCallback(files => { + const sortedFiles = []; + const temp = files.sort((a, b) => a.Name.localeCompare(b.Name)); + + temp.filter(item => item.__hpcc_parentName === "").forEach(parent => { + sortedFiles.push(parent); + const relatedChildren = temp.filter(child => child.__hpcc_parentName === parent.Name); + sortedFiles.push(...relatedChildren); + }); + + return sortedFiles; + }, []); + React.useEffect(() => { if (workunit) { const fetchInfo = singletonDebounce(workunit, "fetchInfo"); @@ -151,10 +166,10 @@ export function useWorkunitSourceFiles(wuid: string): [SourceFile[], Workunit, W }); }); }); - setSourceFiles(sourceFiles); + setSourceFiles(sortFiles(sourceFiles)); }).catch(err => logger.error(err)); } - }, [workunit, state, count]); + }, [count, sortFiles, state, workunit]); return [sourceFiles, workunit, state, inc]; } diff --git a/esp/src/src-react/util/history.ts b/esp/src/src-react/util/history.ts index d4420fc4d7e..6f54845db45 100644 --- a/esp/src/src-react/util/history.ts +++ b/esp/src/src-react/util/history.ts @@ -278,3 +278,11 @@ export function updateState(key: string, val?: string | string[] | number | bool } globalHistory.replaceState(state, ""); } + +export function navCategory(hash: string): string { + let category = hash.split("/").slice(0, 2).join("/"); + if (category.indexOf("?") > -1) { + category = category.substring(0, category.indexOf("?")); + } + return category; +} \ No newline at end of file diff --git a/plugins/javaembed/javaembed.cpp b/plugins/javaembed/javaembed.cpp index 575be35f52c..b21316f65ac 100644 --- a/plugins/javaembed/javaembed.cpp +++ b/plugins/javaembed/javaembed.cpp @@ -787,7 +787,6 @@ static void setupGlobals(CheckedJNIEnv *J) } catch (IException *E) { - Owned e = E; throw makeWrappedExceptionV(E, E->errorCode(), "javaembed: Unable to load Java system classes - is classpath set properly?"); } @@ -803,7 +802,6 @@ static void setupGlobals(CheckedJNIEnv *J) } catch (IException *E) { - Owned e = E; throw makeWrappedExceptionV(E, E->errorCode(), "javaembed: Unable to find HPCC classes - is classpath set properly?"); } } @@ -4537,7 +4535,6 @@ class JavaEmbedImportContext : public CInterfaceOf } catch (IException *E) { - Owned e = E; throw makeWrappedExceptionV(E, E->errorCode(), "Failed to resolve class name %s", classname.str()); } javaClass = (jclass) JNIenv->NewGlobalRef(javaClass, "javaClass"); diff --git a/roxie/ccd/ccdquery.cpp b/roxie/ccd/ccdquery.cpp index 901810b7834..29c34c9fa2d 100644 --- a/roxie/ccd/ccdquery.cpp +++ b/roxie/ccd/ccdquery.cpp @@ -1850,6 +1850,11 @@ class CRoxieServerQueryFactory : public CQueryFactory else return NULL; } + + virtual IPropertyTree *getQueryStats(time_t from, time_t to) override + { + return queryStats->getStats(from, to); + } }; unsigned checkWorkunitVersionConsistency(const IConstWorkUnit *wu) diff --git a/system/jhtree/jhtree.cpp b/system/jhtree/jhtree.cpp index 86279960c2d..e522d11d208 100644 --- a/system/jhtree/jhtree.cpp +++ b/system/jhtree/jhtree.cpp @@ -2639,9 +2639,6 @@ void CNodeCache::getCacheInfo(ICacheInfoRecorder &cacheInfo) //Critical sections are 40bytes on linux so < 0.5% overhead for an 8K page and trivial overhead when constructed (<10ns) static std::atomic lastLockingReportCycles{0}; static std::atomic lastLoadReportCycles{0}; -static std::atomic countExcessiveLock_x1{0}; -static std::atomic countExcessiveLock_x10{0}; -static std::atomic countExcessiveLock_x100{0}; const CJHTreeNode *CNodeCache::getCachedNode(const INodeLoader *keyIndex, unsigned iD, offset_t pos, NodeType type, IContextLogger *ctx, bool isTLK) { @@ -2761,24 +2758,10 @@ const CJHTreeNode *CNodeCache::getCachedNode(const INodeLoader *keyIndex, unsign cycle_t lockingCycles = startLoadCycles - startCycles; if (lockingCycles > traceCacheLockingThreshold) { - if (lockingCycles >= traceCacheLockingThreshold*100) - countExcessiveLock_x100++; - else if (lockingCycles >= traceCacheLockingThreshold*10) - countExcessiveLock_x10++; - else - countExcessiveLock_x1++; - if ((endLoadCycles - lastLockingReportCycles) >= traceCacheLockingFrequency) { lastLockingReportCycles = endLoadCycles; - WARNLOG("CNodeCache::getNode lock(%s) took %lluns counts(>=%lluns, %u, %u, %u) (x1,x10,x100)", cacheTypeText[cacheType], cycle_to_nanosec(lockingCycles), - cycle_to_nanosec(traceCacheLockingThreshold), - countExcessiveLock_x1.load() + countExcessiveLock_x10.load() + countExcessiveLock_x100.load(), - countExcessiveLock_x10.load() + countExcessiveLock_x100.load(), - countExcessiveLock_x100.load()); - countExcessiveLock_x1 = 0; - countExcessiveLock_x10 = 0; - countExcessiveLock_x100 = 0; + WARNLOG("CNodeCache::getNode lock(%s) took %lluns", cacheTypeText[cacheType], cycle_to_nanosec(lockingCycles)); } } cycle_t actualLoadCycles = endLoadCycles - startLoadCycles; @@ -2787,7 +2770,8 @@ const CJHTreeNode *CNodeCache::getCachedNode(const INodeLoader *keyIndex, unsign if ((endLoadCycles - lastLoadReportCycles) >= traceNodeLoadFrequency) { lastLoadReportCycles = endLoadCycles; - WARNLOG("CNodeCache::getNode load(%s %x:%llu) took %lluus size(%u)", cacheTypeText[cacheType], iD, pos, cycle_to_microsec(actualLoadCycles), ownedCacheEntry->queryNode()->getMemSize()); + WARNLOG("CNodeCache::getNode load(%s %x:%llu) took %lluus fetch(%lluus) size(%u)", cacheTypeText[cacheType], iD, pos, + cycle_to_microsec(actualLoadCycles), cycle_to_microsec(fetchCycles), ownedCacheEntry->queryNode()->getMemSize()); } } diff --git a/system/jlib/jexcept.cpp b/system/jlib/jexcept.cpp index 313fbf9bef9..4aecef4f168 100644 --- a/system/jlib/jexcept.cpp +++ b/system/jlib/jexcept.cpp @@ -303,9 +303,14 @@ class jlib_thrown_decl WrappedException: public StringException return str; } protected: - Linked exception; + Owned exception; }; +IException *makeWrappedException(IException *e, int code, const char *why) +{ + return new WrappedException(e, code, why); +} + IException *makeWrappedExceptionVA(IException *e, int code, const char *format, va_list args) { StringBuffer eStr; diff --git a/system/jlib/jexcept.hpp b/system/jlib/jexcept.hpp index 733513d4e49..1621c329c74 100644 --- a/system/jlib/jexcept.hpp +++ b/system/jlib/jexcept.hpp @@ -77,6 +77,7 @@ IException jlib_decl *makeStringExceptionVA(MessageAudience aud, int code, const IException jlib_decl *makeStringException(MessageAudience aud, int code, const char *why); IException jlib_decl *makePrefixedException(const char * prefix, const IException * e); __declspec(noreturn) void jlib_decl throwStringExceptionV(int code, const char *format, ...) __attribute__((format(printf, 2, 3), noreturn)); +IException jlib_decl *makeWrappedException(IException *e, int code, const char *why); IException jlib_decl *makeWrappedExceptionVA(IException *e, int code, const char *why, va_list args) __attribute__((format(printf, 3, 0))); IException jlib_decl *makeWrappedExceptionV(IException *e, int code, const char *why, ...) __attribute__((format(printf, 3, 4))); diff --git a/system/jlib/jfile.cpp b/system/jlib/jfile.cpp index ffe331155f1..bd21866dc1c 100644 --- a/system/jlib/jfile.cpp +++ b/system/jlib/jfile.cpp @@ -1868,6 +1868,27 @@ IFileIO * CFile::openShared(IFOmode mode,IFSHmode share,IFEflags extraFlags) //--------------------------------------------------------------------------- +static int globalFileSyncMaxRetrySecs = fileSyncRetryDisabled; +static std::atomic globalFileSyncMaxRetrySecsConfigured{false}; +static CriticalSection globalFileSyncCS; +static int getGlobalMaxFileSyncSecs() +{ + if (!globalFileSyncMaxRetrySecsConfigured) + { + CriticalBlock b(globalFileSyncCS); + if (!globalFileSyncMaxRetrySecsConfigured) + { + Owned global = getGlobalConfig(); + Owned config = getComponentConfig(); + globalFileSyncMaxRetrySecs = global->getPropInt("expert/@fileSyncMaxRetrySecs", defaultGlobalFileSyncMaxRetrySecs); + globalFileSyncMaxRetrySecs = config->getPropInt("expert/@fileSyncMaxRetrySecs", globalFileSyncMaxRetrySecs); + globalFileSyncMaxRetrySecsConfigured = true; + // NB: -1 == infinite, -2 == disable checking altogether + } + } + return globalFileSyncMaxRetrySecs; +} + extern jlib_decl IFileIO *createIFileIO(IFile * creator,HANDLE handle,IFOmode openmode,IFEflags extraFlags) { @@ -2023,18 +2044,61 @@ void CFileIO::setSize(offset_t pos) //-- Unix implementation ---------------------------------------------------- -static void syncFileData(int fd, bool notReadOnly, IFEflags extraFlags, bool wait_previous=false) +// -2 disabled - don't validate fsync/fdatasync +// -1 retry forever +// 0 no retry +static void retrySync(int fd, int retrySecs, bool dataOnly) { - if (notReadOnly) - { - if (extraFlags & IFEsync) - { #ifdef F_FULLFSYNC - fcntl(fd, F_FULLFSYNC); + // No EIO type retry available + fcntl(fd, F_FULLFSYNC); #else - fdatasync(fd); -#endif + CCycleTimer timer; + unsigned retryMaxMs; + if (retrySecs < 0) + retryMaxMs = UINT_MAX; + else + { + assertex(((unsigned)retrySecs) <= UINT_MAX/1000); + retryMaxMs = ((unsigned)retrySecs) * 1000; + } + unsigned delayMs = 200; // start with .2 secs + unsigned retryAttempts = 0; + while (true) + { + int ret = dataOnly ? fdatasync(fd) : fsync(fd); + if (ret == 0) + break; + if (fileSyncRetryDisabled == retrySecs) // error, but unchecked! Temporary, to allow to be disabled JIC causes unexpected side-effects + break; + if (EIO != errno) + throw makeErrnoException(errno, "retrySync"); + if ((retrySecs >= 0) && timer.elapsedMs() > retryMaxMs) + { + printStackReport(); + Owned e = makeErrnoExceptionV(errno, "retrySync: failed with EIO, retrying after %u seconds (%u retries)", retryMaxMs/1000, retryAttempts); + OWARNLOG(e); + throw e.getClear(); + } + // In the event, not sure I care about burst of logging on retries (there won't be that many and this is fatal last throw of dice) + IWARNLOG("retrySync: failed with EIO, retry: %u", ++retryAttempts); + if (delayMs >= 60000) // cap max delay to 1 minute + MilliSleep(60000); + else + { + MilliSleep(delayMs); + delayMs *= 2; } + } +#endif +} + +static void syncFileData(int fd, bool notReadOnly, IFEflags extraFlags, int syncRetrySecs, bool wait_previous=false) +{ + if (notReadOnly) + { + if (extraFlags & IFEsync) + retrySync(fd, syncRetrySecs, true); #if defined(__linux__) else if (extraFlags & IFEnocache) { @@ -2052,6 +2116,7 @@ static void syncFileData(int fd, bool notReadOnly, IFEflags extraFlags, bool wai #endif } + // More errorno checking TBD CFileIO::CFileIO(IFile * _creator, HANDLE handle, IFOmode _openmode, IFSHmode _sharemode, IFEflags _extraFlags) : creator(_creator), unflushedReadBytes(0), unflushedWriteBytes(0) @@ -2071,6 +2136,22 @@ CFileIO::CFileIO(IFile * _creator, HANDLE handle, IFOmode _openmode, IFSHmode _s extraFlags = static_cast(extraFlags | expertEnableIFileFlagsMask); extraFlags = static_cast(extraFlags & ~expertDisableIFileFlagsMask); + if (isContainerized() && (openmode!=IFOread)) // only containerized (with planes writing to storage types like blob for now) + { + const char *filePath = querySafeFilename(); + if ('/' == filePath[0]) // only for absolute paths + { + unsigned __int64 value; + if (findPlaneAttrFromPath(filePath, FileSyncMaxRetrySecs, getGlobalMaxFileSyncSecs(), value)) // NB: returns only if plane found + { + // fileSyncMaxRetrySecs applies to IFEsync and IFEsyncAtClose + fileSyncMaxRetrySecs = value; + if (fileSyncRetryDisabled != fileSyncMaxRetrySecs) + extraFlags = static_cast(extraFlags | IFEsyncAtClose); + } + } + } + #ifdef CFILEIOTRACE DBGLOG("CFileIO::CfileIO(%d,%d,%d,%d)", handle, _openmode, _sharemode, extraFlags); #endif @@ -2111,7 +2192,9 @@ void CFileIO::close() DBGLOG("CFileIO::close(%d), extraFlags = %d", tmpHandle, extraFlags); #endif if (extraFlags & (IFEnocache | IFEsync)) - syncFileData(tmpHandle, openmode!=IFOread, extraFlags, false); + syncFileData(tmpHandle, openmode!=IFOread, extraFlags, fileSyncMaxRetrySecs, false); + else if (extraFlags & IFEsyncAtClose) + retrySync(tmpHandle, fileSyncMaxRetrySecs, false); if (::close(tmpHandle) < 0) throw makeErrnoExceptionV(errno, "CFileIO::close for file '%s'", querySafeFilename()); @@ -2125,7 +2208,7 @@ void CFileIO::flush() CriticalBlock procedure(cs); - syncFileData(file, true, extraFlags, false); + syncFileData(file, true, extraFlags, fileSyncMaxRetrySecs, false); } @@ -2162,7 +2245,7 @@ size32_t CFileIO::read(offset_t pos, size32_t len, void * data) if (unflushedReadBytes.add_fetch(ret) >= PGCFLUSH_BLKSIZE) { unflushedReadBytes.store(0); - syncFileData(file, false, extraFlags, false); + syncFileData(file, false, extraFlags, 0, false); } } return ret; @@ -2192,7 +2275,7 @@ size32_t CFileIO::write(offset_t pos, size32_t len, const void * data) { unflushedWriteBytes.store(0); // request to write-out dirty pages - syncFileData(file, true, extraFlags, true); + syncFileData(file, true, extraFlags, fileSyncMaxRetrySecs, true); } } return ret; @@ -3099,68 +3182,6 @@ static inline bool isPCFlushAllowed() return false; } -static inline size32_t doread(IFileIOStream * stream,void *dst, size32_t size) -{ - size32_t toread=size; - while (toread) - { - int read = stream->read(toread, dst); - if (!read) - return size-toread; - toread -= read; - dst = (char *) dst + read; - } - return size; -} - - - -CIOStreamReadWriteSeq::CIOStreamReadWriteSeq(IFileIOStream * _stream, offset_t _offset, size32_t _size) -{ - stream.set(_stream); -// stream->setThrowOnError(true); - size = _size; - offset = _offset; // assumption that stream at correct location already -} - -void CIOStreamReadWriteSeq::put(const void *src) -{ - stream->write(size, src); -} - -void CIOStreamReadWriteSeq::putn(const void *src, unsigned n) -{ - stream->write(size*n, src); -} - -void CIOStreamReadWriteSeq::flush() -{ - stream->flush(); -} - -offset_t CIOStreamReadWriteSeq::getPosition() -{ - return stream->tell(); -} - -bool CIOStreamReadWriteSeq::get(void *dst) -{ - return doread(stream,dst,size)==size; -} - -unsigned CIOStreamReadWriteSeq::getn(void *dst, unsigned n) -{ - return doread(stream,dst,size*n)/size; -} - -void CIOStreamReadWriteSeq::reset() -{ - stream->seek(offset, IFSbegin); -} - - - - //-- Helper routines -------------------------------------------------------- size32_t read(IFileIO * in, offset_t pos, size32_t len, MemoryBuffer & buffer) @@ -4590,17 +4611,6 @@ IFileIOStream *createProgressIFileIOStream(IFileIOStream *iFileIOStream, offset_ return new CProgressIFileIOStream(iFileIOStream, totalSize, msg, periodSecs); } -IReadSeq *createReadSeq(IFileIOStream * stream, offset_t offset, size32_t size) -{ - return new CIOStreamReadWriteSeq(stream, offset, size); -} - -IWriteSeq *createWriteSeq(IFileIOStream * stream, size32_t size) -{ - return new CIOStreamReadWriteSeq(stream, 0, size); -} - - extern jlib_decl offset_t filesize(const char *name) { @@ -7937,25 +7947,36 @@ static unsigned jFileHookId = 0; static const std::array planeAttributeTypeStrings = { "blockedFileIOKB", - "blockedRandomIOKB" + "blockedRandomIOKB", + "fileSyncMaxRetrySecs" }; -static std::unordered_map> planeAttributesMap; -static CriticalSection planeAttriubuteMapCrit; +// {prefix, {key1: value1, key2: value2, ...}} +typedef std::pair> PlaneAttributesMapElement; + +static std::unordered_map planeAttributesMap; +static CriticalSection planeAttributeMapCrit; +static constexpr unsigned __int64 unsetPlaneAttrValue = 0xFFFFFFFF00000000; MODULE_INIT(INIT_PRIORITY_STANDARD) { auto updateFunc = [&](const IPropertyTree *oldComponentConfiguration, const IPropertyTree *oldGlobalConfiguration) { - CriticalBlock b(planeAttriubuteMapCrit); + CriticalBlock b(planeAttributeMapCrit); planeAttributesMap.clear(); Owned planesIter = getPlanesIterator(nullptr, nullptr); ForEach(*planesIter) { const IPropertyTree &plane = planesIter->query(); - auto &values = planeAttributesMap[plane.queryProp("@name")]; - values[BlockedSequentialIO] = plane.getPropInt(("@" + std::string(planeAttributeTypeStrings[BlockedSequentialIO])).c_str()) * 1024; - values[BlockedRandomIO] = plane.getPropInt(("@" + std::string(planeAttributeTypeStrings[BlockedRandomIO])).c_str()) * 1024; + PlaneAttributesMapElement &element = planeAttributesMap[plane.queryProp("@name")]; + element.first = plane.queryProp("@prefix"); + auto &values = element.second; + unsigned __int64 value; + value = plane.getPropInt64(("@" + std::string(planeAttributeTypeStrings[BlockedSequentialIO])).c_str(), unsetPlaneAttrValue); + values[BlockedSequentialIO] = (unsetPlaneAttrValue != value) ? value * 1024 : value; + value = plane.getPropInt64(("@" + std::string(planeAttributeTypeStrings[BlockedRandomIO])).c_str(), unsetPlaneAttrValue); + values[BlockedRandomIO] = (unsetPlaneAttrValue != value) ? value * 1024 : value; + values[FileSyncMaxRetrySecs] = plane.getPropInt64(("@" + std::string(planeAttributeTypeStrings[FileSyncMaxRetrySecs])).c_str(), unsetPlaneAttrValue); } // reset defaults @@ -7968,6 +7989,9 @@ MODULE_INIT(INIT_PRIORITY_STANDARD) if (getComponentConfigSP()->getProp("expert/@disableIFileMask", fileFlagsStr.clear()) || getGlobalConfigSP()->getProp("expert/@disableIFileMask", fileFlagsStr)) expertDisableIFileFlagsMask = (IFEflags)strtoul(fileFlagsStr, NULL, 0); + + // clear for getGlobalMaxFileSyncSecs() to re-evaluate + globalFileSyncMaxRetrySecsConfigured = false; }; jFileHookId = installConfigUpdateHook(updateFunc, true); @@ -7988,17 +8012,58 @@ const char *getPlaneAttributeString(PlaneAttributeType attr) unsigned __int64 getPlaneAttributeValue(const char *planeName, PlaneAttributeType planeAttrType, unsigned __int64 defaultValue) { assertex(planeAttrType < PlaneAttributeCount); - CriticalBlock b(planeAttriubuteMapCrit); + CriticalBlock b(planeAttributeMapCrit); auto it = planeAttributesMap.find(planeName); if (it != planeAttributesMap.end()) { - unsigned v = it->second[planeAttrType]; - if (v) // a plane attribute value of 0 is considered as not set + unsigned __int64 v = it->second.second[planeAttrType]; + if (v != unsetPlaneAttrValue) return v; } return defaultValue; } +static PlaneAttributesMapElement *findPlaneElementFromPath(const char *filePath) +{ + for (auto &e: planeAttributesMap) + { + const char *prefix = e.second.first.c_str(); + if (prefix) // sanity check, but should never be null + { + if (startsWith(filePath, prefix)) + return &e.second; + } + } + return nullptr; +} + +const char *findPlaneFromPath(const char *filePath, StringBuffer &result) +{ + CriticalBlock b(planeAttributeMapCrit); + PlaneAttributesMapElement *e = findPlaneElementFromPath(filePath); + if (!e) + return nullptr; + + result.append(e->first.c_str()); + return result; +} + +bool findPlaneAttrFromPath(const char *filePath, PlaneAttributeType planeAttrType, unsigned __int64 defaultValue, unsigned __int64 &resultValue) +{ + CriticalBlock b(planeAttributeMapCrit); + PlaneAttributesMapElement *e = findPlaneElementFromPath(filePath); + if (e) + { + unsigned __int64 value = e->second[planeAttrType]; + if (unsetPlaneAttrValue != value) + resultValue = value; + else + resultValue = defaultValue; + return true; + } + return false; +} + size32_t getBlockedFileIOSize(const char *planeName, size32_t defaultSize) { return (size32_t)getPlaneAttributeValue(planeName, BlockedSequentialIO, defaultSize); @@ -8009,3 +8074,7 @@ size32_t getBlockedRandomIOSize(const char *planeName, size32_t defaultSize) return (size32_t)getPlaneAttributeValue(planeName, BlockedRandomIO, defaultSize); } +int getMaxFileSyncSecs(const char *planeName, int defaultSecs) +{ + return (int)getPlaneAttributeValue(planeName, FileSyncMaxRetrySecs, defaultSecs); +} diff --git a/system/jlib/jfile.hpp b/system/jlib/jfile.hpp index fa2a27b4275..4e93ab84239 100644 --- a/system/jlib/jfile.hpp +++ b/system/jlib/jfile.hpp @@ -46,7 +46,7 @@ enum IFOmode { IFOcreate, IFOread, IFOwrite, IFOreadwrite, IFOcreaterw }; // enum IFSHmode { IFSHnone, IFSHread=0x8, IFSHfull=0x10}; // sharing modes enum IFSmode { IFScurrent = FILE_CURRENT, IFSend = FILE_END, IFSbegin = FILE_BEGIN }; // seek mode enum CFPmode { CFPcontinue, CFPcancel, CFPstop }; // modes for ICopyFileProgress::onProgress return -enum IFEflags { IFEnone=0x0, IFEnocache=0x1, IFEcache=0x2, IFEsync=0x4 }; // mask +enum IFEflags { IFEnone=0x0, IFEnocache=0x1, IFEcache=0x2, IFEsync=0x4, IFEsyncAtClose=0x8 }; // mask constexpr offset_t unknownFileSize = -1; class CDateTime; @@ -296,14 +296,6 @@ extern jlib_decl IFileIO * createIFileIO(MemoryBuffer & buffer); //-- Creation of routines to implement other interfaces on the interfaces above. -interface IReadSeq; -interface IWriteSeq; - -// NB the following are unbuffered -extern jlib_decl IReadSeq *createReadSeq(IFileIOStream * stream, offset_t _offset, size32_t size); // no buffering -extern jlib_decl IWriteSeq *createWriteSeq(IFileIOStream * stream, size32_t size); // no buffering - - extern jlib_decl IDiscretionaryLock *createDiscretionaryLock(IFile *file); extern jlib_decl IDiscretionaryLock *createDiscretionaryLock(IFileIO *fileio); @@ -756,12 +748,19 @@ enum PlaneAttributeType { BlockedSequentialIO, BlockedRandomIO, + FileSyncMaxRetrySecs, PlaneAttributeCount }; extern jlib_decl const char *getPlaneAttributeString(PlaneAttributeType attr); extern jlib_decl unsigned __int64 getPlaneAttributeValue(const char *planeName, PlaneAttributeType planeAttrType, unsigned __int64 defaultValue); +extern jlib_decl const char *findPlaneFromPath(const char *filePath, StringBuffer &result); +//returns true if plane exists, fills resultValue with defaultValue if attribute is unset +extern jlib_decl bool findPlaneAttrFromPath(const char *filePath, PlaneAttributeType planeAttrType, unsigned __int64 defaultValue, unsigned __int64 &resultValue); extern jlib_decl size32_t getBlockedFileIOSize(const char *planeName, size32_t defaultSize=0); extern jlib_decl size32_t getBlockedRandomIOSize(const char *planeName, size32_t defaultSize=0); +constexpr int fileSyncRetryDisabled = -2; +constexpr int defaultGlobalFileSyncMaxRetrySecs = fileSyncRetryDisabled; +extern jlib_decl int getMaxFileSyncSecs(const char *planeName, int defaultSecs = defaultGlobalFileSyncMaxRetrySecs); //---- Pluggable file type related functions ---------------------------------------------- diff --git a/system/jlib/jfile.ipp b/system/jlib/jfile.ipp index 815e8163bf9..023897f0587 100644 --- a/system/jlib/jfile.ipp +++ b/system/jlib/jfile.ipp @@ -87,7 +87,6 @@ protected: unsigned flags; }; - class jlib_decl CFileIO : implements IFileIO, public CInterface { public: @@ -121,6 +120,7 @@ protected: IFOmode openmode; IFEflags extraFlags; FileIOStats stats; + int fileSyncMaxRetrySecs = fileSyncRetryDisabled; // enabled conditionally in ctor RelaxedAtomic unflushedReadBytes; // more: If this recorded flushedReadBytes it could have a slightly lower overhead RelaxedAtomic unflushedWriteBytes; private: @@ -252,34 +252,6 @@ protected: }; -class jlib_decl CIOStreamReadWriteSeq : public IWriteSeq, public IReadSeq, public CInterface -{ -public: - IMPLEMENT_IINTERFACE; - - CIOStreamReadWriteSeq(IFileIOStream * _stream, offset_t _offset, size32_t _size); - - virtual void put(const void *src); - virtual void putn(const void *src, unsigned n); - virtual void flush(); - virtual size32_t getRecordSize() { return size; } - virtual offset_t getPosition(); - - virtual bool get(void *dst); - virtual unsigned getn(void *dst, unsigned n); - virtual void reset(); - virtual void stop() {} // no action required - -private: - offset_t offset; - size32_t size; - Linked stream; -}; - - - - - class jlib_decl DirectBufferI : implements IFileIO, public CInterface { public: diff --git a/system/jlib/jio.cpp b/system/jlib/jio.cpp index 8a51f4e2176..e193660f1d5 100644 --- a/system/jlib/jio.cpp +++ b/system/jlib/jio.cpp @@ -189,433 +189,6 @@ extern jlib_decl size32_t checked_write(const char * filename, int handle, const return (size32_t)ret; } -class CReadSeq : public IReadSeq, public CInterface -{ - int fh; - size32_t size; - char *buffer; - char *ptr; - size32_t bufSize; - size32_t bytesInBuffer; - offset_t startpos; - offset_t endpos; - offset_t nextbufpos; - bool compressed; - void *prev; - size32_t maxcompsize; - bool first; - - inline unsigned remaining() - { - return (unsigned)(buffer+bytesInBuffer-ptr); - } - - size32_t getBytes(void *dst, size32_t _size) - { - size32_t left = remaining(); - size32_t read = 0; - while (_size>left) { - if (left) { - memcpy(dst, ptr, left); - dst = (char *)dst + left; - _size -= left; - read += left; - ptr+=left; - } - refill(); - left = bytesInBuffer; - if (!left) - return read; - } - memcpy(dst, ptr, _size); - ptr += _size; - read += _size; - return read; - } - - void refill() - { - size32_t left = remaining(); - memmove(buffer,ptr,left); - size32_t rd=bufSize-left; - if (endpos-nextbufpos<(offset_t)rd) - rd = (size32_t)(endpos-nextbufpos); - if (rd) - rd = checked_pread("unknown", fh, buffer+left, rd, nextbufpos); - nextbufpos += rd; - bytesInBuffer = left+rd; - ptr = buffer; - } - - -public: - IMPLEMENT_IINTERFACE; - - CReadSeq(int _fh, offset_t _offset, unsigned maxrecs, size32_t _size, size32_t _bufsize, bool _compressed) - { - assertex(_size); - fh = _fh; - size = _size; - bufSize = (_bufsize==(unsigned) -1)?DEFAULTBUFFERSIZE:_bufsize; - bytesInBuffer = 0; - startpos = _offset; - nextbufpos = _offset; - compressed = ((size=MINCOMPRESSEDROWSIZE)&&(size<=MAXCOMPRESSEDROWSIZE))?_compressed:false; - if (compressed) { - maxcompsize = size+size/3+3; // migger than needed - buffer = (char *) malloc(bufSize+size); - prev = buffer+bufSize; - } - else - buffer = (char *) malloc(bufSize); - ptr = buffer; - first = true; - endpos = (maxrecs!=(unsigned)-1)?(_offset+(offset_t)maxrecs*(offset_t)_size):I64C(0x7ffffffffff); - - } - - ~CReadSeq() - { - free(buffer); - } - - virtual bool get(void *dst) - { - if (!compressed) - return getBytes(dst, size)==size; - return (getn(dst,1)==1); - } - - virtual unsigned getn(void *dst, unsigned n) - { - if (!compressed) - return getBytes(dst, size*n)/size; - byte *d = (byte *)dst; - byte *e = d+(size*n); - byte *p = (byte *)prev; - unsigned ret = 0; - while (d!=e) { - if (first) { - if (getBytes(d, size)!=size) - break; - first = false; - } - else { - if (remaining()reset(); // not done itself - return seq; - } - return new CReadSeq(fh, _offset, maxrecs, size, bufsize, compressed); -} - - - -//================================================================================================ - - -class CWriteSeq : public IWriteSeq, public CInterface -{ -private: - int fh; - size32_t size; - char *buffer; - char *ptr; - size32_t bufSize; - offset_t fpos; - bool compressed; - size32_t maxcompsize; - void *prev; - void *aux; - bool first; - - inline size32_t remaining() - { - return (size32_t)(bufSize - (ptr-buffer)); - } - - void putBytes(const void *src, size32_t _size) - { - fpos += _size; - size32_t left = remaining(); - if (_size>left) - { - if (ptr!=buffer) { // don't buffer if entire block - memcpy(ptr, src, left); - ptr += left; - src = (char *)src + left; - _size -= left; - flush(); - left = bufSize; - } - while (_size>=bufSize) // write out directly - { - checked_write("unknown", fh, src, bufSize); // stick to writing bufSize blocks - src = (char *)src + bufSize; - _size -= bufSize; - } - } - memcpy(ptr, src, _size); - ptr += _size; - } - -public: - IMPLEMENT_IINTERFACE; - - CWriteSeq(int _fh, size32_t _size, size32_t _bufsize, bool _compressed) - { - assertex(_fh); - assertex(_size); - fh = _fh; - size = _size; - fpos = 0; - if (_bufsize == (unsigned) -1) - _bufsize = DEFAULTBUFFERSIZE; - bufSize = _bufsize; - - compressed = ((size=MINCOMPRESSEDROWSIZE)&&(size<=MAXCOMPRESSEDROWSIZE))?_compressed:false; - if (compressed) { - maxcompsize = size+size/3+3; // bigger than needed - buffer = (char *) malloc(bufSize+size+maxcompsize); - prev = buffer+bufSize; - aux = (char *)prev+size; - } - else - buffer = (char *) malloc(bufSize); - - ptr = buffer; - first = true; - } - - ~CWriteSeq() - { - free(buffer); - } - - - void put(const void *src) - { - if (compressed) { - if (first) { - first = false; - memcpy(prev,src,size); - } - else if (remaining()>=maxcompsize) { - size32_t sz = DiffCompress(src,ptr,prev,size); - fpos += sz; - ptr += sz; - return; - } - else { - putBytes(aux, DiffCompress(src,aux,prev,size)); - return; - } - } - putBytes(src, size); - } - - void putn(const void *src, unsigned numRecs) - { - if (compressed) { - while (numRecs) { - put(src); - src = (byte *)src+size; - numRecs--; - } - } - else - putBytes(src, size*numRecs); - } - - void flush() - { - if (ptr != buffer) - { - checked_write("unknown", fh, buffer, (size32_t)(ptr-buffer)); - ptr = buffer; - } - } - - offset_t getPosition() - { - return fpos; - } - - virtual size32_t getRecordSize() - { - return size; - } - -}; - -IWriteSeq *createWriteSeq(int fh, size32_t size, size32_t bufsize, bool compressed) -{ - // Async TBD - if (!bufsize) - return new CUnbufferedReadWriteSeq(fh, 0, size); - else - return new CWriteSeq(fh, size, bufsize,compressed); -} - -IWriteSeq *createTeeWriteSeq(IWriteSeq *f1, IWriteSeq *f2) -{ - return new CTeeWriteSeq(f1, f2); -} - -//=========================================================================================== - -CUnbufferedReadWriteSeq::CUnbufferedReadWriteSeq(int _fh, offset_t _offset, size32_t _size) -{ - fh = _fh; - size = _size; - offset = _offset; - fpos = _offset; - -} - -void CUnbufferedReadWriteSeq::put(const void *src) -{ - checked_write("unknown", fh, src, size); - fpos += size; -} - -void CUnbufferedReadWriteSeq::putn(const void *src, unsigned n) -{ - checked_write("unknown", fh, src, size*n); - fpos += size*n; -} - -void CUnbufferedReadWriteSeq::flush() -{} - -offset_t CUnbufferedReadWriteSeq::getPosition() -{ - return fpos; -} - -bool CUnbufferedReadWriteSeq::get(void *dst) -{ - size32_t toread = size; - while (toread) - { - int read = checked_read("unknown", fh, dst, toread); - if (!read) - return false; - toread -= read; - dst = (char *) dst + read; - } - return true; -} - -unsigned CUnbufferedReadWriteSeq::getn(void *dst, unsigned n) -{ - size32_t toread = size*n; - size32_t totread = 0; - while (toread) - { - int read = checked_read("unknown", fh, dst, toread); - if (!read) - break; - toread -= read; - totread += read; - dst = (char *) dst + read; - } - return totread/size; -} - -void CUnbufferedReadWriteSeq::reset() -{ - checked_lseeki64(fh, offset, SEEK_SET); - fpos = offset; -} - -//=========================================================================================== - - -//=========================================================================================== - -CTeeWriteSeq::CTeeWriteSeq(IWriteSeq *_f1, IWriteSeq *_f2) -{ - w1 = _f1; - w1->Link(); - w2 = _f2; - w2->Link(); - assertex(w1->getRecordSize()==w2->getRecordSize()); -} - -CTeeWriteSeq::~CTeeWriteSeq() -{ - w1->Release(); - w2->Release(); -} - -void CTeeWriteSeq::put(const void *src) -{ - w1->put(src); - w2->put(src); -} - -void CTeeWriteSeq::putn(const void *src, unsigned n) -{ - w1->putn(src, n); - w2->putn(src, n); -} - -void CTeeWriteSeq::flush() -{ - w1->flush(); - w2->flush(); -} - -size32_t CTeeWriteSeq::getRecordSize() -{ - return w1->getRecordSize(); -} - -offset_t CTeeWriteSeq::getPosition() -{ - return w1->getPosition(); -} - //================================================================================================== class CFixedRecordSize: public IRecordSize, public CInterface @@ -677,129 +250,6 @@ extern jlib_decl IRecordSize *createDeltaRecordSize(IRecordSize * size, int delt } -//================================================================================================== -// chained routines allowing multiple streams to be concatenated -// all streams assumed to have same record size - - -class CChainedWriteSeq : public IWriteSeq, public CInterface -{ -protected: - IWriteSeq *stream; - IWriteSeqAllocator *allocator; - unsigned num; - size32_t recsize; - offset_t pos; -public: - IMPLEMENT_IINTERFACE; - CChainedWriteSeq(IWriteSeqAllocator *_allocator) - { - allocator = _allocator; - allocator->Link(); - num = 0; - recsize = 0; - pos = 0; - } - virtual ~CChainedWriteSeq() - { - ::Release(stream); - allocator->Release(); - } - void flush() { if (stream) stream->flush(); } - void put(const void *dst) { putn(dst,1); } - void putn(const void *dst, unsigned numrecs) - { - if (numrecs==0) return; - if (stream==NULL) return; // could raise exception instead - byte *out=(byte *)dst; - while (numrecs>num) { - stream->putn(out,num); - pos+=num; - numrecs-=num; - stream->flush(); - IWriteSeq *oldstream=stream; - stream = allocator->next(num); - oldstream->Release(); - if (!stream) { - return; // could raise exception - } - } - stream->putn(out,numrecs); - pos+=numrecs; - } - virtual size32_t getRecordSize() { if ((recsize==0)&&stream) recsize = stream->getRecordSize(); return recsize; } - virtual offset_t getPosition() { return pos; } -}; - - -class CChainedReadSeq : public IReadSeq, public CInterface -{ -protected: - IReadSeq *stream; - IReadSeqAllocator *allocator; - unsigned num; - size32_t recsize; -public: - IMPLEMENT_IINTERFACE; - CChainedReadSeq(IReadSeqAllocator *_allocator) - { - allocator = _allocator; - allocator->Link(); - stream = allocator->next(); - num = 0; - recsize = 0; - } - virtual ~CChainedReadSeq() - { - ::Release(stream); - allocator->Release(); - } - virtual bool get(void *dst) { return (getn(dst,1)==1); } - virtual unsigned getn(void *dst, unsigned n) - { - unsigned done=0; - while (stream&&n) { - unsigned r = stream->getn(dst,n); - if (r==0) { - IReadSeq *oldstream=stream; - stream = allocator->next(); - oldstream->Release(); - } - else { - n-=r; - done+=r; - } - } - return done; - } - virtual size32_t getRecordSize() { return stream->getRecordSize(); } - virtual void reset() { stream->reset(); } - virtual void stop() { stream->stop(); } - -}; - - - -unsigned copySeq(IReadSeq *from,IWriteSeq *to,size32_t bufsize) -{ - size32_t recsize=from->getRecordSize(); - assertex(recsize==to->getRecordSize()); - unsigned nbuf = bufsize/recsize; - if (nbuf==0) - nbuf = 1; - MemoryAttr ma; - byte *buf=(byte *)ma.allocate(nbuf*recsize); - unsigned ret = 0; - for (;;) { - unsigned n = from->getn(buf,nbuf); - if (n==0) - break; - to->putn(buf,n); - ret += n; - } - return ret; -} - ///////////////// CBufferedIOStreamBase::CBufferedIOStreamBase(size32_t _bufferSize) @@ -1072,18 +522,6 @@ class CConcatRowStream : public IRowStream, public CInterface }; -extern jlib_decl IWriteSeq *createChainedWriteSeq(IWriteSeqAllocator *iwsa) -{ - return new CChainedWriteSeq(iwsa); -} - -extern jlib_decl IReadSeq *createChainedReadSeq(IReadSeqAllocator *irsa) -{ - return new CChainedReadSeq(irsa); -} - - - IRowStream *createConcatRowStream(unsigned numstreams,IRowStream** streams,bool grouped) { switch(numstreams) diff --git a/system/jlib/jio.hpp b/system/jlib/jio.hpp index 99ff19041e0..948e9c69245 100644 --- a/system/jlib/jio.hpp +++ b/system/jlib/jio.hpp @@ -47,26 +47,6 @@ interface IRecordSize: public IInterface #endif -interface IReadSeq : public IInterface -{ -// fixed length record read interface - virtual void reset() = 0; - virtual bool get(void *dst) = 0; - virtual unsigned getn(void *dst, unsigned numrecs) = 0; - virtual size32_t getRecordSize() = 0; - virtual void stop() = 0; // indicate finished reading -}; - -interface IWriteSeq : public IInterface -{ -// fixed length record write interface - virtual void flush() = 0; - virtual void put(const void *dst) = 0; - virtual void putn(const void *dst, unsigned numrecs) = 0; - virtual size32_t getRecordSize() = 0; - virtual offset_t getPosition() = 0; -}; - interface ISimpleReadStream : public IInterface { virtual size32_t read(size32_t max_len, void * data) = 0; @@ -109,30 +89,10 @@ interface IReceiver : public IInterface virtual bool takeRecord(offset_t pos) = 0; }; -interface IWriteSeqAllocator : public IInterface -{ - virtual IWriteSeq *next(size32_t &num) = 0; -}; - -interface IReadSeqAllocator : public IInterface -{ - virtual IReadSeq *next() = 0; -}; - - -extern jlib_decl IReadSeq *createReadSeq(int fh, offset_t _offset, size32_t size, size32_t _bufsize = (size32_t)-1, // bufsize in bytes - unsigned maxrecs=(unsigned)-1, bool compress=false); // compression is *not* blocked and needs buffer size -extern jlib_decl IWriteSeq *createWriteSeq(int fh, size32_t size, size32_t bufsize = (size32_t)-1,bool compress=false); // compression is *not* blocked and needs buffer size -extern jlib_decl IWriteSeq *createTeeWriteSeq(IWriteSeq *, IWriteSeq *); -extern jlib_decl IWriteSeq *createChainedWriteSeq(IWriteSeqAllocator *iwsa); -extern jlib_decl IReadSeq *createChainedReadSeq(IReadSeqAllocator *irsa); - extern jlib_decl IRecordSize *createFixedRecordSize(size32_t recsize); extern jlib_decl IRecordSize *createDeltaRecordSize(IRecordSize * size, int delta); -extern jlib_decl unsigned copySeq(IReadSeq *from,IWriteSeq *to,size32_t bufsize); - extern jlib_decl void setIORetryCount(unsigned _ioRetryCount); // default 0 == off, retries if read op. fails extern jlib_decl offset_t checked_lseeki64(int handle, offset_t offset, int origin); extern jlib_decl size32_t checked_write(const char * filename, int handle, const void *buffer, size32_t count); diff --git a/system/jlib/jio.ipp b/system/jlib/jio.ipp index 0c6c597df78..174b2536aa1 100644 --- a/system/jlib/jio.ipp +++ b/system/jlib/jio.ipp @@ -29,52 +29,6 @@ -class CUnbufferedReadWriteSeq : public IWriteSeq, public IReadSeq, public CInterface -{ -private: - offset_t offset; - offset_t fpos; - size32_t size; - int fh; - -public: - IMPLEMENT_IINTERFACE; - - CUnbufferedReadWriteSeq(int _fh, offset_t offset, size32_t _size); - - virtual void put(const void *src); - virtual void putn(const void *src, unsigned n); - virtual void flush(); - virtual size32_t getRecordSize() { return size; } - virtual offset_t getPosition(); - - virtual bool get(void *dst); - virtual unsigned getn(void *dst, unsigned n); - virtual void reset(); - virtual void stop() {} // no action required -}; - -class CTeeWriteSeq : public IWriteSeq, public CInterface -{ -protected: - IWriteSeq *w1; - IWriteSeq *w2; - -public: - IMPLEMENT_IINTERFACE; - - CTeeWriteSeq(IWriteSeq *f1, IWriteSeq *f2); - ~CTeeWriteSeq(); - - virtual void put(const void *src); - virtual void putn(const void *src, unsigned n); - virtual void flush(); - virtual size32_t getRecordSize(); - virtual offset_t getPosition(); -}; - - - class CBufferedIOStreamBase: public CInterface { public: diff --git a/system/jlib/jlog.cpp b/system/jlib/jlog.cpp index 59a7b625a72..ee167a2ee82 100644 --- a/system/jlib/jlog.cpp +++ b/system/jlib/jlog.cpp @@ -3364,7 +3364,7 @@ void ctxlogReport(const LogMsgCategory & cat, const IException * e, const char * { StringBuffer buff; e->errorMessage(buff); - ctxlogReport(cat, e->errorCode(), "%s%s%s", prefix ? prefix : "", prefix ? prefix : " : ", buff.str()); + ctxlogReport(cat, e->errorCode(), "%s%s%s", prefix ? prefix : "", prefix ? " : " : "", buff.str()); } IException * ctxlogReport(IException * e, const char * prefix, LogMsgClass cls) { diff --git a/system/jlib/jlz4.cpp b/system/jlib/jlz4.cpp index 2c32f362c15..6a854c39fbf 100644 --- a/system/jlib/jlz4.cpp +++ b/system/jlib/jlz4.cpp @@ -276,7 +276,7 @@ class jlib_decl CLZ4Expander : public CFcmpExpander for (;;) { - //Try and compress into the current target buffer. If too small increase size and repeat + //Try and decompress into the current target buffer. If too small increase size and repeat written = LZ4_decompress_safe((const char *)in, (char *)target.reserve(maxOut), szchunk, maxOut); if ((int)written > 0) { diff --git a/system/jlib/jlzw.cpp b/system/jlib/jlzw.cpp index 827223950b1..f289c185717 100644 --- a/system/jlib/jlzw.cpp +++ b/system/jlib/jlzw.cpp @@ -2529,7 +2529,7 @@ static bool isCompressedType(__int64 compressedType) return 0 != getCompressedMethod(compressedType); } -bool isCompressedFile(IFileIO *iFileIO, CompressedFileTrailer *trailer=nullptr) +static bool isCompressedFile(IFileIO *iFileIO, CompressedFileTrailer *trailer=nullptr) { if (iFileIO) { @@ -2547,6 +2547,16 @@ bool isCompressedFile(IFileIO *iFileIO, CompressedFileTrailer *trailer=nullptr) return true; } } + else if ((fsize == 0) && trailer) + { + //If the file is empty, but we are expecting a compressed file, fill in the trailer with default information + memset(trailer,0,sizeof(*trailer)); + trailer->crc = ~0U; + trailer->compressedType = LZ4COMPRESSEDFILEFLAG; + trailer->blockSize = LZ4COMPRESSEDFILEBLOCKSIZE; + trailer->recordSize = 0; + return true; + } } return false; } diff --git a/system/jlib/jsort.cpp b/system/jlib/jsort.cpp index 4086911b776..6e810ed7973 100644 --- a/system/jlib/jsort.cpp +++ b/system/jlib/jsort.cpp @@ -696,12 +696,6 @@ bool heap_push_up(unsigned c, unsigned * heap, const void ** rows, ICompare * co typedef void ** VECTOR; -interface IMergeSorter -{ -public: - virtual IWriteSeq *getOutputStream(bool isEOF) = 0; -}; - #define INSERTMAX 10000 #define BUFFSIZE 0x100000 // used for output buffer diff --git a/system/security/cryptohelper/digisign.cpp b/system/security/cryptohelper/digisign.cpp index 41b714bf36e..6d20cb7647e 100755 --- a/system/security/cryptohelper/digisign.cpp +++ b/system/security/cryptohelper/digisign.cpp @@ -271,7 +271,6 @@ IDigitalSignatureManager * createDigitalSignatureManagerInstanceFromFiles(const exceptions.setown(makeMultiException("createDigitalSignatureManagerInstanceFromFiles")); exceptions->append(* makeWrappedExceptionV(e, -1, "createDigitalSignatureManagerInstanceFromFiles:Cannot load public key file")); - e->Release(); } } @@ -287,7 +286,6 @@ IDigitalSignatureManager * createDigitalSignatureManagerInstanceFromFiles(const exceptions.setown(makeMultiException("createDigitalSignatureManagerInstanceFromFiles")); exceptions->append(* makeWrappedExceptionV(e, -1, "createDigitalSignatureManagerInstanceFromFiles:Cannot load private key file")); - e->Release(); } } @@ -331,7 +329,6 @@ IDigitalSignatureManager * createDigitalSignatureManagerInstanceFromKeys(const c exceptions.setown(makeMultiException("createDigitalSignatureManagerInstanceFromKeys")); exceptions->append(* makeWrappedExceptionV(e, -1, "createDigitalSignatureManagerInstanceFromKeys:Cannot load public key")); - e->Release(); } } if (!isEmptyString(privKeyString)) @@ -346,7 +343,6 @@ IDigitalSignatureManager * createDigitalSignatureManagerInstanceFromKeys(const c exceptions.setown(makeMultiException("createDigitalSignatureManagerInstanceFromKeys")); exceptions->append(* makeWrappedExceptionV(e, -1, "createDigitalSignatureManagerInstanceFromKeys:Cannot load private key")); - e->Release(); } } diff --git a/thorlcr/activities/thactivityutil.cpp b/thorlcr/activities/thactivityutil.cpp index 2beb728f227..afa10f9b282 100644 --- a/thorlcr/activities/thactivityutil.cpp +++ b/thorlcr/activities/thactivityutil.cpp @@ -786,6 +786,7 @@ IFileIO *createMultipleWrite(CActivityBase *activity, IPartDescriptor &partDesc, { unsigned compMethod = COMPRESS_METHOD_LZ4; // rowdif used if recordSize > 0, else fallback to compMethod + IFEflags fileIOExtaFlags = IFEnone; if (!ecomp) { if (twFlags & TW_Temporary) @@ -808,7 +809,7 @@ IFileIO *createMultipleWrite(CActivityBase *activity, IPartDescriptor &partDesc, else if (activity->getOptBool(THOROPT_COMP_FORCELZ4HC, false)) compMethod = COMPRESS_METHOD_LZ4HC; } - fileio.setown(createCompressedFileWriter(file, recordSize, 0 != (twFlags & TW_Extend), true, ecomp, compMethod)); + fileio.setown(createCompressedFileWriter(file, recordSize, 0 != (twFlags & TW_Extend), true, ecomp, compMethod, fileIOExtaFlags)); if (!fileio) { compress = false; diff --git a/thorlcr/master/thgraphmanager.cpp b/thorlcr/master/thgraphmanager.cpp index 8f7f0ed6086..955e8fb9716 100644 --- a/thorlcr/master/thgraphmanager.cpp +++ b/thorlcr/master/thgraphmanager.cpp @@ -1345,23 +1345,26 @@ static int recvNextGraph(unsigned timeoutMs, const char *wuid, StringBuffer &ret } -static std::vector> connectedWorkerPods; -void addConnectedWorkerPod(const char *podName, const char *containerName) -{ - connectedWorkerPods.emplace_back(podName, containerName); -} - -void publishPodNames(IWorkUnit *workunit, const char *graphName) +static std::vector connectedWorkers; +void publishPodNames(IWorkUnit *workunit, const char *graphName, const std::vector *_connectedWorkers) { // skip if Thor manager already published (implying worker pods already published too) // NB: this will always associate the new 'graphName' with the manager pod meta info. if (workunit->setContainerizedProcessInfo("Thor", globals->queryProp("@name"), k8s::queryMyPodName(), k8s::queryMyContainerName(), graphName, nullptr)) { - for (unsigned workerNum=0; workerNumsetContainerizedProcessInfo("ThorWorker", globals->queryProp("@name"), workerPodName, workerContainerName, nullptr, std::to_string(workerNum+1).c_str()); + const CConnectedWorkerDetail &worker = connectedWorkers[workerNum]; + workunit->setContainerizedProcessInfo("ThorWorker", globals->queryProp("@name"), worker.podName.c_str(), worker.containerName.c_str(), nullptr, std::to_string(workerNum+1).c_str()); } } } @@ -1485,7 +1488,7 @@ void thorMain(ILogMsgHandler *logHandler, const char *wuid, const char *graphNam { Owned wu = &workunit->lock(); - publishPodNames(wu, currentGraphName); + publishPodNames(wu, currentGraphName, nullptr); } SocketEndpoint dummyAgentEp; jobManager->execute(workunit, currentWuid, currentGraphName, dummyAgentEp); diff --git a/thorlcr/master/thgraphmanager.hpp b/thorlcr/master/thgraphmanager.hpp index 2c53522b21d..c3a50234c91 100644 --- a/thorlcr/master/thgraphmanager.hpp +++ b/thorlcr/master/thgraphmanager.hpp @@ -18,6 +18,9 @@ #ifndef _THGRAPHMANAGER_HPP #define _THGRAPHMANAGER_HPP +#include +#include + class CSDSServerStatus; interface IException; CSDSServerStatus &queryServerStatus(); @@ -31,8 +34,21 @@ void abortThor(IException *e, unsigned errCode, bool abortCurrentJob=true); void setExitCode(int code); int queryExitCode(); -void addConnectedWorkerPod(const char *podName, const char *containerName); -void publishPodNames(IWorkUnit *workunit, const char *graphName); +struct CConnectedWorkerDetail +{ + CConnectedWorkerDetail() {} + CConnectedWorkerDetail(const std::string& h) : host(h) + { + } + CConnectedWorkerDetail(const std::string& h, const std::string& p, const std::string& c) : host(h), podName(p), containerName(c) + { + } + std::string host; + std::string podName; + std::string containerName; + +}; +void publishPodNames(IWorkUnit *workunit, const char *graphName, const std::vector *connectedWorkers); void relayWuidException(IConstWorkUnit *wu, const IException *exception); void auditThorSystemEvent(const char *eventName); void auditThorSystemEvent(const char *eventName, std::initializer_list args); diff --git a/thorlcr/master/thmastermain.cpp b/thorlcr/master/thmastermain.cpp index b9af9d5b57c..0414548c55e 100644 --- a/thorlcr/master/thmastermain.cpp +++ b/thorlcr/master/thmastermain.cpp @@ -19,6 +19,10 @@ #include "platform.h" +#include +#include +#include + #include #include #include @@ -266,8 +270,8 @@ class CRegistryServer : public CSimpleInterface } void connect(unsigned workers) { - IPointerArrayOf connectedWorkers; - connectedWorkers.ensureCapacity(workers); + std::vector connectedWorkers; + connectedWorkers.reserve(workers); unsigned remaining = workers; INode *_sender = nullptr; CMessageBuffer msg; @@ -300,11 +304,15 @@ class CRegistryServer : public CSimpleInterface else { Owned sender = _sender; - if (NotFound != connectedWorkers.find(sender)) + StringBuffer workerEPStr; + sender->endpoint().getEndpointHostText(workerEPStr); + + auto findFunc = [&workerEPStr](const CConnectedWorkerDetail& worker) { - StringBuffer epStr; - throw makeStringExceptionV(TE_AbortException, "Same worker registered twice!! : %s", sender->endpoint().getEndpointHostText(epStr).str()); - } + return streq(worker.host.c_str(), workerEPStr.str()); + }; + if (connectedWorkers.end() != std::find_if(connectedWorkers.begin(), connectedWorkers.end(), findFunc)) + throw makeStringExceptionV(TE_AbortException, "Same worker registered twice!! : %s", workerEPStr.str()); /* NB: in base metal setup, the workers know which worker number they are in advance, and send their workerNum at registration. * In non attached storage setup, they do not send a worker by default and instead are given a # once all are registered @@ -314,42 +322,29 @@ class CRegistryServer : public CSimpleInterface StringBuffer workerPodName, workerContainerName; if (NotFound == workerNum) { - connectedWorkers.append(sender.getLink()); - workerNum = connectedWorkers.ordinality(); if (isContainerized()) { msg.read(workerPodName); msg.read(workerContainerName); - addConnectedWorkerPod(workerPodName, workerContainerName); // NB: these are added in worker # order } + connectedWorkers.emplace_back(workerEPStr.str(), workerPodName.str(), workerContainerName.str()); + PROGLOG("Worker connected from %s", workerEPStr.str()); } else { unsigned pos = workerNum - 1; // NB: workerNum is 1 based - while (connectedWorkers.ordinality() < pos) - connectedWorkers.append(nullptr); - if (connectedWorkers.ordinality() == pos) - connectedWorkers.append(sender.getLink()); + while (connectedWorkers.size() < pos) + connectedWorkers.emplace_back(); + if (connectedWorkers.size() == pos) + connectedWorkers.emplace_back(workerEPStr.str()); else - connectedWorkers.replace(sender.getLink(), pos); + connectedWorkers[pos] = {workerEPStr.str()}; + PROGLOG("Worker %u connected from %s", workerNum, workerEPStr.str()); } - StringBuffer epStr; - PROGLOG("Worker %u connected from %s", workerNum, sender->endpoint().getEndpointHostText(epStr).str()); --remaining; } } - assertex(workers == connectedWorkers.ordinality()); - - if (isContainerized()) - { - unsigned wfid = globals->getPropInt("@wfid"); - const char *wuid = globals->queryProp("@workunit"); - const char *graphName = globals->queryProp("@graphName"); - Owned factory = getWorkUnitFactory(); - Owned workunit = factory->updateWorkUnit(wuid); - addTimeStamp(workunit, wfid, graphName, StWhenK8sReady); - publishPodNames(workunit, graphName); - } + assertex(workers == connectedWorkers.size()); unsigned localThorPortInc = globals->getPropInt("@localThorPortInc", DEFAULT_WORKERPORTINC); unsigned workerBasePort = globals->getPropInt("@slaveport", DEFAULT_THORWORKERPORT); @@ -357,31 +352,43 @@ class CRegistryServer : public CSimpleInterface Owned processGroup; - // NB: in bare metal Thor is bound to a group and cluster/communicator have alreday been setup (see earlier setClusterGroup call) + // NB: in bare metal Thor is bound to a group and cluster/communicator have already been setup (see earlier setClusterGroup call) if (clusterInitialized()) processGroup.set(&queryProcessGroup()); else { - /* sort by {port, ip} - * So that workers are not bunched on same node, but striped across the pod ips - */ - auto compareINodeOrder = [](IInterface * const *ll, IInterface * const *rr) + if (isContainerized()) { - INode *l = (INode *) *ll; - INode *r = (INode *) *rr; - const SocketEndpoint &lep = l->endpoint(); - const SocketEndpoint &rep = r->endpoint(); - if (lep.port < rep.port) - return -1; - else if (lep.port > rep.port) - return 1; - return lep.ipcompare(rep); - }; - connectedWorkers.sort(compareINodeOrder); - processGroup.setown(createIGroup(connectedWorkers.ordinality(), connectedWorkers.getArray())); + // sort by pod+container name so that storage striping doesn't clump too much within single pods + auto sortFunc = [](const CConnectedWorkerDetail& a, const CConnectedWorkerDetail& b) + { + if (a.podName != b.podName) + return a.podName < b.podName; + return a.containerName < b.containerName; + }; + std::sort(connectedWorkers.begin(), connectedWorkers.end(), sortFunc); + } + SocketEndpointArray connectedWorkerEps; + for (const auto &worker: connectedWorkers) + { + SocketEndpoint ep(worker.host.c_str()); + connectedWorkerEps.append(ep); + } + processGroup.setown(createIGroup(connectedWorkerEps)); setupCluster(queryMyNode(), processGroup, channelsPerWorker, workerBasePort, localThorPortInc); } + if (isContainerized()) + { + unsigned wfid = globals->getPropInt("@wfid"); + const char *wuid = globals->queryProp("@workunit"); + const char *graphName = globals->queryProp("@graphName"); + Owned factory = getWorkUnitFactory(); + Owned workunit = factory->updateWorkUnit(wuid); + addTimeStamp(workunit, wfid, graphName, StWhenK8sReady); + publishPodNames(workunit, graphName, &connectedWorkers); + } + PROGLOG("Workers connected, initializing.."); msg.clear(); msg.append(THOR_VERSION_MAJOR).append(THOR_VERSION_MINOR);