Merge remote-tracking branch 'origin/candidate-9.4.x'

Signed-off-by: Gavin Halliday <[email protected]>

build-config.h.cmake

@@ -94,6 +94,14 @@
     #define BUILD_VERSION_POINT ${HPCC_POINT}
 #endif
 
+#ifndef BUILD_MATURITY
+    #define BUILD_MATURITY "${HPCC_MATURITY}"
+#endif
+
+#ifndef BUILD_TAG_TIMESTAMP
+    #define BUILD_TAG_TIMESTAMP "${HPCC_TAG_TIMESTAMP}"
+#endif
+
 #ifndef BASE_BUILD_TAG
     #cmakedefine BASE_BUILD_TAG "${BASE_BUILD_TAG}"
 #endif

common/thorhelper/thorcommon.hpp

@@ -678,7 +678,7 @@ class CStatsContextLogger : public CSimpleInterfaceOf<IContextLogger>
 protected:
     const LogMsgJobInfo job;
     unsigned traceLevel = 1;
-    Owned<ISpan> activeSpan;
+    Owned<ISpan> activeSpan = getNullSpan();
     mutable CRuntimeStatisticCollection stats;
 public:
     CStatsContextLogger(const CRuntimeStatisticCollection  &_mapping, const LogMsgJobInfo & _job=unknownJob) : job(_job), stats(_mapping) {}
@@ -723,26 +723,18 @@ class CStatsContextLogger : public CSimpleInterfaceOf<IContextLogger>
     }
     virtual IProperties * getClientHeaders() const override
     {
-        if (!activeSpan)
-            return nullptr;
         return ::getClientHeaders(activeSpan);
     }
     virtual const char *queryGlobalId() const override
     {
-        if (!activeSpan)
-            return nullptr;
         return activeSpan->queryGlobalId();
     }
     virtual const char *queryLocalId() const override
     {
-        if (!activeSpan)
-            return nullptr;
         return activeSpan->queryLocalId();
     }
     virtual const char *queryCallerId() const override
     {
-        if (!activeSpan)
-            return nullptr;
         return activeSpan->queryCallerId();
     }
     virtual const CRuntimeStatisticCollection &queryStats() const override

common/wuanalysis/anacommon.cpp

@@ -71,7 +71,7 @@ void PerformanceIssue::createException(IWorkUnit * wu, double costRate)
         s.appendf(" cost %.2f", timePenaltyPerHour*costRate);
     }
     we->setExceptionMessage(s.str());
-    we->setExceptionSource("Workunit Analyzer");
+    we->setExceptionSource(CostOptimizerName);
 }
 
 void PerformanceIssue::set(AnalyzerErrorCode _errorCode, stat_type _timePenalty, const char * msg, ...)

common/wuanalysis/anacommon.hpp

@@ -23,6 +23,7 @@
 #include "eclhelper.hpp"
 #include "anaerrorcodes.hpp"
 
+constexpr char CostOptimizerName[] = "Cost Optimizer";
 
 interface IWuScope
 {

common/wuanalysis/anawu.cpp

@@ -2098,7 +2098,7 @@ void WUANALYSIS_API analyseAndPrintIssues(IConstWorkUnit * wu, double costRate,
     if (updatewu)
     {
         Owned<IWorkUnit> lockedwu = &(wu->lock());
-        lockedwu->clearExceptions("Workunit Analyzer");
+        lockedwu->clearExceptions(CostOptimizerName);
         analyser.update(lockedwu, costRate);
     }
 }

dali/base/dadfs.cpp

@@ -316,6 +316,8 @@ class DECL_EXCEPTION CDFS_Exception: implements IDFS_Exception, public CInterfac
             return str.append(": Storage plane missing: ").append(errstr);
         case DFSERR_PhysicalCompressedPartInvalid:
             return str.append(": Compressed part is not in the valid format: ").append(errstr);
+        case DFSERR_InvalidRemoteFileContext:
+            return str.append(": Lookup of remote files must use wsdfs::lookup - file: ").append(errstr);
         }
         return str.append("Unknown DFS Exception");
     }
@@ -8247,6 +8249,11 @@ IDistributedFile *CDistributedFileDirectory::dolookup(CDfsLogicalFileName &_logi
 
 IDistributedFile *CDistributedFileDirectory::lookup(CDfsLogicalFileName &logicalname, IUserDescriptor *user, AccessMode accessMode, bool hold, bool lockSuperOwner, IDistributedFileTransaction *transaction, bool privilegedUser, unsigned timeout)
 {
+    if (logicalname.isRemote())
+    {
+        PrintStackReport(); // to help locate contexts it was called in
+        throw new CDFS_Exception(DFSERR_InvalidRemoteFileContext, logicalname.get());
+    }
     Owned <IDistributedFile>distributedFile = dolookup(logicalname, user, accessMode, hold, lockSuperOwner, transaction, timeout);
     // Restricted access is currently designed to stop users viewing sensitive information. It is not designed to stop users deleting or overwriting existing restricted files
     if (!isWrite(accessMode) && distributedFile && distributedFile->isRestrictedAccess() && !privilegedUser)
@@ -10194,25 +10201,27 @@ class CInitGroups
         switch (groupType)
         {
             case grp_thor:
-                getClusterGroupName(cluster, gname);
-                if (!streq(cluster.queryProp("@name"), gname.str()))
+                getClusterGroupName(cluster, gname); // NB: ensures lowercases
+                if (!strieq(cluster.queryProp("@name"), gname.str()))
                     realCluster = false;
                 if (oldEnvCluster)
                 {
-                    getClusterGroupName(*oldEnvCluster, oldGname);
-                    if (!streq(oldEnvCluster->queryProp("@name"), oldGname.str()))
+                    getClusterGroupName(*oldEnvCluster, oldGname); // NB: ensures lowercases
+                    if (!strieq(oldEnvCluster->queryProp("@name"), oldGname.str()))
                         oldRealCluster = false;
                 }
                 break;
             case grp_thorspares:
-                getClusterSpareGroupName(cluster, gname);
+                getClusterSpareGroupName(cluster, gname); // ensures lowercase
                 oldRealCluster = realCluster = false;
                 break;
             case grp_roxie:
                 gname.append(cluster.queryProp("@name"));
+                gname.toLowerCase();
                 break;
             case grp_dropzone:
                 gname.append(cluster.queryProp("@name"));
+                gname.toLowerCase();
                 oldRealCluster = realCluster = false;
                 defDir = cluster.queryProp("@directory");
                 break;
@@ -10584,8 +10593,12 @@ class CInitGroups
 
     void ensureStorageGroup(bool force, const char * name, unsigned numDevices, const char * path, StringBuffer & messages)
     {
-        Owned<IPropertyTree> newClusterGroup = createStorageGroup(name, numDevices, path);
-        ensureConsistentStorageGroup(force, name, newClusterGroup, messages);
+        //Lower case the group name - see CNamedGroupStore::dolookup which lower cases before resolving.
+        StringBuffer gname;
+        gname.append(name).toLowerCase();
+
+        Owned<IPropertyTree> newClusterGroup = createStorageGroup(gname, numDevices, path);
+        ensureConsistentStorageGroup(force, gname, newClusterGroup, messages);
     }
 
     void constructStorageGroups(bool force, StringBuffer &messages)
@@ -10624,7 +10637,7 @@ class CInitGroups
                 }
                 else if (plane.hasProp("hostGroup"))
                 {
-                    throw makeStringExceptionV(-1, "Use 'hosts' rather than 'hostGroup' for inline list of hosts for plane %s", name);
+                    throw makeStringExceptionV(-1, "Use 'hosts' rather than 'hostGroup' for inline list of hosts for plane %s", gname.str());
                 }
                 else if (plane.hasProp("hosts"))
                 {
@@ -10639,9 +10652,9 @@ class CInitGroups
                 {
                     //Locally mounted, or url accessed storage plane - no associated hosts, localhost used as a placeholder
                     unsigned numDevices = plane.getPropInt("@numDevices", 1);
-                    newClusterGroup.setown(createStorageGroup(name, numDevices, prefix));
+                    newClusterGroup.setown(createStorageGroup(gname, numDevices, prefix));
                 }
-                ensureConsistentStorageGroup(force, name, newClusterGroup, messages);
+                ensureConsistentStorageGroup(force, gname, newClusterGroup, messages);
             }
         }
     }

dali/base/dadfs.hpp

@@ -803,7 +803,8 @@ enum DistributedFileSystemError
     DFSERR_RestrictedFileAccessDenied,
     DFSERR_EmptyStoragePlane,
     DFSERR_MissingStoragePlane,
-    DFSERR_PhysicalCompressedPartInvalid
+    DFSERR_PhysicalCompressedPartInvalid,
+    DFSERR_InvalidRemoteFileContext
 };
 
 

esp/scm/ws_workunits_req_resp.ecm

@@ -939,6 +939,7 @@ ESPresponse [exceptions_inline] WUCreateZAPInfoResponse
 
 ESPrequest [nil_remove] WUCheckFeaturesRequest
 {
+    boolean IncludeFullVersion(false);
 };
 
 ESPresponse [exceptions_inline] WUCheckFeaturesResponse
@@ -948,6 +949,12 @@ ESPresponse [exceptions_inline] WUCheckFeaturesResponse
     int BuildVersionPoint;
     unsigned maxRequestEntityLength;
     ESPStruct DeploymentFeatures Deployment;
+    //The following fields are only generated if IncludeFullVersion is set.  Normally the fields would be versioned,
+    //but this change is applied to a much earlier version and the version number has been incremented several times,
+    //so that approach cannot be used.
+    string BuildVersion;
+    string BuildMaturity;
+    string BuildTagTimestamp;
 };
 
 ESPrequest [nil_remove] WUGetStatsRequest

esp/services/esdl_svc_engine/esdl_binding.cpp

@@ -1573,8 +1573,7 @@ void EsdlServiceImpl::sendTargetSOAP(IEspContext & context,
 
     Owned<ISpan> clientSpan;
     ISpan * activeSpan = context.queryActiveSpan();
-    if (activeSpan)
-        clientSpan.setown(activeSpan->createClientSpan("soapcall"));
+    clientSpan.setown(activeSpan->createClientSpan("soapcall"));
 
     Owned<IProperties> headers = ::getClientHeaders(clientSpan);
     StringBuffer status;

esp/services/ws_ecl/ws_ecl_service.cpp

@@ -1996,12 +1996,9 @@ int CWsEclBinding::submitWsEclWorkunit(IEspContext & context, WsEclWuInfo &wsinf
 
     Owned<ISpan> clientSpan;
     ISpan * activeSpan = context.queryActiveSpan();
-    if (activeSpan)
-    {
-        clientSpan.setown(activeSpan->createClientSpan("wsecl/SubmitWorkunit"));
-        Owned<IProperties> httpHeaders = ::getClientHeaders(clientSpan);
-        recordTraceDebugOptions(workunit, httpHeaders);
-    }
+    clientSpan.setown(activeSpan->createClientSpan("wsecl/SubmitWorkunit"));
+    Owned<IProperties> httpHeaders = ::getClientHeaders(clientSpan);
+    recordTraceDebugOptions(workunit, httpHeaders);
 
     if (httpreq)
     {

esp/services/ws_workunits/ws_workunitsService.cpp

@@ -5085,6 +5085,13 @@ bool CWsWorkunitsEx::onWUCheckFeatures(IEspContext &context, IEspWUCheckFeatures
     resp.setBuildVersionMajor(hpccBuildInfo.buildVersionMajor);
     resp.setBuildVersionMinor(hpccBuildInfo.buildVersionMinor);
     resp.setBuildVersionPoint(hpccBuildInfo.buildVersionPoint);
+    //This does not check version because a consistent version could not be used for all the places it was included
+    if (req.getIncludeFullVersion())
+    {
+        resp.setBuildVersion(hpccBuildInfo.buildTag);
+        resp.setBuildMaturity(hpccBuildInfo.buildMaturity);
+        resp.setBuildTagTimestamp(hpccBuildInfo.buildTagTimestamp);
+    }
     resp.setMaxRequestEntityLength(maxRequestEntityLength);
     resp.updateDeployment().setUseCompression(true);
     return true;

esp/src/eclwatch/stub.js

@@ -30,8 +30,7 @@ define([
     const params = ioQuery.queryToObject(dojo.doc.location.search.substr((dojo.doc.location.search.substr(0, 1) === "?" ? 1 : 0)));
     const hpccWidget = params.Widget ? params.Widget : "HPCCPlatformWidget";
 
-    const store = KeyValStore.userKeyValStore();
-    store.getEx(BuildInfo.ModernMode, { defaultValue: String(true) }).then(modernMode => {
+    Session.fetchModernMode().then(modernMode => {
         if (modernMode === String(true) && hpccWidget !== "IFrameWidget") {
             switch (hpccWidget) {
                 case "WUDetailsWidget":

esp/src/src-react/components/InfoGrid.tsx

@@ -84,6 +84,12 @@ export const InfoGrid: React.FunctionComponent<InfoGridProps> = ({
             },
             Column: { label: nlsHPCC.Col, field: "", width: 36, sortable: false },
             LineNo: { label: nlsHPCC.Line, field: "", width: 36, sortable: false },
+            Activity: {
+                label: nlsHPCC.Activity, field: "", width: 56, sortable: false,
+                formatter: (activityId, row) => {
+                    return activityId ? <Link href={`#/workunits/${wuid}/metrics/a${activityId}`}>a{activityId}</Link> : "";
+                }
+            },
             FileName: { label: nlsHPCC.FileName, field: "", width: 360, sortable: false }
         };
     }, [wuid]);

esp/src/src-react/components/Logs.tsx

@@ -18,7 +18,7 @@ const startTimeOffset = 6 * 60 * 60 * 1000;
 const defaultStartDate = new Date(new Date().getTime() - startTimeOffset);
 
 const FilterFields: Fields = {
-    containerName: { type: "cloud-containername", label: nlsHPCC.ContainerName },
+    components: { type: "cloud-containername", label: nlsHPCC.ContainerName },
     audience: {
         type: "dropdown", label: nlsHPCC.Audience, options: [
             { key: TargetAudience.Operator, text: "Operator" },
@@ -37,9 +37,9 @@ const FilterFields: Fields = {
             { key: LogType.Metric, text: "Metric" },
         ]
     },
-    jobId: { type: "string", label: nlsHPCC.JobID },
-    procId: { type: "string", label: nlsHPCC.ProcessID },
-    threadId: { type: "string", label: nlsHPCC.ThreadID },
+    workunits: { type: "string", label: nlsHPCC.JobID },
+    processid: { type: "string", label: nlsHPCC.ProcessID },
+    threadid: { type: "string", label: nlsHPCC.ThreadID },
     message: { type: "string", label: nlsHPCC.Message },
     StartDate: { type: "datetime", label: nlsHPCC.FromDate },
     EndDate: { type: "datetime", label: nlsHPCC.ToDate },
@@ -117,7 +117,7 @@ export const Logs: React.FunctionComponent<LogsProps> = ({
         return {
             timestamp: { label: nlsHPCC.TimeStamp, width: 140, sortable: false, },
             message: { label: nlsHPCC.Message, sortable: false, },
-            containerName: { label: nlsHPCC.ContainerName, width: 100, sortable: false },
+            components: { label: nlsHPCC.ContainerName, width: 150, sortable: false },
             audience: { label: nlsHPCC.Audience, width: 60, sortable: false, },
             class: {
                 label: nlsHPCC.Class, width: 40, sortable: false,
@@ -127,10 +127,10 @@ export const Logs: React.FunctionComponent<LogsProps> = ({
                     return <span style={styles}>{level}</span>;
                 }
             },
-            jobId: { label: nlsHPCC.JobID, width: 140, sortable: false, hidden: wuid !== undefined, },
-            procId: { label: nlsHPCC.ProcessID, width: 46, sortable: false, },
-            sequence: { label: nlsHPCC.Sequence, width: 70, sortable: false, },
-            threadId: { label: nlsHPCC.ThreadID, width: 60, sortable: false, },
+            workunits: { label: nlsHPCC.JobID, width: 50, sortable: false, hidden: wuid !== undefined, },
+            processid: { label: nlsHPCC.ProcessID, width: 75, sortable: false, },
+            logid: { label: nlsHPCC.Sequence, width: 70, sortable: false, },
+            threadid: { label: nlsHPCC.ThreadID, width: 60, sortable: false, },
         };
     }, [wuid]);
 

esp/src/src-react/components/Title.tsx

@@ -8,12 +8,11 @@ import { cookie } from "dojo/main";
 
 import nlsHPCC from "src/nlsHPCC";
 import * as Utility from "src/Utility";
-import { ModernMode } from "src/BuildInfo";
 
 import { useBanner } from "../hooks/banner";
 import { useECLWatchLogger } from "../hooks/logging";
-import { useBuildInfo } from "../hooks/platform";
-import { useGlobalStore, useUserStore } from "../hooks/store";
+import { useBuildInfo, useModernMode } from "../hooks/platform";
+import { useGlobalStore } from "../hooks/store";
 import { useMyAccount, useUserSession } from "../hooks/user";
 import { replaceUrl } from "../util/history";
 
@@ -71,7 +70,7 @@ export const DevTitle: React.FunctionComponent<DevTitleProps> = ({
 
     const [log, logLastUpdated] = useECLWatchLogger();
 
-    const [_modernMode, setModernMode] = useUserStore(ModernMode, String(true));
+    const { setModernMode } = useModernMode();
     const onTechPreviewClick = React.useCallback(
         (ev?: React.MouseEvent<HTMLButtonElement>, item?: IContextualMenuItem): void => {
             setModernMode(String(false));

esp/src/src-react/components/controls/ComingSoon.tsx

@@ -1,9 +1,7 @@
 import * as React from "react";
 import { IStyle, Toggle } from "@fluentui/react";
 import nlsHPCC from "src/nlsHPCC";
-import { ModernMode } from "src/BuildInfo";
-import { useUserStore } from "../../hooks/store";
-import { useBuildInfo } from "../../hooks/platform";
+import { useBuildInfo, useModernMode } from "../../hooks/platform";
 
 const legacyIndex = {};
 const modernIndex = {};
@@ -75,7 +73,7 @@ export const ComingSoon: React.FunctionComponent<ComingSoon> = ({
 }) => {
 
     const [, { opsCategory }] = useBuildInfo();
-    const [modernMode, setModernMode] = useUserStore(ModernMode, String(defaultValue));
+    const { modernMode, setModernMode } = useModernMode();
 
     React.useEffect(() => {
         if (value !== undefined) {

esp/src/src-react/hooks/platform.ts

@@ -1,8 +1,10 @@
 import * as React from "react";
+import { useConst } from "@fluentui/react-hooks";
 import { scopedLogger } from "@hpcc-js/util";
 import { Topology, WsTopology } from "@hpcc-js/comms";
-import { getBuildInfo, BuildInfo } from "src/Session";
-import { cmake_build_type, containerized } from "src/BuildInfo";
+import { getBuildInfo, BuildInfo, fetchModernMode } from "src/Session";
+import { cmake_build_type, containerized, ModernMode } from "src/BuildInfo";
+import { sessionKeyValStore, userKeyValStore } from "src/KeyValStore";
 
 const logger = scopedLogger("src-react/hooks/platform.ts");
 
@@ -66,3 +68,27 @@ export function useLogicalClusters(): [WsTopology.TpLogicalCluster[] | undefined
 
     return [targetClusters, defaultCluster];
 }
+
+export function useModernMode(): {
+    modernMode: string;
+    setModernMode: (value: string) => void;
+} {
+    const userStore = useConst(() => userKeyValStore());
+    const sessionStore = useConst(() => sessionKeyValStore());
+
+    const [modernMode, setModernMode] = React.useState<string>("");
+
+    React.useEffect(() => {
+        fetchModernMode().then(mode => {
+            setModernMode(String(mode));
+        });
+    }, [sessionStore, userStore]);
+
+    React.useEffect(() => {
+        if (modernMode === "") return;
+        sessionStore.set(ModernMode, modernMode);
+        userStore.set(ModernMode, modernMode);
+    }, [modernMode, sessionStore, userStore]);
+
+    return { modernMode, setModernMode };
+}