Merge remote-tracking branch 'origin/candidate-9.8.x'

Signed-off-by: Gavin Halliday <[email protected]>

# Conflicts:
#	helm/hpcc/Chart.yaml
#	helm/hpcc/templates/_helpers.tpl
#	version.cmake

common/thorhelper/thorsoapcall.cpp

@@ -2629,7 +2629,7 @@ class CWSCAsyncFor : implements IWSCAsyncFor, public CInterface, public CAsyncFo
 
                 StringBuffer spanName;
                 spanName.appendf("%s %s %s:%d", getWsCallTypeName(master->wscType), master->service.str(), url.host.str(), url.port);
-                OwnedSpanScope requestSpan = master->activitySpanScope->createClientSpan(spanName.str());
+                OwnedActiveSpanScope requestSpan = master->activitySpanScope->createClientSpan(spanName.str());
 
                 setSpanURLAttributes(requestSpan, url);
                 requestSpan->setSpanAttribute("request.type", getWsCallTypeName(master->wscType));

dali/daliadmin/daadmin.cpp

@@ -16,6 +16,7 @@
 ############################################################################## */
 
 #include <unordered_map>
+#include <unordered_set>
 #include <string>
 
 #include "platform.h"
@@ -3499,4 +3500,76 @@ void cleanJobQueues(bool dryRun)
     }
 }
 
+void cleanGeneratedDlls(bool dryRun, bool backup)
+{
+    PROGLOG("Gathering workunits for referencd generated dlls");
+    CCycleTimer timer;
+    Owned<IRemoteConnection> conn = querySDS().connect("/", myProcessSession(), 0, SDS_LOCK_TIMEOUT);
+    if (!conn)
+    {
+        WARNLOG("Failed to connect to /WorkUnits");
+        return;
+    }
+    IPropertyTree *root = conn->queryRoot();
+    IPropertyTree *wuidsTree = root->queryPropTree("WorkUnits");
+    if (!wuidsTree)
+    {
+        PROGLOG("No WorkUnits found");
+        return;
+    }
+    Owned<IPropertyTreeIterator> wuidIter = wuidsTree->getElements("*");
+    std::unordered_set<std::string> referencedGDlls;
+    ForEach(*wuidIter)
+    {
+        IPropertyTree &wuid = wuidIter->query();
+        Owned<IPropertyTreeIterator> gdIter = wuid.getElements("Query/Associated/File[@type='dll']");
+        ForEach(*gdIter)
+        {
+            IPropertyTree &gd = gdIter->query();
+            const char *fullPath = gd.queryProp("@filename");
+            const char *filename = pathTail(fullPath);
+            if (filename)
+                referencedGDlls.emplace(filename);
+        }
+    }
+    PROGLOG("Found %u workunits that reference generated dlls, took: %u ms", (unsigned)referencedGDlls.size(), timer.elapsedMs());
+    PROGLOG("Scanning GeneratedDlls");
+    timer.reset();
+    IPropertyTree *generatedDllsTree = root->queryPropTree("GeneratedDlls");
+    if (!generatedDllsTree)
+    {
+        PROGLOG("No GeneratedDlls found");
+        return;
+    }
+    std::vector<IPropertyTree *> gDllsToRemove;
+    unsigned totalGDlls = 0;
+    Owned<IPropertyTreeIterator> gDlls = generatedDllsTree->getElements("*");
+    ForEach(*gDlls)
+    {
+        ++totalGDlls;
+        IPropertyTree &gDll = gDlls->query();
+        const char *name = gDll.queryProp("@name");
+        if (referencedGDlls.find(name) == referencedGDlls.end())
+            gDllsToRemove.push_back(&gDll);
+    }
+    PROGLOG("Found %u GeneratedDlls, %u not referenced, took: %u ms", totalGDlls, (unsigned)gDllsToRemove.size(), timer.elapsedMs());
+    if (!dryRun)
+    {
+        if (backup)
+        {
+            StringBuffer bakName;
+            Owned<IFileIO> iFileIO = createUniqueFile(NULL, "daliadmin_generateddlls", "bak", bakName);
+            if (!iFileIO)
+                throw makeStringException(0, "Failed to create backup file");
+            PROGLOG("Saving backup of GeneratedDlls to %s", bakName.str());
+            saveXML(*iFileIO, generatedDllsTree, 2);
+        }
+        PROGLOG("Deleting %u unreferenced GeneratedDlls", (unsigned)gDllsToRemove.size());
+        timer.reset();
+        for (auto &item: gDllsToRemove)
+            generatedDllsTree->removeTree(item);
+        PROGLOG("Removed %u unreferenced GeneratedDlls, took: %u ms", (unsigned)gDllsToRemove.size(), timer.elapsedMs());
+    }
+}
+
 } // namespace daadmin

dali/daliadmin/daadmin.hpp

@@ -97,5 +97,6 @@ extern DALIADMIN_API void daliping(const char *dalis, unsigned connecttime, unsi
 extern DALIADMIN_API void validateStore(bool fix, bool deleteFiles, bool verbose);
 extern DALIADMIN_API void removeOrphanedGlobalVariables(bool dryrun, bool reconstruct);
 extern DALIADMIN_API void cleanJobQueues(bool dryRun);
+extern DALIADMIN_API void cleanGeneratedDlls(bool dryRun, bool backup);
 
 } // namespace daadmin

dali/daliadmin/daliadmin.cpp

@@ -94,6 +94,7 @@ void usage(const char *exe)
   printf("  auditlog <fromdate> <todate> <match>\n");
   printf("  cleanglobalwuid [dryrun] [noreconstruct]\n");
   printf("  cleanjobqueues [dryrun]\n");
+  printf("  cleangenerateddlls [dryrun] [nobackup]\n");
   printf("  clusterlist <mask>              -- list clusters   (mask optional)\n");
   printf("  coalesce                        -- force transaction coalesce\n");
   printf("  dalilocks [ <ip-pattern> ] [ files ] -- get all locked files/xpaths\n");
@@ -597,6 +598,20 @@ int main(int argc, const char* argv[])
                         bool dryRun = np>0 && strieq("dryrun", params.item(1));
                         cleanJobQueues(dryRun);
                     }
+                    else if (strieq(cmd, "cleangenerateddlls"))
+                    {
+                        bool dryRun = false;
+                        bool backup = true; // default
+                        for (unsigned i=1; i<params.ordinality(); i++)
+                        {
+                            const char *param = params.item(i);
+                            if (strieq("dryrun", param))
+                                dryRun = true;
+                            else if (strieq("nobackup", param))
+                                backup = false;
+                        }
+                        cleanGeneratedDlls(dryRun, backup);
+                    }
                     else if (strieq(cmd, "remotetest"))
                         remoteTest(params.item(1), true);
                     else

ecl/eclagent/eclagent.cpp

@@ -2156,7 +2156,7 @@ void EclAgent::runProcess(IEclProcess *process)
     allocatorMetaCache.setown(createRowAllocatorCache(this));
 
     Owned<IProperties> traceHeaders = extractTraceDebugOptions(queryWorkUnit());
-    OwnedSpanScope requestSpan = queryTraceManager().createServerSpan("run_workunit", traceHeaders);
+    OwnedActiveSpanScope requestSpan = queryTraceManager().createServerSpan("run_workunit", traceHeaders);
     ContextSpanScope spanScope(updateDummyContextLogger(), requestSpan);
     requestSpan->setSpanAttribute("hpcc.wuid", queryWorkUnit()->queryWuid());
 

esp/platform/espcontext.cpp

@@ -89,7 +89,7 @@ class CEspContext : public CInterface, implements IEspContext
     Owned<IEspSecureContextEx> m_secureContext;
 
     StringAttr   m_transactionID;
-    OwnedSpanScope m_requestSpan;    // When the context is destroy the span will end.
+    OwnedActiveSpanScope m_requestSpan;    // When the context is destroy the span will end.
     IHttpMessage* m_request;
 
 public:

esp/services/esdl_svc_engine/esdl_binding.cpp

@@ -1572,7 +1572,7 @@ void EsdlServiceImpl::sendTargetSOAP(IEspContext & context,
     }
 
     ISpan * activeSpan = context.queryActiveSpan();
-    OwnedSpanScope clientSpan(activeSpan->createClientSpan("soapcall"));
+    OwnedActiveSpanScope clientSpan(activeSpan->createClientSpan("soapcall"));
 
     Owned<IProperties> headers = ::getClientHeaders(clientSpan);
     StringBuffer status;

esp/services/ws_ecl/ws_ecl_service.cpp

@@ -2069,7 +2069,7 @@ int CWsEclBinding::submitWsEclWorkunit(IEspContext & context, WsEclWuInfo &wsinf
     bool noTimeout = false;
 
     ISpan * activeSpan = context.queryActiveSpan();
-    OwnedSpanScope clientSpan(activeSpan->createClientSpan("run_workunit"));
+    OwnedActiveSpanScope clientSpan(activeSpan->createClientSpan("run_workunit"));
     Owned<IProperties> httpHeaders = ::getClientHeaders(clientSpan);
     recordTraceDebugOptions(workunit, httpHeaders);
 

esp/services/ws_workunits/ws_workunitsHelpers.cpp

@@ -3794,7 +3794,7 @@ void WsWuHelpers::submitWsWorkunit(IEspContext& context, IConstWorkUnit* cw, con
         }
 
         ISpan * activeSpan = context.queryActiveSpan();
-        OwnedSpanScope clientSpan(activeSpan->createClientSpan("run_workunit"));
+        OwnedActiveSpanScope clientSpan(activeSpan->createClientSpan("run_workunit"));
         Owned<IProperties> httpHeaders = ::getClientHeaders(clientSpan);
         recordTraceDebugOptions(wu, httpHeaders);
 
@@ -4497,7 +4497,11 @@ void CWsWuFileHelper::zipZAPFiles(const char* parentFolder, const char* zapFiles
     else
         zipCommand.setf("cd %s\nzip -r", parentFolder);
     if (!isEmptyString(passwordReq))
-        zipCommand.append(" --password ").append(passwordReq);
+    {
+        StringBuffer sanitizedPassword;
+        sanitizeCommandArg(passwordReq, sanitizedPassword);
+        zipCommand.append(" --password ").append(sanitizedPassword);
+    }
     zipCommand.append(" ").append(zipFileNameWithFullPath).append(" ").append(zapFiles);
     int zipRet = system(zipCommand);
     if (zipRet != 0)

esp/src/src-react/components/Logs.tsx

@@ -113,7 +113,7 @@ export const Logs: React.FunctionComponent<LogsProps> = ({
 
     const now = React.useMemo(() => new Date(), []);
 
-    const { columns: logColumns } = useLogAccessInfo();
+    const { logsColumns: logColumns } = useLogAccessInfo();
 
     //  Grid ---
     const columns = React.useMemo((): FluentColumns => {

esp/src/src-react/components/Menu.tsx

@@ -2,11 +2,11 @@ import * as React from "react";
 import { IconButton, IContextualMenuItem, INavLink, INavLinkGroup, Link, mergeStyleSets, Nav, Stack } from "@fluentui/react";
 import { useConst } from "@fluentui/react-hooks";
 import nlsHPCC from "src/nlsHPCC";
-import { hasLogAccess } from "src/ESPLog";
 import { containerized, bare_metal } from "src/BuildInfo";
 import { navCategory } from "../util/history";
 import { MainNav, routes } from "../routes";
 import { useFavorite, useFavorites, useHistory } from "../hooks/favorite";
+import { useLogAccessInfo } from "../hooks/platform";
 import { useSessionStore } from "../hooks/store";
 import { useUserTheme } from "../hooks/theme";
 import { useMyAccount } from "../hooks/user";
@@ -295,12 +295,7 @@ export const SubNavigation: React.FunctionComponent<SubNavigationProps> = ({
         }
     }), [theme]);
 
-    const [logsDisabled, setLogsDisabled] = React.useState(true);
-    React.useEffect(() => {
-        hasLogAccess().then(response => {
-            setLogsDisabled(!response);
-        });
-    }, []);
+    const { logsEnabled, logsStatusMessage } = useLogAccessInfo();
     const linkStyle = React.useCallback((disabled) => {
         return disabled ? {
             background: themeV9.colorNeutralBackgroundDisabled,
@@ -326,9 +321,10 @@ export const SubNavigation: React.FunctionComponent<SubNavigationProps> = ({
                 <Stack horizontal>
                     <Stack.Item grow={0} className={navStyles.wrapper}>
                         {subMenuItems[mainNav]?.map((row, idx) => {
-                            const linkDisabled = (row.itemKey === "/topology/logs" && logsDisabled) || (row.itemKey.indexOf("security") > -1 && !isAdmin);
+                            const linkDisabled = (row.itemKey === "/topology/logs" && !logsEnabled) || (row.itemKey.indexOf("security") > -1 && !isAdmin);
                             return <Link
                                 disabled={linkDisabled}
+                                title={row.itemKey === "/topology/logs" && !logsEnabled ? logsStatusMessage : ""}
                                 key={`MenuLink_${idx}`}
                                 href={`#${row.itemKey}`}
                                 className={[

esp/src/src-react/components/WorkunitDetails.tsx

@@ -4,11 +4,10 @@ import { WsWorkunits, WorkunitsService } from "@hpcc-js/comms";
 import { scopedLogger } from "@hpcc-js/util";
 import { SizeMe } from "react-sizeme";
 import nlsHPCC from "src/nlsHPCC";
-import { hasLogAccess } from "src/ESPLog";
 import { wuidToDate, wuidToTime } from "src/Utility";
 import { emptyFilter, formatQuery } from "src/ESPWorkunit";
+import { useLogAccessInfo } from "../hooks/platform";
 import { Variable, useWorkunit, useWorkunitVariables } from "../hooks/workunit";
-import { useDeepEffect } from "../hooks/deepHooks";
 import { DojoAdapter } from "../layouts/DojoAdapter";
 import { FullscreenFrame, FullscreenStack } from "../layouts/Fullscreen";
 import { parseQuery, pushUrl, updateFullscreen } from "../util/history";
@@ -58,7 +57,7 @@ export const WorkunitDetails: React.FunctionComponent<WorkunitDetailsProps> = ({
     const [variables, , , refreshVariables] = useWorkunitVariables(wuid);
     const [otTraceParent, setOtTraceParent] = React.useState("");
     const [logCount, setLogCount] = React.useState<number | string>("*");
-    const [logsDisabled, setLogsDisabled] = React.useState(true);
+    const { logsEnabled, logsStatusMessage } = useLogAccessInfo();
     const [_nextPrev, setNextPrev] = useNextPrev();
 
     const query = React.useMemo(() => {
@@ -121,13 +120,6 @@ export const WorkunitDetails: React.FunctionComponent<WorkunitDetailsProps> = ({
         };
     }, [nextWuid, query, setNextPrev, wuid]);
 
-    useDeepEffect(() => {
-        hasLogAccess().then(response => {
-            setLogsDisabled(!response);
-            return response;
-        });
-    }, [wuid], [queryParams]);
-
     const onTabSelect = React.useCallback((tab: TabInfo) => {
         pushUrl(tab.__state ?? `${parentUrl}/${wuid}/${tab.id}`);
         updateFullscreen(fullscreen);
@@ -174,15 +166,16 @@ export const WorkunitDetails: React.FunctionComponent<WorkunitDetailsProps> = ({
             id: "logs",
             label: nlsHPCC.Logs,
             count: logCount,
-            disabled: logsDisabled
+            tooltipText: !logsEnabled ? (logsStatusMessage || nlsHPCC.LogsDisabled) : null,
+            disabled: !logsEnabled
         }, {
             id: "eclsummary",
             label: nlsHPCC.ECL
         }, {
             id: "xml",
             label: nlsHPCC.XML
         }];
-    }, [logCount, logsDisabled, workunit?.ApplicationValueCount, workunit?.DebugValueCount, workunit?.GraphCount, workunit?.HelpersCount, workunit?.ResourceURLCount, workunit?.ResultCount, workunit?.SourceFileCount, workunit?.VariableCount, workunit?.WorkflowCount, wuid]);
+    }, [logCount, logsEnabled, logsStatusMessage, workunit?.ApplicationValueCount, workunit?.DebugValueCount, workunit?.GraphCount, workunit?.HelpersCount, workunit?.ResourceURLCount, workunit?.ResultCount, workunit?.SourceFileCount, workunit?.VariableCount, workunit?.WorkflowCount, wuid]);
 
     return <FullscreenFrame fullscreen={fullscreen}>
         <SizeMe monitorHeight>{({ size }) =>

esp/src/src-react/components/controls/TabbedPanes/OverflowTabList.tsx

@@ -1,5 +1,5 @@
 import * as React from "react";
-import { Overflow, OverflowItem, SelectTabData, SelectTabEvent, Tab, TabList } from "@fluentui/react-components";
+import { Overflow, OverflowItem, SelectTabData, SelectTabEvent, Tab, TabList, Tooltip } from "@fluentui/react-components";
 import { Count } from "./Count";
 import { TabInfo } from "./TabInfo";
 import { OverflowMenu } from "../OverflowMenu";
@@ -28,7 +28,9 @@ export const OverflowTabList: React.FunctionComponent<OverflowTabListProps> = ({
                 tab.__state = state;
             }
             return <OverflowItem key={tab.id} id={tab.id} priority={tab.id === selected ? 2 : 1}>
-                <Tab value={tab.id} icon={tab.icon} disabled={tab.disabled}>{tab.label}<Count value={tab.count} /></Tab>
+                <Tooltip content={tab.tooltipText || tab.label} relationship="label">
+                    <Tab value={tab.id} icon={tab.icon} disabled={tab.disabled}>{tab.label}<Count value={tab.count} /></Tab>
+                </Tooltip>
             </OverflowItem>;
         }), tabsIndex];
     }, [selected, state, tabs]);

esp/src/src-react/components/controls/TabbedPanes/TabInfo.ts

@@ -6,5 +6,6 @@ export interface TabInfo {
     label: string;
     count?: string | number;
     disabled?: boolean;
+    tooltipText?: string;
     __state?: any;
 }

esp/src/src-react/hooks/platform.ts

@@ -3,6 +3,7 @@ import { Octokit } from "octokit";
 import { useConst } from "@fluentui/react-hooks";
 import { scopedLogger } from "@hpcc-js/util";
 import { LogaccessService, Topology, WsLogaccess, WsTopology, WorkunitsServiceEx } from "@hpcc-js/comms";
+import nlsHPCC from "src/nlsHPCC";
 import { getBuildInfo, BuildInfo, fetchModernMode } from "src/Session";
 import { cmake_build_type, containerized, ModernMode } from "src/BuildInfo";
 import { sessionKeyValStore, userKeyValStore } from "src/KeyValStore";
@@ -209,19 +210,34 @@ export function useModernMode(): {
     return { modernMode, setModernMode };
 }
 
-export function useLogAccessInfo(): {
-    managerType: string;
-    columns: WsLogaccess.Column[]
-} {
-    const [managerType, setManagerType] = React.useState("");
-    const [columns, setColumns] = React.useState<WsLogaccess.Column[]>();
+interface LogAccessInfo {
+    logsEnabled: boolean;
+    logsManagerType: string;
+    logsColumns: WsLogaccess.Column[];
+    logsStatusMessage: string;
+}
+
+export function useLogAccessInfo(): LogAccessInfo {
+    const [logsEnabled, setLogsEnabled] = React.useState(false);
+    const [logsManagerType, setLogsManagerType] = React.useState("");
+    const [logsColumns, setLogsColumns] = React.useState<WsLogaccess.Column[]>();
+    const [logsStatusMessage, setLogsStatusMessage] = React.useState("");
 
     React.useEffect(() => {
         service.GetLogAccessInfo({}).then(response => {
-            setManagerType(response.RemoteLogManagerType ?? "");
-            setColumns(response?.Columns?.Column);
+            if (response.hasOwnProperty("Exceptions")) {
+                setLogsStatusMessage(response["Exceptions"]?.Exception[0]?.Message ?? nlsHPCC.LogAccess_GenericException);
+            } else {
+                if (response.RemoteLogManagerType === null) {
+                    setLogsStatusMessage(nlsHPCC.LogAccess_LoggingNotConfigured);
+                } else {
+                    setLogsEnabled(true);
+                    setLogsManagerType(response.RemoteLogManagerType);
+                    setLogsColumns(response?.Columns?.Column);
+                }
+            }
         });
     }, []);
 
-    return { managerType, columns };
+    return { logsEnabled, logsManagerType, logsColumns, logsStatusMessage };
 }

esp/src/src/nls/hpcc.ts

@@ -496,6 +496,8 @@ export = {
         Location: "Location",
         Lock: "Lock",
         LogAccessType: "Log Access Type",
+        LogAccess_GenericException: "ws_logaccess::GetLogAccessInfo, an exception has occurred.",
+        LogAccess_LoggingNotConfigured: "A logging engine has not been configured.",
         LogDirectory: "Log Directory",
         LogEventType: "Log Event Type",
         LogFile: "Log File",
@@ -529,6 +531,7 @@ export = {
         Login: "Login",
         Logout: "Log Out",
         Logs: "Logs",
+        LogsDisabled: "Logs Disabled",
         LogVisualization: "Log Visualization",
         LogVisualizationUnconfigured: "Log Visualization is not configured, please check your configuration manager settings.",
         log_analysis_1: "log_analysis_1*",