diff --git a/common/thorhelper/thorsoapcall.cpp b/common/thorhelper/thorsoapcall.cpp index c696e5e8624..0daaf9193f3 100644 --- a/common/thorhelper/thorsoapcall.cpp +++ b/common/thorhelper/thorsoapcall.cpp @@ -2629,7 +2629,7 @@ class CWSCAsyncFor : implements IWSCAsyncFor, public CInterface, public CAsyncFo StringBuffer spanName; spanName.appendf("%s %s %s:%d", getWsCallTypeName(master->wscType), master->service.str(), url.host.str(), url.port); - OwnedSpanScope requestSpan = master->activitySpanScope->createClientSpan(spanName.str()); + OwnedActiveSpanScope requestSpan = master->activitySpanScope->createClientSpan(spanName.str()); setSpanURLAttributes(requestSpan, url); requestSpan->setSpanAttribute("request.type", getWsCallTypeName(master->wscType)); diff --git a/dali/daliadmin/daadmin.cpp b/dali/daliadmin/daadmin.cpp index 4274d44db10..cce002d4190 100644 --- a/dali/daliadmin/daadmin.cpp +++ b/dali/daliadmin/daadmin.cpp @@ -16,6 +16,7 @@ ############################################################################## */ #include +#include #include #include "platform.h" @@ -3499,4 +3500,76 @@ void cleanJobQueues(bool dryRun) } } +void cleanGeneratedDlls(bool dryRun, bool backup) +{ + PROGLOG("Gathering workunits for referencd generated dlls"); + CCycleTimer timer; + Owned conn = querySDS().connect("/", myProcessSession(), 0, SDS_LOCK_TIMEOUT); + if (!conn) + { + WARNLOG("Failed to connect to /WorkUnits"); + return; + } + IPropertyTree *root = conn->queryRoot(); + IPropertyTree *wuidsTree = root->queryPropTree("WorkUnits"); + if (!wuidsTree) + { + PROGLOG("No WorkUnits found"); + return; + } + Owned wuidIter = wuidsTree->getElements("*"); + std::unordered_set referencedGDlls; + ForEach(*wuidIter) + { + IPropertyTree &wuid = wuidIter->query(); + Owned gdIter = wuid.getElements("Query/Associated/File[@type='dll']"); + ForEach(*gdIter) + { + IPropertyTree &gd = gdIter->query(); + const char *fullPath = gd.queryProp("@filename"); + const char *filename = pathTail(fullPath); + if (filename) + referencedGDlls.emplace(filename); + } + } + PROGLOG("Found %u workunits that reference generated dlls, took: %u ms", (unsigned)referencedGDlls.size(), timer.elapsedMs()); + PROGLOG("Scanning GeneratedDlls"); + timer.reset(); + IPropertyTree *generatedDllsTree = root->queryPropTree("GeneratedDlls"); + if (!generatedDllsTree) + { + PROGLOG("No GeneratedDlls found"); + return; + } + std::vector gDllsToRemove; + unsigned totalGDlls = 0; + Owned gDlls = generatedDllsTree->getElements("*"); + ForEach(*gDlls) + { + ++totalGDlls; + IPropertyTree &gDll = gDlls->query(); + const char *name = gDll.queryProp("@name"); + if (referencedGDlls.find(name) == referencedGDlls.end()) + gDllsToRemove.push_back(&gDll); + } + PROGLOG("Found %u GeneratedDlls, %u not referenced, took: %u ms", totalGDlls, (unsigned)gDllsToRemove.size(), timer.elapsedMs()); + if (!dryRun) + { + if (backup) + { + StringBuffer bakName; + Owned iFileIO = createUniqueFile(NULL, "daliadmin_generateddlls", "bak", bakName); + if (!iFileIO) + throw makeStringException(0, "Failed to create backup file"); + PROGLOG("Saving backup of GeneratedDlls to %s", bakName.str()); + saveXML(*iFileIO, generatedDllsTree, 2); + } + PROGLOG("Deleting %u unreferenced GeneratedDlls", (unsigned)gDllsToRemove.size()); + timer.reset(); + for (auto &item: gDllsToRemove) + generatedDllsTree->removeTree(item); + PROGLOG("Removed %u unreferenced GeneratedDlls, took: %u ms", (unsigned)gDllsToRemove.size(), timer.elapsedMs()); + } +} + } // namespace daadmin diff --git a/dali/daliadmin/daadmin.hpp b/dali/daliadmin/daadmin.hpp index 1209584dcff..fdeff4f8fd8 100644 --- a/dali/daliadmin/daadmin.hpp +++ b/dali/daliadmin/daadmin.hpp @@ -97,5 +97,6 @@ extern DALIADMIN_API void daliping(const char *dalis, unsigned connecttime, unsi extern DALIADMIN_API void validateStore(bool fix, bool deleteFiles, bool verbose); extern DALIADMIN_API void removeOrphanedGlobalVariables(bool dryrun, bool reconstruct); extern DALIADMIN_API void cleanJobQueues(bool dryRun); +extern DALIADMIN_API void cleanGeneratedDlls(bool dryRun, bool backup); } // namespace daadmin \ No newline at end of file diff --git a/dali/daliadmin/daliadmin.cpp b/dali/daliadmin/daliadmin.cpp index 40f39f45fff..ba43e55a700 100644 --- a/dali/daliadmin/daliadmin.cpp +++ b/dali/daliadmin/daliadmin.cpp @@ -94,6 +94,7 @@ void usage(const char *exe) printf(" auditlog \n"); printf(" cleanglobalwuid [dryrun] [noreconstruct]\n"); printf(" cleanjobqueues [dryrun]\n"); + printf(" cleangenerateddlls [dryrun] [nobackup]\n"); printf(" clusterlist -- list clusters (mask optional)\n"); printf(" coalesce -- force transaction coalesce\n"); printf(" dalilocks [ ] [ files ] -- get all locked files/xpaths\n"); @@ -597,6 +598,20 @@ int main(int argc, const char* argv[]) bool dryRun = np>0 && strieq("dryrun", params.item(1)); cleanJobQueues(dryRun); } + else if (strieq(cmd, "cleangenerateddlls")) + { + bool dryRun = false; + bool backup = true; // default + for (unsigned i=1; i traceHeaders = extractTraceDebugOptions(queryWorkUnit()); - OwnedSpanScope requestSpan = queryTraceManager().createServerSpan("run_workunit", traceHeaders); + OwnedActiveSpanScope requestSpan = queryTraceManager().createServerSpan("run_workunit", traceHeaders); ContextSpanScope spanScope(updateDummyContextLogger(), requestSpan); requestSpan->setSpanAttribute("hpcc.wuid", queryWorkUnit()->queryWuid()); diff --git a/esp/platform/espcontext.cpp b/esp/platform/espcontext.cpp index e5016f6506e..2e3d3dabf05 100755 --- a/esp/platform/espcontext.cpp +++ b/esp/platform/espcontext.cpp @@ -89,7 +89,7 @@ class CEspContext : public CInterface, implements IEspContext Owned m_secureContext; StringAttr m_transactionID; - OwnedSpanScope m_requestSpan; // When the context is destroy the span will end. + OwnedActiveSpanScope m_requestSpan; // When the context is destroy the span will end. IHttpMessage* m_request; public: diff --git a/esp/services/esdl_svc_engine/esdl_binding.cpp b/esp/services/esdl_svc_engine/esdl_binding.cpp index 54783bfafb4..d2110932726 100755 --- a/esp/services/esdl_svc_engine/esdl_binding.cpp +++ b/esp/services/esdl_svc_engine/esdl_binding.cpp @@ -1572,7 +1572,7 @@ void EsdlServiceImpl::sendTargetSOAP(IEspContext & context, } ISpan * activeSpan = context.queryActiveSpan(); - OwnedSpanScope clientSpan(activeSpan->createClientSpan("soapcall")); + OwnedActiveSpanScope clientSpan(activeSpan->createClientSpan("soapcall")); Owned headers = ::getClientHeaders(clientSpan); StringBuffer status; diff --git a/esp/services/ws_ecl/ws_ecl_service.cpp b/esp/services/ws_ecl/ws_ecl_service.cpp index 0c330e61cc8..581da49e666 100644 --- a/esp/services/ws_ecl/ws_ecl_service.cpp +++ b/esp/services/ws_ecl/ws_ecl_service.cpp @@ -2069,7 +2069,7 @@ int CWsEclBinding::submitWsEclWorkunit(IEspContext & context, WsEclWuInfo &wsinf bool noTimeout = false; ISpan * activeSpan = context.queryActiveSpan(); - OwnedSpanScope clientSpan(activeSpan->createClientSpan("run_workunit")); + OwnedActiveSpanScope clientSpan(activeSpan->createClientSpan("run_workunit")); Owned httpHeaders = ::getClientHeaders(clientSpan); recordTraceDebugOptions(workunit, httpHeaders); diff --git a/esp/services/ws_workunits/ws_workunitsHelpers.cpp b/esp/services/ws_workunits/ws_workunitsHelpers.cpp index 4c35663c515..ba7ebd43c58 100644 --- a/esp/services/ws_workunits/ws_workunitsHelpers.cpp +++ b/esp/services/ws_workunits/ws_workunitsHelpers.cpp @@ -3794,7 +3794,7 @@ void WsWuHelpers::submitWsWorkunit(IEspContext& context, IConstWorkUnit* cw, con } ISpan * activeSpan = context.queryActiveSpan(); - OwnedSpanScope clientSpan(activeSpan->createClientSpan("run_workunit")); + OwnedActiveSpanScope clientSpan(activeSpan->createClientSpan("run_workunit")); Owned httpHeaders = ::getClientHeaders(clientSpan); recordTraceDebugOptions(wu, httpHeaders); @@ -4497,7 +4497,11 @@ void CWsWuFileHelper::zipZAPFiles(const char* parentFolder, const char* zapFiles else zipCommand.setf("cd %s\nzip -r", parentFolder); if (!isEmptyString(passwordReq)) - zipCommand.append(" --password ").append(passwordReq); + { + StringBuffer sanitizedPassword; + sanitizeCommandArg(passwordReq, sanitizedPassword); + zipCommand.append(" --password ").append(sanitizedPassword); + } zipCommand.append(" ").append(zipFileNameWithFullPath).append(" ").append(zapFiles); int zipRet = system(zipCommand); if (zipRet != 0) diff --git a/esp/src/src-react/components/Logs.tsx b/esp/src/src-react/components/Logs.tsx index c02f3388e5f..2091699f373 100644 --- a/esp/src/src-react/components/Logs.tsx +++ b/esp/src/src-react/components/Logs.tsx @@ -113,7 +113,7 @@ export const Logs: React.FunctionComponent = ({ const now = React.useMemo(() => new Date(), []); - const { columns: logColumns } = useLogAccessInfo(); + const { logsColumns: logColumns } = useLogAccessInfo(); // Grid --- const columns = React.useMemo((): FluentColumns => { diff --git a/esp/src/src-react/components/Menu.tsx b/esp/src/src-react/components/Menu.tsx index d2c1e3da292..f9bc7b5a5e3 100644 --- a/esp/src/src-react/components/Menu.tsx +++ b/esp/src/src-react/components/Menu.tsx @@ -2,11 +2,11 @@ import * as React from "react"; import { IconButton, IContextualMenuItem, INavLink, INavLinkGroup, Link, mergeStyleSets, Nav, Stack } from "@fluentui/react"; import { useConst } from "@fluentui/react-hooks"; import nlsHPCC from "src/nlsHPCC"; -import { hasLogAccess } from "src/ESPLog"; import { containerized, bare_metal } from "src/BuildInfo"; import { navCategory } from "../util/history"; import { MainNav, routes } from "../routes"; import { useFavorite, useFavorites, useHistory } from "../hooks/favorite"; +import { useLogAccessInfo } from "../hooks/platform"; import { useSessionStore } from "../hooks/store"; import { useUserTheme } from "../hooks/theme"; import { useMyAccount } from "../hooks/user"; @@ -295,12 +295,7 @@ export const SubNavigation: React.FunctionComponent = ({ } }), [theme]); - const [logsDisabled, setLogsDisabled] = React.useState(true); - React.useEffect(() => { - hasLogAccess().then(response => { - setLogsDisabled(!response); - }); - }, []); + const { logsEnabled, logsStatusMessage } = useLogAccessInfo(); const linkStyle = React.useCallback((disabled) => { return disabled ? { background: themeV9.colorNeutralBackgroundDisabled, @@ -326,9 +321,10 @@ export const SubNavigation: React.FunctionComponent = ({ {subMenuItems[mainNav]?.map((row, idx) => { - const linkDisabled = (row.itemKey === "/topology/logs" && logsDisabled) || (row.itemKey.indexOf("security") > -1 && !isAdmin); + const linkDisabled = (row.itemKey === "/topology/logs" && !logsEnabled) || (row.itemKey.indexOf("security") > -1 && !isAdmin); return = ({ const [variables, , , refreshVariables] = useWorkunitVariables(wuid); const [otTraceParent, setOtTraceParent] = React.useState(""); const [logCount, setLogCount] = React.useState("*"); - const [logsDisabled, setLogsDisabled] = React.useState(true); + const { logsEnabled, logsStatusMessage } = useLogAccessInfo(); const [_nextPrev, setNextPrev] = useNextPrev(); const query = React.useMemo(() => { @@ -121,13 +120,6 @@ export const WorkunitDetails: React.FunctionComponent = ({ }; }, [nextWuid, query, setNextPrev, wuid]); - useDeepEffect(() => { - hasLogAccess().then(response => { - setLogsDisabled(!response); - return response; - }); - }, [wuid], [queryParams]); - const onTabSelect = React.useCallback((tab: TabInfo) => { pushUrl(tab.__state ?? `${parentUrl}/${wuid}/${tab.id}`); updateFullscreen(fullscreen); @@ -174,7 +166,8 @@ export const WorkunitDetails: React.FunctionComponent = ({ id: "logs", label: nlsHPCC.Logs, count: logCount, - disabled: logsDisabled + tooltipText: !logsEnabled ? (logsStatusMessage || nlsHPCC.LogsDisabled) : null, + disabled: !logsEnabled }, { id: "eclsummary", label: nlsHPCC.ECL @@ -182,7 +175,7 @@ export const WorkunitDetails: React.FunctionComponent = ({ id: "xml", label: nlsHPCC.XML }]; - }, [logCount, logsDisabled, workunit?.ApplicationValueCount, workunit?.DebugValueCount, workunit?.GraphCount, workunit?.HelpersCount, workunit?.ResourceURLCount, workunit?.ResultCount, workunit?.SourceFileCount, workunit?.VariableCount, workunit?.WorkflowCount, wuid]); + }, [logCount, logsEnabled, logsStatusMessage, workunit?.ApplicationValueCount, workunit?.DebugValueCount, workunit?.GraphCount, workunit?.HelpersCount, workunit?.ResourceURLCount, workunit?.ResultCount, workunit?.SourceFileCount, workunit?.VariableCount, workunit?.WorkflowCount, wuid]); return {({ size }) => diff --git a/esp/src/src-react/components/controls/TabbedPanes/OverflowTabList.tsx b/esp/src/src-react/components/controls/TabbedPanes/OverflowTabList.tsx index e9cf3520ccf..63a15308e08 100644 --- a/esp/src/src-react/components/controls/TabbedPanes/OverflowTabList.tsx +++ b/esp/src/src-react/components/controls/TabbedPanes/OverflowTabList.tsx @@ -1,5 +1,5 @@ import * as React from "react"; -import { Overflow, OverflowItem, SelectTabData, SelectTabEvent, Tab, TabList } from "@fluentui/react-components"; +import { Overflow, OverflowItem, SelectTabData, SelectTabEvent, Tab, TabList, Tooltip } from "@fluentui/react-components"; import { Count } from "./Count"; import { TabInfo } from "./TabInfo"; import { OverflowMenu } from "../OverflowMenu"; @@ -28,7 +28,9 @@ export const OverflowTabList: React.FunctionComponent = ({ tab.__state = state; } return - {tab.label} + + {tab.label} + ; }), tabsIndex]; }, [selected, state, tabs]); diff --git a/esp/src/src-react/components/controls/TabbedPanes/TabInfo.ts b/esp/src/src-react/components/controls/TabbedPanes/TabInfo.ts index 7d87929b3a6..d21edad8120 100644 --- a/esp/src/src-react/components/controls/TabbedPanes/TabInfo.ts +++ b/esp/src/src-react/components/controls/TabbedPanes/TabInfo.ts @@ -6,5 +6,6 @@ export interface TabInfo { label: string; count?: string | number; disabled?: boolean; + tooltipText?: string; __state?: any; } diff --git a/esp/src/src-react/hooks/platform.ts b/esp/src/src-react/hooks/platform.ts index 33f2089c260..e403a37c119 100644 --- a/esp/src/src-react/hooks/platform.ts +++ b/esp/src/src-react/hooks/platform.ts @@ -3,6 +3,7 @@ import { Octokit } from "octokit"; import { useConst } from "@fluentui/react-hooks"; import { scopedLogger } from "@hpcc-js/util"; import { LogaccessService, Topology, WsLogaccess, WsTopology, WorkunitsServiceEx } from "@hpcc-js/comms"; +import nlsHPCC from "src/nlsHPCC"; import { getBuildInfo, BuildInfo, fetchModernMode } from "src/Session"; import { cmake_build_type, containerized, ModernMode } from "src/BuildInfo"; import { sessionKeyValStore, userKeyValStore } from "src/KeyValStore"; @@ -209,19 +210,34 @@ export function useModernMode(): { return { modernMode, setModernMode }; } -export function useLogAccessInfo(): { - managerType: string; - columns: WsLogaccess.Column[] -} { - const [managerType, setManagerType] = React.useState(""); - const [columns, setColumns] = React.useState(); +interface LogAccessInfo { + logsEnabled: boolean; + logsManagerType: string; + logsColumns: WsLogaccess.Column[]; + logsStatusMessage: string; +} + +export function useLogAccessInfo(): LogAccessInfo { + const [logsEnabled, setLogsEnabled] = React.useState(false); + const [logsManagerType, setLogsManagerType] = React.useState(""); + const [logsColumns, setLogsColumns] = React.useState(); + const [logsStatusMessage, setLogsStatusMessage] = React.useState(""); React.useEffect(() => { service.GetLogAccessInfo({}).then(response => { - setManagerType(response.RemoteLogManagerType ?? ""); - setColumns(response?.Columns?.Column); + if (response.hasOwnProperty("Exceptions")) { + setLogsStatusMessage(response["Exceptions"]?.Exception[0]?.Message ?? nlsHPCC.LogAccess_GenericException); + } else { + if (response.RemoteLogManagerType === null) { + setLogsStatusMessage(nlsHPCC.LogAccess_LoggingNotConfigured); + } else { + setLogsEnabled(true); + setLogsManagerType(response.RemoteLogManagerType); + setLogsColumns(response?.Columns?.Column); + } + } }); }, []); - return { managerType, columns }; + return { logsEnabled, logsManagerType, logsColumns, logsStatusMessage }; } \ No newline at end of file diff --git a/esp/src/src/nls/hpcc.ts b/esp/src/src/nls/hpcc.ts index 592e8cb09d3..182faf5b59f 100644 --- a/esp/src/src/nls/hpcc.ts +++ b/esp/src/src/nls/hpcc.ts @@ -496,6 +496,8 @@ export = { Location: "Location", Lock: "Lock", LogAccessType: "Log Access Type", + LogAccess_GenericException: "ws_logaccess::GetLogAccessInfo, an exception has occurred.", + LogAccess_LoggingNotConfigured: "A logging engine has not been configured.", LogDirectory: "Log Directory", LogEventType: "Log Event Type", LogFile: "Log File", @@ -529,6 +531,7 @@ export = { Login: "Login", Logout: "Log Out", Logs: "Logs", + LogsDisabled: "Logs Disabled", LogVisualization: "Log Visualization", LogVisualizationUnconfigured: "Log Visualization is not configured, please check your configuration manager settings.", log_analysis_1: "log_analysis_1*", diff --git a/helm/hpcc/templates/_helpers.tpl b/helm/hpcc/templates/_helpers.tpl index e383472a70f..ae0f48a4318 100644 --- a/helm/hpcc/templates/_helpers.tpl +++ b/helm/hpcc/templates/_helpers.tpl @@ -1991,6 +1991,7 @@ args: [ {{ join " " $args }} ] {{- end -}} {{- end -}} {{- $_ := set $local "dnsNames" (uniq $local.dnsNames ) -}} + {{- $_ := set $local "allDomains" (prepend (default list $issuer.alternativeDomains) $domain ) -}} {{- if $externalCert -}} {{- $_ := set $local "commonName" (mustFirst $local.dnsNames ) -}} {{- else -}} @@ -2026,7 +2027,9 @@ spec: {{- end }} dnsNames: {{- range $dnsName := $local.dnsNames }} - - {{ (printf "%s.%s" $dnsName $domain) | quote }} + {{- range $altDomain := $local.allDomains }} + - {{ (printf "%s.%s" $dnsName $altDomain) | quote }} + {{- end }} {{- end }} {{- if $spiffe }} uris: diff --git a/plugins/fileservices/fileservices.cpp b/plugins/fileservices/fileservices.cpp index b69ef6470b8..bdf896334b2 100644 --- a/plugins/fileservices/fileservices.cpp +++ b/plugins/fileservices/fileservices.cpp @@ -779,7 +779,7 @@ FILESERVICES_API char * FILESERVICES_CALL implementSprayFixed(ICodeContext *ctx, req->setNoCommon(noCommon); - OwnedSpanScope clientSpan = queryThreadedActiveSpan()->createClientSpan("Dfu Spray Fixed"); + OwnedActiveSpanScope clientSpan = queryThreadedActiveSpan()->createClientSpan("Dfu Spray Fixed"); clientSpan->setSpanAttribute("destinationFilename", logicalName); try { @@ -925,7 +925,7 @@ static char * implementSprayVariable(ICodeContext *ctx, const char * sourceIP, c req->setNosplit(true); req->setNoCommon(noCommon); - OwnedSpanScope clientSpan = queryThreadedActiveSpan()->createClientSpan("Dfu Spray Variable"); + OwnedActiveSpanScope clientSpan = queryThreadedActiveSpan()->createClientSpan("Dfu Spray Variable"); clientSpan->setSpanAttribute("destinationFilename", logicalName); try { @@ -1110,7 +1110,7 @@ FILESERVICES_API char * FILESERVICES_CALL implementSprayXml(ICodeContext *ctx, c req->setNoCommon(noCommon); - OwnedSpanScope clientSpan = queryThreadedActiveSpan()->createClientSpan("Dfu Spray Xml"); + OwnedActiveSpanScope clientSpan = queryThreadedActiveSpan()->createClientSpan("Dfu Spray Xml"); clientSpan->setSpanAttribute("destinationFilename", logicalName); try { @@ -1266,7 +1266,7 @@ FILESERVICES_API char * FILESERVICES_CALL implementSprayJson(ICodeContext *ctx, req->setSrcPassword(userPw); } - OwnedSpanScope clientSpan = queryThreadedActiveSpan()->createClientSpan("Dfu Spray Json"); + OwnedActiveSpanScope clientSpan = queryThreadedActiveSpan()->createClientSpan("Dfu Spray Json"); clientSpan->setSpanAttribute("destinationFilename", logicalName); try { @@ -1358,7 +1358,7 @@ static char * implementDespray(ICodeContext *ctx, const char * sourceLogicalName if (maxConnections != -1) req->setMaxConnections(maxConnections); - OwnedSpanScope clientSpan = queryThreadedActiveSpan()->createClientSpan("Dfu Despray"); + OwnedActiveSpanScope clientSpan = queryThreadedActiveSpan()->createClientSpan("Dfu Despray"); clientSpan->setSpanAttribute("sourceFilename", logicalName); try { @@ -1457,7 +1457,7 @@ FILESERVICES_API char * FILESERVICES_CALL implementCopy(ICodeContext *ctx, const req->setWrap(true); req->setExpireDays(expireDays); - OwnedSpanScope clientSpan = queryThreadedActiveSpan()->createClientSpan("Dfu Copy"); + OwnedActiveSpanScope clientSpan = queryThreadedActiveSpan()->createClientSpan("Dfu Copy"); clientSpan->setSpanAttribute("sourceFilename", sourceLogicalName); clientSpan->setSpanAttribute("destinationFilename", destinationLogicalName); try @@ -1565,7 +1565,7 @@ FILESERVICES_API char * FILESERVICES_CALL fsfReplicate(ICodeContext *ctx, const req->setSourceLogicalName(logicalName.str()); - OwnedSpanScope clientSpan = queryThreadedActiveSpan()->createClientSpan("Dfu Spray Fixed"); + OwnedActiveSpanScope clientSpan = queryThreadedActiveSpan()->createClientSpan("Dfu Spray Fixed"); clientSpan->setSpanAttribute("destinationFilename", logicalName); try { @@ -2126,7 +2126,7 @@ FILESERVICES_API char * FILESERVICES_CALL fsfMonitorLogicalFileName(ICodeContex if (shotcount == 0) shotcount = -1; - OwnedSpanScope clientSpan = queryThreadedActiveSpan()->createClientSpan("Monitor Logical Filename"); + OwnedActiveSpanScope clientSpan = queryThreadedActiveSpan()->createClientSpan("Monitor Logical Filename"); clientSpan->setSpanAttribute("filename", lfn); try { @@ -2167,7 +2167,7 @@ FILESERVICES_API char * FILESERVICES_CALL fsfMonitorFile(ICodeContext *ctx, con if (shotcount == 0) shotcount = -1; - OwnedSpanScope clientSpan = queryThreadedActiveSpan()->createClientSpan("Monitor File"); + OwnedActiveSpanScope clientSpan = queryThreadedActiveSpan()->createClientSpan("Monitor File"); clientSpan->setSpanAttribute("filename", filename); try { @@ -2503,7 +2503,7 @@ FILESERVICES_API char * FILESERVICES_CALL fsfRemotePull_impl(ICodeContext *ctx, req->setSrcpassword(userPw); } - OwnedSpanScope clientSpan = queryThreadedActiveSpan()->createClientSpan("Dfu Remote Pull"); + OwnedActiveSpanScope clientSpan = queryThreadedActiveSpan()->createClientSpan("Dfu Remote Pull"); clientSpan->setSpanAttribute("sourceFilename", sourceLogicalName); clientSpan->setSpanAttribute("destinationFilename", destinationLogicalName); try diff --git a/roxie/ccd/ccdcontext.cpp b/roxie/ccd/ccdcontext.cpp index e03853a9537..dd470181f10 100644 --- a/roxie/ccd/ccdcontext.cpp +++ b/roxie/ccd/ccdcontext.cpp @@ -1658,7 +1658,7 @@ class CRoxieContextBase : implements IRoxieAgentContext, implements ICodeContext } else { - OwnedSpanScope graphScope = queryThreadedActiveSpan()->createInternalSpan(name); + OwnedActiveSpanScope graphScope = queryThreadedActiveSpan()->createInternalSpan(name); ProcessInfo startProcessInfo; if (workUnit || statsWu) startProcessInfo.update(ReadAllInfo); diff --git a/roxie/ccd/ccdlistener.cpp b/roxie/ccd/ccdlistener.cpp index 19c4bc81b91..4b19ab3afb2 100644 --- a/roxie/ccd/ccdlistener.cpp +++ b/roxie/ccd/ccdlistener.cpp @@ -1283,7 +1283,7 @@ class RoxieWorkUnitWorker : public RoxieQueryWorker Owned logctx = new StringContextLogger(wuid.get()); Owned traceHeaders = extractTraceDebugOptions(wu); - OwnedSpanScope requestSpan = queryTraceManager().createServerSpan("run_workunit", traceHeaders); + OwnedActiveSpanScope requestSpan = queryTraceManager().createServerSpan("run_workunit", traceHeaders); requestSpan->setSpanAttribute("hpcc.wuid", wuid); ContextSpanScope spanScope(*logctx, requestSpan); @@ -1464,7 +1464,7 @@ class RoxieProtocolMsgContext : implements IHpccProtocolMsgContext, public CInte Owned debugCmdHandler; Owned logctx; Owned queryFactory; - OwnedSpanScope requestSpan; + OwnedActiveSpanScope requestSpan; SocketEndpoint ep; time_t startTime; diff --git a/system/jlib/jstring.cpp b/system/jlib/jstring.cpp index 967d9b50658..93bf77edc00 100644 --- a/system/jlib/jstring.cpp +++ b/system/jlib/jstring.cpp @@ -2997,3 +2997,32 @@ const void * jmemmem(size_t lenHaystack, const void * haystack, size_t lenNeedle return nullptr; } + +/** + * For preventing command injection, sanitize the argument to be passed to the system command. + * - Quote the entire argument with single quotes to prevent interpretation of shell metacharacters. + * - Since a single-quoted string can't contain single quotes, even escaped, replace each single + * quote in the argument with the sequence '"'"' . That closes the single quoted string, appends + * a literal single quote, and reopens the single quoted string + */ +StringBuffer& sanitizeCommandArg(const char* arg, StringBuffer& sanitized) +{ +#if defined(__linux__) || defined(__APPLE__) + if (!isEmptyString(arg)) + { + size_t len = strlen(arg); + sanitized.append('\''); + for (size_t i = 0; i < len; i++) + { + if (arg[i] == '\'') + sanitized.append(R"('"'"')"); + else + sanitized.append(arg[i]); + } + sanitized.append('\''); + } +#else + sanitized.append(arg); +#endif + return sanitized; +} diff --git a/system/jlib/jstring.hpp b/system/jlib/jstring.hpp index 47abe9e1898..33e3761574a 100644 --- a/system/jlib/jstring.hpp +++ b/system/jlib/jstring.hpp @@ -659,4 +659,7 @@ extern jlib_decl void ensureSeparator(StringBuffer & out, char separator); //Search for one block of bytes within another block of bytes - memmem is not standard, so we provide our own extern jlib_decl const void * jmemmem(size_t lenHaystack, const void * haystack, size_t lenNeedle, const void *needle); +// For preventing command injection, sanitize the argument to be passed to the system command +extern jlib_decl StringBuffer& sanitizeCommandArg(const char* arg, StringBuffer& sanitized); + #endif diff --git a/system/jlib/jsuperhash.cpp b/system/jlib/jsuperhash.cpp index 3e26af7b888..d1a9fe2cd3e 100644 --- a/system/jlib/jsuperhash.cpp +++ b/system/jlib/jsuperhash.cpp @@ -86,6 +86,7 @@ void SuperHashTable::init(unsigned initsize) #ifdef TRACE_HASH search_tot = 0; search_num = 0; + search_max = 0; #endif } diff --git a/system/jlib/jtrace.cpp b/system/jlib/jtrace.cpp index 58289da29fb..83954f9a4e0 100644 --- a/system/jlib/jtrace.cpp +++ b/system/jlib/jtrace.cpp @@ -928,12 +928,13 @@ class CNullSpan final : public CInterfaceOf virtual void recordError(const SpanError & error) override {}; virtual void setSpanStatusSuccess(bool spanSucceeded, const char * statusMessage) override {} - virtual const char * queryTraceId() const override { return nullptr; } - virtual const char * querySpanId() const override { return nullptr; } + virtual const char * queryTraceId() const override { return "00000000000000000000000000000000"; } + virtual const char * querySpanId() const override { return "0000000000000000"; } - virtual const char* queryGlobalId() const override { return nullptr; } - virtual const char* queryCallerId() const override { return nullptr; } - virtual const char* queryLocalId() const override { return nullptr; } + // Note: GlobalID & LocalID are created from lnuid, which creates 23 char UIDs (16 rand bytes in base58), and uses "1" for zeroes + virtual const char* queryGlobalId() const override { return "11111111111111111111111"; } + virtual const char* queryCallerId() const override { return ""; } + virtual const char* queryLocalId() const override { return "11111111111111111111111"; } virtual ISpan * createClientSpan(const char * name, const SpanTimeStamp * spanStartTimeStamp = nullptr) override { return getNullSpan(); } virtual ISpan * createInternalSpan(const char * name, const SpanTimeStamp * spanStartTimeStamp = nullptr) override { return getNullSpan(); } @@ -1518,13 +1519,36 @@ ISpan * CTraceManager::createServerSpan(const char * name, const IProperties * h //--------------------------------------------------------------------------------------------------------------------- -OwnedSpanScope::OwnedSpanScope(ISpan * _ptr) : span(_ptr) +ActiveSpanScope::ActiveSpanScope(ISpan * _ptr) : ActiveSpanScope(_ptr, queryThreadedActiveSpan()) {} +ActiveSpanScope::ActiveSpanScope(ISpan * _ptr, ISpan * _prev) : span(_ptr), prevSpan(_prev) +{ + setThreadedActiveSpan(_ptr); +} + +ActiveSpanScope::~ActiveSpanScope() +{ + ISpan* current = queryThreadedActiveSpan(); + if (current != span) + { + const char* currSpanID = current->querySpanId(); + const char* expectedSpanID = span != nullptr ? span->querySpanId() : "0000000000000000"; + + IERRLOG("~ActiveSpanScope: threadActiveSpan has changed unexpectedly, expected: %s actual: %s", expectedSpanID, currSpanID); + return; + } + + setThreadedActiveSpan(prevSpan); +} + +//--------------------------------------------------------------------------------------------------------------------- + +OwnedActiveSpanScope::OwnedActiveSpanScope(ISpan * _ptr) : span(_ptr) { if (_ptr) prevSpan = setThreadedActiveSpan(_ptr); } -void OwnedSpanScope::setown(ISpan * _span) +void OwnedActiveSpanScope::setown(ISpan * _span) { assertex(_span); //Just in case the span is already set, ensure it is ended and that the previous span is restored. @@ -1533,12 +1557,12 @@ void OwnedSpanScope::setown(ISpan * _span) prevSpan = setThreadedActiveSpan(_span); } -void OwnedSpanScope::set(ISpan * _span) +void OwnedActiveSpanScope::set(ISpan * _span) { setown(LINK(_span)); } -void OwnedSpanScope::clear() +void OwnedActiveSpanScope::clear() { if (span) { @@ -1548,7 +1572,35 @@ void OwnedSpanScope::clear() } } -OwnedSpanScope::~OwnedSpanScope() +OwnedActiveSpanScope::~OwnedActiveSpanScope() +{ + clear(); +} + +//--------------------------------------------------------------------------------------------------------------------- + +void OwnedSpanLifetime::setown(ISpan * _span) +{ + assertex(_span); + clear(); + span.setown(_span); +} + +void OwnedSpanLifetime::set(ISpan * _span) +{ + setown(LINK(_span)); +} + +void OwnedSpanLifetime::clear() +{ + if (span) + { + span->endSpan(); + span.clear(); + } +} + +OwnedSpanLifetime::~OwnedSpanLifetime() { clear(); } diff --git a/system/jlib/jtrace.hpp b/system/jlib/jtrace.hpp index 86e2fb66115..7c922cf8c23 100644 --- a/system/jlib/jtrace.hpp +++ b/system/jlib/jtrace.hpp @@ -159,21 +159,85 @@ interface ISpan : extends IInterface virtual const char* queryLocalId() const = 0; }; -class jlib_decl OwnedSpanScope +//------------------------------------------------------------------------------ +// ActiveSpanScope vs OwnedActiveSpanScope Usage: +//------------------------------------------------------------------------------ +// The primary difference between OwnedActiveSpanScope and ActiveSpanScope is that +// OwnedActiveSpanScope controls the lifetime of its ISpan while ActiveSpanScope +// does not. In cases where the ISpan will be used from a single thread and within +// a single scope OwnedActiveSpanScope should be used. For more complicated scenarios, +// involving multiple threads, time sliced work, etc ActiveSpanScope should be used +// to associate that ISpan with each processing thread / unit of work, while an +// OwnedSpanLifetime, likely a class member, should control the ISpan lifetime. +// +// When using ActiveSpanScope another class such as OwnedSpanLifetime should be +// used to control the lifetime of the ISpan and the referenced ISpans lifetime +// should be guaranteed to be longer than the ActiveSpanScopes lifetime. +//------------------------------------------------------------------------------ + +class jlib_decl ActiveSpanScope { public: - OwnedSpanScope() = default; - OwnedSpanScope(ISpan * _ptr); - OwnedSpanScope(const OwnedSpanScope& rhs) = delete; - OwnedSpanScope(OwnedSpanScope&& rhs) = default; - ~OwnedSpanScope(); + // Captures current threadActiveSpan for prevSpan + ActiveSpanScope(ISpan * _ptr); + ActiveSpanScope(ISpan * _ptr, ISpan * _prev); + + ActiveSpanScope(const ActiveSpanScope& rhs) = delete; + ~ActiveSpanScope(); inline ISpan * operator -> () const { return span; } inline operator ISpan *() const { return span; } - inline OwnedSpanScope& operator=(ISpan * ptr) = delete; - inline OwnedSpanScope& operator=(const OwnedSpanScope& rhs) = delete; - inline OwnedSpanScope& operator=(OwnedSpanScope&& rhs) = delete; + inline ActiveSpanScope& operator=(ISpan * ptr) = delete; + inline ActiveSpanScope& operator=(const ActiveSpanScope& rhs) = delete; + + inline bool operator == (ISpan * _ptr) const { return span == _ptr; } + inline bool operator != (ISpan * _ptr) const { return span != _ptr; } +private: + ISpan * span = nullptr; + ISpan * prevSpan = nullptr; +}; + +class jlib_decl OwnedSpanLifetime +{ +public: + OwnedSpanLifetime() = default; + OwnedSpanLifetime(ISpan * _ptr) : span(_ptr) {} + OwnedSpanLifetime(const OwnedSpanLifetime& rhs) = delete; + OwnedSpanLifetime(OwnedSpanLifetime&& rhs) = default; + ~OwnedSpanLifetime(); + + inline ISpan * operator -> () const { return span; } + inline operator ISpan *() const { return span; } + + inline OwnedSpanLifetime& operator=(ISpan * ptr) = delete; + inline OwnedSpanLifetime& operator=(const OwnedSpanLifetime& rhs) = delete; + inline OwnedSpanLifetime& operator=(OwnedSpanLifetime&& rhs) = delete; + + void clear(); + ISpan * query() const { return span; } + void set(ISpan * _span); + void setown(ISpan * _span); + +private: + Owned span; +}; + +class jlib_decl OwnedActiveSpanScope +{ +public: + OwnedActiveSpanScope() = default; + OwnedActiveSpanScope(ISpan * _ptr); + OwnedActiveSpanScope(const OwnedActiveSpanScope& rhs) = delete; + OwnedActiveSpanScope(OwnedActiveSpanScope&& rhs) = default; + ~OwnedActiveSpanScope(); + + inline OwnedActiveSpanScope& operator=(ISpan * ptr) = delete; + inline OwnedActiveSpanScope& operator=(const OwnedActiveSpanScope& rhs) = delete; + inline OwnedActiveSpanScope& operator=(OwnedActiveSpanScope&& rhs) = delete; + + inline ISpan * operator -> () const { return span; } + inline operator ISpan *() const { return span; } void clear(); ISpan * query() const { return span; } @@ -185,6 +249,7 @@ class jlib_decl OwnedSpanScope ISpan * prevSpan = nullptr; }; + extern jlib_decl IProperties * getClientHeaders(const ISpan * span); extern jlib_decl IProperties * getSpanContext(const ISpan * span); diff --git a/testing/helm/tests/multicertdomains.yaml b/testing/helm/tests/multicertdomains.yaml new file mode 100644 index 00000000000..d406fc54119 --- /dev/null +++ b/testing/helm/tests/multicertdomains.yaml @@ -0,0 +1,102 @@ +global: + privileged: true + egress: + restricted: false +certificates: + issuers: + remote: + name: hpcc-remote-issuer + domain: hpcc1 + alternativeDomains: + - foo.hpcc1 + - bar.hpcc1 + ## set enabled to true if adding remoteClients for any components + enabled: true + ## kind can be changed to ClusterIssue to refer to a ClusterIssuer. https://cert-manager.io/docs/reference/api-docs/#cert-manager.io/v1.ClusterIssuer + kind: Issuer + spiffe: false + ## do not define spec (set spec: null), to reference an Issuer resource that already exists in the cluster + ## change spec if you'd like to change how certificates get issued... see ## https://cert-manager.io/docs/configuration/#supported-issuer-types + ## for information on what spec should contain. + spec: + ca: + secretName: hpcc-local-issuer-key-pair # real config would use hpcc-remote, but examples have hpcc-local instructions + + signing: # intended to be used for signing/verification purposes only, e.g. by dafilesrv + name: hpcc-signing-issuer + ## kind can be changed to ClusterIssue to refer to a ClusterIssuer. https://cert-manager.io/docs/reference/api-docs/#cert-manager.io/v1.ClusterIssuer + kind: Issuer + ## do not define spec (set spec: null), to reference an Issuer resource that already exists in the cluster + ## change spec if you'd like to change how certificates get issued... see ## https://cert-manager.io/docs/configuration/#supported-issuer-types + ## for information on what spec should contain. + spec: + ca: + secretName: hpcc-signing-issuer-key-pair + +thor: null +eclagent: null +eclscheduler: null +dafilesrv: null +dfuserver: null + +sasha: + disabled: true + wu-archiver: + disabled: true + dfuwu-archiver: + disabled: true + dfurecovery-archiver: + disabled: true + file-expiry: + disabled: true + +esp: +- name: eclwatch + application: eclwatch + auth: none + replicas: 1 + service: + port: 8888 + servicePort: 18010 + visibility: local +- name: eclqueries + application: eclqueries + auth: none + replicas: 1 + service: + visibility: local + servicePort: 18002 +- name: eclservices + application: eclservices + auth: none + replicas: 1 + service: + servicePort: 8010 + visibility: cluster + +roxie: +- name: roxie1 + disabled: false + prefix: roxie1 + services: + - name: roxie1 + servicePort: 19876 + listenQueue: 200 + numThreads: 30 + visibility: local + trustClients: + - commonName: roxie2.hpcc2 + - name: roxie1backup + servicePort: 19877 + listenQueue: 200 + numThreads: 30 + visibility: local + trustClients: + - commonName: roxie2.hpcc2 + replicas: 1 + numChannels: 1 + serverReplicas: 0 + localAgent: false + traceLevel: 1 + topoServer: + replicas: 1 diff --git a/testing/unittests/jlibtests.cpp b/testing/unittests/jlibtests.cpp index 19d2bfcf4bf..b79268bc0a5 100644 --- a/testing/unittests/jlibtests.cpp +++ b/testing/unittests/jlibtests.cpp @@ -198,7 +198,7 @@ class JlibTraceTest : public CppUnit::TestFixture { Owned emptyMockHTTPHeaders = createProperties(); { - OwnedSpanScope serverSpan = queryTraceManager().createServerSpan("spanWithEventsNoAtts", emptyMockHTTPHeaders); + OwnedActiveSpanScope serverSpan = queryTraceManager().createServerSpan("spanWithEventsNoAtts", emptyMockHTTPHeaders); Owned emptyEventAtts = createProperties(); serverSpan->addSpanEvent("event1", emptyEventAtts); } @@ -208,12 +208,12 @@ class JlibTraceTest : public CppUnit::TestFixture twoEventAtt->setProp("key2", ""); { - OwnedSpanScope serverSpan = queryTraceManager().createServerSpan("spanWithEvent1Att", emptyMockHTTPHeaders); + OwnedActiveSpanScope serverSpan = queryTraceManager().createServerSpan("spanWithEvent1Att", emptyMockHTTPHeaders); serverSpan->addSpanEvent("event2", twoEventAtt); }//{ "type": "span", "name": "spanWithEvents1Att", "trace_id": "3b9f55aaf8fab51fb0d73a32db7d704f", "span_id": "2a25a44ae0b3abe0", "start": 1709696036335278770, "duration": 3363911469, "events":[ { "name": "event2", "time_stamp": 1709696038413023245, "attributes": {"key": "value" } } ] } { - OwnedSpanScope serverSpan = queryTraceManager().createServerSpan("spanWith2Events", emptyMockHTTPHeaders); + OwnedActiveSpanScope serverSpan = queryTraceManager().createServerSpan("spanWith2Events", emptyMockHTTPHeaders); serverSpan->addSpanEvent("event1", twoEventAtt); serverSpan->addSpanEvent("event2", twoEventAtt); }//{ "type": "span", "name": "spanWith2Events", "trace_id": "ff5c5919b9c5f85913652b77f289bf0b", "span_id": "82f91ca1f9d469c1", "start": 1709698012480805016, "duration": 2811601377, "events":[ { "name": "event1", "time_stamp": 1709698013294323139, "attributes": {"key": "value" } },{ "name": "event2", "time_stamp": 1709698014500350802, "attributes": {"key": "value" } } ] } @@ -228,14 +228,14 @@ class JlibTraceTest : public CppUnit::TestFixture { //duration should be at least 125 milliseconds - OwnedSpanScope serverSpan = queryTraceManager().createServerSpan("declaredSpanStartTime", emptyMockHTTPHeaders, &declaredSpanStartTime); + OwnedActiveSpanScope serverSpan = queryTraceManager().createServerSpan("declaredSpanStartTime", emptyMockHTTPHeaders, &declaredSpanStartTime); //{ "type": "span", "name": "declaredSpanStartTime", "trace_id": "0a2eff24e1996540056745aaeb2f5824", "span_id": "46d0faf8b4da893e", //"start": 1702672311203213259, "duration": 125311051 } SpanTimeStamp clientSpanTimeStamp; clientSpanTimeStamp.now(); MilliSleep(20); - OwnedSpanScope clientSpan = serverSpan->createClientSpan("clientSpanStartTime", &clientSpanTimeStamp); + OwnedActiveSpanScope clientSpan = serverSpan->createClientSpan("clientSpanStartTime", &clientSpanTimeStamp); //{ "type": "span", "name": "clientSpanStartTime", "trace_id": "f73b171fdcd120f88ca5b656866befee", "span_id": "7c798125d10ee0ec", //"start": 1727200325699918374, "duration": 20256156, "parent_span_id": "b79fe15b7d727fca" } } @@ -253,13 +253,13 @@ class JlibTraceTest : public CppUnit::TestFixture SpanTimeStamp nowTimeStamp; //not used, printed out as "start" time for manual comparison nowTimeStamp.now(); { - OwnedSpanScope serverSpan = queryTraceManager().createServerSpan("msTickOffsetStartTime", emptyMockHTTPHeaders, &msTickOffsetTimeStamp); + OwnedActiveSpanScope serverSpan = queryTraceManager().createServerSpan("msTickOffsetStartTime", emptyMockHTTPHeaders, &msTickOffsetTimeStamp); unsigned clientStartMS = msTick(); MilliSleep(20); SpanTimeStamp clientSpanTimeStamp; clientSpanTimeStamp.setMSTickTime(clientStartMS); - OwnedSpanScope clientSpan = serverSpan->createClientSpan("clientSpanOffsetTime", &clientSpanTimeStamp); + OwnedActiveSpanScope clientSpan = serverSpan->createClientSpan("clientSpanOffsetTime", &clientSpanTimeStamp); //{ "type": "span", "name": "clientSpanOffsetTime", "trace_id": "9a41723ddc0048d854ab34b79340e749", "span_id": "11af70aa6a6dbee3", //"start": 1727200325770619773, "duration": 20015542, "parent_span_id": "531ad336071f453b" } } @@ -277,7 +277,7 @@ class JlibTraceTest : public CppUnit::TestFixture CPPUNIT_ASSERT_EQUAL_MESSAGE("Unexpected initialized spanTimeStamp", true, uninitializedTS.systemClockTime == std::chrono::nanoseconds::zero()); CPPUNIT_ASSERT_EQUAL_MESSAGE("Unexpected initialized spanTimeStamp", true, uninitializedTS.steadyClockTime == std::chrono::nanoseconds::zero()); { - OwnedSpanScope serverSpan = queryTraceManager().createServerSpan("uninitializeddeclaredSpanStartTime", emptyMockHTTPHeaders, &uninitializedTS); + OwnedActiveSpanScope serverSpan = queryTraceManager().createServerSpan("uninitializeddeclaredSpanStartTime", emptyMockHTTPHeaders, &uninitializedTS); //sleep for 75 milliseconds after span creation, expect at least 75 milliseconds duration output MilliSleep(75); @@ -292,43 +292,43 @@ class JlibTraceTest : public CppUnit::TestFixture { Owned emptyMockHTTPHeaders = createProperties(); { - OwnedSpanScope serverSpan = queryTraceManager().createServerSpan("defaultErrorSpan", emptyMockHTTPHeaders); + OwnedActiveSpanScope serverSpan = queryTraceManager().createServerSpan("defaultErrorSpan", emptyMockHTTPHeaders); serverSpan->recordError(); }//{ "type": "span", "name": "defaultErrorSpan", "trace_id": "209b5d8cea0aec9785d2dfa3117e37ad", "span_id": "ab72e76c2f2466c2", "start": 1709675278129335702, "duration": 188292867932, "status": "Error", "kind": "Server", "instrumented_library": "unittests", "events":[ { "name": "Exception", "time_stamp": 1709675465508149013, "attributes": {"escaped": 0 } } ] } { - OwnedSpanScope serverSpan = queryTraceManager().createServerSpan("defaultErrorSpanStruct", emptyMockHTTPHeaders); + OwnedActiveSpanScope serverSpan = queryTraceManager().createServerSpan("defaultErrorSpanStruct", emptyMockHTTPHeaders); SpanError error; serverSpan->recordError(error); }//{ "type": "span", "name": "defaultErrorSpanStruct", "trace_id": "19803a446b971f2e0bdddc9c00db50fe", "span_id": "04c93a91ab8785a2", "start": 1709675487767044352, "duration": 2287497219, "status": "Error", "kind": "Server", "instrumented_library": "unittests", "events":[ { "name": "Exception", "time_stamp": 1709675489216412154, "attributes": {"escaped": 0 } } ] } { - OwnedSpanScope serverSpan = queryTraceManager().createServerSpan("failedErrorSpanEscaped", emptyMockHTTPHeaders); + OwnedActiveSpanScope serverSpan = queryTraceManager().createServerSpan("failedErrorSpanEscaped", emptyMockHTTPHeaders); serverSpan->recordError(SpanError("hello", -1, true, true)); //error message hello, no error code, error caused failure, and error caused escape }//{ "type": "span", "name": "failedErrorSpanEscaped", "trace_id": "634f386c18a6140544c980e0d5a15905", "span_id": "e2f59c48f63a8f82", "start": 1709675508231168974, "duration": 7731717678, "status": "Error", "kind": "Server", "description": "hello", "instrumented_library": "unittests", "events":[ { "name": "Exception", "time_stamp": 1709675512164430668, "attributes": {"escaped": 1,"message": "hello" } } ] } { - OwnedSpanScope serverSpan = queryTraceManager().createServerSpan("failedErrEscapedMsgErrCode", emptyMockHTTPHeaders); + OwnedActiveSpanScope serverSpan = queryTraceManager().createServerSpan("failedErrEscapedMsgErrCode", emptyMockHTTPHeaders); serverSpan->recordError(SpanError("hello", 34, true, true)); //error message hello, error code 34, error caused failure, and error caused escape }//failedErrEscapedMsgErrCode { - OwnedSpanScope serverSpan = queryTraceManager().createServerSpan("containsErrorAndMessageSpan", emptyMockHTTPHeaders); + OwnedActiveSpanScope serverSpan = queryTraceManager().createServerSpan("containsErrorAndMessageSpan", emptyMockHTTPHeaders); serverSpan->recordError(SpanError("Error Message!!")); }//{ "type": "span", "name": "containsErrorAndMessageSpan", "trace_id": "9a6e00ea309bc0427733f9b2d452f9e2", "span_id": "de63e9c69b64e411", "start": 1709675552302360510, "duration": 5233037523, "status": "Error", "kind": "Server", "description": "Error Message!!", "instrumented_library": "unittests", "events":[ { "name": "Exception", "time_stamp": 1709675555149852711, "attributes": {"escaped": 0,"message": "Error Message!!" } } { - OwnedSpanScope serverSpan = queryTraceManager().createServerSpan("containsErrorAndMessageFailedNotEscapedSpan", emptyMockHTTPHeaders); + OwnedActiveSpanScope serverSpan = queryTraceManager().createServerSpan("containsErrorAndMessageFailedNotEscapedSpan", emptyMockHTTPHeaders); serverSpan->recordError(SpanError("Error Message!!", 23, true, false)); }//{ "type": "span", "name": "containsErrorAndMessageFailedNotEscapedSpan", "trace_id": "02f4b2d215f8230b15063862f8a91e41", "span_id": "c665ec371d6db147", "start": 1709675573581678954, "duration": 3467489486, "status": "Error", "kind": "Server", "description": "Error Message!!", "instrumented_library": "unittests", "events":[ { "name": "Exception", "time_stamp": 1709675576145074240, "attributes": {"code": 23,"escaped": 0,"message": "Error Message!!" } } ] } { - OwnedSpanScope serverSpan = queryTraceManager().createServerSpan("mockExceptionSpanNotFailedNotEscaped", emptyMockHTTPHeaders); + OwnedActiveSpanScope serverSpan = queryTraceManager().createServerSpan("mockExceptionSpanNotFailedNotEscaped", emptyMockHTTPHeaders); serverSpan->recordException( makeStringExceptionV(76,"Mock exception"), false, false); }//{ "type": "span", "name": "mockExceptionSpanNotFailedNotEscaped", "trace_id": "e01766474db05ce9085943fa3955cd73", "span_id": "7da620e96e10e42c", "start": 1709675595987480704, "duration": 2609091267, "status": "Unset", "kind": "Server", "instrumented_library": "unittests", "events":[ { "name": "Exception", "time_stamp": 1709675597728975355, "attributes": {"code": 76,"escaped": 0,"message": "Mock exception" } } ] { - OwnedSpanScope serverSpan = queryTraceManager().createServerSpan("thrownExceptionSpan", emptyMockHTTPHeaders); + OwnedActiveSpanScope serverSpan = queryTraceManager().createServerSpan("thrownExceptionSpan", emptyMockHTTPHeaders); try { throw makeStringExceptionV( 356, "Mock thrown exception"); @@ -347,7 +347,7 @@ class JlibTraceTest : public CppUnit::TestFixture { SpanFlags flags = SpanFlags::EnsureTraceId; Owned emptyMockHTTPHeaders = createProperties(); - OwnedSpanScope serverSpan = queryTraceManager().createServerSpan("mySpan", emptyMockHTTPHeaders, flags); + OwnedActiveSpanScope serverSpan = queryTraceManager().createServerSpan("mySpan", emptyMockHTTPHeaders, flags); DBGLOG("mySpan is alive"); savedSpan.set(serverSpan); }//{ "type": "span", "name": "mySpan", "trace_id": "fe266416e7d588113a5131394d913ab4", "span_id": "7ac62328b04442c5", "start": 1709824793826023368, "duration": 16952 } @@ -363,6 +363,16 @@ class JlibTraceTest : public CppUnit::TestFixture savedSpan.clear(); } + void nullSpanTest() + { + Owned nullSpan = getNullSpan(); + CPPUNIT_ASSERT(nullSpan->queryTraceId() != nullptr); + CPPUNIT_ASSERT(nullSpan->querySpanId() != nullptr); + CPPUNIT_ASSERT(nullSpan->queryGlobalId() != nullptr); + CPPUNIT_ASSERT(nullSpan->queryCallerId() != nullptr); + CPPUNIT_ASSERT(nullSpan->queryLocalId() != nullptr); + } + void testTraceDisableConfig() { Owned testTree = createPTreeFromYAMLString(disableTracingYaml, ipt_none, ptr_ignoreWhiteSpace, nullptr); @@ -376,7 +386,7 @@ class JlibTraceTest : public CppUnit::TestFixture { SpanFlags flags = SpanFlags::EnsureTraceId; Owned emptyMockHTTPHeaders = createProperties(); - OwnedSpanScope serverSpan = queryTraceManager().createServerSpan("noRemoteParentEnsureTraceID", emptyMockHTTPHeaders, flags); + OwnedActiveSpanScope serverSpan = queryTraceManager().createServerSpan("noRemoteParentEnsureTraceID", emptyMockHTTPHeaders, flags); Owned retrievedSpanCtxAttributes = createProperties(); serverSpan->getSpanContext(retrievedSpanCtxAttributes.get()); @@ -389,7 +399,7 @@ class JlibTraceTest : public CppUnit::TestFixture { SpanFlags flags = SpanFlags::EnsureTraceId; Owned emptyMockHTTPHeaders = createProperties(); - OwnedSpanScope serverSpan = queryTraceManager().createServerSpan("noRemoteParentEnsureTraceID", emptyMockHTTPHeaders, flags); + OwnedActiveSpanScope serverSpan = queryTraceManager().createServerSpan("noRemoteParentEnsureTraceID", emptyMockHTTPHeaders, flags); Owned retrievedSpanCtxAttributes = createProperties(); serverSpan->getSpanContext(retrievedSpanCtxAttributes.get()); @@ -409,7 +419,7 @@ class JlibTraceTest : public CppUnit::TestFixture Owned mockHTTPHeaders = createProperties(); createMockHTTPHeaders(mockHTTPHeaders, true); - OwnedSpanScope serverSpan = queryTraceManager().createServerSpan("propegatedServerSpan", mockHTTPHeaders); + OwnedActiveSpanScope serverSpan = queryTraceManager().createServerSpan("propegatedServerSpan", mockHTTPHeaders); //at this point the serverSpan should have the following context attributes //traceID, spanID, remoteParentSpanID, traceFlags, traceState, globalID, callerID @@ -461,7 +471,7 @@ class JlibTraceTest : public CppUnit::TestFixture return; } - OwnedSpanScope nullSpan = getNullSpan(); + OwnedActiveSpanScope nullSpan = getNullSpan(); CPPUNIT_ASSERT_EQUAL_MESSAGE("Unexpected nullptr nullspan detected", true, nullSpan != nullptr); { @@ -469,14 +479,14 @@ class JlibTraceTest : public CppUnit::TestFixture nullSpan->getClientHeaders(headers); } - OwnedSpanScope nullSpanChild = nullSpan->createClientSpan("nullSpanChild"); + OwnedActiveSpanScope nullSpanChild = nullSpan->createClientSpan("nullSpanChild"); CPPUNIT_ASSERT_EQUAL_MESSAGE("Unexpected nullptr nullSpanChild detected", true, nullSpanChild != nullptr); } void testClientSpan() { Owned emptyMockHTTPHeaders = createProperties(); - OwnedSpanScope serverSpan = queryTraceManager().createServerSpan("propegatedServerSpan", emptyMockHTTPHeaders); + OwnedActiveSpanScope serverSpan = queryTraceManager().createServerSpan("propegatedServerSpan", emptyMockHTTPHeaders); Owned retrievedSpanCtxAttributes = createProperties(); serverSpan->getSpanContext(retrievedSpanCtxAttributes); @@ -486,7 +496,7 @@ class JlibTraceTest : public CppUnit::TestFixture const char * serverTraceID = retrievedSpanCtxAttributes->queryProp("traceID"); { - OwnedSpanScope internalSpan = serverSpan->createClientSpan("clientSpan"); + OwnedActiveSpanScope internalSpan = serverSpan->createClientSpan("clientSpan"); //retrieve clientSpan context with the intent to propogate otel and HPCC context { Owned retrievedSpanCtxAttributes = createProperties(); @@ -520,7 +530,7 @@ class JlibTraceTest : public CppUnit::TestFixture void testInternalSpan() { Owned emptyMockHTTPHeaders = createProperties(); - OwnedSpanScope serverSpan = queryTraceManager().createServerSpan("propegatedServerSpan", emptyMockHTTPHeaders); + OwnedActiveSpanScope serverSpan = queryTraceManager().createServerSpan("propegatedServerSpan", emptyMockHTTPHeaders); Owned retrievedSpanCtxAttributes = createProperties(); serverSpan->getSpanContext(retrievedSpanCtxAttributes); @@ -531,7 +541,7 @@ class JlibTraceTest : public CppUnit::TestFixture const char * serverTraceID = retrievedSpanCtxAttributes->queryProp("traceID"); { - OwnedSpanScope internalSpan = serverSpan->createInternalSpan("internalSpan"); + OwnedActiveSpanScope internalSpan = serverSpan->createInternalSpan("internalSpan"); //retrieve internalSpan context with the intent to interrogate attributes { @@ -565,7 +575,7 @@ class JlibTraceTest : public CppUnit::TestFixture void testRootServerSpan() { Owned emptyMockHTTPHeaders = createProperties(); - OwnedSpanScope serverSpan = queryTraceManager().createServerSpan("propegatedServerSpan", emptyMockHTTPHeaders); + OwnedActiveSpanScope serverSpan = queryTraceManager().createServerSpan("propegatedServerSpan", emptyMockHTTPHeaders); //retrieve serverSpan context with the intent to propagate it to a remote child span { @@ -639,7 +649,7 @@ class JlibTraceTest : public CppUnit::TestFixture { Owned mockHTTPHeaders = createProperties(); createMockHTTPHeaders(mockHTTPHeaders, false); - OwnedSpanScope currentSpanScope = queryTraceManager().createServerSpan("currentSpanScope", mockHTTPHeaders); + OwnedActiveSpanScope currentSpanScope = queryTraceManager().createServerSpan("currentSpanScope", mockHTTPHeaders); CPPUNIT_ASSERT_MESSAGE("currentSpanScope Span == nullptr!", currentSpanScope != nullptr); @@ -668,7 +678,7 @@ class JlibTraceTest : public CppUnit::TestFixture { Owned mockHTTPHeaders = createProperties(); createMockHTTPHeaders(mockHTTPHeaders, false); - OwnedSpanScope serverSpan = queryTraceManager().createServerSpan("invalidPropegatedServerSpan", mockHTTPHeaders); + OwnedActiveSpanScope serverSpan = queryTraceManager().createServerSpan("invalidPropegatedServerSpan", mockHTTPHeaders); Owned retrievedSpanCtxAttributes = createProperties(); serverSpan->getClientHeaders(retrievedSpanCtxAttributes.get()); @@ -689,7 +699,7 @@ class JlibTraceTest : public CppUnit::TestFixture Owned mockHTTPHeaders = createProperties(); createMockHTTPHeaders(mockHTTPHeaders, true); - OwnedSpanScope serverSpan = queryTraceManager().createServerSpan("propegatedServerSpan", mockHTTPHeaders); + OwnedActiveSpanScope serverSpan = queryTraceManager().createServerSpan("propegatedServerSpan", mockHTTPHeaders); //at this point the serverSpan should have the following context attributes //remoteParentSpanID, globalID, callerID @@ -710,10 +720,10 @@ class JlibTraceTest : public CppUnit::TestFixture Owned mockHTTPHeaders = createProperties(); createMockHTTPHeaders(mockHTTPHeaders, true); - OwnedSpanScope serverSpan = queryTraceManager().createServerSpan("propegatedServerSpan", mockHTTPHeaders); - OwnedSpanScope clientSpan = serverSpan->createClientSpan("clientSpan"); - OwnedSpanScope internalSpan = clientSpan->createInternalSpan("internalSpan"); - OwnedSpanScope internalSpan2 = internalSpan->createInternalSpan("internalSpan2"); + OwnedActiveSpanScope serverSpan = queryTraceManager().createServerSpan("propegatedServerSpan", mockHTTPHeaders); + OwnedActiveSpanScope clientSpan = serverSpan->createClientSpan("clientSpan"); + OwnedActiveSpanScope internalSpan = clientSpan->createInternalSpan("internalSpan"); + OwnedActiveSpanScope internalSpan2 = internalSpan->createInternalSpan("internalSpan2"); StringBuffer out; out.set("{"); @@ -748,8 +758,8 @@ class JlibTraceTest : public CppUnit::TestFixture Owned mockHTTPHeaders = createProperties(); createMockHTTPHeaders(mockHTTPHeaders, true); //includes global ID - OwnedSpanScope serverSpan = queryTraceManager().createServerSpan("propegatedServerSpan", mockHTTPHeaders); - OwnedSpanScope clientSpan = serverSpan->createClientSpan("clientSpanWithGlobalID"); + OwnedActiveSpanScope serverSpan = queryTraceManager().createServerSpan("propegatedServerSpan", mockHTTPHeaders); + OwnedActiveSpanScope clientSpan = serverSpan->createClientSpan("clientSpanWithGlobalID"); //retrieve serverSpan context with the intent to interrogate attributes { @@ -771,7 +781,7 @@ class JlibTraceTest : public CppUnit::TestFixture Owned mockHTTPHeaders = createProperties(); createMockHTTPHeaders(mockHTTPHeaders, true); - OwnedSpanScope serverSpan = queryTraceManager().createServerSpan("propegatedServerSpan", mockHTTPHeaders); + OwnedActiveSpanScope serverSpan = queryTraceManager().createServerSpan("propegatedServerSpan", mockHTTPHeaders); //at this point the serverSpan should have the following context attributes //traceID, spanID, remoteParentSpanID, traceFlags, traceState, globalID, callerID @@ -817,7 +827,7 @@ class JlibTraceTest : public CppUnit::TestFixture mockHTTPHeadersSA.append("HPCC-Global-Id:someGlobalID"); mockHTTPHeadersSA.append("HPCC-Caller-Id:IncomingCID"); - OwnedSpanScope serverSpan = queryTraceManager().createServerSpan("StringArrayPropegatedServerSpan", mockHTTPHeadersSA); + OwnedActiveSpanScope serverSpan = queryTraceManager().createServerSpan("StringArrayPropegatedServerSpan", mockHTTPHeadersSA); //at this point the serverSpan should have the following context attributes //traceID, spanID, remoteParentSpanID, traceFlags, traceState, globalID, callerID @@ -4464,6 +4474,7 @@ class JLibOpensslAESTest : public CppUnit::TestFixture E->Release(); } } + // Note: Using OpenTelemetry null span } void test() diff --git a/thorlcr/graph/thgraphmaster.cpp b/thorlcr/graph/thgraphmaster.cpp index a13db163620..6ff0b361d91 100644 --- a/thorlcr/graph/thgraphmaster.cpp +++ b/thorlcr/graph/thgraphmaster.cpp @@ -1504,7 +1504,7 @@ CJobMaster::CJobMaster(IConstWorkUnit &_workunit, const char *graphName, ILoaded init(); Owned traceHeaders = extractTraceDebugOptions(workunit); - OwnedSpanScope requestSpan = queryTraceManager().createServerSpan("run_graph", traceHeaders); + OwnedActiveSpanScope requestSpan = queryTraceManager().createServerSpan("run_graph", traceHeaders); ContextSpanScope spanScope(*logctx, requestSpan); requestSpan->setSpanAttribute("hpcc.wuid", workunit->queryWuid()); requestSpan->setSpanAttribute("hpcc.graph", graphName); diff --git a/thorlcr/graph/thgraphslave.cpp b/thorlcr/graph/thgraphslave.cpp index 0b8a725b919..ff11c691ffc 100644 --- a/thorlcr/graph/thgraphslave.cpp +++ b/thorlcr/graph/thgraphslave.cpp @@ -1730,7 +1730,7 @@ CJobSlave::CJobSlave(ISlaveWatchdog *_watchdog, IPropertyTree *_workUnitInfo, co init(); Owned traceHeaders = deserializeTraceDebugOptions(workUnitInfo->queryPropTree("Debug")); - OwnedSpanScope requestSpan = queryTraceManager().createServerSpan("run_graph", traceHeaders); + OwnedActiveSpanScope requestSpan = queryTraceManager().createServerSpan("run_graph", traceHeaders); ContextSpanScope spanScope(*logctx, requestSpan); requestSpan->setSpanAttribute("hpcc.wuid", wuid); requestSpan->setSpanAttribute("hpcc.graph", graphName);