diff --git a/.github/workflows/build-assets.yml b/.github/workflows/build-assets.yml index 7c2d94e91a1..992737387f8 100644 --- a/.github/workflows/build-assets.yml +++ b/.github/workflows/build-assets.yml @@ -157,7 +157,7 @@ jobs: path: ${{ needs.preamble.outputs.folder_platform }} - name: Checkout LN - if: ${{ matrix.ln }} + if: ${{ matrix.ln || matrix.ee }} uses: actions/checkout@v4 with: repository: ${{ github.repository_owner }}/LN diff --git a/common/thorhelper/thorcommon.hpp b/common/thorhelper/thorcommon.hpp index dd833e7977f..2c3a63452e1 100644 --- a/common/thorhelper/thorcommon.hpp +++ b/common/thorhelper/thorcommon.hpp @@ -457,6 +457,10 @@ class CStatsContextLogger : public CSimpleInterfaceOf { activeSpan->setSpanAttribute(name, value); } + virtual void setSpanAttribute(const char *name, __uint64 value) const override + { + activeSpan->setSpanAttribute(name, value); + } virtual const char *queryGlobalId() const override { return activeSpan->queryGlobalId(); diff --git a/dali/base/dadfs.cpp b/dali/base/dadfs.cpp index 19f10e16efd..6e2c6b5f113 100644 --- a/dali/base/dadfs.cpp +++ b/dali/base/dadfs.cpp @@ -9934,6 +9934,7 @@ class CInitGroups StringArray clusternames; unsigned defaultTimeout; bool machinesLoaded; + bool writeLock; GroupType getGroupType(const char *type) { @@ -10009,6 +10010,8 @@ class CInitGroups void addClusterGroup(const char *name, IPropertyTree *newClusterGroup, bool realCluster) { + if (!writeLock) + throw makeStringException(0, "CInitGroups::addClusterGroup called in read-only mode"); VStringBuffer prop("Group[@name=\"%s\"]", name); IPropertyTree *root = groupsconnlock.conn->queryRoot(); IPropertyTree *old = root->queryPropTree(prop.str()); @@ -10366,11 +10369,12 @@ class CInitGroups return root->queryPropTree(xpath.str()); } public: - CInitGroups(unsigned _defaultTimeout) - : groupsconnlock("constructGroup",SDS_GROUPSTORE_ROOT,true,false,false,_defaultTimeout) + CInitGroups(unsigned _defaultTimeout, bool _writeLock) + : groupsconnlock("constructGroup",SDS_GROUPSTORE_ROOT,_writeLock,false,false,_defaultTimeout) { defaultTimeout = _defaultTimeout; machinesLoaded = false; + writeLock = _writeLock; } IPropertyTree *queryCluster(const IPropertyTree *env, const char *_clusterName, const char *type, const char *msg, StringBuffer &messages) @@ -10707,13 +10711,13 @@ class CInitGroups void initClusterGroups(bool force, StringBuffer &response, IPropertyTree *oldEnvironment, unsigned timems) { - CInitGroups init(timems); + CInitGroups init(timems, true); init.constructGroups(force, response, oldEnvironment); } void initClusterAndStoragePlaneGroups(bool force, IPropertyTree *oldEnvironment, unsigned timems) { - CInitGroups init(timems); + CInitGroups init(timems, true); StringBuffer response; init.constructGroups(force, response, oldEnvironment); @@ -10728,19 +10732,19 @@ void initClusterAndStoragePlaneGroups(bool force, IPropertyTree *oldEnvironment, bool resetClusterGroup(const char *clusterName, const char *type, bool spares, StringBuffer &response, unsigned timems) { - CInitGroups init(timems); + CInitGroups init(timems, true); return init.resetClusterGroup(clusterName, type, spares, response); } bool addClusterSpares(const char *clusterName, const char *type, const std::vector &hosts, StringBuffer &response, unsigned timems) { - CInitGroups init(timems); + CInitGroups init(timems, true); return init.addSpares(clusterName, type, hosts, response); } bool removeClusterSpares(const char *clusterName, const char *type, const std::vector &hosts, StringBuffer &response, unsigned timems) { - CInitGroups init(timems); + CInitGroups init(timems, true); return init.removeSpares(clusterName, type, hosts, response); } @@ -10764,7 +10768,7 @@ static IGroup *getClusterNodeGroup(const char *clusterName, const char *type, bo * to DFS and elsewhere. */ Owned nodeGroup = queryNamedGroupStore().lookup(nodeGroupName); - CInitGroups init(timems); + CInitGroups init(timems, false); Owned expandedClusterGroup = init.getGroupFromCluster(type, cluster, true); if (!expandedClusterGroup) throwStringExceptionV(0, "Failed to get group for '%s' cluster '%s'", type, clusterName); diff --git a/docs/EN_US/ContainerizedHPCC/ContainerizedMods/ContainerLogging.xml b/docs/EN_US/ContainerizedHPCC/ContainerizedMods/ContainerLogging.xml index 12349160019..e5c833b1f63 100644 --- a/docs/EN_US/ContainerizedHPCC/ContainerizedMods/ContainerLogging.xml +++ b/docs/EN_US/ContainerizedHPCC/ContainerizedMods/ContainerLogging.xml @@ -630,7 +630,48 @@ ContainerLog (1) up to "report all messages" (100). The default log level is rather high (80) and should be adjusted accordingly. - For example, verbosity should be medium for all components: + These are the available log levels: + + + + CriticalMsgThreshold = 1; + + + + FatalMsgThreshold = 1; + + + + ErrMsgThreshold = 10; + + + + WarnMsgThreshold = 20; + + + + AudMsgThreshold = 30; + + + + ProgressMsgThreshold = 50; + + + + InfoMsgThreshold = 60; + + + + DebugMsgThreshold = 80; + + + + ExtraneousMsgThreshold = 90; + + + + For example, to show only progress and lower level (more critical) + messages set the verbosity to 50: helm install myhpcc ./hpcc --set global.logging.detail="50" diff --git a/docs/EN_US/ECLLanguageReference/ECLR_mods/BltInFunc-DISTRIBUTE.xml b/docs/EN_US/ECLLanguageReference/ECLR_mods/BltInFunc-DISTRIBUTE.xml index e170e58dc2c..b96fcff2f80 100644 --- a/docs/EN_US/ECLLanguageReference/ECLR_mods/BltInFunc-DISTRIBUTE.xml +++ b/docs/EN_US/ECLLanguageReference/ECLR_mods/BltInFunc-DISTRIBUTE.xml @@ -104,16 +104,19 @@ maxskew - A floating point number in the range of zero (0.0) to one - (1.0) specifying the minimum skew to allow (0.1=10%). + A value between zero (0) and one (1.0 = 100%) indicating + the maximum percentage of skew to allow before the job fails (the + default skew is 1.0 / <number of worker nodes on + cluster>). skewlimit - Optional. A floating point number in the range of zero - (0.0) to one (1.0) specifying the maximum skew to allow - (0.1=10%). + Optional. A value between zero (0) and one (1.0 = 100%) + indicating the desired maximum percentage of skew to allow (the + default skew is 1.0 / <number of worker nodes on + cluster>). diff --git a/docs/EN_US/ECLLanguageReference/ECLR_mods/BltInFunc-JOIN.xml b/docs/EN_US/ECLLanguageReference/ECLR_mods/BltInFunc-JOIN.xml index ddf008eae2b..b6a4a7de816 100644 --- a/docs/EN_US/ECLLanguageReference/ECLR_mods/BltInFunc-JOIN.xml +++ b/docs/EN_US/ECLLanguageReference/ECLR_mods/BltInFunc-JOIN.xml @@ -534,7 +534,8 @@ A value between zero (0) and one (1.0 = 100%) indicating the maximum percentage of skew to allow before the job fails (the - default is 0.1 = 10%). + default skew is 1.0 / <number of worker nodes on + cluster>). @@ -542,7 +543,8 @@ Optional. A value between zero (0) and one (1.0 = 100%) indicating the desired maximum percentage of skew to allow (the - default is 0.1 = 10%). + default skew is 1.0 / <number of worker nodes on + cluster>). diff --git a/docs/EN_US/ECLLanguageReference/ECLR_mods/BltInFunc-SORT.xml b/docs/EN_US/ECLLanguageReference/ECLR_mods/BltInFunc-SORT.xml index 4a7dc994ab7..0b693b742b4 100644 --- a/docs/EN_US/ECLLanguageReference/ECLR_mods/BltInFunc-SORT.xml +++ b/docs/EN_US/ECLLanguageReference/ECLR_mods/BltInFunc-SORT.xml @@ -102,7 +102,8 @@ A value between zero (0) and one (1.0 = 100%) indicating the maximum percentage of skew to allow before the job fails (the - default is 0.1 = 10%). + default skew is 1.0 / <number of worker nodes on + cluster>). @@ -110,7 +111,8 @@ Optional. A value between zero (0) and one (1.0 = 100%) indicating the desired maximum percentage of skew to allow (the - default is 0.1 = 10%). + default skew is 1.0 / <number of worker nodes on + cluster>). diff --git a/ecl/eclcc/eclcc.cpp b/ecl/eclcc/eclcc.cpp index 30341d3e4cd..6462f51b58e 100644 --- a/ecl/eclcc/eclcc.cpp +++ b/ecl/eclcc/eclcc.cpp @@ -1602,9 +1602,9 @@ void EclCC::processSingleQuery(const EclRepositoryManager & localRepositoryManag { const BlockIoStats summaryIo = systemIoFinishInfo->querySummaryStats() - systemIoStartInfo->querySummaryStats(); if (summaryIo.rd_sectors) - updateWorkunitStat(instance.wu, SSToperation, scopeName, StSizeOsDiskRead, NULL, summaryIo.rd_sectors * summaryIo.getSectorSize()); + updateWorkunitStat(instance.wu, SSToperation, scopeName, StSizeDiskRead, NULL, summaryIo.rd_sectors * summaryIo.getSectorSize()); if (summaryIo.wr_sectors) - updateWorkunitStat(instance.wu, SSToperation, scopeName, StSizeOsDiskWrite, NULL, summaryIo.wr_sectors * summaryIo.getSectorSize()); + updateWorkunitStat(instance.wu, SSToperation, scopeName, StSizeDiskWrite, NULL, summaryIo.wr_sectors * summaryIo.getSectorSize()); } } diff --git a/esp/applications/eclservices/ldap_authorization_map.yaml b/esp/applications/eclservices/ldap_authorization_map.yaml index 55dadb9c563..d3b1ddcf9cd 100644 --- a/esp/applications/eclservices/ldap_authorization_map.yaml +++ b/esp/applications/eclservices/ldap_authorization_map.yaml @@ -84,6 +84,9 @@ ldap: - path: SashaAccess resource: SashaAccess description: Access to WS Sasha service + - path: PackageMapAccess + resource: PackageMapAccess + description: Access to packagemaps WsWorkunits: Feature: - path: OwnWorkunitsAccess @@ -143,7 +146,11 @@ ldap: - path: ConfigAccess resource: ConfigAccess description: Access to super computer environment - WsPackageProcess: [] + WsPackageProcess: + Feature: + - path: PackageMapAccess + resource: PackageMapAccess + description: Access to packagemaps ws_account: [] ws_access: [] ws_esdlconfig: [] diff --git a/esp/applications/eclwatch/ldap_authorization_map.yaml b/esp/applications/eclwatch/ldap_authorization_map.yaml index 1ce5b440e2a..95a601c37f2 100644 --- a/esp/applications/eclwatch/ldap_authorization_map.yaml +++ b/esp/applications/eclwatch/ldap_authorization_map.yaml @@ -85,6 +85,9 @@ ldap: - path: SashaAccess resource: SashaAccess description: Access to WS Sasha service + - path: PackageMapAccess + resource: PackageMapAccess + description: Access to packagemaps WsWorkunits: Feature: - path: OwnWorkunitsAccess @@ -154,7 +157,11 @@ ldap: - path: ResourceQueryAccess resource: ResourceQueryAccess description: Access to resources - WsPackageProcess: [] + WsPackageProcess: + Feature: + - path: PackageMapAccess + resource: PackageMapAccess + description: Access to packagemaps ws_account: [] ws_access: [] ws_esdlconfig: [] diff --git a/esp/bindings/http/platform/httpservice.cpp b/esp/bindings/http/platform/httpservice.cpp index 4e45abaa7aa..748155df69e 100644 --- a/esp/bindings/http/platform/httpservice.cpp +++ b/esp/bindings/http/platform/httpservice.cpp @@ -153,6 +153,33 @@ static bool authenticateOptionalFailed(IEspContext& ctx, IEspHttpBinding* bindin return false; } +static bool checkHttpPathStaysWithinBounds(const char *path) +{ + if (isEmptyString(path)) + return true; + //The path that follows /esp/files should be relative, not absolute - reject immediately if it is. + if (isAbsolutePath(path)) + return false; + int depth = 0; + StringArray nodes; + nodes.appendList(path, "/"); + ForEachItemIn(i, nodes) + { + const char *node = nodes.item(i); + if (!*node || streq(node, ".")) //empty or "." doesn't advance + continue; + if (!streq(node, "..")) + depth++; + else + { + depth--; + if (depth<0) //only really care that the relative http path doesn't position itself above its own root node + return false; + } + } + return true; +} + EspHttpBinding* CEspHttpServer::getBinding() { EspHttpBinding* thebinding=NULL; @@ -417,7 +444,16 @@ int CEspHttpServer::processRequest() m_response->redirect(*m_request.get(),url); } else + { + if (strieq(methodName.str(), "files_") && !checkHttpPathStaysWithinBounds(pathEx)) + { + AERRLOG("Get File %s: attempted access outside of %sfiles/", pathEx.str(), getCFD()); + m_response->setStatus(HTTP_STATUS_NOT_FOUND); + m_response->send(); + return 0; + } thebinding->onGet(m_request.get(), m_response.get()); + } } else unsupported(); @@ -733,33 +769,6 @@ static void httpGetDirectory(CHttpRequest* request, CHttpResponse* response, con response->send(); } -static bool checkHttpPathStaysWithinBounds(const char *path) -{ - if (!path || !*path) - return true; - //The path that follows /esp/files should be relative, not absolute - reject immediately if it is. - if (isAbsolutePath(path)) - return false; - int depth = 0; - StringArray nodes; - nodes.appendList(path, "/"); - ForEachItemIn(i, nodes) - { - const char *node = nodes.item(i); - if (!*node || streq(node, ".")) //empty or "." doesn't advance - continue; - if (!streq(node, "..")) - depth++; - else - { - depth--; - if (depth<0) //only really care that the relative http path doesn't position itself above its own root node - return false; - } - } - return true; -} - int CEspHttpServer::onGetFile(CHttpRequest* request, CHttpResponse* response, const char *urlpath) { if (!request || !response || !urlpath) @@ -810,6 +819,14 @@ int CEspHttpServer::onGetXslt(CHttpRequest* request, CHttpResponse* response, co if (!request || !response || !path) return -1; + if (!checkHttpPathStaysWithinBounds(path)) + { + AERRLOG("Get File %s: attempted access outside of %sxslt/", path, getCFD()); + response->setStatus(HTTP_STATUS_NOT_FOUND); + response->send(); + return 0; + } + StringBuffer mimetype, etag, lastModified; MemoryBuffer content; bool modified = true; diff --git a/esp/platform/espp.cpp b/esp/platform/espp.cpp index 43366530897..c3139b144c7 100644 --- a/esp/platform/espp.cpp +++ b/esp/platform/espp.cpp @@ -364,6 +364,20 @@ void initializeMetrics(CEspConfig* config) pMetricsTree.setown(getComponentConfigSP()->getPropTree("metrics")); } +#ifdef _DEBUG + if (!pMetricsTree) + { + const char * defaultMetricsYaml = R"!!( +sinks: + - type: log + name: loggingsink + settings: + period: 60 +)!!"; + pMetricsTree.setown(createPTreeFromYAMLString(defaultMetricsYaml)); + } +#endif + if (pMetricsTree != nullptr) { PROGLOG("Metrics initializing..."); diff --git a/esp/scm/ws_workunits.ecm b/esp/scm/ws_workunits.ecm index 25edec054ed..9f5a095cbfb 100644 --- a/esp/scm/ws_workunits.ecm +++ b/esp/scm/ws_workunits.ecm @@ -25,7 +25,7 @@ EspInclude(ws_workunits_queryset_req_resp); ESPservice [ auth_feature("DEFERRED"), //This declares that the method logic handles feature level authorization - version("1.98"), default_client_version("1.98"), cache_group("ESPWsWUs"), + version("1.99"), default_client_version("1.99"), cache_group("ESPWsWUs"), noforms,exceptions_inline("./smc_xslt/exceptions.xslt"),use_method_name] WsWorkunits { ESPmethod [cache_seconds(60), resp_xsl_default("/esp/xslt/workunits.xslt")] WUQuery(WUQueryRequest, WUQueryResponse); diff --git a/esp/scm/ws_workunits_struct.ecm b/esp/scm/ws_workunits_struct.ecm index abd719e5570..3c4114d074b 100644 --- a/esp/scm/ws_workunits_struct.ecm +++ b/esp/scm/ws_workunits_struct.ecm @@ -864,6 +864,7 @@ ESPStruct WUDetailsMetaProperty { string Name; ESPenum WUDetailsAttrValueType ValueType; + [min_ver("1.99")] string Description; }; ESPStruct WUDetailsActivityInfo diff --git a/esp/services/ws_workunits/ws_workunitsService.cpp b/esp/services/ws_workunits/ws_workunitsService.cpp index bcdac0ee2a2..4d77967a471 100644 --- a/esp/services/ws_workunits/ws_workunitsService.cpp +++ b/esp/services/ws_workunits/ws_workunitsService.cpp @@ -4235,7 +4235,7 @@ bool CWsWorkunitsEx::onWUAnalyseHotspot(IEspContext &context, IEspWUAnalyseHotsp } -static void getWUDetailsMetaProperties(IArrayOf & properties) +static void getWUDetailsMetaProperties(double version, IArrayOf & properties) { for (unsigned sk=StKindAll+1; sk & pro Owned property = createWUDetailsMetaProperty("",""); property->setName(s); property->setValueType(CWUDetailsAttrValueType_Single); + if (version >= 1.99) + property->setDescription(queryStatisticDescription((StatisticKind)sk)); properties.append(*property.getClear()); } } @@ -4302,8 +4304,10 @@ bool CWsWorkunitsEx::onWUDetailsMeta(IEspContext &context, IEspWUDetailsMetaRequ { try { + double version = context.getClientVersion(); + IArrayOf properties; - getWUDetailsMetaProperties(properties); + getWUDetailsMetaProperties(version, properties); resp.setProperties(properties); StringArray scopeTypes; @@ -4340,7 +4344,7 @@ class WUDetailsMetaTest : public CppUnit::TestFixture // These calls also check that all the calls required to build WUDetailsMeta // are successful. IArrayOf properties; - getWUDetailsMetaProperties(properties); + getWUDetailsMetaProperties(1.98, properties); unsigned expectedOrdinalityProps = StMax - (StKindAll + 1) + (WaMax-WaKind); ASSERT(properties.ordinality()==expectedOrdinalityProps); diff --git a/esp/src/eclwatch/templates/UserQueryWidget.html b/esp/src/eclwatch/templates/UserQueryWidget.html index 4617a93f3ef..28016977f25 100644 --- a/esp/src/eclwatch/templates/UserQueryWidget.html +++ b/esp/src/eclwatch/templates/UserQueryWidget.html @@ -146,12 +146,12 @@
-
-
+
+

${i18n.PleaseSelectAUserOrGroup}

- - - + + +
diff --git a/esp/src/src-react/components/Permissions.tsx b/esp/src/src-react/components/Permissions.tsx index 98c2f9039e6..33345e78233 100644 --- a/esp/src/src-react/components/Permissions.tsx +++ b/esp/src/src-react/components/Permissions.tsx @@ -9,6 +9,7 @@ import { useConfirm } from "../hooks/confirm"; import { useBuildInfo } from "../hooks/platform"; import { DojoGrid, selector, tree } from "./DojoGrid"; import { AddPermissionForm } from "./forms/AddPermission"; +import { CheckPermissionsForm } from "./forms/CheckPermissions"; import { HolyGrail } from "../layouts/HolyGrail"; import { pushUrl } from "../util/history"; @@ -35,6 +36,7 @@ export const Permissions: React.FunctionComponent = ({ const [selection, setSelection] = React.useState([]); const [showAddPermission, setShowAddPermission] = React.useState(false); + const [showCheckFilePermissions, setShowCheckFilePermissions] = React.useState(false); const [scopeScansEnabled, setScopeScansEnabled] = React.useState(false); const [modulesDn, setModulesDn] = React.useState(""); const [uiState, setUIState] = React.useState({ ...defaultUIState }); @@ -203,7 +205,7 @@ export const Permissions: React.FunctionComponent = ({ { key: "fileScopeDefaults", text: nlsHPCC.FileScopeDefaultPermissions, onClick: (evt, item) => pushUrl(`/${opsCategory}/security/permissions/_/File%20Scopes`), disabled: !uiState.fileScope }, { key: "workunitScopeDefaults", text: nlsHPCC.WorkUnitScopeDefaultPermissions, onClick: (evt, item) => pushUrl(`/${opsCategory}/security/permissions/_/Workunit%20Scopes`), disabled: !uiState.workunitScope }, { key: "physicalFiles", text: nlsHPCC.PhysicalFiles, onClick: (evt, item) => pushUrl(`/${opsCategory}/security/permissions/file/File%20Scopes`), disabled: !uiState.fileScope }, - { key: "checkFilePermissions", text: nlsHPCC.CheckFilePermissions, disabled: !uiState.fileScope }, + { key: "checkFilePermissions", text: nlsHPCC.CheckFilePermissions, onClick: () => setShowCheckFilePermissions(true), disabled: !uiState.fileScope }, { key: "codeGenerator", text: nlsHPCC.CodeGenerator, onClick: (evt, item) => pushUrl(`/${opsCategory}/security/permissions/_/${modulesDn}`), disabled: !uiState.repositoryModule }, ], }, @@ -223,6 +225,7 @@ export const Permissions: React.FunctionComponent = ({ + refreshTable()} /> ; diff --git a/esp/src/src-react/components/WorkunitDetails.tsx b/esp/src/src-react/components/WorkunitDetails.tsx index d4b779309f2..df196748aeb 100644 --- a/esp/src/src-react/components/WorkunitDetails.tsx +++ b/esp/src/src-react/components/WorkunitDetails.tsx @@ -105,9 +105,6 @@ export const WorkunitDetails: React.FunctionComponent = ({ }, { id: "eclsummary", label: nlsHPCC.ECL - }, { - id: "eclsummaryold", - label: "L" + nlsHPCC.ECL }, { id: "xml", label: nlsHPCC.XML diff --git a/esp/src/src-react/components/forms/CheckPermissions.tsx b/esp/src/src-react/components/forms/CheckPermissions.tsx new file mode 100644 index 00000000000..b917fdad3cc --- /dev/null +++ b/esp/src/src-react/components/forms/CheckPermissions.tsx @@ -0,0 +1,151 @@ +import * as React from "react"; +import { ComboBox, DefaultButton, IDropdownOption, MessageBar, MessageBarType, PrimaryButton, TextField } from "@fluentui/react"; +import { scopedLogger } from "@hpcc-js/util"; +import { useForm, Controller } from "react-hook-form"; +import nlsHPCC from "src/nlsHPCC"; +import { MessageBox } from "../../layouts/MessageBox"; +import { FilePermission } from "src/ws_access"; + +const logger = scopedLogger("src-react/components/forms/CheckPermissions.tsx"); + +interface CheckPermissionsFormValues { + FileName: string; + UserName: string; + GroupName: string; +} + +const defaultValues: CheckPermissionsFormValues = { + FileName: "", + UserName: "", + GroupName: "", +}; + +interface CheckPermissionsFormProps { + refreshGrid?: () => void; + showForm: boolean; + setShowForm: (_: boolean) => void; +} + +export const CheckPermissionsForm: React.FunctionComponent = ({ + refreshGrid, + showForm, + setShowForm +}) => { + + const { handleSubmit, control, reset, setValue, watch } = useForm({ defaultValues }); + + const [showError, setShowError] = React.useState(false); + const [errorMessage, setErrorMessage] = React.useState(""); + const [userOptions, setUserOptions] = React.useState([]); + const [groupOptions, setGroupOptions] = React.useState([]); + const [filePermissionResponse, setFilePermissionResponse] = React.useState(""); + + React.useEffect(() => { + const fetchData = async () => { + try { + const { FilePermissionResponse } = await FilePermission({}); + const { Users, Groups } = FilePermissionResponse; + const sortedUsers = Users?.User?.sort((a, b) => a.username.localeCompare(b.username)); + const sortedGroups = Groups?.Group?.sort((a, b) => a.name.localeCompare(b.name)); + + setUserOptions(sortedUsers?.map((user: { username: any; }) => { + return { key: user.username, text: user.username }; + }) ?? []); + + setGroupOptions(sortedGroups?.map(group => { + return { key: group.name, text: group.name }; + }) ?? []); + } catch (error) { + logger.error(error); + } + }; + + fetchData(); + }, []); + + const closeForm = React.useCallback(() => { + reset(defaultValues); + setShowForm(false); + setFilePermissionResponse(""); + }, [reset, setShowForm]); + + const onSubmit = async (data: CheckPermissionsFormValues) => { + try { + const { FileName, UserName, GroupName } = data; + logger.info(`Checking permissions for file ${FileName} for UserName: ${UserName} and GroupName: ${GroupName}`); + const response = await FilePermission({ + request: { + FileName, + UserName, + GroupName + } + }); + setFilePermissionResponse(response.FilePermissionResponse.UserPermission); + } catch (error) { + logger.error(error); + setErrorMessage(nlsHPCC.FilePermissionError); + setShowError(true); + } + }; + + return ( + + + + }> + } + rules={{ + required: nlsHPCC.ValidationErrorRequired + }} + /> + { + setValue("UserName", option?.key.toString() || ""); + setValue("GroupName", ""); + }} + /> + { + setValue("GroupName", option?.key.toString() || ""); + setValue("UserName", ""); + }} + /> + {filePermissionResponse && ( + + )} + {showError && +
+ setShowError(false)} dismissButtonAriaLabel="Close"> + {errorMessage} + +
+ } + + ); +}; \ No newline at end of file diff --git a/esp/src/src-react/components/forms/landing-zone/BlobImportForm.tsx b/esp/src/src-react/components/forms/landing-zone/BlobImportForm.tsx index 0ca2e32c585..2ce53e8f583 100644 --- a/esp/src/src-react/components/forms/landing-zone/BlobImportForm.tsx +++ b/esp/src/src-react/components/forms/landing-zone/BlobImportForm.tsx @@ -85,9 +85,8 @@ export const BlobImportForm: React.FunctionComponent = ({ request = data; if (!isContainer) { request["sourceIP"] = file.SourceIP; - } else { - request["sourcePlane"] = file.SourcePlane; } + request["sourcePlane"] = file.SourcePlane; request["sourcePath"] = file.SourceFile; request["fullPath"] = file.SourceFile; requests.push(FileSpray.SprayFixed({ diff --git a/esp/src/src-react/components/forms/landing-zone/DelimitedImportForm.tsx b/esp/src/src-react/components/forms/landing-zone/DelimitedImportForm.tsx index 13ffd9b8ccf..6e4d8accf49 100644 --- a/esp/src/src-react/components/forms/landing-zone/DelimitedImportForm.tsx +++ b/esp/src/src-react/components/forms/landing-zone/DelimitedImportForm.tsx @@ -102,9 +102,8 @@ export const DelimitedImportForm: React.FunctionComponent = ({ request = data; if (!isContainer) { request["sourceIP"] = file.SourceIP; - } else { - request["sourcePlane"] = file.SourcePlane; } + request["sourcePlane"] = file.SourcePlane; request["sourcePath"] = file.SourceFile; request["sourceRecordSize"] = file.RecordSize; request["destLogicalName"] = data.namePrefix + (( diff --git a/esp/src/src-react/components/forms/landing-zone/VariableImportForm.tsx b/esp/src/src-react/components/forms/landing-zone/VariableImportForm.tsx index 9f6842a579a..d9042425a31 100644 --- a/esp/src/src-react/components/forms/landing-zone/VariableImportForm.tsx +++ b/esp/src/src-react/components/forms/landing-zone/VariableImportForm.tsx @@ -84,9 +84,8 @@ export const VariableImportForm: React.FunctionComponent = ({ request = data; if (!isContainer) { request["sourceIP"] = file.SourceIP; - } else { - request["sourcePlane"] = file.SourcePlane; } + request["sourcePlane"] = file.SourcePlane; request["sourcePath"] = file.SourceFile; request["destLogicalName"] = data.namePrefix + (( data.namePrefix && data.namePrefix.substring(-2) !== "::" && diff --git a/esp/src/src-react/util/history.ts b/esp/src/src-react/util/history.ts index 9521b9902f4..e46d91e61a6 100644 --- a/esp/src/src-react/util/history.ts +++ b/esp/src/src-react/util/history.ts @@ -53,7 +53,8 @@ export function parseSearch>(_: strin return { ...parsed } as unknown as T; } -export function parseSort(_: string): QuerySortItem { +export function parseSort(_?: string): QuerySortItem | undefined { + if (!_) return undefined; const filter = parse(pick(_.substring(1), ["sortBy"])); let descending = false; let sortBy = filter?.sortBy?.toString(); diff --git a/esp/src/src/nls/hpcc.ts b/esp/src/src/nls/hpcc.ts index cfb38f32e89..c000f4b115e 100644 --- a/esp/src/src/nls/hpcc.ts +++ b/esp/src/src/nls/hpcc.ts @@ -338,7 +338,8 @@ export = { FileScopeDefaultPermissions: "File Scope Default Permissions", FileSize: "File Size", FilesNoPackage: "Files without matching package definitions", - FilePermission: "File Permission", + FilePermission: "File Permission", + FilePermissionError: "Error occurred while fetching file permissions.", FilesWarning: "The number of files returned is too large. Only the first 100,000 files sorted by date/time modified were returned. If you wish to limit results, set a filter.", FilesWithUnknownSize: "Files With Unknown Size", FileType: "File Type", diff --git a/roxie/ccd/ccd.hpp b/roxie/ccd/ccd.hpp index 04a7fdfd482..eafe07d9ae7 100644 --- a/roxie/ccd/ccd.hpp +++ b/roxie/ccd/ccd.hpp @@ -621,7 +621,7 @@ class ContextLogger : implements IRoxieContextLogger, public CInterface log.item(idx).writeXML(writer); } writer.outputEndArray("Log"); - }; + } virtual void CTXLOGva(const LogMsgCategory & cat, const LogMsgJobInfo & job, LogMsgCode code, const char *format, va_list args) const override __attribute__((format(printf,5,0))) { @@ -761,6 +761,10 @@ class ContextLogger : implements IRoxieContextLogger, public CInterface { activeSpan->setSpanAttribute(name, value); } + virtual void setSpanAttribute(const char *name, __uint64 value) const override + { + activeSpan->setSpanAttribute(name, value); + } virtual const char *queryGlobalId() const override { return activeSpan->queryGlobalId(); @@ -781,6 +785,7 @@ class ContextLogger : implements IRoxieContextLogger, public CInterface { stats.recordStatistics(progress, false); } + void exportStatsToSpan(bool failed, unsigned elapsed, unsigned memused, unsigned agentsDuplicates, unsigned agentsResends); }; class StringContextLogger : public ContextLogger diff --git a/roxie/ccd/ccdcontext.cpp b/roxie/ccd/ccdcontext.cpp index cd34a14624a..68169884ae0 100644 --- a/roxie/ccd/ccdcontext.cpp +++ b/roxie/ccd/ccdcontext.cpp @@ -1403,6 +1403,10 @@ class CRoxieContextBase : implements IRoxieAgentContext, implements ICodeContext { logctx.setSpanAttribute(name, value); } + virtual void setSpanAttribute(const char *name, __uint64 value) const override + { + logctx.setSpanAttribute(name, value); + } virtual const char *queryGlobalId() const override { return logctx.queryGlobalId(); diff --git a/roxie/ccd/ccdlistener.cpp b/roxie/ccd/ccdlistener.cpp index f99df009bb0..f8c0838c1b1 100644 --- a/roxie/ccd/ccdlistener.cpp +++ b/roxie/ccd/ccdlistener.cpp @@ -943,6 +943,27 @@ extern void updateAffinity(unsigned __int64 affinity) RoxieListener::updateAffinity(); } +//-------------------------------------------------------------------------------------------------------------------- + +void ContextLogger::exportStatsToSpan(bool failed, unsigned elapsed, unsigned memused, unsigned agentsDuplicates, unsigned agentsResends) +{ + if (activeSpan->isRecording()) + { + setSpanAttribute("time_elapsed", elapsed*1000000ULL); + if (failed) + setSpanAttribute("num_failures", 1); + if (memused) + setSpanAttribute("size_peak_row_memory", memused * 0x100000); + + StringBuffer prefix(""); + stats.exportToSpan(activeSpan, prefix); + } +} + +//-------------------------------------------------------------------------------------------------------------------- + + + class RoxieWorkUnitListener : public RoxieListener { Owned queue; @@ -1321,6 +1342,7 @@ class RoxieWorkUnitWorker : public RoxieQueryWorker logctx.CTXLOG("COMPLETE: %s%s complete in %u msecs memory=%u Mb priority=%d agentsreply=%u duplicatePackets=%u resentPackets=%u%s", wuid.get(), txidInfo.str(), elapsed, memused, priority, agentsReplyLen, agentsDuplicates, agentsResends, s.str()); } + logctx.exportStatsToSpan(failed, elapsed, memused, agentsDuplicates, agentsResends); } private: @@ -1589,31 +1611,37 @@ class RoxieProtocolMsgContext : implements IHpccProtocolMsgContext, public CInte queryFactory->noteQuery(startTime, failed, elapsed, memused, agentsReplyLen, bytesOut); queryFactory.clear(); } - if (logctx && logctx->queryTraceLevel() && (logFullQueries() || logctx->intercept)) + if (logctx) { - if (queryName.get()) + if (logctx->queryTraceLevel() && (logFullQueries() || logctx->intercept)) { - StringBuffer s; - logctx->getStats(s); - - const char * callerId = logctx->queryCallerId(); - StringBuffer txIds; - if (!isEmptyString(callerId)) - txIds.appendf("caller: %s", callerId); - const char *localId = logctx->queryLocalId(); - if (localId && *localId) + if (queryName.get()) { + StringBuffer s; + logctx->getStats(s); + + const char * callerId = logctx->queryCallerId(); + StringBuffer txIds; + if (!isEmptyString(callerId)) + txIds.appendf("caller: %s", callerId); + const char *localId = logctx->queryLocalId(); + if (localId && *localId) + { + if (txIds.length()) + txIds.append(", "); + txIds.append("local: ").append(localId); + } if (txIds.length()) - txIds.append(", "); - txIds.append("local: ").append(localId); + txIds.insert(0, '[').append(']'); + if (requestArraySize > 1) + logctx->CTXLOG("COMPLETE: %s(x%u) %s%s from %s complete in %u msecs memory=%u Mb priority=%d agentsreply=%u duplicatePackets=%u resentPackets=%u resultsize=%u continue=%d%s", queryName.get(), requestArraySize, uid.get(), txIds.str(), peer, elapsed, memused, getQueryPriority(), agentsReplyLen, agentsDuplicates, agentsResends, bytesOut, continuationNeeded, s.str()); + else + logctx->CTXLOG("COMPLETE: %s %s%s from %s complete in %u msecs memory=%u Mb priority=%d agentsreply=%u duplicatePackets=%u resentPackets=%u resultsize=%u continue=%d%s", queryName.get(), uid.get(), txIds.str(), peer, elapsed, memused, getQueryPriority(), agentsReplyLen, agentsDuplicates, agentsResends, bytesOut, continuationNeeded, s.str()); + } - if (txIds.length()) - txIds.insert(0, '[').append(']'); - if (requestArraySize > 1) - logctx->CTXLOG("COMPLETE: %s(x%u) %s%s from %s complete in %u msecs memory=%u Mb priority=%d agentsreply=%u duplicatePackets=%u resentPackets=%u resultsize=%u continue=%d%s", queryName.get(), requestArraySize, uid.get(), txIds.str(), peer, elapsed, memused, getQueryPriority(), agentsReplyLen, agentsDuplicates, agentsResends, bytesOut, continuationNeeded, s.str()); - else - logctx->CTXLOG("COMPLETE: %s %s%s from %s complete in %u msecs memory=%u Mb priority=%d agentsreply=%u duplicatePackets=%u resentPackets=%u resultsize=%u continue=%d%s", queryName.get(), uid.get(), txIds.str(), peer, elapsed, memused, getQueryPriority(), agentsReplyLen, agentsDuplicates, agentsResends, bytesOut, continuationNeeded, s.str()); } + + logctx->exportStatsToSpan(failed, elapsed, memused, agentsDuplicates, agentsResends); } } }; diff --git a/roxie/ccd/ccdserver.cpp b/roxie/ccd/ccdserver.cpp index c3b5fe22ac9..67e88252f3e 100644 --- a/roxie/ccd/ccdserver.cpp +++ b/roxie/ccd/ccdserver.cpp @@ -296,6 +296,10 @@ class IndirectAgentContext : implements IRoxieAgentContext, public CInterface { ctx->setSpanAttribute(name, value); } + virtual void setSpanAttribute(const char *name, __uint64 value) const override + { + ctx->setSpanAttribute(name, value); + } virtual const char *queryGlobalId() const { return ctx->queryGlobalId(); @@ -1444,6 +1448,10 @@ class CRoxieServerActivity : implements CInterfaceOf, impl { ctx->setSpanAttribute(name, value); } + virtual void setSpanAttribute(const char *name, __uint64 value) const override + { + ctx->setSpanAttribute(name, value); + } virtual const char *queryGlobalId() const override { return ctx ? ctx->queryGlobalId() : nullptr; diff --git a/system/jlib/jlog.cpp b/system/jlib/jlog.cpp index ee6de055063..6039e30ba71 100644 --- a/system/jlib/jlog.cpp +++ b/system/jlib/jlog.cpp @@ -2893,6 +2893,10 @@ class DummyLogCtx : implements IContextLogger { activeSpan->setSpanAttribute(name, value); } + virtual void setSpanAttribute(const char *name, __uint64 value) const override + { + activeSpan->setSpanAttribute(name, value); + } virtual const char *queryGlobalId() const override { return activeSpan->queryGlobalId(); diff --git a/system/jlib/jlog.hpp b/system/jlib/jlog.hpp index ca142787684..02d5dfff2e6 100644 --- a/system/jlib/jlog.hpp +++ b/system/jlib/jlog.hpp @@ -1276,6 +1276,7 @@ interface jlib_decl IContextLogger : extends IInterface virtual IProperties * getSpanContext() const = 0; virtual IProperties * getClientHeaders() const = 0; virtual void setSpanAttribute(const char *name, const char *value) const = 0; + virtual void setSpanAttribute(const char *name, __uint64 value) const = 0; virtual void recordStatistics(IStatisticGatherer &progress) const = 0; virtual const LogMsgJobInfo & queryJob() const { return unknownJob; } }; diff --git a/system/jlib/jstatcodes.h b/system/jlib/jstatcodes.h index 23a74dfa659..2314a94ab65 100644 --- a/system/jlib/jstatcodes.h +++ b/system/jlib/jstatcodes.h @@ -301,6 +301,7 @@ enum StatisticKind StNumAckRetries, StSizeContinuationData, StNumContinuationRequests, + StNumFailures, StMax, //For any quantity there is potentially the following variants. diff --git a/system/jlib/jstats.cpp b/system/jlib/jstats.cpp index 604ec7224ec..0f5b4618932 100644 --- a/system/jlib/jstats.cpp +++ b/system/jlib/jstats.cpp @@ -439,6 +439,16 @@ StringBuffer & formatStatistic(StringBuffer & out, unsigned __int64 value, Stati return formatStatistic(out, value, queryMeasure(kind)); } +constexpr static stat_type usTimestampThreshold = 100ULL * 365ULL * 24ULL * 3600'000000ULL; // 100 years in us +// Support the gradual conversion of timestamps from us to ns. If a timestamp is in the old format (before 2070) +// then it is assumed to be in us, otherwise ns. +stat_type normalizeTimestampToNs(stat_type value) +{ + if (value < usTimestampThreshold) + return value * 1000; + return value; +} + //-------------------------------------------------------------------------------------------------------------------- stat_type readStatisticValue(const char * cur, const char * * end, StatisticMeasure measure) @@ -759,7 +769,7 @@ StatisticMeasure queryMeasure(const char * measure, StatisticMeasure dft) #define NODESTAT(y) STAT(Node, y, SMeasureNode, StatsMergeKeepNonZero) #define PERSTAT(y) STAT(Per, y, SMeasurePercent, StatsMergeReplace) #define IPV4STAT(y) STAT(IPV4, y, SMeasureIPV4, StatsMergeKeepNonZero) -#define CYCLESTAT(y) St##Cycle##y##Cycles, SMeasureCycle, StatsMergeSum, St##Time##y, St##Cycle##y##Cycles, { NAMES(Cycle, y##Cycles) }, { TAGS(Cycle, y##Cycles) } +#define CYCLESTAT(y) St##Cycle##y##Cycles, SMeasureCycle, StatsMergeSum, St##Time##y, St##Cycle##y##Cycles, { NAMES(Cycle, y##Cycles) }, { TAGS(Cycle, y##Cycles) }, "" #define ENUMSTAT(y) STAT(Enum, y, SMeasureEnum, StatsMergeKeepNonZero) #define COSTSTAT(y) STAT(Cost, y, SMeasureCost, StatsMergeSum) #define PEAKSIZESTAT(y) STAT(Size, y, SMeasureSize, StatsMergeMax) @@ -775,190 +785,195 @@ class StatisticMeta StatisticKind rawKind; const char * names[StNextModifier/StVariantScale]; const char * tags[StNextModifier/StVariantScale]; + const char * description; }; +constexpr const char * UNUSED = ""; + //The order of entries in this table must match the order in the enumeration static const constexpr StatisticMeta statsMetaData[StMax] = { - { StKindNone, SMeasureNone, StatsMergeSum, StKindNone, StKindNone, { "none" }, { "@none" } }, - { StKindAll, SMeasureAll, StatsMergeSum, StKindAll, StKindAll, { "all" }, { "@all" } }, - { WHENFIRSTSTAT(GraphStarted) }, // Deprecated - use WhenStart - { WHENLASTSTAT(GraphFinished) }, // Deprecated - use WhenFinished - { WHENFIRSTSTAT(FirstRow) }, - { WHENFIRSTSTAT(QueryStarted) }, // Deprecated - use WhenStart - { WHENLASTSTAT(QueryFinished) }, // Deprecated - use WhenFinished - { WHENFIRSTSTAT(Created) }, - { WHENFIRSTSTAT(Compiled) }, - { WHENFIRSTSTAT(WorkunitModified) }, - { TIMESTAT(Elapsed) }, - { TIMESTAT(LocalExecute) }, - { TIMESTAT(TotalExecute) }, - { TIMESTAT(Remaining) }, - { SIZESTAT(GeneratedCpp) }, - { SIZESTAT(PeakMemory) }, - { SIZESTAT(MaxRowSize) }, - { NUMSTAT(RowsProcessed) }, - { NUMSTAT(Slaves) }, - { NUMSTAT(Starts) }, - { NUMSTAT(Stops) }, - { NUMSTAT(IndexSeeks) }, - { NUMSTAT(IndexScans) }, - { NUMSTAT(IndexWildSeeks) }, - { NUMSTAT(IndexSkips) }, - { NUMSTAT(IndexNullSkips) }, - { NUMSTAT(IndexMerges) }, - { NUMSTAT(IndexMergeCompares) }, - { NUMSTAT(PreFiltered) }, - { NUMSTAT(PostFiltered) }, - { NUMSTAT(BlobCacheHits) }, - { NUMSTAT(LeafCacheHits) }, - { NUMSTAT(NodeCacheHits) }, - { NUMSTAT(BlobCacheAdds) }, - { NUMSTAT(LeafCacheAdds) }, - { NUMSTAT(NodeCacheAdds) }, - { NUMSTAT(PreloadCacheHits) }, - { NUMSTAT(PreloadCacheAdds) }, - { NUMSTAT(ServerCacheHits) }, - { NUMSTAT(IndexAccepted) }, - { NUMSTAT(IndexRejected) }, - { NUMSTAT(AtmostTriggered) }, - { NUMSTAT(DiskSeeks) }, - { NUMSTAT(Iterations) }, - { LOADSTAT(WhileSorting) }, - { NUMSTAT(LeftRows) }, - { NUMSTAT(RightRows) }, - { PERSTAT(Replicated) }, - { NUMSTAT(DiskRowsRead) }, - { NUMSTAT(IndexRowsRead) }, - { NUMSTAT(DiskAccepted) }, - { NUMSTAT(DiskRejected) }, - { TIMESTAT(Soapcall) }, - { TIMESTAT(FirstExecute) }, - { TIMESTAT(DiskReadIO) }, - { TIMESTAT(DiskWriteIO) }, - { SIZESTAT(DiskRead) }, - { SIZESTAT(DiskWrite) }, + { StKindNone, SMeasureNone, StatsMergeSum, StKindNone, StKindNone, { "none" }, { "@none" }, nullptr }, + { StKindAll, SMeasureAll, StatsMergeSum, StKindAll, StKindAll, { "all" }, { "@all" }, nullptr }, + { WHENFIRSTSTAT(GraphStarted), "The time when a graph started./nDeprecated" }, // Deprecated - use WhenStart + { WHENLASTSTAT(GraphFinished), "The time when a graph finished./nDeprecated" }, // Deprecated - use WhenFinished + { WHENFIRSTSTAT(FirstRow), "The time when the first row is processed by an activity" }, + { WHENFIRSTSTAT(QueryStarted), "The time when a query started./nDeprecated" }, // Deprecated - use WhenStart + { WHENLASTSTAT(QueryFinished), "The time when a query finished./nDeprecated" }, // Deprecated - use WhenFinished + { WHENFIRSTSTAT(Created), "The time when an item was created" }, + { WHENFIRSTSTAT(Compiled), "The time a workunit started being compiled" }, + { WHENFIRSTSTAT(WorkunitModified), UNUSED }, + { TIMESTAT(Elapsed), "The elapsed time between starting and finishing\nFor child queries this may be significantly larger than TimeTotalExecute" }, + { TIMESTAT(LocalExecute), "The time spent executing this activity not including its inputs\nSort activities by local execute time to help isolate potential processing bottlenecks" }, + { TIMESTAT(TotalExecute), "The time spent executing this activity and its inputs\nSort activities by total execute time to find sections of a query that are a bottleneck" }, + { TIMESTAT(Remaining), UNUSED }, + { SIZESTAT(GeneratedCpp), "The size of the generated c++ file" }, + { SIZESTAT(PeakMemory), "The peak memory used while processing this item" }, + { SIZESTAT(MaxRowSize), "The high water mark of the memory used for representing rows (roxiemem)" }, + { NUMSTAT(RowsProcessed), "The number of rows processed" }, + { NUMSTAT(Slaves), "The number of parallel execution processes used to execute an activity" }, + { NUMSTAT(Starts), "The number of times the activity has started executing\nAn activity is active if this does not match NumStops" }, + { NUMSTAT(Stops), "The number of times the activity has stopped executing\nAn activity is active if this is less than NumStarts" }, + { NUMSTAT(IndexSeeks), "The number of keyed lookups on an index\nThese correspond to KEYED() filters on indexes. A single keyed filter may result in multiple seeks if a leading component is not single-valued" }, + { NUMSTAT(IndexScans), "The number of index scans\nHow many entries are sequentially examined after an initial seek (including wild seeks). Large numbers compared to the number of seeks may indicate extra keyed filters would be worthwhile" }, + { NUMSTAT(IndexWildSeeks), "The number of seeks caused by WILD() filters\nThe number of keyed lookups that had to search for the next potential match. If this is a high proportion of NumIndexScans it may suggest poor key design" }, + { NUMSTAT(IndexSkips), "The number of smart-stepping operations that increment the next match" }, + { NUMSTAT(IndexNullSkips), "The number of smart-stepping operations that had no effect\nIf this is large compare to NumIndexSkips it suggests the priority may not be set correctly" }, + { NUMSTAT(IndexMerges), "The number of merges set up when smart stepping"}, + { NUMSTAT(IndexMergeCompares), "The number of merge comparisons when smart stepping" }, + { NUMSTAT(PreFiltered), "The number of LEFT rows filtered before performing a keyed lookup" }, + { NUMSTAT(PostFiltered), "The number of index matches filtered by the transform and the non-keyed filter" }, + { NUMSTAT(BlobCacheHits), "The number of times a blob was resolved in the cache" }, + { NUMSTAT(LeafCacheHits), "The number of times a leaf node was resolved in the cache" }, + { NUMSTAT(NodeCacheHits), "The number of times a branch node was resolved in the cache" }, + { NUMSTAT(BlobCacheAdds), "The number of times a blob was read from disk rather than the cache" }, + { NUMSTAT(LeafCacheAdds), "The number of times a leaf node was read from disk rather than the cache\nIf this number is high it may be worth experimenting with the leaf cache size (the branch cache size is more important)" }, + { NUMSTAT(NodeCacheAdds), "The number of times a branch node was read from disk rather than the cache\nBranch cache hits are significant for performance. If this number is high it is likely to be worth increasing the node cache size. If this number does not increase once the system is warmed up it may be worth reducing the cache size" }, + { NUMSTAT(PreloadCacheHits), UNUSED }, + { NUMSTAT(PreloadCacheAdds), UNUSED }, + { NUMSTAT(ServerCacheHits), UNUSED }, + { NUMSTAT(IndexAccepted), "The number of KEYED JOIN matches that return a result from the TRANSFORM" }, + { NUMSTAT(IndexRejected), "The number of KEYED JOIN matches that are skipped by the TRANSFORM" }, + { NUMSTAT(AtmostTriggered), "The number of times ATMOST on a JOIN causes it to fail to match" }, + { NUMSTAT(DiskSeeks), "The number of FETCHES from disk" }, + { NUMSTAT(Iterations), "The number of LOOP iterations executed" }, + { LOADSTAT(WhileSorting), UNUSED }, + { NUMSTAT(LeftRows), "The number of LEFT rows processed" }, + { NUMSTAT(RightRows), "The number of RIGHT rows processed" }, + { PERSTAT(Replicated), "The percentage replication complete" }, + { NUMSTAT(DiskRowsRead), "The number of rows read from the file" }, + { NUMSTAT(IndexRowsRead), "The number of rows read from the index" }, + { NUMSTAT(DiskAccepted), "The number of disk rows that return a result from the TRANSFORM" }, + { NUMSTAT(DiskRejected), "The number of disk rows that are skipped by the TRANSFORM" }, + { TIMESTAT(Soapcall), "The time taken to execute a SOAPCALL" }, + { TIMESTAT(FirstExecute), "The time taken to return the first row from this activity" }, + { TIMESTAT(DiskReadIO), "Total time spent reading from disk" }, + { TIMESTAT(DiskWriteIO), "Total time spent writing to disk" }, + { SIZESTAT(DiskRead), "Total size of data read from disk" }, + { SIZESTAT(DiskWrite), "Total size of data written to disk" }, { CYCLESTAT(DiskReadIO) }, { CYCLESTAT(DiskWriteIO) }, - { NUMSTAT(DiskReads) }, - { NUMSTAT(DiskWrites) }, - { NUMSTAT(Spills) }, - { TIMESTAT(SpillElapsed) }, - { TIMESTAT(SortElapsed) }, - { NUMSTAT(Groups) }, - { NUMSTAT(GroupMax) }, - { SIZESTAT(SpillFile) }, + { NUMSTAT(DiskReads), "The number of disk read operations" }, + { NUMSTAT(DiskWrites), "The number of disk write operations" }, + { NUMSTAT(Spills), "The number of times the activity spilt to disk"}, + { TIMESTAT(SpillElapsed), "Time spent spilling rows from memory to disk" }, //MORE: Do we have a similar stat for SpillRead? + { TIMESTAT(SortElapsed), "Time spent sorting rows in memory" }, + { NUMSTAT(Groups), "The number of groups processed by this activity" }, + { NUMSTAT(GroupMax), "The size of the largest group processed by this activity\nA skew in group size can cause a skew in processing time. A large skew may indicate some special values would benefit from special casing" }, + { SIZESTAT(SpillFile), "Total size of data spilled to disk" }, { CYCLESTAT(SpillElapsed) }, { CYCLESTAT(SortElapsed) }, - { NUMSTAT(Strands) }, + { NUMSTAT(Strands), "The number of parallel execution strands\n(A partially implemented feature to allow parallel execution within an activity)" }, { CYCLESTAT(TotalExecute) }, - { NUMSTAT(Executions) }, - { TIMESTAT(TotalNested) }, + { NUMSTAT(Executions), "The number of times a graph has been executed" }, + { TIMESTAT(TotalNested), UNUSED }, { CYCLESTAT(LocalExecute) }, - { NUMSTAT(Compares) }, - { NUMSTAT(ScansPerRow) }, - { NUMSTAT(Allocations) }, - { NUMSTAT(AllocationScans) }, - { NUMSTAT(DiskRetries) }, + { NUMSTAT(Compares), UNUSED }, + { NUMSTAT(ScansPerRow), UNUSED }, + { NUMSTAT(Allocations), "The number of allocations from the row memory" }, + { NUMSTAT(AllocationScans), "The number of scans within the memory manager when allocating row memory\nOnly applies to the scanning heap manager (not used by default)" }, + { NUMSTAT(DiskRetries), "The number of times an I/O operation was retried\nIf this is non-zero it may suggest a problem with the underlying disk storage" }, { CYCLESTAT(Elapsed) }, { CYCLESTAT(Remaining) }, { CYCLESTAT(Soapcall) }, { CYCLESTAT(FirstExecute) }, { CYCLESTAT(TotalNested) }, - { TIMESTAT(Generate) }, + { TIMESTAT(Generate), "Time taken to generate the c++ code from the parsed ECL" }, { CYCLESTAT(Generate) }, - { WHENFIRSTSTAT(Started) }, - { WHENLASTSTAT(Finished) }, - { NUMSTAT(AnalyseExprs) }, - { NUMSTAT(TransformExprs) }, - { NUMSTAT(UniqueAnalyseExprs) }, - { NUMSTAT(UniqueTransformExprs) }, - { NUMSTAT(DuplicateKeys) }, - { NUMSTAT(AttribsProcessed) }, - { NUMSTAT(AttribsSimplified) }, - { NUMSTAT(AttribsFromCache) }, - { NUMSTAT(SmartJoinDegradedToLocal) }, - { NUMSTAT(SmartJoinSlavesDegradedToStd) }, - { NUMSTAT(AttribsSimplifiedTooComplex) }, - { NUMSTAT(SysContextSwitches) }, - { TIMESTAT(OsUser) }, - { TIMESTAT(OsSystem) }, - { TIMESTAT(OsTotal) }, + { WHENFIRSTSTAT(Started), "Time when this activity or operation started" }, + { WHENLASTSTAT(Finished), "Time when this activity or operation finished" }, + { NUMSTAT(AnalyseExprs), "Code generator internal\nThe number of expressions that were processed by transformer::analyse()" }, + { NUMSTAT(TransformExprs), "Code generator internal\nThe number of expressions that were processed by transformer::transform()" }, + { NUMSTAT(UniqueAnalyseExprs), "Code generator internal\nThe number of unique expressions that were processed by transformer::analyse()" }, + { NUMSTAT(UniqueTransformExprs), "Code generator internal\nThe number of unique expressions that were processed by transformer::transform()" }, + { NUMSTAT(DuplicateKeys), "The number of duplicate keys that were present in the index" }, + { NUMSTAT(AttribsProcessed), "The number of attributes processed when parsing the ECL" }, + { NUMSTAT(AttribsSimplified), UNUSED }, + { NUMSTAT(AttribsFromCache), UNUSED }, + { NUMSTAT(SmartJoinDegradedToLocal), "The number of times a global smart-join switched to a LOCAL JOIN (with distribute)\nThis will be 0 or 1 unless the activity is within a LOOP" }, + { NUMSTAT(SmartJoinSlavesDegradedToStd), "The number of times a global smart-join degraded to a standard join" }, + { NUMSTAT(AttribsSimplifiedTooComplex), UNUSED }, + { NUMSTAT(SysContextSwitches), "The number of context switches that occurred when processing" }, + { TIMESTAT(OsUser), "Total elapsed user-space time" }, + { TIMESTAT(OsSystem), "Total time spent in the system/kernel" }, + { TIMESTAT(OsTotal), "Total elapsed time according to the OS\nIncludes system, user, idle and iowait times" }, { CYCLESTAT(OsUser) }, { CYCLESTAT(OsSystem) }, { CYCLESTAT(OsTotal) }, - { NUMSTAT(ContextSwitches) }, - { TIMESTAT(User) }, - { TIMESTAT(System) }, - { TIMESTAT(Total) }, + //The following seem to be duplicates of the values above + { NUMSTAT(ContextSwitches), "The number of context switches that occurred when processing" }, + { TIMESTAT(User), "Total elapsed user-space time" }, + { TIMESTAT(System), "Total time spent in the system/kernel" }, + { TIMESTAT(Total), "Total elapsed time according to the OS\nInclude system,user,idle and iowait times" }, { CYCLESTAT(User) }, { CYCLESTAT(System) }, { CYCLESTAT(Total) }, - { SIZESTAT(OsDiskRead) }, - { SIZESTAT(OsDiskWrite) }, - { TIMESTAT(Blocked) }, + { SIZESTAT(OsDiskRead), UNUSED }, + { SIZESTAT(OsDiskWrite), UNUSED }, + { TIMESTAT(Blocked), "Time spent blocked waiting for another operation to complete" }, { CYCLESTAT(Blocked) }, - { COSTSTAT(Execute) }, - { SIZESTAT(AgentReply) }, - { TIMESTAT(AgentWait) }, + { COSTSTAT(Execute), "The CPU cost of executing" }, + { SIZESTAT(AgentReply), "Size of data sent from the workers to the agent" }, + { TIMESTAT(AgentWait), "Time that the agent spent waiting for a reply from the workers" }, { CYCLESTAT(AgentWait) }, - { COSTSTAT(FileAccess) }, - { NUMSTAT(Pods) }, - { COSTSTAT(Compile) }, - { TIMESTAT(NodeLoad) }, + { COSTSTAT(FileAccess), "The transactional cost of any file operations" }, + { NUMSTAT(Pods), "The number of pods used" }, + { COSTSTAT(Compile), "The cost to compile this workunit"}, + { TIMESTAT(NodeLoad), "Time spent reading branch nodes from disk and decompressing them" }, { CYCLESTAT(NodeLoad) }, - { TIMESTAT(LeafLoad) }, + { TIMESTAT(LeafLoad), "Time spent reading leaf nodes from disk and decompressing them\nIf this is a high proportion of the time (especially compared to TimeLeafRead) then consider using the new index compression formats" }, { CYCLESTAT(LeafLoad) }, - { TIMESTAT(BlobLoad) }, + { TIMESTAT(BlobLoad), "Time spent reading blob nodes from disk and decompressing them" }, { CYCLESTAT(BlobLoad) }, - { TIMESTAT(Dependencies) }, + { TIMESTAT(Dependencies), "Time spent processing dependencies for this activity"}, { CYCLESTAT(Dependencies) }, - { TIMESTAT(Start) }, + { TIMESTAT(Start), "Time taken to start an activity\nThis includes the time spent processing dependencies" }, { CYCLESTAT(Start) }, - { ENUMSTAT(ActivityCharacteristics) }, - { TIMESTAT(NodeRead) }, + { ENUMSTAT(ActivityCharacteristics), "A bitfield describing characteristics of the activity\nurgentStart = 0x01, hasRowLatency = 0x02, hasDependencies = 0x04, slowDependencies = 0x08" }, + { TIMESTAT(NodeRead), "Time spent reading branch nodes from disk (including linux page cache)" }, { CYCLESTAT(NodeRead) }, - { TIMESTAT(LeafRead) }, + { TIMESTAT(LeafRead), "Time spent reading leaf nodes from disk (including linux page cache)" }, { CYCLESTAT(LeafRead) }, - { TIMESTAT(BlobRead) }, + { TIMESTAT(BlobRead), "Time spent reading blob from disk (including linux page cache)" }, { CYCLESTAT(BlobRead) }, - { NUMSTAT(NodeDiskFetches) }, - { NUMSTAT(LeafDiskFetches) }, - { NUMSTAT(BlobDiskFetches) }, - { TIMESTAT(NodeFetch) }, + { NUMSTAT(NodeDiskFetches), "Number of times a branch node was read from disk rather than the linux page cache" }, + { NUMSTAT(LeafDiskFetches), "Number of times a leaf node was read from disk rather than the linux page cache\nIf this is a significant proportion of NumLeafAdds then consider allocating more memory, or reducing the leaf cache size" }, + { NUMSTAT(BlobDiskFetches), "Number of times a blob was read from disk rather than the linux page cache" }, + { TIMESTAT(NodeFetch), "Time spent reading branch nodes from disk (EXCLUDING the linux page cache)" }, { CYCLESTAT(NodeFetch) }, - { TIMESTAT(LeafFetch) }, + { TIMESTAT(LeafFetch), "Time spent reading leaf nodes from disk (EXCLUDING the linux page cache)" }, { CYCLESTAT(LeafFetch) }, - { TIMESTAT(BlobFetch) }, + { TIMESTAT(BlobFetch), "Time spent reading blobs from disk (EXCLUDING the linux page cache)" }, { CYCLESTAT(BlobFetch) }, - { PEAKSIZESTAT(GraphSpill) }, - { TIMESTAT(AgentQueue) }, + { PEAKSIZESTAT(GraphSpill), "Peak size of spill memory usage" }, + { TIMESTAT(AgentQueue), "Time worker items were received and queued before being processed\nThis may indicate that the primary node on a channel was down, or that the workers are overloaded with requests" }, { CYCLESTAT(AgentQueue) }, - { TIMESTAT(IBYTIDelay) }, + { TIMESTAT(IBYTIDelay), "Time spent waiting for another worker to start processing a request\nA non-zero value indicates that the primary node on a channel was down or very busy" }, { CYCLESTAT(IBYTIDelay) }, - { WHENFIRSTSTAT(Queued) }, - { WHENFIRSTSTAT(Dequeued) }, - { WHENFIRSTSTAT(K8sLaunched) }, - { WHENFIRSTSTAT(K8sStarted) }, - { WHENFIRSTSTAT(K8sReady) }, - { NUMSTAT(SocketWrites) }, - { SIZESTAT(SocketWrite) }, - { TIMESTAT(SocketWriteIO) }, + { WHENFIRSTSTAT(Queued), "The time when this item was added to a queue" }, + { WHENFIRSTSTAT(Dequeued), "The time when this item was removed from a queue" }, + { WHENFIRSTSTAT(K8sLaunched), "The time when the K8s job to process this item was launched" }, + { WHENFIRSTSTAT(K8sStarted), "The time when the K8s job to process this item started executing/nThe difference between the K8sStarted and K8sLaunched indicates how long Kubernetes took to resource and initialised the job" }, + { WHENFIRSTSTAT(K8sReady), "The time when the Thor job is ready to process\nThe difference with K8sStarted indicates how long it took to resource and start the slave processes" }, + { NUMSTAT(SocketWrites), "The number of writes to the client socket" }, + { SIZESTAT(SocketWrite), "The size of data written to the client socket" }, + { TIMESTAT(SocketWriteIO), "The total time spent writing data to the client socket" }, { CYCLESTAT(SocketWriteIO) }, - { NUMSTAT(SocketReads) }, - { SIZESTAT(SocketRead) }, - { TIMESTAT(SocketReadIO) }, + { NUMSTAT(SocketReads), "The number of reads from the client socket" }, + { SIZESTAT(SocketRead), "The size of data read from the client socket" }, + { TIMESTAT(SocketReadIO), "The total time spent reading data from the client socket" }, { CYCLESTAT(SocketReadIO) }, - { SIZESTAT(Memory) }, - { SIZESTAT(RowMemory) }, - { SIZESTAT(PeakRowMemory) }, - { SIZESTAT(AgentSend) }, - { TIMESTAT(IndexCacheBlocked) }, + { SIZESTAT(Memory), "The total memory allocated from the system" }, + { SIZESTAT(RowMemory), "The size of memory used to store rows" }, + { SIZESTAT(PeakRowMemory), "The peak memory used to store rows" }, + { SIZESTAT(AgentSend), "The size of data sent to the agent from the server" }, + { TIMESTAT(IndexCacheBlocked), "The time spent waiting to access the index page cache" }, { CYCLESTAT(IndexCacheBlocked) }, - { TIMESTAT(AgentProcess) }, + { TIMESTAT(AgentProcess), "The total time spent by the agents processing requests" }, { CYCLESTAT(AgentProcess) }, - { NUMSTAT(AckRetries) }, - { SIZESTAT(ContinuationData) }, - { NUMSTAT(ContinuationRequests) }, + { NUMSTAT(AckRetries), "The number of times the server failed to receive a response from an agent within the expected time" }, + { SIZESTAT(ContinuationData), "The total size of continuation data sent from agent to the server\nA large number may indicate a poor filter, or merging from many different index locations" }, + { NUMSTAT(ContinuationRequests), "The number of times the agent indicated there was more data to be returned" }, + { NUMSTAT(Failures), "The number of times a query has failed" }, }; static MapStringTo statisticNameMap(true); @@ -1067,6 +1082,14 @@ const char * queryStatisticName(StatisticKind kind) } +const char * queryStatisticDescription(StatisticKind kind) +{ + StatisticKind rawkind = (StatisticKind)(kind & StKindMask); + if (rawkind >= StKindNone && rawkind < StMax) + return statsMetaData[rawkind].description; + return nullptr; +} + unsigned __int64 convertMeasure(StatisticMeasure from, StatisticMeasure to, unsigned __int64 value) { if (from == to) @@ -2796,6 +2819,42 @@ StringBuffer & CRuntimeStatisticCollection::toStr(StringBuffer &str) const return str; } +//MORE: This could be commoned up with the toStr() method by using a visitor pattern +void CRuntimeStatisticCollection::exportToSpan(ISpan * span, StringBuffer & prefix) const +{ + unsigned lenPrefix = prefix.length(); + ForEachItem(iStat) + { + StatisticKind kind = getKind(iStat); + StatisticKind serialKind = querySerializedKind(kind); + if (kind != serialKind) + continue; // ignore - we will roll this one into the corresponding serialized value's output + unsigned __int64 value = values[iStat].get(); + StatisticKind rawKind = queryRawKind(kind); + if (kind != rawKind) + { + // roll raw values into the corresponding serialized value, if present... + unsigned __int64 rawValue = getStatisticValue(rawKind); + if (rawValue) + value += convertMeasure(rawKind, kind, rawValue); + } + if (value) + { + //Convert timestamp to nanoseconds so it is reported consistently. + if (queryMeasure(serialKind) == SMeasureTimestampUs) + value = normalizeTimestampToNs(value); + + const char * name = queryStatisticName(serialKind); + getSnakeCase(prefix, name); + span->setSpanAttribute(prefix, value); + prefix.setLength(lenPrefix); + } + } + CNestedRuntimeStatisticMap *qn = queryNested(); + if (qn) + qn->exportToSpan(span, prefix); +} + void CRuntimeStatisticCollection::deserialize(MemoryBuffer& in) { unsigned numValid; @@ -3204,6 +3263,14 @@ StringBuffer & CNestedRuntimeStatisticCollection::toStr(StringBuffer &str) const return str.append(" }"); } +void CNestedRuntimeStatisticCollection::exportToSpan(ISpan * span, StringBuffer & prefix) const +{ + unsigned lenPrefix = prefix.length(); + scope.getScopeText(prefix).append("."); + stats->exportToSpan(span, prefix); + prefix.setLength(lenPrefix); +} + StringBuffer & CNestedRuntimeStatisticCollection::toXML(StringBuffer &str) const { str.append(" AggregateUpdatedCallBackFunc; @@ -631,6 +632,9 @@ class jlib_decl CRuntimeStatisticCollection void merge(const CRuntimeStatisticCollection & other, unsigned node = 0); void updateDelta(CRuntimeStatisticCollection & target, const CRuntimeStatisticCollection & source); + // Add the statistics to a span + void exportToSpan(ISpan * span, StringBuffer & prefix) const; + // Print out collected stats to string StringBuffer &toStr(StringBuffer &str) const; // Print out collected stats to string as XML @@ -754,6 +758,7 @@ class CNestedRuntimeStatisticCollection : public CInterface void set(const CNestedRuntimeStatisticCollection & other, unsigned node); void merge(const CNestedRuntimeStatisticCollection & other, unsigned node); void recordStatistics(IStatisticGatherer & target, bool clear) const; + void exportToSpan(ISpan * span, StringBuffer & prefix) const; StringBuffer & toStr(StringBuffer &str) const; StringBuffer & toXML(StringBuffer &str) const; void updateDelta(CNestedRuntimeStatisticCollection & target, const CNestedRuntimeStatisticCollection & source); @@ -776,6 +781,7 @@ class CNestedRuntimeStatisticMap void merge(const CNestedRuntimeStatisticMap & other, unsigned node); void set(const CNestedRuntimeStatisticMap & other, unsigned node); void recordStatistics(IStatisticGatherer & target, bool clear) const; + void exportToSpan(ISpan * span, StringBuffer & prefix) const; StringBuffer & toStr(StringBuffer &str) const; StringBuffer & toXML(StringBuffer &str) const; void updateDelta(CNestedRuntimeStatisticMap & target, const CNestedRuntimeStatisticMap & source); @@ -879,12 +885,14 @@ extern jlib_decl StringBuffer & formatStatistic(StringBuffer & out, unsigned __i extern jlib_decl StringBuffer & formatStatistic(StringBuffer & out, unsigned __int64 value, StatisticKind kind); extern jlib_decl void formatTimeStampAsLocalTime(StringBuffer & out, unsigned __int64 value); extern jlib_decl stat_type readStatisticValue(const char * cur, const char * * end, StatisticMeasure measure); +extern jlib_decl stat_type normalizeTimestampToNs(stat_type value); extern jlib_decl unsigned __int64 mergeStatisticValue(unsigned __int64 prevValue, unsigned __int64 newValue, StatsMergeAction mergeAction); extern jlib_decl StatisticMeasure queryMeasure(StatisticKind kind); extern jlib_decl const char * queryStatisticName(StatisticKind kind); extern jlib_decl void queryLongStatisticName(StringBuffer & out, StatisticKind kind); +extern jlib_decl const char * queryStatisticDescription(StatisticKind kind); extern jlib_decl const char * queryTreeTag(StatisticKind kind); extern jlib_decl const char * queryCreatorTypeName(StatisticCreatorType sct); extern jlib_decl const char * queryScopeTypeName(StatisticScopeType sst); diff --git a/system/jlib/jstring.cpp b/system/jlib/jstring.cpp index 4048c887d4f..5a1be75faf6 100644 --- a/system/jlib/jstring.cpp +++ b/system/jlib/jstring.cpp @@ -2843,6 +2843,29 @@ void processOptionString(const char * options, optionCallback callback) } } +void getSnakeCase(StringBuffer & out, const char * camelValue) +{ + if (isEmptyString(camelValue)) + return; + + out.append((char)tolower(*camelValue++)); + + for (;;) + { + byte next = *camelValue++; + if (!next) + break; + + if (isupper(next)) + { + out.append('_'); + out.append((char)tolower(next)); + } + else + out.append((char)next); + } +} + /** * stristr - Case insensitive strstr() * @haystack: Where we will search for our @needle diff --git a/system/jlib/jstring.hpp b/system/jlib/jstring.hpp index 162679a9e13..5a153555041 100644 --- a/system/jlib/jstring.hpp +++ b/system/jlib/jstring.hpp @@ -640,5 +640,6 @@ using optionCallback = std::function 0) { - StringBuffer encoded; - encodeJSON(encoded, val.c_str()); - out.appendf("\"%s\": \"%s\"", kv.first.c_str(), encoded.str()); + switch (value.index()) + { + case opentelemetry::sdk::common::kTypeBool: + case opentelemetry::sdk::common::kTypeInt: + case opentelemetry::sdk::common::kTypeUInt: + case opentelemetry::sdk::common::kTypeInt64: + case opentelemetry::sdk::common::kTypeDouble: + case opentelemetry::sdk::common::kTypeUInt64: + out.appendf("\"%s\": %s", kv.first.c_str(), val.c_str()); + break; + case opentelemetry::sdk::common::kTypeString: + { + StringBuffer encoded; + encodeJSON(encoded, val.c_str()); + out.appendf("\"%s\": \"%s\"", kv.first.c_str(), encoded.str()); + break; + } + case opentelemetry::sdk::common::kTypeSpanBool: + case opentelemetry::sdk::common::kTypeSpanInt: + case opentelemetry::sdk::common::kTypeSpanUInt: + case opentelemetry::sdk::common::kTypeSpanInt64: + case opentelemetry::sdk::common::kTypeSpanDouble: + case opentelemetry::sdk::common::kTypeSpanString: + case opentelemetry::sdk::common::kTypeSpanUInt64: + case opentelemetry::sdk::common::kTypeSpanByte: + //MORE: These should be output as lists of values. + //Implement when we have a need. + default: + UNIMPLEMENTED; + } } } out.append(" }"); @@ -558,6 +586,12 @@ class CSpan : public CInterfaceOf span->SetAttribute(key, val); } + void setSpanAttribute(const char *name, __uint64 value) override + { + if (span && !isEmptyString(name)) + span->SetAttribute(name, value); + } + void addSpanEvent(const char * eventName, IProperties * attributes) override { if (span && !isEmptyString(eventName)) @@ -696,6 +730,11 @@ class CSpan : public CInterfaceOf } } + virtual bool isRecording() const override + { + return span ? span->IsRecording() : false; + } + protected: CSpan(const char * spanName) { @@ -790,6 +829,7 @@ class CNullSpan : public CInterfaceOf CNullSpan() = default; virtual void setSpanAttribute(const char * key, const char * val) override {} + virtual void setSpanAttribute(const char *name, __uint64 value) override {} virtual void setSpanAttributes(const IProperties * attributes) override {} virtual void addSpanEvent(const char * eventName) override {} virtual void addSpanEvent(const char * eventName, IProperties * attributes) override {}; @@ -798,6 +838,7 @@ class CNullSpan : public CInterfaceOf virtual void toString(StringBuffer & out) const override {} virtual void getLogPrefix(StringBuffer & out) const override {} + virtual bool isRecording() const { return false; } virtual const char* queryGlobalId() const override { return nullptr; } virtual const char* queryCallerId() const override { return nullptr; } diff --git a/system/jlib/jtrace.hpp b/system/jlib/jtrace.hpp index 994ba3eed60..c2cdeb2bae1 100644 --- a/system/jlib/jtrace.hpp +++ b/system/jlib/jtrace.hpp @@ -58,6 +58,7 @@ BITMASK_ENUM(SpanFlags); interface ISpan : extends IInterface { virtual void setSpanAttribute(const char * key, const char * val) = 0; + virtual void setSpanAttribute(const char *name, __uint64 value) = 0; virtual void setSpanAttributes(const IProperties * attributes) = 0; virtual void addSpanEvent(const char * eventName) = 0; virtual void addSpanEvent(const char * eventName, IProperties * attributes) = 0; @@ -65,6 +66,7 @@ interface ISpan : extends IInterface virtual void getClientHeaders(IProperties * clientHeaders) const = 0; virtual void toString(StringBuffer & out) const = 0; virtual void getLogPrefix(StringBuffer & out) const = 0; + virtual bool isRecording() const = 0; // Is it worth adding any events/attributes to this span? virtual ISpan * createClientSpan(const char * name) = 0; virtual ISpan * createInternalSpan(const char * name) = 0; diff --git a/system/metrics/sinks/log/logSink.cpp b/system/metrics/sinks/log/logSink.cpp index 30c51eaf922..af6b12a78b6 100644 --- a/system/metrics/sinks/log/logSink.cpp +++ b/system/metrics/sinks/log/logSink.cpp @@ -47,57 +47,85 @@ void LogMetricSink::doCollection() void LogMetricSink::writeLogEntry(const std::shared_ptr &pMetric) { - std::string name = pMetric->queryName(); + const std::string & name = pMetric->queryName(); + __uint64 metricValue = pMetric->queryValue(); + if (ignoreZeroMetrics && (metricValue == 0)) + return; + const auto & metaData = pMetric->queryMetaData(); + const std::string * searchKey = &name; + //Create a unique id from a combination of the name and the labels so that we can check whether this metric instance has changed since the last time it was logged + std::string uid; + if (!metaData.empty()) + { + uid.append(name); + for (auto &metaDataIt: metaData) + uid.append("/").append(metaDataIt.value.c_str()); + searchKey = &uid; + } + + auto match = alreadySeen.find(*searchKey); + const auto notFound = alreadySeen.end(); + //If the values haven't changed then avoid logging an update to the logs + if ((match != notFound) && (match->second == metricValue)) + return; + + StringBuffer output; + + StringBuffer labels; for (auto &metaDataIt: metaData) { - name.append(".").append(metaDataIt.value); + if (labels.length() > 0) + labels.append(","); + labels.appendf("{ \"name\":\"%s\", \"value\": \"%s\" }", metaDataIt.key.c_str(), metaDataIt.value.c_str()); } + if (labels.length()) + output.appendf("\"labels\": [%s], ", labels.str()); + const char *unitsStr = pManager->queryUnitsString(pMetric->queryUnits()); if (pMetric->queryMetricType() != METRICS_HISTOGRAM) { - __uint64 metricValue = pMetric->queryValue(); - if (!ignoreZeroMetrics || metricValue) - { - if (!isEmptyString(unitsStr)) - { - name.append(".").append(unitsStr); - } - LOG(MCmonitorMetric, "name=%s,value=%" I64F "d", name.c_str(), metricValue); - } + output.appendf("\"value\": %" I64F "d", metricValue); } else { + output.append("\"sum\": ").append(metricValue).append(", "); + + StringBuffer valueText; std::vector<__uint64> values = pMetric->queryHistogramValues(); - std::vector<__uint64> limits = pMetric->queryHistogramBucketLimits(); - size_t countBucketValues = values.size(); + __uint64 cumulative = 0; + for (size_t i=0; i < values.size(); ++i) + { + if (valueText.length() > 0) + valueText.append(", "); + cumulative += values[i]; + valueText.appendf("%" I64F "d", values[i]); + } + output.append("\"count\": ").append(cumulative).append(", "); - // If not ignoring or measurements exist, output the log entries - __uint64 sum = pMetric->queryValue(); - if (!ignoreZeroMetrics || sum) + output.append("\"counts\": [").append(valueText).append("]"); + + //Only output the limits the first time this metric is traced + if (match == notFound) { - __uint64 cumulative = 0; - for (size_t i=0; i < countBucketValues - 1; ++i) + std::vector<__uint64> limits = pMetric->queryHistogramBucketLimits(); + StringBuffer limitText; + for (size_t i=0; i < limits.size(); ++i) { - cumulative += values[i]; - if (!ignoreZeroMetrics || values[i]) - { - LOG(MCmonitorMetric, "name=%s, bucket le %" I64F "d=%" I64F "d", name.c_str(), limits[i], cumulative); - } + if (limitText.length() > 0) + limitText.append(", "); + limitText.appendf("%" I64F "d", limits[i]); } - - // The inf bucket count is the last element in the array of values returned. - // Add it to the cumulative count and print the value - cumulative += values[countBucketValues - 1]; - LOG(MCmonitorMetric, "name=%s, bucket inf=%" I64F "d", name.c_str(), cumulative); - - // sum - total of all observations - LOG(MCmonitorMetric, "name=%s, sum=%" I64F "d", name.c_str(), sum); - - // count - total of all bucket counts (same as inf) - LOG(MCmonitorMetric, "name=%s, count=%" I64F "d", name.c_str(), cumulative); + output.append(", \"limits\": [").append(limitText).append("]"); } } + + LOG(MCmonitorMetric, "{ \"type\": \"metric\", \"name\": \"%s%s%s\", %s }", name.c_str(), unitsStr ? "." : "", unitsStr ? unitsStr : "", output.str()); + + if (match == notFound) + alreadySeen.insert({ *searchKey, metricValue}); + else + match->second = metricValue; } diff --git a/system/metrics/sinks/log/logSink.hpp b/system/metrics/sinks/log/logSink.hpp index 1e19e603512..55d885ec0ca 100644 --- a/system/metrics/sinks/log/logSink.hpp +++ b/system/metrics/sinks/log/logSink.hpp @@ -14,6 +14,7 @@ #pragma once +#include #include "jmetrics.hpp" #include "jptree.hpp" #include "jstring.hpp" @@ -41,5 +42,6 @@ class LOGSINK_API LogMetricSink : public PeriodicMetricSink void writeHistogramLogEntry(const std::shared_ptr &pMetric); protected: + std::unordered_map alreadySeen; bool ignoreZeroMetrics; }; diff --git a/thorlcr/activities/csvread/thcsvrslave.cpp b/thorlcr/activities/csvread/thcsvrslave.cpp index b2c34630652..c682dc8230d 100644 --- a/thorlcr/activities/csvread/thcsvrslave.cpp +++ b/thorlcr/activities/csvread/thcsvrslave.cpp @@ -249,17 +249,27 @@ class CCsvReadSlaveActivity : public CDiskReadSlaveActivityBase if (sentHeaderLines->testSet(subFile)) return; - /* Before we can send state of headerLines of subfiles, - * need to have received any updates from previous worker. - * The previous worker will have sent updates as it progressed, - * and info. re. all files it is not dealing with (and all remaining if stopped) */ - while (true) + /* NB: we are here because this worker has consumed all remaining header lines for this subfile. + * It must now inform the next worker so it can make progress on this subfile asap. + * + * The other subfile header line info. will be communicated as this worker makes progress + * through the subfiles, or when it stops (see sendRemainingHeaderLines()). + */ + + // JCSMORE: only left in for testing, should be removed (see HPCC-31160) + if (getOptBool("csvWaitAllSubs")) { - unsigned which = gotHeaderLines->scan(0, false); - if (which == subFiles) // all received - break; - getHeaderLines(which); + // This causes this worker to block until the previous worker has processed the headerlines for all subfiles. + // In effect, causing workers to process the csv read sequentially, massively slowing down throughput. + while (true) + { + unsigned which = gotHeaderLines->scan(0, false); + if (which >= subFiles) // all received + break; + getHeaderLines(which); + } } + bool someLeft=false; unsigned hL=0; for (; hL, { exception.setown(e); EXCLOG(e, "CRowProcessor"); + owner.broadcaster->cancel(e); } } } *rowProcessor;