From e6e1bac33cdf316d0302e1cc1dbd2ee2cf9af391 Mon Sep 17 00:00:00 2001 From: Richard Chapman Date: Wed, 7 Aug 2024 14:39:19 +0100 Subject: [PATCH 01/33] HPCC-32400 unittests crashes on mac arm64 if tests are dynamically loaded Signed-off-by: Richard Chapman --- testing/unittests/unittests.cpp | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/testing/unittests/unittests.cpp b/testing/unittests/unittests.cpp index 7b0a2512b6d..c11154dfe2c 100644 --- a/testing/unittests/unittests.cpp +++ b/testing/unittests/unittests.cpp @@ -56,6 +56,7 @@ void usage() " -e --exact Match subsequent test names exactly\n" " -h --help Display this help text\n" " -l --list List matching tests but do not execute them\n" + " -u --unload Unload dynamically-loaded dlls before termination (may crash on some systems)\n" " -x --exclude Exclude subsequent test names\n" "\n"); } @@ -95,7 +96,7 @@ LoadedObject *loadDll(const char *thisDll) return NULL; } -void loadDlls(IArray &objects, const char * libDirectory) +void loadDlls(IArray &objects, const char * libDirectory, bool optUnloadDlls) { const char * mask = "*" SharedObjectExtension; Owned libDir = createIFile(libDirectory); @@ -108,7 +109,7 @@ void loadDlls(IArray &objects, const char * libDirectory) if (!strstr(thisDll, "py3embed")) // ... best to load neither... { LoadedObject *loaded = loadDll(thisDll); - if (loaded) + if (loaded && optUnloadDlls) objects.append(*loaded); } } @@ -133,6 +134,7 @@ int main(int argc, const char *argv[]) bool verbose = false; bool list = false; bool useDefaultLocations = true; + bool unloadDlls = false; //NB: not actually used for now, but required initialization for anything that may call getGlobalConfig*() or getComponentConfig*() Owned globals = loadConfiguration(defaultYaml, argv, "unittests", nullptr, nullptr, nullptr, nullptr, false); @@ -152,6 +154,8 @@ int main(int argc, const char *argv[]) includeAll = true; else if (streq(arg, "-l") || streq(arg, "--list")) list = true; + else if (streq(arg, "-u") || streq(arg, "-unload")) + unloadDlls = true; else if (streq(arg, "-d") || streq(arg, "--load")) { useDefaultLocations = false; @@ -220,7 +224,7 @@ int main(int argc, const char *argv[]) DBGLOG("Specified library location %s not found", location); break; case fileBool::foundYes: - loadDlls(objects, location); + loadDlls(objects, location, unloadDlls); break; case fileBool::foundNo: LoadedObject *loaded = loadDll(location); From 2755147765516a5708825ed299a2f763dd41cf0c Mon Sep 17 00:00:00 2001 From: Richard Chapman Date: Wed, 7 Aug 2024 15:09:09 +0100 Subject: [PATCH 02/33] HPCC-32401 IKeyManagerTests are slow (and should be renamed) Signed-off-by: Richard Chapman --- system/jhtree/jhtree.cpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/system/jhtree/jhtree.cpp b/system/jhtree/jhtree.cpp index 024cbd59b4a..86279960c2d 100644 --- a/system/jhtree/jhtree.cpp +++ b/system/jhtree/jhtree.cpp @@ -3434,9 +3434,9 @@ class TestIndexWriteArg : public CThorIndexWriteArg }; -class IKeyManagerTest : public CppUnit::TestFixture +class IKeyManagerSlowTest : public CppUnit::TestFixture { - CPPUNIT_TEST_SUITE( IKeyManagerTest ); + CPPUNIT_TEST_SUITE( IKeyManagerSlowTest ); CPPUNIT_TEST(testStepping); CPPUNIT_TEST(testKeys); CPPUNIT_TEST_SUITE_END(); @@ -3842,7 +3842,7 @@ class IKeyManagerTest : public CppUnit::TestFixture } }; -CPPUNIT_TEST_SUITE_REGISTRATION( IKeyManagerTest ); -CPPUNIT_TEST_SUITE_NAMED_REGISTRATION( IKeyManagerTest, "IKeyManagerTest" ); +CPPUNIT_TEST_SUITE_REGISTRATION( IKeyManagerSlowTest ); +CPPUNIT_TEST_SUITE_NAMED_REGISTRATION( IKeyManagerSlowTest, "IKeyManagerSlowTest" ); #endif From 2314af35d80aafba385289993d0263489c4b18be Mon Sep 17 00:00:00 2001 From: Rodrigo Pastrana Date: Wed, 7 Aug 2024 21:46:16 -0400 Subject: [PATCH 03/33] HPCC-32411 Update ZAP log column mode name - Updates Zap report field name to coincide with eclwatch Signed-off-by: Rodrigo Pastrana --- esp/services/ws_workunits/ws_workunitsHelpers.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/esp/services/ws_workunits/ws_workunitsHelpers.hpp b/esp/services/ws_workunits/ws_workunitsHelpers.hpp index ae040b8aae8..d8b68ce724b 100644 --- a/esp/services/ws_workunits/ws_workunitsHelpers.hpp +++ b/esp/services/ws_workunits/ws_workunitsHelpers.hpp @@ -219,7 +219,7 @@ struct WUComponentLogOptions populateTimeRange(start, end, bufferSecs); //int 0 ==MIN, 1==DEFAULT, 2==ALL, 3==CUSTOM - int colMode = zapHttpRequest->getParameterInt("LogFilter_ColumnMode", -1); + int colMode = zapHttpRequest->getParameterInt("LogFilter_SelectColumnMode", -1); if (colMode != -1) { StringArray customFields; //comma delimited list of available columns, only if ColumnMode==3 From ee61ccba9cdba7e7918a3dc14808b7de5eaa5ce3 Mon Sep 17 00:00:00 2001 From: Gordon Smith Date: Fri, 2 Aug 2024 16:32:52 +0100 Subject: [PATCH 04/33] HPCC-32417 Add visibility changed event to DockPanel Fix linter issue (introduced by wasm version bump) Bump wasm package (avoid additional warnings) Signed-off-by: Gordon Smith --- esp/src/eslint/index.js | 3 ++ esp/src/package-lock.json | 25 +++++------- esp/src/package.json | 2 +- esp/src/src-react/components/ECLArchive.tsx | 2 +- esp/src/src-react/components/Frame.tsx | 2 +- esp/src/src-react/components/Metrics.tsx | 2 +- esp/src/src-react/layouts/DockPanel.tsx | 45 ++++++++++++++++++--- 7 files changed, 57 insertions(+), 24 deletions(-) diff --git a/esp/src/eslint/index.js b/esp/src/eslint/index.js index 3a433ad8a60..c3bdb1cd045 100644 --- a/esp/src/eslint/index.js +++ b/esp/src/eslint/index.js @@ -2,6 +2,9 @@ module.exports = { rules: { "no-src-react": { + meta: { + fixable: "code" + }, create: function (context) { return { ImportDeclaration(node) { diff --git a/esp/src/package-lock.json b/esp/src/package-lock.json index e15b7782995..ecb1390c775 100644 --- a/esp/src/package-lock.json +++ b/esp/src/package-lock.json @@ -31,7 +31,7 @@ "@hpcc-js/timeline": "2.53.0", "@hpcc-js/tree": "2.41.0", "@hpcc-js/util": "2.52.0", - "@hpcc-js/wasm": "2.18.1", + "@hpcc-js/wasm": "2.18.2", "@kubernetes/client-node": "0.20.0", "clipboard": "2.0.11", "d3-dsv": "3.0.1", @@ -89,12 +89,13 @@ "dev": true }, "node_modules/@75lb/deep-merge": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/@75lb/deep-merge/-/deep-merge-1.1.1.tgz", - "integrity": "sha512-xvgv6pkMGBA6GwdyJbNAnDmfAIR/DfWhrj9jgWh3TY7gRm3KO46x/GPjRg6wJ0nOepwqrNxFfojebh0Df4h4Tw==", + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/@75lb/deep-merge/-/deep-merge-1.1.2.tgz", + "integrity": "sha512-08K9ou5VNbheZFxM5tDWoqjA3ImC50DiuuJ2tj1yEPRfkp8lLLg6XAaJ4On+a0yAXor/8ay5gHnAIshRM44Kpw==", "dev": true, + "license": "MIT", "dependencies": { - "lodash.assignwith": "^4.2.0", + "lodash": "^4.17.21", "typical": "^7.1.1" }, "engines": { @@ -2334,9 +2335,9 @@ "license": "0BSD" }, "node_modules/@hpcc-js/wasm": { - "version": "2.18.1", - "resolved": "https://registry.npmjs.org/@hpcc-js/wasm/-/wasm-2.18.1.tgz", - "integrity": "sha512-fT8NCOTaF0NDnT+ZwWpV2VQ6ywFEqw+fG87GSPNQemEmg7FFqUaKRQOW9MBICrkZcXaJBb7VHo1t5UF6bi/JgQ==", + "version": "2.18.2", + "resolved": "https://registry.npmjs.org/@hpcc-js/wasm/-/wasm-2.18.2.tgz", + "integrity": "sha512-9FIpuXvIsIY3UbUd/HZPPiaZe6IFIuA6k5j9Lh54QcINP1s9hbMr/na0xjt+qRPXlwZdrOz3zQJBzHEEQDKnCw==", "license": "Apache-2.0", "dependencies": { "yargs": "17.7.2" @@ -8065,12 +8066,6 @@ "resolved": "https://registry.npmjs.org/lodash-es/-/lodash-es-4.17.21.tgz", "integrity": "sha512-mKnC+QJ9pWVzv+C4/U3rRsHapFfHvQFoFB92e52xeyGMcX6/OlIl78je1u8vePzYZSkkogMPJ2yjxxsb89cxyw==" }, - "node_modules/lodash.assignwith": { - "version": "4.2.0", - "resolved": "https://registry.npmjs.org/lodash.assignwith/-/lodash.assignwith-4.2.0.tgz", - "integrity": "sha512-ZznplvbvtjK2gMvnQ1BR/zqPFZmS6jbK4p+6Up4xcRYA7yMIwxHCfbTcrYxXKzzqLsQ05eJPVznEW3tuwV7k1g==", - "dev": true - }, "node_modules/lodash.camelcase": { "version": "4.3.0", "resolved": "https://registry.npmjs.org/lodash.camelcase/-/lodash.camelcase-4.3.0.tgz", @@ -12322,4 +12317,4 @@ } } } -} \ No newline at end of file +} diff --git a/esp/src/package.json b/esp/src/package.json index c1ec1651488..e9d7224d6f4 100644 --- a/esp/src/package.json +++ b/esp/src/package.json @@ -57,7 +57,7 @@ "@hpcc-js/timeline": "2.53.0", "@hpcc-js/tree": "2.41.0", "@hpcc-js/util": "2.52.0", - "@hpcc-js/wasm": "2.18.1", + "@hpcc-js/wasm": "2.18.2", "@kubernetes/client-node": "0.20.0", "clipboard": "2.0.11", "d3-dsv": "3.0.1", diff --git a/esp/src/src-react/components/ECLArchive.tsx b/esp/src/src-react/components/ECLArchive.tsx index 8d63d3ae8f5..a84ceaae3ac 100644 --- a/esp/src/src-react/components/ECLArchive.tsx +++ b/esp/src/src-react/components/ECLArchive.tsx @@ -111,7 +111,7 @@ export const ECLArchive: React.FunctionComponent = ({ return } main={ - + { // Only render after archive is loaded (to ensure it "defaults to open") --- archive?.modAttrs.length && diff --git a/esp/src/src-react/components/Frame.tsx b/esp/src/src-react/components/Frame.tsx index 9821ffdfd43..76b39bd3a5d 100644 --- a/esp/src/src-react/components/Frame.tsx +++ b/esp/src/src-react/components/Frame.tsx @@ -88,7 +88,7 @@ export const Frame: React.FunctionComponent = () => { router.resolve(hashHistory.location).then(setBody); - userKeyValStore().get("user_cookie_consent") + userKeyValStore().get(USER_COOKIE_CONSENT) .then((resp) => { setShowCookieConsent(resp === "1"); }) diff --git a/esp/src/src-react/components/Metrics.tsx b/esp/src/src-react/components/Metrics.tsx index f0df3a94f6a..40413d012c7 100644 --- a/esp/src/src-react/components/Metrics.tsx +++ b/esp/src/src-react/components/Metrics.tsx @@ -634,7 +634,7 @@ export const Metrics: React.FunctionComponent = ({ } main={ - + diff --git a/esp/src/src-react/layouts/DockPanel.tsx b/esp/src/src-react/layouts/DockPanel.tsx index 8ebea0e1c74..d359e0397b9 100644 --- a/esp/src/src-react/layouts/DockPanel.tsx +++ b/esp/src/src-react/layouts/DockPanel.tsx @@ -3,8 +3,8 @@ import * as ReactDOM from "react-dom"; import { Theme, ThemeProvider } from "@fluentui/react"; import { useConst } from "@fluentui/react-hooks"; import { FluentProvider, Theme as ThemeV9 } from "@fluentui/react-components"; -import { HTMLWidget, Widget } from "@hpcc-js/common"; -import { DockPanel as HPCCDockPanel, IClosable } from "@hpcc-js/phosphor"; +import { HTMLWidget, Widget, Utility } from "@hpcc-js/common"; +import { DockPanel as HPCCDockPanel, IClosable, WidgetAdapter } from "@hpcc-js/phosphor"; import { compare2 } from "@hpcc-js/util"; import { lightTheme, lightThemeV9 } from "../themes"; import { useUserTheme } from "../hooks/theme"; @@ -96,6 +96,7 @@ export class ResetableDockPanel extends HPCCDockPanel { protected _origLayout: DockPanelLayout | undefined; protected _lastLayout: DockPanelLayout | undefined; + protected _visibility: { [id: string]: boolean }; resetLayout() { if (this._origLayout) { @@ -118,8 +119,19 @@ export class ResetableDockPanel extends HPCCDockPanel { return formatLayout(this.layout()) ?? this._lastLayout ?? this._origLayout; } + getVisibility() { + return this._visibility; + } + render(callback?: (w: Widget) => void) { - const retVal = super.render(); + const retVal = this._visibility !== undefined ? super.render() : super.render(() => { + if (this._visibility === undefined) { + this._visibility = {}; + this.widgetAdapters().forEach(wa => { + this._visibility[wa.widget.id()] = wa.isVisible; + }); + } + }); if (this._origLayout === undefined) { this._origLayout = formatLayout(this.layout()); } @@ -130,9 +142,27 @@ export class ResetableDockPanel extends HPCCDockPanel { } // Events --- + childActivation(w: Widget, wa: WidgetAdapter) { + } + + childVisibility(w: Widget, visible: boolean, wa: WidgetAdapter) { + if (this._visibility && this._visibility[w.id()] !== visible) { + this._visibility[w.id()] = visible; + this._lazyVisibilityChanged(); + } + } + layoutChanged() { this._lastLayout = this.getLayout(); } + + // Exposed Events --- + private _lazyVisibilityChanged = Utility.debounce(async () => { + this.visibilityChanged(this._visibility); + }, 60); + + visibilityChanged(visibility: { [id: string]: boolean }) { + } } interface DockPanelItemProps { @@ -154,14 +184,16 @@ export const DockPanelItem: React.FunctionComponent = ({ interface DockPanelProps { layout?: object; hideSingleTabs?: boolean; - onDockPanelCreate?: (dockpanel: ResetableDockPanel) => void; + onCreate?: (dockpanel: ResetableDockPanel) => void; + onVisibilityChanged?: (visibility: { [id: string]: boolean }) => void; children?: React.ReactElement | React.ReactElement[]; } export const DockPanel: React.FunctionComponent = ({ layout, hideSingleTabs, - onDockPanelCreate, + onCreate: onDockPanelCreate, + onVisibilityChanged: onDockPanelVisibilityChanged, children }) => { const items = React.useMemo(() => { @@ -179,6 +211,9 @@ export const DockPanel: React.FunctionComponent = ({ onDockPanelCreate(retVal); }, 0); } + if (onDockPanelVisibilityChanged) { + retVal.on("visibilityChanged", visibility => onDockPanelVisibilityChanged(visibility), true); + } return retVal; }); From 2311d12cdde170bce5aea19e76076359367c88ac Mon Sep 17 00:00:00 2001 From: Gordon Smith Date: Fri, 2 Aug 2024 16:32:52 +0100 Subject: [PATCH 05/33] HPCC-32418 Make toolbar optional for all source editor components Signed-off-by: Gordon Smith --- esp/src/src-react/components/SourceEditor.tsx | 32 +++++++++++++------ 1 file changed, 22 insertions(+), 10 deletions(-) diff --git a/esp/src/src-react/components/SourceEditor.tsx b/esp/src/src-react/components/SourceEditor.tsx index 5e070c6f419..5164848df25 100644 --- a/esp/src/src-react/components/SourceEditor.tsx +++ b/esp/src/src-react/components/SourceEditor.tsx @@ -147,38 +147,44 @@ export const SourceEditor: React.FunctionComponent = ({ interface TextSourceEditorProps { text: string; readonly?: boolean; + toolbar?: boolean; } export const TextSourceEditor: React.FunctionComponent = ({ text = "", - readonly = false + readonly, + toolbar }) => { - return ; + return ; }; interface XMLSourceEditorProps { text: string; readonly?: boolean; + toolbar?: boolean; } export const XMLSourceEditor: React.FunctionComponent = ({ text = "", - readonly = false + readonly, + toolbar }) => { - return ; + return ; }; interface JSONSourceEditorProps { json?: object; readonly?: boolean; + toolbar?: boolean; onChange?: (obj: object) => void; } export const JSONSourceEditor: React.FunctionComponent = ({ json, - readonly = false, + readonly, + toolbar, onChange = (obj: object) => { } }) => { @@ -197,7 +203,7 @@ export const JSONSourceEditor: React.FunctionComponent = } }, [onChange]); - return ; + return ; }; export interface WUXMLSourceEditorProps { @@ -215,10 +221,12 @@ export const WUXMLSourceEditor: React.FunctionComponent export interface WUResourceEditorProps { src: string; + toolbar?: boolean; } export const WUResourceEditor: React.FunctionComponent = ({ - src + src, + toolbar }) => { const [text, setText] = React.useState(""); @@ -231,7 +239,7 @@ export const WUResourceEditor: React.FunctionComponent = }); }, [src]); - return ; + return ; }; interface ECLSourceEditorProps { @@ -266,6 +274,7 @@ interface FetchEditor { url: string; wuid?: string; readonly?: boolean; + toolbar?: boolean; mode?: "ecl" | "xml" | "text"; } @@ -273,6 +282,7 @@ export const FetchEditor: React.FunctionComponent = ({ url, wuid, readonly = true, + toolbar, mode = "text" }) => { @@ -293,11 +303,12 @@ export const FetchEditor: React.FunctionComponent = ({ } }, [url, wuid]); - return ; + return ; }; interface SQLSourceEditorProps { sql: string; + readonly?: boolean; toolbar?: boolean; onSqlChange?: (sql: string) => void; onFetchHints?: (cm: any, option: any) => Promise; @@ -306,11 +317,12 @@ interface SQLSourceEditorProps { export const SQLSourceEditor: React.FunctionComponent = ({ sql, + readonly, toolbar, onSqlChange, onFetchHints, onSubmit }) => { - return ; + return ; }; From 8501cd0e6041a81d1c3233935b6e399264c04263 Mon Sep 17 00:00:00 2001 From: Gordon Smith Date: Thu, 8 Aug 2024 17:20:19 +0100 Subject: [PATCH 06/33] Split off 9.2.112 Signed-off-by: Gordon Smith --- helm/hpcc/Chart.yaml | 4 ++-- helm/hpcc/templates/_helpers.tpl | 2 +- version.cmake | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/helm/hpcc/Chart.yaml b/helm/hpcc/Chart.yaml index 54a76856aaf..541ec0b4765 100644 --- a/helm/hpcc/Chart.yaml +++ b/helm/hpcc/Chart.yaml @@ -6,9 +6,9 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. -version: 9.2.111-closedown0 +version: 9.2.113-closedown0 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. -appVersion: 9.2.111-closedown0 +appVersion: 9.2.113-closedown0 diff --git a/helm/hpcc/templates/_helpers.tpl b/helm/hpcc/templates/_helpers.tpl index 313cafb9530..c35fa1d92b0 100644 --- a/helm/hpcc/templates/_helpers.tpl +++ b/helm/hpcc/templates/_helpers.tpl @@ -1361,7 +1361,7 @@ Pass in dict with .root, .visibility defined {{- end -}} {{- define "hpcc.generateHelmVersion" -}} -helmVersion: 9.2.111-closedown0 +helmVersion: 9.2.113-closedown0 {{- end -}} {{/* diff --git a/version.cmake b/version.cmake index 375f81065b7..3ff7c348647 100644 --- a/version.cmake +++ b/version.cmake @@ -5,8 +5,8 @@ set ( HPCC_NAME "Community Edition" ) set ( HPCC_PROJECT "community" ) set ( HPCC_MAJOR 9 ) set ( HPCC_MINOR 2 ) -set ( HPCC_POINT 111 ) +set ( HPCC_POINT 113 ) set ( HPCC_MATURITY "closedown" ) set ( HPCC_SEQUENCE 0 ) -set ( HPCC_TAG_TIMESTAMP "2024-08-02T10:48:12Z" ) +set ( HPCC_TAG_TIMESTAMP "2024-08-08T16:20:19Z" ) ### From 2e827e20c65605a9d26334c44c09f53346ccf516 Mon Sep 17 00:00:00 2001 From: James McMullan Date: Thu, 8 Aug 2024 14:53:07 -0400 Subject: [PATCH 07/33] HPCC4-32419 Jirabot Improve transition flow and input handling - Removed the need for the issue transition map - Various security improvements around input handling Signed-off-by: James McMullan James.McMullan@lexisnexis.com --- .github/workflows/jirabot.yml | 80 ++++++++++++++++++----------------- 1 file changed, 41 insertions(+), 39 deletions(-) diff --git a/.github/workflows/jirabot.yml b/.github/workflows/jirabot.yml index 277f6e7afb7..2f1db9c80f2 100644 --- a/.github/workflows/jirabot.yml +++ b/.github/workflows/jirabot.yml @@ -38,33 +38,61 @@ jobs: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} GHUB_JIRA_USER_MAP: ${{ vars.GHUB_JIRA_USER_MAP }} JIRA_ISSUE_PROPERTY_MAP: ${{ vars.JIRA_ISSUE_PROPERTY_MAP }} - JIRA_ISSUE_TRANSITION_MAP: ${{ vars.JIRA_ISSUE_TRANSITION_MAP }} run: | import os import re import time import sys import json + import subprocess + from email.utils import parseaddr from atlassian.jira import Jira - def updateIssue(jira, issue, prAuthor : str, transitionMap: dict, propertyMap: dict, pull_url: str) -> str: + def sanitizeInput(input: str, inputType: str) -> str: + if inputType.lower() == 'email': + # Return the email address only, returns '' if not valid or found + return parseaddr(input)[1] + else: + return '' + + def updateIssue(jira, issue, prAuthor : str, propertyMap: dict, pull_url: str) -> str: result = '' issueName = issue['key'] issueFields = issue['fields'] - statusName = str(issueFields['status']['name']) - transition = transitionMap.get(statusName, None) + # Need to update user first in case we are starting from Unresourced + if prAuthor: + assignee = issueFields['assignee'] + if assignee is None: + assigneeId = '' + assigneeEmail = '' + else: + assigneeId = assignee['accountId'] + assigneeEmail = assignee["emailAddress"] + + assigneeEmail = sanitizeInput(assigneeEmail, 'email') + + prAuthorId = prAuthor["accountId"] + prAuthorEmail = prAuthor["emailAddress"] + prAuthorEmail = sanitizeInput(prAuthorEmail, 'email') + + if assigneeId is None or assigneeId == '': + jira.assign_issue(issueName, prAuthorId) + result += 'Assigning user: ' + prAuthorEmail + '\n' + elif assigneeId != prAuthorId: + result += 'Changing assignee from: ' + assigneeEmail + ' to: ' + prAuthorEmail + '\n' + jira.assign_issue(issueName, prAuthorId) - if transition == None: - print('Error: Unable to find transition for status: ' + statusName) - elif transition != '': + transitionFlow = ['Merge Pending'] + for desiredStatus in transitionFlow: try: - jira.issue_transition(issueName, transition) - result += 'Workflow Transition: ' + transition + '\n' + transitionId = jira.get_transition_id_to_status_name(issueName, desiredStatus) + jira.set_issue_status_by_transition_id(issueName, transitionId) + result += 'Workflow Transition To: ' + desiredStatus + '\n' except Exception as error: transitions = jira.get_issue_transitions(issueName) - result += 'Error: Transition: "' + transition + '" failed with: "' + str(error) + '" Valid transitions=' + str(transitions) + '\n' + result += 'Error: Transitioning to: "' + desiredStatus + '" failed with: "' + str(error) + '" Valid transitions=' + str(transitions) + '\n' prFieldName = propertyMap.get('pullRequestFieldName', 'customfield_10010') @@ -80,24 +108,6 @@ jobs: elif currentPR is not None and currentPR != pull_url: result += 'Additional PR: ' + pull_url + '\n' - if prAuthor: - assignee = issueFields['assignee'] - if assignee is None: - assigneeId = '' - assigneeEmail = '' - else: - assigneeId = assignee['accountId'] - assigneeEmail = assignee["emailAddress"] - - prAuthorId = prAuthor["accountId"] - prAuthorEmail = prAuthor["emailAddress"] - if assigneeId is None or assigneeId == '': - jira.assign_issue(issueName, prAuthorId) - result += 'Assigning user: ' + prAuthorEmail + '\n' - elif assigneeId != prAuthorId: - result += 'Changing assignee from: ' + assigneeEmail + ' to: ' + prAuthorEmail + '\n' - jira.assign_issue(issueName, prAuthorId) - return result jirabot_user = os.environ['JIRABOT_USERNAME'] @@ -110,7 +120,6 @@ jobs: github_token = os.environ['GITHUB_TOKEN'] comments_url = os.environ['COMMENTS_URL'] - print("%s %s %s" % (title, prAuthor, comments_url)) result = '' issuem = re.search("(HPCC|HH|IDE|EPE|ML|HPCC4J|JAPI)-[0-9]+", title) if issuem: @@ -131,7 +140,7 @@ jobs: if userSearchResults and len(userSearchResults) > 0: jiraUser = userSearchResults[0] else: - print('Error: Unable to find Jira user: ' + prAuthor + ' continuing without assigning') + print('Error: Unable to map GitHub user to Jira user, continuing without assigning') if not jira.issue_exists(issue_name): sys.exit('Error: Unable to find Jira issue: ' + issue_name) @@ -140,17 +149,12 @@ jobs: result = 'Jirabot Action Result:\n' - transitionMap = json.loads(os.environ['JIRA_ISSUE_TRANSITION_MAP']) - if not isinstance(transitionMap, dict): - print('Error: JIRA_ISSUE_TRANSITION_MAP is not a valid JSON object, ignoring.') - transitionMap = {} - jiraIssuePropertyMap = json.loads(os.environ['JIRA_ISSUE_PROPERTY_MAP']) if not isinstance(jiraIssuePropertyMap, dict): print('Error: JIRA_ISSUE_PROPERTY_MAP is not a valid JSON object, ignoring.') jiraIssuePropertyMap = {} - result += updateIssue(jira, issue, jiraUser, transitionMap, jiraIssuePropertyMap, pull_url) + result += updateIssue(jira, issue, jiraUser, jiraIssuePropertyMap, pull_url) jira.issue_add_comment(issue_name, result) result = 'Jira Issue: ' + jira_url + '/browse/' + issue_name + '\n\n' + result @@ -158,9 +162,7 @@ jobs: # Escape the result for JSON result = json.dumps(result) - curlCommand = 'curl -X POST %s -H "Content-Type: application/json" -H "Authorization: token %s" --data \'{ "body": %s }\'' % ( comments_url, github_token, result ) - print(curlCommand) - os.system(curlCommand) + subprocess.run(['curl', '-X', 'POST', comments_url, '-H', 'Content-Type: application/json', '-H', f'Authorization: token {github_token}', '--data', f'{{ "body": {result} }}'], check=True) else: print('Unable to find Jira issue name in title') From a2049d5f6b8cf1ce6c0198e0a055101668c30426 Mon Sep 17 00:00:00 2001 From: Ken Rowland Date: Wed, 14 Aug 2024 15:22:33 -0400 Subject: [PATCH 08/33] HPCC-32436 LDAP should not fall back to an insecure connection Removed fall back to an insecure connection when configured for secure connections and there is a failure during init Signed-Off-By Kenneth Rowland kenneth.rowland@lexisnexisrisk.com --- system/security/LdapSecurity/ldaputils.cpp | 5 ----- 1 file changed, 5 deletions(-) diff --git a/system/security/LdapSecurity/ldaputils.cpp b/system/security/LdapSecurity/ldaputils.cpp index fb68659f0e7..f8b2a6e4d0d 100644 --- a/system/security/LdapSecurity/ldaputils.cpp +++ b/system/security/LdapSecurity/ldaputils.cpp @@ -296,11 +296,6 @@ int LdapUtils::getServerInfo(const char* ldapserver, const char* userDN, const c if (nullptr == ld) { ld = ldapInitAndSimpleBind(ldapserver, userDN, pwd, ldapprotocol, ldapport, cipherSuite, timeout, &err); - if(nullptr == ld && strieq(ldapprotocol,"ldaps")) - { - //if that failed, and was for ldaps, see if we can do anonymous bind using ldap/389 - ld = ldapInitAndSimpleBind(ldapserver, nullptr, nullptr, "ldap", 389, cipherSuite, timeout, &err); - } // for new versions of openldap, version 2.2.* if(nullptr == ld && err == LDAP_PROTOCOL_ERROR && stype != ACTIVE_DIRECTORY) From 1a0da02d0009f3b5b6e00b71e8cebcb9ea05531e Mon Sep 17 00:00:00 2001 From: Jake Smith Date: Thu, 15 Aug 2024 16:42:13 +0100 Subject: [PATCH 09/33] Split off 9.8.14 Signed-off-by: Jake Smith --- helm/hpcc/Chart.yaml | 4 ++-- helm/hpcc/templates/_helpers.tpl | 2 +- version.cmake | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/helm/hpcc/Chart.yaml b/helm/hpcc/Chart.yaml index 7f55a6deb84..379a6c6bd0c 100644 --- a/helm/hpcc/Chart.yaml +++ b/helm/hpcc/Chart.yaml @@ -6,9 +6,9 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. -version: 9.8.13-closedown0 +version: 9.8.15-closedown0 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. -appVersion: 9.8.13-closedown0 +appVersion: 9.8.15-closedown0 diff --git a/helm/hpcc/templates/_helpers.tpl b/helm/hpcc/templates/_helpers.tpl index 53f11be68b8..50d6cf06176 100644 --- a/helm/hpcc/templates/_helpers.tpl +++ b/helm/hpcc/templates/_helpers.tpl @@ -1477,7 +1477,7 @@ Pass in dict with .root, .visibility defined {{- end -}} {{- define "hpcc.generateHelmVersion" -}} -helmVersion: 9.8.13-closedown0 +helmVersion: 9.8.15-closedown0 {{- end -}} {{/* diff --git a/version.cmake b/version.cmake index 57750c465c8..b7f7a740551 100644 --- a/version.cmake +++ b/version.cmake @@ -5,8 +5,8 @@ set ( HPCC_NAME "Community Edition" ) set ( HPCC_PROJECT "community" ) set ( HPCC_MAJOR 9 ) set ( HPCC_MINOR 8 ) -set ( HPCC_POINT 13 ) +set ( HPCC_POINT 15 ) set ( HPCC_MATURITY "closedown" ) set ( HPCC_SEQUENCE 0 ) -set ( HPCC_TAG_TIMESTAMP "2024-08-08T16:17:22Z" ) +set ( HPCC_TAG_TIMESTAMP "2024-08-15T15:42:13Z" ) ### From b1ac7e689c12b7699a30aad0a16280ccc7a300b2 Mon Sep 17 00:00:00 2001 From: Jake Smith Date: Thu, 15 Aug 2024 16:43:09 +0100 Subject: [PATCH 10/33] Split off 9.6.40 Signed-off-by: Jake Smith --- helm/hpcc/Chart.yaml | 4 ++-- helm/hpcc/templates/_helpers.tpl | 2 +- version.cmake | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/helm/hpcc/Chart.yaml b/helm/hpcc/Chart.yaml index 1ccd1c3af10..71d9c5bc171 100644 --- a/helm/hpcc/Chart.yaml +++ b/helm/hpcc/Chart.yaml @@ -6,9 +6,9 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. -version: 9.6.39-closedown0 +version: 9.6.41-closedown0 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. -appVersion: 9.6.39-closedown0 +appVersion: 9.6.41-closedown0 diff --git a/helm/hpcc/templates/_helpers.tpl b/helm/hpcc/templates/_helpers.tpl index 298e37c5550..167cea0ec5d 100644 --- a/helm/hpcc/templates/_helpers.tpl +++ b/helm/hpcc/templates/_helpers.tpl @@ -1477,7 +1477,7 @@ Pass in dict with .root, .visibility defined {{- end -}} {{- define "hpcc.generateHelmVersion" -}} -helmVersion: 9.6.39-closedown0 +helmVersion: 9.6.41-closedown0 {{- end -}} {{/* diff --git a/version.cmake b/version.cmake index 90d26bd3200..ed620f6e7b4 100644 --- a/version.cmake +++ b/version.cmake @@ -5,8 +5,8 @@ set ( HPCC_NAME "Community Edition" ) set ( HPCC_PROJECT "community" ) set ( HPCC_MAJOR 9 ) set ( HPCC_MINOR 6 ) -set ( HPCC_POINT 39 ) +set ( HPCC_POINT 41 ) set ( HPCC_MATURITY "closedown" ) set ( HPCC_SEQUENCE 0 ) -set ( HPCC_TAG_TIMESTAMP "2024-08-08T16:18:20Z" ) +set ( HPCC_TAG_TIMESTAMP "2024-08-15T15:43:09Z" ) ### From 952a8b692b5575a4dca447557d5679ec29ef7699 Mon Sep 17 00:00:00 2001 From: Jake Smith Date: Thu, 15 Aug 2024 16:44:19 +0100 Subject: [PATCH 11/33] Split off 9.4.88 Signed-off-by: Jake Smith --- helm/hpcc/Chart.yaml | 4 ++-- helm/hpcc/templates/_helpers.tpl | 2 +- version.cmake | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/helm/hpcc/Chart.yaml b/helm/hpcc/Chart.yaml index f6fd5069a08..6b890d3a750 100644 --- a/helm/hpcc/Chart.yaml +++ b/helm/hpcc/Chart.yaml @@ -6,9 +6,9 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. -version: 9.4.87-closedown0 +version: 9.4.89-closedown0 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. -appVersion: 9.4.87-closedown0 +appVersion: 9.4.89-closedown0 diff --git a/helm/hpcc/templates/_helpers.tpl b/helm/hpcc/templates/_helpers.tpl index 112bf329345..6a04a569072 100644 --- a/helm/hpcc/templates/_helpers.tpl +++ b/helm/hpcc/templates/_helpers.tpl @@ -1473,7 +1473,7 @@ Pass in dict with .root, .visibility defined {{- end -}} {{- define "hpcc.generateHelmVersion" -}} -helmVersion: 9.4.87-closedown0 +helmVersion: 9.4.89-closedown0 {{- end -}} {{/* diff --git a/version.cmake b/version.cmake index 36b64302db2..d1c48e4651d 100644 --- a/version.cmake +++ b/version.cmake @@ -5,8 +5,8 @@ set ( HPCC_NAME "Community Edition" ) set ( HPCC_PROJECT "community" ) set ( HPCC_MAJOR 9 ) set ( HPCC_MINOR 4 ) -set ( HPCC_POINT 87 ) +set ( HPCC_POINT 89 ) set ( HPCC_MATURITY "closedown" ) set ( HPCC_SEQUENCE 0 ) -set ( HPCC_TAG_TIMESTAMP "2024-08-08T16:19:19Z" ) +set ( HPCC_TAG_TIMESTAMP "2024-08-15T15:44:19Z" ) ### From 431e4e30ba6c842a9252dc060176a0337acb3e9a Mon Sep 17 00:00:00 2001 From: Jake Smith Date: Thu, 15 Aug 2024 16:45:07 +0100 Subject: [PATCH 12/33] Split off 9.2.114 Signed-off-by: Jake Smith --- helm/hpcc/Chart.yaml | 4 ++-- helm/hpcc/templates/_helpers.tpl | 2 +- version.cmake | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/helm/hpcc/Chart.yaml b/helm/hpcc/Chart.yaml index 541ec0b4765..38711d74f46 100644 --- a/helm/hpcc/Chart.yaml +++ b/helm/hpcc/Chart.yaml @@ -6,9 +6,9 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. -version: 9.2.113-closedown0 +version: 9.2.115-closedown0 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. -appVersion: 9.2.113-closedown0 +appVersion: 9.2.115-closedown0 diff --git a/helm/hpcc/templates/_helpers.tpl b/helm/hpcc/templates/_helpers.tpl index c35fa1d92b0..00103293fa4 100644 --- a/helm/hpcc/templates/_helpers.tpl +++ b/helm/hpcc/templates/_helpers.tpl @@ -1361,7 +1361,7 @@ Pass in dict with .root, .visibility defined {{- end -}} {{- define "hpcc.generateHelmVersion" -}} -helmVersion: 9.2.113-closedown0 +helmVersion: 9.2.115-closedown0 {{- end -}} {{/* diff --git a/version.cmake b/version.cmake index 3ff7c348647..9b6027ac6bc 100644 --- a/version.cmake +++ b/version.cmake @@ -5,8 +5,8 @@ set ( HPCC_NAME "Community Edition" ) set ( HPCC_PROJECT "community" ) set ( HPCC_MAJOR 9 ) set ( HPCC_MINOR 2 ) -set ( HPCC_POINT 113 ) +set ( HPCC_POINT 115 ) set ( HPCC_MATURITY "closedown" ) set ( HPCC_SEQUENCE 0 ) -set ( HPCC_TAG_TIMESTAMP "2024-08-08T16:20:19Z" ) +set ( HPCC_TAG_TIMESTAMP "2024-08-15T15:45:07Z" ) ### From 56fafb8ad6529a9d731f6cde6612201b3ff053b8 Mon Sep 17 00:00:00 2001 From: Jake Smith Date: Thu, 15 Aug 2024 17:57:49 +0100 Subject: [PATCH 13/33] HPCC-32446 Identify the Thor instance name in interface incompatibility error Signed-off-by: Jake Smith --- thorlcr/master/thgraphmanager.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/thorlcr/master/thgraphmanager.cpp b/thorlcr/master/thgraphmanager.cpp index 5eb01feaca4..9707b5a48d1 100644 --- a/thorlcr/master/thgraphmanager.cpp +++ b/thorlcr/master/thgraphmanager.cpp @@ -545,7 +545,7 @@ bool CJobManager::execute(IConstWorkUnit *workunit, const char *wuid, const char if (workunit->getCodeVersion() == 0) throw makeStringException(0, "Attempting to execute a workunit that hasn't been compiled"); if ((workunit->getCodeVersion() > ACTIVITY_INTERFACE_VERSION) || (workunit->getCodeVersion() < MIN_ACTIVITY_INTERFACE_VERSION)) - throw MakeStringException(0, "Workunit was compiled for eclagent interface version %d, this thor requires version %d..%d", workunit->getCodeVersion(), MIN_ACTIVITY_INTERFACE_VERSION, ACTIVITY_INTERFACE_VERSION); + throw MakeStringException(0, "Workunit was compiled for eclagent interface version %d, this thor (%s) requires version %d..%d", workunit->getCodeVersion(), globals->queryProp("@name"), MIN_ACTIVITY_INTERFACE_VERSION, ACTIVITY_INTERFACE_VERSION); if (workunit->getCodeVersion() == 652) { // Any workunit compiled using eclcc 7.12.0-7.12.18 is not compatible @@ -557,7 +557,7 @@ bool CJobManager::execute(IConstWorkUnit *workunit, const char *wuid, const char const char *point = version + strlen("7.12."); unsigned pointVer = atoi(point); if (pointVer <= 18) - throw MakeStringException(0, "Workunit was compiled by eclcc version %s which is not compatible with this runtime", buildVersion.str()); + throw MakeStringException(0, "Workunit was compiled by eclcc version %s which is not compatible with this thor (%s)", buildVersion.str(), globals->queryProp("@name")); } } From 6e9fee15c6c5fcc508348dc281791ea647b5a349 Mon Sep 17 00:00:00 2001 From: Gavin Halliday Date: Fri, 16 Aug 2024 14:23:46 +0100 Subject: [PATCH 14/33] HPCC-32447 Fix very slow spilling of rows containing very large child datasets Signed-off-by: Gavin Halliday --- system/jlib/jstream.cpp | 11 +++- testing/unittests/jstreamtests.cpp | 96 ++++++++++++++++++++++++++++++ 2 files changed, 105 insertions(+), 2 deletions(-) diff --git a/system/jlib/jstream.cpp b/system/jlib/jstream.cpp index 8f31718a8e1..c0ea60f0618 100644 --- a/system/jlib/jstream.cpp +++ b/system/jlib/jstream.cpp @@ -896,8 +896,15 @@ class CBlockedSerialOutputStream final : public CInterfaceOf 1M + size32_t alignment = buffer.length() / 4; + if (alignment < 32) + alignment = 32; + newLength += (alignment - 1); + newLength -= newLength % alignment; MemoryAttr expandedBuffer(newLength); memcpy(expandedBuffer.mem(), data(0), bufferOffset); diff --git a/testing/unittests/jstreamtests.cpp b/testing/unittests/jstreamtests.cpp index dda4523835d..23ee3dccd8b 100644 --- a/testing/unittests/jstreamtests.cpp +++ b/testing/unittests/jstreamtests.cpp @@ -282,6 +282,87 @@ class VariableDataProvider : public CDataProvider }; +//A very large row (because of a large embedded dataset) +//100 bytes of data then 500K rows of 100 bytes +class LargeRowDataProvider : public CDataProvider +{ +public: + LargeRowDataProvider(bool _useCount, unsigned _numChildren) : useCount(_useCount), numChildren(_numChildren) + { + name.append("Large_").append(useCount ? 'C' : 'S').append(numChildren); + } + + virtual size32_t create(IBufferedSerialOutputStream * target, unsigned row) + { + //Output (row, (string)row, (row % 7)items of (row, row*2, row*3)) + byte mainRow[100]; + unsigned childRow[25]; + + for (size32_t i=0; i < sizeof(mainRow); i++) + mainRow[i] = (byte)(i * row); + target->put(sizeof(mainRow), mainRow); + + size32_t childCount = numChildren + row; + size32_t childSize = sizeof(childRow) * childCount; + if (useCount) + target->put(4, &childCount); + else + target->suspend(sizeof(size32_t)); + + unsigned next = 1234 + row * 31419264U; + for (unsigned i=0; i < childCount; i++) + { + for (size32_t i=0; i < sizeof(mainRow)/sizeof(next); i++) + { + childRow[i] = next; + next *= 0x13894225; + next += row; + } + target->put(sizeof(childRow), &childRow); + } + if (!useCount) + target->resume(sizeof(childSize), &childSize); + + return sizeof(mainRow) + 4 + childSize; + } + + virtual size32_t check(IBufferedSerialInputStream * source, unsigned row) + { + byte mainRow[100]; + unsigned childRow[25]; + + source->read(sizeof(mainRow), &mainRow); + for (size32_t i=0; i < sizeof(mainRow); i++) + assertex(mainRow[i] == (byte)(i * row)); + + size32_t childCount = numChildren + row; + size32_t childSize = sizeof(childRow) * childCount; + size32_t size; + source->read(sizeof(size), &size); + if (useCount) + assertex(size == childCount); + else + assertex(size == childSize); + + unsigned next = 1234 + row * 31419264U; + for (unsigned i=0; i < childCount; i++) + { + source->read(sizeof(childRow), &childRow); + for (size32_t i=0; i < sizeof(mainRow)/sizeof(next); i++) + { + assertex(childRow[i] == next); + next *= 0x13894225; + next += row; + } + } + return sizeof(mainRow) + 4 + childSize; + } + +protected: + bool useCount; + unsigned numChildren = 10'000'000; +}; + class NullOuputStream : public CInterfaceOf { virtual size32_t write(size32_t len, const void * ptr) { return len; } @@ -305,6 +386,7 @@ class JlibStreamStressTest : public CppUnit::TestFixture CPPUNIT_TEST(testEvenSequentialStream); // write a file and read results after each flush CPPUNIT_TEST(testParallelStream); // write a file and read in parallel from a separate thread CPPUNIT_TEST(testThreadedWriteStream); // write a file using a threaded writer + CPPUNIT_TEST(testPathologicalRows); // 1M child rows, total row size 100MB //MORE: //Threaded writer //Threaded reader @@ -735,6 +817,20 @@ class JlibStreamStressTest : public CppUnit::TestFixture } } + void testPathologicalRows() + { + LargeRowDataProvider largeCount50K(true, 50'000); + LargeRowDataProvider largeCount10M(true, 10'000'000); + LargeRowDataProvider largeSize50K(false, 50'000); + LargeRowDataProvider largeSize10M(false, 10'000'000); + + ICompressHandler * lz4 = queryCompressHandler(COMPRESS_METHOD_LZ4); + + runSimpleStream(nullptr, largeCount50K, 0x100000, 0x100000, 2000); + runSimpleStream(nullptr, largeCount10M, 0x100000, 0x100000, 10); + runSimpleStream(nullptr, largeSize50K, 0x100000, 0x100000, 2000); + runSimpleStream(nullptr, largeSize10M, 0x100000, 0x100000, 10); + } void testIncSequentialStream() { From 76dca1dcabdc3cadf8aab1d3a11963bf6261cea5 Mon Sep 17 00:00:00 2001 From: M Kelly Date: Mon, 19 Aug 2024 08:24:47 -0400 Subject: [PATCH 15/33] HPCC-32454 SYS: swap and mem util stats fixup previous change Signed-off-by: M Kelly --- system/jlib/jdebug.cpp | 38 ++++++++++++++++++++++++++++++++++---- 1 file changed, 34 insertions(+), 4 deletions(-) diff --git a/system/jlib/jdebug.cpp b/system/jlib/jdebug.cpp index 17154d4a99c..79c9c36bbb7 100644 --- a/system/jlib/jdebug.cpp +++ b/system/jlib/jdebug.cpp @@ -1454,10 +1454,40 @@ static void getMemUsage(unsigned &inuse,unsigned &active,unsigned &total,unsigne } inuse = total-free-cached; - // not sure if a bug in kernel or container or ... - // but sometimes we see swapfree > 0 when swaptotal == 0 - if ((swapfree + swapcached) >= swaptotal) - swapinuse = 0; + // swapinuse = swaptotal-swapfree-swapcached; + + // sometimes in containers [under mem pressure ?] we see from /proc/meminfo - + + // SwapCached: 0 kB + // SwapTotal: 0 kB + // SwapFree: 18446744073709551496 kB + // or - + // SwapCached: 0 kB + // SwapTotal: 0 kB + // SwapFree: 120 kB + + // and from free cmd - + + // free -m + // total used free shared buff/cache available + // Mem: 43008 17616 5375 0 20015 25391 + // Swap: 0 18014398509481984 0 + + // if swapfree > 0 when swaptotal == 0 - + // *might* indicate kernel is pushing exe/mmapped pages out of memory to make room + // for other things and this can affect performance + + // not sure why SwapFree value is not always valid/accurate + // vmstat shows more reasonable swpd value, but walks all /proc//stat files + + // SwapCached: Memory that is present within main memory, but also in the swapfile + + if ((swapfree + swapcached) > swaptotal) + { + swapinuse = swapfree + swapcached; + if (swapinuse > total) + swapinuse = active; // something more reasonable ... + } else swapinuse = swaptotal-swapfree-swapcached; #endif From a8502f128a1e694f4d38382037f9cc1c384e362b Mon Sep 17 00:00:00 2001 From: "Dan S. Camper" Date: Mon, 19 Aug 2024 10:46:33 -0500 Subject: [PATCH 16/33] HPCC-32461 REGEXREPLACE fails to handle unknown replacement groups --- rtl/eclrtl/eclregex.cpp | 8 ++++---- testing/regress/ecl/key/regex3.xml | 6 ++++++ testing/regress/ecl/regex3.ecl | 4 ++++ 3 files changed, 14 insertions(+), 4 deletions(-) diff --git a/rtl/eclrtl/eclregex.cpp b/rtl/eclrtl/eclregex.cpp index 9e3714795f3..b867df4af9b 100644 --- a/rtl/eclrtl/eclregex.cpp +++ b/rtl/eclrtl/eclregex.cpp @@ -440,7 +440,7 @@ class CCompiledStrRegExpr : implements ICompiledStrRegExpr if (numMatches > 0) { - uint32_t replaceOptions = PCRE2_SUBSTITUTE_MATCHED|PCRE2_SUBSTITUTE_GLOBAL|PCRE2_SUBSTITUTE_EXTENDED; + uint32_t replaceOptions = PCRE2_SUBSTITUTE_MATCHED|PCRE2_SUBSTITUTE_GLOBAL|PCRE2_SUBSTITUTE_EXTENDED|PCRE2_SUBSTITUTE_UNKNOWN_UNSET|PCRE2_SUBSTITUTE_UNSET_EMPTY; PCRE2_SIZE pcreSize = 0; // Call substitute once to get the size of the output (pushed into pcreSize); @@ -516,7 +516,7 @@ class CCompiledStrRegExpr : implements ICompiledStrRegExpr if (numMatches > 0) { - uint32_t replaceOptions = PCRE2_SUBSTITUTE_MATCHED|PCRE2_SUBSTITUTE_GLOBAL|PCRE2_SUBSTITUTE_EXTENDED; + uint32_t replaceOptions = PCRE2_SUBSTITUTE_MATCHED|PCRE2_SUBSTITUTE_GLOBAL|PCRE2_SUBSTITUTE_EXTENDED|PCRE2_SUBSTITUTE_UNKNOWN_UNSET|PCRE2_SUBSTITUTE_UNSET_EMPTY; PCRE2_SIZE pcreSize = 0; // Call substitute once to get the size of the output and see if it will fit within fixedOutLen; @@ -965,7 +965,7 @@ class CCompiledUStrRegExpr : implements ICompiledUStrRegExpr if (numMatches > 0) { - uint32_t replaceOptions = PCRE2_SUBSTITUTE_MATCHED|PCRE2_SUBSTITUTE_GLOBAL|PCRE2_SUBSTITUTE_EXTENDED; + uint32_t replaceOptions = PCRE2_SUBSTITUTE_MATCHED|PCRE2_SUBSTITUTE_GLOBAL|PCRE2_SUBSTITUTE_EXTENDED|PCRE2_SUBSTITUTE_UNKNOWN_UNSET|PCRE2_SUBSTITUTE_UNSET_EMPTY; PCRE2_SIZE pcreSize = 0; // Call substitute once to get the size of the output, then allocate memory for it; @@ -1033,7 +1033,7 @@ class CCompiledUStrRegExpr : implements ICompiledUStrRegExpr if (numMatches > 0) { - uint32_t replaceOptions = PCRE2_SUBSTITUTE_MATCHED|PCRE2_SUBSTITUTE_GLOBAL|PCRE2_SUBSTITUTE_EXTENDED; + uint32_t replaceOptions = PCRE2_SUBSTITUTE_MATCHED|PCRE2_SUBSTITUTE_GLOBAL|PCRE2_SUBSTITUTE_EXTENDED|PCRE2_SUBSTITUTE_UNKNOWN_UNSET|PCRE2_SUBSTITUTE_UNSET_EMPTY; PCRE2_SIZE pcreSize = 0; // Call substitute once to get the size of the output and see if it will fit within fixedOutLen; diff --git a/testing/regress/ecl/key/regex3.xml b/testing/regress/ecl/key/regex3.xml index a4ac707da50..4f4f55c3b10 100644 --- a/testing/regress/ecl/key/regex3.xml +++ b/testing/regress/ecl/key/regex3.xml @@ -25,3 +25,9 @@ true + + NM + + + JustOneWord + diff --git a/testing/regress/ecl/regex3.ecl b/testing/regress/ecl/regex3.ecl index aac3d3488d4..2eb88cec623 100644 --- a/testing/regress/ecl/regex3.ecl +++ b/testing/regress/ecl/regex3.ecl @@ -66,3 +66,7 @@ REGEXREPLACE('\'', 'Dan\'s', '\\\'') = 'Dan\'s'; REGEXREPLACE(NOFOLD('\''), 'Dan\'s', '\\\'') = 'Dan\'s'; REGEXREPLACE(u'\'', u'Dan\'s', u'\\\'') = u'Dan\'s'; REGEXREPLACE(NOFOLD(u'\''), u'Dan\'s', u'\\\'') = u'Dan\'s'; + +// HPCC-32461 +REGEXREPLACE('\\b(N)EW (M)EXICO\\b|\\b(U)NITED (S)TATES\\b','NEW MEXICO','$1$2$3'); +REGEXREPLACE('(\\w+)', 'JustOneWord', '$2 $1'); From c0e20f604ab9d71a4e4712b54752a5f29f802ddb Mon Sep 17 00:00:00 2001 From: Anthony Fishbeck Date: Mon, 19 Aug 2024 14:30:16 -0400 Subject: [PATCH 17/33] HPCC-32462 Wildcard certs should work with ClusterIssuers Signed-off-by: Anthony Fishbeck --- helm/hpcc/templates/issuers.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/helm/hpcc/templates/issuers.yaml b/helm/hpcc/templates/issuers.yaml index 9b52fb85800..77dc5e2b3bf 100644 --- a/helm/hpcc/templates/issuers.yaml +++ b/helm/hpcc/templates/issuers.yaml @@ -119,8 +119,8 @@ spec: --- {{- end }} {{- end }} - {{ include "hpcc.addWildIssuerCertificate" (dict "root" .root "issuerKeyName" .issuerKeyName "me" .me ) }} {{- end }} + {{ include "hpcc.addWildIssuerCertificate" (dict "root" .root "issuerKeyName" .issuerKeyName "me" .me ) }} {{- end }} {{- template "hpcc.ensureNoResourceValidationFlag" ( dict "root" $ ) }} From a4cc067029ae61a635754b6ff7caee61e9e11860 Mon Sep 17 00:00:00 2001 From: Jeremy Clements <79224539+jeclrsg@users.noreply.github.com> Date: Mon, 19 Aug 2024 15:16:00 -0400 Subject: [PATCH 18/33] HPCC-32425 ECL Watch v9 copy to clipboard missing columns fixes some missing headers and columns when using the copy selection to clipboard from the Files, Workunits and Queries list pages Signed-off-by: Jeremy Clements <79224539+jeclrsg@users.noreply.github.com> --- esp/src/src-react/components/Files.tsx | 7 ++++++- esp/src/src-react/components/Queries.tsx | 11 +++++++---- esp/src/src-react/components/Workunits.tsx | 18 ++++++++---------- 3 files changed, 21 insertions(+), 15 deletions(-) diff --git a/esp/src/src-react/components/Files.tsx b/esp/src/src-react/components/Files.tsx index 08f5f953c34..6cc33214cbb 100644 --- a/esp/src/src-react/components/Files.tsx +++ b/esp/src/src-react/components/Files.tsx @@ -156,6 +156,7 @@ export const Files: React.FunctionComponent = ({ } return ""; }, + field: nlsHPCC.Protected, }, IsCompressed: { headerIcon: "ZipFolder", @@ -168,6 +169,7 @@ export const Files: React.FunctionComponent = ({ } return ""; }, + field: nlsHPCC.Compressed, }, Name: { label: nlsHPCC.LogicalName, @@ -194,18 +196,21 @@ export const Files: React.FunctionComponent = ({ formatter: (value, row) => { return Utility.formatNum(row.IntRecordCount); }, + csvFormatter: (value, row) => row.IntRecordCount, }, FileSize: { label: nlsHPCC.Size, formatter: (value, row) => { return Utility.convertedSize(row.IntSize); }, + csvFormatter: (value, row) => row.IntSize, }, CompressedFileSizeString: { label: nlsHPCC.CompressedSize, formatter: (value, row) => { return Utility.convertedSize(row.CompressedFileSize); - } + }, + csvFormatter: (value, row) => row.CompressedFileSize, }, Parts: { label: nlsHPCC.Parts, width: 40, diff --git a/esp/src/src-react/components/Queries.tsx b/esp/src/src-react/components/Queries.tsx index e11938a5734..ff45b2a7f6d 100644 --- a/esp/src/src-react/components/Queries.tsx +++ b/esp/src/src-react/components/Queries.tsx @@ -112,7 +112,8 @@ export const Queries: React.FunctionComponent = ({ return ; } return ""; - } + }, + field: nlsHPCC.Suspended, }, ErrorCount: { headerIcon: "Warning", @@ -124,7 +125,8 @@ export const Queries: React.FunctionComponent = ({ return ; } return ""; - } + }, + field: nlsHPCC.ErrorWarnings, }, MixedNodeStates: { headerIcon: "Error", @@ -136,7 +138,7 @@ export const Queries: React.FunctionComponent = ({ return ; } return ""; - } + }, }, Activated: { headerIcon: "SkypeCircleCheck", @@ -147,7 +149,8 @@ export const Queries: React.FunctionComponent = ({ return ; } return ""; - } + }, + field: nlsHPCC.Active, }, Id: { label: nlsHPCC.ID, diff --git a/esp/src/src-react/components/Workunits.tsx b/esp/src/src-react/components/Workunits.tsx index 0fce3d69f16..ab47cf34673 100644 --- a/esp/src/src-react/components/Workunits.tsx +++ b/esp/src/src-react/components/Workunits.tsx @@ -116,7 +116,8 @@ export const Workunits: React.FunctionComponent = ({ return ; } return ""; - } + }, + field: nlsHPCC.Protected, }, Wuid: { label: nlsHPCC.WUID, width: 120, @@ -143,23 +144,20 @@ export const Workunits: React.FunctionComponent = ({ "Compile Cost": { label: nlsHPCC.CompileCost, width: 100, justify: "right", - formatter: (cost, row) => { - return `${formatCost(row.CompileCost)}`; - } + formatter: (cost, row) => `${formatCost(row.CompileCost)}`, + csvFormatter: (cost, row) => row.CompileCost, }, "Execution Cost": { label: nlsHPCC.ExecuteCost, width: 100, justify: "right", - formatter: (cost, row) => { - return `${formatCost(row.ExecuteCost)}`; - } + formatter: (cost, row) => `${formatCost(row.ExecuteCost)}`, + csvFormatter: (cost, row) => row.ExecuteCost, }, "File Access Cost": { label: nlsHPCC.FileAccessCost, width: 100, justify: "right", - formatter: (cost, row) => { - return `${formatCost(row.FileAccessCost)}`; - } + formatter: (cost, row) => `${formatCost(row.FileAccessCost)}`, + csvFormatter: (cost, row) => row.FileAccessCost, } }; }, [filter]); From e22748bd4febab5dcf8d83f2ea8f311405a01d8c Mon Sep 17 00:00:00 2001 From: Gavin Halliday Date: Tue, 20 Aug 2024 09:55:01 +0100 Subject: [PATCH 19/33] HPCC-32464 Allow components to start if config watcher cannot be created Signed-off-by: Gavin Halliday --- system/jlib/jptree.cpp | 24 ++++++++++++++++-------- 1 file changed, 16 insertions(+), 8 deletions(-) diff --git a/system/jlib/jptree.cpp b/system/jlib/jptree.cpp index 4aed0538292..d7ba2871c78 100644 --- a/system/jlib/jptree.cpp +++ b/system/jlib/jptree.cpp @@ -8782,14 +8782,22 @@ class CConfigUpdater : public CInterface absoluteConfigFilename.set(std::get<0>(result).c_str()); } }; - fileWatcher.setown(createFileEventWatcher(updateFunc)); - - // watch the path, not the filename, because the filename might not be seen if directories are moved, softlinks are changed.. - StringBuffer path, filename; - splitFilename(absoluteConfigFilename, nullptr, &path, &filename, &filename); - configFilename.set(filename); - fileWatcher->add(path, FileWatchEvents::anyChange); - fileWatcher->start(); + try + { + fileWatcher.setown(createFileEventWatcher(updateFunc)); + + // watch the path, not the filename, because the filename might not be seen if directories are moved, softlinks are changed.. + StringBuffer path, filename; + splitFilename(absoluteConfigFilename, nullptr, &path, &filename, &filename); + configFilename.set(filename); + fileWatcher->add(path, FileWatchEvents::anyChange); + fileWatcher->start(); + } + catch (IException * e) + { + OERRLOG(e, "Failed to start file watcher"); + e->Release(); + } return true; } void executeCallbacks(IPropertyTree *oldComponentConfiguration, IPropertyTree *oldGlobalConfiguration) From ddc3346ad71c5a62699fa7c8a476797e494cf3d5 Mon Sep 17 00:00:00 2001 From: Gavin Halliday Date: Wed, 24 Jul 2024 16:22:53 +0100 Subject: [PATCH 20/33] HPCC-31726 Optimize MALLOC_ARENA_MAX for containerized systems Signed-off-by: Gavin Halliday --- helm/hpcc/templates/_helpers.tpl | 7 ++++--- helm/hpcc/templates/dafilesrv.yaml | 2 +- helm/hpcc/templates/dali.yaml | 2 +- helm/hpcc/templates/dfuserver.yaml | 2 +- helm/hpcc/templates/eclagent.yaml | 4 ++-- helm/hpcc/templates/eclccserver.yaml | 4 ++-- helm/hpcc/templates/eclscheduler.yaml | 2 +- helm/hpcc/templates/esp.yaml | 2 +- helm/hpcc/templates/localroxie.yaml | 2 +- helm/hpcc/templates/roxie.yaml | 6 +++--- helm/hpcc/templates/thor.yaml | 11 ++++++----- testing/helm/run.sh | 4 ++-- 12 files changed, 25 insertions(+), 23 deletions(-) diff --git a/helm/hpcc/templates/_helpers.tpl b/helm/hpcc/templates/_helpers.tpl index 7307367562b..fbcf50926b6 100644 --- a/helm/hpcc/templates/_helpers.tpl +++ b/helm/hpcc/templates/_helpers.tpl @@ -1426,7 +1426,7 @@ Pass in dict with root, me and dali if container in dali pod {{- include "hpcc.addResources" (dict "me" .me.resources "root" .root) | indent 2 }} {{- include "hpcc.addSecurityContext" . | indent 2 }} env: -{{ include "hpcc.mergeEnvironments" $env | indent 2 -}} +{{ include "hpcc.mergeEnvironments" (dict "env" $env "defaultArenas" 2) | indent 2 -}} - name: "SENTINEL" value: "/tmp/{{ $serviceName }}.sentinel" {{- with (dict "name" $serviceName) }} @@ -2507,8 +2507,9 @@ global.noResourceValidation flag. This behavior can be overridden by the caller A template to output a merged environment. Pass in a list with global then local environments. Only the last specified value for each named environment variable will be output */}} {{- define "hpcc.mergeEnvironments" -}} -{{- $result := dict "MALLOC_ARENA_MAX" 8 -}}{{- /* HPCC arena default, can be overriden by component config */ -}} -{{- range . -}} +{{- $defaultArenas := .defaultArenas | default 1 -}} +{{- $result := dict "MALLOC_ARENA_MAX" $defaultArenas -}}{{- /* HPCC arena default, can be overridden by component config */ -}} +{{- range .env -}} {{- $_ := set $result .name .value -}} {{- end -}} {{- range $key,$value := $result -}} diff --git a/helm/hpcc/templates/dafilesrv.yaml b/helm/hpcc/templates/dafilesrv.yaml index e378c759bf5..9287ac61907 100644 --- a/helm/hpcc/templates/dafilesrv.yaml +++ b/helm/hpcc/templates/dafilesrv.yaml @@ -69,7 +69,7 @@ spec: {{ include "hpcc.configArg" . }} ] env: -{{ include "hpcc.mergeEnvironments" $env | indent 8 -}} +{{ include "hpcc.mergeEnvironments" (dict "env" $env "defaultArenas" 2) | indent 8 -}} - name: "SENTINEL" value: "/tmp/{{ .name }}.sentinel" {{ include "hpcc.addSentinelProbes" . | indent 8 }} diff --git a/helm/hpcc/templates/dali.yaml b/helm/hpcc/templates/dali.yaml index 2adfbbd328d..38c62df33be 100644 --- a/helm/hpcc/templates/dali.yaml +++ b/helm/hpcc/templates/dali.yaml @@ -122,7 +122,7 @@ spec: {{ include "hpcc.configArg" $dali }} ] env: -{{ include "hpcc.mergeEnvironments" $env | indent 8 -}} +{{ include "hpcc.mergeEnvironments" (dict "env" $env "defaultArenas" 4) | indent 8 -}} - name: "SENTINEL" value: "/tmp/{{ $dali.name }}.sentinel" {{ include "hpcc.addSentinelProbes" $dali | indent 8 }} diff --git a/helm/hpcc/templates/dfuserver.yaml b/helm/hpcc/templates/dfuserver.yaml index 4622e253e30..ea62d531b0c 100644 --- a/helm/hpcc/templates/dfuserver.yaml +++ b/helm/hpcc/templates/dfuserver.yaml @@ -79,7 +79,7 @@ spec: {{ include "hpcc.daliArg" (dict "root" $ "component" "DFU Server" "optional" false) }} ] env: -{{ include "hpcc.mergeEnvironments" $env | indent 8 -}} +{{ include "hpcc.mergeEnvironments" (dict "env" $env "defaultArenas" 1) | indent 8 -}} - name: "SENTINEL" value: "/tmp/{{ .name }}.sentinel" {{ include "hpcc.addSentinelProbes" . | indent 8 }} diff --git a/helm/hpcc/templates/eclagent.yaml b/helm/hpcc/templates/eclagent.yaml index 954184a51dc..7153b5317ba 100644 --- a/helm/hpcc/templates/eclagent.yaml +++ b/helm/hpcc/templates/eclagent.yaml @@ -88,7 +88,7 @@ data: {{- $appCmd := printf "%s %s %s _HPCC_ARGS_" $apptype (include "hpcc.configArg" .me) (include "hpcc.daliArg" (dict "root" .root "component" "ECL Agent" "optional" false )) }} {{ include "hpcc.addCommandAndLifecycle" (. | merge (dict "command" $appCmd)) | indent 12 }} env: -{{ include "hpcc.mergeEnvironments" .env | indent 12 }} +{{ include "hpcc.mergeEnvironments" (dict "env" .env "defaultArenas" 1) | indent 12 }} workingDir: /var/lib/HPCCSystems volumeMounts: {{ include "hpcc.addConfigMapVolumeMount" .me | indent 12 }} @@ -161,7 +161,7 @@ spec: {{ include "hpcc.daliArg" (dict "root" $ "component" "ECL Agent" "optional" false) }} ] env: -{{ include "hpcc.mergeEnvironments" $env | indent 8 -}} +{{ include "hpcc.mergeEnvironments" (dict "env" $env "defaultArenas" 1) | indent 8 -}} - name: "SENTINEL" value: "/tmp/{{ .name }}.sentinel" {{ include "hpcc.addSentinelProbes" . | indent 8 }} diff --git a/helm/hpcc/templates/eclccserver.yaml b/helm/hpcc/templates/eclccserver.yaml index ccfbd73a288..6b294146471 100644 --- a/helm/hpcc/templates/eclccserver.yaml +++ b/helm/hpcc/templates/eclccserver.yaml @@ -88,7 +88,7 @@ data: {{- $eclccserverCmd := printf "eclccserver %s %s _HPCC_ARGS_" (include "hpcc.configArg" .me) (include "hpcc.daliArg" (dict "root" .root "component" "ECLCC Server" "optional" false)) }} {{ include "hpcc.addCommandAndLifecycle" (. | merge (dict "command" $eclccserverCmd)) | indent 12 }} env: -{{ include "hpcc.mergeEnvironments" .env | indent 12 }} +{{ include "hpcc.mergeEnvironments" (dict "env" .env "defaultArenas" 1) | indent 12 }} workingDir: /tmp volumeMounts: {{ include "hpcc.addConfigMapVolumeMount" .me | indent 12 }} @@ -171,7 +171,7 @@ spec: {{ include "hpcc.daliArg" (dict "root" $ "component" "ECLCC Server" "optional" false) }} ] env: -{{ include "hpcc.mergeEnvironments" $env | indent 8 -}} +{{ include "hpcc.mergeEnvironments" (dict "env" $env "defaultArenas" 1) | indent 8 -}} - name: "SENTINEL" value: "/tmp/{{ .name }}.sentinel" {{ include "hpcc.addSentinelProbes" . | indent 8 }} diff --git a/helm/hpcc/templates/eclscheduler.yaml b/helm/hpcc/templates/eclscheduler.yaml index 9738de3223d..6586c57a752 100644 --- a/helm/hpcc/templates/eclscheduler.yaml +++ b/helm/hpcc/templates/eclscheduler.yaml @@ -88,7 +88,7 @@ spec: {{ include "hpcc.daliArg" (dict "root" $ "component" "ECL Scheduler" "optional" false) }} ] env: -{{ include "hpcc.mergeEnvironments" $env | indent 8 -}} +{{ include "hpcc.mergeEnvironments" (dict "env" $env "defaultArenas" 1) | indent 8 -}} - name: "SENTINEL" value: "/tmp/{{ .name }}.sentinel" {{ include "hpcc.addSentinelProbes" . | indent 8 }} diff --git a/helm/hpcc/templates/esp.yaml b/helm/hpcc/templates/esp.yaml index 6b9fc36695a..aa3435d8e15 100644 --- a/helm/hpcc/templates/esp.yaml +++ b/helm/hpcc/templates/esp.yaml @@ -153,7 +153,7 @@ spec: {{ include "hpcc.daliArg" (dict "root" $ "component" $application "optional" (has $application (list "esdl" "ldapenvironment" "loggingservice")) )}} ] env: -{{ include "hpcc.mergeEnvironments" $env | indent 8 -}} +{{ include "hpcc.mergeEnvironments" (dict "env" $env "defaultArenas" 4) | indent 8 -}} - name: "SENTINEL" value: "/tmp/{{ .name }}.sentinel" {{ include "hpcc.addSentinelProbes" . | indent 8 }} diff --git a/helm/hpcc/templates/localroxie.yaml b/helm/hpcc/templates/localroxie.yaml index c764ab1f8f3..2abd3857bf0 100644 --- a/helm/hpcc/templates/localroxie.yaml +++ b/helm/hpcc/templates/localroxie.yaml @@ -99,7 +99,7 @@ spec: "--resolveLocally=false" ] env: -{{ include "hpcc.mergeEnvironments" $env | indent 8 -}} +{{ include "hpcc.mergeEnvironments" (dict "env" $env "defaultArenas" 8) | indent 8 -}} - name: "SENTINEL" value: "/tmp/{{ $roxie.name }}.sentinel" {{- $local := dict "first" true }} diff --git a/helm/hpcc/templates/roxie.yaml b/helm/hpcc/templates/roxie.yaml index 3fb3c33f137..7610e46451b 100644 --- a/helm/hpcc/templates/roxie.yaml +++ b/helm/hpcc/templates/roxie.yaml @@ -155,7 +155,7 @@ spec: {{ include "hpcc.configArg" $toposerver }} ] env: -{{ include "hpcc.mergeEnvironments" $env | indent 8 -}} +{{ include "hpcc.mergeEnvironments" (dict "env" $env "defaultArenas" 1) | indent 8 -}} - name: "SENTINEL" value: "/tmp/{{ $commonCtx.toponame }}.sentinel" volumeMounts: @@ -277,7 +277,7 @@ spec: "--server=true" ] env: -{{ include "hpcc.mergeEnvironments" $env | indent 8 -}} +{{ include "hpcc.mergeEnvironments" (dict "env" $env "defaultArenas" 8) | indent 8 -}} - name: "SENTINEL" value: "/tmp/{{ $roxie.name }}.sentinel" {{- $local := dict "first" true }} @@ -383,7 +383,7 @@ spec: "--server={{ not $roxie.serverReplicas }}", ] env: -{{ include "hpcc.mergeEnvironments" $env | indent 8 -}} +{{ include "hpcc.mergeEnvironments" (dict "env" $env "defaultArenas" 8) | indent 8 -}} - name: "SENTINEL" value: "/tmp/{{ $roxie.name }}.sentinel" {{- if not $roxie.serverReplicas }} diff --git a/helm/hpcc/templates/thor.yaml b/helm/hpcc/templates/thor.yaml index 0f65429e5ee..a382d51707f 100644 --- a/helm/hpcc/templates/thor.yaml +++ b/helm/hpcc/templates/thor.yaml @@ -115,7 +115,7 @@ data: {{- $agentCmd := printf "%s %s %s _HPCC_ARGS_" $eclAgentType (include "hpcc.configArg" .me) (include "hpcc.daliArg" (dict "root" .root "component" "Thor" "optional" false)) }} {{ include "hpcc.addCommandAndLifecycle" (. | merge (dict "command" $agentCmd)) | indent 12 }} env: -{{ include "hpcc.mergeEnvironments" .env | indent 12 }} +{{ include "hpcc.mergeEnvironments" (dict "env" .env "defaultArenas" 1) | indent 12 }} workingDir: /var/lib/HPCCSystems volumeMounts: {{ include "hpcc.addConfigMapVolumeMount" .me | indent 12 }} @@ -182,7 +182,7 @@ data: {{- $thorManagerCmd := printf "thormaster_lcr %s %s _HPCC_ARGS_" (include "hpcc.configArg" .me) (include "hpcc.daliArg" (dict "root" .root "component" "Thor" "optional" false)) }} {{ include "hpcc.addCommandAndLifecycle" (. | merge (dict "command" $thorManagerCmd)) | indent 12 }} env: -{{ include "hpcc.mergeEnvironments" .env | indent 12 }} +{{ include "hpcc.mergeEnvironments" (dict "env" .env "defaultArenas" 2) | indent 12 }} workingDir: /var/lib/HPCCSystems volumeMounts: {{ include "hpcc.addConfigMapVolumeMount" .me | indent 12 }} @@ -251,7 +251,8 @@ data: {{- $thorWorkerCmd := printf "thorslave_lcr %s %s _HPCC_ARGS_ --slaveport=%d" (include "hpcc.configArg" $configCtx.me) (include "hpcc.daliArg" (dict "root" $configCtx.root "component" "Thor" "optional" false)) $slavePort }} {{ include "hpcc.addCommandAndLifecycle" ($configCtx | merge (dict "command" $thorWorkerCmd)) | indent 12 }} env: -{{ include "hpcc.mergeEnvironments" (append ($configCtx.me.env | default list) (dict "name" "MY_CONTAINER_NAME" "value" (printf "%s-%d" $thorWorkerJobName $containerNum))) | indent 12 }} +{{- $env := append ($configCtx.me.env | default list) (dict "name" "MY_CONTAINER_NAME" "value" (printf "%s-%d" $thorWorkerJobName $containerNum)) }} +{{ include "hpcc.mergeEnvironments" (dict "env" $env "defaultArenas" 8) | indent 12 }} workingDir: /var/lib/HPCCSystems volumeMounts: {{ include "hpcc.addConfigMapVolumeMount" (deepCopy $configCtx.me | merge (dict "tmpSubPath" $containerNum)) | indent 12 }} @@ -379,7 +380,7 @@ spec: {{ printf "\"--name=%s\"" .name }} ] env: -{{ include "hpcc.mergeEnvironments" $env | indent 8 -}} +{{ include "hpcc.mergeEnvironments" (dict "env" $env "defaultArenas" 1) | indent 8 -}} - name: "SENTINEL" value: "/tmp/{{ $commonCtx.eclAgentName }}.sentinel" {{- include "hpcc.addSecurityContext" $commonCtx | indent 8 }} @@ -442,7 +443,7 @@ spec: {{ printf "\"--name=%s\"" .name }} ] env: -{{ include "hpcc.mergeEnvironments" $env | indent 8 -}} +{{ include "hpcc.mergeEnvironments" (dict "env" $env "defaultArenas" 1) | indent 8 -}} - name: "SENTINEL" value: "/tmp/{{ $commonCtx.thorAgentName }}.sentinel" {{- include "hpcc.addSecurityContext" $commonCtx | indent 8 }} diff --git a/testing/helm/run.sh b/testing/helm/run.sh index 27db917d9f6..d2ec9212ab0 100755 --- a/testing/helm/run.sh +++ b/testing/helm/run.sh @@ -71,7 +71,7 @@ if type kubeval >/dev/null 2> /dev/null; then fi if type kube-score >/dev/null 2> /dev/null; then - echo Running kube-score... + echo Running $(kube-score version) # Note we force all replicas to be > 1 as some checks are not done on replicas=1 cases e.g. antiaffinity helm template $hpccchart ${options} | sed "s/replicas: 1/replicas: 2/" | \ kube-score score --output-format ci \ @@ -82,7 +82,7 @@ if type kube-score >/dev/null 2> /dev/null; then - >results.txt 2>errors.txt if [ $? -ne 0 ] then - echo $file failed + echo Running kube-score failed cat errors.txt cat results.txt failed=1 From 0a6a75107c0b1cc6c7d3d22e91afa426a8787a13 Mon Sep 17 00:00:00 2001 From: Gavin Halliday Date: Mon, 19 Aug 2024 11:37:38 +0100 Subject: [PATCH 21/33] HPCC-32410 Avoid crash if roxie detects a dali disconnect when reloading queries Signed-off-by: Gavin Halliday --- dali/base/dacsds.cpp | 60 +++++++++++++++++++++++++++++++++----------- dali/base/dacsds.ipp | 1 + dali/base/dasds.cpp | 6 +++++ dali/base/dasds.hpp | 1 + 4 files changed, 53 insertions(+), 15 deletions(-) diff --git a/dali/base/dacsds.cpp b/dali/base/dacsds.cpp index 3a23fdf5e4e..8577ad9ac6f 100644 --- a/dali/base/dacsds.cpp +++ b/dali/base/dacsds.cpp @@ -38,8 +38,6 @@ static unsigned clientThrottleLimit; static unsigned clientThrottleDelay; -static ISDSManager *SDSManager=NULL; - static CriticalSection SDScrit; #define CHECK_CONNECTED(XSTR) \ @@ -1290,6 +1288,12 @@ CClientSDSManager::CClientSDSManager() } CClientSDSManager::~CClientSDSManager() +{ + closedown(); + ::Release(properties); +} + +void CClientSDSManager::closedown() { CriticalBlock block(connections.crit); SuperHashIteratorOf iter(connections.queryBaseTable()); @@ -1298,7 +1302,6 @@ CClientSDSManager::~CClientSDSManager() CRemoteConnection &conn = (CRemoteConnection &) iter.query(); conn.setConnected(false); } - ::Release(properties); } bool CClientSDSManager::sendRequest(CMessageBuffer &mb, bool throttle) @@ -2236,32 +2239,59 @@ bool CClientSDSManager::updateEnvironment(IPropertyTree *newEnv, bool forceGroup ////////////// +static ISDSManager * activeSDSManager=NULL; +static ISDSManager * savedSDSManager=NULL; + +MODULE_INIT(INIT_PRIORITY_STANDARD) +{ + return true; +} +MODULE_EXIT() +{ + delete activeSDSManager; + activeSDSManager = nullptr; + delete savedSDSManager; + savedSDSManager = nullptr; +} + ISDSManager &querySDS() { CriticalBlock block(SDScrit); - if (SDSManager) - return *SDSManager; + if (activeSDSManager) + return *activeSDSManager; else if (!queryCoven().inCoven()) { - if (!SDSManager) - SDSManager = new CClientSDSManager(); + if (!activeSDSManager) + activeSDSManager = new CClientSDSManager(); - return *SDSManager; + return *activeSDSManager; } else { - SDSManager = &querySDSServer(); - return *SDSManager; + activeSDSManager = &querySDSServer(); + return *activeSDSManager; } } void closeSDS() { CriticalBlock block(SDScrit); - if (SDSManager) { + + //In roxie this is called when connection to dali is lost, but other threads can still be processing + //CRemoteConnections (see HPCC-32410), which uses an ISDSManager member - accessing a stale manager. + //There can be similar issues at closedown if threads have not been cleaned up properly. + //Do not delete the active SDS manager immediately - save it so that it is deleted on the next call/closedown. + ISDSManager * toDelete = savedSDSManager; + savedSDSManager = activeSDSManager; + activeSDSManager = nullptr; + if (savedSDSManager || toDelete) + { assertex(!queryCoven().inCoven()); // only called by client - try { - delete SDSManager; + try + { + if (savedSDSManager) + savedSDSManager->closedown(); + delete toDelete; } catch (IMP_Exception *e) { @@ -2270,11 +2300,11 @@ void closeSDS() EXCLOG(e, "closeSDS"); e->Release(); } - catch (IDaliClient_Exception *e) { + catch (IDaliClient_Exception *e) + { if (e->errorCode()!=DCERR_server_closed) throw; e->Release(); } - SDSManager = NULL; } } diff --git a/dali/base/dacsds.ipp b/dali/base/dacsds.ipp index e4b9590b955..3bda3dc649d 100644 --- a/dali/base/dacsds.ipp +++ b/dali/base/dacsds.ipp @@ -418,6 +418,7 @@ public: virtual void setConfigOpt(const char *opt, const char *value); virtual unsigned queryCount(const char *xpath); virtual bool updateEnvironment(IPropertyTree *newEnv, bool forceGroupUpdate, StringBuffer &response); + virtual void closedown() override; private: void noteDisconnected(CRemoteConnection &connection); diff --git a/dali/base/dasds.cpp b/dali/base/dasds.cpp index e5daa59a8a3..89caaa9efc5 100644 --- a/dali/base/dasds.cpp +++ b/dali/base/dasds.cpp @@ -2031,6 +2031,7 @@ class CCovenSDSManager : public CSDSManagerBase, implements ISDSManagerServer, i virtual void setConfigOpt(const char *opt, const char *value); virtual unsigned queryCount(const char *xpath); virtual bool updateEnvironment(IPropertyTree *newEnv, bool forceGroupUpdate, StringBuffer &response); + virtual void closedown() override; // ISubscriptionManager impl. virtual void add(ISubscription *subs,SubscriptionId id); @@ -6062,6 +6063,11 @@ CCovenSDSManager::~CCovenSDSManager() config.Release(); } +void CCovenSDSManager::closedown() +{ + //Should never be called - but do not assert since it is harmless and it is better not to report +} + void CCovenSDSManager::validateDeltaBackup() { // check consistency of delta diff --git a/dali/base/dasds.hpp b/dali/base/dasds.hpp index 91a9e6b5048..1a114271bcf 100644 --- a/dali/base/dasds.hpp +++ b/dali/base/dasds.hpp @@ -118,6 +118,7 @@ interface ISDSManager virtual void setConfigOpt(const char *opt, const char *value) = 0; virtual unsigned queryCount(const char *xpath) = 0; virtual bool updateEnvironment(IPropertyTree *newEnv, bool forceGroupUpdate, StringBuffer &response) = 0; + virtual void closedown() = 0; }; extern da_decl const char *queryNotifyHandlerName(IPropertyTree *tree); From e1627ec0cb480d24fe85477eb0f722568d9be43d Mon Sep 17 00:00:00 2001 From: Gavin Halliday Date: Tue, 20 Aug 2024 14:50:44 +0100 Subject: [PATCH 22/33] HPCC-32466 Protect dali against a crash processing a zero length packet Signed-off-by: Gavin Halliday --- system/jlib/jbuff.cpp | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/system/jlib/jbuff.cpp b/system/jlib/jbuff.cpp index e1e535a087c..dc95a5129a4 100644 --- a/system/jlib/jbuff.cpp +++ b/system/jlib/jbuff.cpp @@ -62,12 +62,9 @@ constexpr unsigned BUFF_DOUBLE_LIMIT=4096; constexpr unsigned BUFF_FIRST_CHUNK_SIZE=8; constexpr unsigned BUFF_DETACH_GRANULARITY=16; - -#ifdef _DEBUG +//Always check the length before reading - so ensure that serialization problems are caught +//The overhead is trivial.... #define CHECKREADPOS(len) assertex(readPos+(len)<=length()) -#else -#define CHECKREADPOS(len) -#endif //----------------------------------------------------------------------- From fb4f76dd7be15b6fa1659638fda249848fcd111c Mon Sep 17 00:00:00 2001 From: Jake Smith Date: Wed, 21 Aug 2024 17:22:37 +0100 Subject: [PATCH 23/33] HPCC-32486 Use new compressing lookahead classes for HD spilling Signed-off-by: Jake Smith --- .../hashdistrib/thhashdistribslave.cpp | 15 ++++++++++++++- thorlcr/activities/thactivityutil.cpp | 16 +--------------- thorlcr/thorutil/thbuf.cpp | 16 ++++++++++++++++ thorlcr/thorutil/thbuf.hpp | 1 + 4 files changed, 32 insertions(+), 16 deletions(-) diff --git a/thorlcr/activities/hashdistrib/thhashdistribslave.cpp b/thorlcr/activities/hashdistrib/thhashdistribslave.cpp index da214c0f16a..f34d124b803 100644 --- a/thorlcr/activities/hashdistrib/thhashdistribslave.cpp +++ b/thorlcr/activities/hashdistrib/thhashdistribslave.cpp @@ -1072,6 +1072,8 @@ class CDistributorBase : implements IHashDistributor, implements IExceptionHandl StringAttr id; // for tracing ICompressHandler *compressHandler; StringBuffer compressOptions; + LookAheadOptions options; + bool newLookAhead = false; public: IMPLEMENT_IINTERFACE_USING(CInterface); @@ -1126,6 +1128,10 @@ class CDistributorBase : implements IHashDistributor, implements IExceptionHandl ::ActPrintLog(activity, thorDetailedLogLevel, "inputBufferSize : %d, bucketSendSize = %d, pullBufferSize=%d", inputBufferSize, bucketSendSize, pullBufferSize); targetWriterLimit = activity->getOptUInt(THOROPT_HDIST_TARGETWRITELIMIT); ::ActPrintLog(activity, thorDetailedLogLevel, "targetWriterLimit : %d", targetWriterLimit); + + newLookAhead = activity->getOptBool("newlookahead", false); + if (newLookAhead) + populateLookAheadOptions(*activity, options); } virtual void beforeDispose() @@ -1187,7 +1193,14 @@ class CDistributorBase : implements IHashDistributor, implements IExceptionHandl { StringBuffer temp; GetTempFilePath(temp,"hddrecvbuff"); - piperd.setown(createSmartBuffer(activity, temp.str(), pullBufferSize, rowIf)); + if (newLookAhead) + { + options.totalCompressionBufferSize = pullBufferSize; // hd option overrides defaults + ICompressHandler *compressHandler = pullBufferSize ? queryDefaultCompressHandler() : nullptr; + piperd.setown(createCompressedSpillingRowStream(activity, temp.str(), false, rowIf, options, compressHandler)); + } + else + piperd.setown(createSmartBuffer(activity, temp.str(), pullBufferSize, rowIf)); } else piperd.setown(createSmartInMemoryBuffer(activity, rowIf, pullBufferSize)); diff --git a/thorlcr/activities/thactivityutil.cpp b/thorlcr/activities/thactivityutil.cpp index fe9a960bb74..2beb728f227 100644 --- a/thorlcr/activities/thactivityutil.cpp +++ b/thorlcr/activities/thactivityutil.cpp @@ -224,21 +224,7 @@ class CRowStreamLookAhead : public CSimpleInterfaceOf allowspill = true; } - // for "newlookahead" only - if (isContainerized()) - { - // JCSMORE - add CJobBase::getTempBlockSize() to calc. once. - StringBuffer planeName; - if (!getDefaultPlane(planeName, "@tempPlane", "temp")) - getDefaultPlane(planeName, "@spillPlane", "spill"); - size32_t blockedSequentialIOSize = getPlaneAttributeValue(planeName, BlockedSequentialIO, (size32_t)-1); - if ((size32_t)-1 != blockedSequentialIOSize) - options.storageBlockSize = blockedSequentialIOSize; - } - options.totalCompressionBufferSize = activity.getOptInt(THOROPT_LOOKAHEAD_COMPRESSIONTOTALK, options.totalCompressionBufferSize / 1024) * 1024; - options.inMemMaxMem = activity.getOptInt(THOROPT_LOOKAHEAD_MAXROWMEMK, options.inMemMaxMem / 1024) * 1024; - options.writeAheadSize = activity.getOptInt64(THOROPT_LOOKAHEAD_WRITEAHEADK, options.writeAheadSize / 1024) * 1024; - options.tempFileGranularity = activity.getOptInt64(THOROPT_LOOKAHEAD_TEMPFILE_GRANULARITY, options.tempFileGranularity / 0x100000) * 0x100000; + populateLookAheadOptions(activity, options); } ~CRowStreamLookAhead() { diff --git a/thorlcr/thorutil/thbuf.cpp b/thorlcr/thorutil/thbuf.cpp index 2e45c7c78d8..7c5d533fa77 100644 --- a/thorlcr/thorutil/thbuf.cpp +++ b/thorlcr/thorutil/thbuf.cpp @@ -2957,3 +2957,19 @@ class CRCFileStream: public CSimpleInterface, implements IFileIOStream } }; +void populateLookAheadOptions(CActivityBase &activity, LookAheadOptions &options) +{ + if (isContainerized()) + { + StringBuffer planeName; + if (!getDefaultPlane(planeName, "@tempPlane", "temp")) + getDefaultPlane(planeName, "@spillPlane", "spill"); + size32_t blockedSequentialIOSize = getPlaneAttributeValue(planeName, BlockedSequentialIO, (size32_t)-1); + if ((size32_t)-1 != blockedSequentialIOSize) + options.storageBlockSize = blockedSequentialIOSize; + } + options.totalCompressionBufferSize = activity.getOptInt(THOROPT_LOOKAHEAD_COMPRESSIONTOTALK, options.totalCompressionBufferSize / 1024) * 1024; + options.inMemMaxMem = activity.getOptInt(THOROPT_LOOKAHEAD_MAXROWMEMK, options.inMemMaxMem / 1024) * 1024; + options.writeAheadSize = activity.getOptInt64(THOROPT_LOOKAHEAD_WRITEAHEADK, options.writeAheadSize / 1024) * 1024; + options.tempFileGranularity = activity.getOptInt64(THOROPT_LOOKAHEAD_TEMPFILE_GRANULARITY, options.tempFileGranularity / 0x100000) * 0x100000; +} diff --git a/thorlcr/thorutil/thbuf.hpp b/thorlcr/thorutil/thbuf.hpp index fb5a66af8fa..69a28f04824 100644 --- a/thorlcr/thorutil/thbuf.hpp +++ b/thorlcr/thorutil/thbuf.hpp @@ -132,5 +132,6 @@ interface IRowMultiWriterReader : extends IRowStream #define DEFAULT_WR_WRITE_GRANULARITY 1000 // Amount writers buffer up before committing to output extern graph_decl IRowMultiWriterReader *createSharedWriteBuffer(CActivityBase *activity, IThorRowInterfaces *rowif, unsigned limit, unsigned readGranularity=DEFAULT_WR_READ_GRANULARITY, unsigned writeGranularity=DEFAULT_WR_WRITE_GRANULARITY); +extern graph_decl void populateLookAheadOptions(CActivityBase &activity, LookAheadOptions &options); #endif From 5884fe91e1d434cc728d97b09123c13cbd3cb800 Mon Sep 17 00:00:00 2001 From: Jake Smith Date: Tue, 13 Aug 2024 14:47:01 +0100 Subject: [PATCH 24/33] HPCC-32421 Suppress foreign groups Prevent foreign groups from being published to Dali. The code used to try to find a matching group (for naming purposes), and then create a unique one. What made it worse, was that if the foreign group was from an AKS system, it would have ports associated with it, and the published groups never have ports, consequently it never matched and always added a new unique (anon) group. This caused a gradual build up of new groups (under /Group), which cause delays for everything that looked up a group elsewhere. Signed-off-by: Jake Smith --- dali/base/dadfs.cpp | 20 ++++++++++++-- dali/base/dafdesc.cpp | 63 +++++++++++++++++++++++++------------------ dali/base/dafdesc.hpp | 6 +++-- 3 files changed, 59 insertions(+), 30 deletions(-) diff --git a/dali/base/dadfs.cpp b/dali/base/dadfs.cpp index 48ec2a1bb63..faa42e68a9e 100644 --- a/dali/base/dadfs.cpp +++ b/dali/base/dadfs.cpp @@ -3839,6 +3839,9 @@ protected: friend class CDistributedFilePart; unsigned nc = fdesc->numClusters(); if (nc) { + unsigned flags = 0; + if (FileDescriptorFlags::none != (FileDescriptorFlags::foreign & fdesc->getFlags())) + flags = IFDSF_FOREIGN_GROUP; for (unsigned i=0;iqueryClusterGroup(i), fdesc->queryPartDiskMapping(i), - &queryNamedGroupStore() + &queryNamedGroupStore(), + flags ); if (!cluster->queryGroup(&queryNamedGroupStore())) @@ -7466,8 +7470,20 @@ class CNamedGroupIterator: implements INamedGroupIterator, public CInterface public: IMPLEMENT_IINTERFACE; CNamedGroupIterator(IRemoteConnection *_conn,IGroup *_matchgroup=NULL,bool _exactmatch=false) - : conn(_conn), matchgroup(_matchgroup) + : conn(_conn) { + if (_matchgroup) + { + // the matchgroup may contain ports, but they are never part of published groups and are not to be used for matching + SocketEndpointArray epa; + for (unsigned i=0; i<_matchgroup->ordinality(); i++) + { + SocketEndpoint ep = _matchgroup->queryNode(i).endpoint(); + ep.port = 0; + epa.append(ep); + } + matchgroup.setown(createIGroup(epa)); + } exactmatch = _exactmatch; pe.setown(conn->queryRoot()->getElements("Group")); } diff --git a/dali/base/dafdesc.cpp b/dali/base/dafdesc.cpp index 825833e086a..b03967a19c2 100644 --- a/dali/base/dafdesc.cpp +++ b/dali/base/dafdesc.cpp @@ -417,7 +417,7 @@ struct CClusterInfo: implements IClusterInfo, public CInterface name.clear(); } StringBuffer gname; - if (resolver->find(group,gname,true)||(group->ordinality()>1)) + if (resolver->find(group,gname,!foreignGroup)||(group->ordinality()>1)) name.set(gname); } } @@ -451,11 +451,13 @@ struct CClusterInfo: implements IClusterInfo, public CInterface checkClusterName(resolver); } - CClusterInfo(const char *_name,IGroup *_group,const ClusterPartDiskMapSpec &_mspec,INamedGroupStore *resolver) + CClusterInfo(const char *_name,IGroup *_group,const ClusterPartDiskMapSpec &_mspec,INamedGroupStore *resolver,unsigned flags) : name(_name),group(_group) { name.toLowerCase(); mspec =_mspec; + if (flags & IFDSF_FOREIGN_GROUP) + foreignGroup = true; checkClusterName(resolver); checkStriped(); } @@ -617,9 +619,10 @@ struct CClusterInfo: implements IClusterInfo, public CInterface IClusterInfo *createClusterInfo(const char *name, IGroup *grp, const ClusterPartDiskMapSpec &mspec, - INamedGroupStore *resolver) + INamedGroupStore *resolver, + unsigned flags) { - return new CClusterInfo(name,grp,mspec,resolver); + return new CClusterInfo(name,grp,mspec,resolver,flags); } IClusterInfo *deserializeClusterInfo(MemoryBuffer &mb, INamedGroupStore *resolver) @@ -1281,33 +1284,39 @@ class CFileDescriptor: public CFileDescriptorBase, implements ISuperFileDescrip cluster->getReplicateDir(repDir, os); setReplicateFilename(fullpath,queryDrive(idx,copy),baseDir.str(),repDir.str()); - const char *planeName = cluster->queryGroupName(); - if (!isEmptyString(planeName)) + // The following code manipulates the directory for striping and aliasing if necessary. + // To do so, it needs the plane details. + // Normally, the plane name is obtained from IClusterInfo, however, if this file is foreign, + // then the IClusterInfo's will have no resolved names (aka groups) because the remote groups + // don't exist in the client environment. Instead, if the foreign file came from k8s, it will + // have remoteStoragePlane serialized/set. + Owned plane; + if (remoteStoragePlane) + plane.set(remoteStoragePlane); + else { -#ifdef _CONTAINERIZED - Owned plane = getDataStoragePlane(planeName, false); -#else - Owned plane = remoteStoragePlane.getLink(); -#endif - if (plane) + const char *planeName = cluster->queryGroupName(); + if (!isEmptyString(planeName)) + plane.setown(getDataStoragePlane(planeName, false)); + } + if (plane) + { + StringBuffer planePrefix(plane->queryPrefix()); + Owned alias = plane->getAliasMatch(accessMode); + if (alias) { - StringBuffer planePrefix(plane->queryPrefix()); - Owned alias = plane->getAliasMatch(accessMode); - if (alias) + StringBuffer tmp; + StringBuffer newPlanePrefix(alias->queryPrefix()); + if (setReplicateDir(fullpath, tmp, false, planePrefix, newPlanePrefix)) { - StringBuffer tmp; - StringBuffer newPlanePrefix(alias->queryPrefix()); - if (setReplicateDir(fullpath, tmp, false, planePrefix, newPlanePrefix)) - { - planePrefix.swapWith(newPlanePrefix); - fullpath.swapWith(tmp); - } + planePrefix.swapWith(newPlanePrefix); + fullpath.swapWith(tmp); } - StringBuffer stripeDir; - addStripeDirectory(stripeDir, fullpath, planePrefix, idx, lfnHash, cluster->queryPartDiskMapping().numStripedDevices); - if (!stripeDir.isEmpty()) - fullpath.swapWith(stripeDir); } + StringBuffer stripeDir; + addStripeDirectory(stripeDir, fullpath, planePrefix, idx, lfnHash, cluster->queryPartDiskMapping().numStripedDevices); + if (!stripeDir.isEmpty()) + fullpath.swapWith(stripeDir); } } @@ -1633,6 +1642,8 @@ class CFileDescriptor: public CFileDescriptorBase, implements ISuperFileDescrip attr.setown(createPTreeFromIPT(at)); else attr.setown(createPTree("Attr")); + if (flags & IFDSF_FOREIGN_GROUP) + setFlags(static_cast(fileFlags | FileDescriptorFlags::foreign)); if (attr->hasProp("@lfnHash")) // potentially missing for meta coming from a legacy Dali lfnHash = attr->getPropInt("@lfnHash"); else if (tracename.length()) diff --git a/dali/base/dafdesc.hpp b/dali/base/dafdesc.hpp index c88a6adb34e..535178828e2 100644 --- a/dali/base/dafdesc.hpp +++ b/dali/base/dafdesc.hpp @@ -190,7 +190,8 @@ typedef IIteratorOf IPartDescriptorIterator; enum class FileDescriptorFlags { none = 0x00, - dirperpart = 0x01 + dirperpart = 0x01, + foreign = 0x02 }; BITMASK_ENUM(FileDescriptorFlags); @@ -351,7 +352,8 @@ interface IStoragePlane: extends IInterface IClusterInfo *createClusterInfo(const char *grpname, // NULL if roxie label set IGroup *grp, const ClusterPartDiskMapSpec &mspec, - INamedGroupStore *resolver=NULL + INamedGroupStore *resolver=NULL, + unsigned flags=0 ); IClusterInfo *createRoxieClusterInfo(const char *label, const ClusterPartDiskMapSpec &mspec From 939af2176d0683845c27c2cc68b650f0626c7751 Mon Sep 17 00:00:00 2001 From: Jack Del Vecchio Date: Thu, 22 Aug 2024 13:18:25 +0000 Subject: [PATCH 25/33] HPCC-32491 HThor crashes reading local files with forceRemoteRead --- ecl/hthor/hthor.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ecl/hthor/hthor.cpp b/ecl/hthor/hthor.cpp index ee9dc1bc5ab..166cea2c0a7 100644 --- a/ecl/hthor/hthor.cpp +++ b/ecl/hthor/hthor.cpp @@ -8668,7 +8668,8 @@ bool CHThorDiskReadBaseActivity::openNext() StringBuffer tmp; remoteFileIO->addVirtualFieldMapping("logicalFilename", logicalFileName.str()); remoteFileIO->addVirtualFieldMapping("baseFpos", tmp.clear().append(offsetOfPart).str()); - remoteFileIO->addVirtualFieldMapping("partNum", tmp.clear().append(curPart->getPartIndex()).str()); + if (curPart) + remoteFileIO->addVirtualFieldMapping("partNum", tmp.clear().append(curPart->getPartIndex()).str()); try { From 5d8b5863988966133749dad3ca396797b8e932d8 Mon Sep 17 00:00:00 2001 From: Jake Smith Date: Thu, 22 Aug 2024 15:27:25 +0100 Subject: [PATCH 26/33] HPCC-32495 Rename splitterSpill option Also rename the force* (compression) options and compressSortOverflow compressAllOutputs Signed-off-by: Jake Smith --- thorlcr/thorutil/thormisc.hpp | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/thorlcr/thorutil/thormisc.hpp b/thorlcr/thorutil/thormisc.hpp index 4d6ffabe197..4e85c73d6f5 100644 --- a/thorlcr/thorutil/thormisc.hpp +++ b/thorlcr/thorutil/thormisc.hpp @@ -54,7 +54,7 @@ #define THOROPT_HDIST_TARGETWRITELIMIT "hdTargetLimit" // Limit # of writer threads working on a single target (default = unbound, but picks round-robin) #define THOROPT_HDIST_COMP "v9_4_hdCompressorType" // Distribute compressor to use (default = "LZ4") #define THOROPT_HDIST_COMPOPTIONS "v9_4_hdCompressorOptions" // Distribute compressor options, e.g. AES key (default = "") -#define THOROPT_SPLITTER_SPILL "splitterSpill" // Force splitters to spill or not, default is to adhere to helper setting (default = -1) +#define THOROPT_SPLITTER_SPILL "v9_4_splitterSpill" // Force splitters to spill or not, default is to adhere to helper setting (default = -1) #define THOROPT_LOOP_MAX_EMPTY "loopMaxEmpty" // Max # of iterations that LOOP can cycle through with 0 results before errors (default = 1000) #define THOROPT_SMALLSORT "smallSortThreshold" // Use minisort approach, if estimate size of data to sort is below this setting (default = 0) #define THOROPT_PARALLEL_FUNNEL "parallelFunnel" // Use parallel funnel impl. if !ordered (default = true) @@ -66,10 +66,10 @@ #define THOROPT_LKJOIN_LOCALFAILOVER "lkjoin_localfailover" // Force SMART to failover to distributed local lookup join (for testing only) (default = false) #define THOROPT_LKJOIN_HASHJOINFAILOVER "lkjoin_hashjoinfailover" // Force SMART to failover to hash join (for testing only) (default = false) #define THOROPT_MAX_KERNLOG "max_kern_level" // Max kernel logging level, to push to workunit, -1 to disable (default = 3) -#define THOROPT_COMP_FORCELZW "forceLZW" // Forces file compression to use LZW (default = false) -#define THOROPT_COMP_FORCEFLZ "forceFLZ" // Forces file compression to use FLZ (default = false) -#define THOROPT_COMP_FORCELZ4 "forceLZ4" // Forces file compression to use LZ4 (default = false) -#define THOROPT_COMP_FORCELZ4HC "forceLZ4HC" // Forces file compression to use LZ4HC (default = false) +#define THOROPT_COMP_FORCELZW "v9_4_forceLZW" // Forces file compression to use LZW (default = false) +#define THOROPT_COMP_FORCEFLZ "v9_4_forceFLZ" // Forces file compression to use FLZ (default = false) +#define THOROPT_COMP_FORCELZ4 "v9_4_forceLZ4" // Forces file compression to use LZ4 (default = false) +#define THOROPT_COMP_FORCELZ4HC "v9_4_forceLZ4HC" // Forces file compression to use LZ4HC (default = false) #define THOROPT_TRACE_ENABLED "traceEnabled" // Output from TRACE activity enabled (default = false) #define THOROPT_TRACE_LIMIT "traceLimit" // Number of rows from TRACE activity (default = 10) #define THOROPT_READ_CRC "crcReadEnabled" // Enabled CRC validation on disk reads if file CRC are available (default = true) @@ -105,7 +105,7 @@ #define THOROPT_VALIDATE_FILE_TYPE "validateFileType" // validate file type compatibility, e.g. if on fire error if XML reading CSV (default = true) #define THOROPT_MIN_REMOTE_CQ_INDEX_SIZE_MB "minRemoteCQIndexSizeMb" // minimum size of index file to enable server side handling (default = 0, meaning use heuristic to determin) #define THOROPT_KJ_ASSUME_PRIMARY "keyedJoinAssumePrimary" // assume primary part exists (don't check when mapping, which can be slow) -#define THOROPT_COMPRESS_SORTOVERFLOW "compressSortOverflow" // If global sort spills, compress the merged overflow file (default = true) +#define THOROPT_COMPRESS_SORTOVERFLOW "v9_4_compressSortOverflow" // If global sort spills, compress the merged overflow file (default = true) #define THOROPT_TIME_ACTIVITIES "timeActivities" // Time activities (default=true) #define THOROPT_MAX_ACTIVITY_CORES "maxActivityCores" // controls number of default threads to use for very parallel phases (like sort/parallel join helper). (default = # of h/w cores) #define THOROPT_THOR_ROWCRC "THOR_ROWCRC" // Use a CRC checking row allocator (default=false) @@ -114,7 +114,7 @@ #define THOROPT_FAIL_ON_LEAKS "failOnLeaks" // If any leaks are detected at the end of graph, fail the query (default=false) #define THOROPT_SOAP_TRACE_LEVEL "soapTraceLevel" // The trace SOAP level (default=1) #define THOROPT_SORT_ALGORITHM "sortAlgorithm" // The algorithm used to sort records (quicksort/mergesort) -#define THOROPT_COMPRESS_ALLFILES "compressAllOutputs" // Compress all output files (default: bare-metal=off, cloud=on) +#define THOROPT_COMPRESS_ALLFILES "v9_4_compressAllOutputs" // Compress all output files (default: bare-metal=off, cloud=on) #define THOROPT_AVOID_RENAME "avoidRename" // Avoid rename, write directly to target physical filenames (no temp file) From 9e7f3d7e87cf9603adf9dffaa7875559f7bbbef1 Mon Sep 17 00:00:00 2001 From: Jeremy Clements <79224539+jeclrsg@users.noreply.github.com> Date: Thu, 22 Aug 2024 10:54:59 -0400 Subject: [PATCH 27/33] HPCC-32474 ECL Watch v9 allow selection of grid row contents modifies the style of FluentUI grid rows to allow selection of text Signed-off-by: Jeremy Clements <79224539+jeclrsg@users.noreply.github.com> --- esp/src/src-react/components/controls/Grid.tsx | 1 + 1 file changed, 1 insertion(+) diff --git a/esp/src/src-react/components/controls/Grid.tsx b/esp/src/src-react/components/controls/Grid.tsx index 7a445acac93..c781322a0f2 100644 --- a/esp/src/src-react/components/controls/Grid.tsx +++ b/esp/src/src-react/components/controls/Grid.tsx @@ -122,6 +122,7 @@ const gridStyles = (height: string): Partial => { maxHeight: height, selectors: { ".ms-DetailsHeader-cellName": { fontSize: "13.5px" }, + ".ms-DetailsRow": { userSelect: "text" }, ".ms-DetailsRow-cell:has(.bgFilled)": { color: "white", boxShadow: "inset 1px 0 var(--colorNeutralBackground1), inset -1px 1px var(--colorNeutralBackground1)" }, ".ms-DetailsRow-cell:has(.bgGreen)": { background: "green" }, ".ms-DetailsRow-cell:has(.bgOrange)": { background: "orange" }, From 8be1a41cc21663d5c34cb80c377bc8d403dd0468 Mon Sep 17 00:00:00 2001 From: Jack Del Vecchio Date: Thu, 22 Aug 2024 16:07:42 +0000 Subject: [PATCH 28/33] HPCC-32507 Parquet Plugin not reading Arrow Partitioned files --- plugins/parquet/parquet.ecllib | 2 +- plugins/parquet/parquetembed.cpp | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/plugins/parquet/parquet.ecllib b/plugins/parquet/parquet.ecllib index 66529ac50dc..48f26678058 100644 --- a/plugins/parquet/parquet.ecllib +++ b/plugins/parquet/parquet.ecllib @@ -43,7 +43,7 @@ EXPORT ParquetIO := MODULE ENDMACRO; EXPORT Read(resultLayout, basePath, partitionFieldList) := FUNCTIONMACRO - LOCAL STREAMED DATASET(resultLayout) _DoParquetReadPartition() := EMBED(parquet: activity, option('readdirectorypartition'), location(basePath)), partitionFields(partitionFieldList) + LOCAL STREAMED DATASET(resultLayout) _DoParquetReadPartition() := EMBED(parquet: activity, option('readdirectorypartition'), location(basePath), partitionFields(partitionFieldList)) ENDEMBED; RETURN _DoParquetReadPartition(); ENDMACRO; diff --git a/plugins/parquet/parquetembed.cpp b/plugins/parquet/parquetembed.cpp index 473f56e145d..107ade09d7f 100644 --- a/plugins/parquet/parquetembed.cpp +++ b/plugins/parquet/parquetembed.cpp @@ -462,6 +462,8 @@ __int64 ParquetReader::next(TableColumns *&nextTable) if (endsWithIgnoreCase(partOption.c_str(), "partition")) { PARQUET_ASSIGN_OR_THROW(table, queryRows()); // Sets rowsProcessed to current row in table corresponding to startRow + rowsCount = table->num_rows(); + splitTable(table); } else { From fef64a08cd1e3762cad6f5929d9f6f73e403567c Mon Sep 17 00:00:00 2001 From: "Dan S. Camper" Date: Wed, 7 Aug 2024 08:46:47 -0500 Subject: [PATCH 29/33] HPCC-32390 Add timer support to core regex helper code --- ecl/hql/hqlatoms.cpp | 2 + ecl/hql/hqlatoms.hpp | 1 + ecl/hql/hqlutil.cpp | 1 + ecl/hql/hqlutil.hpp | 2 +- ecl/hqlcpp/hqlcpp.cpp | 44 ++++++++++++-- ecl/hqlcpp/hqlcpp.ipp | 2 + ecl/hqlcpp/hqlcppsys.ecl | 30 +++++----- ecl/hqlcpp/hqlhtcpp.cpp | 12 +++- rtl/eclrtl/eclregex.cpp | 119 ++++++++++++++++++++++++++++++++------ rtl/eclrtl/eclrtl.hpp | 23 ++++++-- rtl/eclrtl/eclrtl_imp.hpp | 72 +++++++++++++++++++++++ rtl/include/eclhelper.hpp | 26 ++++++++- 12 files changed, 289 insertions(+), 45 deletions(-) diff --git a/ecl/hql/hqlatoms.cpp b/ecl/hql/hqlatoms.cpp index c88fc79fdbe..88aa73edc0a 100644 --- a/ecl/hql/hqlatoms.cpp +++ b/ecl/hql/hqlatoms.cpp @@ -452,6 +452,7 @@ IAtom * _threadlocal_Atom; IAtom * thresholdAtom; IAtom * throwAtom; IAtom * timeAtom; +IAtom * timerAtom; IAtom * timeoutAtom; IAtom * timeLimitAtom; IAtom * timestampAtom; @@ -933,6 +934,7 @@ MODULE_INIT(INIT_PRIORITY_HQLATOM) MAKEATOM(threshold); MAKEATOM(throw); MAKEATOM(time); + MAKEATOM(timer); MAKEATOM(timeout); MAKEATOM(timeLimit); MAKEATOM(timestamp); diff --git a/ecl/hql/hqlatoms.hpp b/ecl/hql/hqlatoms.hpp index 9aba3ba4c33..6f8bb00157a 100644 --- a/ecl/hql/hqlatoms.hpp +++ b/ecl/hql/hqlatoms.hpp @@ -457,6 +457,7 @@ extern HQL_API IAtom * _threadlocal_Atom; extern HQL_API IAtom * thresholdAtom; extern HQL_API IAtom * throwAtom; extern HQL_API IAtom * timeAtom; +extern HQL_API IAtom * timerAtom; extern HQL_API IAtom * timeoutAtom; extern HQL_API IAtom * timeLimitAtom; extern HQL_API IAtom * timestampAtom; diff --git a/ecl/hql/hqlutil.cpp b/ecl/hql/hqlutil.cpp index 86fb4d19742..a5dac66b852 100644 --- a/ecl/hql/hqlutil.cpp +++ b/ecl/hql/hqlutil.cpp @@ -3282,6 +3282,7 @@ bool isDependentOnParameter(IHqlExpression * expr) return checker.isDependent(expr); } +// Returns true only when we need to build timing mechanisms around the calls bool isTimed(IHqlExpression * expr) { switch (expr->getOperator()) diff --git a/ecl/hql/hqlutil.hpp b/ecl/hql/hqlutil.hpp index fec7673ddf0..96a74cfcf11 100644 --- a/ecl/hql/hqlutil.hpp +++ b/ecl/hql/hqlutil.hpp @@ -262,7 +262,7 @@ extern HQL_API IException * checkRegexSyntax(IHqlExpression * expr); inline bool isInternalEmbedAttr(IAtom *name) { return name == languageAtom || name == projectedAtom || name == streamedAtom || name == _linkCounted_Atom || - name == importAtom || name==foldAtom || name==timeAtom || name==_prebind_Atom || name==_precompile_Atom || name==_original_Atom || + name == importAtom || name==foldAtom || name==timeAtom || name==timerAtom || name==_prebind_Atom || name==_precompile_Atom || name==_original_Atom || name == _threadlocal_Atom || name == _singletonEmbedContext_Atom || name == activityAtom || name == localAtom || name == parallelAtom; } diff --git a/ecl/hqlcpp/hqlcpp.cpp b/ecl/hqlcpp/hqlcpp.cpp index 3189ca0955c..9dc01e7e5da 100644 --- a/ecl/hqlcpp/hqlcpp.cpp +++ b/ecl/hqlcpp/hqlcpp.cpp @@ -2232,19 +2232,40 @@ void HqlCppTranslator::callProcedure(BuildCtx & ctx, IIdAtom * name, HqlExprArra CHqlBoundExpr boundTimer, boundStart; IHqlExpression * external = funcdef->queryChild(0); - bool isTimed = external->hasAttribute(timeAtom); - if (isTimed) + bool needsStartStopTimer = external->hasAttribute(timeAtom); + if (needsStartStopTimer) { StringBuffer nameTemp; - const char * name = str(external->queryId()); + const char * timerName = str(external->queryId()); if (getStringValue(nameTemp, queryAttributeChild(external, timeAtom, 0)).length()) - name = nameTemp; - buildStartTimer(ctx, boundTimer, boundStart, name); + timerName = nameTemp; + buildStartTimer(ctx, boundTimer, boundStart, timerName); + } + else if (external->hasAttribute(timerAtom)) + { + // Timing performed by helper, but we need to build the timer that will be passed to the helper + StringBuffer nameTemp; + const char * timerName = str(external->queryId()); + if (getStringValue(nameTemp, queryAttributeChild(external, timerAtom, 0)).length()) + timerName = nameTemp; + buildHelperTimer(ctx, boundTimer, timerName); + // We need to patch the args to include the timer; helper timers will always be the first argument + // passed to the helper function, which means that it is either the first argument in the + // array (for a bare function call) or the second argument (for a method call) + bool isMethod = external->hasAttribute(methodAtom) || external->hasAttribute(omethodAtom); + unsigned int timerPos = (isMethod ? 1 : 0); + HqlExprArray newArgs; + unwindChildren(newArgs, call); + if (timerPos < newArgs.length()) + newArgs.add(*LINK(boundTimer.expr), timerPos); + else + newArgs.append(*LINK(boundTimer.expr)); + call.setown(bindTranslatedFunctionCall(name, newArgs)); } ctx.addExpr(call); - if (isTimed) + if (needsStartStopTimer) { buildStopTimer(ctx, boundTimer, boundStart); } @@ -6098,6 +6119,17 @@ void HqlCppTranslator::doBuildCall(BuildCtx & ctx, const CHqlBoundTarget * tgt, buildExpr(ctx, expr->queryChild(firstParam++), bound); args.append(*bound.expr.getClear()); } + if (external->hasAttribute(timerAtom)) + { + // helper timers will always be the first argument passed to the helper function + CHqlBoundExpr boundTimer; + StringBuffer nameTemp; + const char * name = str(external->queryId()); + if (getStringValue(nameTemp, queryAttributeChild(external, timerAtom, 0)).length()) + name = nameTemp; + buildHelperTimer(ctx, boundTimer, name); + args.append(*LINK(boundTimer.expr)); + } if (external->hasAttribute(userMatchFunctionAtom)) { //MORE: Test valid in this location... diff --git a/ecl/hqlcpp/hqlcpp.ipp b/ecl/hqlcpp/hqlcpp.ipp index ce13dba6eef..388e25bde0c 100644 --- a/ecl/hqlcpp/hqlcpp.ipp +++ b/ecl/hqlcpp/hqlcpp.ipp @@ -1675,6 +1675,8 @@ public: IHqlExpression * getFirstCharacter(IHqlExpression * source); bool hasAddress(BuildCtx & ctx, IHqlExpression * expr); + void buildTimerBase(BuildCtx & ctx, CHqlBoundExpr & boundTimer, const char * name); + void buildHelperTimer(BuildCtx & ctx, CHqlBoundExpr & boundTimer, const char * name); void buildStartTimer(BuildCtx & ctx, CHqlBoundExpr & boundTimer, CHqlBoundExpr & boundStart, const char * name); void buildStopTimer(BuildCtx & ctx, const CHqlBoundExpr & boundTimer, const CHqlBoundExpr & boundStart); diff --git a/ecl/hqlcpp/hqlcppsys.ecl b/ecl/hqlcpp/hqlcppsys.ecl index 4919983331a..8938288bec1 100644 --- a/ecl/hqlcpp/hqlcppsys.ecl +++ b/ecl/hqlcpp/hqlcppsys.ecl @@ -553,29 +553,29 @@ const char * cppSystemText[] = { " unicode wregexReplaceX(unsigned8 compiled, const unicode text, const unicode replace) : eclrtl,pure,include='eclrtl.hpp',library='eclrtl',entrypoint='rtlWRegExprReplace';", " unicode wregexGetFindStr(unsigned8 compiled, unsigned4 idx): eclrtl,pure,include='eclrtl.hpp',library='eclrtl',entrypoint='rtlWRegExprGetFindStr';", - " regexNewSetStrPattern(const varstring _pattern, boolean isCaseSensitive) : omethod,entrypoint='setPattern',time('CompileStringRegex');" - " regexNewStrFind(boolean _compiled, const string _search, boolean _cloneSearch) : omethod,entrypoint='find',time('REGEXFIND');" + " regexNewSetStrPattern(const varstring _pattern, boolean isCaseSensitive) : omethod,entrypoint='setPatternTimed',timer('CompileStringRegex');" + " regexNewStrFind(boolean _compiled, const string _search, boolean _cloneSearch) : omethod,entrypoint='findTimed',timer('REGEXFIND');" " boolean regexNewStrFound() : method,pure,entrypoint='found';" " string regexNewStrFoundX(unsigned4 idx) : method,pure,entrypoint='getMatchX';" - " string regexNewStrReplaceX(const string _search, const string _replace) : method,pure,entrypoint='replace',time('REGEXREPLACE');" - " regexNewStrReplaceFixed(noconst string _tgt, const string _search, const string _replace) : method,pure,entrypoint='replaceFixed',time('REGEXREPLACE');" - " set of string regexMatchSet(const string _search) : method,pure,entrypoint='getMatchSet',time('REGEXFINDSET');" + " string regexNewStrReplaceX(const string _search, const string _replace) : method,pure,entrypoint='replaceTimed',timer('REGEXREPLACE');" + " regexNewStrReplaceFixed(noconst string _tgt, const string _search, const string _replace) : method,pure,entrypoint='replaceFixedTimed',timer('REGEXREPLACE');" + " set of string regexMatchSet(const string _search) : method,pure,entrypoint='getMatchSetTimed',timer('REGEXFINDSET');" - " regexNewSetUStrPattern(const varunicode _pattern, boolean isCaseSensitive) : omethod,entrypoint='setPattern',time('CompileUnicodeRegex');" - " regexNewUStrFind(boolean _compiled, const unicode _search) : omethod,entrypoint='find',time('REGEXFIND');" + " regexNewSetUStrPattern(const varunicode _pattern, boolean isCaseSensitive) : omethod,entrypoint='setPatternTimed',timer('CompileUnicodeRegex');" + " regexNewUStrFind(boolean _compiled, const unicode _search) : omethod,entrypoint='findTimed',timer('REGEXFIND');" " boolean regexNewUStrFound() : method,pure,entrypoint='found';" " unicode regexNewUStrFoundX(unsigned4 idx) : method,pure,entrypoint='getMatchX';" - " unicode regexNewUStrReplaceX(const unicode _search, const unicode _replace) : method,pure,entrypoint='replace',time('REGEXREPLACE');" - " regexNewUStrReplaceFixed(noconst unicode _tgt, const unicode _search, const unicode _replace) : method,pure,entrypoint='replaceFixed',time('REGEXREPLACE');" - " set of unicode regexUStrMatchSet(const unicode _search) : method,pure,entrypoint='getMatchSet',time('REGEXFINDSET');" + " unicode regexNewUStrReplaceX(const unicode _search, const unicode _replace) : method,pure,entrypoint='replaceTimed',timer('REGEXREPLACE');" + " regexNewUStrReplaceFixed(noconst unicode _tgt, const unicode _search, const unicode _replace) : method,pure,entrypoint='replaceFixedTimed',timer('REGEXREPLACE');" + " set of unicode regexUStrMatchSet(const unicode _search) : method,pure,entrypoint='getMatchSetTimed',timer('REGEXFINDSET');" - " regexNewSetU8StrPattern(const utf8 _pattern, boolean isCaseSensitive) : omethod,entrypoint='setPattern',time('CompileUTF8Regex');" - " regexNewU8StrFind(boolean _compiled, const utf8 _search, boolean _cloneSearch) : omethod,entrypoint='find',time('REGEXFIND');" + " regexNewSetU8StrPattern(const utf8 _pattern, boolean isCaseSensitive) : omethod,entrypoint='setPatternTimed',timer('CompileUTF8Regex');" + " regexNewU8StrFind(boolean _compiled, const utf8 _search, boolean _cloneSearch) : omethod,entrypoint='findTimed',timer('REGEXFIND');" " boolean regexNewU8StrFound() : method,pure,entrypoint='found';" " utf8 regexNewU8StrFoundX(unsigned4 idx) : method,pure,entrypoint='getMatchX';" - " utf8 regexNewU8StrReplaceX(const utf8 _search, const utf8 _replace) : method,pure,entrypoint='replace',time('REGEXREPLACE');" - " regexNewU8StrReplaceFixed(noconst utf8 _tgt, const utf8 _search, const utf8 _replace) : method,pure,entrypoint='replaceFixed',time('REGEXREPLACE');" - " set of utf8 regexU8StrMatchSet(const utf8 _search) : method,pure,entrypoint='getMatchSet',time('REGEXFINDSET');" + " utf8 regexNewU8StrReplaceX(const utf8 _search, const utf8 _replace) : method,pure,entrypoint='replaceTimed',timer('REGEXREPLACE');" + " regexNewU8StrReplaceFixed(noconst utf8 _tgt, const utf8 _search, const utf8 _replace) : method,pure,entrypoint='replaceFixedTimed',timer('REGEXREPLACE');" + " set of utf8 regexU8StrMatchSet(const utf8 _search) : method,pure,entrypoint='getMatchSetTimed',timer('REGEXFINDSET');" //clibrary functions that are called from the code generation " free(noconst data1 src) : eclrtl,library='eclrtl',entrypoint='rtlFree';", diff --git a/ecl/hqlcpp/hqlhtcpp.cpp b/ecl/hqlcpp/hqlhtcpp.cpp index 36abc0f758e..4904276a649 100644 --- a/ecl/hqlcpp/hqlhtcpp.cpp +++ b/ecl/hqlcpp/hqlhtcpp.cpp @@ -18789,7 +18789,7 @@ void HqlCppTranslator::doBuildExprRegexFindSet(BuildCtx & ctx, IHqlExpression * //--------------------------------------------------------------------------- -void HqlCppTranslator::buildStartTimer(BuildCtx & ctx, CHqlBoundExpr & boundTimer, CHqlBoundExpr & boundStart, const char * name) +void HqlCppTranslator::buildTimerBase(BuildCtx & ctx, CHqlBoundExpr & boundTimer, const char * name) { BuildCtx * initCtx = &ctx; BuildCtx * declareCtx = &ctx; @@ -18813,6 +18813,16 @@ void HqlCppTranslator::buildStartTimer(BuildCtx & ctx, CHqlBoundExpr & boundTime declareCtx->associateExpr(call, boundTimer); initCtx->addAssign(boundTimer.expr, call); } +} + +void HqlCppTranslator::buildHelperTimer(BuildCtx & ctx, CHqlBoundExpr & boundTimer, const char * name) +{ + buildTimerBase(ctx, boundTimer, name); +} + +void HqlCppTranslator::buildStartTimer(BuildCtx & ctx, CHqlBoundExpr & boundTimer, CHqlBoundExpr & boundStart, const char * name) +{ + buildTimerBase(ctx, boundTimer, name); HqlExprArray nowArgs; nowArgs.append(*boundTimer.getTranslatedExpr()); diff --git a/rtl/eclrtl/eclregex.cpp b/rtl/eclrtl/eclregex.cpp index 9e3714795f3..156e7339a33 100644 --- a/rtl/eclrtl/eclregex.cpp +++ b/rtl/eclrtl/eclregex.cpp @@ -26,6 +26,7 @@ #include "pcre2.h" #include "platform.h" +#include "eclhelper.hpp" #include "eclrtl.hpp" #include "eclrtl_imp.hpp" #include "jhash.hpp" @@ -287,7 +288,7 @@ static void initMaxCacheSize() //--------------------------------------------------------------------------- -class CStrRegExprFindInstance : implements IStrRegExprFindInstance +class CStrRegExprFindInstance final : implements IStrRegExprFindInstance { private: bool matched = false; @@ -362,7 +363,7 @@ class CStrRegExprFindInstance : implements IStrRegExprFindInstance } } - char const * findvstr(unsigned outlen, char * out, unsigned n = 0) + char const * findvstr(unsigned outlen, char * out, unsigned n) { if (matched && (n < pcre2_get_ovector_count_8(matchData))) { @@ -384,7 +385,7 @@ class CStrRegExprFindInstance : implements IStrRegExprFindInstance //--------------------------------------------------------------------------- -class CCompiledStrRegExpr : implements ICompiledStrRegExpr +class CCompiledStrRegExpr final : implements ICompiledStrRegExpr { private: std::shared_ptr compiledRegex = nullptr; @@ -417,7 +418,7 @@ class CCompiledStrRegExpr : implements ICompiledStrRegExpr //ICompiledStrRegExpr - void replace(size32_t & outlen, char * & out, size32_t slen, char const * str, size32_t rlen, char const * replace) const + void replace(size32_t & outlen, char * & out, size32_t slen, char const * str, size32_t rlen, char const * rstr) const { outlen = 0; PCRE2MatchData8 matchData = pcre2_match_data_create_from_pattern_8(compiledRegex.get(), pcre2GeneralContext8); @@ -426,7 +427,7 @@ class CCompiledStrRegExpr : implements ICompiledStrRegExpr // (slen and rlen) are in code points (characters), not bytes; we need to convert these to a // byte count for PCRE2 size32_t sourceSize = (isUTF8Enabled ? rtlUtf8Size(slen, str) : slen); - size32_t replaceSize = (isUTF8Enabled ? rtlUtf8Size(rlen, replace) : rlen); + size32_t replaceSize = (isUTF8Enabled ? rtlUtf8Size(rlen, rstr) : rlen); // Execute an explicit match first to see if we match at all; if we do, matchData will be populated // with data that can be used by pcre2_substitute to bypass some work @@ -445,7 +446,7 @@ class CCompiledStrRegExpr : implements ICompiledStrRegExpr // Call substitute once to get the size of the output (pushed into pcreSize); // note that pcreSize will include space for a terminating null character even though we don't want it - int replaceResult = pcre2_substitute_8(compiledRegex.get(), (PCRE2_SPTR8)str, sourceSize, 0, replaceOptions|PCRE2_SUBSTITUTE_OVERFLOW_LENGTH, matchData, pcre2MatchContext8, (PCRE2_SPTR8)replace, replaceSize, nullptr, &pcreSize); + int replaceResult = pcre2_substitute_8(compiledRegex.get(), (PCRE2_SPTR8)str, sourceSize, 0, replaceOptions|PCRE2_SUBSTITUTE_OVERFLOW_LENGTH, matchData, pcre2MatchContext8, (PCRE2_SPTR8)rstr, replaceSize, nullptr, &pcreSize); if (replaceResult < 0 && replaceResult != PCRE2_ERROR_NOMEMORY) { @@ -458,7 +459,7 @@ class CCompiledStrRegExpr : implements ICompiledStrRegExpr { out = (char *)rtlMalloc(pcreSize); - replaceResult = pcre2_substitute_8(compiledRegex.get(), (PCRE2_SPTR8)str, sourceSize, 0, replaceOptions, matchData, pcre2MatchContext8, (PCRE2_SPTR8)replace, replaceSize, (PCRE2_UCHAR8 *)out, &pcreSize); + replaceResult = pcre2_substitute_8(compiledRegex.get(), (PCRE2_SPTR8)str, sourceSize, 0, replaceOptions, matchData, pcre2MatchContext8, (PCRE2_SPTR8)rstr, replaceSize, (PCRE2_UCHAR8 *)out, &pcreSize); // Note that, weirdly, pcreSize will now contain the number of code points // in the result *excluding* the null terminator, so pcreSize will @@ -489,9 +490,15 @@ class CCompiledStrRegExpr : implements ICompiledStrRegExpr } } + void replaceTimed(ISectionTimer * timer, size32_t & outlen, char * & out, size32_t slen, char const * str, size32_t rlen, char const * rstr) const + { + SectionTimerBlock sectionTimer(timer); + replace(outlen, out, slen, str, rlen, rstr); + } + // This method supports "fixed length UTF-8" even though that isn't really a thing; // it's here more for completeness, in case we ever implement some version of it - void replaceFixed(size32_t tlen, char * tgt, size32_t slen, char const * str, size32_t rlen, char const * replace) const + void replaceFixed(size32_t tlen, char * tgt, size32_t slen, char const * str, size32_t rlen, char const * rstr) const { if (tlen == 0) return; @@ -502,7 +509,7 @@ class CCompiledStrRegExpr : implements ICompiledStrRegExpr // (slen and rlen) are in code points (characters), not bytes; we need to convert these to a // byte count for PCRE2 size32_t sourceSize = (isUTF8Enabled ? rtlUtf8Size(slen, str) : slen); - size32_t replaceSize = (isUTF8Enabled ? rtlUtf8Size(rlen, replace) : rlen); + size32_t replaceSize = (isUTF8Enabled ? rtlUtf8Size(rlen, rstr) : rlen); // Execute an explicit match first to see if we match at all; if we do, matchData will be populated // with data that can be used by pcre2_substitute to bypass some work @@ -523,7 +530,7 @@ class CCompiledStrRegExpr : implements ICompiledStrRegExpr // if it does then we can substitute within the given buffer and then pad with spaces, if not then // we have to allocate memory, substitute into that memory, then copy into the given buffer; // note that pcreSize will include space for a terminating null character even though we don't want it - int replaceResult = pcre2_substitute_8(compiledRegex.get(), (PCRE2_SPTR8)str, sourceSize, 0, replaceOptions|PCRE2_SUBSTITUTE_OVERFLOW_LENGTH, matchData, pcre2MatchContext8, (PCRE2_SPTR8)replace, replaceSize, nullptr, &pcreSize); + int replaceResult = pcre2_substitute_8(compiledRegex.get(), (PCRE2_SPTR8)str, sourceSize, 0, replaceOptions|PCRE2_SUBSTITUTE_OVERFLOW_LENGTH, matchData, pcre2MatchContext8, (PCRE2_SPTR8)rstr, replaceSize, nullptr, &pcreSize); if (replaceResult < 0 && replaceResult != PCRE2_ERROR_NOMEMORY) { @@ -548,7 +555,7 @@ class CCompiledStrRegExpr : implements ICompiledStrRegExpr replaceBuffer = (char *)tempBuffer.data(); } - replaceResult = pcre2_substitute_8(compiledRegex.get(), (PCRE2_SPTR8)str, sourceSize, 0, replaceOptions, matchData, pcre2MatchContext8, (PCRE2_SPTR8)replace, replaceSize, (PCRE2_UCHAR8 *)replaceBuffer, &pcreSize); + replaceResult = pcre2_substitute_8(compiledRegex.get(), (PCRE2_SPTR8)str, sourceSize, 0, replaceOptions, matchData, pcre2MatchContext8, (PCRE2_SPTR8)rstr, replaceSize, (PCRE2_UCHAR8 *)replaceBuffer, &pcreSize); if (replaceResult < 0) { @@ -620,12 +627,24 @@ class CCompiledStrRegExpr : implements ICompiledStrRegExpr } } + void replaceFixedTimed(ISectionTimer * timer, size32_t tlen, char * tgt, size32_t slen, char const * str, size32_t rlen, char const * rstr) const + { + SectionTimerBlock sectionTimer(timer); + replaceFixed(tlen, tgt, slen, str, rlen, rstr); + } + IStrRegExprFindInstance * find(const char * str, size32_t from, size32_t len, bool needToKeepSearchString) const { CStrRegExprFindInstance * findInst = new CStrRegExprFindInstance(compiledRegex, str, from, len, needToKeepSearchString); return findInst; } + IStrRegExprFindInstance * findTimed(ISectionTimer * timer, const char * str, size32_t from, size32_t len, bool needToKeepSearchString) const + { + SectionTimerBlock sectionTimer(timer); + return find(str, from, len, needToKeepSearchString); + } + void getMatchSet(bool & __isAllResult, size32_t & __resultBytes, void * & __result, size32_t _subjectLen, const char * _subject) { rtlRowBuilder out; @@ -690,6 +709,12 @@ class CCompiledStrRegExpr : implements ICompiledStrRegExpr __result = out.detachdata(); }; + void getMatchSetTimed(ISectionTimer * timer, bool & __isAllResult, size32_t & __resultBytes, void * & __result, size32_t _subjectLen, const char * _subject) + { + SectionTimerBlock sectionTimer(timer); + getMatchSet(__isAllResult, __resultBytes, __result, _subjectLen, _subject); + } + }; //--------------------------------------------------------------------------- @@ -749,11 +774,23 @@ ECLRTL_API ICompiledStrRegExpr * rtlCreateCompiledStrRegExpr(const char * regExp return fetchOrCreateCompiledStrRegExpr(strlen(regExpr), regExpr, isCaseSensitive); } +ECLRTL_API ICompiledStrRegExpr * rtlCreateCompiledStrRegExprTimed(ISectionTimer * timer, const char * regExpr, bool isCaseSensitive) +{ + SectionTimerBlock sectionTimer(timer); + return rtlCreateCompiledStrRegExpr(regExpr, isCaseSensitive); +} + ECLRTL_API ICompiledStrRegExpr * rtlCreateCompiledStrRegExpr(int regExprLength, const char * regExpr, bool isCaseSensitive) { return fetchOrCreateCompiledStrRegExpr(regExprLength, regExpr, isCaseSensitive); } +ECLRTL_API ICompiledStrRegExpr * rtlCreateCompiledStrRegExprTimed(ISectionTimer * timer, int regExprLength, const char * regExpr, bool isCaseSensitive) +{ + SectionTimerBlock sectionTimer(timer); + return rtlCreateCompiledStrRegExpr(regExprLength, regExpr, isCaseSensitive); +} + ECLRTL_API void rtlDestroyCompiledStrRegExpr(ICompiledStrRegExpr * compiledExpr) { if (compiledExpr) @@ -824,11 +861,23 @@ ECLRTL_API ICompiledStrRegExpr * rtlCreateCompiledU8StrRegExpr(const char * regE return fetchOrCreateCompiledU8StrRegExpr(rtlUtf8Length(regExpr), regExpr, isCaseSensitive); } +ECLRTL_API ICompiledStrRegExpr * rtlCreateCompiledU8StrRegExprTimed(ISectionTimer * timer, const char * regExpr, bool isCaseSensitive) +{ + SectionTimerBlock sectionTimer(timer); + return rtlCreateCompiledU8StrRegExpr(regExpr, isCaseSensitive); +} + ECLRTL_API ICompiledStrRegExpr * rtlCreateCompiledU8StrRegExpr(int regExprLength, const char * regExpr, bool isCaseSensitive) { return fetchOrCreateCompiledU8StrRegExpr(regExprLength, regExpr, isCaseSensitive); } +ECLRTL_API ICompiledStrRegExpr * rtlCreateCompiledU8StrRegExprTimed(ISectionTimer * timer, int regExprLength, const char * regExpr, bool isCaseSensitive) +{ + SectionTimerBlock sectionTimer(timer); + return rtlCreateCompiledU8StrRegExpr(regExprLength, regExpr, isCaseSensitive); +} + ECLRTL_API void rtlDestroyCompiledU8StrRegExpr(ICompiledStrRegExpr * compiledExpr) { if (compiledExpr) @@ -847,7 +896,7 @@ ECLRTL_API void rtlDestroyU8StrRegExprFindInstance(IStrRegExprFindInstance * fin #ifdef _USE_ICU -class CUStrRegExprFindInstance : implements IUStrRegExprFindInstance +class CUStrRegExprFindInstance final : implements IUStrRegExprFindInstance { private: bool matched = false; @@ -898,7 +947,7 @@ class CUStrRegExprFindInstance : implements IUStrRegExprFindInstance } } - UChar const * findvstr(unsigned outlen, UChar * out, unsigned n = 0) + UChar const * findvstr(unsigned outlen, UChar * out, unsigned n) { if (matched && (n < pcre2_get_ovector_count_16(matchData))) { @@ -920,7 +969,7 @@ class CUStrRegExprFindInstance : implements IUStrRegExprFindInstance //--------------------------------------------------------------------------- -class CCompiledUStrRegExpr : implements ICompiledUStrRegExpr +class CCompiledUStrRegExpr final : implements ICompiledUStrRegExpr { private: std::shared_ptr compiledRegex = nullptr; @@ -948,7 +997,7 @@ class CCompiledUStrRegExpr : implements ICompiledUStrRegExpr std::shared_ptr getCompiledRegex() const { return compiledRegex; } - void replace(size32_t & outlen, UChar * & out, size32_t slen, const UChar * str, size32_t rlen, UChar const * replace) const + void replace(size32_t & outlen, UChar * & out, size32_t slen, const UChar * str, size32_t rlen, UChar const * rstr) const { outlen = 0; PCRE2MatchData16 matchData = pcre2_match_data_create_from_pattern_16(compiledRegex.get(), pcre2GeneralContext16); @@ -972,7 +1021,7 @@ class CCompiledUStrRegExpr : implements ICompiledUStrRegExpr // Note that pcreSize will include space for a terminating null character; // we have to allocate memory for that byte to avoid a buffer overrun, // but we won't count that terminating byte - int replaceResult = pcre2_substitute_16(compiledRegex.get(), (PCRE2_SPTR16)str, slen, 0, replaceOptions|PCRE2_SUBSTITUTE_OVERFLOW_LENGTH, matchData, pcre2MatchContext16, (PCRE2_SPTR16)replace, rlen, nullptr, &pcreSize); + int replaceResult = pcre2_substitute_16(compiledRegex.get(), (PCRE2_SPTR16)str, slen, 0, replaceOptions|PCRE2_SUBSTITUTE_OVERFLOW_LENGTH, matchData, pcre2MatchContext16, (PCRE2_SPTR16)rstr, rlen, nullptr, &pcreSize); if (replaceResult < 0 && replaceResult != PCRE2_ERROR_NOMEMORY) { @@ -984,7 +1033,7 @@ class CCompiledUStrRegExpr : implements ICompiledUStrRegExpr { out = (UChar *)rtlMalloc(pcreSize * sizeof(UChar)); - replaceResult = pcre2_substitute_16(compiledRegex.get(), (PCRE2_SPTR16)str, slen, 0, replaceOptions, matchData, pcre2MatchContext16, (PCRE2_SPTR16)replace, rlen, (PCRE2_UCHAR16 *)out, &pcreSize); + replaceResult = pcre2_substitute_16(compiledRegex.get(), (PCRE2_SPTR16)str, slen, 0, replaceOptions, matchData, pcre2MatchContext16, (PCRE2_SPTR16)rstr, rlen, (PCRE2_UCHAR16 *)out, &pcreSize); // Note that, weirdly, pcreSize will now contain the number of code points // in the result *excluding* the null terminator, so pcreSize will @@ -1014,6 +1063,12 @@ class CCompiledUStrRegExpr : implements ICompiledUStrRegExpr } } + void replaceTimed(ISectionTimer * timer, size32_t & outlen, UChar * & out, size32_t slen, const UChar * str, size32_t rlen, UChar const * rstr) const + { + SectionTimerBlock sectionTimer(timer); + replace(outlen, out, slen, str, rlen, rstr); + } + void replaceFixed(size32_t tlen, UChar * tgt, size32_t slen, UChar const * str, size32_t rlen, UChar const * replace) const { if (tlen == 0) @@ -1112,12 +1167,24 @@ class CCompiledUStrRegExpr : implements ICompiledUStrRegExpr } } + void replaceFixedTimed(ISectionTimer * timer, size32_t tlen, UChar * tgt, size32_t slen, UChar const * str, size32_t rlen, UChar const * replace) const + { + SectionTimerBlock sectionTimer(timer); + replaceFixed(tlen, tgt, slen, str, rlen, replace); + } + IUStrRegExprFindInstance * find(const UChar * str, size32_t from, size32_t len) const { CUStrRegExprFindInstance * findInst = new CUStrRegExprFindInstance(compiledRegex, str, from, len); return findInst; } + IUStrRegExprFindInstance * findTimed(ISectionTimer * timer, const UChar * str, size32_t from, size32_t len) const + { + SectionTimerBlock sectionTimer(timer); + return find(str, from, len); + } + void getMatchSet(bool & __isAllResult, size32_t & __resultBytes, void * & __result, size32_t _subjectLen, const UChar * _subject) { rtlRowBuilder out; @@ -1180,6 +1247,12 @@ class CCompiledUStrRegExpr : implements ICompiledUStrRegExpr __resultBytes = outBytes; __result = out.detachdata(); } + + void getMatchSetTimed(ISectionTimer * timer, bool & __isAllResult, size32_t & __resultBytes, void * & __result, size32_t _subjectLen, const UChar * _subject) + { + SectionTimerBlock sectionTimer(timer); + getMatchSet(__isAllResult, __resultBytes, __result, _subjectLen, _subject); + } }; //--------------------------------------------------------------------------- @@ -1240,11 +1313,23 @@ ECLRTL_API ICompiledUStrRegExpr * rtlCreateCompiledUStrRegExpr(const UChar * reg return fetchOrCreateCompiledUStrRegExpr(rtlUnicodeStrlen(regExpr), regExpr, isCaseSensitive); } +ECLRTL_API ICompiledUStrRegExpr * rtlCreateCompiledUStrRegExprTimed(ISectionTimer * timer, const UChar * regExpr, bool isCaseSensitive) +{ + SectionTimerBlock sectionTimer(timer); + return rtlCreateCompiledUStrRegExpr(regExpr, isCaseSensitive); +} + ECLRTL_API ICompiledUStrRegExpr * rtlCreateCompiledUStrRegExpr(int regExprLength, const UChar * regExpr, bool isCaseSensitive) { return fetchOrCreateCompiledUStrRegExpr(regExprLength, regExpr, isCaseSensitive); } +ECLRTL_API ICompiledUStrRegExpr * rtlCreateCompiledUStrRegExprTimed(ISectionTimer * timer, int regExprLength, const UChar * regExpr, bool isCaseSensitive) +{ + SectionTimerBlock sectionTimer(timer); + return rtlCreateCompiledUStrRegExpr(regExprLength, regExpr, isCaseSensitive); +} + ECLRTL_API void rtlDestroyCompiledUStrRegExpr(ICompiledUStrRegExpr * compiledExpr) { if (compiledExpr) diff --git a/rtl/eclrtl/eclrtl.hpp b/rtl/eclrtl/eclrtl.hpp index 97beb5ff748..9635ae23287 100644 --- a/rtl/eclrtl/eclrtl.hpp +++ b/rtl/eclrtl/eclrtl.hpp @@ -70,6 +70,7 @@ interface IOutputRowDeserializer; interface IAtom; interface IRecordSize; interface IException; +interface ISectionTimer; class StringBuffer; class MemoryBuffer; @@ -86,10 +87,14 @@ interface IStrRegExprFindInstance interface ICompiledStrRegExpr { - virtual void replace(size32_t & outlen, char * & out, size32_t slen, char const * str, size32_t rlen, char const * replace) const = 0; - virtual void replaceFixed(size32_t tlen, char * tgt, size32_t slen, char const * str, size32_t rlen, char const * replace) const = 0; + virtual void replace(size32_t & outlen, char * & out, size32_t slen, char const * str, size32_t rlen, char const * rstr) const = 0; + virtual void replaceFixed(size32_t tlen, char * tgt, size32_t slen, char const * str, size32_t rlen, char const * rstr) const = 0; virtual IStrRegExprFindInstance * find(const char * str, size32_t from, size32_t len, bool needToKeepSearchString) const = 0; virtual void getMatchSet(bool & __isAllResult, size32_t & __resultBytes, void * & __result, size32_t _srcLen, const char * _search) = 0; + virtual void replaceTimed(ISectionTimer * timer, size32_t & outlen, char * & out, size32_t slen, char const * str, size32_t rlen, char const * rstr) const = 0; + virtual void replaceFixedTimed(ISectionTimer * timer, size32_t tlen, char * tgt, size32_t slen, char const * str, size32_t rlen, char const * rstr) const = 0; + virtual IStrRegExprFindInstance * findTimed(ISectionTimer * timer, const char * str, size32_t from, size32_t len, bool needToKeepSearchString) const = 0; + virtual void getMatchSetTimed(ISectionTimer * timer, bool & __isAllResult, size32_t & __resultBytes, void * & __result, size32_t _srcLen, const char * _search) = 0; }; // RegEx Compiler for unicode strings @@ -101,10 +106,14 @@ interface IUStrRegExprFindInstance interface ICompiledUStrRegExpr { - virtual void replace(size32_t & outlen, UChar * & out, size32_t slen, UChar const * str, size32_t rlen, UChar const * replace) const = 0; - virtual void replaceFixed(size32_t tlen, UChar * tgt, size32_t slen, UChar const * str, size32_t rlen, UChar const * replace) const = 0; + virtual void replace(size32_t & outlen, UChar * & out, size32_t slen, UChar const * str, size32_t rlen, UChar const * rstr) const = 0; + virtual void replaceFixed(size32_t tlen, UChar * tgt, size32_t slen, UChar const * str, size32_t rlen, UChar const * rstr) const = 0; virtual IUStrRegExprFindInstance * find(const UChar * str, size32_t from, size32_t len) const = 0; virtual void getMatchSet(bool & __isAllResult, size32_t & __resultBytes, void * & __result, size32_t _srcLen, const UChar * _search) = 0; + virtual void replaceTimed(ISectionTimer * timer, size32_t & outlen, UChar * & out, size32_t slen, UChar const * str, size32_t rlen, UChar const * rstr) const = 0; + virtual void replaceFixedTimed(ISectionTimer * timer, size32_t tlen, UChar * tgt, size32_t slen, UChar const * str, size32_t rlen, UChar const * rstr) const = 0; + virtual IUStrRegExprFindInstance * findTimed(ISectionTimer * timer, const UChar * str, size32_t from, size32_t len) const = 0; + virtual void getMatchSetTimed(ISectionTimer * timer, bool & __isAllResult, size32_t & __resultBytes, void * & __result, size32_t _srcLen, const UChar * _search) = 0; }; //----------------------------------------------------------------------------- @@ -707,17 +716,23 @@ ECLRTL_API void rtlUtf8SpaceFill(unsigned tlne, char * tgt, unsigned idx); ECLRTL_API ICompiledStrRegExpr * rtlCreateCompiledStrRegExpr(const char * regExpr, bool isCaseSensitive); +ECLRTL_API ICompiledStrRegExpr * rtlCreateCompiledStrRegExprTimed(ISectionTimer * timer, const char * regExpr, bool isCaseSensitive); ECLRTL_API ICompiledStrRegExpr * rtlCreateCompiledStrRegExpr(int regExprLength, const char * regExpr, bool isCaseSensitive); +ECLRTL_API ICompiledStrRegExpr * rtlCreateCompiledStrRegExprTimed(ISectionTimer * timer, int regExprLength, const char * regExpr, bool isCaseSensitive); ECLRTL_API void rtlDestroyCompiledStrRegExpr(ICompiledStrRegExpr * compiled); ECLRTL_API void rtlDestroyStrRegExprFindInstance(IStrRegExprFindInstance * compiled); ECLRTL_API ICompiledStrRegExpr * rtlCreateCompiledU8StrRegExpr(const char * regExpr, bool isCaseSensitive); +ECLRTL_API ICompiledStrRegExpr * rtlCreateCompiledU8StrRegExprTimed(ISectionTimer * timer, const char * regExpr, bool isCaseSensitive); ECLRTL_API ICompiledStrRegExpr * rtlCreateCompiledU8StrRegExpr(int regExprLength, const char * regExpr, bool isCaseSensitive); +ECLRTL_API ICompiledStrRegExpr * rtlCreateCompiledU8StrRegExprTimed(ISectionTimer * timer, int regExprLength, const char * regExpr, bool isCaseSensitive); ECLRTL_API void rtlDestroyCompiledU8StrRegExpr(ICompiledStrRegExpr * compiled); ECLRTL_API void rtlDestroyU8StrRegExprFindInstance(IStrRegExprFindInstance * compiled); ECLRTL_API ICompiledUStrRegExpr * rtlCreateCompiledUStrRegExpr(const UChar * regExpr, bool isCaseSensitive); +ECLRTL_API ICompiledUStrRegExpr * rtlCreateCompiledUStrRegExprTimed(ISectionTimer * timer, const UChar * regExpr, bool isCaseSensitive); ECLRTL_API ICompiledUStrRegExpr * rtlCreateCompiledUStrRegExpr(int regExprLength, const UChar * regExpr, bool isCaseSensitive); +ECLRTL_API ICompiledUStrRegExpr * rtlCreateCompiledUStrRegExprTimed(ISectionTimer * timer, int regExprLength, const UChar * regExpr, bool isCaseSensitive); ECLRTL_API void rtlDestroyCompiledUStrRegExpr(ICompiledUStrRegExpr * compiled); ECLRTL_API void rtlDestroyUStrRegExprFindInstance(IUStrRegExprFindInstance * compiled); diff --git a/rtl/eclrtl/eclrtl_imp.hpp b/rtl/eclrtl/eclrtl_imp.hpp index dc4f53642ea..a7fdf36209b 100644 --- a/rtl/eclrtl/eclrtl_imp.hpp +++ b/rtl/eclrtl/eclrtl_imp.hpp @@ -132,6 +132,14 @@ class ECLRTL_API rtlCompiledStrRegex regex = compiled; } + inline void setPatternTimed(ISectionTimer * timer, const char * pattern, bool isCaseSensitive) + { + ICompiledStrRegExpr * compiled = rtlCreateCompiledStrRegExprTimed(timer, pattern, isCaseSensitive); + if (regex) + rtlDestroyCompiledStrRegExpr(regex); + regex = compiled; + } + inline void setPattern(unsigned int patternLen, const char * pattern, bool isCaseSensitive) { ICompiledStrRegExpr * compiled = rtlCreateCompiledStrRegExpr(patternLen, pattern, isCaseSensitive); @@ -140,6 +148,14 @@ class ECLRTL_API rtlCompiledStrRegex regex = compiled; } + inline void setPatternTimed(ISectionTimer * timer, unsigned int patternLen, const char * pattern, bool isCaseSensitive) + { + ICompiledStrRegExpr * compiled = rtlCreateCompiledStrRegExprTimed(timer, patternLen, pattern, isCaseSensitive); + if (regex) + rtlDestroyCompiledStrRegExpr(regex); + regex = compiled; + } + private: ICompiledStrRegExpr * regex; }; @@ -160,6 +176,14 @@ class ECLRTL_API rtlStrRegexFindInstance rtlDestroyStrRegExprFindInstance(instance); instance = search; } + + void findTimed(ISectionTimer * timer, const rtlCompiledStrRegex & regex, size32_t len, const char * str, bool needToKeepSearchString) + { + IStrRegExprFindInstance * search = regex->findTimed(timer, str, 0, len, needToKeepSearchString); + if (instance) + rtlDestroyStrRegExprFindInstance(instance); + instance = search; + } private: IStrRegExprFindInstance * instance; @@ -186,6 +210,14 @@ class ECLRTL_API rtlCompiledUStrRegex instance = compiled; } + inline void setPatternTimed(ISectionTimer * timer, const UChar * pattern, bool isCaseSensitive) + { + ICompiledUStrRegExpr * compiled = rtlCreateCompiledUStrRegExprTimed(timer, pattern, isCaseSensitive); + if (instance) + rtlDestroyCompiledUStrRegExpr(instance); + instance = compiled; + } + inline void setPattern(unsigned int patternLen, const UChar * pattern, bool isCaseSensitive) { ICompiledUStrRegExpr * compiled = rtlCreateCompiledUStrRegExpr(patternLen, pattern, isCaseSensitive); @@ -194,6 +226,14 @@ class ECLRTL_API rtlCompiledUStrRegex instance = compiled; } + inline void setPatternTimed(ISectionTimer * timer, unsigned int patternLen, const UChar * pattern, bool isCaseSensitive) + { + ICompiledUStrRegExpr * compiled = rtlCreateCompiledUStrRegExprTimed(timer, patternLen, pattern, isCaseSensitive); + if (instance) + rtlDestroyCompiledUStrRegExpr(instance); + instance = compiled; + } + private: ICompiledUStrRegExpr * instance; }; @@ -218,6 +258,14 @@ class ECLRTL_API rtlCompiledU8StrRegex rtlDestroyCompiledU8StrRegExpr(instance); instance = compiled; } + + inline void setPatternTimed(ISectionTimer * timer, const char * pattern, bool isCaseSensitive) + { + ICompiledStrRegExpr * compiled = rtlCreateCompiledU8StrRegExprTimed(timer, pattern, isCaseSensitive); + if (instance) + rtlDestroyCompiledU8StrRegExpr(instance); + instance = compiled; + } inline void setPattern(unsigned int patternLen, const char * pattern, bool isCaseSensitive) { @@ -226,6 +274,14 @@ class ECLRTL_API rtlCompiledU8StrRegex rtlDestroyCompiledU8StrRegExpr(instance); instance = compiled; } + + inline void setPatternTimed(ISectionTimer * timer, unsigned int patternLen, const char * pattern, bool isCaseSensitive) + { + ICompiledStrRegExpr * compiled = rtlCreateCompiledU8StrRegExprTimed(timer, patternLen, pattern, isCaseSensitive); + if (instance) + rtlDestroyCompiledU8StrRegExpr(instance); + instance = compiled; + } private: ICompiledStrRegExpr * instance; @@ -247,6 +303,14 @@ class ECLRTL_API rtlUStrRegexFindInstance rtlDestroyUStrRegExprFindInstance(instance); instance = search; } + + void findTimed(ISectionTimer * timer, const rtlCompiledUStrRegex & regex, size32_t len, const UChar * str) + { + IUStrRegExprFindInstance * search = regex->findTimed(timer, str, 0, len); + if (instance) + rtlDestroyUStrRegExprFindInstance(instance); + instance = search; + } private: IUStrRegExprFindInstance * instance; @@ -268,6 +332,14 @@ class ECLRTL_API rtlU8StrRegexFindInstance rtlDestroyU8StrRegExprFindInstance(instance); instance = search; } + + void findTimed(ISectionTimer * timer, const rtlCompiledU8StrRegex & regex, size32_t len, const char * str, bool needToKeepSearchString) + { + IStrRegExprFindInstance * search = regex->findTimed(timer, str, 0, len, needToKeepSearchString); + if (instance) + rtlDestroyU8StrRegExprFindInstance(instance); + instance = search; + } private: IStrRegExprFindInstance * instance; diff --git a/rtl/include/eclhelper.hpp b/rtl/include/eclhelper.hpp index cc0c0c843ba..b312b7a3409 100644 --- a/rtl/include/eclhelper.hpp +++ b/rtl/include/eclhelper.hpp @@ -59,7 +59,7 @@ typedef unsigned short UChar; //Should be incremented whenever the virtuals in the context or a helper are changed, so //that a work unit can't be rerun. Try as hard as possible to retain compatibility. -#define ACTIVITY_INTERFACE_VERSION 654 +#define ACTIVITY_INTERFACE_VERSION 655 #define MIN_ACTIVITY_INTERFACE_VERSION 650 //minimum value that is compatible with current interface typedef unsigned char byte; @@ -3019,6 +3019,30 @@ class XmlChildIterator constexpr unsigned daliResultOutputMax = 2000; // MB constexpr unsigned futureResultOutputMax = 100; // MB +//------------------------------------------------------------------------------------------------ + +// RAII class to start and stop a section timer +class SectionTimerBlock +{ +private: + ISectionTimer * timer; + unsigned long long cycles = 0; + +public: + SectionTimerBlock(ISectionTimer * _timer) : timer(_timer) + { + if (timer) + cycles = timer->getStartCycles(); + } + ~SectionTimerBlock() + { + if (timer) + timer->noteSectionTime(cycles); + } +}; + +//------------------------------------------------------------------------------------------------ + #ifdef STARTQUERY_EXPORTS #define STARTQUERY_API DECL_EXPORT #else From 074095fc0c974ba393b211de8b71ead61c7f1e43 Mon Sep 17 00:00:00 2001 From: M Kelly Date: Thu, 22 Aug 2024 15:07:23 -0400 Subject: [PATCH 30/33] HPCC-32512 Fix LD_LIBRARY_PATH for internal ssh cmd from frunssh and rsync Signed-off-by: M Kelly --- common/remote/rmtssh.cpp | 3 ++- initfiles/bin/init_thorslave.in | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/common/remote/rmtssh.cpp b/common/remote/rmtssh.cpp index 86682580c7e..f156e2dbd8c 100644 --- a/common/remote/rmtssh.cpp +++ b/common/remote/rmtssh.cpp @@ -99,7 +99,8 @@ class CFRunSSH: public CInterface, implements IFRunSSH break; case 's': { // ssh params bool usepssh = !password.isEmpty(); - cmdbuf.appendf("%s -o LogLevel=QUIET -o StrictHostKeyChecking=%s -o BatchMode=yes ",usepssh?"pssh":"ssh",strict?"yes":"no"); + // reset LD_LIBRARY_PATH here so ssh cmd itself doesn't use HPCC libssl/crypto as they may be different + cmdbuf.appendf("%s -o LogLevel=QUIET -o StrictHostKeyChecking=%s -o BatchMode=yes ",usepssh?"pssh":"LD_LIBRARY_PATH=: ssh",strict?"yes":"no"); if (!identityfile.isEmpty()) cmdbuf.appendf("-i %s ",identityfile.get()); if (background) diff --git a/initfiles/bin/init_thorslave.in b/initfiles/bin/init_thorslave.in index 1048ce6f9ac..ad845f0a74f 100755 --- a/initfiles/bin/init_thorslave.in +++ b/initfiles/bin/init_thorslave.in @@ -102,7 +102,8 @@ start_slaves() rsync_att=3 rsync_stat=1 while [[ $rsync_stat -ne 0 && $rsync_att -gt 0 ]] ; do - rsync -e "ssh -o LogLevel=QUIET -o StrictHostKeyChecking=no" --timeout=60 $master:$instancedir/slaves $slavesfname + # reset LD_LIBRARY_PATH here so ssh cmd doesn't use HPCC libssl/crypto as they may be different + LD_LIBRARY_PATH=: rsync -e "ssh -o LogLevel=QUIET -o StrictHostKeyChecking=no" --timeout=60 $master:$instancedir/slaves $slavesfname rsync_stat=$? ((rsync_att--)) log "rsync returns ${rsync_stat}" From 91af2a3d34d23cf89f5e4f9b9abfc1397720b878 Mon Sep 17 00:00:00 2001 From: Jake Smith Date: Thu, 22 Aug 2024 15:14:52 +0100 Subject: [PATCH 31/33] HPCC-32493 Prevent thor k8s crash in disconnectLogMsgManagerFromDali Signed-off-by: Jake Smith --- dali/base/daclient.cpp | 2 ++ system/mp/mplog.cpp | 8 ++++++++ 2 files changed, 10 insertions(+) diff --git a/dali/base/daclient.cpp b/dali/base/daclient.cpp index f8adc17e90a..7a55e473dc1 100644 --- a/dali/base/daclient.cpp +++ b/dali/base/daclient.cpp @@ -270,6 +270,8 @@ void connectLogMsgManagerToDali() void disconnectLogMsgManagerFromDali() { + if (isContainerized()) + return; // we do not redirect logging between components in containerized environments (this is used for audit->dali in BM) disconnectLogMsgManagerFromParentOwn(daliClientLoggingParent); daliClientLoggingParent = 0; } diff --git a/system/mp/mplog.cpp b/system/mp/mplog.cpp index 86b338e85c1..2483ad015b1 100644 --- a/system/mp/mplog.cpp +++ b/system/mp/mplog.cpp @@ -532,6 +532,8 @@ aindex_t LogMsgParentReceiverThread::findParent(const INode * node) const bool connectLogMsgManagerToParent(INode * parentNode) { + if (isContainerized()) + return false; assertex(parentReceiver); MPLogId pid = parentReceiver->getNextId(); return parentReceiver->addParentToManager(0, pid, parentNode, false); @@ -539,6 +541,8 @@ bool connectLogMsgManagerToParent(INode * parentNode) bool connectLogMsgManagerToParentOwn(INode * parentNode) { + if (isContainerized()) + return false; bool ret = connectLogMsgManagerToParent(parentNode); parentNode->Release(); return ret; @@ -546,11 +550,15 @@ bool connectLogMsgManagerToParentOwn(INode * parentNode) bool disconnectLogMsgManagerFromParent(INode * parentNode) { + if (isContainerized()) + return false; return parentReceiver->removeParentFromManager(parentNode, false); } bool disconnectLogMsgManagerFromParentOwn(INode * parentNode) { + if (isContainerized()) + return false; bool ret = disconnectLogMsgManagerFromParent(parentNode); parentNode->Release(); return ret; From e75fc2be1614bda0714f7df24f4cd3596db894a6 Mon Sep 17 00:00:00 2001 From: M Kelly Date: Fri, 23 Aug 2024 08:17:57 -0400 Subject: [PATCH 32/33] HPCC-32512 Fix LD_LIBRARY_PATH for internal ssh cmd from frunssh and rsync 2 Signed-off-by: M Kelly --- common/remote/rmtssh.cpp | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/common/remote/rmtssh.cpp b/common/remote/rmtssh.cpp index f156e2dbd8c..fe08594b4ad 100644 --- a/common/remote/rmtssh.cpp +++ b/common/remote/rmtssh.cpp @@ -99,8 +99,7 @@ class CFRunSSH: public CInterface, implements IFRunSSH break; case 's': { // ssh params bool usepssh = !password.isEmpty(); - // reset LD_LIBRARY_PATH here so ssh cmd itself doesn't use HPCC libssl/crypto as they may be different - cmdbuf.appendf("%s -o LogLevel=QUIET -o StrictHostKeyChecking=%s -o BatchMode=yes ",usepssh?"pssh":"LD_LIBRARY_PATH=: ssh",strict?"yes":"no"); + cmdbuf.appendf("%s -o LogLevel=QUIET -o StrictHostKeyChecking=%s -o BatchMode=yes ",usepssh?"pssh":"ssh",strict?"yes":"no"); if (!identityfile.isEmpty()) cmdbuf.appendf("-i %s ",identityfile.get()); if (background) @@ -423,6 +422,8 @@ class CFRunSSH: public CInterface, implements IFRunSSH printf("%s\n",cmdline.str()); else { Owned pipe = createPipeProcess(); + // reset LD_LIBRARY_PATH here so ssh cmd itself doesn't use HPCC libssl/crypto as they may be different + pipe->setenv("LD_LIBRARY_PATH", ":"); if (pipe->run((verbose&&!usepssh)?"FRUNSSH":NULL,cmdline.str(),workdir, useplink, // for some reason plink needs input handle true,true)) { From f8ce1f0b7a737b8008524544b38afd6044bc4223 Mon Sep 17 00:00:00 2001 From: Jake Smith Date: Fri, 23 Aug 2024 10:05:59 +0100 Subject: [PATCH 33/33] HPCC-32492 Prevent failing unrelated thor instance marking jobs failed Also, track engine(thor) instance session in agent, so that if it disappears unexpectedly whilst running, it aborts the job and marks as session died unexpectedly. Signed-off-by: Jake Smith --- common/workunit/workunit.cpp | 55 ++++++++++++++++++++++++++----- common/workunit/workunit.hpp | 2 ++ common/workunit/workunit.ipp | 4 ++- ecl/agentexec/agentexec.cpp | 7 +--- plugins/cassandra/cassandrawu.cpp | 2 +- thorlcr/master/thgraphmanager.cpp | 3 ++ 6 files changed, 57 insertions(+), 16 deletions(-) diff --git a/common/workunit/workunit.cpp b/common/workunit/workunit.cpp index a5e97aff12b..7fa15bd12ed 100644 --- a/common/workunit/workunit.cpp +++ b/common/workunit/workunit.cpp @@ -4341,6 +4341,8 @@ class CLockedWorkUnit : implements ILocalWorkUnit, implements IExtendedWUInterfa { return c->getStateEx(str); } virtual __int64 getAgentSession() const { return c->getAgentSession(); } + virtual __int64 getEngineSession() const + { return c->getEngineSession(); } virtual unsigned getAgentPID() const { return c->getAgentPID(); } virtual const char *queryStateDesc() const @@ -4497,6 +4499,8 @@ class CLockedWorkUnit : implements ILocalWorkUnit, implements IExtendedWUInterfa { c->setStateEx(text); } virtual void setAgentSession(__int64 sessionId) { c->setAgentSession(sessionId); } + virtual void setEngineSession(__int64 sessionId) + { c->setEngineSession(sessionId); } virtual void setStatistic(StatisticCreatorType creatorType, const char * creator, StatisticScopeType scopeType, const char * scope, StatisticKind kind, const char * optDescription, unsigned __int64 value, unsigned __int64 count, unsigned __int64 maxValue, StatsMergeAction mergeAction) { c->setStatistic(creatorType, creator, scopeType, scope, kind, optDescription, value, count, maxValue, mergeAction); } virtual void setTracingValue(const char * propname, const char * value) @@ -5918,9 +5922,11 @@ void CWorkUnitFactory::clearAborting(const char *wuid) } } -void CWorkUnitFactory::reportAbnormalTermination(const char *wuid, WUState &state, SessionId agent) +void CWorkUnitFactory::reportAbnormalTermination(const char *wuid, WUState &state, SessionId sessionId, const char *sessionText) { - WARNLOG("reportAbnormalTermination: session stopped unexpectedly: %" I64F "d state: %d", (__int64) agent, (int) state); + StringBuffer sessionMessage(sessionText); + sessionMessage.appendf(" session stopped unexpectedly [sessionId=%" I64F "d]", (__int64) sessionId); + WARNLOG("reportAbnormalTermination: %s - state: %d", sessionText, (int) state); bool isEcl = false; switch (state) { @@ -5937,7 +5943,10 @@ void CWorkUnitFactory::reportAbnormalTermination(const char *wuid, WUState &stat wu->setState(state); Owned e = wu->createException(); e->setExceptionCode(isEcl ? 1001 : 1000); - e->setExceptionMessage(isEcl ? "EclCC terminated unexpectedly" : "Workunit terminated unexpectedly"); + StringBuffer exceptionText; + exceptionText.append(isEcl ? "EclCC terminated unexpectedly" : "Workunit terminated unexpectedly"); + exceptionText.append(" (").append(sessionMessage).append(")"); + e->setExceptionMessage(exceptionText); } static CriticalSection deleteDllLock; @@ -6422,8 +6431,11 @@ class CDaliWorkUnitFactory : public CWorkUnitFactory, implements IDaliClientShut LocalIAbortHandler abortHandler(*waiter); if (conn) { - SessionId agent = -1; + SessionId agentSessionID = -1; + SessionId engineSessionID = -1; bool agentSessionStopped = false; + bool engineSessionStopped = false; + bool queryRuntimeSessionStopped = false; unsigned start = msTick(); for (;;) { @@ -6452,22 +6464,37 @@ class CDaliWorkUnitFactory : public CWorkUnitFactory, implements IDaliClientShut case WUStateAborting: if (agentSessionStopped) { - reportAbnormalTermination(wuid, ret, agent); + reportAbnormalTermination(wuid, ret, agentSessionID, "Agent"); + return ret; + } + if (engineSessionStopped) + { + reportAbnormalTermination(wuid, ret, engineSessionID, "Engine"); return ret; } if (queryDaliServerVersion().compare("2.1")>=0) { - agent = conn->queryRoot()->getPropInt64("@agentSession", -1); - if((agent>0) && querySessionManager().sessionStopped(agent, 0)) + agentSessionID = conn->queryRoot()->getPropInt64("@agentSession", -1); + if((agentSessionID>0) && querySessionManager().sessionStopped(agentSessionID, 0)) { agentSessionStopped = true; conn->reload(); continue; } + engineSessionID = conn->queryRoot()->getPropInt64("@engineSession", -1); + if((engineSessionID>0) && querySessionManager().sessionStopped(engineSessionID, 0)) + { + engineSessionStopped = true; + conn->reload(); + continue; + } } break; } - agentSessionStopped = false; // reset for state changes such as WUStateWait then WUStateRunning again + // reset for state changes such as WUStateWait then WUStateRunning again + agentSessionStopped = false; + engineSessionStopped = false; + unsigned waited = msTick() - start; if (timeout==-1 || waited + 20000 < timeout) { @@ -7694,6 +7721,12 @@ void CLocalWorkUnit::setAgentSession(__int64 sessionId) p->setPropInt64("@agentSession", sessionId); } +void CLocalWorkUnit::setEngineSession(__int64 sessionId) +{ + CriticalBlock block(crit); + p->setPropInt64("@engineSession", sessionId); +} + bool CLocalWorkUnit::getIsQueryService() const { CriticalBlock block(crit); @@ -7795,6 +7828,12 @@ __int64 CLocalWorkUnit::getAgentSession() const return p->getPropInt64("@agentSession", -1); } +__int64 CLocalWorkUnit::getEngineSession() const +{ + CriticalBlock block(crit); + return p->getPropInt64("@engineSession", -1); +} + unsigned CLocalWorkUnit::getAgentPID() const { CriticalBlock block(crit); diff --git a/common/workunit/workunit.hpp b/common/workunit/workunit.hpp index d96471b5077..b8da9047823 100644 --- a/common/workunit/workunit.hpp +++ b/common/workunit/workunit.hpp @@ -1251,6 +1251,7 @@ interface IConstWorkUnit : extends IConstWorkUnitInfo virtual IStringVal & getWorkunitDistributedAccessToken(IStringVal & datoken) const = 0; virtual IStringVal & getStateEx(IStringVal & str) const = 0; virtual __int64 getAgentSession() const = 0; + virtual __int64 getEngineSession() const = 0; virtual unsigned getAgentPID() const = 0; virtual IConstWUResult * getTemporaryByName(const char * name) const = 0; virtual IConstWUResultIterator & getTemporaries() const = 0; @@ -1340,6 +1341,7 @@ interface IWorkUnit : extends IConstWorkUnit virtual void setState(WUState state) = 0; virtual void setStateEx(const char * text) = 0; // Indicates why blocked virtual void setAgentSession(__int64 sessionId) = 0; + virtual void setEngineSession(__int64 sessionId) = 0; virtual void setStatistic(StatisticCreatorType creatorType, const char * creator, StatisticScopeType scopeType, const char * scope, StatisticKind kind, const char * optDescription, unsigned __int64 value, unsigned __int64 count, unsigned __int64 maxValue, StatsMergeAction mergeAction) = 0; virtual void setTracingValue(const char * propname, const char * value) = 0; virtual void setTracingValueInt(const char * propname, int value) = 0; diff --git a/common/workunit/workunit.ipp b/common/workunit/workunit.ipp index f83956af090..5213b437896 100644 --- a/common/workunit/workunit.ipp +++ b/common/workunit/workunit.ipp @@ -257,6 +257,7 @@ public: virtual WUState getState() const; virtual IStringVal & getStateEx(IStringVal & str) const; virtual __int64 getAgentSession() const; + virtual __int64 getEngineSession() const; virtual unsigned getAgentPID() const; virtual const char *queryStateDesc() const; virtual IConstWUResult * getTemporaryByName(const char * name) const; @@ -336,6 +337,7 @@ public: void setState(WUState state); void setStateEx(const char * text); void setAgentSession(__int64 sessionId); + void setEngineSession(__int64 sessionId); bool setDistributedAccessToken(const char * user); void setStatistic(StatisticCreatorType creatorType, const char * creator, StatisticScopeType scopeType, const char * scope, StatisticKind kind, const char * optDescription, unsigned __int64 value, unsigned __int64 count, unsigned __int64 maxValue, StatsMergeAction mergeAction); void setTracingValue(const char * propname, const char * value); @@ -621,7 +623,7 @@ public: } protected: - void reportAbnormalTermination(const char *wuid, WUState &state, SessionId agent); + void reportAbnormalTermination(const char *wuid, WUState &state, SessionId agent, const char *sessionText); // These need to be implemented by the derived classes virtual CLocalWorkUnit* _createWorkUnit(const char *wuid, ISecManager *secmgr, ISecUser *secuser) = 0; diff --git a/ecl/agentexec/agentexec.cpp b/ecl/agentexec/agentexec.cpp index 5582af12177..e0935dba13b 100644 --- a/ecl/agentexec/agentexec.cpp +++ b/ecl/agentexec/agentexec.cpp @@ -342,13 +342,8 @@ class WaitThread : public CInterfaceOf Owned cw = factory->openWorkUnit(wuid); if (cw) { - // if either a) NOT a thoragent with useChildProcesses=false (default in k8s config) or b) is still in an executing state - if (!sharedK8sJob || (cw->getState() == WUStateRunning) || (cw->getState() == WUStateBlocked) || (cw->getState() == WUStateWait)) + if (!sharedK8sJob && ((cw->getState() == WUStateRunning) || (cw->getState() == WUStateBlocked) || (cw->getState() == WUStateWait))) { - // For a shared k8s job, i.e. where this agent is thoragent launching shared (multiJobLinger) k8s jobs - // the job agent should handle the job state. - // In that scenario, this is a fallback that should only come into effect if the job workflow instance has failed to handle the exception - // e.g. because it abruptly disappeared. Owned workunit = &cw->lock(); // recheck now locked if ((workunit->getState() == WUStateRunning) || (workunit->getState() == WUStateBlocked) || (workunit->getState() == WUStateWait)) diff --git a/plugins/cassandra/cassandrawu.cpp b/plugins/cassandra/cassandrawu.cpp index 293b0c21ea1..10cb6239430 100644 --- a/plugins/cassandra/cassandrawu.cpp +++ b/plugins/cassandra/cassandrawu.cpp @@ -3873,7 +3873,7 @@ class CCasssandraWorkUnitFactory : public CWorkUnitFactory, implements ICassandr case WUStateAborting: if (agentSessionStopped) { - reportAbnormalTermination(wuid, state, agent); + reportAbnormalTermination(wuid, state, agent, "Agent"); return state; } if (queryDaliServerVersion().compare("2.1")>=0) diff --git a/thorlcr/master/thgraphmanager.cpp b/thorlcr/master/thgraphmanager.cpp index 142724fd868..33c580dfd16 100644 --- a/thorlcr/master/thgraphmanager.cpp +++ b/thorlcr/master/thgraphmanager.cpp @@ -1114,6 +1114,7 @@ bool CJobManager::executeGraph(IConstWorkUnit &workunit, const char *graphName, wu->setStatistic(queryStatisticsComponentType(), queryStatisticsComponentName(), SSTgraph, graphScope, StWhenStarted, NULL, startTs, 1, 0, StatsMergeAppend); //Could use addTimeStamp(wu, SSTgraph, graphName, StWhenStarted, wfid) if start time could be this point wu->setState(WUStateRunning); + wu->setEngineSession(myProcessSession()); VStringBuffer version("%d.%d", THOR_VERSION_MAJOR, THOR_VERSION_MINOR); wu->setDebugValue("ThorVersion", version.str(), true); @@ -1140,6 +1141,8 @@ bool CJobManager::executeGraph(IConstWorkUnit &workunit, const char *graphName, wu->setStatistic(queryStatisticsComponentType(), queryStatisticsComponentName(), SSTgraph, graphScope, StCostExecute, NULL, cost, 1, 0, StatsMergeReplace); if (globals->getPropBool("@watchdogProgressEnabled")) queryDeMonServer()->updateAggregates(wu); + // clear engine session, otherwise agent may consider a failure beyond this point for an unrelated job caused by this instance + wu->setEngineSession(-1); removeJob(*job); }