From 2314af35d80aafba385289993d0263489c4b18be Mon Sep 17 00:00:00 2001 From: Rodrigo Pastrana Date: Wed, 7 Aug 2024 21:46:16 -0400 Subject: [PATCH 01/25] HPCC-32411 Update ZAP log column mode name - Updates Zap report field name to coincide with eclwatch Signed-off-by: Rodrigo Pastrana --- esp/services/ws_workunits/ws_workunitsHelpers.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/esp/services/ws_workunits/ws_workunitsHelpers.hpp b/esp/services/ws_workunits/ws_workunitsHelpers.hpp index ae040b8aae8..d8b68ce724b 100644 --- a/esp/services/ws_workunits/ws_workunitsHelpers.hpp +++ b/esp/services/ws_workunits/ws_workunitsHelpers.hpp @@ -219,7 +219,7 @@ struct WUComponentLogOptions populateTimeRange(start, end, bufferSecs); //int 0 ==MIN, 1==DEFAULT, 2==ALL, 3==CUSTOM - int colMode = zapHttpRequest->getParameterInt("LogFilter_ColumnMode", -1); + int colMode = zapHttpRequest->getParameterInt("LogFilter_SelectColumnMode", -1); if (colMode != -1) { StringArray customFields; //comma delimited list of available columns, only if ColumnMode==3 From ee61ccba9cdba7e7918a3dc14808b7de5eaa5ce3 Mon Sep 17 00:00:00 2001 From: Gordon Smith Date: Fri, 2 Aug 2024 16:32:52 +0100 Subject: [PATCH 02/25] HPCC-32417 Add visibility changed event to DockPanel Fix linter issue (introduced by wasm version bump) Bump wasm package (avoid additional warnings) Signed-off-by: Gordon Smith --- esp/src/eslint/index.js | 3 ++ esp/src/package-lock.json | 25 +++++------- esp/src/package.json | 2 +- esp/src/src-react/components/ECLArchive.tsx | 2 +- esp/src/src-react/components/Frame.tsx | 2 +- esp/src/src-react/components/Metrics.tsx | 2 +- esp/src/src-react/layouts/DockPanel.tsx | 45 ++++++++++++++++++--- 7 files changed, 57 insertions(+), 24 deletions(-) diff --git a/esp/src/eslint/index.js b/esp/src/eslint/index.js index 3a433ad8a60..c3bdb1cd045 100644 --- a/esp/src/eslint/index.js +++ b/esp/src/eslint/index.js @@ -2,6 +2,9 @@ module.exports = { rules: { "no-src-react": { + meta: { + fixable: "code" + }, create: function (context) { return { ImportDeclaration(node) { diff --git a/esp/src/package-lock.json b/esp/src/package-lock.json index e15b7782995..ecb1390c775 100644 --- a/esp/src/package-lock.json +++ b/esp/src/package-lock.json @@ -31,7 +31,7 @@ "@hpcc-js/timeline": "2.53.0", "@hpcc-js/tree": "2.41.0", "@hpcc-js/util": "2.52.0", - "@hpcc-js/wasm": "2.18.1", + "@hpcc-js/wasm": "2.18.2", "@kubernetes/client-node": "0.20.0", "clipboard": "2.0.11", "d3-dsv": "3.0.1", @@ -89,12 +89,13 @@ "dev": true }, "node_modules/@75lb/deep-merge": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/@75lb/deep-merge/-/deep-merge-1.1.1.tgz", - "integrity": "sha512-xvgv6pkMGBA6GwdyJbNAnDmfAIR/DfWhrj9jgWh3TY7gRm3KO46x/GPjRg6wJ0nOepwqrNxFfojebh0Df4h4Tw==", + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/@75lb/deep-merge/-/deep-merge-1.1.2.tgz", + "integrity": "sha512-08K9ou5VNbheZFxM5tDWoqjA3ImC50DiuuJ2tj1yEPRfkp8lLLg6XAaJ4On+a0yAXor/8ay5gHnAIshRM44Kpw==", "dev": true, + "license": "MIT", "dependencies": { - "lodash.assignwith": "^4.2.0", + "lodash": "^4.17.21", "typical": "^7.1.1" }, "engines": { @@ -2334,9 +2335,9 @@ "license": "0BSD" }, "node_modules/@hpcc-js/wasm": { - "version": "2.18.1", - "resolved": "https://registry.npmjs.org/@hpcc-js/wasm/-/wasm-2.18.1.tgz", - "integrity": "sha512-fT8NCOTaF0NDnT+ZwWpV2VQ6ywFEqw+fG87GSPNQemEmg7FFqUaKRQOW9MBICrkZcXaJBb7VHo1t5UF6bi/JgQ==", + "version": "2.18.2", + "resolved": "https://registry.npmjs.org/@hpcc-js/wasm/-/wasm-2.18.2.tgz", + "integrity": "sha512-9FIpuXvIsIY3UbUd/HZPPiaZe6IFIuA6k5j9Lh54QcINP1s9hbMr/na0xjt+qRPXlwZdrOz3zQJBzHEEQDKnCw==", "license": "Apache-2.0", "dependencies": { "yargs": "17.7.2" @@ -8065,12 +8066,6 @@ "resolved": "https://registry.npmjs.org/lodash-es/-/lodash-es-4.17.21.tgz", "integrity": "sha512-mKnC+QJ9pWVzv+C4/U3rRsHapFfHvQFoFB92e52xeyGMcX6/OlIl78je1u8vePzYZSkkogMPJ2yjxxsb89cxyw==" }, - "node_modules/lodash.assignwith": { - "version": "4.2.0", - "resolved": "https://registry.npmjs.org/lodash.assignwith/-/lodash.assignwith-4.2.0.tgz", - "integrity": "sha512-ZznplvbvtjK2gMvnQ1BR/zqPFZmS6jbK4p+6Up4xcRYA7yMIwxHCfbTcrYxXKzzqLsQ05eJPVznEW3tuwV7k1g==", - "dev": true - }, "node_modules/lodash.camelcase": { "version": "4.3.0", "resolved": "https://registry.npmjs.org/lodash.camelcase/-/lodash.camelcase-4.3.0.tgz", @@ -12322,4 +12317,4 @@ } } } -} \ No newline at end of file +} diff --git a/esp/src/package.json b/esp/src/package.json index c1ec1651488..e9d7224d6f4 100644 --- a/esp/src/package.json +++ b/esp/src/package.json @@ -57,7 +57,7 @@ "@hpcc-js/timeline": "2.53.0", "@hpcc-js/tree": "2.41.0", "@hpcc-js/util": "2.52.0", - "@hpcc-js/wasm": "2.18.1", + "@hpcc-js/wasm": "2.18.2", "@kubernetes/client-node": "0.20.0", "clipboard": "2.0.11", "d3-dsv": "3.0.1", diff --git a/esp/src/src-react/components/ECLArchive.tsx b/esp/src/src-react/components/ECLArchive.tsx index 8d63d3ae8f5..a84ceaae3ac 100644 --- a/esp/src/src-react/components/ECLArchive.tsx +++ b/esp/src/src-react/components/ECLArchive.tsx @@ -111,7 +111,7 @@ export const ECLArchive: React.FunctionComponent = ({ return } main={ - + { // Only render after archive is loaded (to ensure it "defaults to open") --- archive?.modAttrs.length && diff --git a/esp/src/src-react/components/Frame.tsx b/esp/src/src-react/components/Frame.tsx index 9821ffdfd43..76b39bd3a5d 100644 --- a/esp/src/src-react/components/Frame.tsx +++ b/esp/src/src-react/components/Frame.tsx @@ -88,7 +88,7 @@ export const Frame: React.FunctionComponent = () => { router.resolve(hashHistory.location).then(setBody); - userKeyValStore().get("user_cookie_consent") + userKeyValStore().get(USER_COOKIE_CONSENT) .then((resp) => { setShowCookieConsent(resp === "1"); }) diff --git a/esp/src/src-react/components/Metrics.tsx b/esp/src/src-react/components/Metrics.tsx index f0df3a94f6a..40413d012c7 100644 --- a/esp/src/src-react/components/Metrics.tsx +++ b/esp/src/src-react/components/Metrics.tsx @@ -634,7 +634,7 @@ export const Metrics: React.FunctionComponent = ({ } main={ - + diff --git a/esp/src/src-react/layouts/DockPanel.tsx b/esp/src/src-react/layouts/DockPanel.tsx index 8ebea0e1c74..d359e0397b9 100644 --- a/esp/src/src-react/layouts/DockPanel.tsx +++ b/esp/src/src-react/layouts/DockPanel.tsx @@ -3,8 +3,8 @@ import * as ReactDOM from "react-dom"; import { Theme, ThemeProvider } from "@fluentui/react"; import { useConst } from "@fluentui/react-hooks"; import { FluentProvider, Theme as ThemeV9 } from "@fluentui/react-components"; -import { HTMLWidget, Widget } from "@hpcc-js/common"; -import { DockPanel as HPCCDockPanel, IClosable } from "@hpcc-js/phosphor"; +import { HTMLWidget, Widget, Utility } from "@hpcc-js/common"; +import { DockPanel as HPCCDockPanel, IClosable, WidgetAdapter } from "@hpcc-js/phosphor"; import { compare2 } from "@hpcc-js/util"; import { lightTheme, lightThemeV9 } from "../themes"; import { useUserTheme } from "../hooks/theme"; @@ -96,6 +96,7 @@ export class ResetableDockPanel extends HPCCDockPanel { protected _origLayout: DockPanelLayout | undefined; protected _lastLayout: DockPanelLayout | undefined; + protected _visibility: { [id: string]: boolean }; resetLayout() { if (this._origLayout) { @@ -118,8 +119,19 @@ export class ResetableDockPanel extends HPCCDockPanel { return formatLayout(this.layout()) ?? this._lastLayout ?? this._origLayout; } + getVisibility() { + return this._visibility; + } + render(callback?: (w: Widget) => void) { - const retVal = super.render(); + const retVal = this._visibility !== undefined ? super.render() : super.render(() => { + if (this._visibility === undefined) { + this._visibility = {}; + this.widgetAdapters().forEach(wa => { + this._visibility[wa.widget.id()] = wa.isVisible; + }); + } + }); if (this._origLayout === undefined) { this._origLayout = formatLayout(this.layout()); } @@ -130,9 +142,27 @@ export class ResetableDockPanel extends HPCCDockPanel { } // Events --- + childActivation(w: Widget, wa: WidgetAdapter) { + } + + childVisibility(w: Widget, visible: boolean, wa: WidgetAdapter) { + if (this._visibility && this._visibility[w.id()] !== visible) { + this._visibility[w.id()] = visible; + this._lazyVisibilityChanged(); + } + } + layoutChanged() { this._lastLayout = this.getLayout(); } + + // Exposed Events --- + private _lazyVisibilityChanged = Utility.debounce(async () => { + this.visibilityChanged(this._visibility); + }, 60); + + visibilityChanged(visibility: { [id: string]: boolean }) { + } } interface DockPanelItemProps { @@ -154,14 +184,16 @@ export const DockPanelItem: React.FunctionComponent = ({ interface DockPanelProps { layout?: object; hideSingleTabs?: boolean; - onDockPanelCreate?: (dockpanel: ResetableDockPanel) => void; + onCreate?: (dockpanel: ResetableDockPanel) => void; + onVisibilityChanged?: (visibility: { [id: string]: boolean }) => void; children?: React.ReactElement | React.ReactElement[]; } export const DockPanel: React.FunctionComponent = ({ layout, hideSingleTabs, - onDockPanelCreate, + onCreate: onDockPanelCreate, + onVisibilityChanged: onDockPanelVisibilityChanged, children }) => { const items = React.useMemo(() => { @@ -179,6 +211,9 @@ export const DockPanel: React.FunctionComponent = ({ onDockPanelCreate(retVal); }, 0); } + if (onDockPanelVisibilityChanged) { + retVal.on("visibilityChanged", visibility => onDockPanelVisibilityChanged(visibility), true); + } return retVal; }); From 2311d12cdde170bce5aea19e76076359367c88ac Mon Sep 17 00:00:00 2001 From: Gordon Smith Date: Fri, 2 Aug 2024 16:32:52 +0100 Subject: [PATCH 03/25] HPCC-32418 Make toolbar optional for all source editor components Signed-off-by: Gordon Smith --- esp/src/src-react/components/SourceEditor.tsx | 32 +++++++++++++------ 1 file changed, 22 insertions(+), 10 deletions(-) diff --git a/esp/src/src-react/components/SourceEditor.tsx b/esp/src/src-react/components/SourceEditor.tsx index 5e070c6f419..5164848df25 100644 --- a/esp/src/src-react/components/SourceEditor.tsx +++ b/esp/src/src-react/components/SourceEditor.tsx @@ -147,38 +147,44 @@ export const SourceEditor: React.FunctionComponent = ({ interface TextSourceEditorProps { text: string; readonly?: boolean; + toolbar?: boolean; } export const TextSourceEditor: React.FunctionComponent = ({ text = "", - readonly = false + readonly, + toolbar }) => { - return ; + return ; }; interface XMLSourceEditorProps { text: string; readonly?: boolean; + toolbar?: boolean; } export const XMLSourceEditor: React.FunctionComponent = ({ text = "", - readonly = false + readonly, + toolbar }) => { - return ; + return ; }; interface JSONSourceEditorProps { json?: object; readonly?: boolean; + toolbar?: boolean; onChange?: (obj: object) => void; } export const JSONSourceEditor: React.FunctionComponent = ({ json, - readonly = false, + readonly, + toolbar, onChange = (obj: object) => { } }) => { @@ -197,7 +203,7 @@ export const JSONSourceEditor: React.FunctionComponent = } }, [onChange]); - return ; + return ; }; export interface WUXMLSourceEditorProps { @@ -215,10 +221,12 @@ export const WUXMLSourceEditor: React.FunctionComponent export interface WUResourceEditorProps { src: string; + toolbar?: boolean; } export const WUResourceEditor: React.FunctionComponent = ({ - src + src, + toolbar }) => { const [text, setText] = React.useState(""); @@ -231,7 +239,7 @@ export const WUResourceEditor: React.FunctionComponent = }); }, [src]); - return ; + return ; }; interface ECLSourceEditorProps { @@ -266,6 +274,7 @@ interface FetchEditor { url: string; wuid?: string; readonly?: boolean; + toolbar?: boolean; mode?: "ecl" | "xml" | "text"; } @@ -273,6 +282,7 @@ export const FetchEditor: React.FunctionComponent = ({ url, wuid, readonly = true, + toolbar, mode = "text" }) => { @@ -293,11 +303,12 @@ export const FetchEditor: React.FunctionComponent = ({ } }, [url, wuid]); - return ; + return ; }; interface SQLSourceEditorProps { sql: string; + readonly?: boolean; toolbar?: boolean; onSqlChange?: (sql: string) => void; onFetchHints?: (cm: any, option: any) => Promise; @@ -306,11 +317,12 @@ interface SQLSourceEditorProps { export const SQLSourceEditor: React.FunctionComponent = ({ sql, + readonly, toolbar, onSqlChange, onFetchHints, onSubmit }) => { - return ; + return ; }; From 8501cd0e6041a81d1c3233935b6e399264c04263 Mon Sep 17 00:00:00 2001 From: Gordon Smith Date: Thu, 8 Aug 2024 17:20:19 +0100 Subject: [PATCH 04/25] Split off 9.2.112 Signed-off-by: Gordon Smith --- helm/hpcc/Chart.yaml | 4 ++-- helm/hpcc/templates/_helpers.tpl | 2 +- version.cmake | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/helm/hpcc/Chart.yaml b/helm/hpcc/Chart.yaml index 54a76856aaf..541ec0b4765 100644 --- a/helm/hpcc/Chart.yaml +++ b/helm/hpcc/Chart.yaml @@ -6,9 +6,9 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. -version: 9.2.111-closedown0 +version: 9.2.113-closedown0 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. -appVersion: 9.2.111-closedown0 +appVersion: 9.2.113-closedown0 diff --git a/helm/hpcc/templates/_helpers.tpl b/helm/hpcc/templates/_helpers.tpl index 313cafb9530..c35fa1d92b0 100644 --- a/helm/hpcc/templates/_helpers.tpl +++ b/helm/hpcc/templates/_helpers.tpl @@ -1361,7 +1361,7 @@ Pass in dict with .root, .visibility defined {{- end -}} {{- define "hpcc.generateHelmVersion" -}} -helmVersion: 9.2.111-closedown0 +helmVersion: 9.2.113-closedown0 {{- end -}} {{/* diff --git a/version.cmake b/version.cmake index 375f81065b7..3ff7c348647 100644 --- a/version.cmake +++ b/version.cmake @@ -5,8 +5,8 @@ set ( HPCC_NAME "Community Edition" ) set ( HPCC_PROJECT "community" ) set ( HPCC_MAJOR 9 ) set ( HPCC_MINOR 2 ) -set ( HPCC_POINT 111 ) +set ( HPCC_POINT 113 ) set ( HPCC_MATURITY "closedown" ) set ( HPCC_SEQUENCE 0 ) -set ( HPCC_TAG_TIMESTAMP "2024-08-02T10:48:12Z" ) +set ( HPCC_TAG_TIMESTAMP "2024-08-08T16:20:19Z" ) ### From 2e827e20c65605a9d26334c44c09f53346ccf516 Mon Sep 17 00:00:00 2001 From: James McMullan Date: Thu, 8 Aug 2024 14:53:07 -0400 Subject: [PATCH 05/25] HPCC4-32419 Jirabot Improve transition flow and input handling - Removed the need for the issue transition map - Various security improvements around input handling Signed-off-by: James McMullan James.McMullan@lexisnexis.com --- .github/workflows/jirabot.yml | 80 ++++++++++++++++++----------------- 1 file changed, 41 insertions(+), 39 deletions(-) diff --git a/.github/workflows/jirabot.yml b/.github/workflows/jirabot.yml index 277f6e7afb7..2f1db9c80f2 100644 --- a/.github/workflows/jirabot.yml +++ b/.github/workflows/jirabot.yml @@ -38,33 +38,61 @@ jobs: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} GHUB_JIRA_USER_MAP: ${{ vars.GHUB_JIRA_USER_MAP }} JIRA_ISSUE_PROPERTY_MAP: ${{ vars.JIRA_ISSUE_PROPERTY_MAP }} - JIRA_ISSUE_TRANSITION_MAP: ${{ vars.JIRA_ISSUE_TRANSITION_MAP }} run: | import os import re import time import sys import json + import subprocess + from email.utils import parseaddr from atlassian.jira import Jira - def updateIssue(jira, issue, prAuthor : str, transitionMap: dict, propertyMap: dict, pull_url: str) -> str: + def sanitizeInput(input: str, inputType: str) -> str: + if inputType.lower() == 'email': + # Return the email address only, returns '' if not valid or found + return parseaddr(input)[1] + else: + return '' + + def updateIssue(jira, issue, prAuthor : str, propertyMap: dict, pull_url: str) -> str: result = '' issueName = issue['key'] issueFields = issue['fields'] - statusName = str(issueFields['status']['name']) - transition = transitionMap.get(statusName, None) + # Need to update user first in case we are starting from Unresourced + if prAuthor: + assignee = issueFields['assignee'] + if assignee is None: + assigneeId = '' + assigneeEmail = '' + else: + assigneeId = assignee['accountId'] + assigneeEmail = assignee["emailAddress"] + + assigneeEmail = sanitizeInput(assigneeEmail, 'email') + + prAuthorId = prAuthor["accountId"] + prAuthorEmail = prAuthor["emailAddress"] + prAuthorEmail = sanitizeInput(prAuthorEmail, 'email') + + if assigneeId is None or assigneeId == '': + jira.assign_issue(issueName, prAuthorId) + result += 'Assigning user: ' + prAuthorEmail + '\n' + elif assigneeId != prAuthorId: + result += 'Changing assignee from: ' + assigneeEmail + ' to: ' + prAuthorEmail + '\n' + jira.assign_issue(issueName, prAuthorId) - if transition == None: - print('Error: Unable to find transition for status: ' + statusName) - elif transition != '': + transitionFlow = ['Merge Pending'] + for desiredStatus in transitionFlow: try: - jira.issue_transition(issueName, transition) - result += 'Workflow Transition: ' + transition + '\n' + transitionId = jira.get_transition_id_to_status_name(issueName, desiredStatus) + jira.set_issue_status_by_transition_id(issueName, transitionId) + result += 'Workflow Transition To: ' + desiredStatus + '\n' except Exception as error: transitions = jira.get_issue_transitions(issueName) - result += 'Error: Transition: "' + transition + '" failed with: "' + str(error) + '" Valid transitions=' + str(transitions) + '\n' + result += 'Error: Transitioning to: "' + desiredStatus + '" failed with: "' + str(error) + '" Valid transitions=' + str(transitions) + '\n' prFieldName = propertyMap.get('pullRequestFieldName', 'customfield_10010') @@ -80,24 +108,6 @@ jobs: elif currentPR is not None and currentPR != pull_url: result += 'Additional PR: ' + pull_url + '\n' - if prAuthor: - assignee = issueFields['assignee'] - if assignee is None: - assigneeId = '' - assigneeEmail = '' - else: - assigneeId = assignee['accountId'] - assigneeEmail = assignee["emailAddress"] - - prAuthorId = prAuthor["accountId"] - prAuthorEmail = prAuthor["emailAddress"] - if assigneeId is None or assigneeId == '': - jira.assign_issue(issueName, prAuthorId) - result += 'Assigning user: ' + prAuthorEmail + '\n' - elif assigneeId != prAuthorId: - result += 'Changing assignee from: ' + assigneeEmail + ' to: ' + prAuthorEmail + '\n' - jira.assign_issue(issueName, prAuthorId) - return result jirabot_user = os.environ['JIRABOT_USERNAME'] @@ -110,7 +120,6 @@ jobs: github_token = os.environ['GITHUB_TOKEN'] comments_url = os.environ['COMMENTS_URL'] - print("%s %s %s" % (title, prAuthor, comments_url)) result = '' issuem = re.search("(HPCC|HH|IDE|EPE|ML|HPCC4J|JAPI)-[0-9]+", title) if issuem: @@ -131,7 +140,7 @@ jobs: if userSearchResults and len(userSearchResults) > 0: jiraUser = userSearchResults[0] else: - print('Error: Unable to find Jira user: ' + prAuthor + ' continuing without assigning') + print('Error: Unable to map GitHub user to Jira user, continuing without assigning') if not jira.issue_exists(issue_name): sys.exit('Error: Unable to find Jira issue: ' + issue_name) @@ -140,17 +149,12 @@ jobs: result = 'Jirabot Action Result:\n' - transitionMap = json.loads(os.environ['JIRA_ISSUE_TRANSITION_MAP']) - if not isinstance(transitionMap, dict): - print('Error: JIRA_ISSUE_TRANSITION_MAP is not a valid JSON object, ignoring.') - transitionMap = {} - jiraIssuePropertyMap = json.loads(os.environ['JIRA_ISSUE_PROPERTY_MAP']) if not isinstance(jiraIssuePropertyMap, dict): print('Error: JIRA_ISSUE_PROPERTY_MAP is not a valid JSON object, ignoring.') jiraIssuePropertyMap = {} - result += updateIssue(jira, issue, jiraUser, transitionMap, jiraIssuePropertyMap, pull_url) + result += updateIssue(jira, issue, jiraUser, jiraIssuePropertyMap, pull_url) jira.issue_add_comment(issue_name, result) result = 'Jira Issue: ' + jira_url + '/browse/' + issue_name + '\n\n' + result @@ -158,9 +162,7 @@ jobs: # Escape the result for JSON result = json.dumps(result) - curlCommand = 'curl -X POST %s -H "Content-Type: application/json" -H "Authorization: token %s" --data \'{ "body": %s }\'' % ( comments_url, github_token, result ) - print(curlCommand) - os.system(curlCommand) + subprocess.run(['curl', '-X', 'POST', comments_url, '-H', 'Content-Type: application/json', '-H', f'Authorization: token {github_token}', '--data', f'{{ "body": {result} }}'], check=True) else: print('Unable to find Jira issue name in title') From b1ac7e689c12b7699a30aad0a16280ccc7a300b2 Mon Sep 17 00:00:00 2001 From: Jake Smith Date: Thu, 15 Aug 2024 16:43:09 +0100 Subject: [PATCH 06/25] Split off 9.6.40 Signed-off-by: Jake Smith --- helm/hpcc/Chart.yaml | 4 ++-- helm/hpcc/templates/_helpers.tpl | 2 +- version.cmake | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/helm/hpcc/Chart.yaml b/helm/hpcc/Chart.yaml index 1ccd1c3af10..71d9c5bc171 100644 --- a/helm/hpcc/Chart.yaml +++ b/helm/hpcc/Chart.yaml @@ -6,9 +6,9 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. -version: 9.6.39-closedown0 +version: 9.6.41-closedown0 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. -appVersion: 9.6.39-closedown0 +appVersion: 9.6.41-closedown0 diff --git a/helm/hpcc/templates/_helpers.tpl b/helm/hpcc/templates/_helpers.tpl index 298e37c5550..167cea0ec5d 100644 --- a/helm/hpcc/templates/_helpers.tpl +++ b/helm/hpcc/templates/_helpers.tpl @@ -1477,7 +1477,7 @@ Pass in dict with .root, .visibility defined {{- end -}} {{- define "hpcc.generateHelmVersion" -}} -helmVersion: 9.6.39-closedown0 +helmVersion: 9.6.41-closedown0 {{- end -}} {{/* diff --git a/version.cmake b/version.cmake index 90d26bd3200..ed620f6e7b4 100644 --- a/version.cmake +++ b/version.cmake @@ -5,8 +5,8 @@ set ( HPCC_NAME "Community Edition" ) set ( HPCC_PROJECT "community" ) set ( HPCC_MAJOR 9 ) set ( HPCC_MINOR 6 ) -set ( HPCC_POINT 39 ) +set ( HPCC_POINT 41 ) set ( HPCC_MATURITY "closedown" ) set ( HPCC_SEQUENCE 0 ) -set ( HPCC_TAG_TIMESTAMP "2024-08-08T16:18:20Z" ) +set ( HPCC_TAG_TIMESTAMP "2024-08-15T15:43:09Z" ) ### From 952a8b692b5575a4dca447557d5679ec29ef7699 Mon Sep 17 00:00:00 2001 From: Jake Smith Date: Thu, 15 Aug 2024 16:44:19 +0100 Subject: [PATCH 07/25] Split off 9.4.88 Signed-off-by: Jake Smith --- helm/hpcc/Chart.yaml | 4 ++-- helm/hpcc/templates/_helpers.tpl | 2 +- version.cmake | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/helm/hpcc/Chart.yaml b/helm/hpcc/Chart.yaml index f6fd5069a08..6b890d3a750 100644 --- a/helm/hpcc/Chart.yaml +++ b/helm/hpcc/Chart.yaml @@ -6,9 +6,9 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. -version: 9.4.87-closedown0 +version: 9.4.89-closedown0 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. -appVersion: 9.4.87-closedown0 +appVersion: 9.4.89-closedown0 diff --git a/helm/hpcc/templates/_helpers.tpl b/helm/hpcc/templates/_helpers.tpl index 112bf329345..6a04a569072 100644 --- a/helm/hpcc/templates/_helpers.tpl +++ b/helm/hpcc/templates/_helpers.tpl @@ -1473,7 +1473,7 @@ Pass in dict with .root, .visibility defined {{- end -}} {{- define "hpcc.generateHelmVersion" -}} -helmVersion: 9.4.87-closedown0 +helmVersion: 9.4.89-closedown0 {{- end -}} {{/* diff --git a/version.cmake b/version.cmake index 36b64302db2..d1c48e4651d 100644 --- a/version.cmake +++ b/version.cmake @@ -5,8 +5,8 @@ set ( HPCC_NAME "Community Edition" ) set ( HPCC_PROJECT "community" ) set ( HPCC_MAJOR 9 ) set ( HPCC_MINOR 4 ) -set ( HPCC_POINT 87 ) +set ( HPCC_POINT 89 ) set ( HPCC_MATURITY "closedown" ) set ( HPCC_SEQUENCE 0 ) -set ( HPCC_TAG_TIMESTAMP "2024-08-08T16:19:19Z" ) +set ( HPCC_TAG_TIMESTAMP "2024-08-15T15:44:19Z" ) ### From 431e4e30ba6c842a9252dc060176a0337acb3e9a Mon Sep 17 00:00:00 2001 From: Jake Smith Date: Thu, 15 Aug 2024 16:45:07 +0100 Subject: [PATCH 08/25] Split off 9.2.114 Signed-off-by: Jake Smith --- helm/hpcc/Chart.yaml | 4 ++-- helm/hpcc/templates/_helpers.tpl | 2 +- version.cmake | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/helm/hpcc/Chart.yaml b/helm/hpcc/Chart.yaml index 541ec0b4765..38711d74f46 100644 --- a/helm/hpcc/Chart.yaml +++ b/helm/hpcc/Chart.yaml @@ -6,9 +6,9 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. -version: 9.2.113-closedown0 +version: 9.2.115-closedown0 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. -appVersion: 9.2.113-closedown0 +appVersion: 9.2.115-closedown0 diff --git a/helm/hpcc/templates/_helpers.tpl b/helm/hpcc/templates/_helpers.tpl index c35fa1d92b0..00103293fa4 100644 --- a/helm/hpcc/templates/_helpers.tpl +++ b/helm/hpcc/templates/_helpers.tpl @@ -1361,7 +1361,7 @@ Pass in dict with .root, .visibility defined {{- end -}} {{- define "hpcc.generateHelmVersion" -}} -helmVersion: 9.2.113-closedown0 +helmVersion: 9.2.115-closedown0 {{- end -}} {{/* diff --git a/version.cmake b/version.cmake index 3ff7c348647..9b6027ac6bc 100644 --- a/version.cmake +++ b/version.cmake @@ -5,8 +5,8 @@ set ( HPCC_NAME "Community Edition" ) set ( HPCC_PROJECT "community" ) set ( HPCC_MAJOR 9 ) set ( HPCC_MINOR 2 ) -set ( HPCC_POINT 113 ) +set ( HPCC_POINT 115 ) set ( HPCC_MATURITY "closedown" ) set ( HPCC_SEQUENCE 0 ) -set ( HPCC_TAG_TIMESTAMP "2024-08-08T16:20:19Z" ) +set ( HPCC_TAG_TIMESTAMP "2024-08-15T15:45:07Z" ) ### From 56fafb8ad6529a9d731f6cde6612201b3ff053b8 Mon Sep 17 00:00:00 2001 From: Jake Smith Date: Thu, 15 Aug 2024 17:57:49 +0100 Subject: [PATCH 09/25] HPCC-32446 Identify the Thor instance name in interface incompatibility error Signed-off-by: Jake Smith --- thorlcr/master/thgraphmanager.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/thorlcr/master/thgraphmanager.cpp b/thorlcr/master/thgraphmanager.cpp index 5eb01feaca4..9707b5a48d1 100644 --- a/thorlcr/master/thgraphmanager.cpp +++ b/thorlcr/master/thgraphmanager.cpp @@ -545,7 +545,7 @@ bool CJobManager::execute(IConstWorkUnit *workunit, const char *wuid, const char if (workunit->getCodeVersion() == 0) throw makeStringException(0, "Attempting to execute a workunit that hasn't been compiled"); if ((workunit->getCodeVersion() > ACTIVITY_INTERFACE_VERSION) || (workunit->getCodeVersion() < MIN_ACTIVITY_INTERFACE_VERSION)) - throw MakeStringException(0, "Workunit was compiled for eclagent interface version %d, this thor requires version %d..%d", workunit->getCodeVersion(), MIN_ACTIVITY_INTERFACE_VERSION, ACTIVITY_INTERFACE_VERSION); + throw MakeStringException(0, "Workunit was compiled for eclagent interface version %d, this thor (%s) requires version %d..%d", workunit->getCodeVersion(), globals->queryProp("@name"), MIN_ACTIVITY_INTERFACE_VERSION, ACTIVITY_INTERFACE_VERSION); if (workunit->getCodeVersion() == 652) { // Any workunit compiled using eclcc 7.12.0-7.12.18 is not compatible @@ -557,7 +557,7 @@ bool CJobManager::execute(IConstWorkUnit *workunit, const char *wuid, const char const char *point = version + strlen("7.12."); unsigned pointVer = atoi(point); if (pointVer <= 18) - throw MakeStringException(0, "Workunit was compiled by eclcc version %s which is not compatible with this runtime", buildVersion.str()); + throw MakeStringException(0, "Workunit was compiled by eclcc version %s which is not compatible with this thor (%s)", buildVersion.str(), globals->queryProp("@name")); } } From 6e9fee15c6c5fcc508348dc281791ea647b5a349 Mon Sep 17 00:00:00 2001 From: Gavin Halliday Date: Fri, 16 Aug 2024 14:23:46 +0100 Subject: [PATCH 10/25] HPCC-32447 Fix very slow spilling of rows containing very large child datasets Signed-off-by: Gavin Halliday --- system/jlib/jstream.cpp | 11 +++- testing/unittests/jstreamtests.cpp | 96 ++++++++++++++++++++++++++++++ 2 files changed, 105 insertions(+), 2 deletions(-) diff --git a/system/jlib/jstream.cpp b/system/jlib/jstream.cpp index 8f31718a8e1..c0ea60f0618 100644 --- a/system/jlib/jstream.cpp +++ b/system/jlib/jstream.cpp @@ -896,8 +896,15 @@ class CBlockedSerialOutputStream final : public CInterfaceOf 1M + size32_t alignment = buffer.length() / 4; + if (alignment < 32) + alignment = 32; + newLength += (alignment - 1); + newLength -= newLength % alignment; MemoryAttr expandedBuffer(newLength); memcpy(expandedBuffer.mem(), data(0), bufferOffset); diff --git a/testing/unittests/jstreamtests.cpp b/testing/unittests/jstreamtests.cpp index dda4523835d..23ee3dccd8b 100644 --- a/testing/unittests/jstreamtests.cpp +++ b/testing/unittests/jstreamtests.cpp @@ -282,6 +282,87 @@ class VariableDataProvider : public CDataProvider }; +//A very large row (because of a large embedded dataset) +//100 bytes of data then 500K rows of 100 bytes +class LargeRowDataProvider : public CDataProvider +{ +public: + LargeRowDataProvider(bool _useCount, unsigned _numChildren) : useCount(_useCount), numChildren(_numChildren) + { + name.append("Large_").append(useCount ? 'C' : 'S').append(numChildren); + } + + virtual size32_t create(IBufferedSerialOutputStream * target, unsigned row) + { + //Output (row, (string)row, (row % 7)items of (row, row*2, row*3)) + byte mainRow[100]; + unsigned childRow[25]; + + for (size32_t i=0; i < sizeof(mainRow); i++) + mainRow[i] = (byte)(i * row); + target->put(sizeof(mainRow), mainRow); + + size32_t childCount = numChildren + row; + size32_t childSize = sizeof(childRow) * childCount; + if (useCount) + target->put(4, &childCount); + else + target->suspend(sizeof(size32_t)); + + unsigned next = 1234 + row * 31419264U; + for (unsigned i=0; i < childCount; i++) + { + for (size32_t i=0; i < sizeof(mainRow)/sizeof(next); i++) + { + childRow[i] = next; + next *= 0x13894225; + next += row; + } + target->put(sizeof(childRow), &childRow); + } + if (!useCount) + target->resume(sizeof(childSize), &childSize); + + return sizeof(mainRow) + 4 + childSize; + } + + virtual size32_t check(IBufferedSerialInputStream * source, unsigned row) + { + byte mainRow[100]; + unsigned childRow[25]; + + source->read(sizeof(mainRow), &mainRow); + for (size32_t i=0; i < sizeof(mainRow); i++) + assertex(mainRow[i] == (byte)(i * row)); + + size32_t childCount = numChildren + row; + size32_t childSize = sizeof(childRow) * childCount; + size32_t size; + source->read(sizeof(size), &size); + if (useCount) + assertex(size == childCount); + else + assertex(size == childSize); + + unsigned next = 1234 + row * 31419264U; + for (unsigned i=0; i < childCount; i++) + { + source->read(sizeof(childRow), &childRow); + for (size32_t i=0; i < sizeof(mainRow)/sizeof(next); i++) + { + assertex(childRow[i] == next); + next *= 0x13894225; + next += row; + } + } + return sizeof(mainRow) + 4 + childSize; + } + +protected: + bool useCount; + unsigned numChildren = 10'000'000; +}; + class NullOuputStream : public CInterfaceOf { virtual size32_t write(size32_t len, const void * ptr) { return len; } @@ -305,6 +386,7 @@ class JlibStreamStressTest : public CppUnit::TestFixture CPPUNIT_TEST(testEvenSequentialStream); // write a file and read results after each flush CPPUNIT_TEST(testParallelStream); // write a file and read in parallel from a separate thread CPPUNIT_TEST(testThreadedWriteStream); // write a file using a threaded writer + CPPUNIT_TEST(testPathologicalRows); // 1M child rows, total row size 100MB //MORE: //Threaded writer //Threaded reader @@ -735,6 +817,20 @@ class JlibStreamStressTest : public CppUnit::TestFixture } } + void testPathologicalRows() + { + LargeRowDataProvider largeCount50K(true, 50'000); + LargeRowDataProvider largeCount10M(true, 10'000'000); + LargeRowDataProvider largeSize50K(false, 50'000); + LargeRowDataProvider largeSize10M(false, 10'000'000); + + ICompressHandler * lz4 = queryCompressHandler(COMPRESS_METHOD_LZ4); + + runSimpleStream(nullptr, largeCount50K, 0x100000, 0x100000, 2000); + runSimpleStream(nullptr, largeCount10M, 0x100000, 0x100000, 10); + runSimpleStream(nullptr, largeSize50K, 0x100000, 0x100000, 2000); + runSimpleStream(nullptr, largeSize10M, 0x100000, 0x100000, 10); + } void testIncSequentialStream() { From 76dca1dcabdc3cadf8aab1d3a11963bf6261cea5 Mon Sep 17 00:00:00 2001 From: M Kelly Date: Mon, 19 Aug 2024 08:24:47 -0400 Subject: [PATCH 11/25] HPCC-32454 SYS: swap and mem util stats fixup previous change Signed-off-by: M Kelly --- system/jlib/jdebug.cpp | 38 ++++++++++++++++++++++++++++++++++---- 1 file changed, 34 insertions(+), 4 deletions(-) diff --git a/system/jlib/jdebug.cpp b/system/jlib/jdebug.cpp index 17154d4a99c..79c9c36bbb7 100644 --- a/system/jlib/jdebug.cpp +++ b/system/jlib/jdebug.cpp @@ -1454,10 +1454,40 @@ static void getMemUsage(unsigned &inuse,unsigned &active,unsigned &total,unsigne } inuse = total-free-cached; - // not sure if a bug in kernel or container or ... - // but sometimes we see swapfree > 0 when swaptotal == 0 - if ((swapfree + swapcached) >= swaptotal) - swapinuse = 0; + // swapinuse = swaptotal-swapfree-swapcached; + + // sometimes in containers [under mem pressure ?] we see from /proc/meminfo - + + // SwapCached: 0 kB + // SwapTotal: 0 kB + // SwapFree: 18446744073709551496 kB + // or - + // SwapCached: 0 kB + // SwapTotal: 0 kB + // SwapFree: 120 kB + + // and from free cmd - + + // free -m + // total used free shared buff/cache available + // Mem: 43008 17616 5375 0 20015 25391 + // Swap: 0 18014398509481984 0 + + // if swapfree > 0 when swaptotal == 0 - + // *might* indicate kernel is pushing exe/mmapped pages out of memory to make room + // for other things and this can affect performance + + // not sure why SwapFree value is not always valid/accurate + // vmstat shows more reasonable swpd value, but walks all /proc//stat files + + // SwapCached: Memory that is present within main memory, but also in the swapfile + + if ((swapfree + swapcached) > swaptotal) + { + swapinuse = swapfree + swapcached; + if (swapinuse > total) + swapinuse = active; // something more reasonable ... + } else swapinuse = swaptotal-swapfree-swapcached; #endif From c0e20f604ab9d71a4e4712b54752a5f29f802ddb Mon Sep 17 00:00:00 2001 From: Anthony Fishbeck Date: Mon, 19 Aug 2024 14:30:16 -0400 Subject: [PATCH 12/25] HPCC-32462 Wildcard certs should work with ClusterIssuers Signed-off-by: Anthony Fishbeck --- helm/hpcc/templates/issuers.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/helm/hpcc/templates/issuers.yaml b/helm/hpcc/templates/issuers.yaml index 9b52fb85800..77dc5e2b3bf 100644 --- a/helm/hpcc/templates/issuers.yaml +++ b/helm/hpcc/templates/issuers.yaml @@ -119,8 +119,8 @@ spec: --- {{- end }} {{- end }} - {{ include "hpcc.addWildIssuerCertificate" (dict "root" .root "issuerKeyName" .issuerKeyName "me" .me ) }} {{- end }} + {{ include "hpcc.addWildIssuerCertificate" (dict "root" .root "issuerKeyName" .issuerKeyName "me" .me ) }} {{- end }} {{- template "hpcc.ensureNoResourceValidationFlag" ( dict "root" $ ) }} From a4cc067029ae61a635754b6ff7caee61e9e11860 Mon Sep 17 00:00:00 2001 From: Jeremy Clements <79224539+jeclrsg@users.noreply.github.com> Date: Mon, 19 Aug 2024 15:16:00 -0400 Subject: [PATCH 13/25] HPCC-32425 ECL Watch v9 copy to clipboard missing columns fixes some missing headers and columns when using the copy selection to clipboard from the Files, Workunits and Queries list pages Signed-off-by: Jeremy Clements <79224539+jeclrsg@users.noreply.github.com> --- esp/src/src-react/components/Files.tsx | 7 ++++++- esp/src/src-react/components/Queries.tsx | 11 +++++++---- esp/src/src-react/components/Workunits.tsx | 18 ++++++++---------- 3 files changed, 21 insertions(+), 15 deletions(-) diff --git a/esp/src/src-react/components/Files.tsx b/esp/src/src-react/components/Files.tsx index 08f5f953c34..6cc33214cbb 100644 --- a/esp/src/src-react/components/Files.tsx +++ b/esp/src/src-react/components/Files.tsx @@ -156,6 +156,7 @@ export const Files: React.FunctionComponent = ({ } return ""; }, + field: nlsHPCC.Protected, }, IsCompressed: { headerIcon: "ZipFolder", @@ -168,6 +169,7 @@ export const Files: React.FunctionComponent = ({ } return ""; }, + field: nlsHPCC.Compressed, }, Name: { label: nlsHPCC.LogicalName, @@ -194,18 +196,21 @@ export const Files: React.FunctionComponent = ({ formatter: (value, row) => { return Utility.formatNum(row.IntRecordCount); }, + csvFormatter: (value, row) => row.IntRecordCount, }, FileSize: { label: nlsHPCC.Size, formatter: (value, row) => { return Utility.convertedSize(row.IntSize); }, + csvFormatter: (value, row) => row.IntSize, }, CompressedFileSizeString: { label: nlsHPCC.CompressedSize, formatter: (value, row) => { return Utility.convertedSize(row.CompressedFileSize); - } + }, + csvFormatter: (value, row) => row.CompressedFileSize, }, Parts: { label: nlsHPCC.Parts, width: 40, diff --git a/esp/src/src-react/components/Queries.tsx b/esp/src/src-react/components/Queries.tsx index e11938a5734..ff45b2a7f6d 100644 --- a/esp/src/src-react/components/Queries.tsx +++ b/esp/src/src-react/components/Queries.tsx @@ -112,7 +112,8 @@ export const Queries: React.FunctionComponent = ({ return ; } return ""; - } + }, + field: nlsHPCC.Suspended, }, ErrorCount: { headerIcon: "Warning", @@ -124,7 +125,8 @@ export const Queries: React.FunctionComponent = ({ return ; } return ""; - } + }, + field: nlsHPCC.ErrorWarnings, }, MixedNodeStates: { headerIcon: "Error", @@ -136,7 +138,7 @@ export const Queries: React.FunctionComponent = ({ return ; } return ""; - } + }, }, Activated: { headerIcon: "SkypeCircleCheck", @@ -147,7 +149,8 @@ export const Queries: React.FunctionComponent = ({ return ; } return ""; - } + }, + field: nlsHPCC.Active, }, Id: { label: nlsHPCC.ID, diff --git a/esp/src/src-react/components/Workunits.tsx b/esp/src/src-react/components/Workunits.tsx index 0fce3d69f16..ab47cf34673 100644 --- a/esp/src/src-react/components/Workunits.tsx +++ b/esp/src/src-react/components/Workunits.tsx @@ -116,7 +116,8 @@ export const Workunits: React.FunctionComponent = ({ return ; } return ""; - } + }, + field: nlsHPCC.Protected, }, Wuid: { label: nlsHPCC.WUID, width: 120, @@ -143,23 +144,20 @@ export const Workunits: React.FunctionComponent = ({ "Compile Cost": { label: nlsHPCC.CompileCost, width: 100, justify: "right", - formatter: (cost, row) => { - return `${formatCost(row.CompileCost)}`; - } + formatter: (cost, row) => `${formatCost(row.CompileCost)}`, + csvFormatter: (cost, row) => row.CompileCost, }, "Execution Cost": { label: nlsHPCC.ExecuteCost, width: 100, justify: "right", - formatter: (cost, row) => { - return `${formatCost(row.ExecuteCost)}`; - } + formatter: (cost, row) => `${formatCost(row.ExecuteCost)}`, + csvFormatter: (cost, row) => row.ExecuteCost, }, "File Access Cost": { label: nlsHPCC.FileAccessCost, width: 100, justify: "right", - formatter: (cost, row) => { - return `${formatCost(row.FileAccessCost)}`; - } + formatter: (cost, row) => `${formatCost(row.FileAccessCost)}`, + csvFormatter: (cost, row) => row.FileAccessCost, } }; }, [filter]); From e22748bd4febab5dcf8d83f2ea8f311405a01d8c Mon Sep 17 00:00:00 2001 From: Gavin Halliday Date: Tue, 20 Aug 2024 09:55:01 +0100 Subject: [PATCH 14/25] HPCC-32464 Allow components to start if config watcher cannot be created Signed-off-by: Gavin Halliday --- system/jlib/jptree.cpp | 24 ++++++++++++++++-------- 1 file changed, 16 insertions(+), 8 deletions(-) diff --git a/system/jlib/jptree.cpp b/system/jlib/jptree.cpp index 4aed0538292..d7ba2871c78 100644 --- a/system/jlib/jptree.cpp +++ b/system/jlib/jptree.cpp @@ -8782,14 +8782,22 @@ class CConfigUpdater : public CInterface absoluteConfigFilename.set(std::get<0>(result).c_str()); } }; - fileWatcher.setown(createFileEventWatcher(updateFunc)); - - // watch the path, not the filename, because the filename might not be seen if directories are moved, softlinks are changed.. - StringBuffer path, filename; - splitFilename(absoluteConfigFilename, nullptr, &path, &filename, &filename); - configFilename.set(filename); - fileWatcher->add(path, FileWatchEvents::anyChange); - fileWatcher->start(); + try + { + fileWatcher.setown(createFileEventWatcher(updateFunc)); + + // watch the path, not the filename, because the filename might not be seen if directories are moved, softlinks are changed.. + StringBuffer path, filename; + splitFilename(absoluteConfigFilename, nullptr, &path, &filename, &filename); + configFilename.set(filename); + fileWatcher->add(path, FileWatchEvents::anyChange); + fileWatcher->start(); + } + catch (IException * e) + { + OERRLOG(e, "Failed to start file watcher"); + e->Release(); + } return true; } void executeCallbacks(IPropertyTree *oldComponentConfiguration, IPropertyTree *oldGlobalConfiguration) From 0a6a75107c0b1cc6c7d3d22e91afa426a8787a13 Mon Sep 17 00:00:00 2001 From: Gavin Halliday Date: Mon, 19 Aug 2024 11:37:38 +0100 Subject: [PATCH 15/25] HPCC-32410 Avoid crash if roxie detects a dali disconnect when reloading queries Signed-off-by: Gavin Halliday --- dali/base/dacsds.cpp | 60 +++++++++++++++++++++++++++++++++----------- dali/base/dacsds.ipp | 1 + dali/base/dasds.cpp | 6 +++++ dali/base/dasds.hpp | 1 + 4 files changed, 53 insertions(+), 15 deletions(-) diff --git a/dali/base/dacsds.cpp b/dali/base/dacsds.cpp index 3a23fdf5e4e..8577ad9ac6f 100644 --- a/dali/base/dacsds.cpp +++ b/dali/base/dacsds.cpp @@ -38,8 +38,6 @@ static unsigned clientThrottleLimit; static unsigned clientThrottleDelay; -static ISDSManager *SDSManager=NULL; - static CriticalSection SDScrit; #define CHECK_CONNECTED(XSTR) \ @@ -1290,6 +1288,12 @@ CClientSDSManager::CClientSDSManager() } CClientSDSManager::~CClientSDSManager() +{ + closedown(); + ::Release(properties); +} + +void CClientSDSManager::closedown() { CriticalBlock block(connections.crit); SuperHashIteratorOf iter(connections.queryBaseTable()); @@ -1298,7 +1302,6 @@ CClientSDSManager::~CClientSDSManager() CRemoteConnection &conn = (CRemoteConnection &) iter.query(); conn.setConnected(false); } - ::Release(properties); } bool CClientSDSManager::sendRequest(CMessageBuffer &mb, bool throttle) @@ -2236,32 +2239,59 @@ bool CClientSDSManager::updateEnvironment(IPropertyTree *newEnv, bool forceGroup ////////////// +static ISDSManager * activeSDSManager=NULL; +static ISDSManager * savedSDSManager=NULL; + +MODULE_INIT(INIT_PRIORITY_STANDARD) +{ + return true; +} +MODULE_EXIT() +{ + delete activeSDSManager; + activeSDSManager = nullptr; + delete savedSDSManager; + savedSDSManager = nullptr; +} + ISDSManager &querySDS() { CriticalBlock block(SDScrit); - if (SDSManager) - return *SDSManager; + if (activeSDSManager) + return *activeSDSManager; else if (!queryCoven().inCoven()) { - if (!SDSManager) - SDSManager = new CClientSDSManager(); + if (!activeSDSManager) + activeSDSManager = new CClientSDSManager(); - return *SDSManager; + return *activeSDSManager; } else { - SDSManager = &querySDSServer(); - return *SDSManager; + activeSDSManager = &querySDSServer(); + return *activeSDSManager; } } void closeSDS() { CriticalBlock block(SDScrit); - if (SDSManager) { + + //In roxie this is called when connection to dali is lost, but other threads can still be processing + //CRemoteConnections (see HPCC-32410), which uses an ISDSManager member - accessing a stale manager. + //There can be similar issues at closedown if threads have not been cleaned up properly. + //Do not delete the active SDS manager immediately - save it so that it is deleted on the next call/closedown. + ISDSManager * toDelete = savedSDSManager; + savedSDSManager = activeSDSManager; + activeSDSManager = nullptr; + if (savedSDSManager || toDelete) + { assertex(!queryCoven().inCoven()); // only called by client - try { - delete SDSManager; + try + { + if (savedSDSManager) + savedSDSManager->closedown(); + delete toDelete; } catch (IMP_Exception *e) { @@ -2270,11 +2300,11 @@ void closeSDS() EXCLOG(e, "closeSDS"); e->Release(); } - catch (IDaliClient_Exception *e) { + catch (IDaliClient_Exception *e) + { if (e->errorCode()!=DCERR_server_closed) throw; e->Release(); } - SDSManager = NULL; } } diff --git a/dali/base/dacsds.ipp b/dali/base/dacsds.ipp index e4b9590b955..3bda3dc649d 100644 --- a/dali/base/dacsds.ipp +++ b/dali/base/dacsds.ipp @@ -418,6 +418,7 @@ public: virtual void setConfigOpt(const char *opt, const char *value); virtual unsigned queryCount(const char *xpath); virtual bool updateEnvironment(IPropertyTree *newEnv, bool forceGroupUpdate, StringBuffer &response); + virtual void closedown() override; private: void noteDisconnected(CRemoteConnection &connection); diff --git a/dali/base/dasds.cpp b/dali/base/dasds.cpp index e5daa59a8a3..89caaa9efc5 100644 --- a/dali/base/dasds.cpp +++ b/dali/base/dasds.cpp @@ -2031,6 +2031,7 @@ class CCovenSDSManager : public CSDSManagerBase, implements ISDSManagerServer, i virtual void setConfigOpt(const char *opt, const char *value); virtual unsigned queryCount(const char *xpath); virtual bool updateEnvironment(IPropertyTree *newEnv, bool forceGroupUpdate, StringBuffer &response); + virtual void closedown() override; // ISubscriptionManager impl. virtual void add(ISubscription *subs,SubscriptionId id); @@ -6062,6 +6063,11 @@ CCovenSDSManager::~CCovenSDSManager() config.Release(); } +void CCovenSDSManager::closedown() +{ + //Should never be called - but do not assert since it is harmless and it is better not to report +} + void CCovenSDSManager::validateDeltaBackup() { // check consistency of delta diff --git a/dali/base/dasds.hpp b/dali/base/dasds.hpp index 91a9e6b5048..1a114271bcf 100644 --- a/dali/base/dasds.hpp +++ b/dali/base/dasds.hpp @@ -118,6 +118,7 @@ interface ISDSManager virtual void setConfigOpt(const char *opt, const char *value) = 0; virtual unsigned queryCount(const char *xpath) = 0; virtual bool updateEnvironment(IPropertyTree *newEnv, bool forceGroupUpdate, StringBuffer &response) = 0; + virtual void closedown() = 0; }; extern da_decl const char *queryNotifyHandlerName(IPropertyTree *tree); From e1627ec0cb480d24fe85477eb0f722568d9be43d Mon Sep 17 00:00:00 2001 From: Gavin Halliday Date: Tue, 20 Aug 2024 14:50:44 +0100 Subject: [PATCH 16/25] HPCC-32466 Protect dali against a crash processing a zero length packet Signed-off-by: Gavin Halliday --- system/jlib/jbuff.cpp | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/system/jlib/jbuff.cpp b/system/jlib/jbuff.cpp index e1e535a087c..dc95a5129a4 100644 --- a/system/jlib/jbuff.cpp +++ b/system/jlib/jbuff.cpp @@ -62,12 +62,9 @@ constexpr unsigned BUFF_DOUBLE_LIMIT=4096; constexpr unsigned BUFF_FIRST_CHUNK_SIZE=8; constexpr unsigned BUFF_DETACH_GRANULARITY=16; - -#ifdef _DEBUG +//Always check the length before reading - so ensure that serialization problems are caught +//The overhead is trivial.... #define CHECKREADPOS(len) assertex(readPos+(len)<=length()) -#else -#define CHECKREADPOS(len) -#endif //----------------------------------------------------------------------- From 5884fe91e1d434cc728d97b09123c13cbd3cb800 Mon Sep 17 00:00:00 2001 From: Jake Smith Date: Tue, 13 Aug 2024 14:47:01 +0100 Subject: [PATCH 17/25] HPCC-32421 Suppress foreign groups Prevent foreign groups from being published to Dali. The code used to try to find a matching group (for naming purposes), and then create a unique one. What made it worse, was that if the foreign group was from an AKS system, it would have ports associated with it, and the published groups never have ports, consequently it never matched and always added a new unique (anon) group. This caused a gradual build up of new groups (under /Group), which cause delays for everything that looked up a group elsewhere. Signed-off-by: Jake Smith --- dali/base/dadfs.cpp | 20 ++++++++++++-- dali/base/dafdesc.cpp | 63 +++++++++++++++++++++++++------------------ dali/base/dafdesc.hpp | 6 +++-- 3 files changed, 59 insertions(+), 30 deletions(-) diff --git a/dali/base/dadfs.cpp b/dali/base/dadfs.cpp index 48ec2a1bb63..faa42e68a9e 100644 --- a/dali/base/dadfs.cpp +++ b/dali/base/dadfs.cpp @@ -3839,6 +3839,9 @@ protected: friend class CDistributedFilePart; unsigned nc = fdesc->numClusters(); if (nc) { + unsigned flags = 0; + if (FileDescriptorFlags::none != (FileDescriptorFlags::foreign & fdesc->getFlags())) + flags = IFDSF_FOREIGN_GROUP; for (unsigned i=0;iqueryClusterGroup(i), fdesc->queryPartDiskMapping(i), - &queryNamedGroupStore() + &queryNamedGroupStore(), + flags ); if (!cluster->queryGroup(&queryNamedGroupStore())) @@ -7466,8 +7470,20 @@ class CNamedGroupIterator: implements INamedGroupIterator, public CInterface public: IMPLEMENT_IINTERFACE; CNamedGroupIterator(IRemoteConnection *_conn,IGroup *_matchgroup=NULL,bool _exactmatch=false) - : conn(_conn), matchgroup(_matchgroup) + : conn(_conn) { + if (_matchgroup) + { + // the matchgroup may contain ports, but they are never part of published groups and are not to be used for matching + SocketEndpointArray epa; + for (unsigned i=0; i<_matchgroup->ordinality(); i++) + { + SocketEndpoint ep = _matchgroup->queryNode(i).endpoint(); + ep.port = 0; + epa.append(ep); + } + matchgroup.setown(createIGroup(epa)); + } exactmatch = _exactmatch; pe.setown(conn->queryRoot()->getElements("Group")); } diff --git a/dali/base/dafdesc.cpp b/dali/base/dafdesc.cpp index 825833e086a..b03967a19c2 100644 --- a/dali/base/dafdesc.cpp +++ b/dali/base/dafdesc.cpp @@ -417,7 +417,7 @@ struct CClusterInfo: implements IClusterInfo, public CInterface name.clear(); } StringBuffer gname; - if (resolver->find(group,gname,true)||(group->ordinality()>1)) + if (resolver->find(group,gname,!foreignGroup)||(group->ordinality()>1)) name.set(gname); } } @@ -451,11 +451,13 @@ struct CClusterInfo: implements IClusterInfo, public CInterface checkClusterName(resolver); } - CClusterInfo(const char *_name,IGroup *_group,const ClusterPartDiskMapSpec &_mspec,INamedGroupStore *resolver) + CClusterInfo(const char *_name,IGroup *_group,const ClusterPartDiskMapSpec &_mspec,INamedGroupStore *resolver,unsigned flags) : name(_name),group(_group) { name.toLowerCase(); mspec =_mspec; + if (flags & IFDSF_FOREIGN_GROUP) + foreignGroup = true; checkClusterName(resolver); checkStriped(); } @@ -617,9 +619,10 @@ struct CClusterInfo: implements IClusterInfo, public CInterface IClusterInfo *createClusterInfo(const char *name, IGroup *grp, const ClusterPartDiskMapSpec &mspec, - INamedGroupStore *resolver) + INamedGroupStore *resolver, + unsigned flags) { - return new CClusterInfo(name,grp,mspec,resolver); + return new CClusterInfo(name,grp,mspec,resolver,flags); } IClusterInfo *deserializeClusterInfo(MemoryBuffer &mb, INamedGroupStore *resolver) @@ -1281,33 +1284,39 @@ class CFileDescriptor: public CFileDescriptorBase, implements ISuperFileDescrip cluster->getReplicateDir(repDir, os); setReplicateFilename(fullpath,queryDrive(idx,copy),baseDir.str(),repDir.str()); - const char *planeName = cluster->queryGroupName(); - if (!isEmptyString(planeName)) + // The following code manipulates the directory for striping and aliasing if necessary. + // To do so, it needs the plane details. + // Normally, the plane name is obtained from IClusterInfo, however, if this file is foreign, + // then the IClusterInfo's will have no resolved names (aka groups) because the remote groups + // don't exist in the client environment. Instead, if the foreign file came from k8s, it will + // have remoteStoragePlane serialized/set. + Owned plane; + if (remoteStoragePlane) + plane.set(remoteStoragePlane); + else { -#ifdef _CONTAINERIZED - Owned plane = getDataStoragePlane(planeName, false); -#else - Owned plane = remoteStoragePlane.getLink(); -#endif - if (plane) + const char *planeName = cluster->queryGroupName(); + if (!isEmptyString(planeName)) + plane.setown(getDataStoragePlane(planeName, false)); + } + if (plane) + { + StringBuffer planePrefix(plane->queryPrefix()); + Owned alias = plane->getAliasMatch(accessMode); + if (alias) { - StringBuffer planePrefix(plane->queryPrefix()); - Owned alias = plane->getAliasMatch(accessMode); - if (alias) + StringBuffer tmp; + StringBuffer newPlanePrefix(alias->queryPrefix()); + if (setReplicateDir(fullpath, tmp, false, planePrefix, newPlanePrefix)) { - StringBuffer tmp; - StringBuffer newPlanePrefix(alias->queryPrefix()); - if (setReplicateDir(fullpath, tmp, false, planePrefix, newPlanePrefix)) - { - planePrefix.swapWith(newPlanePrefix); - fullpath.swapWith(tmp); - } + planePrefix.swapWith(newPlanePrefix); + fullpath.swapWith(tmp); } - StringBuffer stripeDir; - addStripeDirectory(stripeDir, fullpath, planePrefix, idx, lfnHash, cluster->queryPartDiskMapping().numStripedDevices); - if (!stripeDir.isEmpty()) - fullpath.swapWith(stripeDir); } + StringBuffer stripeDir; + addStripeDirectory(stripeDir, fullpath, planePrefix, idx, lfnHash, cluster->queryPartDiskMapping().numStripedDevices); + if (!stripeDir.isEmpty()) + fullpath.swapWith(stripeDir); } } @@ -1633,6 +1642,8 @@ class CFileDescriptor: public CFileDescriptorBase, implements ISuperFileDescrip attr.setown(createPTreeFromIPT(at)); else attr.setown(createPTree("Attr")); + if (flags & IFDSF_FOREIGN_GROUP) + setFlags(static_cast(fileFlags | FileDescriptorFlags::foreign)); if (attr->hasProp("@lfnHash")) // potentially missing for meta coming from a legacy Dali lfnHash = attr->getPropInt("@lfnHash"); else if (tracename.length()) diff --git a/dali/base/dafdesc.hpp b/dali/base/dafdesc.hpp index c88a6adb34e..535178828e2 100644 --- a/dali/base/dafdesc.hpp +++ b/dali/base/dafdesc.hpp @@ -190,7 +190,8 @@ typedef IIteratorOf IPartDescriptorIterator; enum class FileDescriptorFlags { none = 0x00, - dirperpart = 0x01 + dirperpart = 0x01, + foreign = 0x02 }; BITMASK_ENUM(FileDescriptorFlags); @@ -351,7 +352,8 @@ interface IStoragePlane: extends IInterface IClusterInfo *createClusterInfo(const char *grpname, // NULL if roxie label set IGroup *grp, const ClusterPartDiskMapSpec &mspec, - INamedGroupStore *resolver=NULL + INamedGroupStore *resolver=NULL, + unsigned flags=0 ); IClusterInfo *createRoxieClusterInfo(const char *label, const ClusterPartDiskMapSpec &mspec From 939af2176d0683845c27c2cc68b650f0626c7751 Mon Sep 17 00:00:00 2001 From: Jack Del Vecchio Date: Thu, 22 Aug 2024 13:18:25 +0000 Subject: [PATCH 18/25] HPCC-32491 HThor crashes reading local files with forceRemoteRead --- ecl/hthor/hthor.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ecl/hthor/hthor.cpp b/ecl/hthor/hthor.cpp index ee9dc1bc5ab..166cea2c0a7 100644 --- a/ecl/hthor/hthor.cpp +++ b/ecl/hthor/hthor.cpp @@ -8668,7 +8668,8 @@ bool CHThorDiskReadBaseActivity::openNext() StringBuffer tmp; remoteFileIO->addVirtualFieldMapping("logicalFilename", logicalFileName.str()); remoteFileIO->addVirtualFieldMapping("baseFpos", tmp.clear().append(offsetOfPart).str()); - remoteFileIO->addVirtualFieldMapping("partNum", tmp.clear().append(curPart->getPartIndex()).str()); + if (curPart) + remoteFileIO->addVirtualFieldMapping("partNum", tmp.clear().append(curPart->getPartIndex()).str()); try { From 5d8b5863988966133749dad3ca396797b8e932d8 Mon Sep 17 00:00:00 2001 From: Jake Smith Date: Thu, 22 Aug 2024 15:27:25 +0100 Subject: [PATCH 19/25] HPCC-32495 Rename splitterSpill option Also rename the force* (compression) options and compressSortOverflow compressAllOutputs Signed-off-by: Jake Smith --- thorlcr/thorutil/thormisc.hpp | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/thorlcr/thorutil/thormisc.hpp b/thorlcr/thorutil/thormisc.hpp index 4d6ffabe197..4e85c73d6f5 100644 --- a/thorlcr/thorutil/thormisc.hpp +++ b/thorlcr/thorutil/thormisc.hpp @@ -54,7 +54,7 @@ #define THOROPT_HDIST_TARGETWRITELIMIT "hdTargetLimit" // Limit # of writer threads working on a single target (default = unbound, but picks round-robin) #define THOROPT_HDIST_COMP "v9_4_hdCompressorType" // Distribute compressor to use (default = "LZ4") #define THOROPT_HDIST_COMPOPTIONS "v9_4_hdCompressorOptions" // Distribute compressor options, e.g. AES key (default = "") -#define THOROPT_SPLITTER_SPILL "splitterSpill" // Force splitters to spill or not, default is to adhere to helper setting (default = -1) +#define THOROPT_SPLITTER_SPILL "v9_4_splitterSpill" // Force splitters to spill or not, default is to adhere to helper setting (default = -1) #define THOROPT_LOOP_MAX_EMPTY "loopMaxEmpty" // Max # of iterations that LOOP can cycle through with 0 results before errors (default = 1000) #define THOROPT_SMALLSORT "smallSortThreshold" // Use minisort approach, if estimate size of data to sort is below this setting (default = 0) #define THOROPT_PARALLEL_FUNNEL "parallelFunnel" // Use parallel funnel impl. if !ordered (default = true) @@ -66,10 +66,10 @@ #define THOROPT_LKJOIN_LOCALFAILOVER "lkjoin_localfailover" // Force SMART to failover to distributed local lookup join (for testing only) (default = false) #define THOROPT_LKJOIN_HASHJOINFAILOVER "lkjoin_hashjoinfailover" // Force SMART to failover to hash join (for testing only) (default = false) #define THOROPT_MAX_KERNLOG "max_kern_level" // Max kernel logging level, to push to workunit, -1 to disable (default = 3) -#define THOROPT_COMP_FORCELZW "forceLZW" // Forces file compression to use LZW (default = false) -#define THOROPT_COMP_FORCEFLZ "forceFLZ" // Forces file compression to use FLZ (default = false) -#define THOROPT_COMP_FORCELZ4 "forceLZ4" // Forces file compression to use LZ4 (default = false) -#define THOROPT_COMP_FORCELZ4HC "forceLZ4HC" // Forces file compression to use LZ4HC (default = false) +#define THOROPT_COMP_FORCELZW "v9_4_forceLZW" // Forces file compression to use LZW (default = false) +#define THOROPT_COMP_FORCEFLZ "v9_4_forceFLZ" // Forces file compression to use FLZ (default = false) +#define THOROPT_COMP_FORCELZ4 "v9_4_forceLZ4" // Forces file compression to use LZ4 (default = false) +#define THOROPT_COMP_FORCELZ4HC "v9_4_forceLZ4HC" // Forces file compression to use LZ4HC (default = false) #define THOROPT_TRACE_ENABLED "traceEnabled" // Output from TRACE activity enabled (default = false) #define THOROPT_TRACE_LIMIT "traceLimit" // Number of rows from TRACE activity (default = 10) #define THOROPT_READ_CRC "crcReadEnabled" // Enabled CRC validation on disk reads if file CRC are available (default = true) @@ -105,7 +105,7 @@ #define THOROPT_VALIDATE_FILE_TYPE "validateFileType" // validate file type compatibility, e.g. if on fire error if XML reading CSV (default = true) #define THOROPT_MIN_REMOTE_CQ_INDEX_SIZE_MB "minRemoteCQIndexSizeMb" // minimum size of index file to enable server side handling (default = 0, meaning use heuristic to determin) #define THOROPT_KJ_ASSUME_PRIMARY "keyedJoinAssumePrimary" // assume primary part exists (don't check when mapping, which can be slow) -#define THOROPT_COMPRESS_SORTOVERFLOW "compressSortOverflow" // If global sort spills, compress the merged overflow file (default = true) +#define THOROPT_COMPRESS_SORTOVERFLOW "v9_4_compressSortOverflow" // If global sort spills, compress the merged overflow file (default = true) #define THOROPT_TIME_ACTIVITIES "timeActivities" // Time activities (default=true) #define THOROPT_MAX_ACTIVITY_CORES "maxActivityCores" // controls number of default threads to use for very parallel phases (like sort/parallel join helper). (default = # of h/w cores) #define THOROPT_THOR_ROWCRC "THOR_ROWCRC" // Use a CRC checking row allocator (default=false) @@ -114,7 +114,7 @@ #define THOROPT_FAIL_ON_LEAKS "failOnLeaks" // If any leaks are detected at the end of graph, fail the query (default=false) #define THOROPT_SOAP_TRACE_LEVEL "soapTraceLevel" // The trace SOAP level (default=1) #define THOROPT_SORT_ALGORITHM "sortAlgorithm" // The algorithm used to sort records (quicksort/mergesort) -#define THOROPT_COMPRESS_ALLFILES "compressAllOutputs" // Compress all output files (default: bare-metal=off, cloud=on) +#define THOROPT_COMPRESS_ALLFILES "v9_4_compressAllOutputs" // Compress all output files (default: bare-metal=off, cloud=on) #define THOROPT_AVOID_RENAME "avoidRename" // Avoid rename, write directly to target physical filenames (no temp file) From 9e7f3d7e87cf9603adf9dffaa7875559f7bbbef1 Mon Sep 17 00:00:00 2001 From: Jeremy Clements <79224539+jeclrsg@users.noreply.github.com> Date: Thu, 22 Aug 2024 10:54:59 -0400 Subject: [PATCH 20/25] HPCC-32474 ECL Watch v9 allow selection of grid row contents modifies the style of FluentUI grid rows to allow selection of text Signed-off-by: Jeremy Clements <79224539+jeclrsg@users.noreply.github.com> --- esp/src/src-react/components/controls/Grid.tsx | 1 + 1 file changed, 1 insertion(+) diff --git a/esp/src/src-react/components/controls/Grid.tsx b/esp/src/src-react/components/controls/Grid.tsx index 7a445acac93..c781322a0f2 100644 --- a/esp/src/src-react/components/controls/Grid.tsx +++ b/esp/src/src-react/components/controls/Grid.tsx @@ -122,6 +122,7 @@ const gridStyles = (height: string): Partial => { maxHeight: height, selectors: { ".ms-DetailsHeader-cellName": { fontSize: "13.5px" }, + ".ms-DetailsRow": { userSelect: "text" }, ".ms-DetailsRow-cell:has(.bgFilled)": { color: "white", boxShadow: "inset 1px 0 var(--colorNeutralBackground1), inset -1px 1px var(--colorNeutralBackground1)" }, ".ms-DetailsRow-cell:has(.bgGreen)": { background: "green" }, ".ms-DetailsRow-cell:has(.bgOrange)": { background: "orange" }, From 8be1a41cc21663d5c34cb80c377bc8d403dd0468 Mon Sep 17 00:00:00 2001 From: Jack Del Vecchio Date: Thu, 22 Aug 2024 16:07:42 +0000 Subject: [PATCH 21/25] HPCC-32507 Parquet Plugin not reading Arrow Partitioned files --- plugins/parquet/parquet.ecllib | 2 +- plugins/parquet/parquetembed.cpp | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/plugins/parquet/parquet.ecllib b/plugins/parquet/parquet.ecllib index 66529ac50dc..48f26678058 100644 --- a/plugins/parquet/parquet.ecllib +++ b/plugins/parquet/parquet.ecllib @@ -43,7 +43,7 @@ EXPORT ParquetIO := MODULE ENDMACRO; EXPORT Read(resultLayout, basePath, partitionFieldList) := FUNCTIONMACRO - LOCAL STREAMED DATASET(resultLayout) _DoParquetReadPartition() := EMBED(parquet: activity, option('readdirectorypartition'), location(basePath)), partitionFields(partitionFieldList) + LOCAL STREAMED DATASET(resultLayout) _DoParquetReadPartition() := EMBED(parquet: activity, option('readdirectorypartition'), location(basePath), partitionFields(partitionFieldList)) ENDEMBED; RETURN _DoParquetReadPartition(); ENDMACRO; diff --git a/plugins/parquet/parquetembed.cpp b/plugins/parquet/parquetembed.cpp index 473f56e145d..107ade09d7f 100644 --- a/plugins/parquet/parquetembed.cpp +++ b/plugins/parquet/parquetembed.cpp @@ -462,6 +462,8 @@ __int64 ParquetReader::next(TableColumns *&nextTable) if (endsWithIgnoreCase(partOption.c_str(), "partition")) { PARQUET_ASSIGN_OR_THROW(table, queryRows()); // Sets rowsProcessed to current row in table corresponding to startRow + rowsCount = table->num_rows(); + splitTable(table); } else { From 074095fc0c974ba393b211de8b71ead61c7f1e43 Mon Sep 17 00:00:00 2001 From: M Kelly Date: Thu, 22 Aug 2024 15:07:23 -0400 Subject: [PATCH 22/25] HPCC-32512 Fix LD_LIBRARY_PATH for internal ssh cmd from frunssh and rsync Signed-off-by: M Kelly --- common/remote/rmtssh.cpp | 3 ++- initfiles/bin/init_thorslave.in | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/common/remote/rmtssh.cpp b/common/remote/rmtssh.cpp index 86682580c7e..f156e2dbd8c 100644 --- a/common/remote/rmtssh.cpp +++ b/common/remote/rmtssh.cpp @@ -99,7 +99,8 @@ class CFRunSSH: public CInterface, implements IFRunSSH break; case 's': { // ssh params bool usepssh = !password.isEmpty(); - cmdbuf.appendf("%s -o LogLevel=QUIET -o StrictHostKeyChecking=%s -o BatchMode=yes ",usepssh?"pssh":"ssh",strict?"yes":"no"); + // reset LD_LIBRARY_PATH here so ssh cmd itself doesn't use HPCC libssl/crypto as they may be different + cmdbuf.appendf("%s -o LogLevel=QUIET -o StrictHostKeyChecking=%s -o BatchMode=yes ",usepssh?"pssh":"LD_LIBRARY_PATH=: ssh",strict?"yes":"no"); if (!identityfile.isEmpty()) cmdbuf.appendf("-i %s ",identityfile.get()); if (background) diff --git a/initfiles/bin/init_thorslave.in b/initfiles/bin/init_thorslave.in index 1048ce6f9ac..ad845f0a74f 100755 --- a/initfiles/bin/init_thorslave.in +++ b/initfiles/bin/init_thorslave.in @@ -102,7 +102,8 @@ start_slaves() rsync_att=3 rsync_stat=1 while [[ $rsync_stat -ne 0 && $rsync_att -gt 0 ]] ; do - rsync -e "ssh -o LogLevel=QUIET -o StrictHostKeyChecking=no" --timeout=60 $master:$instancedir/slaves $slavesfname + # reset LD_LIBRARY_PATH here so ssh cmd doesn't use HPCC libssl/crypto as they may be different + LD_LIBRARY_PATH=: rsync -e "ssh -o LogLevel=QUIET -o StrictHostKeyChecking=no" --timeout=60 $master:$instancedir/slaves $slavesfname rsync_stat=$? ((rsync_att--)) log "rsync returns ${rsync_stat}" From 91af2a3d34d23cf89f5e4f9b9abfc1397720b878 Mon Sep 17 00:00:00 2001 From: Jake Smith Date: Thu, 22 Aug 2024 15:14:52 +0100 Subject: [PATCH 23/25] HPCC-32493 Prevent thor k8s crash in disconnectLogMsgManagerFromDali Signed-off-by: Jake Smith --- dali/base/daclient.cpp | 2 ++ system/mp/mplog.cpp | 8 ++++++++ 2 files changed, 10 insertions(+) diff --git a/dali/base/daclient.cpp b/dali/base/daclient.cpp index f8adc17e90a..7a55e473dc1 100644 --- a/dali/base/daclient.cpp +++ b/dali/base/daclient.cpp @@ -270,6 +270,8 @@ void connectLogMsgManagerToDali() void disconnectLogMsgManagerFromDali() { + if (isContainerized()) + return; // we do not redirect logging between components in containerized environments (this is used for audit->dali in BM) disconnectLogMsgManagerFromParentOwn(daliClientLoggingParent); daliClientLoggingParent = 0; } diff --git a/system/mp/mplog.cpp b/system/mp/mplog.cpp index 86b338e85c1..2483ad015b1 100644 --- a/system/mp/mplog.cpp +++ b/system/mp/mplog.cpp @@ -532,6 +532,8 @@ aindex_t LogMsgParentReceiverThread::findParent(const INode * node) const bool connectLogMsgManagerToParent(INode * parentNode) { + if (isContainerized()) + return false; assertex(parentReceiver); MPLogId pid = parentReceiver->getNextId(); return parentReceiver->addParentToManager(0, pid, parentNode, false); @@ -539,6 +541,8 @@ bool connectLogMsgManagerToParent(INode * parentNode) bool connectLogMsgManagerToParentOwn(INode * parentNode) { + if (isContainerized()) + return false; bool ret = connectLogMsgManagerToParent(parentNode); parentNode->Release(); return ret; @@ -546,11 +550,15 @@ bool connectLogMsgManagerToParentOwn(INode * parentNode) bool disconnectLogMsgManagerFromParent(INode * parentNode) { + if (isContainerized()) + return false; return parentReceiver->removeParentFromManager(parentNode, false); } bool disconnectLogMsgManagerFromParentOwn(INode * parentNode) { + if (isContainerized()) + return false; bool ret = disconnectLogMsgManagerFromParent(parentNode); parentNode->Release(); return ret; From e75fc2be1614bda0714f7df24f4cd3596db894a6 Mon Sep 17 00:00:00 2001 From: M Kelly Date: Fri, 23 Aug 2024 08:17:57 -0400 Subject: [PATCH 24/25] HPCC-32512 Fix LD_LIBRARY_PATH for internal ssh cmd from frunssh and rsync 2 Signed-off-by: M Kelly --- common/remote/rmtssh.cpp | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/common/remote/rmtssh.cpp b/common/remote/rmtssh.cpp index f156e2dbd8c..fe08594b4ad 100644 --- a/common/remote/rmtssh.cpp +++ b/common/remote/rmtssh.cpp @@ -99,8 +99,7 @@ class CFRunSSH: public CInterface, implements IFRunSSH break; case 's': { // ssh params bool usepssh = !password.isEmpty(); - // reset LD_LIBRARY_PATH here so ssh cmd itself doesn't use HPCC libssl/crypto as they may be different - cmdbuf.appendf("%s -o LogLevel=QUIET -o StrictHostKeyChecking=%s -o BatchMode=yes ",usepssh?"pssh":"LD_LIBRARY_PATH=: ssh",strict?"yes":"no"); + cmdbuf.appendf("%s -o LogLevel=QUIET -o StrictHostKeyChecking=%s -o BatchMode=yes ",usepssh?"pssh":"ssh",strict?"yes":"no"); if (!identityfile.isEmpty()) cmdbuf.appendf("-i %s ",identityfile.get()); if (background) @@ -423,6 +422,8 @@ class CFRunSSH: public CInterface, implements IFRunSSH printf("%s\n",cmdline.str()); else { Owned pipe = createPipeProcess(); + // reset LD_LIBRARY_PATH here so ssh cmd itself doesn't use HPCC libssl/crypto as they may be different + pipe->setenv("LD_LIBRARY_PATH", ":"); if (pipe->run((verbose&&!usepssh)?"FRUNSSH":NULL,cmdline.str(),workdir, useplink, // for some reason plink needs input handle true,true)) { From f8ce1f0b7a737b8008524544b38afd6044bc4223 Mon Sep 17 00:00:00 2001 From: Jake Smith Date: Fri, 23 Aug 2024 10:05:59 +0100 Subject: [PATCH 25/25] HPCC-32492 Prevent failing unrelated thor instance marking jobs failed Also, track engine(thor) instance session in agent, so that if it disappears unexpectedly whilst running, it aborts the job and marks as session died unexpectedly. Signed-off-by: Jake Smith --- common/workunit/workunit.cpp | 55 ++++++++++++++++++++++++++----- common/workunit/workunit.hpp | 2 ++ common/workunit/workunit.ipp | 4 ++- ecl/agentexec/agentexec.cpp | 7 +--- plugins/cassandra/cassandrawu.cpp | 2 +- thorlcr/master/thgraphmanager.cpp | 3 ++ 6 files changed, 57 insertions(+), 16 deletions(-) diff --git a/common/workunit/workunit.cpp b/common/workunit/workunit.cpp index a5e97aff12b..7fa15bd12ed 100644 --- a/common/workunit/workunit.cpp +++ b/common/workunit/workunit.cpp @@ -4341,6 +4341,8 @@ class CLockedWorkUnit : implements ILocalWorkUnit, implements IExtendedWUInterfa { return c->getStateEx(str); } virtual __int64 getAgentSession() const { return c->getAgentSession(); } + virtual __int64 getEngineSession() const + { return c->getEngineSession(); } virtual unsigned getAgentPID() const { return c->getAgentPID(); } virtual const char *queryStateDesc() const @@ -4497,6 +4499,8 @@ class CLockedWorkUnit : implements ILocalWorkUnit, implements IExtendedWUInterfa { c->setStateEx(text); } virtual void setAgentSession(__int64 sessionId) { c->setAgentSession(sessionId); } + virtual void setEngineSession(__int64 sessionId) + { c->setEngineSession(sessionId); } virtual void setStatistic(StatisticCreatorType creatorType, const char * creator, StatisticScopeType scopeType, const char * scope, StatisticKind kind, const char * optDescription, unsigned __int64 value, unsigned __int64 count, unsigned __int64 maxValue, StatsMergeAction mergeAction) { c->setStatistic(creatorType, creator, scopeType, scope, kind, optDescription, value, count, maxValue, mergeAction); } virtual void setTracingValue(const char * propname, const char * value) @@ -5918,9 +5922,11 @@ void CWorkUnitFactory::clearAborting(const char *wuid) } } -void CWorkUnitFactory::reportAbnormalTermination(const char *wuid, WUState &state, SessionId agent) +void CWorkUnitFactory::reportAbnormalTermination(const char *wuid, WUState &state, SessionId sessionId, const char *sessionText) { - WARNLOG("reportAbnormalTermination: session stopped unexpectedly: %" I64F "d state: %d", (__int64) agent, (int) state); + StringBuffer sessionMessage(sessionText); + sessionMessage.appendf(" session stopped unexpectedly [sessionId=%" I64F "d]", (__int64) sessionId); + WARNLOG("reportAbnormalTermination: %s - state: %d", sessionText, (int) state); bool isEcl = false; switch (state) { @@ -5937,7 +5943,10 @@ void CWorkUnitFactory::reportAbnormalTermination(const char *wuid, WUState &stat wu->setState(state); Owned e = wu->createException(); e->setExceptionCode(isEcl ? 1001 : 1000); - e->setExceptionMessage(isEcl ? "EclCC terminated unexpectedly" : "Workunit terminated unexpectedly"); + StringBuffer exceptionText; + exceptionText.append(isEcl ? "EclCC terminated unexpectedly" : "Workunit terminated unexpectedly"); + exceptionText.append(" (").append(sessionMessage).append(")"); + e->setExceptionMessage(exceptionText); } static CriticalSection deleteDllLock; @@ -6422,8 +6431,11 @@ class CDaliWorkUnitFactory : public CWorkUnitFactory, implements IDaliClientShut LocalIAbortHandler abortHandler(*waiter); if (conn) { - SessionId agent = -1; + SessionId agentSessionID = -1; + SessionId engineSessionID = -1; bool agentSessionStopped = false; + bool engineSessionStopped = false; + bool queryRuntimeSessionStopped = false; unsigned start = msTick(); for (;;) { @@ -6452,22 +6464,37 @@ class CDaliWorkUnitFactory : public CWorkUnitFactory, implements IDaliClientShut case WUStateAborting: if (agentSessionStopped) { - reportAbnormalTermination(wuid, ret, agent); + reportAbnormalTermination(wuid, ret, agentSessionID, "Agent"); + return ret; + } + if (engineSessionStopped) + { + reportAbnormalTermination(wuid, ret, engineSessionID, "Engine"); return ret; } if (queryDaliServerVersion().compare("2.1")>=0) { - agent = conn->queryRoot()->getPropInt64("@agentSession", -1); - if((agent>0) && querySessionManager().sessionStopped(agent, 0)) + agentSessionID = conn->queryRoot()->getPropInt64("@agentSession", -1); + if((agentSessionID>0) && querySessionManager().sessionStopped(agentSessionID, 0)) { agentSessionStopped = true; conn->reload(); continue; } + engineSessionID = conn->queryRoot()->getPropInt64("@engineSession", -1); + if((engineSessionID>0) && querySessionManager().sessionStopped(engineSessionID, 0)) + { + engineSessionStopped = true; + conn->reload(); + continue; + } } break; } - agentSessionStopped = false; // reset for state changes such as WUStateWait then WUStateRunning again + // reset for state changes such as WUStateWait then WUStateRunning again + agentSessionStopped = false; + engineSessionStopped = false; + unsigned waited = msTick() - start; if (timeout==-1 || waited + 20000 < timeout) { @@ -7694,6 +7721,12 @@ void CLocalWorkUnit::setAgentSession(__int64 sessionId) p->setPropInt64("@agentSession", sessionId); } +void CLocalWorkUnit::setEngineSession(__int64 sessionId) +{ + CriticalBlock block(crit); + p->setPropInt64("@engineSession", sessionId); +} + bool CLocalWorkUnit::getIsQueryService() const { CriticalBlock block(crit); @@ -7795,6 +7828,12 @@ __int64 CLocalWorkUnit::getAgentSession() const return p->getPropInt64("@agentSession", -1); } +__int64 CLocalWorkUnit::getEngineSession() const +{ + CriticalBlock block(crit); + return p->getPropInt64("@engineSession", -1); +} + unsigned CLocalWorkUnit::getAgentPID() const { CriticalBlock block(crit); diff --git a/common/workunit/workunit.hpp b/common/workunit/workunit.hpp index d96471b5077..b8da9047823 100644 --- a/common/workunit/workunit.hpp +++ b/common/workunit/workunit.hpp @@ -1251,6 +1251,7 @@ interface IConstWorkUnit : extends IConstWorkUnitInfo virtual IStringVal & getWorkunitDistributedAccessToken(IStringVal & datoken) const = 0; virtual IStringVal & getStateEx(IStringVal & str) const = 0; virtual __int64 getAgentSession() const = 0; + virtual __int64 getEngineSession() const = 0; virtual unsigned getAgentPID() const = 0; virtual IConstWUResult * getTemporaryByName(const char * name) const = 0; virtual IConstWUResultIterator & getTemporaries() const = 0; @@ -1340,6 +1341,7 @@ interface IWorkUnit : extends IConstWorkUnit virtual void setState(WUState state) = 0; virtual void setStateEx(const char * text) = 0; // Indicates why blocked virtual void setAgentSession(__int64 sessionId) = 0; + virtual void setEngineSession(__int64 sessionId) = 0; virtual void setStatistic(StatisticCreatorType creatorType, const char * creator, StatisticScopeType scopeType, const char * scope, StatisticKind kind, const char * optDescription, unsigned __int64 value, unsigned __int64 count, unsigned __int64 maxValue, StatsMergeAction mergeAction) = 0; virtual void setTracingValue(const char * propname, const char * value) = 0; virtual void setTracingValueInt(const char * propname, int value) = 0; diff --git a/common/workunit/workunit.ipp b/common/workunit/workunit.ipp index f83956af090..5213b437896 100644 --- a/common/workunit/workunit.ipp +++ b/common/workunit/workunit.ipp @@ -257,6 +257,7 @@ public: virtual WUState getState() const; virtual IStringVal & getStateEx(IStringVal & str) const; virtual __int64 getAgentSession() const; + virtual __int64 getEngineSession() const; virtual unsigned getAgentPID() const; virtual const char *queryStateDesc() const; virtual IConstWUResult * getTemporaryByName(const char * name) const; @@ -336,6 +337,7 @@ public: void setState(WUState state); void setStateEx(const char * text); void setAgentSession(__int64 sessionId); + void setEngineSession(__int64 sessionId); bool setDistributedAccessToken(const char * user); void setStatistic(StatisticCreatorType creatorType, const char * creator, StatisticScopeType scopeType, const char * scope, StatisticKind kind, const char * optDescription, unsigned __int64 value, unsigned __int64 count, unsigned __int64 maxValue, StatsMergeAction mergeAction); void setTracingValue(const char * propname, const char * value); @@ -621,7 +623,7 @@ public: } protected: - void reportAbnormalTermination(const char *wuid, WUState &state, SessionId agent); + void reportAbnormalTermination(const char *wuid, WUState &state, SessionId agent, const char *sessionText); // These need to be implemented by the derived classes virtual CLocalWorkUnit* _createWorkUnit(const char *wuid, ISecManager *secmgr, ISecUser *secuser) = 0; diff --git a/ecl/agentexec/agentexec.cpp b/ecl/agentexec/agentexec.cpp index 5582af12177..e0935dba13b 100644 --- a/ecl/agentexec/agentexec.cpp +++ b/ecl/agentexec/agentexec.cpp @@ -342,13 +342,8 @@ class WaitThread : public CInterfaceOf Owned cw = factory->openWorkUnit(wuid); if (cw) { - // if either a) NOT a thoragent with useChildProcesses=false (default in k8s config) or b) is still in an executing state - if (!sharedK8sJob || (cw->getState() == WUStateRunning) || (cw->getState() == WUStateBlocked) || (cw->getState() == WUStateWait)) + if (!sharedK8sJob && ((cw->getState() == WUStateRunning) || (cw->getState() == WUStateBlocked) || (cw->getState() == WUStateWait))) { - // For a shared k8s job, i.e. where this agent is thoragent launching shared (multiJobLinger) k8s jobs - // the job agent should handle the job state. - // In that scenario, this is a fallback that should only come into effect if the job workflow instance has failed to handle the exception - // e.g. because it abruptly disappeared. Owned workunit = &cw->lock(); // recheck now locked if ((workunit->getState() == WUStateRunning) || (workunit->getState() == WUStateBlocked) || (workunit->getState() == WUStateWait)) diff --git a/plugins/cassandra/cassandrawu.cpp b/plugins/cassandra/cassandrawu.cpp index 293b0c21ea1..10cb6239430 100644 --- a/plugins/cassandra/cassandrawu.cpp +++ b/plugins/cassandra/cassandrawu.cpp @@ -3873,7 +3873,7 @@ class CCasssandraWorkUnitFactory : public CWorkUnitFactory, implements ICassandr case WUStateAborting: if (agentSessionStopped) { - reportAbnormalTermination(wuid, state, agent); + reportAbnormalTermination(wuid, state, agent, "Agent"); return state; } if (queryDaliServerVersion().compare("2.1")>=0) diff --git a/thorlcr/master/thgraphmanager.cpp b/thorlcr/master/thgraphmanager.cpp index 142724fd868..33c580dfd16 100644 --- a/thorlcr/master/thgraphmanager.cpp +++ b/thorlcr/master/thgraphmanager.cpp @@ -1114,6 +1114,7 @@ bool CJobManager::executeGraph(IConstWorkUnit &workunit, const char *graphName, wu->setStatistic(queryStatisticsComponentType(), queryStatisticsComponentName(), SSTgraph, graphScope, StWhenStarted, NULL, startTs, 1, 0, StatsMergeAppend); //Could use addTimeStamp(wu, SSTgraph, graphName, StWhenStarted, wfid) if start time could be this point wu->setState(WUStateRunning); + wu->setEngineSession(myProcessSession()); VStringBuffer version("%d.%d", THOR_VERSION_MAJOR, THOR_VERSION_MINOR); wu->setDebugValue("ThorVersion", version.str(), true); @@ -1140,6 +1141,8 @@ bool CJobManager::executeGraph(IConstWorkUnit &workunit, const char *graphName, wu->setStatistic(queryStatisticsComponentType(), queryStatisticsComponentName(), SSTgraph, graphScope, StCostExecute, NULL, cost, 1, 0, StatsMergeReplace); if (globals->getPropBool("@watchdogProgressEnabled")) queryDeMonServer()->updateAggregates(wu); + // clear engine session, otherwise agent may consider a failure beyond this point for an unrelated job caused by this instance + wu->setEngineSession(-1); removeJob(*job); }