From 29b70fca2f87d10a879676f653d8ea49844f543d Mon Sep 17 00:00:00 2001 From: Gordon Smith Date: Wed, 4 Dec 2024 12:07:55 +0000 Subject: [PATCH 1/7] HPCC-33066: Add basic ECL Watch UI tests Signed-off-by: Gordon Smith --- .github/workflows/build-test-eclwatch.yml | 8 +- esp/src/.gitignore | 4 + esp/src/lws.config.js | 4 +- esp/src/package-lock.json | 116 ++++++++++++++++++++-- esp/src/package.json | 10 +- esp/src/playwright.config.ts | 46 +++++++++ esp/src/tests/eclwatch-v5.spec.ts | 31 ++++++ esp/src/tests/eclwatch-v9.spec.ts | 42 ++++++++ 8 files changed, 249 insertions(+), 12 deletions(-) create mode 100644 esp/src/playwright.config.ts create mode 100644 esp/src/tests/eclwatch-v5.spec.ts create mode 100644 esp/src/tests/eclwatch-v9.spec.ts diff --git a/.github/workflows/build-test-eclwatch.yml b/.github/workflows/build-test-eclwatch.yml index 7a9c561af5f..ff4d53c36c1 100644 --- a/.github/workflows/build-test-eclwatch.yml +++ b/.github/workflows/build-test-eclwatch.yml @@ -25,7 +25,7 @@ jobs: build: strategy: matrix: - node: ["20", "18", "16"] + node: ["22", "20", "18"] fail-fast: false name: "Check eclwatch and npm" needs: pre_job @@ -48,6 +48,12 @@ jobs: - name: Install Dependencies working-directory: ./esp/src run: npm ci + - name: Lint + working-directory: ./esp/src + run: npm run lint + - name: Install Playwright browsers + working-directory: ./esp/src + run: npx playwright install --with-deps - name: Build working-directory: ./esp/src run: npm run build diff --git a/esp/src/.gitignore b/esp/src/.gitignore index a2270e87528..09c76804ba1 100644 --- a/esp/src/.gitignore +++ b/esp/src/.gitignore @@ -1,7 +1,11 @@ +blob-report/ build/ hpcc-js/ lib/ node_modules/ +playwright/.cache/ +playwright-report/ +test-results/ types/ .vscode/* !.vscode/tasks.json diff --git a/esp/src/lws.config.js b/esp/src/lws.config.js index 39dfa94dcb8..00826fdf310 100644 --- a/esp/src/lws.config.js +++ b/esp/src/lws.config.js @@ -1,6 +1,6 @@ const fs = require("fs"); -let ip = "192.168.99.103"; +let ip = "https://play.hpccsystems.com:18010"; if (fs.existsSync("./lws.target.txt")) { ip = fs.readFileSync("./lws.target.txt").toString().replace("\r\n", "\n").split("\n")[0]; } @@ -68,5 +68,5 @@ let rewrite = [ module.exports = { port: 8080, rewrite: rewrite, - stack: ['lws-basic-auth', 'lws-request-monitor', 'lws-log', 'lws-cors', 'lws-json', 'lws-compress', 'lws-rewrite', 'lws-blacklist', 'lws-conditional-get', 'lws-mime', 'lws-range', 'lws-spa', 'lws-static', 'lws-index'] + stack: ["lws-basic-auth", "lws-request-monitor", "lws-log", "lws-cors", "lws-json", "lws-compress", "lws-rewrite", "lws-blacklist", "lws-conditional-get", "lws-mime", "lws-range", "lws-spa", "lws-static", "lws-index"] }; \ No newline at end of file diff --git a/esp/src/package-lock.json b/esp/src/package-lock.json index 4fcce04e973..7571e9c3011 100644 --- a/esp/src/package-lock.json +++ b/esp/src/package-lock.json @@ -57,13 +57,16 @@ "xstyle": "0.3.3" }, "devDependencies": { + "@playwright/test": "^1.49.0", "@simbathesailor/use-what-changed": "^2.0.0", "@types/dojo": "1.9.48", + "@types/node": "^22.10.1", "@types/react": "17.0.80", "@types/react-dom": "17.0.25", "@typescript-eslint/eslint-plugin": "6.21.0", "@typescript-eslint/parser": "6.21.0", "copyfiles": "2.4.1", + "cross-env": "^7.0.3", "css-loader": "6.10.0", "dojo-webpack-plugin": "3.0.6", "eslint": "8.57.0", @@ -2626,6 +2629,21 @@ "openid-client": "^5.3.0" } }, + "node_modules/@kubernetes/client-node/node_modules/@types/node": { + "version": "20.17.9", + "resolved": "https://registry.npmjs.org/@types/node/-/node-20.17.9.tgz", + "integrity": "sha512-0JOXkRyLanfGPE2QRCwgxhzlBAvaRdCNMcvbd7jFfpmD4eEXll7LRwy5ymJmyeZqk7Nh7eD2LeUyQ68BbndmXw==", + "license": "MIT", + "dependencies": { + "undici-types": "~6.19.2" + } + }, + "node_modules/@kubernetes/client-node/node_modules/undici-types": { + "version": "6.19.8", + "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-6.19.8.tgz", + "integrity": "sha512-ve2KP6f/JnbPBFyobGHuerC9g1FYGn/F8n1LWTwNxCEzd6IfqTwUQcNXgEtmmQ6DlRrC1hrSrBnCZPokRrDHjw==", + "license": "MIT" + }, "node_modules/@leichtgewicht/ip-codec": { "version": "2.0.5", "resolved": "https://registry.npmjs.org/@leichtgewicht/ip-codec/-/ip-codec-2.0.5.tgz", @@ -3143,6 +3161,22 @@ "node": ">=14" } }, + "node_modules/@playwright/test": { + "version": "1.49.0", + "resolved": "https://registry.npmjs.org/@playwright/test/-/test-1.49.0.tgz", + "integrity": "sha512-DMulbwQURa8rNIQrf94+jPJQ4FmOVdpE5ZppRNvWVjvhC+6sOeo28r8MgIpQRYouXRtt/FCCXU7zn20jnHR4Qw==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "playwright": "1.49.0" + }, + "bin": { + "playwright": "cli.js" + }, + "engines": { + "node": ">=18" + } + }, "node_modules/@simbathesailor/use-what-changed": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/@simbathesailor/use-what-changed/-/use-what-changed-2.0.0.tgz", @@ -3442,11 +3476,12 @@ "dev": true }, "node_modules/@types/node": { - "version": "20.10.5", - "resolved": "https://registry.npmjs.org/@types/node/-/node-20.10.5.tgz", - "integrity": "sha512-nNPsNE65wjMxEKI93yOP+NPGGBJz/PoN3kZsVLee0XMiJolxSekEVD8wRwBUBqkwc7UWop0edW50yrCQW4CyRw==", + "version": "22.10.1", + "resolved": "https://registry.npmjs.org/@types/node/-/node-22.10.1.tgz", + "integrity": "sha512-qKgsUwfHZV2WCWLAnVP1JqnpE6Im6h3Y0+fYgMTasNQ7V++CBX5OT1as0g0f+OyubbFqhf6XVNIsmN4IIhEgGQ==", + "license": "MIT", "dependencies": { - "undici-types": "~5.26.4" + "undici-types": "~6.20.0" } }, "node_modules/@types/node-forge": { @@ -5078,6 +5113,25 @@ "node": ">=10" } }, + "node_modules/cross-env": { + "version": "7.0.3", + "resolved": "https://registry.npmjs.org/cross-env/-/cross-env-7.0.3.tgz", + "integrity": "sha512-+/HKd6EgcQCJGh2PSjZuUitQBQynKor4wrFbRg4DtAgS1aWO+gU52xpH7M9ScGgXSYmAVS9bIJ8EzuaGw0oNAw==", + "dev": true, + "license": "MIT", + "dependencies": { + "cross-spawn": "^7.0.1" + }, + "bin": { + "cross-env": "src/bin/cross-env.js", + "cross-env-shell": "src/bin/cross-env-shell.js" + }, + "engines": { + "node": ">=10.14", + "npm": ">=6", + "yarn": ">=1" + } + }, "node_modules/cross-spawn": { "version": "7.0.3", "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.3.tgz", @@ -9371,6 +9425,53 @@ "node": ">=8" } }, + "node_modules/playwright": { + "version": "1.49.0", + "resolved": "https://registry.npmjs.org/playwright/-/playwright-1.49.0.tgz", + "integrity": "sha512-eKpmys0UFDnfNb3vfsf8Vx2LEOtflgRebl0Im2eQQnYMA4Aqd+Zw8bEOB+7ZKvN76901mRnqdsiOGKxzVTbi7A==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "playwright-core": "1.49.0" + }, + "bin": { + "playwright": "cli.js" + }, + "engines": { + "node": ">=18" + }, + "optionalDependencies": { + "fsevents": "2.3.2" + } + }, + "node_modules/playwright-core": { + "version": "1.49.0", + "resolved": "https://registry.npmjs.org/playwright-core/-/playwright-core-1.49.0.tgz", + "integrity": "sha512-R+3KKTQF3npy5GTiKH/T+kdhoJfJojjHESR1YEWhYuEKRVfVaxH3+4+GvXE5xyCngCxhxnykk0Vlah9v8fs3jA==", + "dev": true, + "license": "Apache-2.0", + "bin": { + "playwright-core": "cli.js" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/playwright/node_modules/fsevents": { + "version": "2.3.2", + "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.2.tgz", + "integrity": "sha512-xiqMQR4xAeHTuB9uWm+fFRcIOgKBMiOBP+eXiyT7jsgVCq1bkVygt00oASowB7EdtpOHaaPgKt812P9ab+DDKA==", + "dev": true, + "hasInstallScript": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": "^8.16.0 || ^10.6.0 || >=11.0.0" + } + }, "node_modules/postcss": { "version": "8.4.33", "resolved": "https://registry.npmjs.org/postcss/-/postcss-8.4.33.tgz", @@ -11539,9 +11640,10 @@ } }, "node_modules/undici-types": { - "version": "5.26.5", - "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-5.26.5.tgz", - "integrity": "sha512-JlCMO+ehdEIKqlFxk6IfVoAUVmgz7cU7zD/h9XZ0qzeosSHmUJVOzSQvvYSYWXkFXC+IfLKSIffhv0sVZup6pA==" + "version": "6.20.0", + "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-6.20.0.tgz", + "integrity": "sha512-Ny6QZ2Nju20vw1SRHe3d9jVu6gJ+4e3+MMpqu7pqE5HT6WsTSlce++GQmK5UXS8mzV8DSYHrQH+Xrf2jVcuKNg==", + "license": "MIT" }, "node_modules/universal-github-app-jwt": { "version": "1.1.1", diff --git a/esp/src/package.json b/esp/src/package.json index db7708ef45d..43d56c2c664 100644 --- a/esp/src/package.json +++ b/esp/src/package.json @@ -28,8 +28,11 @@ "dev-start": "run-p bundle-watch dev-start-ws", "dev-start-verbose": "ws --verbose.include request response", "rm-hpcc": "rimraf ./node_modules/@hpcc-js", - "start": "webpack serve --env development --config webpack.config.js", - "test": "run-s lint", + "start": "ws", + "test": "npx playwright test", + "test-ci": "cross-env CI=1 npx playwright test", + "test-codegen": "npx playwright codegen", + "test-interactive": "npx playwright test --ui", "update": "npx npm-check-updates -u -t minor", "update-major": "npx npm-check-updates -u" }, @@ -83,13 +86,16 @@ "xstyle": "0.3.3" }, "devDependencies": { + "@playwright/test": "^1.49.0", "@simbathesailor/use-what-changed": "^2.0.0", "@types/dojo": "1.9.48", + "@types/node": "^22.10.1", "@types/react": "17.0.80", "@types/react-dom": "17.0.25", "@typescript-eslint/eslint-plugin": "6.21.0", "@typescript-eslint/parser": "6.21.0", "copyfiles": "2.4.1", + "cross-env": "^7.0.3", "css-loader": "6.10.0", "dojo-webpack-plugin": "3.0.6", "eslint": "8.57.0", diff --git a/esp/src/playwright.config.ts b/esp/src/playwright.config.ts new file mode 100644 index 00000000000..00031962989 --- /dev/null +++ b/esp/src/playwright.config.ts @@ -0,0 +1,46 @@ +import { defineConfig, devices } from "@playwright/test"; + +const baseURL = process.env.CI ? "https://play.hpccsystems.com:18010" : "http://127.0.0.1:8080"; + +/** + * See https://playwright.dev/docs/test-configuration. + */ +export default defineConfig({ + testDir: "./tests", + fullyParallel: true, + forbidOnly: !!process.env.CI, + retries: process.env.CI ? 2 : 0, + workers: process.env.CI ? 4 : undefined, + reporter: "html", + use: { + baseURL, + trace: "on-first-retry", + ignoreHTTPSErrors: true + }, + + projects: [ + { + name: "chromium", + use: { ...devices["Desktop Chrome"] }, + }, + + { + name: "firefox", + use: { ...devices["Desktop Firefox"] }, + }, + + { + name: "webkit", + use: { ...devices["Desktop Safari"] }, + }, + + ], + + /* Run your local dev server before starting the tests */ + webServer: { + command: "npm run start", + url: baseURL, + reuseExistingServer: true, + ignoreHTTPSErrors: true, + }, +}); diff --git a/esp/src/tests/eclwatch-v5.spec.ts b/esp/src/tests/eclwatch-v5.spec.ts new file mode 100644 index 00000000000..01f1eccc486 --- /dev/null +++ b/esp/src/tests/eclwatch-v5.spec.ts @@ -0,0 +1,31 @@ +import { test, expect } from "@playwright/test"; + +test.describe("ECLWatch V5", () => { + test.beforeEach(async ({ page }) => { + await page.goto("/esp/files/index.html"); + await page.evaluate(() => { + sessionStorage.setItem("ECLWatch:ModernMode-9.0", "false"); + }); + }); + + test("Basic Frame", async ({ page }) => { + await page.goto("/esp/files/stub.htm"); + await expect(page.locator("#stubStackController_stub_Main span").first()).toBeVisible(); + await expect(page.getByLabel("Advanced")).toBeVisible(); + }); + + test("Activities", async ({ page }) => { + await page.goto("/esp/files/stub.htm"); + await expect(page.locator("#stub_Main-DLStackController_stub_Main-DL_Activity_label")).toBeVisible(); + await expect(page.getByLabel("Auto Refresh")).toBeVisible(); + await expect(page.getByLabel("Maximize/Restore")).toBeVisible(); + await expect(page.locator("i")).toBeVisible(); + await expect(page.locator("svg").filter({ hasText: "%hthor" })).toBeVisible(); + await expect(page.getByRole("img", { name: "Priority" })).toBeVisible(); + await expect(page.getByText("Target/Wuid")).toBeVisible(); + await expect(page.getByText("Graph")).toBeVisible(); + await expect(page.getByText("State")).toBeVisible(); + await expect(page.getByText("Owner")).toBeVisible(); + await expect(page.getByText("Job Name")).toBeVisible(); + }); +}); diff --git a/esp/src/tests/eclwatch-v9.spec.ts b/esp/src/tests/eclwatch-v9.spec.ts new file mode 100644 index 00000000000..745a274254d --- /dev/null +++ b/esp/src/tests/eclwatch-v9.spec.ts @@ -0,0 +1,42 @@ +import { test, expect } from "@playwright/test"; + +test.describe("ECLWatch V9", () => { + + test("Basic Frame", async ({ page }) => { + await page.goto("/esp/files/index.html#/activities"); + await expect(page.getByRole("link", { name: "ECL Watch" })).toBeVisible(); + await expect(page.locator("button").filter({ hasText: "" })).toBeVisible(); + await expect(page.getByRole("button", { name: "Advanced" })).toBeVisible(); + await expect(page.getByTitle("Activities")).toBeVisible(); + await expect(page.getByRole("link", { name: "ECL", exact: true })).toBeVisible(); + await expect(page.getByRole("link", { name: "Files" })).toBeVisible(); + await expect(page.getByRole("link", { name: "Published Queries" })).toBeVisible(); + await expect(page.getByRole("button", { name: "History" })).toBeVisible(); + await expect(page.getByRole("button", { name: "Add to favorites" })).toBeVisible(); + await expect(page.locator("a").filter({ hasText: /^Activities$/ })).toBeVisible(); + await expect(page.getByRole("link", { name: "Event Scheduler" })).toBeVisible(); + }); + + test("Activities", async ({ page }) => { + await page.goto("/esp/files/index.html#/activities"); + await page.getByTitle("Disk Usage").locator("i").click(); + await expect(page.locator("svg").filter({ hasText: "%hthor" })).toBeVisible(); + await expect(page.locator(".reflex-splitter")).toBeVisible(); + await expect(page.getByRole("menubar")).toBeVisible(); + await expect(page.getByRole("menuitem", { name: "Refresh" })).toBeVisible(); + await expect(page.locator("button").filter({ hasText: "" })).toBeVisible(); + await expect(page.locator("button").filter({ hasText: "" })).toBeVisible(); + await expect(page.getByRole("columnheader", { name: "Priority" }).locator("div").first()).toBeVisible(); + await expect(page.getByText("Target/Wuid")).toBeVisible(); + await expect(page.getByText("Graph")).toBeVisible(); + await expect(page.getByText("State")).toBeVisible(); + await expect(page.getByText("Owner")).toBeVisible(); + await expect(page.getByText("Job Name")).toBeVisible(); + await expect(page.getByRole("gridcell", { name: "HThorServer - hthor" })).toBeVisible(); + await expect(page.getByRole("gridcell", { name: "ThorMaster - thor", exact: true })).toBeVisible(); + await expect(page.getByRole("gridcell", { name: "ThorMaster - thor_roxie" })).toBeVisible(); + await expect(page.getByRole("gridcell", { name: "RoxieServer - roxie" })).toBeVisible(); + await expect(page.getByRole("gridcell", { name: "myeclccserver - hthor." })).toBeVisible(); + await expect(page.getByRole("gridcell", { name: "mydfuserver - dfuserver_queue" })).toBeVisible(); + }); +}); From ee42588ebefe17996723e185ad55475a86986c5e Mon Sep 17 00:00:00 2001 From: Jake Smith Date: Tue, 10 Dec 2024 14:40:34 +0000 Subject: [PATCH 2/7] HPCC-33100 Fix data loss, streaming from dafilesrv and network failure. If there is a broken socket whilst streaming from dafilesrv, the underlying mechanism will dispose of the socket and reconnect and retry the command. If the server side remained running and did not receive notification of the closed socket, the client can reconnect to the same server and the retry will cause the next chunk of data in the stream to be retrieved, thereby losing the earlier one that hit the socket issues. Fix by avoiding the reconnect/retry mechanism at this level, instead, on failure during streaming, establish a new stream at the valid position using the serialized cursor. Signed-off-by: Jake Smith --- fs/dafsclient/rmtfile.cpp | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/fs/dafsclient/rmtfile.cpp b/fs/dafsclient/rmtfile.cpp index 7b876e63883..fa8b494e6f7 100644 --- a/fs/dafsclient/rmtfile.cpp +++ b/fs/dafsclient/rmtfile.cpp @@ -2316,9 +2316,19 @@ class CRemoteFilteredFileIOBase : public CRemoteBase, implements IRemoteFileIO mrequest.append((RemoteFileCommandType)RFCStreamRead); VStringBuffer json("{ \"handle\" : %u, \"format\" : \"binary\" }", handle); mrequest.append(json.length(), json.str()); - sendRemoteCommand(mrequest, newReply); - unsigned newHandle; - newReply.read(newHandle); + unsigned newHandle = 0; + try + { + sendRemoteCommand(mrequest, newReply, false); + newReply.read(newHandle); + } + catch (IJSOCK_Exception *e) + { + // will trigger new request with cursor + EXCLOG(e, "CRemoteFilteredFileIOBase:: socket failure whilst streaming, will attempt to reconnect with cursor"); + newHandle = 0; + e->Release(); + } if (newHandle == handle) { reply.swapWith(newReply); From 3a59a0781fbc73b440bf62ac3e7a177847a499e5 Mon Sep 17 00:00:00 2001 From: Jeremy Clements <79224539+jeclrsg@users.noreply.github.com> Date: Tue, 10 Dec 2024 16:20:17 -0500 Subject: [PATCH 3/7] HPCC-33083 ECL Watch v9 playground submit feedback Fixes an issue in the ECL Watch v9 UI where submitting a WU in the playground was inconsistent to the v5 UI. Now both the graph & results components will update to indicate work is being done. Signed-off-by: Jeremy Clements <79224539+jeclrsg@users.noreply.github.com> --- .../src-react/components/ECLPlayground.tsx | 54 ++++++++++++------- 1 file changed, 35 insertions(+), 19 deletions(-) diff --git a/esp/src/src-react/components/ECLPlayground.tsx b/esp/src/src-react/components/ECLPlayground.tsx index 9fe22b20ec7..ceca0b9eed9 100644 --- a/esp/src/src-react/components/ECLPlayground.tsx +++ b/esp/src/src-react/components/ECLPlayground.tsx @@ -1,13 +1,13 @@ import * as React from "react"; import { ReflexContainer, ReflexElement, ReflexSplitter } from "../layouts/react-reflex"; -import { IconButton, IIconProps, Link, Dropdown, IDropdownOption, TextField, useTheme } from "@fluentui/react"; +import { IconButton, IIconProps, Link, Dropdown, IDropdownOption, Spinner, SpinnerSize, TextField, useTheme } from "@fluentui/react"; import { Button } from "@fluentui/react-components"; import { CheckmarkCircleRegular, DismissCircleRegular, QuestionCircleRegular } from "@fluentui/react-icons"; import { scopedLogger } from "@hpcc-js/util"; import { useOnEvent } from "@fluentui/react-hooks"; import { mergeStyleSets } from "@fluentui/style-utilities"; import { ECLEditor, IPosition } from "@hpcc-js/codemirror"; -import { Workunit, WUUpdate, WorkunitsService } from "@hpcc-js/comms"; +import { Workunit, WUUpdate, WorkunitsService, WUStateID } from "@hpcc-js/comms"; import { HolyGrail } from "../layouts/HolyGrail"; import { DojoAdapter } from "../layouts/DojoAdapter"; import { pushUrl } from "../util/history"; @@ -238,7 +238,6 @@ const ECLEditorToolbar: React.FunctionComponent = ({ if (document.location.hash.includes("play")) { if (wu.isFailed()) { pushUrl(`/play/${wu.Wuid}`); - setWorkunit(wu); displayErrors(wu, editor); setOutputMode(OutputMode.ERRORS); } else if (wu.isComplete()) { @@ -247,7 +246,6 @@ const ECLEditorToolbar: React.FunctionComponent = ({ wu.publish(queryName); setWuState("Published"); } - setWorkunit(wu); setOutputMode(OutputMode.RESULTS); } } else { @@ -255,16 +253,17 @@ const ECLEditorToolbar: React.FunctionComponent = ({ logger.info(`${nlsHPCC.Playground} ${nlsHPCC.Finished} (${wu.Wuid})`); } } - }, [editor, queryName, setOutputMode, setWorkunit]); + }, [editor, queryName, setOutputMode]); const submitWU = React.useCallback(async () => { const wu = await Workunit.create({ baseUrl: "" }); + setWorkunit(wu); await wu.update({ Jobname: queryName, QueryText: editor.ecl() }); await wu.submit(cluster); wu.watchUntilComplete(changes => playgroundResults(wu)); - }, [cluster, editor, playgroundResults, queryName]); + }, [cluster, editor, playgroundResults, queryName, setWorkunit]); const publishWU = React.useCallback(async () => { if (queryName === "") { @@ -274,13 +273,14 @@ const ECLEditorToolbar: React.FunctionComponent = ({ setQueryNameErrorMsg(""); const wu = await Workunit.create({ baseUrl: "" }); + setWorkunit(wu); await wu.update({ Jobname: queryName, QueryText: editor.ecl() }); await wu.submit(cluster, WUUpdate.Action.Compile); wu.watchUntilComplete(changes => playgroundResults(wu, "publish")); } - }, [cluster, editor, playgroundResults, queryName, setQueryNameErrorMsg]); + }, [cluster, editor, playgroundResults, queryName, setQueryNameErrorMsg, setWorkunit]); const checkSyntax = React.useCallback(() => { service.WUSyntaxCheckECL({ @@ -470,6 +470,12 @@ export const ECLPlayground: React.FunctionComponent = (props }, [editor]); useOnEvent(document, "eclwatch-theme-toggle", handleThemeToggle); + const submissionComplete = React.useMemo(() => { + return workunit?.StateID === WUStateID.Completed || + workunit?.StateID === WUStateID.Failed || + (workunit?.ActionEx === "compile" && workunit?.StateID === WUStateID.Compiled); + }, [workunit?.StateID, workunit?.ActionEx]); + const handleEclChange = React.useMemo(() => debounce((evt) => { if (editor.hasFocus()) { setSyntaxStatusIcon(SyntaxCheckResult.Unknown); @@ -507,23 +513,33 @@ export const ECLPlayground: React.FunctionComponent = (props - + {submissionComplete ? + + :
+ +
+ } -
- {outputMode === OutputMode.ERRORS ? ( - - - ) : outputMode === OutputMode.RESULTS ? ( - - - ) : outputMode === OutputMode.VIS ? ( - - ) : null} -
+ {submissionComplete ? +
+ {outputMode === OutputMode.ERRORS ? ( + + + ) : outputMode === OutputMode.RESULTS ? ( + + + ) : outputMode === OutputMode.VIS ? ( + + ) : null} +
+ :
+ +
+ } ; From bb1a9e1871ff8836b6d68fc4c12bd78902b185b5 Mon Sep 17 00:00:00 2001 From: Jake Smith Date: Mon, 9 Dec 2024 13:06:58 +0000 Subject: [PATCH 4/7] HPCC-33022 Fix pathological dali load time JPTree (used by Dali SDS) could perform horrendously slowly when dealing with a large list of child nodes sharing a common prefix. This was due to how the hashing function was used (no seed number provided). With many entries having the same prefix, it caused a high number of hash collisions, resulting in excessive scanning when adding new members. This particularly affected the /GeneratedDlls section of Dali. After fixing the issue, the time to load this section dropped from over 4700 seconds to 17 seconds. Also add a 'loadxml' utility to daliadmin, that loads and times loading an xml file, and traces the amount of memory it consumes. Signed-off-by: Jake Smith --- dali/daliadmin/daadmin.cpp | 75 ++++++++++++++++++++++++++++++++++++ dali/daliadmin/daadmin.hpp | 1 + dali/daliadmin/daliadmin.cpp | 15 ++++++++ system/jlib/jhash.cpp | 2 +- system/jlib/jptree.cpp | 2 +- system/jlib/jptree.ipp | 6 +-- system/jlib/jstats.cpp | 2 +- system/jlib/jsuperhash.cpp | 22 +++++------ system/jlib/jsuperhash.hpp | 18 +++++---- 9 files changed, 118 insertions(+), 25 deletions(-) diff --git a/dali/daliadmin/daadmin.cpp b/dali/daliadmin/daadmin.cpp index 91f11f4a1d6..cee095e3321 100644 --- a/dali/daliadmin/daadmin.cpp +++ b/dali/daliadmin/daadmin.cpp @@ -2472,6 +2472,81 @@ void xmlSize(const char *filename, double pc) } } +void loadXMLTest(const char *filename, bool parseOnly, bool useLowMemPTree, bool saveFormattedXML) +{ + OwnedIFile iFile = createIFile(filename); + OwnedIFileIO iFileIO = iFile->open(IFOread); + if (!iFileIO) + { + WARNLOG("File '%s' not found", filename); + return; + } + + class CDummyPTreeMaker : public CSimpleInterfaceOf + { + StringBuffer xpath; + unsigned level = 0; + public: + virtual IPropertyTree *queryRoot() override { return nullptr; } + virtual IPropertyTree *queryCurrentNode() override { return nullptr; } + virtual void reset() override { } + virtual IPropertyTree *create(const char *tag) override { return nullptr; } + // IPTreeNotifyEvent impl. + virtual void beginNode(const char *tag, bool sequence, offset_t startOffset) override { } + virtual void newAttribute(const char *name, const char *value) override { } + virtual void beginNodeContent(const char *tag) override { } + virtual void endNode(const char *tag, unsigned length, const void *value, bool binary, offset_t endOffset) override { } + }; + + byte flags=ipt_none; + PTreeReaderOptions readFlags=ptr_ignoreWhiteSpace; + Owned iMaker; + if (!parseOnly) + { + PROGLOG("Creating property tree from file: %s", filename); + byte flags = ipt_none; + if (useLowMemPTree) + { + PROGLOG("Using low memory property trees"); + flags = ipt_lowmem; + } + iMaker.setown(createPTreeMaker(flags)); + } + else + { + PROGLOG("Reading property tree from file (without creating it): %s", filename); + iMaker.setown(new CDummyPTreeMaker()); + } + + offset_t fSize = iFileIO->size(); + OwnedIFileIOStream stream = createIOStream(iFileIO); + OwnedIFileIOStream progressedIFileIOStream = createProgressIFileIOStream(stream, fSize, "Load progress", 1); + Owned reader = createXMLStreamReader(*progressedIFileIOStream, *iMaker, readFlags); + + ProcessInfo memInfo(ReadMemoryInfo); + __uint64 rss = memInfo.getActiveResidentMemory(); + CCycleTimer timer; + reader->load(); + memInfo.update(ReadMemoryInfo); + __uint64 rssUsed = memInfo.getActiveResidentMemory() - rss; + reader.clear(); + progressedIFileIOStream.clear(); + PROGLOG("Load took: %.2f - RSS consumed: %.2f MB", (float)timer.elapsedMs()/1000, (float)rssUsed/0x100000); + + if (!parseOnly && saveFormattedXML) + { + assertex(iMaker->queryRoot()); + StringBuffer outFilename(filename); + outFilename.append(".out.xml"); + PROGLOG("Saving to %s", outFilename.str()); + timer.reset(); + saveXML(outFilename, iMaker->queryRoot(), 2); + PROGLOG("Save took: %.2f", (float)timer.elapsedMs()/1000); + } + + ::LINK(iMaker->queryRoot()); // intentionally leak (avoid time clearing up) +} + void translateToXpath(const char *logicalfile, DfsXmlBranchKind tailType) { CDfsLogicalFileName lfn; diff --git a/dali/daliadmin/daadmin.hpp b/dali/daliadmin/daadmin.hpp index 7a86e86ab30..9230e245d41 100644 --- a/dali/daliadmin/daadmin.hpp +++ b/dali/daliadmin/daadmin.hpp @@ -28,6 +28,7 @@ namespace daadmin extern DALIADMIN_API void setDaliConnectTimeoutMs(unsigned timeoutMs); extern DALIADMIN_API void xmlSize(const char *filename, double pc); +extern DALIADMIN_API void loadXMLTest(const char *filename, bool parseOnly, bool useLowMemPTree, bool saveFormattedXML); extern DALIADMIN_API void translateToXpath(const char *logicalfile, DfsXmlBranchKind tailType = DXB_File); extern DALIADMIN_API void exportToFile(const char *path, const char *filename, bool safe = false); diff --git a/dali/daliadmin/daliadmin.cpp b/dali/daliadmin/daliadmin.cpp index d70ceb728ed..c402fa6f62c 100644 --- a/dali/daliadmin/daliadmin.cpp +++ b/dali/daliadmin/daliadmin.cpp @@ -97,6 +97,9 @@ void usage(const char *exe) printf(" mpping -- time MP connect\n"); printf(" daliping [ ] -- time dali server connect\n"); printf(" getxref -- get all XREF information\n"); + printf(" loadxml [--lowmem[= ] [ files ] -- get all locked files/xpaths\n"); printf(" unlock <[path|file]> -- unlocks either matching xpath(s) or matching logical file(s), can contain wildcards\n"); printf(" validatestore [fix=]\n" @@ -226,6 +229,18 @@ int main(int argc, const char* argv[]) } else if (strieq(cmd, "remotetest")) remoteTest(params.item(1), false); + else if (strieq(cmd, "loadxml")) + { + bool useLowMemPTree = false; + bool saveFormatedTree = false; + bool parseOnly = getComponentConfigSP()->getPropBool("@parseonly"); + if (!parseOnly) + { + useLowMemPTree = getComponentConfigSP()->getPropBool("@lowmem"); + saveFormatedTree = getComponentConfigSP()->getPropBool("@savexml"); + } + loadXMLTest(params.item(1), parseOnly, useLowMemPTree, saveFormatedTree); + } else { UERRLOG("Unknown command %s",cmd); diff --git a/system/jlib/jhash.cpp b/system/jlib/jhash.cpp index 28bce3d24df..da42a193ef2 100644 --- a/system/jlib/jhash.cpp +++ b/system/jlib/jhash.cpp @@ -305,7 +305,7 @@ bool HashTable::keyeq(const void *key1, const void *key2, int ksize) const unsigned HashTable::hash(const void *key, int ksize) const { - unsigned h = 0x811C9DC5; + unsigned h = fnvInitialHash32; unsigned char *bp = (unsigned char *) key; if (ksize<=0) { diff --git a/system/jlib/jptree.cpp b/system/jlib/jptree.cpp index 1b02941f990..5b4151b2461 100644 --- a/system/jlib/jptree.cpp +++ b/system/jlib/jptree.cpp @@ -3847,7 +3847,7 @@ unsigned CAtomPTree::queryHash() const { const char *_name = name.get(); size32_t nl = strlen(_name); - return isnocase() ? hashnc((const byte *) _name, nl, 0): hashc((const byte *) _name, nl, 0); + return isnocase() ? hashnc((const byte *) _name, nl, fnvInitialHash32): hashc((const byte *) _name, nl, fnvInitialHash32); } } diff --git a/system/jlib/jptree.ipp b/system/jlib/jptree.ipp index f4037783958..fd5cb63fd89 100644 --- a/system/jlib/jptree.ipp +++ b/system/jlib/jptree.ipp @@ -58,7 +58,7 @@ protected: virtual unsigned getHashFromElement(const void *e) const override; virtual unsigned getHashFromFindParam(const void *fp) const override { - return hashcz((const unsigned char *)fp, 0); + return hashcz((const unsigned char *)fp, fnvInitialHash32); } virtual bool matchesFindParam(const void *e, const void *fp, unsigned fphash) const override { @@ -108,7 +108,7 @@ public: // SuperHashTable definitions virtual unsigned getHashFromFindParam(const void *fp) const override { - return hashncz((const unsigned char *)fp, 0); + return hashncz((const unsigned char *)fp, fnvInitialHash32); } virtual bool matchesFindParam(const void *e, const void *fp, unsigned fphash) const override { @@ -844,7 +844,7 @@ public: const char *myname = queryName(); assert(myname); size32_t nl = strlen(myname); - return isnocase() ? hashnc((const byte *)myname, nl, 0): hashc((const byte *)myname, nl, 0); + return isnocase() ? hashnc((const byte *)myname, nl, fnvInitialHash32): hashc((const byte *)myname, nl, fnvInitialHash32); } virtual void setName(const char *_name) override; virtual void setAttribute(const char *attr, const char *val, bool encoded) override; diff --git a/system/jlib/jstats.cpp b/system/jlib/jstats.cpp index 79d2944334d..283d148ee00 100644 --- a/system/jlib/jstats.cpp +++ b/system/jlib/jstats.cpp @@ -1290,7 +1290,7 @@ void StatisticsMapping::createMappings() } const unsigned * kinds = indexToKind.getArray(); - hashcode = hashc((const byte *)kinds, indexToKind.ordinality() * sizeof(unsigned), 0x811C9DC5); + hashcode = hashc((const byte *)kinds, indexToKind.ordinality() * sizeof(unsigned), fnvInitialHash32); //All StatisticsMapping objects are assumed to be static, and never destroyed. const StatisticsMapping * existing = allStatsMappings[hashcode]; diff --git a/system/jlib/jsuperhash.cpp b/system/jlib/jsuperhash.cpp index a1e2b22d229..3e26af7b888 100644 --- a/system/jlib/jsuperhash.cpp +++ b/system/jlib/jsuperhash.cpp @@ -33,8 +33,8 @@ //#define MY_TRACE_HASH #ifdef MY_TRACE_HASH -int my_search_tot = 0; -int my_search_num = 0; +static unsigned my_search_tot = 0; +static unsigned my_search_num = 0; #endif //-- SuperHashTable --------------------------------------------------- @@ -104,13 +104,13 @@ void SuperHashTable::dumpStats() const { #ifdef TRACE_HASH if (tablecount && search_tot && search_num) - printf("Hash table %d entries, %d size, average search length %d(%d/%d) max %d\n", tablecount, tablesize, - (int) (search_tot/search_num), search_tot, search_num, search_max); + printf("Hash table %u entries, %u size, average search length %" I64F "u(%" I64F "u/%u) max %u\n", tablecount, tablesize, + search_tot/search_num, search_tot, search_num, search_max); #endif } #ifdef TRACE_HASH -void SuperHashTable::note_searchlen(int len) const +void SuperHashTable::note_searchlen(unsigned len) const { search_tot += len; search_num++; @@ -151,8 +151,8 @@ unsigned SuperHashTable::doFind(unsigned findHash, const void * findParam) const } #ifdef MY_TRACE_HASH my_search_num++; - if(my_search_num != 0) - printf("Hash table average search length %d\n", (int) (my_search_tot/my_search_num)); + if (my_search_num != 0) + printf("Hash table average search length %u\n", my_search_tot/my_search_num); #endif #ifdef TRACE_HASH note_searchlen(searchlen); @@ -193,8 +193,8 @@ unsigned SuperHashTable::doFindElement(unsigned v, const void * findET) const } #ifdef MY_TRACE_HASH my_search_num++; - if(my_search_num != 0) - printf("Hash table average search length %d\n", (int) (my_search_tot/my_search_num)); + if (my_search_num != 0) + printf("Hash table average search length %u\n", my_search_tot/my_search_num); #endif #ifdef TRACE_HASH note_searchlen(searchlen); @@ -233,8 +233,8 @@ unsigned SuperHashTable::doFindNew(unsigned v) const } #ifdef MY_TRACE_HASH my_search_num++; - if(my_search_num != 0) - printf("Hash table average search length %d\n", (int) (my_search_tot/my_search_num)); + if (my_search_num != 0) + printf("Hash table average search length %u\n", my_search_tot/my_search_num); #endif #ifdef TRACE_HASH note_searchlen(searchlen); diff --git a/system/jlib/jsuperhash.hpp b/system/jlib/jsuperhash.hpp index 9936e86112f..2c82a8fb5e8 100644 --- a/system/jlib/jsuperhash.hpp +++ b/system/jlib/jsuperhash.hpp @@ -28,6 +28,8 @@ #include "jstring.hpp" #include "jmutex.hpp" +constexpr unsigned fnvInitialHash32 = 0x811C9DC5; + extern jlib_decl unsigned hashc( const unsigned char *k, unsigned length, unsigned initval); extern jlib_decl unsigned hashnc( const unsigned char *k, unsigned length, unsigned initval); extern jlib_decl unsigned hashcz( const unsigned char *k, unsigned initval); @@ -88,7 +90,7 @@ class jlib_decl SuperHashTable : public CInterface void doKill(void); void expand(); void expand(unsigned newsize); - void note_searchlen(int) const; + void note_searchlen(unsigned) const; virtual void onAdd(void *et) = 0; virtual void onRemove(void *et) = 0; @@ -105,9 +107,9 @@ class jlib_decl SuperHashTable : public CInterface unsigned tablesize; unsigned tablecount; #ifdef TRACE_HASH - mutable int search_tot; - mutable int search_num; - mutable int search_max; + mutable unsigned __int64 search_tot; + mutable unsigned search_num; + mutable unsigned search_max; #endif }; @@ -514,9 +516,9 @@ class jlib_decl AtomRefTable : public SuperHashTableOfkeyPtr()), key, l+1); if (nocase) - hke->hashValue = hashnc((const unsigned char *)key, l, 0); + hke->hashValue = hashnc((const unsigned char *)key, l, fnvInitialHash32); else - hke->hashValue = hashc((const unsigned char *)key, l, 0); + hke->hashValue = hashc((const unsigned char *)key, l, fnvInitialHash32); hke->linkCount = 0; return hke; } @@ -609,9 +611,9 @@ class jlib_decl AtomRefTable : public SuperHashTableOf Date: Wed, 11 Dec 2024 16:06:15 +0000 Subject: [PATCH 5/7] HPCC-33113 Ensure empty compressed part size is correct When writing a logical file that is wider than Thor with, Thor pads the file at publish time with empty parts. If compressed it creates an empty compressed file, which will typically be non-zero in size (e.g. contains compressed header). The logical file part meta data did not relect this correct physical size. Consequently, as a result of the check added in HPCC-33064, reading these files was provoking this error check. Signed-off-by: Jake Smith --- thorlcr/activities/thdiskbase.cpp | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/thorlcr/activities/thdiskbase.cpp b/thorlcr/activities/thdiskbase.cpp index dae7d5d870f..ba5f23b0498 100644 --- a/thorlcr/activities/thdiskbase.cpp +++ b/thorlcr/activities/thdiskbase.cpp @@ -293,6 +293,9 @@ void CWriteMasterBase::publish() compMethod = COMPRESS_METHOD_LZ4HC; bool blockCompressed; bool compressed = fileDesc->isCompressed(&blockCompressed); + + // NB: it would be far preferable to avoid this and have the file reference a group with the correct number of parts + // Possibly could use subgroup syntax: 'data[1..n]' for (unsigned clusterIdx=0; clusterIdxnumClusters(); clusterIdx++) { StringBuffer clusterName; @@ -305,6 +308,7 @@ void CWriteMasterBase::publish() p += queryJob().querySlaves(); IPartDescriptor *partDesc = fileDesc->queryPart(p); CDateTime createTime, modifiedTime; + offset_t compSize = 0; for (unsigned c=0; cnumCopies(); c++) { RemoteFilename rfn; @@ -316,7 +320,7 @@ void CWriteMasterBase::publish() ensureDirectoryForFile(path.str()); OwnedIFile iFile = createIFile(path.str()); Owned iFileIO; - if (compressed) + if (compressed) // NB: this would not be necessary if all builds have the changes in HPCC-32651 iFileIO.setown(createCompressedFileWriter(iFile, recordSize, false, true, NULL, compMethod)); else iFileIO.setown(iFile->open(IFOcreate)); @@ -324,7 +328,10 @@ void CWriteMasterBase::publish() iFileIO.clear(); // ensure copies have matching datestamps, as they would do normally (backupnode expects it) if (0 == c) + { iFile->getTime(&createTime, &modifiedTime, NULL); + compSize = iFile->size(); + } else iFile->setTime(&createTime, &modifiedTime, NULL); } @@ -345,7 +352,7 @@ void CWriteMasterBase::publish() props.setPropInt64("@recordCount", 0); props.setPropInt64("@size", 0); if (compressed) - props.setPropInt64("@compressedSize", 0); + props.setPropInt64("@compressedSize", compSize); p++; } } From b91afa51250bbc77037f87a62c141fddb495f5b8 Mon Sep 17 00:00:00 2001 From: Jake Smith Date: Wed, 11 Dec 2024 17:59:10 +0000 Subject: [PATCH 6/7] HPCC-33116 Suppress file check if invalid @compressedSize(0) Signed-off-by: Jake Smith --- dali/base/dadfs.cpp | 39 +++++++++++++++++++ dali/base/dadfs.hpp | 4 ++ ecl/hthor/hthor.cpp | 10 ++--- .../activities/diskread/thdiskreadslave.cpp | 10 ++--- 4 files changed, 49 insertions(+), 14 deletions(-) diff --git a/dali/base/dadfs.cpp b/dali/base/dadfs.cpp index 7b008f187b4..57017be1448 100644 --- a/dali/base/dadfs.cpp +++ b/dali/base/dadfs.cpp @@ -13807,6 +13807,45 @@ void configurePreferredPlanes() } } +static bool doesPhysicalMatchMeta(IPropertyTree &partProps, IFile &iFile, offset_t expectedSize, offset_t &actualSize) +{ + // NB: temporary workaround for 'narrow' files publishing extra empty parts with the wrong @compressedSize(0) + // causing a new check introduced in HPCC-33064 to be hit (fixed in HPCC-33113, but will continue to affect exiting files) + unsigned __int64 size = partProps.getPropInt64("@size", unknownFileSize); + unsigned __int64 compressedSize = partProps.getPropInt64("@compressedSize", unknownFileSize); + if ((0 == size) && (0 == compressedSize)) + { + actualSize = unknownFileSize; + return true; + } + + if (expectedSize != unknownFileSize) + { + actualSize = iFile.size(); + if (actualSize != expectedSize) + return false; + } + else + actualSize = unknownFileSize; + + return true; +} + +bool doesPhysicalMatchMeta(IPartDescriptor &partDesc, IFile &iFile, offset_t &expectedSize, offset_t &actualSize) +{ + IPropertyTree &partProps = partDesc.queryProperties(); + expectedSize = partDesc.getDiskSize(false, false); + return doesPhysicalMatchMeta(partProps, iFile, expectedSize, actualSize); +} + +bool doesPhysicalMatchMeta(IDistributedFilePart &part, IFile &iFile, offset_t &expectedSize, offset_t &actualSize) +{ + IPropertyTree &partProps = part.queryAttributes(); + expectedSize = part.getDiskSize(false, false); + return doesPhysicalMatchMeta(partProps, iFile, expectedSize, actualSize); +} + + #ifdef _USE_CPPUNIT /* * This method removes files only logically. removeEntry() used to do that, but the only diff --git a/dali/base/dadfs.hpp b/dali/base/dadfs.hpp index 6cde9f34875..4e3f5fd2124 100644 --- a/dali/base/dadfs.hpp +++ b/dali/base/dadfs.hpp @@ -925,4 +925,8 @@ inline cost_type getLegacyWriteCost(const IPropertyTree & fileAttr, Source sourc else return 0; } + +extern da_decl bool doesPhysicalMatchMeta(IPartDescriptor &partDesc, IFile &iFile, offset_t &expectedSize, offset_t &actualSize); +extern da_decl bool doesPhysicalMatchMeta(IDistributedFilePart &partDesc, IFile &iFile, offset_t &expectedSize, offset_t &actualSize); + #endif diff --git a/ecl/hthor/hthor.cpp b/ecl/hthor/hthor.cpp index 06267a4908d..24d6f9027bc 100644 --- a/ecl/hthor/hthor.cpp +++ b/ecl/hthor/hthor.cpp @@ -8737,13 +8737,9 @@ bool CHThorDiskReadBaseActivity::openNext() if (curPart) { - offset_t expectedSize = curPart->getDiskSize(false, false); - if (expectedSize != unknownFileSize) - { - offset_t actualSize = inputfile->size(); - if(actualSize != expectedSize) - throw MakeStringException(0, "File size mismatch: file %s was supposed to be %" I64F "d bytes but appears to be %" I64F "d bytes", inputfile->queryFilename(), expectedSize, actualSize); - } + offset_t expectedSize, actualSize; + if (!doesPhysicalMatchMeta(*curPart, *inputfile, expectedSize, actualSize)) + throw makeStringExceptionV(0, "File size mismatch: file %s was supposed to be %" I64F "d bytes but appears to be %" I64F "d bytes", inputfile->queryFilename(), expectedSize, actualSize); } if (compressed) diff --git a/thorlcr/activities/diskread/thdiskreadslave.cpp b/thorlcr/activities/diskread/thdiskreadslave.cpp index c09718a041d..dd244fc6c5a 100644 --- a/thorlcr/activities/diskread/thdiskreadslave.cpp +++ b/thorlcr/activities/diskread/thdiskreadslave.cpp @@ -366,13 +366,9 @@ void CDiskRecordPartHandler::open() rwFlags |= DEFAULT_RWFLAGS; - offset_t expectedSize = partDesc->getDiskSize(false, false); - if (expectedSize != unknownFileSize) - { - offset_t actualSize = iFile->size(); - if(actualSize != expectedSize) - throw MakeStringException(0, "File size mismatch: file %s was supposed to be %" I64F "d bytes but appears to be %" I64F "d bytes", iFile->queryFilename(), expectedSize, actualSize); - } + offset_t expectedSize, actualSize; + if (!doesPhysicalMatchMeta(*partDesc, *iFile, expectedSize, actualSize)) + throw MakeActivityException(&activity, 0, "File size mismatch: file %s was supposed to be %" I64F "d bytes but appears to be %" I64F "d bytes", iFile->queryFilename(), expectedSize, actualSize); if (compressed) { From 5eeb875356f0ca56f78738cac6d0e47a0e471f8d Mon Sep 17 00:00:00 2001 From: Gordon Smith Date: Thu, 12 Dec 2024 11:20:22 +0000 Subject: [PATCH 7/7] HPCC-33117 Switch Workunits page to use @hpcc-js/comms fully WU Monitoring is opt in, rather than opt out, which is preferable for the React WUs page. Signed-off-by: Gordon Smith --- esp/src/src-react/comms/workunit.ts | 71 ++++++++++++++++++++++ esp/src/src-react/components/Workunits.tsx | 17 +++--- 2 files changed, 80 insertions(+), 8 deletions(-) create mode 100644 esp/src/src-react/comms/workunit.ts diff --git a/esp/src/src-react/comms/workunit.ts b/esp/src/src-react/comms/workunit.ts new file mode 100644 index 00000000000..26debedffa9 --- /dev/null +++ b/esp/src/src-react/comms/workunit.ts @@ -0,0 +1,71 @@ +import { Workunit, WorkunitsService, type WsWorkunits } from "@hpcc-js/comms"; +import { Thenable } from "src/store/Deferred"; +import { Paged } from "src/store/Paged"; +import { BaseStore } from "src/store/Store"; +import { wuidToDateTime } from "src/Utility"; + +const service = new WorkunitsService({ baseUrl: "" }); + +export type WUQueryStore = BaseStore; + +export function CreateWUQueryStore(): BaseStore { + const store = new Paged({ + start: "PageStartFrom", + count: "PageSize", + sortBy: "Sortby", + descending: "Descending" + }, "Wuid", (request, abortSignal): Thenable<{ data: Workunit[], total: number }> => { + if (request.Sortby && request.Sortby === "TotalClusterTime") { + request.Sortby = "ClusterTime"; + } + return service.WUQuery(request, abortSignal).then(response => { + const page = { + start: undefined, + end: undefined + }; + const data = response.Workunits.ECLWorkunit.map((wu): Workunit => { + const start = wuidToDateTime(wu.Wuid); + if (!page.start || page.start > start) { + page.start = start; + } + let timePartsSection = 0; + const end = new Date(start); + const timeParts = wu.TotalClusterTime?.split(":") ?? []; + while (timeParts.length) { + const timePart = timeParts.pop(); + switch (timePartsSection) { + case 0: + end.setSeconds(end.getSeconds() + +timePart); + break; + case 1: + end.setMinutes(end.getMinutes() + +timePart); + break; + case 2: + end.setHours(end.getHours() + +timePart); + break; + case 3: + end.setDate(end.getDate() + +timePart); + break; + } + ++timePartsSection; + } + if (!page.end || page.end < end) { + page.end = end; + } + const retVal = Workunit.attach(service, wu.Wuid, wu); + // HPCC-33121 - Move to @hpcc-js/comms --- + retVal["__timeline_timings"] = { + start, + end, + page + }; + return retVal; + }); + return { + data, + total: response.NumWUs + }; + }); + }); + return store; +} diff --git a/esp/src/src-react/components/Workunits.tsx b/esp/src/src-react/components/Workunits.tsx index 5826e790dde..b53ba30dabc 100644 --- a/esp/src/src-react/components/Workunits.tsx +++ b/esp/src/src-react/components/Workunits.tsx @@ -1,8 +1,9 @@ import * as React from "react"; import { CommandBar, ContextualMenuItemType, DetailsRow, ICommandBarItemProps, IDetailsRowProps, Icon, Image, Link } from "@fluentui/react"; import { hsl as d3Hsl } from "@hpcc-js/common"; +import { Workunit } from "@hpcc-js/comms"; import { SizeMe } from "react-sizeme"; -import { CreateWUQueryStore, defaultSort, emptyFilter, Get, WUQueryStore, formatQuery } from "src/ESPWorkunit"; +import { defaultSort, emptyFilter, getStateImage, WUQueryStore, formatQuery } from "src/ESPWorkunit"; import * as WsWorkunits from "src/WsWorkunits"; import { formatCost } from "src/Session"; import { userKeyValStore } from "src/KeyValStore"; @@ -14,6 +15,7 @@ import { useLogicalClustersPalette } from "../hooks/platform"; import { calcSearch, pushParams } from "../util/history"; import { useHasFocus, useIsMounted } from "../hooks/util"; import { HolyGrail } from "../layouts/HolyGrail"; +import { CreateWUQueryStore } from "../comms/workunit"; import { FluentPagedGrid, FluentPagedFooter, useCopyButtons, useFluentStoreState, FluentColumns } from "./controls/Grid"; import { Fields } from "./forms/Fields"; import { Filter } from "./forms/Filter"; @@ -122,11 +124,10 @@ export const Workunits: React.FunctionComponent = ({ Wuid: { label: nlsHPCC.WUID, width: 120, sortable: true, - formatter: (Wuid, row) => { - const wu = Get(Wuid); + formatter: (Wuid: string, wu: Workunit) => { const search = calcSearch(filter); return <> - +   {Wuid} ; @@ -294,10 +295,10 @@ export const Workunits: React.FunctionComponent = ({ }, [selection]); const renderRowTimings = React.useCallback((props: IDetailsRowProps, size: { readonly width: number; readonly height: number; }) => { - if (showTimeline && props?.item?.timings) { - const total = props.item.timings.page.end - props.item.timings.page.start; - const startPct = 100 - (props.item.timings.start - props.item.timings.page.start) / total * 100; - const endPct = 100 - (props.item.timings.end - props.item.timings.page.start) / total * 100; + if (showTimeline && props?.item?.__timeline_timings) { + const total = props.item.__timeline_timings.page.end - props.item.__timeline_timings.page.start; + const startPct = 100 - (props.item.__timeline_timings.start - props.item.__timeline_timings.page.start) / total * 100; + const endPct = 100 - (props.item.__timeline_timings.end - props.item.__timeline_timings.page.start) / total * 100; const backgroundColor = palette(props.item.Cluster); const borderColor = d3Hsl(backgroundColor).darker().toString();