diff --git a/.github/workflows/build-test-eclwatch.yml b/.github/workflows/build-test-eclwatch.yml index 34b6fd1553f..61012f4197c 100644 --- a/.github/workflows/build-test-eclwatch.yml +++ b/.github/workflows/build-test-eclwatch.yml @@ -25,7 +25,7 @@ jobs: build: strategy: matrix: - node: ["20", "18", "16"] + node: ["22", "20", "18"] fail-fast: false name: "Check eclwatch and npm" needs: pre_job @@ -44,6 +44,12 @@ jobs: - name: Install Dependencies working-directory: ./esp/src run: npm ci + - name: Lint + working-directory: ./esp/src + run: npm run lint + - name: Install Playwright browsers + working-directory: ./esp/src + run: npx playwright install --with-deps - name: Build working-directory: ./esp/src run: npm run build diff --git a/dali/daliadmin/daadmin.cpp b/dali/daliadmin/daadmin.cpp index d1ea8bd9b02..7b8d2e38a36 100644 --- a/dali/daliadmin/daadmin.cpp +++ b/dali/daliadmin/daadmin.cpp @@ -2557,6 +2557,81 @@ void xmlSize(const char *filename, double pc) } } +void loadXMLTest(const char *filename, bool parseOnly, bool useLowMemPTree, bool saveFormattedXML) +{ + OwnedIFile iFile = createIFile(filename); + OwnedIFileIO iFileIO = iFile->open(IFOread); + if (!iFileIO) + { + WARNLOG("File '%s' not found", filename); + return; + } + + class CDummyPTreeMaker : public CSimpleInterfaceOf + { + StringBuffer xpath; + unsigned level = 0; + public: + virtual IPropertyTree *queryRoot() override { return nullptr; } + virtual IPropertyTree *queryCurrentNode() override { return nullptr; } + virtual void reset() override { } + virtual IPropertyTree *create(const char *tag) override { return nullptr; } + // IPTreeNotifyEvent impl. + virtual void beginNode(const char *tag, bool sequence, offset_t startOffset) override { } + virtual void newAttribute(const char *name, const char *value) override { } + virtual void beginNodeContent(const char *tag) override { } + virtual void endNode(const char *tag, unsigned length, const void *value, bool binary, offset_t endOffset) override { } + }; + + byte flags=ipt_none; + PTreeReaderOptions readFlags=ptr_ignoreWhiteSpace; + Owned iMaker; + if (!parseOnly) + { + PROGLOG("Creating property tree from file: %s", filename); + byte flags = ipt_none; + if (useLowMemPTree) + { + PROGLOG("Using low memory property trees"); + flags = ipt_lowmem; + } + iMaker.setown(createPTreeMaker(flags)); + } + else + { + PROGLOG("Reading property tree from file (without creating it): %s", filename); + iMaker.setown(new CDummyPTreeMaker()); + } + + offset_t fSize = iFileIO->size(); + OwnedIFileIOStream stream = createIOStream(iFileIO); + OwnedIFileIOStream progressedIFileIOStream = createProgressIFileIOStream(stream, fSize, "Load progress", 1); + Owned reader = createXMLStreamReader(*progressedIFileIOStream, *iMaker, readFlags); + + ProcessInfo memInfo(ReadMemoryInfo); + __uint64 rss = memInfo.getActiveResidentMemory(); + CCycleTimer timer; + reader->load(); + memInfo.update(ReadMemoryInfo); + __uint64 rssUsed = memInfo.getActiveResidentMemory() - rss; + reader.clear(); + progressedIFileIOStream.clear(); + PROGLOG("Load took: %.2f - RSS consumed: %.2f MB", (float)timer.elapsedMs()/1000, (float)rssUsed/0x100000); + + if (!parseOnly && saveFormattedXML) + { + assertex(iMaker->queryRoot()); + StringBuffer outFilename(filename); + outFilename.append(".out.xml"); + PROGLOG("Saving to %s", outFilename.str()); + timer.reset(); + saveXML(outFilename, iMaker->queryRoot(), 2); + PROGLOG("Save took: %.2f", (float)timer.elapsedMs()/1000); + } + + ::LINK(iMaker->queryRoot()); // intentionally leak (avoid time clearing up) +} + void translateToXpath(const char *logicalfile, DfsXmlBranchKind tailType) { CDfsLogicalFileName lfn; diff --git a/dali/daliadmin/daadmin.hpp b/dali/daliadmin/daadmin.hpp index 687d0d882b2..e6ba91fa7ee 100644 --- a/dali/daliadmin/daadmin.hpp +++ b/dali/daliadmin/daadmin.hpp @@ -28,6 +28,7 @@ namespace daadmin extern DALIADMIN_API void setDaliConnectTimeoutMs(unsigned timeoutMs); extern DALIADMIN_API void xmlSize(const char *filename, double pc); +extern DALIADMIN_API void loadXMLTest(const char *filename, bool parseOnly, bool useLowMemPTree, bool saveFormattedXML); extern DALIADMIN_API void translateToXpath(const char *logicalfile, DfsXmlBranchKind tailType = DXB_File); extern DALIADMIN_API void exportToFile(const char *path, const char *filename, bool safe = false); diff --git a/dali/daliadmin/daliadmin.cpp b/dali/daliadmin/daliadmin.cpp index 0491b7d42f1..4a22ce29b3c 100644 --- a/dali/daliadmin/daliadmin.cpp +++ b/dali/daliadmin/daliadmin.cpp @@ -98,6 +98,9 @@ void usage(const char *exe) printf(" dalilocks [ ] [ files ] -- get all locked files/xpaths\n"); printf(" daliping [ ] -- time dali server connect\n"); printf(" getxref -- get all XREF information\n"); + printf(" loadxml [--lowmem[= [] [dryrun] [createmaps] [listonly] [verbose]\n"); printf(" mpping -- time MP connect\n"); printf(" serverlist -- list server IPs (mask optional)\n"); @@ -229,6 +232,18 @@ int main(int argc, const char* argv[]) } else if (strieq(cmd, "remotetest")) remoteTest(params.item(1), false); + else if (strieq(cmd, "loadxml")) + { + bool useLowMemPTree = false; + bool saveFormatedTree = false; + bool parseOnly = getComponentConfigSP()->getPropBool("@parseonly"); + if (!parseOnly) + { + useLowMemPTree = getComponentConfigSP()->getPropBool("@lowmem"); + saveFormatedTree = getComponentConfigSP()->getPropBool("@savexml"); + } + loadXMLTest(params.item(1), parseOnly, useLowMemPTree, saveFormatedTree); + } else { UERRLOG("Unknown command %s",cmd); diff --git a/esp/src/.gitignore b/esp/src/.gitignore index a2270e87528..09c76804ba1 100644 --- a/esp/src/.gitignore +++ b/esp/src/.gitignore @@ -1,7 +1,11 @@ +blob-report/ build/ hpcc-js/ lib/ node_modules/ +playwright/.cache/ +playwright-report/ +test-results/ types/ .vscode/* !.vscode/tasks.json diff --git a/esp/src/lws.config.js b/esp/src/lws.config.js index 39dfa94dcb8..00826fdf310 100644 --- a/esp/src/lws.config.js +++ b/esp/src/lws.config.js @@ -1,6 +1,6 @@ const fs = require("fs"); -let ip = "192.168.99.103"; +let ip = "https://play.hpccsystems.com:18010"; if (fs.existsSync("./lws.target.txt")) { ip = fs.readFileSync("./lws.target.txt").toString().replace("\r\n", "\n").split("\n")[0]; } @@ -68,5 +68,5 @@ let rewrite = [ module.exports = { port: 8080, rewrite: rewrite, - stack: ['lws-basic-auth', 'lws-request-monitor', 'lws-log', 'lws-cors', 'lws-json', 'lws-compress', 'lws-rewrite', 'lws-blacklist', 'lws-conditional-get', 'lws-mime', 'lws-range', 'lws-spa', 'lws-static', 'lws-index'] + stack: ["lws-basic-auth", "lws-request-monitor", "lws-log", "lws-cors", "lws-json", "lws-compress", "lws-rewrite", "lws-blacklist", "lws-conditional-get", "lws-mime", "lws-range", "lws-spa", "lws-static", "lws-index"] }; \ No newline at end of file diff --git a/esp/src/package-lock.json b/esp/src/package-lock.json index e54c6aba8ef..297253ae249 100644 --- a/esp/src/package-lock.json +++ b/esp/src/package-lock.json @@ -57,13 +57,16 @@ "xstyle": "0.3.3" }, "devDependencies": { + "@playwright/test": "^1.49.0", "@simbathesailor/use-what-changed": "^2.0.0", "@types/dojo": "1.9.48", + "@types/node": "^22.10.1", "@types/react": "17.0.80", "@types/react-dom": "17.0.25", "@typescript-eslint/eslint-plugin": "6.21.0", "@typescript-eslint/parser": "6.21.0", "copyfiles": "2.4.1", + "cross-env": "^7.0.3", "css-loader": "6.10.0", "dojo-webpack-plugin": "3.0.6", "eslint": "8.57.0", @@ -2626,6 +2629,21 @@ "openid-client": "^5.3.0" } }, + "node_modules/@kubernetes/client-node/node_modules/@types/node": { + "version": "20.17.9", + "resolved": "https://registry.npmjs.org/@types/node/-/node-20.17.9.tgz", + "integrity": "sha512-0JOXkRyLanfGPE2QRCwgxhzlBAvaRdCNMcvbd7jFfpmD4eEXll7LRwy5ymJmyeZqk7Nh7eD2LeUyQ68BbndmXw==", + "license": "MIT", + "dependencies": { + "undici-types": "~6.19.2" + } + }, + "node_modules/@kubernetes/client-node/node_modules/undici-types": { + "version": "6.19.8", + "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-6.19.8.tgz", + "integrity": "sha512-ve2KP6f/JnbPBFyobGHuerC9g1FYGn/F8n1LWTwNxCEzd6IfqTwUQcNXgEtmmQ6DlRrC1hrSrBnCZPokRrDHjw==", + "license": "MIT" + }, "node_modules/@leichtgewicht/ip-codec": { "version": "2.0.5", "resolved": "https://registry.npmjs.org/@leichtgewicht/ip-codec/-/ip-codec-2.0.5.tgz", @@ -3143,6 +3161,22 @@ "node": ">=14" } }, + "node_modules/@playwright/test": { + "version": "1.49.0", + "resolved": "https://registry.npmjs.org/@playwright/test/-/test-1.49.0.tgz", + "integrity": "sha512-DMulbwQURa8rNIQrf94+jPJQ4FmOVdpE5ZppRNvWVjvhC+6sOeo28r8MgIpQRYouXRtt/FCCXU7zn20jnHR4Qw==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "playwright": "1.49.0" + }, + "bin": { + "playwright": "cli.js" + }, + "engines": { + "node": ">=18" + } + }, "node_modules/@simbathesailor/use-what-changed": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/@simbathesailor/use-what-changed/-/use-what-changed-2.0.0.tgz", @@ -3442,11 +3476,12 @@ "dev": true }, "node_modules/@types/node": { - "version": "20.10.5", - "resolved": "https://registry.npmjs.org/@types/node/-/node-20.10.5.tgz", - "integrity": "sha512-nNPsNE65wjMxEKI93yOP+NPGGBJz/PoN3kZsVLee0XMiJolxSekEVD8wRwBUBqkwc7UWop0edW50yrCQW4CyRw==", + "version": "22.10.1", + "resolved": "https://registry.npmjs.org/@types/node/-/node-22.10.1.tgz", + "integrity": "sha512-qKgsUwfHZV2WCWLAnVP1JqnpE6Im6h3Y0+fYgMTasNQ7V++CBX5OT1as0g0f+OyubbFqhf6XVNIsmN4IIhEgGQ==", + "license": "MIT", "dependencies": { - "undici-types": "~5.26.4" + "undici-types": "~6.20.0" } }, "node_modules/@types/node-forge": { @@ -5078,6 +5113,25 @@ "node": ">=10" } }, + "node_modules/cross-env": { + "version": "7.0.3", + "resolved": "https://registry.npmjs.org/cross-env/-/cross-env-7.0.3.tgz", + "integrity": "sha512-+/HKd6EgcQCJGh2PSjZuUitQBQynKor4wrFbRg4DtAgS1aWO+gU52xpH7M9ScGgXSYmAVS9bIJ8EzuaGw0oNAw==", + "dev": true, + "license": "MIT", + "dependencies": { + "cross-spawn": "^7.0.1" + }, + "bin": { + "cross-env": "src/bin/cross-env.js", + "cross-env-shell": "src/bin/cross-env-shell.js" + }, + "engines": { + "node": ">=10.14", + "npm": ">=6", + "yarn": ">=1" + } + }, "node_modules/cross-spawn": { "version": "7.0.3", "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.3.tgz", @@ -9371,6 +9425,53 @@ "node": ">=8" } }, + "node_modules/playwright": { + "version": "1.49.0", + "resolved": "https://registry.npmjs.org/playwright/-/playwright-1.49.0.tgz", + "integrity": "sha512-eKpmys0UFDnfNb3vfsf8Vx2LEOtflgRebl0Im2eQQnYMA4Aqd+Zw8bEOB+7ZKvN76901mRnqdsiOGKxzVTbi7A==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "playwright-core": "1.49.0" + }, + "bin": { + "playwright": "cli.js" + }, + "engines": { + "node": ">=18" + }, + "optionalDependencies": { + "fsevents": "2.3.2" + } + }, + "node_modules/playwright-core": { + "version": "1.49.0", + "resolved": "https://registry.npmjs.org/playwright-core/-/playwright-core-1.49.0.tgz", + "integrity": "sha512-R+3KKTQF3npy5GTiKH/T+kdhoJfJojjHESR1YEWhYuEKRVfVaxH3+4+GvXE5xyCngCxhxnykk0Vlah9v8fs3jA==", + "dev": true, + "license": "Apache-2.0", + "bin": { + "playwright-core": "cli.js" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/playwright/node_modules/fsevents": { + "version": "2.3.2", + "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.2.tgz", + "integrity": "sha512-xiqMQR4xAeHTuB9uWm+fFRcIOgKBMiOBP+eXiyT7jsgVCq1bkVygt00oASowB7EdtpOHaaPgKt812P9ab+DDKA==", + "dev": true, + "hasInstallScript": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": "^8.16.0 || ^10.6.0 || >=11.0.0" + } + }, "node_modules/postcss": { "version": "8.4.33", "resolved": "https://registry.npmjs.org/postcss/-/postcss-8.4.33.tgz", @@ -11539,9 +11640,10 @@ } }, "node_modules/undici-types": { - "version": "5.26.5", - "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-5.26.5.tgz", - "integrity": "sha512-JlCMO+ehdEIKqlFxk6IfVoAUVmgz7cU7zD/h9XZ0qzeosSHmUJVOzSQvvYSYWXkFXC+IfLKSIffhv0sVZup6pA==" + "version": "6.20.0", + "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-6.20.0.tgz", + "integrity": "sha512-Ny6QZ2Nju20vw1SRHe3d9jVu6gJ+4e3+MMpqu7pqE5HT6WsTSlce++GQmK5UXS8mzV8DSYHrQH+Xrf2jVcuKNg==", + "license": "MIT" }, "node_modules/universal-github-app-jwt": { "version": "1.1.1", diff --git a/esp/src/package.json b/esp/src/package.json index 2cb35d04463..b663f6391f6 100644 --- a/esp/src/package.json +++ b/esp/src/package.json @@ -28,8 +28,11 @@ "dev-start": "run-p bundle-watch dev-start-ws", "dev-start-verbose": "ws --verbose.include request response", "rm-hpcc": "rimraf ./node_modules/@hpcc-js", - "start": "webpack serve --env development --config webpack.config.js", - "test": "run-s lint", + "start": "ws", + "test": "npx playwright test", + "test-ci": "cross-env CI=1 npx playwright test", + "test-codegen": "npx playwright codegen", + "test-interactive": "npx playwright test --ui", "update": "npx npm-check-updates -u -t minor", "update-major": "npx npm-check-updates -u" }, @@ -83,13 +86,16 @@ "xstyle": "0.3.3" }, "devDependencies": { + "@playwright/test": "^1.49.0", "@simbathesailor/use-what-changed": "^2.0.0", "@types/dojo": "1.9.48", + "@types/node": "^22.10.1", "@types/react": "17.0.80", "@types/react-dom": "17.0.25", "@typescript-eslint/eslint-plugin": "6.21.0", "@typescript-eslint/parser": "6.21.0", "copyfiles": "2.4.1", + "cross-env": "^7.0.3", "css-loader": "6.10.0", "dojo-webpack-plugin": "3.0.6", "eslint": "8.57.0", diff --git a/esp/src/playwright.config.ts b/esp/src/playwright.config.ts new file mode 100644 index 00000000000..00031962989 --- /dev/null +++ b/esp/src/playwright.config.ts @@ -0,0 +1,46 @@ +import { defineConfig, devices } from "@playwright/test"; + +const baseURL = process.env.CI ? "https://play.hpccsystems.com:18010" : "http://127.0.0.1:8080"; + +/** + * See https://playwright.dev/docs/test-configuration. + */ +export default defineConfig({ + testDir: "./tests", + fullyParallel: true, + forbidOnly: !!process.env.CI, + retries: process.env.CI ? 2 : 0, + workers: process.env.CI ? 4 : undefined, + reporter: "html", + use: { + baseURL, + trace: "on-first-retry", + ignoreHTTPSErrors: true + }, + + projects: [ + { + name: "chromium", + use: { ...devices["Desktop Chrome"] }, + }, + + { + name: "firefox", + use: { ...devices["Desktop Firefox"] }, + }, + + { + name: "webkit", + use: { ...devices["Desktop Safari"] }, + }, + + ], + + /* Run your local dev server before starting the tests */ + webServer: { + command: "npm run start", + url: baseURL, + reuseExistingServer: true, + ignoreHTTPSErrors: true, + }, +}); diff --git a/esp/src/src-react/comms/workunit.ts b/esp/src/src-react/comms/workunit.ts new file mode 100644 index 00000000000..26debedffa9 --- /dev/null +++ b/esp/src/src-react/comms/workunit.ts @@ -0,0 +1,71 @@ +import { Workunit, WorkunitsService, type WsWorkunits } from "@hpcc-js/comms"; +import { Thenable } from "src/store/Deferred"; +import { Paged } from "src/store/Paged"; +import { BaseStore } from "src/store/Store"; +import { wuidToDateTime } from "src/Utility"; + +const service = new WorkunitsService({ baseUrl: "" }); + +export type WUQueryStore = BaseStore; + +export function CreateWUQueryStore(): BaseStore { + const store = new Paged({ + start: "PageStartFrom", + count: "PageSize", + sortBy: "Sortby", + descending: "Descending" + }, "Wuid", (request, abortSignal): Thenable<{ data: Workunit[], total: number }> => { + if (request.Sortby && request.Sortby === "TotalClusterTime") { + request.Sortby = "ClusterTime"; + } + return service.WUQuery(request, abortSignal).then(response => { + const page = { + start: undefined, + end: undefined + }; + const data = response.Workunits.ECLWorkunit.map((wu): Workunit => { + const start = wuidToDateTime(wu.Wuid); + if (!page.start || page.start > start) { + page.start = start; + } + let timePartsSection = 0; + const end = new Date(start); + const timeParts = wu.TotalClusterTime?.split(":") ?? []; + while (timeParts.length) { + const timePart = timeParts.pop(); + switch (timePartsSection) { + case 0: + end.setSeconds(end.getSeconds() + +timePart); + break; + case 1: + end.setMinutes(end.getMinutes() + +timePart); + break; + case 2: + end.setHours(end.getHours() + +timePart); + break; + case 3: + end.setDate(end.getDate() + +timePart); + break; + } + ++timePartsSection; + } + if (!page.end || page.end < end) { + page.end = end; + } + const retVal = Workunit.attach(service, wu.Wuid, wu); + // HPCC-33121 - Move to @hpcc-js/comms --- + retVal["__timeline_timings"] = { + start, + end, + page + }; + return retVal; + }); + return { + data, + total: response.NumWUs + }; + }); + }); + return store; +} diff --git a/esp/src/src-react/components/ECLPlayground.tsx b/esp/src/src-react/components/ECLPlayground.tsx index 9fe22b20ec7..ceca0b9eed9 100644 --- a/esp/src/src-react/components/ECLPlayground.tsx +++ b/esp/src/src-react/components/ECLPlayground.tsx @@ -1,13 +1,13 @@ import * as React from "react"; import { ReflexContainer, ReflexElement, ReflexSplitter } from "../layouts/react-reflex"; -import { IconButton, IIconProps, Link, Dropdown, IDropdownOption, TextField, useTheme } from "@fluentui/react"; +import { IconButton, IIconProps, Link, Dropdown, IDropdownOption, Spinner, SpinnerSize, TextField, useTheme } from "@fluentui/react"; import { Button } from "@fluentui/react-components"; import { CheckmarkCircleRegular, DismissCircleRegular, QuestionCircleRegular } from "@fluentui/react-icons"; import { scopedLogger } from "@hpcc-js/util"; import { useOnEvent } from "@fluentui/react-hooks"; import { mergeStyleSets } from "@fluentui/style-utilities"; import { ECLEditor, IPosition } from "@hpcc-js/codemirror"; -import { Workunit, WUUpdate, WorkunitsService } from "@hpcc-js/comms"; +import { Workunit, WUUpdate, WorkunitsService, WUStateID } from "@hpcc-js/comms"; import { HolyGrail } from "../layouts/HolyGrail"; import { DojoAdapter } from "../layouts/DojoAdapter"; import { pushUrl } from "../util/history"; @@ -238,7 +238,6 @@ const ECLEditorToolbar: React.FunctionComponent = ({ if (document.location.hash.includes("play")) { if (wu.isFailed()) { pushUrl(`/play/${wu.Wuid}`); - setWorkunit(wu); displayErrors(wu, editor); setOutputMode(OutputMode.ERRORS); } else if (wu.isComplete()) { @@ -247,7 +246,6 @@ const ECLEditorToolbar: React.FunctionComponent = ({ wu.publish(queryName); setWuState("Published"); } - setWorkunit(wu); setOutputMode(OutputMode.RESULTS); } } else { @@ -255,16 +253,17 @@ const ECLEditorToolbar: React.FunctionComponent = ({ logger.info(`${nlsHPCC.Playground} ${nlsHPCC.Finished} (${wu.Wuid})`); } } - }, [editor, queryName, setOutputMode, setWorkunit]); + }, [editor, queryName, setOutputMode]); const submitWU = React.useCallback(async () => { const wu = await Workunit.create({ baseUrl: "" }); + setWorkunit(wu); await wu.update({ Jobname: queryName, QueryText: editor.ecl() }); await wu.submit(cluster); wu.watchUntilComplete(changes => playgroundResults(wu)); - }, [cluster, editor, playgroundResults, queryName]); + }, [cluster, editor, playgroundResults, queryName, setWorkunit]); const publishWU = React.useCallback(async () => { if (queryName === "") { @@ -274,13 +273,14 @@ const ECLEditorToolbar: React.FunctionComponent = ({ setQueryNameErrorMsg(""); const wu = await Workunit.create({ baseUrl: "" }); + setWorkunit(wu); await wu.update({ Jobname: queryName, QueryText: editor.ecl() }); await wu.submit(cluster, WUUpdate.Action.Compile); wu.watchUntilComplete(changes => playgroundResults(wu, "publish")); } - }, [cluster, editor, playgroundResults, queryName, setQueryNameErrorMsg]); + }, [cluster, editor, playgroundResults, queryName, setQueryNameErrorMsg, setWorkunit]); const checkSyntax = React.useCallback(() => { service.WUSyntaxCheckECL({ @@ -470,6 +470,12 @@ export const ECLPlayground: React.FunctionComponent = (props }, [editor]); useOnEvent(document, "eclwatch-theme-toggle", handleThemeToggle); + const submissionComplete = React.useMemo(() => { + return workunit?.StateID === WUStateID.Completed || + workunit?.StateID === WUStateID.Failed || + (workunit?.ActionEx === "compile" && workunit?.StateID === WUStateID.Compiled); + }, [workunit?.StateID, workunit?.ActionEx]); + const handleEclChange = React.useMemo(() => debounce((evt) => { if (editor.hasFocus()) { setSyntaxStatusIcon(SyntaxCheckResult.Unknown); @@ -507,23 +513,33 @@ export const ECLPlayground: React.FunctionComponent = (props - + {submissionComplete ? + + :
+ +
+ } -
- {outputMode === OutputMode.ERRORS ? ( - - - ) : outputMode === OutputMode.RESULTS ? ( - - - ) : outputMode === OutputMode.VIS ? ( - - ) : null} -
+ {submissionComplete ? +
+ {outputMode === OutputMode.ERRORS ? ( + + + ) : outputMode === OutputMode.RESULTS ? ( + + + ) : outputMode === OutputMode.VIS ? ( + + ) : null} +
+ :
+ +
+ } ; diff --git a/esp/src/src-react/components/Workunits.tsx b/esp/src/src-react/components/Workunits.tsx index 5826e790dde..b53ba30dabc 100644 --- a/esp/src/src-react/components/Workunits.tsx +++ b/esp/src/src-react/components/Workunits.tsx @@ -1,8 +1,9 @@ import * as React from "react"; import { CommandBar, ContextualMenuItemType, DetailsRow, ICommandBarItemProps, IDetailsRowProps, Icon, Image, Link } from "@fluentui/react"; import { hsl as d3Hsl } from "@hpcc-js/common"; +import { Workunit } from "@hpcc-js/comms"; import { SizeMe } from "react-sizeme"; -import { CreateWUQueryStore, defaultSort, emptyFilter, Get, WUQueryStore, formatQuery } from "src/ESPWorkunit"; +import { defaultSort, emptyFilter, getStateImage, WUQueryStore, formatQuery } from "src/ESPWorkunit"; import * as WsWorkunits from "src/WsWorkunits"; import { formatCost } from "src/Session"; import { userKeyValStore } from "src/KeyValStore"; @@ -14,6 +15,7 @@ import { useLogicalClustersPalette } from "../hooks/platform"; import { calcSearch, pushParams } from "../util/history"; import { useHasFocus, useIsMounted } from "../hooks/util"; import { HolyGrail } from "../layouts/HolyGrail"; +import { CreateWUQueryStore } from "../comms/workunit"; import { FluentPagedGrid, FluentPagedFooter, useCopyButtons, useFluentStoreState, FluentColumns } from "./controls/Grid"; import { Fields } from "./forms/Fields"; import { Filter } from "./forms/Filter"; @@ -122,11 +124,10 @@ export const Workunits: React.FunctionComponent = ({ Wuid: { label: nlsHPCC.WUID, width: 120, sortable: true, - formatter: (Wuid, row) => { - const wu = Get(Wuid); + formatter: (Wuid: string, wu: Workunit) => { const search = calcSearch(filter); return <> - +   {Wuid} ; @@ -294,10 +295,10 @@ export const Workunits: React.FunctionComponent = ({ }, [selection]); const renderRowTimings = React.useCallback((props: IDetailsRowProps, size: { readonly width: number; readonly height: number; }) => { - if (showTimeline && props?.item?.timings) { - const total = props.item.timings.page.end - props.item.timings.page.start; - const startPct = 100 - (props.item.timings.start - props.item.timings.page.start) / total * 100; - const endPct = 100 - (props.item.timings.end - props.item.timings.page.start) / total * 100; + if (showTimeline && props?.item?.__timeline_timings) { + const total = props.item.__timeline_timings.page.end - props.item.__timeline_timings.page.start; + const startPct = 100 - (props.item.__timeline_timings.start - props.item.__timeline_timings.page.start) / total * 100; + const endPct = 100 - (props.item.__timeline_timings.end - props.item.__timeline_timings.page.start) / total * 100; const backgroundColor = palette(props.item.Cluster); const borderColor = d3Hsl(backgroundColor).darker().toString(); diff --git a/esp/src/tests/eclwatch-v5.spec.ts b/esp/src/tests/eclwatch-v5.spec.ts new file mode 100644 index 00000000000..01f1eccc486 --- /dev/null +++ b/esp/src/tests/eclwatch-v5.spec.ts @@ -0,0 +1,31 @@ +import { test, expect } from "@playwright/test"; + +test.describe("ECLWatch V5", () => { + test.beforeEach(async ({ page }) => { + await page.goto("/esp/files/index.html"); + await page.evaluate(() => { + sessionStorage.setItem("ECLWatch:ModernMode-9.0", "false"); + }); + }); + + test("Basic Frame", async ({ page }) => { + await page.goto("/esp/files/stub.htm"); + await expect(page.locator("#stubStackController_stub_Main span").first()).toBeVisible(); + await expect(page.getByLabel("Advanced")).toBeVisible(); + }); + + test("Activities", async ({ page }) => { + await page.goto("/esp/files/stub.htm"); + await expect(page.locator("#stub_Main-DLStackController_stub_Main-DL_Activity_label")).toBeVisible(); + await expect(page.getByLabel("Auto Refresh")).toBeVisible(); + await expect(page.getByLabel("Maximize/Restore")).toBeVisible(); + await expect(page.locator("i")).toBeVisible(); + await expect(page.locator("svg").filter({ hasText: "%hthor" })).toBeVisible(); + await expect(page.getByRole("img", { name: "Priority" })).toBeVisible(); + await expect(page.getByText("Target/Wuid")).toBeVisible(); + await expect(page.getByText("Graph")).toBeVisible(); + await expect(page.getByText("State")).toBeVisible(); + await expect(page.getByText("Owner")).toBeVisible(); + await expect(page.getByText("Job Name")).toBeVisible(); + }); +}); diff --git a/esp/src/tests/eclwatch-v9.spec.ts b/esp/src/tests/eclwatch-v9.spec.ts new file mode 100644 index 00000000000..745a274254d --- /dev/null +++ b/esp/src/tests/eclwatch-v9.spec.ts @@ -0,0 +1,42 @@ +import { test, expect } from "@playwright/test"; + +test.describe("ECLWatch V9", () => { + + test("Basic Frame", async ({ page }) => { + await page.goto("/esp/files/index.html#/activities"); + await expect(page.getByRole("link", { name: "ECL Watch" })).toBeVisible(); + await expect(page.locator("button").filter({ hasText: "" })).toBeVisible(); + await expect(page.getByRole("button", { name: "Advanced" })).toBeVisible(); + await expect(page.getByTitle("Activities")).toBeVisible(); + await expect(page.getByRole("link", { name: "ECL", exact: true })).toBeVisible(); + await expect(page.getByRole("link", { name: "Files" })).toBeVisible(); + await expect(page.getByRole("link", { name: "Published Queries" })).toBeVisible(); + await expect(page.getByRole("button", { name: "History" })).toBeVisible(); + await expect(page.getByRole("button", { name: "Add to favorites" })).toBeVisible(); + await expect(page.locator("a").filter({ hasText: /^Activities$/ })).toBeVisible(); + await expect(page.getByRole("link", { name: "Event Scheduler" })).toBeVisible(); + }); + + test("Activities", async ({ page }) => { + await page.goto("/esp/files/index.html#/activities"); + await page.getByTitle("Disk Usage").locator("i").click(); + await expect(page.locator("svg").filter({ hasText: "%hthor" })).toBeVisible(); + await expect(page.locator(".reflex-splitter")).toBeVisible(); + await expect(page.getByRole("menubar")).toBeVisible(); + await expect(page.getByRole("menuitem", { name: "Refresh" })).toBeVisible(); + await expect(page.locator("button").filter({ hasText: "" })).toBeVisible(); + await expect(page.locator("button").filter({ hasText: "" })).toBeVisible(); + await expect(page.getByRole("columnheader", { name: "Priority" }).locator("div").first()).toBeVisible(); + await expect(page.getByText("Target/Wuid")).toBeVisible(); + await expect(page.getByText("Graph")).toBeVisible(); + await expect(page.getByText("State")).toBeVisible(); + await expect(page.getByText("Owner")).toBeVisible(); + await expect(page.getByText("Job Name")).toBeVisible(); + await expect(page.getByRole("gridcell", { name: "HThorServer - hthor" })).toBeVisible(); + await expect(page.getByRole("gridcell", { name: "ThorMaster - thor", exact: true })).toBeVisible(); + await expect(page.getByRole("gridcell", { name: "ThorMaster - thor_roxie" })).toBeVisible(); + await expect(page.getByRole("gridcell", { name: "RoxieServer - roxie" })).toBeVisible(); + await expect(page.getByRole("gridcell", { name: "myeclccserver - hthor." })).toBeVisible(); + await expect(page.getByRole("gridcell", { name: "mydfuserver - dfuserver_queue" })).toBeVisible(); + }); +}); diff --git a/fs/dafsclient/rmtfile.cpp b/fs/dafsclient/rmtfile.cpp index 58bf77f0118..d7132bad273 100644 --- a/fs/dafsclient/rmtfile.cpp +++ b/fs/dafsclient/rmtfile.cpp @@ -2395,9 +2395,19 @@ class CRemoteFilteredFileIOBase : public CRemoteBase, implements IRemoteFileIO mrequest.append((RemoteFileCommandType)RFCStreamRead); VStringBuffer json("{ \"handle\" : %u, \"format\" : \"binary\" }", handle); mrequest.append(json.length(), json.str()); - sendRemoteCommand(mrequest, newReply); - unsigned newHandle; - newReply.read(newHandle); + unsigned newHandle = 0; + try + { + sendRemoteCommand(mrequest, newReply, false); + newReply.read(newHandle); + } + catch (IJSOCK_Exception *e) + { + // will trigger new request with cursor + EXCLOG(e, "CRemoteFilteredFileIOBase:: socket failure whilst streaming, will attempt to reconnect with cursor"); + newHandle = 0; + e->Release(); + } if (newHandle == handle) { reply.swapWith(newReply); diff --git a/system/jlib/jhash.cpp b/system/jlib/jhash.cpp index 28bce3d24df..da42a193ef2 100644 --- a/system/jlib/jhash.cpp +++ b/system/jlib/jhash.cpp @@ -305,7 +305,7 @@ bool HashTable::keyeq(const void *key1, const void *key2, int ksize) const unsigned HashTable::hash(const void *key, int ksize) const { - unsigned h = 0x811C9DC5; + unsigned h = fnvInitialHash32; unsigned char *bp = (unsigned char *) key; if (ksize<=0) { diff --git a/system/jlib/jptree.cpp b/system/jlib/jptree.cpp index 70d483495ad..d2a8197dfc1 100644 --- a/system/jlib/jptree.cpp +++ b/system/jlib/jptree.cpp @@ -3848,7 +3848,7 @@ unsigned CAtomPTree::queryHash() const { const char *_name = name.get(); size32_t nl = strlen(_name); - return isnocase() ? hashnc((const byte *) _name, nl, 0): hashc((const byte *) _name, nl, 0); + return isnocase() ? hashnc((const byte *) _name, nl, fnvInitialHash32): hashc((const byte *) _name, nl, fnvInitialHash32); } } diff --git a/system/jlib/jptree.ipp b/system/jlib/jptree.ipp index f4037783958..fd5cb63fd89 100644 --- a/system/jlib/jptree.ipp +++ b/system/jlib/jptree.ipp @@ -58,7 +58,7 @@ protected: virtual unsigned getHashFromElement(const void *e) const override; virtual unsigned getHashFromFindParam(const void *fp) const override { - return hashcz((const unsigned char *)fp, 0); + return hashcz((const unsigned char *)fp, fnvInitialHash32); } virtual bool matchesFindParam(const void *e, const void *fp, unsigned fphash) const override { @@ -108,7 +108,7 @@ public: // SuperHashTable definitions virtual unsigned getHashFromFindParam(const void *fp) const override { - return hashncz((const unsigned char *)fp, 0); + return hashncz((const unsigned char *)fp, fnvInitialHash32); } virtual bool matchesFindParam(const void *e, const void *fp, unsigned fphash) const override { @@ -844,7 +844,7 @@ public: const char *myname = queryName(); assert(myname); size32_t nl = strlen(myname); - return isnocase() ? hashnc((const byte *)myname, nl, 0): hashc((const byte *)myname, nl, 0); + return isnocase() ? hashnc((const byte *)myname, nl, fnvInitialHash32): hashc((const byte *)myname, nl, fnvInitialHash32); } virtual void setName(const char *_name) override; virtual void setAttribute(const char *attr, const char *val, bool encoded) override; diff --git a/system/jlib/jstats.cpp b/system/jlib/jstats.cpp index 0dca5030096..08f1e2498ed 100644 --- a/system/jlib/jstats.cpp +++ b/system/jlib/jstats.cpp @@ -1342,7 +1342,7 @@ void StatisticsMapping::createMappings() } const unsigned * kinds = indexToKind.getArray(); - hashcode = hashc((const byte *)kinds, indexToKind.ordinality() * sizeof(unsigned), 0x811C9DC5); + hashcode = hashc((const byte *)kinds, indexToKind.ordinality() * sizeof(unsigned), fnvInitialHash32); //All StatisticsMapping objects are assumed to be static, and never destroyed. const StatisticsMapping * existing = allStatsMappings[hashcode]; diff --git a/system/jlib/jsuperhash.cpp b/system/jlib/jsuperhash.cpp index a1e2b22d229..3e26af7b888 100644 --- a/system/jlib/jsuperhash.cpp +++ b/system/jlib/jsuperhash.cpp @@ -33,8 +33,8 @@ //#define MY_TRACE_HASH #ifdef MY_TRACE_HASH -int my_search_tot = 0; -int my_search_num = 0; +static unsigned my_search_tot = 0; +static unsigned my_search_num = 0; #endif //-- SuperHashTable --------------------------------------------------- @@ -104,13 +104,13 @@ void SuperHashTable::dumpStats() const { #ifdef TRACE_HASH if (tablecount && search_tot && search_num) - printf("Hash table %d entries, %d size, average search length %d(%d/%d) max %d\n", tablecount, tablesize, - (int) (search_tot/search_num), search_tot, search_num, search_max); + printf("Hash table %u entries, %u size, average search length %" I64F "u(%" I64F "u/%u) max %u\n", tablecount, tablesize, + search_tot/search_num, search_tot, search_num, search_max); #endif } #ifdef TRACE_HASH -void SuperHashTable::note_searchlen(int len) const +void SuperHashTable::note_searchlen(unsigned len) const { search_tot += len; search_num++; @@ -151,8 +151,8 @@ unsigned SuperHashTable::doFind(unsigned findHash, const void * findParam) const } #ifdef MY_TRACE_HASH my_search_num++; - if(my_search_num != 0) - printf("Hash table average search length %d\n", (int) (my_search_tot/my_search_num)); + if (my_search_num != 0) + printf("Hash table average search length %u\n", my_search_tot/my_search_num); #endif #ifdef TRACE_HASH note_searchlen(searchlen); @@ -193,8 +193,8 @@ unsigned SuperHashTable::doFindElement(unsigned v, const void * findET) const } #ifdef MY_TRACE_HASH my_search_num++; - if(my_search_num != 0) - printf("Hash table average search length %d\n", (int) (my_search_tot/my_search_num)); + if (my_search_num != 0) + printf("Hash table average search length %u\n", my_search_tot/my_search_num); #endif #ifdef TRACE_HASH note_searchlen(searchlen); @@ -233,8 +233,8 @@ unsigned SuperHashTable::doFindNew(unsigned v) const } #ifdef MY_TRACE_HASH my_search_num++; - if(my_search_num != 0) - printf("Hash table average search length %d\n", (int) (my_search_tot/my_search_num)); + if (my_search_num != 0) + printf("Hash table average search length %u\n", my_search_tot/my_search_num); #endif #ifdef TRACE_HASH note_searchlen(searchlen); diff --git a/system/jlib/jsuperhash.hpp b/system/jlib/jsuperhash.hpp index 2cd6216f412..2b004b0dcaa 100644 --- a/system/jlib/jsuperhash.hpp +++ b/system/jlib/jsuperhash.hpp @@ -28,6 +28,8 @@ #include "jstring.hpp" #include "jmutex.hpp" +constexpr unsigned fnvInitialHash32 = 0x811C9DC5; + extern jlib_decl unsigned hashc( const unsigned char *k, unsigned length, unsigned initval); extern jlib_decl unsigned hashnc( const unsigned char *k, unsigned length, unsigned initval); extern jlib_decl unsigned hashcz( const unsigned char *k, unsigned initval); @@ -88,7 +90,7 @@ class jlib_decl SuperHashTable : public CInterface void doKill(void); void expand(); void expand(unsigned newsize); - void note_searchlen(int) const; + void note_searchlen(unsigned) const; virtual void onAdd(void *et) = 0; virtual void onRemove(void *et) = 0; @@ -105,9 +107,9 @@ class jlib_decl SuperHashTable : public CInterface unsigned tablesize; unsigned tablecount; #ifdef TRACE_HASH - mutable int search_tot; - mutable int search_num; - mutable int search_max; + mutable unsigned __int64 search_tot; + mutable unsigned search_num; + mutable unsigned search_max; #endif }; @@ -514,9 +516,9 @@ class jlib_decl AtomRefTable : public SuperHashTableOfkeyPtr()), key, l+1); if (nocase) - hke->hashValue = hashnc((const unsigned char *)key, l, 0); + hke->hashValue = hashnc((const unsigned char *)key, l, fnvInitialHash32); else - hke->hashValue = hashc((const unsigned char *)key, l, 0); + hke->hashValue = hashc((const unsigned char *)key, l, fnvInitialHash32); hke->linkCount = 0; return hke; } @@ -609,9 +611,9 @@ class jlib_decl AtomRefTable : public SuperHashTableOfisCompressed(&blockCompressed); + + // NB: it would be far preferable to avoid this and have the file reference a group with the correct number of parts + // Possibly could use subgroup syntax: 'data[1..n]' for (unsigned clusterIdx=0; clusterIdxnumClusters(); clusterIdx++) { StringBuffer clusterName; @@ -336,6 +339,7 @@ void CWriteMasterBase::publish() p += queryJob().querySlaves(); IPartDescriptor *partDesc = fileDesc->queryPart(p); CDateTime createTime, modifiedTime; + offset_t compSize = 0; for (unsigned c=0; cnumCopies(); c++) { RemoteFilename rfn; @@ -347,7 +351,7 @@ void CWriteMasterBase::publish() ensureDirectoryForFile(path.str()); OwnedIFile iFile = createIFile(path.str()); Owned iFileIO; - if (compressed) + if (compressed) // NB: this would not be necessary if all builds have the changes in HPCC-32651 iFileIO.setown(createCompressedFileWriter(iFile, recordSize, false, true, NULL, compMethod)); else iFileIO.setown(iFile->open(IFOcreate)); @@ -355,7 +359,10 @@ void CWriteMasterBase::publish() iFileIO.clear(); // ensure copies have matching datestamps, as they would do normally (backupnode expects it) if (0 == c) + { iFile->getTime(&createTime, &modifiedTime, NULL); + compSize = iFile->size(); + } else iFile->setTime(&createTime, &modifiedTime, NULL); } @@ -376,7 +383,7 @@ void CWriteMasterBase::publish() props.setPropInt64("@recordCount", 0); props.setPropInt64("@size", 0); if (compressed) - props.setPropInt64("@compressedSize", 0); + props.setPropInt64("@compressedSize", compSize); p++; } }