Merge remote-tracking branch 'origin/candidate-9.4.x' into candidate-…

…9.6.x

Signed-off-by: Jake Smith <[email protected]>

.github/workflows/build-test-eclwatch.yml

@@ -25,7 +25,7 @@ jobs:
   build:
     strategy:
       matrix:
-        node: ["20", "18", "16"]
+        node: ["22", "20", "18"]
       fail-fast: false
     name: "Check eclwatch and npm"
     needs: pre_job
@@ -44,6 +44,12 @@ jobs:
       - name: Install Dependencies
         working-directory: ./esp/src
         run: npm ci
+      - name: Lint
+        working-directory: ./esp/src
+        run: npm run lint
+      - name: Install Playwright browsers 
+        working-directory: ./esp/src
+        run: npx playwright install --with-deps
       - name: Build
         working-directory: ./esp/src
         run: npm run build

dali/daliadmin/daadmin.cpp

@@ -2557,6 +2557,81 @@ void xmlSize(const char *filename, double pc)
     }
 }
 
+void loadXMLTest(const char *filename, bool parseOnly, bool useLowMemPTree, bool saveFormattedXML)
+{
+    OwnedIFile iFile = createIFile(filename);
+    OwnedIFileIO iFileIO = iFile->open(IFOread);
+    if (!iFileIO)
+    {
+        WARNLOG("File '%s' not found", filename);
+        return;
+    }
+
+    class CDummyPTreeMaker : public CSimpleInterfaceOf<IPTreeMaker>
+    {
+        StringBuffer xpath;
+        unsigned level = 0;
+    public:
+        virtual IPropertyTree *queryRoot() override { return nullptr; }
+        virtual IPropertyTree *queryCurrentNode() override { return nullptr; }
+        virtual void reset() override { }
+        virtual IPropertyTree *create(const char *tag) override { return nullptr; }
+        // IPTreeNotifyEvent impl.
+        virtual void beginNode(const char *tag, bool sequence, offset_t startOffset) override { }
+        virtual void newAttribute(const char *name, const char *value) override { }
+        virtual void beginNodeContent(const char *tag) override { }
+        virtual void endNode(const char *tag, unsigned length, const void *value, bool binary, offset_t endOffset) override { }
+    };
+
+    byte flags=ipt_none;
+    PTreeReaderOptions readFlags=ptr_ignoreWhiteSpace;
+    Owned<IPTreeMaker> iMaker;
+    if (!parseOnly)
+    {
+        PROGLOG("Creating property tree from file: %s", filename);
+        byte flags = ipt_none;
+        if (useLowMemPTree)
+        {
+            PROGLOG("Using low memory property trees");
+            flags = ipt_lowmem;
+        }
+        iMaker.setown(createPTreeMaker(flags));
+    }
+    else
+    {
+        PROGLOG("Reading property tree from file (without creating it): %s", filename);
+        iMaker.setown(new CDummyPTreeMaker());
+    }
+
+    offset_t fSize = iFileIO->size();
+    OwnedIFileIOStream stream = createIOStream(iFileIO);
+    OwnedIFileIOStream progressedIFileIOStream = createProgressIFileIOStream(stream, fSize, "Load progress", 1);
+    Owned<IPTreeReader> reader = createXMLStreamReader(*progressedIFileIOStream, *iMaker, readFlags);
+
+    ProcessInfo memInfo(ReadMemoryInfo);
+    __uint64 rss = memInfo.getActiveResidentMemory();
+    CCycleTimer timer;
+    reader->load();
+    memInfo.update(ReadMemoryInfo);
+    __uint64 rssUsed = memInfo.getActiveResidentMemory() - rss;
+    reader.clear();
+    progressedIFileIOStream.clear();
+    PROGLOG("Load took: %.2f - RSS consumed: %.2f MB", (float)timer.elapsedMs()/1000, (float)rssUsed/0x100000);
+
+    if (!parseOnly && saveFormattedXML)
+    {
+        assertex(iMaker->queryRoot());
+        StringBuffer outFilename(filename);
+        outFilename.append(".out.xml");
+        PROGLOG("Saving to %s", outFilename.str());
+        timer.reset();
+        saveXML(outFilename, iMaker->queryRoot(), 2);
+        PROGLOG("Save took: %.2f", (float)timer.elapsedMs()/1000);
+    }
+
+    ::LINK(iMaker->queryRoot()); // intentionally leak (avoid time clearing up)
+}
+
 void translateToXpath(const char *logicalfile, DfsXmlBranchKind tailType)
 {
     CDfsLogicalFileName lfn;

dali/daliadmin/daadmin.hpp

@@ -28,6 +28,7 @@ namespace daadmin
 
 extern DALIADMIN_API void setDaliConnectTimeoutMs(unsigned timeoutMs);
 extern DALIADMIN_API void xmlSize(const char *filename, double pc);
+extern DALIADMIN_API void loadXMLTest(const char *filename, bool parseOnly, bool useLowMemPTree, bool saveFormattedXML);
 extern DALIADMIN_API void translateToXpath(const char *logicalfile, DfsXmlBranchKind tailType = DXB_File);
 
 extern DALIADMIN_API void exportToFile(const char *path, const char *filename, bool safe = false);

dali/daliadmin/daliadmin.cpp

@@ -98,6 +98,9 @@ void usage(const char *exe)
   printf("  dalilocks [ <ip-pattern> ] [ files ] -- get all locked files/xpaths\n");
   printf("  daliping [ <num> ]              -- time dali server connect\n");
   printf("  getxref <destxmlfile>           -- get all XREF information\n");
+  printf("  loadxml <srcxmlfile> [--lowmem[=<true|false]]    -- use lowmem AtomPTree's\n"
+         "                       [--parseonly[=<true|false]] -- parse the xml file, don't load it into dali\n"
+         "                       [--savexml[=<true|false]]   -- save and time the parsed xml\n");
   printf("  migratefiles <src-group> <target-group> [<filemask>] [dryrun] [createmaps] [listonly] [verbose]\n");
   printf("  mpping <server-ip>              -- time MP connect\n");
   printf("  serverlist <mask>               -- list server IPs (mask optional)\n");
@@ -229,6 +232,18 @@ int main(int argc, const char* argv[])
                 }
                 else if (strieq(cmd, "remotetest"))
                     remoteTest(params.item(1), false);
+                else if (strieq(cmd, "loadxml"))
+                {
+                    bool useLowMemPTree = false;
+                    bool saveFormatedTree = false;
+                    bool parseOnly = getComponentConfigSP()->getPropBool("@parseonly");
+                    if (!parseOnly)
+                    {
+                        useLowMemPTree = getComponentConfigSP()->getPropBool("@lowmem");
+                        saveFormatedTree = getComponentConfigSP()->getPropBool("@savexml");
+                    }
+                    loadXMLTest(params.item(1), parseOnly, useLowMemPTree, saveFormatedTree);
+                }
                 else
                 {
                     UERRLOG("Unknown command %s",cmd);

esp/src/.gitignore

@@ -1,7 +1,11 @@
+blob-report/
 build/
 hpcc-js/
 lib/
 node_modules/
+playwright/.cache/
+playwright-report/
+test-results/
 types/
 .vscode/*
 !.vscode/tasks.json

esp/src/lws.config.js

@@ -1,6 +1,6 @@
 const fs = require("fs");
 
-let ip = "192.168.99.103";
+let ip = "https://play.hpccsystems.com:18010";
 if (fs.existsSync("./lws.target.txt")) {
     ip = fs.readFileSync("./lws.target.txt").toString().replace("\r\n", "\n").split("\n")[0];
 }
@@ -68,5 +68,5 @@ let rewrite = [
 module.exports = {
     port: 8080,
     rewrite: rewrite,
-    stack: ['lws-basic-auth', 'lws-request-monitor', 'lws-log', 'lws-cors', 'lws-json', 'lws-compress', 'lws-rewrite', 'lws-blacklist', 'lws-conditional-get', 'lws-mime', 'lws-range', 'lws-spa', 'lws-static', 'lws-index']
+    stack: ["lws-basic-auth", "lws-request-monitor", "lws-log", "lws-cors", "lws-json", "lws-compress", "lws-rewrite", "lws-blacklist", "lws-conditional-get", "lws-mime", "lws-range", "lws-spa", "lws-static", "lws-index"]
 };

esp/src/package.json

@@ -28,8 +28,11 @@
     "dev-start": "run-p bundle-watch dev-start-ws",
     "dev-start-verbose": "ws --verbose.include request response",
     "rm-hpcc": "rimraf ./node_modules/@hpcc-js",
-    "start": "webpack serve --env development --config webpack.config.js",
-    "test": "run-s lint",
+    "start": "ws",
+    "test": "npx playwright test",
+    "test-ci": "cross-env CI=1 npx playwright test",
+    "test-codegen": "npx playwright codegen",
+    "test-interactive": "npx playwright test --ui",
     "update": "npx npm-check-updates -u -t minor",
     "update-major": "npx npm-check-updates -u"
   },
@@ -83,13 +86,16 @@
     "xstyle": "0.3.3"
   },
   "devDependencies": {
+    "@playwright/test": "^1.49.0",
     "@simbathesailor/use-what-changed": "^2.0.0",
     "@types/dojo": "1.9.48",
+    "@types/node": "^22.10.1",
     "@types/react": "17.0.80",
     "@types/react-dom": "17.0.25",
     "@typescript-eslint/eslint-plugin": "6.21.0",
     "@typescript-eslint/parser": "6.21.0",
     "copyfiles": "2.4.1",
+    "cross-env": "^7.0.3",
     "css-loader": "6.10.0",
     "dojo-webpack-plugin": "3.0.6",
     "eslint": "8.57.0",