Merge remote-tracking branch 'origin/candidate-9.8.x'

Signed-off-by: Jake Smith <[email protected]>

.github/workflows/build-test-eclwatch.yml

@@ -25,7 +25,7 @@ jobs:
   build:
     strategy:
       matrix:
-        node: ["20", "18", "16"]
+        node: ["22", "20", "18"]
       fail-fast: false
     name: "Check eclwatch and npm"
     needs: pre_job
@@ -44,6 +44,12 @@ jobs:
       - name: Install Dependencies
         working-directory: ./esp/src
         run: npm ci
+      - name: Lint
+        working-directory: ./esp/src
+        run: npm run lint
+      - name: Install Playwright browsers 
+        working-directory: ./esp/src
+        run: npx playwright install --with-deps
       - name: Build
         working-directory: ./esp/src
         run: npm run build

dali/base/dadfs.cpp

@@ -13849,6 +13849,45 @@ void configurePreferredPlanes()
     }
 }
 
+static bool doesPhysicalMatchMeta(IPropertyTree &partProps, IFile &iFile, offset_t expectedSize, offset_t &actualSize)
+{
+    // NB: temporary workaround for 'narrow' files publishing extra empty parts with the wrong @compressedSize(0)
+    // causing a new check introduced in HPCC-33064 to be hit (fixed in HPCC-33113, but will continue to affect exiting files)
+    unsigned __int64 size = partProps.getPropInt64("@size", unknownFileSize);
+    unsigned __int64 compressedSize = partProps.getPropInt64("@compressedSize", unknownFileSize);
+    if ((0 == size) && (0 == compressedSize))
+    {
+        actualSize = unknownFileSize;
+        return true;
+    }
+
+    if (expectedSize != unknownFileSize)
+    {
+        actualSize = iFile.size();
+        if (actualSize != expectedSize)
+            return false;
+    }
+    else
+        actualSize = unknownFileSize;
+
+    return true;
+}
+
+bool doesPhysicalMatchMeta(IPartDescriptor &partDesc, IFile &iFile, offset_t &expectedSize, offset_t &actualSize)
+{
+    IPropertyTree &partProps = partDesc.queryProperties();
+    expectedSize = partDesc.getDiskSize(false, false);
+    return doesPhysicalMatchMeta(partProps, iFile, expectedSize, actualSize);
+}
+
+bool doesPhysicalMatchMeta(IDistributedFilePart &part, IFile &iFile, offset_t &expectedSize, offset_t &actualSize)
+{
+    IPropertyTree &partProps = part.queryAttributes();
+    expectedSize = part.getDiskSize(false, false);
+    return doesPhysicalMatchMeta(partProps, iFile, expectedSize, actualSize);
+}
+
+
 #ifdef _USE_CPPUNIT
 /*
  * This method removes files only logically. removeEntry() used to do that, but the only

dali/base/dadfs.hpp

@@ -925,4 +925,8 @@ inline cost_type getLegacyWriteCost(const IPropertyTree & fileAttr, Source sourc
     else
         return 0;
 }
+
+extern da_decl bool doesPhysicalMatchMeta(IPartDescriptor &partDesc, IFile &iFile, offset_t &expectedSize, offset_t &actualSize);
+extern da_decl bool doesPhysicalMatchMeta(IDistributedFilePart &partDesc, IFile &iFile, offset_t &expectedSize, offset_t &actualSize);
+
 #endif

dali/daliadmin/daadmin.cpp

@@ -2557,6 +2557,81 @@ void xmlSize(const char *filename, double pc)
     }
 }
 
+void loadXMLTest(const char *filename, bool parseOnly, bool useLowMemPTree, bool saveFormattedXML)
+{
+    OwnedIFile iFile = createIFile(filename);
+    OwnedIFileIO iFileIO = iFile->open(IFOread);
+    if (!iFileIO)
+    {
+        WARNLOG("File '%s' not found", filename);
+        return;
+    }
+
+    class CDummyPTreeMaker : public CSimpleInterfaceOf<IPTreeMaker>
+    {
+        StringBuffer xpath;
+        unsigned level = 0;
+    public:
+        virtual IPropertyTree *queryRoot() override { return nullptr; }
+        virtual IPropertyTree *queryCurrentNode() override { return nullptr; }
+        virtual void reset() override { }
+        virtual IPropertyTree *create(const char *tag) override { return nullptr; }
+        // IPTreeNotifyEvent impl.
+        virtual void beginNode(const char *tag, bool sequence, offset_t startOffset) override { }
+        virtual void newAttribute(const char *name, const char *value) override { }
+        virtual void beginNodeContent(const char *tag) override { }
+        virtual void endNode(const char *tag, unsigned length, const void *value, bool binary, offset_t endOffset) override { }
+    };
+
+    byte flags=ipt_none;
+    PTreeReaderOptions readFlags=ptr_ignoreWhiteSpace;
+    Owned<IPTreeMaker> iMaker;
+    if (!parseOnly)
+    {
+        PROGLOG("Creating property tree from file: %s", filename);
+        byte flags = ipt_none;
+        if (useLowMemPTree)
+        {
+            PROGLOG("Using low memory property trees");
+            flags = ipt_lowmem;
+        }
+        iMaker.setown(createPTreeMaker(flags));
+    }
+    else
+    {
+        PROGLOG("Reading property tree from file (without creating it): %s", filename);
+        iMaker.setown(new CDummyPTreeMaker());
+    }
+
+    offset_t fSize = iFileIO->size();
+    OwnedIFileIOStream stream = createIOStream(iFileIO);
+    OwnedIFileIOStream progressedIFileIOStream = createProgressIFileIOStream(stream, fSize, "Load progress", 1);
+    Owned<IPTreeReader> reader = createXMLStreamReader(*progressedIFileIOStream, *iMaker, readFlags);
+
+    ProcessInfo memInfo(ReadMemoryInfo);
+    __uint64 rss = memInfo.getActiveResidentMemory();
+    CCycleTimer timer;
+    reader->load();
+    memInfo.update(ReadMemoryInfo);
+    __uint64 rssUsed = memInfo.getActiveResidentMemory() - rss;
+    reader.clear();
+    progressedIFileIOStream.clear();
+    PROGLOG("Load took: %.2f - RSS consumed: %.2f MB", (float)timer.elapsedMs()/1000, (float)rssUsed/0x100000);
+
+    if (!parseOnly && saveFormattedXML)
+    {
+        assertex(iMaker->queryRoot());
+        StringBuffer outFilename(filename);
+        outFilename.append(".out.xml");
+        PROGLOG("Saving to %s", outFilename.str());
+        timer.reset();
+        saveXML(outFilename, iMaker->queryRoot(), 2);
+        PROGLOG("Save took: %.2f", (float)timer.elapsedMs()/1000);
+    }
+
+    ::LINK(iMaker->queryRoot()); // intentionally leak (avoid time clearing up)
+}
+
 void translateToXpath(const char *logicalfile, DfsXmlBranchKind tailType)
 {
     CDfsLogicalFileName lfn;

dali/daliadmin/daadmin.hpp

@@ -28,6 +28,7 @@ namespace daadmin
 
 extern DALIADMIN_API void setDaliConnectTimeoutMs(unsigned timeoutMs);
 extern DALIADMIN_API void xmlSize(const char *filename, double pc);
+extern DALIADMIN_API void loadXMLTest(const char *filename, bool parseOnly, bool useLowMemPTree, bool saveFormattedXML);
 extern DALIADMIN_API void translateToXpath(const char *logicalfile, DfsXmlBranchKind tailType = DXB_File);
 
 extern DALIADMIN_API void exportToFile(const char *path, const char *filename, bool safe = false);

dali/daliadmin/daliadmin.cpp

@@ -99,6 +99,9 @@ void usage(const char *exe)
   printf("  dalilocks [ <ip-pattern> ] [ files ] -- get all locked files/xpaths\n");
   printf("  daliping [ <num> ]              -- time dali server connect\n");
   printf("  getxref <destxmlfile>           -- get all XREF information\n");
+  printf("  loadxml <srcxmlfile> [--lowmem[=<true|false]]    -- use lowmem AtomPTree's\n"
+         "                       [--parseonly[=<true|false]] -- parse the xml file, don't load it into dali\n"
+         "                       [--savexml[=<true|false]]   -- save and time the parsed xml\n");
   printf("  migratefiles <src-group> <target-group> [<filemask>] [dryrun] [createmaps] [listonly] [verbose]\n");
   printf("  mpping <server-ip>              -- time MP connect\n");
   printf("  serverlist <mask>               -- list server IPs (mask optional)\n");
@@ -230,6 +233,18 @@ int main(int argc, const char* argv[])
                 }
                 else if (strieq(cmd, "remotetest"))
                     remoteTest(params.item(1), false);
+                else if (strieq(cmd, "loadxml"))
+                {
+                    bool useLowMemPTree = false;
+                    bool saveFormatedTree = false;
+                    bool parseOnly = getComponentConfigSP()->getPropBool("@parseonly");
+                    if (!parseOnly)
+                    {
+                        useLowMemPTree = getComponentConfigSP()->getPropBool("@lowmem");
+                        saveFormatedTree = getComponentConfigSP()->getPropBool("@savexml");
+                    }
+                    loadXMLTest(params.item(1), parseOnly, useLowMemPTree, saveFormatedTree);
+                }
                 else
                 {
                     UERRLOG("Unknown command %s",cmd);

docs/EN_US/ContainerizedHPCC/ContainerizedHPCCSystemsPlatform.xml

@@ -227,4 +227,6 @@
 
   <xi:include href="ContainerizedHPCC/ContainerizedMods/ContainerLogging.xml"
               xmlns:xi="http://www.w3.org/2001/XInclude" />
+  <xi:include href="ContainerizedHPCC/ContainerizedMods/TroubleshootingHelmDeployments.xml"
+              xmlns:xi="http://www.w3.org/2001/XInclude" />			  
 </book>