HPCC-33120 Add daliadmin cleangenerateddlls utility command.

daliadmin <daliip> cleangenerateddlls [dryrun] [nobackup] The command will scan workunits and generated dll meta data, and remove all unrefenced generated dlls. Signed-off-by: Jake Smith <[email protected]>
hpcc-systems · Dec 12, 2024 · 028115a · 028115a
1 parent c120cbc
commit 028115a
Show file tree

Hide file tree

Showing 4 changed files with 89 additions and 0 deletions.
diff --git a/dali/daliadmin/daadmin.cpp b/dali/daliadmin/daadmin.cpp
@@ -16,6 +16,7 @@
 ############################################################################## */
 
 #include <unordered_map>
+#include <unordered_set>
 #include <string>
 
 #include "platform.h"
@@ -3424,4 +3425,75 @@ void cleanJobQueues(bool dryRun)
     }
 }
 
+void cleanGeneratedDlls(bool dryRun, bool backup)
+{
+    PROGLOG("Gathering workunits for referencd generated dlls");
+    CCycleTimer timer;
+    Owned<IRemoteConnection> conn = querySDS().connect("/", myProcessSession(), 0, SDS_LOCK_TIMEOUT);
+    if (!conn)
+    {
+        WARNLOG("Failed to connect to /WorkUnits");
+        return;
+    }
+    IPropertyTree *root = conn->queryRoot();
+    IPropertyTree *wuidsTree = root->queryPropTree("WorkUnits");
+    if (!wuidsTree)
+    {
+        PROGLOG("No WorkUnits found");
+        return;
+    }
+    Owned<IPropertyTreeIterator> wuidIter = wuidsTree->getElements("*");
+    std::unordered_set<std::string> referencedGDlls;
+    ForEach(*wuidIter)
+    {
+        IPropertyTree &wuid = wuidIter->query();
+        Owned<IPropertyTreeIterator> gdIter = wuid.getElements("Query/Associated/File[@type='dll']");
+        ForEach(*gdIter)
+        {
+            IPropertyTree &gd = gdIter->query();
+            const char *fullPath = gd.queryProp("@filename");
+            const char *filename = pathTail(fullPath);
+            referencedGDlls.emplace(filename);
+        }
+    }
+    PROGLOG("Found %u workunits that reference generated dlls, took: %u ms", (unsigned)referencedGDlls.size(), timer.elapsedMs());
+    PROGLOG("Scanning GeneratedDlls");
+    timer.reset();
+    IPropertyTree *generatedDllsTree = root->queryPropTree("GeneratedDlls");
+    if (!generatedDllsTree)
+    {
+        PROGLOG("No GeneratedDlls found");
+        return;
+    }
+    std::vector<IPropertyTree *> gDllsToRemove;
+    unsigned totalGDlls = 0;
+    Owned<IPropertyTreeIterator> gDlls = generatedDllsTree->getElements("*");
+    ForEach(*gDlls)
+    {
+        ++totalGDlls;
+        IPropertyTree &gDll = gDlls->query();
+        const char *name = gDll.queryProp("@name");
+        if (referencedGDlls.find(name) == referencedGDlls.end())
+            gDllsToRemove.push_back(&gDll);
+    }
+    PROGLOG("Found %u GeneratedDlls, %u not referenced, took: %u ms", totalGDlls, (unsigned)gDllsToRemove.size(), timer.elapsedMs());
+    if (!dryRun)
+    {
+        if (backup)
+        {
+            StringBuffer bakName;
+            Owned<IFileIO> iFileIO = createUniqueFile(NULL, "daliadmin_generateddlls", "bak", bakName);
+            if (!iFileIO)
+                throw makeStringException(0, "Failed to create backup file");
+            PROGLOG("Saving backup of GeneratedDlls to %s", bakName.str());
+            saveXML(*iFileIO, generatedDllsTree, 2);
+        }
+        PROGLOG("Deleting %u unreferenced GeneratedDlls", (unsigned)gDllsToRemove.size());
+        timer.reset();
+        for (auto &item: gDllsToRemove)
+            generatedDllsTree->removeTree(item);
+        PROGLOG("Removed %u unreferenced GeneratedDlls, took: %u ms", (unsigned)gDllsToRemove.size(), timer.elapsedMs());
+    }
+}
+
 } // namespace daadmin
diff --git a/dali/daliadmin/daadmin.hpp b/dali/daliadmin/daadmin.hpp
@@ -96,5 +96,6 @@ extern DALIADMIN_API void daliping(const char *dalis, unsigned connecttime, unsi
 extern DALIADMIN_API void validateStore(bool fix, bool deleteFiles, bool verbose);
 extern DALIADMIN_API void removeOrphanedGlobalVariables(bool dryrun, bool reconstruct);
 extern DALIADMIN_API void cleanJobQueues(bool dryRun);
+extern DALIADMIN_API void cleanGeneratedDlls(bool dryRun, bool backup);
 
 } // namespace daadmin
diff --git a/dali/daliadmin/daliadmin.cpp b/dali/daliadmin/daliadmin.cpp
@@ -94,6 +94,7 @@ void usage(const char *exe)
   printf("  auditlog <fromdate> <todate> <match>\n");
   printf("  cleanglobalwuid [dryrun] [noreconstruct]\n");
   printf("  cleanjobqueues [dryrun]\n");
+  printf("  cleangenerateddlls [dryrun] [nobackup]\n");
   printf("  clusterlist <mask>              -- list clusters   (mask optional)\n");
   printf("  coalesce                        -- force transaction coalesce\n");
   printf("  dalilocks [ <ip-pattern> ] [ files ] -- get all locked files/xpaths\n");
@@ -582,6 +583,20 @@ int main(int argc, const char* argv[])
                         bool dryRun = np>0 && strieq("dryrun", params.item(1));
                         cleanJobQueues(dryRun);
                     }
+                    else if (strieq(cmd, "cleangenerateddlls"))
+                    {
+                        bool dryRun = false;
+                        bool backup = true; // default
+                        for (unsigned i=1; i<params.ordinality(); i++)
+                        {
+                            const char *param = params.item(i);
+                            if (strieq("dryrun", param))
+                                dryRun = true;
+                            else if (strieq("nobackup", param))
+                                backup = false;
+                        }
+                        cleanGeneratedDlls(dryRun, backup);
+                    }
                     else if (strieq(cmd, "remotetest"))
                         remoteTest(params.item(1), true);
                     else

diff --git a/system/jlib/jsuperhash.cpp b/system/jlib/jsuperhash.cpp
@@ -86,6 +86,7 @@ void SuperHashTable::init(unsigned initsize)
 #ifdef TRACE_HASH
     search_tot = 0;
     search_num = 0;
+    search_max = 0;
 #endif
 }