HPCC-30365 Add XREF Sasha service to K8s

dali/base/dadfs.cpp

@@ -273,7 +273,7 @@ RemoteFilename &constructPartFilename(IGroup *grp,unsigned partno,unsigned partm
     return rfn;
 }
 
-RemoteFilename &constructPartFilename(IGroup *grp,unsigned partNo,unsigned copy,unsigned max,unsigned lfnHash,int replicateOffset,bool dirPerPart,const char *lname,const char *prefix,const char *pmask,IStoragePlane *plane,RemoteFilename &rfn)
+RemoteFilename &constructPartFilename(IGroup *grp,unsigned partNo,unsigned copy,unsigned max,unsigned lfnHash,int replicateOffset,bool dirPerPart,const char *lname,const char *prefix,const char *pmask,unsigned numDevices,RemoteFilename &rfn)
 {
     partNo--;
     StringBuffer partName;
@@ -286,8 +286,8 @@ RemoteFilename &constructPartFilename(IGroup *grp,unsigned partNo,unsigned copy,
         }
         lname = expandMask(partName, pmask, partNo, max);
     }
-    // Get stripeNum from storage plane
-    unsigned stripeNum = calcStripeNumber(partNo+1, lfnHash, plane->numDevices());
+
+    unsigned stripeNum = calcStripeNumber(partNo+1, lfnHash, numDevices);
 
     StringBuffer fullname;
     makePhysicalPartName(lname, partNo+1, max, fullname, 0, DFD_OSdefault, prefix, dirPerPart, stripeNum);

dali/base/dadfs.hpp

@@ -811,7 +811,7 @@ enum DistributedFileSystemError
 
 
 // utility routines (used by xref and dfu)
-extern da_decl RemoteFilename &constructPartFilename(IGroup *grp,unsigned partNo,unsigned copy,unsigned max,unsigned lfnHash,int replicateOffset,bool dirPerPart,const char *lname,const char *prefix,const char *pmask,IStoragePlane *plane,RemoteFilename &rfn);
+extern da_decl RemoteFilename &constructPartFilename(IGroup *grp,unsigned partNo,unsigned copy,unsigned max,unsigned lfnHash,int replicateOffset,bool dirPerPart,const char *lname,const char *prefix,const char *pmask,unsigned numDevices,RemoteFilename &rfn);
 extern da_decl RemoteFilename &deprecatedConstructPartFilename(IGroup *grp,unsigned partno,unsigned partmax,const char *name,const char *partmask,const char *partdir,unsigned copy,ClusterPartDiskMapSpec &mspec,RemoteFilename &rfn);
 // legacy version
 inline RemoteFilename &deprecatedConstructPartFilename(IGroup *grp,unsigned partno,unsigned partmax,const char *name,const char *partmask,const char *partdir,bool replicate,int replicateoffset,RemoteFilename &rfn,bool localmount=false)

dali/base/dafdesc.cpp

@@ -3579,7 +3579,10 @@ void GroupInformation::createStoragePlane(IPropertyTree * storage, unsigned copy
     StringBuffer mirrorname;
     const char * planeName = name;
     if (copy != 0)
+    {
         planeName = mirrorname.append(name).append("_mirror");
+        plane->setPropBool("@copy", true);
+    }
 
     plane->setProp("@name", planeName);
 

dali/datest/datest.cmake

@@ -38,7 +38,6 @@ include_directories (
          ${HPCC_SOURCE_DIR}/fs/dafsstream
          ${HPCC_SOURCE_DIR}/rtl/include
          ${HPCC_SOURCE_DIR}/rtl/eclrtl
-         ${HPCC_SOURCE_DIR}/dali/dfuXRefLib
     )
 
 HPCC_ADD_EXECUTABLE ( datest ${SRCS} )
@@ -51,8 +50,7 @@ target_link_libraries ( datest
          dafsstream
          eclrtl
          wsdfuaccess 
-         dalibase
-         dfuXRefLib
+         dalibase 
          ${CppUnit_LIBRARIES}
     )
 

dali/datest/datest.cpp

@@ -37,8 +37,6 @@
 
 #include "rtlformat.hpp"
 
-#include "dfuxreflib.hpp"
-
 #include "jptree.hpp"
 #include "wsdfuaccess.hpp"
 
@@ -80,10 +78,9 @@ static void addTestFile(const char *name,unsigned n)
     StringBuffer partmask;
     getPartMask(partmask,name,n);
     StringBuffer path;
-    IStoragePlane *plane = getDataStoragePlane("mystorageplane", true);
     for (unsigned m=0; m<n; m++) {
         RemoteFilename rfn;
-        constructPartFilename(group,m+1,1,n,0,0,false,"",dir.str(),partmask.str(),plane,rfn);
+        constructPartFilename(group,m+1,1,n,0,0,false,"",dir.str(),partmask.str(),0,rfn);
         rfn.getLocalPath(path.clear());
         Owned<IPropertyTree> pp = createPTree("Part");
         pp->setPropInt64("@size",1234*(m+1));
@@ -3320,108 +3317,17 @@ void testlockprop(const char *lfn)
     printf("done\n");
 }
 
-void TestParseFileName()
-{
-    StringBuffer mname;
-    unsigned num;
-    unsigned max;
-    bool replicate;
-
-    // Standard file name with multiple parts
-    assertex(testParseFileName("/var/lib/HPCCSystems/hpcc-data/test/myname._1_of_3", mname.clear(), num, max, replicate));
-    assertex(strcmp(mname.str(), "/var/lib/HPCCSystems/hpcc-data/test/myname._$P$_of_3")==0);
-    assertex(num==1);
-    assertex(max==3);
-    assertex(replicate==true);
-
-    // Standard file name with multiple parts is striped across directories, storage plane has numDevices set for it
-    assertex(testParseFileName("/var/lib/HPCCSystems/hpcc-data-two/d1/test/myname._10_of_30", mname.clear(), num, max, replicate));
-    assertex(strcmp(mname.str(), "/var/lib/HPCCSystems/hpcc-data-two/d$P$/test/myname._$P$_of_30")==0);
-    assertex(num==10);
-    assertex(max==30);
-    assertex(replicate==true);
-
-    // Standard file name with multiple parts is striped across directories and each part has its own directory, storage plane has numDevices set for it
-    assertex(testParseFileName("/var/lib/HPCCSystems/hpcc-data-two/d1/test/42/myname._42_of_100", mname.clear(), num, max, replicate));
-    assertex(strcmp(mname.str(), "/var/lib/HPCCSystems/hpcc-data-two/d$P$/test/$P$/myname._$P$_of_100")==0);
-    assertex(num==42);
-    assertex(max==100);
-    assertex(replicate==true);
-
-    // Test a longer part number
-    assertex(testParseFileName("/var/lib/HPCCSystems/hpcc-data-two/d110/test/12345/myname._12345_of_100000", mname.clear(), num, max, replicate));
-    assertex(strcmp(mname.str(), "/var/lib/HPCCSystems/hpcc-data-two/d$P$/test/$P$/myname._$P$_of_100000")==0);
-    assertex(num==12345);
-    assertex(max==100000);
-    assertex(replicate==true);
-
-    // A file without a storage plane throws an exception
-    try {
-        testParseFileName("/test/myname._1_of_3", mname.clear(), num, max, replicate);
-        assertex(false);
-    } catch (IException *e) {
-        StringBuffer msg;
-        e->errorMessage(msg);
-        e->Release();
-        assertex(strcmp(msg.str(), "Could not find matching prefix in plane definition for file /test/myname._1_of_3")==0);
-    }
-
-    // A file with a storage plane that has numDevices>1 but no stripe number in the path throws an exception
-    try {
-        testParseFileName("/var/lib/HPCCSystems/hpcc-data-two/test/myname.42_of_100", mname.clear(), num, max, replicate);
-        assertex(false);
-    } catch (IException *e) {
-        StringBuffer msg;
-        e->errorMessage(msg);
-        e->Release();
-        assertex(strcmp(msg.str(), "In storage plane definition numDevices>1, but no stripe sub-directory found in file /var/lib/HPCCSystems/hpcc-data-two/test/myname.42_of_100")==0);
-    }
-
-    // A file where the dir-per-part number does not match the part number
-    try {
-        testParseFileName("/var/lib/HPCCSystems/hpcc-data-two/d1/test/42/myname._43_of_100", mname.clear(), num, max, replicate);
-        assertex(false);
-    } catch (IException *e) {
-        StringBuffer msg;
-        e->errorMessage(msg);
-        e->Release();
-        assertex(strcmp(msg.str(), "Dir-per-part # does not match part # of file /var/lib/HPCCSystems/hpcc-data-two/d1/test/42/myname._43_of_100")==0);
-    }
-
-    printf("All parseFileName tests passed\n");
-}
-
 void usage(const char *error=NULL)
 {
     if (error) printf("%s\n", error);
     printf("usage: DATEST <server_ip:port>* [/test <name> [<test params...>] [/NITER <iterations>]\n");
-    printf("where name = RANDTEST | DFS | QTEST | QTEST2 | SESSION | LOCKS | SDS1 | SDS2 | XPATHS| STRESS | STRESS2 | SHUTDOWN | EXTERNAL | SUBLOCKS | SUBSCRIPTION | CONNECTIONSUBS | MULTIFILE | NODESUBS | DFUSTREAMREAD | DFUSTREAMWRITE | DFUSTREAMCOPY | PARSEFILENAME\n");
+    printf("where name = RANDTEST | DFS | QTEST | QTEST2 | SESSION | LOCKS | SDS1 | SDS2 | XPATHS| STRESS | STRESS2 | SHUTDOWN | EXTERNAL | SUBLOCKS | SUBSCRIPTION | CONNECTIONSUBS | MULTIFILE | NODESUBS | DFUSTREAMREAD | DFUSTREAMWRITE | DFUSTREAMCOPY\n");
     printf("eg:  datest . /test QTEST put          -- one coven server running locally, running qtest with param \"put\"\n");
     printf("     datest eq0001016 eq0001017        -- two coven servers, use default test %s\n", DEFAULT_TEST);
 }
 
 struct ReleaseAtomBlock { ~ReleaseAtomBlock() { releaseAtoms(); } };
 
-static constexpr const char * defaultYaml = R"!!(
-version: "1.0"
-datest:
-  name: datest
-global:
-  storage:
-    planes:
-    - name: mystorageplane
-      storageClass: ""
-      storageSize: 1Gi
-      prefix: "/var/lib/HPCCSystems/hpcc-data"
-      category: data
-    - name: mystripedplane
-      storageClass: ""
-      storageSize: 1Gi
-      prefix: "/var/lib/HPCCSystems/hpcc-data-two"
-      numDevices: 2
-      category: data
-)!!";
-
 int main(int argc, char* argv[])
 {   
     ReleaseAtomBlock rABlock;
@@ -3430,10 +3336,6 @@ int main(int argc, char* argv[])
     EnableSEHtoExceptionMapping();
 
     try {
-        //NB: required initialization for anything that may call getGlobalConfig*() or getComponentConfig*()
-        Owned<IPropertyTree> globals = loadConfiguration(defaultYaml, (const char **)argv, "datest", nullptr, nullptr, nullptr, nullptr, false);
-        initializeStorageGroups(true);
-
         StringBuffer cmd;
         splitFilename(argv[0], NULL, NULL, &cmd, NULL);
         StringBuffer lf;
@@ -3573,7 +3475,6 @@ int main(int argc, char* argv[])
                 case 10: TestSubLocks(); break;
                 case 11: TestSDS3(group); break;
                 case 12: TestNodeSubs(); break;
-                case 13: TestParseFileName(); break;
                 }
             }
             else if (TEST("DFS"))
@@ -3628,8 +3529,6 @@ int main(int argc, char* argv[])
                 testDfuStreamWrite(testParams.ordinality() ? testParams.item(0) : nullptr);
             else if (TEST("DFUSTREAMCOPY"))
                 testDfuStreamCopy(testParams.ordinality() ? testParams.item(0) : nullptr);
-            else if (TEST("PARSEFILENAME"))
-                TestParseFileName();
 //          else if (TEST("DALILOG"))
 //              testDaliLog(testParams.ordinality()&&0!=atoi(testParams.item(0)));
             else

dali/dfuXRefLib/CMakeLists.txt

@@ -46,6 +46,7 @@ include_directories (
          ./../../common/environment 
          ./../../common/workunit
          ./../../system/security/shared
+         ${HPCC_SOURCE_DIR}/testing/unittests
     )
 
 ADD_DEFINITIONS( -D_USRDLL -DDFUXREFLIB_EXPORTS )
@@ -57,7 +58,8 @@ target_link_libraries ( dfuXRefLib
          mp 
          hrpc 
          dafsclient 
-         dalibase 
+         dalibase
+         ${CppUnit_LIBRARIES}
     )
 
 if (NOT CONTAINERIZED)