HPCC-30365 Add XREF Sasha service to K8s

dali/base/dadfs.cpp

@@ -245,9 +245,9 @@ extern da_decl cost_type calcDiskWriteCost(const StringArray & clusters, stat_ty
     return writeCost;
 }
 
-// JCSMORE - I suspect this function should be removed/deprecated. It does not deal with dirPerPart or striping.
-// makePhysicalPartName supports both, but does not deal with groups/endpoints)
-RemoteFilename &constructPartFilename(IGroup *grp,unsigned partno,unsigned partmax,const char *name,const char *partmask,const char *partdir,unsigned copy,ClusterPartDiskMapSpec &mspec,RemoteFilename &rfn)
+
+// Deprecated and should be removed and new feature tested
+RemoteFilename &deprecatedConstructPartFilename(IGroup *grp,unsigned partno,unsigned partmax,const char *name,const char *partmask,const char *partdir,unsigned copy,ClusterPartDiskMapSpec &mspec,RemoteFilename &rfn)
 {
     partno--;
     StringBuffer tmp;
@@ -273,6 +273,45 @@ RemoteFilename &constructPartFilename(IGroup *grp,unsigned partno,unsigned partm
     return rfn;
 }
 
+RemoteFilename &constructPartFilename(IGroup *grp,unsigned partNo,unsigned copy,unsigned max,unsigned lfnHash,int replicateOffset,bool dirPerPart,const char *lname,const char *prefix,const char *pmask,unsigned numDevices,RemoteFilename &rfn)
+{
+    partNo--;
+    StringBuffer partName;
+    if (!lname||!*lname)
+    {
+        if (!pmask)
+        {
+            pmask = "!ERROR!._$P$_of_$N$";
+            IERRLOG("No partmask for constructPartFilename");
+        }
+        lname = expandMask(partName, pmask, partNo, max);
+    }
+
+    unsigned stripeNum = calcStripeNumber(partNo+1, lfnHash, numDevices);
+
+    StringBuffer fullname;
+    makePhysicalPartName(lname, partNo+1, max, fullname, 0, DFD_OSdefault, prefix, dirPerPart, stripeNum);
+
+    // revisit: constructPartFilename should be refactored not to deal with replicate directories, by pre-determining the alternate prefix if copy>0
+    // If copy>0 it could do calPartLocation, find the replicate plane, get it's prefix, and pass to makePhysicalPartName
+    unsigned n = 0;
+    if (!isContainerized())
+    {
+        ClusterPartDiskMapSpec mspec;
+        mspec.replicateOffset = replicateOffset;
+        unsigned d;
+        mspec.calcPartLocation(partNo, max, copy, grp?grp->ordinality():max, n, d);
+        setReplicateFilename(fullname, d);
+    }
+
+    SocketEndpoint ep;
+    if (grp)
+        ep = grp->queryNode(n).endpoint();
+    rfn.setPath(ep, fullname.toLowerCase().str());
+
+    return rfn;
+}
+
 inline void LOGPTREE(const char *title,IPropertyTree *pt)
 {
     StringBuffer buf;

dali/base/dadfs.hpp

@@ -811,14 +811,15 @@ enum DistributedFileSystemError
 
 
 // utility routines (used by xref and dfu)
-extern da_decl RemoteFilename &constructPartFilename(IGroup *grp,unsigned partno,unsigned partmax,const char *name,const char *partmask,const char *partdir,unsigned copy,ClusterPartDiskMapSpec &mspec,RemoteFilename &rfn);
+extern da_decl RemoteFilename &constructPartFilename(IGroup *grp,unsigned partNo,unsigned copy,unsigned max,unsigned lfnHash,int replicateOffset,bool dirPerPart,const char *lname,const char *prefix,const char *pmask,unsigned numDevices,RemoteFilename &rfn);
+extern da_decl RemoteFilename &deprecatedConstructPartFilename(IGroup *grp,unsigned partno,unsigned partmax,const char *name,const char *partmask,const char *partdir,unsigned copy,ClusterPartDiskMapSpec &mspec,RemoteFilename &rfn);
 // legacy version
-inline RemoteFilename &constructPartFilename(IGroup *grp,unsigned partno,unsigned partmax,const char *name,const char *partmask,const char *partdir,bool replicate,int replicateoffset,RemoteFilename &rfn,bool localmount=false)
+inline RemoteFilename &deprecatedConstructPartFilename(IGroup *grp,unsigned partno,unsigned partmax,const char *name,const char *partmask,const char *partdir,bool replicate,int replicateoffset,RemoteFilename &rfn,bool localmount=false)
 {
     // local mount ignored!
     ClusterPartDiskMapSpec mspec;
     mspec.replicateOffset = replicateoffset;
-    return constructPartFilename(grp,partno,partmax,name,partmask,partdir,replicate?1:0,mspec,rfn);
+    return deprecatedConstructPartFilename(grp,partno,partmax,name,partmask,partdir,replicate?1:0,mspec,rfn);
 }
 
 extern da_decl IDFPartFilter *createPartFilter(const char *filter);

dali/base/dafdesc.cpp

@@ -3576,10 +3576,19 @@ void GroupInformation::createStoragePlane(IPropertyTree * storage, unsigned copy
         return;
 
     IPropertyTree * plane = storage->addPropTree("planes");
+
+    // Revisit: Ignore hthor planes when running XRef on storage planes
+    if (startsWith(name, "hthor"))
+        plane->setPropBool("@hthorplane", true);
+
     StringBuffer mirrorname;
     const char * planeName = name;
     if (copy != 0)
+    {
         planeName = mirrorname.append(name).append("_mirror");
+        // Set copy to true to be able to avoid running XRef on mirror planes
+        plane->setPropBool("@copy", true);
+    }
 
     plane->setProp("@name", planeName);
 

dali/datest/datest.cpp

@@ -80,7 +80,7 @@ static void addTestFile(const char *name,unsigned n)
     StringBuffer path;
     for (unsigned m=0; m<n; m++) {
         RemoteFilename rfn;
-        constructPartFilename(group,m+1,n,NULL,partmask.str(),dir.str(),false,1,rfn);
+        constructPartFilename(group,m+1,1,n,0,0,false,"",dir.str(),partmask.str(),0,rfn);
         rfn.getLocalPath(path.clear());
         Owned<IPropertyTree> pp = createPTree("Part");
         pp->setPropInt64("@size",1234*(m+1));
@@ -3328,7 +3328,6 @@ void usage(const char *error=NULL)
 
 struct ReleaseAtomBlock { ~ReleaseAtomBlock() { releaseAtoms(); } };
 
-
 int main(int argc, char* argv[])
 {   
     ReleaseAtomBlock rABlock;

dali/dfuXRefLib/CMakeLists.txt

@@ -46,6 +46,7 @@ include_directories (
          ./../../common/environment 
          ./../../common/workunit
          ./../../system/security/shared
+         ${HPCC_SOURCE_DIR}/testing/unittests
     )
 
 ADD_DEFINITIONS( -D_USRDLL -DDFUXREFLIB_EXPORTS )
@@ -57,7 +58,8 @@ target_link_libraries ( dfuXRefLib
          mp 
          hrpc 
          dafsclient 
-         dalibase 
+         dalibase
+         ${CppUnit_LIBRARIES}
     )
 
 if (NOT CONTAINERIZED)