diff --git a/dali/base/dadfs.cpp b/dali/base/dadfs.cpp index 0160d4a7371..94d3cd6e90a 100644 --- a/dali/base/dadfs.cpp +++ b/dali/base/dadfs.cpp @@ -245,9 +245,9 @@ extern da_decl cost_type calcDiskWriteCost(const StringArray & clusters, stat_ty return writeCost; } -// JCSMORE - I suspect this function should be removed/deprecated. It does not deal with dirPerPart or striping. -// makePhysicalPartName supports both, but does not deal with groups/endpoints) -RemoteFilename &constructPartFilename(IGroup *grp,unsigned partno,unsigned partmax,const char *name,const char *partmask,const char *partdir,unsigned copy,ClusterPartDiskMapSpec &mspec,RemoteFilename &rfn) + +// Deprecated and should be removed and new feature tested +RemoteFilename &deprecatedConstructPartFilename(IGroup *grp,unsigned partno,unsigned partmax,const char *name,const char *partmask,const char *partdir,unsigned copy,ClusterPartDiskMapSpec &mspec,RemoteFilename &rfn) { partno--; StringBuffer tmp; @@ -273,6 +273,45 @@ RemoteFilename &constructPartFilename(IGroup *grp,unsigned partno,unsigned partm return rfn; } +RemoteFilename &constructPartFilename(IGroup *grp,unsigned partNo,unsigned copy,unsigned max,unsigned lfnHash,int replicateOffset,bool dirPerPart,const char *lname,const char *prefix,const char *pmask,unsigned numDevices,RemoteFilename &rfn) +{ + partNo--; + StringBuffer partName; + if (!lname||!*lname) + { + if (!pmask) + { + pmask = "!ERROR!._$P$_of_$N$"; + IERRLOG("No partmask for constructPartFilename"); + } + lname = expandMask(partName, pmask, partNo, max); + } + + unsigned stripeNum = calcStripeNumber(partNo+1, lfnHash, numDevices); + + StringBuffer fullname; + makePhysicalPartName(lname, partNo+1, max, fullname, 0, DFD_OSdefault, prefix, dirPerPart, stripeNum); + + // revisit: constructPartFilename should be refactored not to deal with replicate directories, by pre-determining the alternate prefix if copy>0 + // If copy>0 it could do calPartLocation, find the replicate plane, get it's prefix, and pass to makePhysicalPartName + unsigned n = 0; + if (!isContainerized()) + { + ClusterPartDiskMapSpec mspec; + mspec.replicateOffset = replicateOffset; + unsigned d; + mspec.calcPartLocation(partNo, max, copy, grp?grp->ordinality():max, n, d); + setReplicateFilename(fullname, d); + } + + SocketEndpoint ep; + if (grp) + ep = grp->queryNode(n).endpoint(); + rfn.setPath(ep, fullname.toLowerCase().str()); + + return rfn; +} + inline void LOGPTREE(const char *title,IPropertyTree *pt) { StringBuffer buf; diff --git a/dali/base/dadfs.hpp b/dali/base/dadfs.hpp index 6cde9f34875..e79a439a0cf 100644 --- a/dali/base/dadfs.hpp +++ b/dali/base/dadfs.hpp @@ -811,14 +811,15 @@ enum DistributedFileSystemError // utility routines (used by xref and dfu) -extern da_decl RemoteFilename &constructPartFilename(IGroup *grp,unsigned partno,unsigned partmax,const char *name,const char *partmask,const char *partdir,unsigned copy,ClusterPartDiskMapSpec &mspec,RemoteFilename &rfn); +extern da_decl RemoteFilename &constructPartFilename(IGroup *grp,unsigned partNo,unsigned copy,unsigned max,unsigned lfnHash,int replicateOffset,bool dirPerPart,const char *lname,const char *prefix,const char *pmask,unsigned numDevices,RemoteFilename &rfn); +extern da_decl RemoteFilename &deprecatedConstructPartFilename(IGroup *grp,unsigned partno,unsigned partmax,const char *name,const char *partmask,const char *partdir,unsigned copy,ClusterPartDiskMapSpec &mspec,RemoteFilename &rfn); // legacy version -inline RemoteFilename &constructPartFilename(IGroup *grp,unsigned partno,unsigned partmax,const char *name,const char *partmask,const char *partdir,bool replicate,int replicateoffset,RemoteFilename &rfn,bool localmount=false) +inline RemoteFilename &deprecatedConstructPartFilename(IGroup *grp,unsigned partno,unsigned partmax,const char *name,const char *partmask,const char *partdir,bool replicate,int replicateoffset,RemoteFilename &rfn,bool localmount=false) { // local mount ignored! ClusterPartDiskMapSpec mspec; mspec.replicateOffset = replicateoffset; - return constructPartFilename(grp,partno,partmax,name,partmask,partdir,replicate?1:0,mspec,rfn); + return deprecatedConstructPartFilename(grp,partno,partmax,name,partmask,partdir,replicate?1:0,mspec,rfn); } extern da_decl IDFPartFilter *createPartFilter(const char *filter); diff --git a/dali/base/dafdesc.cpp b/dali/base/dafdesc.cpp index df1c816f8a4..5aedb9988c4 100644 --- a/dali/base/dafdesc.cpp +++ b/dali/base/dafdesc.cpp @@ -3579,7 +3579,10 @@ void GroupInformation::createStoragePlane(IPropertyTree * storage, unsigned copy StringBuffer mirrorname; const char * planeName = name; if (copy != 0) + { planeName = mirrorname.append(name).append("_mirror"); + plane->setPropBool("@copy", true); + } plane->setProp("@name", planeName); diff --git a/dali/datest/datest.cpp b/dali/datest/datest.cpp index 84db25fe817..62a0468da85 100644 --- a/dali/datest/datest.cpp +++ b/dali/datest/datest.cpp @@ -80,7 +80,7 @@ static void addTestFile(const char *name,unsigned n) StringBuffer path; for (unsigned m=0; m pp = createPTree("Part"); pp->setPropInt64("@size",1234*(m+1)); @@ -3328,7 +3328,6 @@ void usage(const char *error=NULL) struct ReleaseAtomBlock { ~ReleaseAtomBlock() { releaseAtoms(); } }; - int main(int argc, char* argv[]) { ReleaseAtomBlock rABlock; diff --git a/dali/dfuXRefLib/CMakeLists.txt b/dali/dfuXRefLib/CMakeLists.txt index 7e931f84074..b99d9d29c77 100644 --- a/dali/dfuXRefLib/CMakeLists.txt +++ b/dali/dfuXRefLib/CMakeLists.txt @@ -46,6 +46,7 @@ include_directories ( ./../../common/environment ./../../common/workunit ./../../system/security/shared + ${HPCC_SOURCE_DIR}/testing/unittests ) ADD_DEFINITIONS( -D_USRDLL -DDFUXREFLIB_EXPORTS ) @@ -57,7 +58,8 @@ target_link_libraries ( dfuXRefLib mp hrpc dafsclient - dalibase + dalibase + ${CppUnit_LIBRARIES} ) if (NOT CONTAINERIZED) diff --git a/dali/dfuXRefLib/dfuxreflib.cpp b/dali/dfuXRefLib/dfuxreflib.cpp index e858a0cf496..51011ef2a94 100644 --- a/dali/dfuXRefLib/dfuxreflib.cpp +++ b/dali/dfuXRefLib/dfuxreflib.cpp @@ -32,6 +32,7 @@ #include "rmtfile.hpp" #include "dautils.hpp" #include "jptree.hpp" +#include "jcontainerized.hpp" #include "XRefNodeManager.hpp" @@ -647,7 +648,7 @@ struct CLogicalNameEntry: public CInterface if (partmask&&*partmask) { if (!containsPathSepChar(partmask)) { const char *dir = file.queryProp("@directory"); - if (dir&&*dir) + if (dir&&*dir) tmp.append(dir).append(getPathSepChar(dir)); } tmp.append(partmask); @@ -655,6 +656,17 @@ struct CLogicalNameEntry: public CInterface tmp.toLowerCase(); substnum(tmp,"$n$",max); dirpartmask.set(tmp.str()); + replicateOffset = file.getPropInt("@replicateOffset",1); + lfnHash = file.getPropInt("@lfnHash"); + plane.setown(getDataStoragePlane(grpname, false)); + if (!plane) + manager.error(_lname, "Plane definition \"%s\" is missing for File", grpname.str()); + prefix = plane->queryPrefix(); + if (prefix.isEmpty()) + manager.error(_lname, "Prefix definition for plane \"%s\" is missing for File", grpname.str()); + // Get dir-per-part # from file metadata or storage plane info + FileDescriptorFlags flags = static_cast(file.getPropInt("Attr/@flags")); + dirPerPart = FileDescriptorFlags::none != (flags & FileDescriptorFlags::dirperpart) ? true : max>1?plane->queryDirPerPart():false; } ~CLogicalNameEntry() { @@ -757,6 +769,16 @@ struct CLogicalNameEntry: public CInterface return false; } + RemoteFilename &constructPartFilename(unsigned partNo, bool replicate, RemoteFilename &rfn) + { + if (!plane) + throw makeStringExceptionV(0, "Plane definition \"%s\" is missing for File", grpname.str()); + unsigned numDevices = plane->numDevices(); + bool r = replicate?1:0; + + return ::constructPartFilename(grp, partNo, r, max, lfnHash, replicateOffset, dirPerPart, lname, prefix, pmask, numDevices, rfn); + } + void resolve(CFileEntry *entry); IPropertyTree *addFileBranch(IPropertyTree *dst,unsigned flags); @@ -793,6 +815,11 @@ struct CLogicalNameEntry: public CInterface bool grouped; StringAttr dirpartmask; CXRefManagerBase &manager; + bool dirPerPart; + int replicateOffset; + unsigned lfnHash; + StringAttr prefix; + Owned plane; }; @@ -914,51 +941,120 @@ static void constructPartname(const char *filename,unsigned n, StringBuffer &pn, } } -static bool parseFileName(const char *name,StringBuffer &mname,unsigned &num,unsigned &max,bool &replicate) +static void parseFileName(const char *name,StringBuffer &mname,unsigned &num,unsigned &max,unsigned &stripeNum,unsigned &dirPerPart,bool &replicate) { // takes filename and creates mask filename with $P$ extension + const char *filename = name; StringBuffer nonrepdir; - replicate = setReplicateDir(name,nonrepdir,false); - if (replicate) - name = nonrepdir.str(); + if (!isContainerized()) + { + replicate = setReplicateDir(name,nonrepdir,false); + if (replicate) + name = nonrepdir.str(); + } + else + replicate = false; // Replicate dir is not supported in containerized environment + + Owned planesIter = getPlanesIterator("data", nullptr); + ForEach(*planesIter) + { + const IPropertyTree &plane = planesIter->query(); + const char *prefix = plane.queryProp("@prefix"); + size_t prefixLen = strlen(prefix); + if (startsWith(name, prefix) && name[prefixLen] == PATHSEPCHAR) + { + mname.ensureCapacity(strlen(name)); + mname.append(prefix).append(PATHSEPCHAR); + name += prefixLen + 1; + stripeNum = 0; + if (plane.getPropInt("@numDevices") > 1) + { + if (*name!='d') + throw makeStringExceptionV(-1, "In storage plane definition numDevices>1, but no stripe sub-directory found in file %s", filename); + name++; + if (*name==PATHSEPCHAR) + throw makeStringExceptionV(-1, "In storage plane definition numDevices>1, but no stripe sub-directory found in file %s", filename); + while (*name&&isdigit(*name)) + { + stripeNum = stripeNum*10+(*name-'0'); + name++; + } + if (*name!=PATHSEPCHAR) + throw makeStringExceptionV(-1, "In storage plane definition numDevices>1, but no stripe sub-directory found in file %s", filename); + name++; + if (stripeNum>=plane.getPropInt("@numDevices")) + throw makeStringExceptionV(-1, "Stripe number in file %s is greater than numDevices in storage plane definition", filename); + } + break; + } + } + + if (mname.isEmpty()) + throw makeStringExceptionV(-1, "Could not find matching prefix in plane definition for file %s", filename); + num = 0; max = 0; - for (;;) { - char c=*name; + dirPerPart = 0; + const char * cur = name; + for (;;) + { + char c=*cur; if (!c) break; - if ((c=='.')&&(name[1]=='_')) { + if ((c=='.')&&(cur[1]=='_')) + { unsigned pn = 0; - const char *s = name+2; - while (*s&&isdigit(*s)) { + const char *s = cur+2; + while (*s&&isdigit(*s)) + { pn = pn*10+(*s-'0'); s++; } - if (pn&&(memicmp(s,"_of_",4)==0)) { + + // Check for dir-per-part number + const char *tailSlash = cur; + while (tailSlash!=name&&*tailSlash!='/') + tailSlash--; + const char *d = tailSlash-1; + for (int i=1;d!=name&&isdigit(*d);i*=10,d--) + dirPerPart += (*d-'0')*i; + // If a dir-per-part number was found, check that it matches the part number + if (*d=='/') + { + if (dirPerPart!=pn) + throw makeStringExceptionV(-1, "Dir-per-part # does not match part # of file %s", filename); + mname.append((d+1)-name, name).append(cur-(tailSlash+1), tailSlash+1); + } + else + mname.append(cur-name,name); + + if (pn&&(memicmp(s,"_of_",4)==0)) + { unsigned mn = 0; s += 4; - while (*s&&isdigit(*s)) { + while (*s&&isdigit(*s)) + { mn = mn*10+(*s-'0'); s++; } - if ((mn!=0)&&((*s==0)||(*s=='.'))&&(mn>=pn)) { // NB allow trailing extension + if ((mn!=0)&&((*s==0)||(*s=='.'))&&(mn>=pn)) + { + // NB allow trailing extension mname.append("._$P$_of_").append(mn); if (*s) mname.append(s); num = pn; max = mn; - return true; } + else + throw makeStringExceptionV(-1, "Incorrect max part number(%d) and part number (%d) in file %s", mn, pn, filename); } + else + throw makeStringExceptionV(-1, "Missing part number in file %s", filename); } - mname.append(c); - name++; + cur++; } - return false; -} - - - +} class COrphanEntry: public CInterface @@ -1517,18 +1613,14 @@ void loadFromDFS(CXRefManagerBase &manager,IGroup *grp,unsigned numdirs,const ch } else { bool replicate=false; - const char *partname = part.queryProp("@name"); - const char *partmask = file.queryProp("@partmask"); - const char *partdir = file.queryProp("@directory"); - int replicateoffset = file.getPropInt("@replicateOffset",1); for (;;) { - RemoteFilename rfn; + RemoteFilename rfn; IGroup *grp = lnentry->queryGroup(); if (!grp) { manager.warn(lnentry->lname.get(),"No group found, ignoring logical file"); return; } - constructPartFilename(grp,partno,numparts,partname,partmask,partdir,replicate,replicateoffset,rfn); + lnentry->constructPartFilename(partno, replicate, rfn); SocketEndpoint rep=rfn.queryEndpoint(); if (manager.EndpointTable.find(rep)!=NULL) { rfn.getLocalPath(localname.clear()); @@ -1544,9 +1636,9 @@ void loadFromDFS(CXRefManagerBase &manager,IGroup *grp,unsigned numdirs,const ch if (dirmatch) { rfn.getRemotePath(remotename.clear()); remotename.toLowerCase(); - + CFileEntry *entry= new CFileEntry(remotename.str(),lnentry,partno,replicate,part.getPropInt64("@size", -1),part.getPropInt("@rowCompression", 0)!=0,part.getPropInt("@compressedSize", -1)); - + CFileEntry *oldentry= manager.filemap.find(remotename.str()); if (oldentry) { @@ -1569,7 +1661,6 @@ void loadFromDFS(CXRefManagerBase &manager,IGroup *grp,unsigned numdirs,const ch else { manager.filemap.add(*entry); } - } else { lnentry->outsidedir++; @@ -1581,33 +1672,32 @@ void loadFromDFS(CXRefManagerBase &manager,IGroup *grp,unsigned numdirs,const ch if (lnentry->outsidenodes.find(rep)==NotFound) lnentry->outsidenodes.append(rep); } + if (isContainerized()) + { + // NB: Replication not supported in containerized version + break; + } if (replicate) break; replicate = true; - } + } } } } scanner(manager,grp,numdirs,dirbaselist); - + manager.log("Loading Files branch from SDS"); Owned conn = querySDS().connect(SDS_DFS_ROOT,myProcessSession(),RTM_LOCK_READ, INFINITE); if (!conn) { throw MakeStringException(-1,"Could not connect to Files"); - } conn->changeMode(RTM_NONE); manager.log("Files loaded, scanning"); scanner.scan(conn); manager.log("Scanning done"); - } - - - - class CPhysicalXREF { unsigned numdirs; @@ -1685,13 +1775,14 @@ class CPhysicalXREF StringBuffer orphanname; unsigned m; unsigned n; + unsigned stripeNum; + unsigned dirPerPart; bool replicate; - parseFileName(fullname,orphanname,n,m,replicate); - if (n>m) - manager.error(fullname, "Part %d: number greater than max %d",n+1,m); - else { - orphanname.toLowerCase(); + try + { + parseFileName(fullname,orphanname,n,m,stripeNum,dirPerPart,replicate); + orphanname.toLowerCase(); COrphanEntryPtr *entryp = manager.orphanmap.getValue(orphanname.str()); COrphanEntryPtr entry; @@ -1709,6 +1800,13 @@ class CPhysicalXREF n--; entry->add(ep,n,replicate,sz); } + catch (IException *e) + { + StringBuffer s; + e->errorMessage(s.clear()); + e->Release(); + manager.error(fullname, "%s", s.str()); + } } } @@ -1867,7 +1965,11 @@ class CXRefManager: public CXRefManagerBase Owned results; { CriticalUnblock unblock(manager.logsect); - unsigned short port = getDafsPort(node.endpoint(),numfails,&crit); + unsigned short port = 0; + // localhost signifies that this is a locally hosted or mounted storage + // which is standard in the cloud and otherwise needs to be reached via dafilesrv + if (!strieq(msg, "localhost")) + port = getDafsPort(node.endpoint(),numfails,&crit); results.setown(getDirectory(dirlist,&node,port)); } manager.log("Crossreferencing %s",msg.str()); @@ -2596,59 +2698,93 @@ class CXRefManager: public CXRefManagerBase msgcallback.set(_msgcallback); IPropertyTree *out=NULL; - - Owned g; - unsigned j; - if (!nclusters) { - error("XREF","No clusters specified\n"); + + if (!nclusters) + { + error("XREF","No storage planes specified\n"); return NULL; } - if (!numdirs) { + if (!numdirs) + { error("XREF","No directories specified\n"); return NULL; } - for (j=0;j gsub = queryNamedGroupStore().lookup(clusters[j]); - if (!gsub) { - error(clusters[j],"Could not find cluster group"); - return NULL; - } - if (!g) - g.set(gsub.get()); - else - g.setown(g->combine(gsub.get())); - } - totalSizeOrphans =0; - totalNumOrphans = 0; + totalSizeOrphans = 0; + totalNumOrphans = 0; logicalnamelist.kill(); dirlist.kill(); orphanlist.kill(); - - const char* cluster = clusters[0]; - loadFromDFS(*this,g,numdirs,dirbaselist,cluster); - xrefRemoteDirectories(g,numdirs,dirbaselist,numthreads); + if (!isContainerized()) + { + Owned g; + unsigned j; + + for (j=0;j gsub = queryNamedGroupStore().lookup(clusters[j]); + if (!gsub) + { + error(clusters[j], "Could not find cluster group"); + return NULL; + } + if (!g) + g.set(gsub.get()); + else + g.setown(g->combine(gsub.get())); + } + + const char* cluster = clusters[0]; + loadFromDFS(*this, g, numdirs, dirbaselist, cluster); + xrefRemoteDirectories(g, numdirs, dirbaselist, numthreads); + } + else + { + const char *storageDir[1]; + for (int i = 0; i < nclusters; i++) + { + DBGLOG("CONTAINERIZED(CXRefManager::process)"); + + const char *storagePlaneName = clusters[i]; // clusters holds a list of storage plane names + Owned storagePlane = getStoragePlane(storagePlaneName); + if (!storagePlane) + { + error("XREF", "Could not find storage plane definition for %s", storagePlaneName); + return NULL; + } + Owned g = queryNamedGroupStore().lookup(storagePlaneName); + if (!g) + { + error("XREF", "Could not find cluster group for storage plane %s", storagePlaneName); + return NULL; + } + storageDir[0] = storagePlane->queryProp("@prefix"); + loadFromDFS(*this, g, 1, storageDir, storagePlaneName); + xrefRemoteDirectories(g, 1, storageDir, numthreads); + } + } + StringBuffer filename; filename.clear().append("xrefrpt"); addFileTimestamp(filename, true); filename.append(".txt"); - - if (flags&PMtextoutput) + + if (flags&PMtextoutput) outputTextReport(filename.str()); filename.clear().append("xrefrpt"); addFileTimestamp(filename, true); filename.append(".txt"); - if (flags&PMcsvoutput) + if (flags&PMcsvoutput) outputCsvReport(filename.str()); - if (flags&PMbackupoutput) + if (flags&PMbackupoutput) outputBackupReport(); - if (flags&PMtreeoutput) - out = outputTree(); + if (flags&PMtreeoutput) + out = outputTree(); logicalnamemap.kill(); filemap.kill(); @@ -2678,12 +2814,11 @@ IPropertyTree * runXRef(unsigned nclusters,const char **clusters,IXRefProgressC #endif // assume all nodes same OS Owned group = queryNamedGroupStore().lookup(clusters[0]); -#ifdef _CONTAINERIZED - WARNLOG("CONTAINERIZED(runXRef calls queryOS())"); -#else - if (group) + if (isContainerized()) + WARNLOG("CONTAINERIZED(runXRef calls queryOS())"); + else if (group) islinux = queryOS(group->queryNode(0).endpoint())==MachineOsLinux; -#endif + dirs[0] = queryBaseDirectory(grp_unknown, 0,islinux?DFD_OSunix:DFD_OSwindows); // MORE - should use the info from the group store dirs[1] = queryBaseDirectory(grp_unknown, 1,islinux?DFD_OSunix:DFD_OSwindows); numdirs = 2; @@ -2904,3 +3039,182 @@ IPropertyTree * RunProcess(XRefCmd cmd, unsigned nclusters,const char **clusters } return nullptr; } + +#ifdef _USE_CPPUNIT + +#include "unittests.hpp" + +#define CPPUNIT_ASSERT_EQUAL_STR(x, y) CPPUNIT_ASSERT_EQUAL(std::string(x ? x : ""),std::string(y ? y : "")) +class DFUXrefLibTests : public CppUnit::TestFixture +{ + CPPUNIT_TEST_SUITE(DFUXrefLibTests); + CPPUNIT_TEST(TestParseFileName); + CPPUNIT_TEST_SUITE_END(); + +protected: + void TestParseFileName() + { + StringBuffer mname; + unsigned num; + unsigned max; + unsigned stripeNum; + unsigned dirPerPart; + bool replicate; + + // Standard file name with multiple parts + parseFileName("/var/lib/HPCCSystems/hpcc-data/test/myname._1_of_3", mname.clear(), num, max, stripeNum, dirPerPart, replicate); + CPPUNIT_ASSERT_EQUAL_STR("/var/lib/HPCCSystems/hpcc-data/test/myname._$P$_of_3",mname.str()); + CPPUNIT_ASSERT_EQUAL_MESSAGE("Part number is incorrect",(unsigned)1,num); + CPPUNIT_ASSERT_EQUAL_MESSAGE("Max part number is incorrect",(unsigned)3,max); + CPPUNIT_ASSERT_EQUAL_MESSAGE("Stripe number is incorrect",(unsigned)0,stripeNum); + CPPUNIT_ASSERT_EQUAL_MESSAGE("Dir-per-part number is incorrect",(unsigned)0,dirPerPart); + CPPUNIT_ASSERT_EQUAL_MESSAGE("Replicate is incorrect",true,replicate); + + // Standard file name with multiple parts is striped across directories, storage plane has numDevices set to 111 + parseFileName("/var/lib/HPCCSystems/hpcc-data-two/d1/test/myname._10_of_30", mname.clear(), num, max, stripeNum, dirPerPart, replicate); + CPPUNIT_ASSERT_EQUAL_STR("/var/lib/HPCCSystems/hpcc-data-two/test/myname._$P$_of_30",mname.str()); + CPPUNIT_ASSERT_EQUAL_MESSAGE("Part number is incorrect",(unsigned)10,num); + CPPUNIT_ASSERT_EQUAL_MESSAGE("Max part number is incorrect",(unsigned)30,max); + CPPUNIT_ASSERT_EQUAL_MESSAGE("Stripe number is incorrect",(unsigned)1,stripeNum); + CPPUNIT_ASSERT_EQUAL_MESSAGE("Dir-per-part number is incorrect",(unsigned)0,dirPerPart); + CPPUNIT_ASSERT_EQUAL_MESSAGE("Replicate is incorrect",true,replicate); + + // Standard file name with multiple parts is striped across directories and each part has its own directory, storage plane has numDevices set to 111 + parseFileName("/var/lib/HPCCSystems/hpcc-data-two/d1/test/42/myname._42_of_100", mname.clear(), num, max, stripeNum, dirPerPart, replicate); + CPPUNIT_ASSERT_EQUAL_STR("/var/lib/HPCCSystems/hpcc-data-two/test/myname._$P$_of_100",mname.str()); + CPPUNIT_ASSERT_EQUAL_MESSAGE("Part number is incorrect",(unsigned)42,num); + CPPUNIT_ASSERT_EQUAL_MESSAGE("Max part number is incorrect",(unsigned)100,max); + CPPUNIT_ASSERT_EQUAL_MESSAGE("Stripe number is incorrect",(unsigned)1,stripeNum); + CPPUNIT_ASSERT_EQUAL_MESSAGE("Dir-per-part number is incorrect",(unsigned)42,dirPerPart); + CPPUNIT_ASSERT_EQUAL_MESSAGE("Replicate is incorrect",true,replicate); + + // Test a longer part number + parseFileName("/var/lib/HPCCSystems/hpcc-data-two/d110/test/12345/myname._12345_of_100000", mname.clear(), num, max, stripeNum, dirPerPart, replicate); + CPPUNIT_ASSERT_EQUAL_STR("/var/lib/HPCCSystems/hpcc-data-two/test/myname._$P$_of_100000",mname.str()); + CPPUNIT_ASSERT_EQUAL_MESSAGE("Part number is incorrect",(unsigned)12345,num); + CPPUNIT_ASSERT_EQUAL_MESSAGE("Max part number is incorrect",(unsigned)100000,max); + CPPUNIT_ASSERT_EQUAL_MESSAGE("Stripe number is incorrect",(unsigned)110,stripeNum); + CPPUNIT_ASSERT_EQUAL_MESSAGE("Dir-per-part number is incorrect",(unsigned)12345,dirPerPart); + CPPUNIT_ASSERT_EQUAL_MESSAGE("Replicate is incorrect",true,replicate); + + // A file without a storage plane throws an exception + try + { + parseFileName("/test/myname._1_of_3", mname.clear(), num, max, stripeNum, dirPerPart, replicate); + CPPUNIT_ASSERT(false); + } + catch (IException *e) + { + StringBuffer msg; + e->errorMessage(msg); + e->Release(); + CPPUNIT_ASSERT_EQUAL_STR("Could not find matching prefix in plane definition for file /test/myname._1_of_3",msg.str()); + } + + // A file with a storage plane that has numDevices>1 but no stripe number in the path throws an exception + try + { + parseFileName("/var/lib/HPCCSystems/hpcc-data-two/test/myname.42_of_100", mname.clear(), num, max, stripeNum, dirPerPart, replicate); + CPPUNIT_ASSERT(false); + } + catch (IException *e) + { + StringBuffer msg; + e->errorMessage(msg); + e->Release(); + CPPUNIT_ASSERT_EQUAL_STR("In storage plane definition numDevices>1, but no stripe sub-directory found in file /var/lib/HPCCSystems/hpcc-data-two/test/myname.42_of_100",msg.str()); + } + + // A file with a storage plane that has numDevices>1 but no stripe number in the path throws an exception + try + { + parseFileName("/var/lib/HPCCSystems/hpcc-data-two/d/test/myname.42_of_100", mname.clear(), num, max, stripeNum, dirPerPart, replicate); + CPPUNIT_ASSERT(false); + } + catch (IException *e) + { + StringBuffer msg; + e->errorMessage(msg); + e->Release(); + CPPUNIT_ASSERT_EQUAL_STR("In storage plane definition numDevices>1, but no stripe sub-directory found in file /var/lib/HPCCSystems/hpcc-data-two/d/test/myname.42_of_100",msg.str()); + } + + // A file with a storage plane that has numDevices>1 but no stripe number in the path throws an exception + try + { + parseFileName("/var/lib/HPCCSystems/hpcc-data-two/datadir/test/myname.42_of_100", mname.clear(), num, max, stripeNum, dirPerPart, replicate); + CPPUNIT_ASSERT(false); + } + catch (IException *e) + { + StringBuffer msg; + e->errorMessage(msg); + e->Release(); + CPPUNIT_ASSERT_EQUAL_STR("In storage plane definition numDevices>1, but no stripe sub-directory found in file /var/lib/HPCCSystems/hpcc-data-two/datadir/test/myname.42_of_100",msg.str()); + } + + // A file where the stripe number is equal to numDevices(111) throws an exception + try + { + parseFileName("/var/lib/HPCCSystems/hpcc-data-two/d111/test/myname.42_of_100", mname.clear(), num, max, stripeNum, dirPerPart, replicate); + CPPUNIT_ASSERT(false); + } + catch (IException *e) + { + StringBuffer msg; + e->errorMessage(msg); + e->Release(); + CPPUNIT_ASSERT_EQUAL_STR("Stripe number in file /var/lib/HPCCSystems/hpcc-data-two/d111/test/myname.42_of_100 is greater than numDevices in storage plane definition",msg.str()); + } + + // A file where the dir-per-part number does not match the part number + try + { + parseFileName("/var/lib/HPCCSystems/hpcc-data-two/d1/test/42/myname._43_of_100", mname.clear(), num, max, stripeNum, dirPerPart, replicate); + CPPUNIT_ASSERT(false); + } + catch (IException *e) + { + StringBuffer msg; + e->errorMessage(msg); + e->Release(); + CPPUNIT_ASSERT_EQUAL_STR("Dir-per-part # does not match part # of file /var/lib/HPCCSystems/hpcc-data-two/d1/test/42/myname._43_of_100",msg.str()); + } + + // A file where the part number is greater than the max part number + try + { + parseFileName("/var/lib/HPCCSystems/hpcc-data-two/d1/test/1000/myname._1000_of_100", mname.clear(), num, max, stripeNum, dirPerPart, replicate); + CPPUNIT_ASSERT(false); + } + catch (IException *e) + { + StringBuffer msg; + e->errorMessage(msg); + e->Release(); + CPPUNIT_ASSERT_EQUAL_STR("Incorrect max part number(100) and part number (1000) in file /var/lib/HPCCSystems/hpcc-data-two/d1/test/1000/myname._1000_of_100",msg.str()); + } + + // A file where the part number is missing + try + { + parseFileName("/var/lib/HPCCSystems/hpcc-data-two/d1/test/myname._of_100", mname.clear(), num, max, stripeNum, dirPerPart, replicate); + CPPUNIT_ASSERT(false); + } + catch (IException *e) + { + StringBuffer msg; + e->errorMessage(msg); + e->Release(); + CPPUNIT_ASSERT_EQUAL_STR("Missing part number in file /var/lib/HPCCSystems/hpcc-data-two/d1/test/myname._of_100",msg.str()); + } + + printf("All parseFileName tests passed\n"); + } + +}; + +CPPUNIT_TEST_SUITE_REGISTRATION(DFUXrefLibTests); +CPPUNIT_TEST_SUITE_NAMED_REGISTRATION(DFUXrefLibTests, "DFUXrefLibTests"); + +#endif // _USE_CPPUNIT diff --git a/dali/sasha/saserver.cpp b/dali/sasha/saserver.cpp index 16f808542cd..5a426fb1633 100644 --- a/dali/sasha/saserver.cpp +++ b/dali/sasha/saserver.cpp @@ -435,8 +435,8 @@ int main(int argc, const char* argv[]) servers.append(*createSashaFileExpiryServer()); else if (strieq(service, "thor-qmon")) servers.append(*createSashaQMonitorServer()); - //else if (strieq(service, "xref")) // TODO - // servers.append(*createSashaXrefServer()); + else if (strieq(service, "xref")) + servers.append(*createSashaXrefServer()); else throw makeStringExceptionV(0, "Unrecognised 'service': %s", service); #else diff --git a/esp/services/ws_dfu/ws_dfuXRefService.cpp b/esp/services/ws_dfu/ws_dfuXRefService.cpp index 759df1a7144..670a76d480f 100644 --- a/esp/services/ws_dfu/ws_dfuXRefService.cpp +++ b/esp/services/ws_dfu/ws_dfuXRefService.cpp @@ -587,55 +587,25 @@ void CWsDfuXRefEx::addXRefNode(const char* name, IPropertyTree* pXRefNodeTree) } } -bool CWsDfuXRefEx::addUniqueXRefNode(const char *processName, BoolHash &uniqueProcesses, IPropertyTree *xrefNodeTree) -{ - if (isEmptyString(processName)) - return false; - bool *found = uniqueProcesses.getValue(processName); - if (found && *found) - return false; - uniqueProcesses.setValue(processName, true); - addXRefNode(processName, xrefNodeTree); - return true; -} - bool CWsDfuXRefEx::onDFUXRefList(IEspContext &context, IEspDFUXRefListRequest &req, IEspDFUXRefListResponse &resp) { try { -#ifdef _CONTAINERIZED - IERRLOG("CONTAINERIZED(CWsDfuXRefEx::onDFUXRefList)"); -#else context.ensureFeatureAccess(XREF_FEATURE_URL, SecAccess_Read, ECLWATCH_DFU_XREF_ACCESS_DENIED, "WsDfuXRef::DFUXRefList: Permission denied."); - CConstWUClusterInfoArray clusters; - getEnvironmentClusterInfo(clusters); - - BoolHash uniqueProcesses; + Owned planesIter = getPlanesIterator("data", nullptr); Owned xrefNodeTree = createPTree("XRefNodes"); - ForEachItemIn(c, clusters) + + ForEach(*planesIter) { - IConstWUClusterInfo &cluster = clusters.item(c); - switch (cluster.getPlatform()) - { - case ThorLCRCluster: - { - const StringArray &primaryThorProcesses = cluster.getPrimaryThorProcesses(); - ForEachItemIn(i, primaryThorProcesses) - addUniqueXRefNode(primaryThorProcesses.item(i), uniqueProcesses, xrefNodeTree); - } - break; - case RoxieCluster: - SCMStringBuffer roxieProcess; - addUniqueXRefNode(cluster.getRoxieProcess(roxieProcess).str(), uniqueProcesses, xrefNodeTree); - break; - } + IPropertyTree &item = planesIter->query(); + bool isCopy = item.getPropBool("@copy", false); + if (!isCopy) + addXRefNode(item.queryProp("@name"), xrefNodeTree); } - addXRefNode("SuperFiles", xrefNodeTree); StringBuffer buf; resp.setDFUXRefListResult(formatResult(context, xrefNodeTree, buf)); -#endif } catch(IException *e) { diff --git a/esp/src/src-react/components/Xrefs.tsx b/esp/src/src-react/components/Xrefs.tsx index 39f37e1a09b..56e7273807b 100644 --- a/esp/src/src-react/components/Xrefs.tsx +++ b/esp/src/src-react/components/Xrefs.tsx @@ -66,13 +66,21 @@ export const Xrefs: React.FunctionComponent = ({ .then(({ DFUXRefListResponse }) => { const xrefNodes = DFUXRefListResponse?.DFUXRefListResult?.XRefNode; if (xrefNodes) { - setData(xrefNodes.map((item, idx) => { - return { - name: item.Name, - modified: item.Modified, - status: item.Status - }; - })); + if (Array.isArray(xrefNodes)) { + setData(xrefNodes.map((item, idx) => { + return { + name: item.Name, + modified: item.Modified, + status: item.Status + }; + })); + } else { + setData([{ + name: xrefNodes.Name, + modified: xrefNodes.Modified, + status: xrefNodes.Status + }]); + } } }) .catch(err => logger.error(err)) diff --git a/helm/examples/vault-pki-remote/values-hpcc1.yaml b/helm/examples/vault-pki-remote/values-hpcc1.yaml index a21e9ccc765..97e58232591 100644 --- a/helm/examples/vault-pki-remote/values-hpcc1.yaml +++ b/helm/examples/vault-pki-remote/values-hpcc1.yaml @@ -52,6 +52,8 @@ sasha: disabled: true file-expiry: disabled: true + xref: + disabled: true esp: - name: eclwatch diff --git a/helm/hpcc/templates/_helpers.tpl b/helm/hpcc/templates/_helpers.tpl index e383472a70f..1f180d1173e 100644 --- a/helm/hpcc/templates/_helpers.tpl +++ b/helm/hpcc/templates/_helpers.tpl @@ -1687,7 +1687,7 @@ prometheusMetricsReporter: "yes" {{- end -}} {{/* -Return access permssions for a given service +Return access permissions for a given service */}} {{- define "hpcc.getSashaServiceAccess" }} {{- if (eq "coalescer" .name) -}} @@ -1702,6 +1702,8 @@ dali dali data {{- else if (eq "thor-qmon" .name) -}} dali queues +{{- else if (eq "xref" .name) -}} +dali dalidata {{- else -}} {{- $_ := fail (printf "Unknown sasha service:" .name ) -}} {{- end -}} diff --git a/helm/hpcc/values.schema.json b/helm/hpcc/values.schema.json index 321ea56c09e..4abce8eb2cf 100644 --- a/helm/hpcc/values.schema.json +++ b/helm/hpcc/values.schema.json @@ -2994,6 +2994,14 @@ }, "additionalProperties": false }, + "sasha-xref": { + "type": "object", + "allOf": [{ "$ref": "#/definitions/sashacommon" }], + "properties": { + "disabled": {} + }, + "additionalProperties": false + }, "sashaservice": { "oneOf": [ { @@ -3018,6 +3026,9 @@ "thor-qmon": { "$ref": "#/definitions/sasha-thor-qmon" }, + "xref": { + "$ref": "#/definitions/sasha-xref" + }, "disabled": { "type": "boolean" } diff --git a/helm/hpcc/values.yaml b/helm/hpcc/values.yaml index b6a6d05ccba..34207c5a3f1 100644 --- a/helm/hpcc/values.yaml +++ b/helm/hpcc/values.yaml @@ -509,6 +509,10 @@ sasha: #switchMinTime: 1 #queues: "*" + xref: {} # NB: no properties defined, use defaults + #disabled: true + #interval: 1 + #user: sasha dfuserver: - name: dfuserver diff --git a/testing/unittests/dalitests.cpp b/testing/unittests/dalitests.cpp index c10a7529d2d..434e873d16d 100644 --- a/testing/unittests/dalitests.cpp +++ b/testing/unittests/dalitests.cpp @@ -477,9 +477,11 @@ class CDaliTestsStress : public CppUnit::TestFixture ClusterPartDiskMapSpec mspec; Owned grp = createIGroup("10.150.10.1-3"); RemoteFilename rfn; + IStoragePlane *plane = getDataStoragePlane("mystorageplane", true); for (unsigned i=0;i<3;i++) - for (unsigned ic=0;ic globals = loadConfiguration(defaultYaml, argv, "unittests", nullptr, nullptr, nullptr, nullptr, false); for (int argNo = 1; argNo < argc; argNo++)