Skip to content

Commit

Permalink
review update
Browse files Browse the repository at this point in the history 
Signed-off-by: Anthony Fishbeck <[email protected]>
  • Loading branch information
@afishbeck
afishbeck committed Oct 1, 2023
1 parent d048f94 commit 85f3860
Show file tree
Hide file tree
Showing 13 changed files with 46 additions and 60 deletions.
44 changes: 18 additions & 26 deletions common/pkgfiles/referencedfilelist.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -163,8 +163,8 @@ class ReferencedFile : implements IReferencedFile, public CInterface

void processLocalFileInfo(IDistributedFile *df, const StringArray &locations, const char *srcCluster, StringArray *subfiles);
void processLocalFileInfo(IDistributedFile *df, const char *dstCluster, const char *srcCluster, StringArray *subfiles);
void processForeignFileTree(IPropertyTree *tree, const char *srcCluster, StringArray *subfiles);
void processRemoteFileTree(IPropertyTree *tree, const char *srcCluster, StringArray *subfiles);
void processForeignFileTree(const IPropertyTree *tree, const char *srcCluster, StringArray *subfiles);
void processRemoteFileTree(const IPropertyTree *tree, const char *srcCluster, StringArray *subfiles);

void resolveLocal(const StringArray &locations, const char *srcCluster, IUserDescriptor *user, StringArray *subfiles);
void resolveLocal(const char *dstCluster, const char *srcCluster, IUserDescriptor *user, StringArray *subfiles);
Expand Down Expand Up @@ -344,7 +344,7 @@ void ReferencedFile::processLocalFileInfo(IDistributedFile *df, const char *dstC
processLocalFileInfo(df, locations, srcCluster, subfiles);
}

void ReferencedFile::processForeignFileTree(IPropertyTree *tree, const char *srcCluster, StringArray *subfiles)
void ReferencedFile::processForeignFileTree(const IPropertyTree *tree, const char *srcCluster, StringArray *subfiles)
{
flags |= RefFileResolvedForeign;
if (fileSrcCluster.length())
Expand Down Expand Up @@ -378,7 +378,17 @@ void ReferencedFile::processForeignFileTree(IPropertyTree *tree, const char *src
}
}

void ReferencedFile::processRemoteFileTree(IPropertyTree *tree, const char *srcCluster, StringArray *subfiles)
StringBuffer &makeRemoteLFN(StringBuffer &remoteLfn, const char *remoteStorageName, const char *remotePrefix, const char *lfn)
{
if (isEmptyString(remoteStorageName))
return remoteLfn;
remoteLfn.append("remote::").append(remoteStorageName).append("::");
if (remotePrefix && *remotePrefix)
remoteLfn.append(remotePrefix).append("::");
return remoteLfn.append(lfn);
}

void ReferencedFile::processRemoteFileTree(const IPropertyTree *tree, const char *srcCluster, StringArray *subfiles)
{
flags |= RefFileResolvedRemote;
if (fileSrcCluster.length())
Expand All @@ -394,7 +404,7 @@ void ReferencedFile::processRemoteFileTree(IPropertyTree *tree, const char *srcC
const char *lfn = it->query().queryProp("@name");
StringBuffer remoteLfn;
if (flags & RefFileLFNForeign)
lfn = remoteLfn.append("remote::").append(this->remoteStorage).append("::").append(lfn).str();
lfn = makeRemoteLFN(remoteLfn, this->remoteStorage, nullptr, lfn).str();
subfiles->append(lfn);
if (trackSubFiles)
subFileNames.append(lfn);
Expand All @@ -403,7 +413,6 @@ void ReferencedFile::processRemoteFileTree(IPropertyTree *tree, const char *srcC
}
else if (srcCluster && *srcCluster)
{

VStringBuffer xpath("Cluster[@name='%s']", srcCluster);
if (!tree->hasProp(xpath))
flags |= RefFileNotOnSource;
Expand Down Expand Up @@ -501,8 +510,6 @@ void ReferencedFile::resolveForeign(IUserDescriptor *user, INode *remote, const
Owned<IPropertyTree> tree = getFileOrProvidedForeignFileTree(user, remote, remotePrefix);
if (tree)
{
DBGLOG("RemoteDALI xml");
dbglogXML(tree, 1);
processForeignFileTree(tree, srcCluster, subfiles);
return;
}
Expand All @@ -521,10 +528,8 @@ void ReferencedFile::resolveForeign(IUserDescriptor *user, INode *remote, const

IPropertyTree *ReferencedFile::getRemoteStorageFileTree(IUserDescriptor *user, const char *remoteStorageName, const char *remotePrefix)
{
VStringBuffer remoteLFN("remote::%s", remoteStorageName);
if (remotePrefix && *remotePrefix)
remoteLFN.append("::").append(remotePrefix);
remoteLFN.append("::").append(logicalName);
StringBuffer remoteLFN;
makeRemoteLFN(remoteLFN, remoteStorageName, remotePrefix, logicalName);

Owned<wsdfs::IDFSFile> dfsFile = wsdfs::lookupDFSFile(remoteLFN.str(), AccessMode::readSequential, INFINITE, wsdfs::keepAliveExpiryFrequency, user);
IPropertyTree *tree = (dfsFile) ? dfsFile->queryFileMeta() : nullptr;
Expand Down Expand Up @@ -558,8 +563,6 @@ void ReferencedFile::resolveRemote(IUserDescriptor *user, const char *remoteStor
if (tree)
{
remoteStorage.set(remoteStorageName);
DBGLOG("RemoteStorage xml");
dbglogXML(tree, 1);
processRemoteFileTree(tree, srcCluster, subfiles);
return;
}
Expand All @@ -571,7 +574,6 @@ void ReferencedFile::resolveRemote(IUserDescriptor *user, const char *remoteStor

flags |= RefFileNotFound;

StringBuffer dest;
DBGLOG("RemoteStorage ReferencedFile not found %s [remoteStorage=%s, prefix=%s]", logicalName.str(), remoteStorageName, remotePrefix);
}

Expand All @@ -593,15 +595,6 @@ void ReferencedFile::resolveLocalOrForeign(const StringArray &locations, const c
resolveLocal(locations, srcCluster, user, subfiles);
}

/*
void ReferencedFile::resolve(const char *dstCluster, const char *srcCluster, IUserDescriptor *user, INode *remote, const char *remotePrefix, bool checkLocalFirst, StringArray *subfiles, bool _trackSubFiles, bool resolveLFNForeign)
{
StringArray locations;
if (!isEmptyString(dstCluster))
locations.append(dstCluster);
}
*/

static void setRoxieClusterPartDiskMapping(const char *clusterName, const char *defaultFolder, const char *defaultReplicateFolder, bool supercopy, IDFUfileSpec *wuFSpecDest, IDFUoptions *wuOptions)
{
ClusterPartDiskMapSpec spec;
Expand Down Expand Up @@ -685,8 +678,7 @@ static void dfuCopy(const IPropertyTree *directories, IDFUWorkUnit *publisherWu,
logicalName.setForeign(ep,false);
}
}
//Pass forceAllowForeign=true because: 1. we need backward compatibility of configurations, 2. this may be the only place we want to allow foreiegn (to copy files in)
Owned<IDistributedFile> file = wsdfs::lookup(logicalName, user, AccessMode::tbdRead, false, false, nullptr, defaultPrivilegedUser, INFINITE, true);
Owned<IDistributedFile> file = wsdfs::lookup(logicalName, user, AccessMode::tbdRead, false, false, nullptr, defaultPrivilegedUser, INFINITE);
if (!file)
throw MakeStringException(-1, "ReferencedFile failed to find file: %s", logicalName.get());

Expand Down
10 changes: 5 additions & 5 deletions dali/dfu/dfurun.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1064,7 +1064,7 @@ class CDFUengine: public CInterface, implements IDFUengine
}

// first see if target exists (and remove if does and overwrite specified)
Owned<IDistributedFile> dfile = wsdfs::lookup(dlfn,ctx.user,AccessMode::tbdWrite,false,false,nullptr,defaultPrivilegedUser,INFINITE,false);
Owned<IDistributedFile> dfile = wsdfs::lookup(dlfn,ctx.user,AccessMode::tbdWrite,false,false,nullptr,defaultPrivilegedUser,INFINITE);
if (dfile) {
if (!ctx.superoptions->getOverwrite())
throw MakeStringException(-1,"Destination file %s already exists",dlfn.get());
Expand Down Expand Up @@ -1365,7 +1365,7 @@ class CDFUengine: public CInterface, implements IDFUengine
}
srcFile.setown(wsdfs::lookup(tmp.str(),userdesc,
(cmd==DFUcmd_move)||(cmd==DFUcmd_rename)||((cmd==DFUcmd_copy)&&multiclusterinsert) ? AccessMode::tbdWrite : AccessMode::tbdRead,
false,false,nullptr,true, INFINITE,false));
false,false,nullptr,true, INFINITE));

if (!srcFile)
throw MakeStringException(-1,"Source file %s could not be found",tmp.str());
Expand Down Expand Up @@ -1528,7 +1528,7 @@ class CDFUengine: public CInterface, implements IDFUengine
}
else if (multiclustermerge)
{
dstFile.setown(wsdfs::lookup(tmp.str(),userdesc,AccessMode::tbdWrite,false,false,nullptr,defaultPrivilegedUser,INFINITE,false));
dstFile.setown(wsdfs::lookup(tmp.str(),userdesc,AccessMode::tbdWrite,false,false,nullptr,defaultPrivilegedUser,INFINITE));
if (!dstFile)
throw MakeStringException(-1,"Destination for merge %s does not exist",tmp.str());
StringBuffer err;
Expand All @@ -1537,7 +1537,7 @@ class CDFUengine: public CInterface, implements IDFUengine
}
else
{
Owned<IDistributedFile> oldfile = wsdfs::lookup(tmp.str(),userdesc,AccessMode::tbdWrite,false,false,nullptr,defaultPrivilegedUser,INFINITE,false);
Owned<IDistributedFile> oldfile = wsdfs::lookup(tmp.str(),userdesc,AccessMode::tbdWrite,false,false,nullptr,defaultPrivilegedUser,INFINITE);
if (oldfile)
{
StringBuffer reason;
Expand Down Expand Up @@ -1743,7 +1743,7 @@ class CDFUengine: public CInterface, implements IDFUengine
destination->getLogicalName(toname);
if (toname.length()) {
unsigned start = msTick();
Owned<IDistributedFile> newfile = wsdfs::lookup(toname.str(),userdesc,AccessMode::tbdWrite,false,false,nullptr,defaultPrivilegedUser,INFINITE,false);
Owned<IDistributedFile> newfile = wsdfs::lookup(toname.str(),userdesc,AccessMode::tbdWrite,false,false,nullptr,defaultPrivilegedUser,INFINITE);
if (newfile) {
// check for rename into multicluster
CDfsLogicalFileName dstlfn;
Expand Down
12 changes: 3 additions & 9 deletions dali/dfu/dfuutil.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -895,7 +895,6 @@ class CFileCloner

StringBuffer s;
Owned<IPropertyTree> ftree;
Owned<IPropertyTree> filePlane;
IPropertyTree *attsrc = nullptr;
if (remoteStorage || srcLFN.isRemote())
{
Expand All @@ -905,14 +904,9 @@ class CFileCloner
srcLFN.get(remoteLFN);

Owned<wsdfs::IDFSFile> dfsFile = wsdfs::lookupDFSFile(remoteLFN.str(), AccessMode::readSequential, INFINITE, wsdfs::keepAliveExpiryFrequency, foreignuserdesc);
IPropertyTree *tree = (dfsFile) ? dfsFile->queryFileMeta() : nullptr;
if (tree)
ftree.setown(tree->getPropTree("File"));
if (!ftree.get())
throw MakeStringException(-1,"Source file %s could not be found in Remote Storage", remoteLFN.str()); //remote scope already included in remoteLFN
const char *remotePlaneName = ftree->queryProp("@group");
VStringBuffer planeXPath("planes[@name=\"%s\"]", remotePlaneName);
filePlane.set(dfsFile->queryCommonMeta()->queryPropTree(planeXPath));
if (!dfsFile)
throw makeStringExceptionV(-1,"Source file %s could not be found in Remote Storage", remoteLFN.str()); //remote scope already included in remoteLFN
ftree.setown(dfsFile->queryFileMeta()->getPropTree("File"));
}
else
{
Expand Down
2 changes: 1 addition & 1 deletion dali/dfu/dfuwu.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -983,7 +983,7 @@ class CDFUfileSpec: public CLinkedDFUWUchild, implements IDFUfileSpec
parent->getPassword(password);
}
userdesc->set(username.str(),password.str());
Owned<IDistributedFile> file = wsdfs::lookup(lfn,userdesc,AccessMode::tbdRead,false,false,nullptr,defaultPrivilegedUser,INFINITE,false);
Owned<IDistributedFile> file = wsdfs::lookup(lfn,userdesc,AccessMode::tbdRead,false,false,nullptr,defaultPrivilegedUser,INFINITE);
if (file)
return file->getFileDescriptor();
}
Expand Down
6 changes: 3 additions & 3 deletions ecl/eclagent/eclagent.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1441,7 +1441,7 @@ bool EclAgent::fileExists(const char *name)
StringBuffer lfn;
expandLogicalName(lfn, name);

Owned<IDistributedFile> f = wsdfs::lookup(lfn.str(), queryUserDescriptor(), AccessMode::tbdRead, false, false, nullptr, defaultPrivilegedUser, INFINITE, false);
Owned<IDistributedFile> f = wsdfs::lookup(lfn.str(), queryUserDescriptor(), AccessMode::tbdRead, false, false, nullptr, defaultPrivilegedUser, INFINITE);
if (f)
return true;
return false;
Expand Down Expand Up @@ -2774,7 +2774,7 @@ unsigned __int64 EclAgent::getDatasetHash(const char * logicalName, unsigned __i
return crc;
}

Owned<IDistributedFile> file = wsdfs::lookup(fullname.str(),queryUserDescriptor(), AccessMode::tbdRead, false, false, nullptr, defaultPrivilegedUser, INFINITE, false);
Owned<IDistributedFile> file = wsdfs::lookup(fullname.str(),queryUserDescriptor(), AccessMode::tbdRead, false, false, nullptr, defaultPrivilegedUser, INFINITE);
if (file)
{
WorkunitUpdate wu = updateWorkUnit();
Expand Down Expand Up @@ -3084,7 +3084,7 @@ void EclAgent::deleteLRUPersists(const char * logicalName, unsigned keep)
MilliSleep(PERSIST_LOCK_SLEEP + (getRandom()%PERSIST_LOCK_SLEEP));
persistLock.setown(getPersistReadLock(goer));
}
Owned<IDistributedFile> f = wsdfs::lookup(goer, queryUserDescriptor(), AccessMode::tbdWrite, false, false, nullptr, defaultPrivilegedUser, INFINITE, false);
Owned<IDistributedFile> f = wsdfs::lookup(goer, queryUserDescriptor(), AccessMode::tbdWrite, false, false, nullptr, defaultPrivilegedUser, INFINITE);
if (!f)
goto restart; // Persist has been deleted since last checked - repeat the whole process
const char *newAccessTime = f->queryAttributes().queryProp("@accessed");
Expand Down
2 changes: 1 addition & 1 deletion ecl/eclcc/eclcc.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2553,7 +2553,7 @@ IHqlExpression *EclCC::lookupDFSlayout(const char *filename, IErrorReceiver &err
// Look up the file in Dali
try
{
Owned<IDistributedFile> dfsFile = wsdfs::lookup(filename, udesc, AccessMode::tbdRead, false, false, nullptr, defaultPrivilegedUser, INFINITE,false);
Owned<IDistributedFile> dfsFile = wsdfs::lookup(filename, udesc, AccessMode::tbdRead, false, false, nullptr, defaultPrivilegedUser, INFINITE);
if (dfsFile)
{
const char *recordECL = dfsFile->queryAttributes().queryProp("ECL");
Expand Down
2 changes: 1 addition & 1 deletion ecl/hthor/hthor.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1105,7 +1105,7 @@ CHThorIndexWriteActivity::CHThorIndexWriteActivity(IAgentContext &_agent, unsign
expandLogicalFilename(lfn, fname, agent.queryWorkUnit(), agent.queryResolveFilesLocally(), false);
if (!agent.queryResolveFilesLocally())
{
Owned<IDistributedFile> f = wsdfs::lookup(lfn, agent.queryCodeContext()->queryUserDescriptor(), AccessMode::tbdWrite, false, false, nullptr, defaultNonPrivilegedUser, INFINITE, false);
Owned<IDistributedFile> f = wsdfs::lookup(lfn, agent.queryCodeContext()->queryUserDescriptor(), AccessMode::tbdWrite, false, false, nullptr, defaultNonPrivilegedUser, INFINITE);

if (f)
{
Expand Down
12 changes: 6 additions & 6 deletions esp/clients/ws_dfsclient/ws_dfsclient.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -782,14 +782,14 @@ IDistributedFile *lookupLegacyDFSFile(const char *logicalName, AccessMode access
return createLegacyDFSFile(dfsFile);
}

IDistributedFile *lookup(CDfsLogicalFileName &lfn, IUserDescriptor *user, AccessMode accessMode, bool hold, bool lockSuperOwner, IDistributedFileTransaction *transaction, bool priviledged, unsigned timeout, bool forceAllowForeign)
IDistributedFile *lookup(CDfsLogicalFileName &lfn, IUserDescriptor *user, AccessMode accessMode, bool hold, bool lockSuperOwner, IDistributedFileTransaction *transaction, bool priviledged, unsigned timeout)
{
bool viaDali = false;

bool isForeign = false;
try { isForeign = lfn.isForeign(); }
catch(IException *e) { e->Release(); } // catch and ignore multi lfn case, will be checked later
if (isForeign && !forceAllowForeign)
if (isForeign)
{
// default denied in cloud, allowed in bare-metal
bool allow = isContainerized() ? false : true;
Expand Down Expand Up @@ -825,11 +825,11 @@ IDistributedFile *lookup(CDfsLogicalFileName &lfn, IUserDescriptor *user, Access
return wsdfs::lookupLegacyDFSFile(lfn.get(), accessMode, timeout, wsdfs::keepAliveExpiryFrequency, user);
}

IDistributedFile *lookup(const char *logicalFilename, IUserDescriptor *user, AccessMode accessMode, bool hold, bool lockSuperOwner, IDistributedFileTransaction *transaction, bool priviledged, unsigned timeout, bool forceAllowForeign)
IDistributedFile *lookup(const char *logicalFilename, IUserDescriptor *user, AccessMode accessMode, bool hold, bool lockSuperOwner, IDistributedFileTransaction *transaction, bool priviledged, unsigned timeout)
{
CDfsLogicalFileName lfn;
lfn.set(logicalFilename);
return lookup(lfn, user, accessMode, hold, lockSuperOwner, transaction, priviledged, timeout, forceAllowForeign);
return lookup(lfn, user, accessMode, hold, lockSuperOwner, transaction, priviledged, timeout);
}


Expand Down Expand Up @@ -884,7 +884,7 @@ class CLocalOrDistributedFile: implements ILocalOrDistributedFile, public CInter
if (gotlocal)
{
if (!write && !onlylocal) // MORE - this means the dali access checks not happening... maybe that's ok?
dfile.setown(wsdfs::lookup(lfn, user, accessMode, false, false, nullptr, isPrivilegedUser, INFINITE, false));
dfile.setown(wsdfs::lookup(lfn, user, accessMode, false, false, nullptr, isPrivilegedUser, INFINITE));
Owned<IFile> file = getPartFile(0,0);
if (file.get())
{
Expand All @@ -908,7 +908,7 @@ class CLocalOrDistributedFile: implements ILocalOrDistributedFile, public CInter
}
else
{
dfile.setown(wsdfs::lookup(lfn, user, accessMode, false, false, nullptr, isPrivilegedUser, INFINITE, false));
dfile.setown(wsdfs::lookup(lfn, user, accessMode, false, false, nullptr, isPrivilegedUser, INFINITE));
if (dfile.get())
return true;
}
Expand Down
4 changes: 2 additions & 2 deletions esp/clients/ws_dfsclient/ws_dfsclient.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -51,8 +51,8 @@ WS_DFSCLIENT_API IDFSFile *lookupDFSFile(const char *logicalName, AccessMode acc
WS_DFSCLIENT_API IDistributedFile *createLegacyDFSFile(IDFSFile *dfsFile);
WS_DFSCLIENT_API IDistributedFile *lookupLegacyDFSFile(const char *logicalName, AccessMode accessMode, unsigned timeoutSecs, unsigned keepAliveExpiryFrequency, IUserDescriptor *userDesc);

WS_DFSCLIENT_API IDistributedFile *lookup(CDfsLogicalFileName &lfn, IUserDescriptor *user, AccessMode accessMode, bool hold, bool lockSuperOwner, IDistributedFileTransaction *transaction, bool priviledged, unsigned timeout, bool forceAllowForeign);
WS_DFSCLIENT_API IDistributedFile *lookup(const char *logicalFilename, IUserDescriptor *user, AccessMode accessMode, bool hold, bool lockSuperOwner, IDistributedFileTransaction *transaction, bool priviledged, unsigned timeout, bool forceAllowForeign);
WS_DFSCLIENT_API IDistributedFile *lookup(CDfsLogicalFileName &lfn, IUserDescriptor *user, AccessMode accessMode, bool hold, bool lockSuperOwner, IDistributedFileTransaction *transaction, bool priviledged, unsigned timeout);
WS_DFSCLIENT_API IDistributedFile *lookup(const char *logicalFilename, IUserDescriptor *user, AccessMode accessMode, bool hold, bool lockSuperOwner, IDistributedFileTransaction *transaction, bool priviledged, unsigned timeout);


} // end of namespace wsdfs
Expand Down
2 changes: 1 addition & 1 deletion esp/services/ws_fs/ws_fsService.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2752,7 +2752,7 @@ bool CFileSprayEx::onCopy(IEspContext &context, IEspCopy &req, IEspCopyResponse
logicalName.setForeign(ep,false);
}

Owned<IDistributedFile> file = wsdfs::lookup(logicalName, udesc, AccessMode::tbdRead, false, false, nullptr, defaultPrivilegedUser, INFINITE, false);
Owned<IDistributedFile> file = wsdfs::lookup(logicalName, udesc, AccessMode::tbdRead, false, false, nullptr, defaultPrivilegedUser, INFINITE);
if (!file)
throw MakeStringException(ECLWATCH_FILE_NOT_EXIST, "Failed to find file: %s", logicalName.get());

Expand Down
Loading

0 comments on commit 85f3860

Please sign in to comment.