From 31baac7eddf1848cc853d5f6876e8118e049364d Mon Sep 17 00:00:00 2001 From: wangkx Date: Tue, 30 Jan 2024 15:19:28 -0500 Subject: [PATCH] HPCC-31213 Add plane name to MoveExternalFile The STD.File.MoveExternalFile moves files from one landing zone folder to anothor. The existing method can only specify a landing zone using host name. In this PR, a landing zone can be specified by plane name. Revise based on review: 1. Change the checkExternalFileRights() to call getScopePermissions always. 2. Pass correct scopes to the checkExternalFileRights(). 3. Add try/catch to log failures (not throw exceptions). Signed-off-by: wangkx --- ecllibrary/std/File.ecl | 5 +- plugins/fileservices/fileservices.cpp | 163 ++++++++++++++++++------ plugins/fileservices/fileservices.hpp | 1 + plugins/proxies/lib_fileservices.ecllib | 2 +- 4 files changed, 128 insertions(+), 43 deletions(-) diff --git a/ecllibrary/std/File.ecl b/ecllibrary/std/File.ecl index 4371f36dfa7..022882aa363 100644 --- a/ecllibrary/std/File.ecl +++ b/ecllibrary/std/File.ecl @@ -358,9 +358,10 @@ EXPORT RfsAction(varstring server, varstring query) := * @param location The IP address of the remote machine. * @param frompath The path/name of the file to move. * @param topath The path/name of the target file. + * @param planename The name of the data plane containing the file. */ -EXPORT MoveExternalFile(varstring location, varstring frompath, varstring topath) := - lib_fileservices.FileServices.MoveExternalFile(location, frompath, topath); +EXPORT MoveExternalFile(varstring location, varstring frompath, varstring topath, varstring planename) := + lib_fileservices.FileServices.MoveExternalFile(location, frompath, topath, planename); /** * Removes a single physical file from a remote machine. The diff --git a/plugins/fileservices/fileservices.cpp b/plugins/fileservices/fileservices.cpp index b32257f26b6..a6140fbe2c4 100644 --- a/plugins/fileservices/fileservices.cpp +++ b/plugins/fileservices/fileservices.cpp @@ -2726,7 +2726,7 @@ FILESERVICES_API char * FILESERVICES_CALL fsfResolveHostName(const char *hostna return ret.detach(); } -static void checkExternalFileRights(ICodeContext *ctx, CDfsLogicalFileName &lfn, bool rd,bool wr) +static void checkExternalFileRights(ICodeContext *ctx,const char *scope,bool rd,bool wr) { Linked udesc = ctx->queryUserDescriptor(); unsigned auditflags = 0; @@ -2734,62 +2734,144 @@ static void checkExternalFileRights(ICodeContext *ctx, CDfsLogicalFileName &lfn, auditflags |= (DALI_LDAP_AUDIT_REPORT|DALI_LDAP_READ_WANTED); if (wr) auditflags |= (DALI_LDAP_AUDIT_REPORT|DALI_LDAP_WRITE_WANTED); - SecAccessFlags perm = queryDistributedFileDirectory().getFilePermissions(lfn.get(),udesc,auditflags); + SecAccessFlags perm = queryDistributedFileDirectory().getFScopePermissions(scope,udesc,auditflags); if (wr) { if (!HASWRITEPERMISSION(perm)) { - throw MakeStringException(-1,"Write permission denied for %s", lfn.get()); + throw makeStringExceptionV(-1,"Write permission denied for scope %s", scope); } } if (rd) { if (!HASREADPERMISSION(perm)) { - throw MakeStringException(-1,"Read permission denied for %s", lfn.get()); + throw makeStringExceptionV(-1,"Read permission denied for scope %s", scope); } } } -FILESERVICES_API void FILESERVICES_CALL fsMoveExternalFile(ICodeContext * ctx,const char *location,const char *frompath,const char *topath) +static void checkExternalFilePath(ICodeContext *ctx,IPropertyTree *plane,const char *host, + const char *path,bool rd,bool wr,RemoteFilename &rfn) { - SocketEndpoint ep(location); + if (containsRelPaths(path)) //Detect a path like: a/../../../f + throw makeStringExceptionV(-1,"Invalid file path %s",path); + + CDfsLogicalFileName dlfn; + if (plane) + { + //allow relative path if a plane name is specified. + if (isAbsolutePath(path)) + { + const char *relativePath = getRelativePath(path,plane->queryProp("@prefix")); + if (nullptr == relativePath) + throw makeStringExceptionV(-1,"Invalid plane path %s.",path); + path = relativePath; + } + dlfn.setPlaneExternal(plane->queryProp("@name"),path); + } + else + dlfn.setExternal(host,path); //for backward compatibility + dlfn.getExternalFilename(rfn); + + Owned f = createIFile(rfn); + if (rd && !f->exists()) + throw makeStringExceptionV(-1,"File %s not found",path); + + if (f->exists() && (f->isDirectory()==fileBool::foundYes)) + checkExternalFileRights(ctx,dlfn.get(),rd,wr); //check if the folder can be read/writen. + else + { + StringBuffer scopes; + checkExternalFileRights(ctx,dlfn.getScopes(scopes),rd,wr); + } +} + +static IPropertyTree *checkPlaneOrHost(const char *planeName,const char *host) +{ + if (!isEmptyString(planeName)) + { + Owned plane = getDropZonePlane(planeName); //Only support DropZone for now. + if (!plane) + throw makeStringExceptionV(-1,"DropZone %s not found.",planeName); + return plane.getClear(); + } + SocketEndpoint ep(host); if (ep.isNull()) - throw MakeStringException(-1,"fsMoveExternalFile: Cannot resolve location %s",location); - CDfsLogicalFileName from; - from.setExternal(location,frompath); - CDfsLogicalFileName to; - to.setExternal(location,topath); - checkExternalFileRights(ctx,from,true,true); - checkExternalFileRights(ctx,to,false,true); - RemoteFilename fromrfn; - fromrfn.setPath(ep,frompath); - RemoteFilename torfn; - torfn.setPath(ep,topath); - Owned fileto = createIFile(torfn); - if (fileto->exists()) - throw MakeStringException(-1,"fsMoveExternalFile: Destination %s already exists", topath); - fileto.clear(); - Owned file = createIFile(fromrfn); - file->move(topath); + throw makeStringExceptionV(-1,"Cannot resolve host %s",host); + return nullptr; +} + +static void implementMoveExternalFile(ICodeContext *ctx,const char *location, + const char *frompath,const char *topath,const char *planename) +{ StringBuffer s("MoveExternalFile ('"); - s.append(location).append(',').append(frompath).append(',').append(topath).append(") done"); - WUmessage(ctx,SeverityInformation,NULL,s.str()); - AuditMessage(ctx,"MoveExternalFile",frompath,topath); + try + { + Owned plane = checkPlaneOrHost(planename,location); + RemoteFilename fromrfn,torfn; + checkExternalFilePath(ctx,plane,location,frompath,true,true,fromrfn); + checkExternalFilePath(ctx,plane,location,topath,false,true,torfn); + + Owned fileto = createIFile(torfn); + if (fileto->exists()) + throw makeStringExceptionV(-1,"Destination %s already exists",topath); + fileto.clear(); + Owned file = createIFile(fromrfn); + file->move(topath); + s.append(location).append(',').append(frompath).append(',').append(topath).append(") done"); + AuditMessage(ctx,"MoveExternalFile",frompath,topath); + } + catch (IException *e) + { + s.append(location).append(',').append(frompath).append(',').append(topath).append("), "); + e->errorMessage(s); + e->Release(); + } + WUmessage(ctx,SeverityInformation,nullptr,s.str()); +} + +FILESERVICES_API void FILESERVICES_CALL fsMoveExternalFile(ICodeContext *ctx,const char *location,const char *frompath,const char *topath) +{ + implementMoveExternalFile(ctx,location,frompath,topath,nullptr); +} + +FILESERVICES_API void FILESERVICES_CALL fsMoveExternalFile_v2(ICodeContext *ctx,const char *location, + const char *frompath,const char *topath,const char *planename) +{ + implementMoveExternalFile(ctx,location,frompath,topath,planename); } FILESERVICES_API void FILESERVICES_CALL fsDeleteExternalFile(ICodeContext * ctx,const char *location,const char *path) { - SocketEndpoint ep(location); - if (ep.isNull()) - throw MakeStringException(-1,"fsDeleteExternalFile: Cannot resolve location %s",location); - CDfsLogicalFileName lfn; - lfn.setExternal(location,path); - checkExternalFileRights(ctx,lfn,false,true); - RemoteFilename rfn; - rfn.setPath(ep,path); - Owned file = createIFile(rfn); - file->remove(); StringBuffer s("DeleteExternalFile ('"); - s.append(location).append(',').append(path).append(") done"); - WUmessage(ctx,SeverityInformation,NULL,s.str()); - AuditMessage(ctx,"DeleteExternalFile",path); + try + { + SocketEndpoint ep(location); + if (ep.isNull()) + throw makeStringExceptionV(-1,"Cannot resolve location %s",location); + CDfsLogicalFileName lfn; + lfn.setExternal(location,path); + + RemoteFilename rfn; + lfn.getExternalFilename(rfn); + Owned file = createIFile(rfn); + if (!file->exists()) + throw makeStringExceptionV(-1,"File %s not found",path); + if (file->isDirectory()==fileBool::foundYes) + checkExternalFileRights(ctx,lfn.get(),false,true); + else + { + StringBuffer scopes; + checkExternalFileRights(ctx,lfn.getScopes(scopes),false,true); + } + file->remove(); + s.append(location).append(',').append(path).append(") done"); + AuditMessage(ctx,"DeleteExternalFile",path); + } + catch (IException *e) + { + s.append(location).append(',').append(path).append("), "); + e->errorMessage(s); + e->Release(); + } + WUmessage(ctx,SeverityInformation,nullptr,s.str()); } FILESERVICES_API void FILESERVICES_CALL fsCreateExternalDirectory(ICodeContext * ctx,const char *location,const char *_path) @@ -2805,7 +2887,8 @@ FILESERVICES_API void FILESERVICES_CALL fsCreateExternalDirectory(ICodeContext if (isPathSepChar(path.charAt(path.length()-1))) path.remove(path.length()-1, 1); lfn.setExternal(location,path); - checkExternalFileRights(ctx,lfn,false,true); + StringBuffer scopes; + checkExternalFileRights(ctx,lfn.getScopes(scopes),false,true); RemoteFilename rfn; rfn.setPath(ep,path); Owned file = createIFile(rfn); diff --git a/plugins/fileservices/fileservices.hpp b/plugins/fileservices/fileservices.hpp index db1c52a049f..432e655a862 100644 --- a/plugins/fileservices/fileservices.hpp +++ b/plugins/fileservices/fileservices.hpp @@ -183,6 +183,7 @@ FILESERVICES_API void FILESERVICES_CALL fsRfsAction(const char *server, const ch FILESERVICES_API char * FILESERVICES_CALL fsfGetHostName(const char *ipaddress); FILESERVICES_API char * FILESERVICES_CALL fsfResolveHostName(const char *hostname); FILESERVICES_API void FILESERVICES_CALL fsMoveExternalFile(ICodeContext * ctx,const char *location,const char *frompath,const char *topath); +FILESERVICES_API void FILESERVICES_CALL fsMoveExternalFile_v2(ICodeContext *ctx,const char *location,const char *frompath,const char *topath,const char *planename); FILESERVICES_API void FILESERVICES_CALL fsDeleteExternalFile(ICodeContext * ctx,const char *location,const char *path); FILESERVICES_API void FILESERVICES_CALL fsCreateExternalDirectory(ICodeContext * ctx,const char *location,const char *path); FILESERVICES_API char * FILESERVICES_CALL fsfGetLogicalFileAttribute(ICodeContext * ctx,const char *lfn,const char *attrname); diff --git a/plugins/proxies/lib_fileservices.ecllib b/plugins/proxies/lib_fileservices.ecllib index 33e8412ce2e..cf1a85c9f39 100644 --- a/plugins/proxies/lib_fileservices.ecllib +++ b/plugins/proxies/lib_fileservices.ecllib @@ -99,7 +99,7 @@ export FileServices := SERVICE : plugin('fileservices'), time RfsAction( const varstring server, const varstring query): c,entrypoint='fsRfsAction'; varstring GetHostName( const varstring ipaddress ): c,entrypoint='fsfGetHostName'; varstring ResolveHostName( const varstring hostname ): c,entrypoint='fsfResolveHostName'; - MoveExternalFile(const varstring location, const varstring frompath, const varstring topath): c,action,context,entrypoint='fsMoveExternalFile'; + MoveExternalFile(const varstring location, const varstring frompath, const varstring topath, const varstring planename=''): c,action,context,entrypoint='fsMoveExternalFile_v2'; DeleteExternalFile(const varstring location, const varstring path): c,action,context,entrypoint='fsDeleteExternalFile'; CreateExternalDirectory(const varstring location, const varstring path): c,action,context,entrypoint='fsCreateExternalDirectory'; varstring GetLogicalFileAttribute(const varstring lfn,const varstring attrname) : c,context,entrypoint='fsfGetLogicalFileAttribute';