From 6005c35bdc9fe7d5769e936ab82fa0dfe6a015ab Mon Sep 17 00:00:00 2001 From: wangkx Date: Thu, 19 Oct 2023 16:46:46 -0400 Subject: [PATCH] HPCC-30579 Retrieve remote storage targets in ESP FileSpray 1. Move the code for checking the allowForeign from the ws_dfsclient lib into the dali lib so that it can be used by the ws_fs lib. 2. Add new ESP FileSpray method GetRemoteTargets. Signed-off-by: wangkx --- dali/base/dautils.cpp | 15 +++++++++++++++ dali/base/dautils.hpp | 1 + esp/clients/ws_dfsclient/ws_dfsclient.cpp | 21 +++------------------ esp/scm/ws_fs.ecm | 13 ++++++++++++- esp/services/ws_fs/ws_fsService.cpp | 21 +++++++++++++++++++++ esp/services/ws_fs/ws_fsService.hpp | 1 + system/jlib/jfile.cpp | 5 +++++ system/jlib/jfile.hpp | 1 + 8 files changed, 59 insertions(+), 19 deletions(-) diff --git a/dali/base/dautils.cpp b/dali/base/dautils.cpp index 2fe6abc5b12..596cc7eb703 100644 --- a/dali/base/dautils.cpp +++ b/dali/base/dautils.cpp @@ -161,6 +161,21 @@ IPropertyTree * findDropZonePlane(const char * path, const char * host, bool ipM return findPlane("lz", path, host, ipMatch, mustMatch); } +bool allowForeign() +{ + StringBuffer optValue; + // NB: component setting takes precedence over global + getComponentConfigSP()->getProp("expert/@allowForeign", optValue); + if (!optValue.isEmpty()) + return strToBool(optValue); + + getGlobalConfigSP()->getProp("expert/@allowForeign", optValue); + if (!optValue.isEmpty()) + return strToBool(optValue); + // default denied in cloud, allowed in bare-metal + return isContainerized() ? false : true; +} + extern da_decl const char *queryDfsXmlBranchName(DfsXmlBranchKind kind) { switch (kind) { diff --git a/dali/base/dautils.hpp b/dali/base/dautils.hpp index 10c145de248..05103115e12 100644 --- a/dali/base/dautils.hpp +++ b/dali/base/dautils.hpp @@ -552,6 +552,7 @@ extern da_decl bool isHostInPlane(IPropertyTree *plane, const char *host, bool i extern da_decl bool getPlaneHost(StringBuffer &host, IPropertyTree *plane, unsigned which); extern da_decl void getPlaneHosts(StringArray &hosts, IPropertyTree *plane); extern da_decl bool isPathInPlane(IPropertyTree *plane, const char *path); +extern da_decl bool allowForeign(); extern da_decl void setPageCacheTimeoutMilliSeconds(unsigned timeoutSeconds); extern da_decl void setMaxPageCacheItems(unsigned _maxPageCacheItems); extern da_decl IRemoteConnection* connectXPathOrFile(const char* path, bool safe, StringBuffer& xpath); diff --git a/esp/clients/ws_dfsclient/ws_dfsclient.cpp b/esp/clients/ws_dfsclient/ws_dfsclient.cpp index 677e14f3535..e162d2854c7 100644 --- a/esp/clients/ws_dfsclient/ws_dfsclient.cpp +++ b/esp/clients/ws_dfsclient/ws_dfsclient.cpp @@ -807,24 +807,9 @@ IDistributedFile *lookup(CDfsLogicalFileName &lfn, IUserDescriptor *user, Access bool isForeign = false; try { isForeign = lfn.isForeign(); } catch(IException *e) { e->Release(); } // catch and ignore multi lfn case, will be checked later - if (isForeign) - { - // default denied in cloud, allowed in bare-metal - bool allow = isContainerized() ? false : true; - StringBuffer optValue; - // NB: component setting takes precedence over global - getComponentConfigSP()->getProp("expert/@allowForeign", optValue.clear()); - if (optValue.length()) - allow = strToBool(optValue); - else - { - getGlobalConfigSP()->getProp("expert/@allowForeign", optValue); - if (optValue.length()) - allow = strToBool(optValue); - } - if (!allow) - throw makeStringExceptionV(0, "foreign access is not permitted from this system (file='%s')", lfn.get()); - } + if (isForeign && !allowForeign()) + throw makeStringExceptionV(0, "foreign access is not permitted from this system (file='%s')", lfn.get()); + // DFS service currently only supports remote files if (isWrite(accessMode)) viaDali = true; diff --git a/esp/scm/ws_fs.ecm b/esp/scm/ws_fs.ecm index 6aab351fb1c..0243b320fcd 100644 --- a/esp/scm/ws_fs.ecm +++ b/esp/scm/ws_fs.ecm @@ -714,9 +714,19 @@ ESPresponse [exceptions_inline, nil_remove] GetDFUServerQueuesResponse ESParray Names; }; +ESPrequest GetRemoteTargetsRequest +{ +}; + +ESPresponse [exceptions_inline, nil_remove] GetRemoteTargetsResponse +{ + ESParray TargetNames; + bool AllowForeign; +}; + ESPservice [ auth_feature("DEFERRED"), - version("1.25"), + version("1.26"), exceptions_inline("./smc_xslt/exceptions.xslt")] FileSpray { ESPmethod EchoDateTime(EchoDateTime, EchoDateTimeResponse); @@ -751,6 +761,7 @@ ESPservice [ ESPmethod [min_ver("1.13")] DropZoneFileSearch(DropZoneFileSearchRequest, DropZoneFileSearchResponse); ESPmethod GetSprayTargets(GetSprayTargetsRequest, GetSprayTargetsResponse); ESPmethod [min_ver("1.14")] GetDFUServerQueues(GetDFUServerQueuesRequest, GetDFUServerQueuesResponse); + ESPmethod [min_ver("1.26")] GetRemoteTargets(GetRemoteTargetsRequest, GetRemoteTargetsResponse); }; SCMexportdef(FileSpray); diff --git a/esp/services/ws_fs/ws_fsService.cpp b/esp/services/ws_fs/ws_fsService.cpp index dd41506ce37..8f93fb9c642 100644 --- a/esp/services/ws_fs/ws_fsService.cpp +++ b/esp/services/ws_fs/ws_fsService.cpp @@ -3818,3 +3818,24 @@ bool CFileSprayEx::onGetDFUServerQueues(IEspContext &context, IEspGetDFUServerQu return true; } + +bool CFileSprayEx::onGetRemoteTargets(IEspContext &context, IEspGetRemoteTargetsRequest &req, IEspGetRemoteTargetsResponse &resp) +{ + try + { + context.ensureFeatureAccess(FILE_SPRAY_URL, SecAccess_Read, ECLWATCH_FILE_SPRAY_ACCESS_DENIED, "Permission denied."); + + StringArray remoteTargetNames; + Owned remoteItr = getRemoteStoragesIterator(); + ForEach(*remoteItr) + remoteTargetNames.append(remoteItr->query().queryProp("@name")); + resp.setTargetNames(remoteTargetNames); + resp.setAllowForeign(allowForeign()); + } + catch(IException* e) + { + FORWARDEXCEPTION(context, e, ECLWATCH_INTERNAL_ERROR); + } + + return true; +} diff --git a/esp/services/ws_fs/ws_fsService.hpp b/esp/services/ws_fs/ws_fsService.hpp index eaf05c12342..e62e43a79d8 100644 --- a/esp/services/ws_fs/ws_fsService.hpp +++ b/esp/services/ws_fs/ws_fsService.hpp @@ -138,6 +138,7 @@ class CFileSprayEx : public CFileSpray virtual bool onDeleteDropZoneFiles(IEspContext &context, IEspDeleteDropZoneFilesRequest &req, IEspDFUWorkunitsActionResponse &resp); virtual bool onGetSprayTargets(IEspContext &context, IEspGetSprayTargetsRequest &req, IEspGetSprayTargetsResponse &resp); virtual bool onGetDFUServerQueues(IEspContext &context, IEspGetDFUServerQueuesRequest &req, IEspGetDFUServerQueuesResponse &resp); + virtual bool onGetRemoteTargets(IEspContext &context, IEspGetRemoteTargetsRequest &req, IEspGetRemoteTargetsResponse &resp); protected: StringBuffer m_QueueLabel; diff --git a/system/jlib/jfile.cpp b/system/jlib/jfile.cpp index fa928407783..ed4aa7c1cb8 100644 --- a/system/jlib/jfile.cpp +++ b/system/jlib/jfile.cpp @@ -7657,6 +7657,11 @@ IPropertyTree * getRemoteStorage(const char * name) return global->getPropTree(xpath); } +IPropertyTreeIterator * getRemoteStoragesIterator() +{ + return getGlobalConfigSP()->getElements("storage/remote"); +} + IPropertyTreeIterator * getPlanesIterator(const char * category, const char *name) { StringBuffer xpath("storage/planes"); diff --git a/system/jlib/jfile.hpp b/system/jlib/jfile.hpp index c415685c9bb..509200dd1c4 100644 --- a/system/jlib/jfile.hpp +++ b/system/jlib/jfile.hpp @@ -782,6 +782,7 @@ interface IPropertyTreeIterator; extern jlib_decl IPropertyTree * getHostGroup(const char * name, bool required); extern jlib_decl IPropertyTree * getStoragePlane(const char * name); extern jlib_decl IPropertyTree * getRemoteStorage(const char * name); +extern jlib_decl IPropertyTreeIterator * getRemoteStoragesIterator(); extern jlib_decl IPropertyTreeIterator * getPlanesIterator(const char * category, const char *name); extern jlib_decl IFileIO *createBlockedIO(IFileIO *base, size32_t blockSize);