HPCC-30579 Retrieve remote storage targets in ESP FileSpray

1. Move the code for checking the allowForeign from the ws_dfsclient lib into the dali lib so that it can be used by the ws_fs lib. 2. Add new ESP FileSpray method GetRemoteTargets. Signed-off-by: wangkx <[email protected]>
hpcc-systems · Nov 1, 2023 · 6005c35 · 6005c35
1 parent e09fb04
commit 6005c35
Show file tree

Hide file tree

Showing 8 changed files with 59 additions and 19 deletions.
diff --git a/dali/base/dautils.cpp b/dali/base/dautils.cpp
@@ -161,6 +161,21 @@ IPropertyTree * findDropZonePlane(const char * path, const char * host, bool ipM
     return findPlane("lz", path, host, ipMatch, mustMatch);
 }
 
+bool allowForeign()
+{
+    StringBuffer optValue;
+    // NB: component setting takes precedence over global
+    getComponentConfigSP()->getProp("expert/@allowForeign", optValue);
+    if (!optValue.isEmpty())
+        return strToBool(optValue);
+
+    getGlobalConfigSP()->getProp("expert/@allowForeign", optValue);
+    if (!optValue.isEmpty())
+        return strToBool(optValue);
+    // default denied in cloud, allowed in bare-metal
+    return isContainerized() ? false : true;
+}
+
 extern da_decl const char *queryDfsXmlBranchName(DfsXmlBranchKind kind)
 {
     switch (kind) {

diff --git a/dali/base/dautils.hpp b/dali/base/dautils.hpp
@@ -552,6 +552,7 @@ extern da_decl bool isHostInPlane(IPropertyTree *plane, const char *host, bool i
 extern da_decl bool getPlaneHost(StringBuffer &host, IPropertyTree *plane, unsigned which);
 extern da_decl void getPlaneHosts(StringArray &hosts, IPropertyTree *plane);
 extern da_decl bool isPathInPlane(IPropertyTree *plane, const char *path);
+extern da_decl bool allowForeign();
 extern da_decl void setPageCacheTimeoutMilliSeconds(unsigned timeoutSeconds);
 extern da_decl void setMaxPageCacheItems(unsigned _maxPageCacheItems);
 extern da_decl IRemoteConnection* connectXPathOrFile(const char* path, bool safe, StringBuffer& xpath);

diff --git a/esp/clients/ws_dfsclient/ws_dfsclient.cpp b/esp/clients/ws_dfsclient/ws_dfsclient.cpp
@@ -807,24 +807,9 @@ IDistributedFile *lookup(CDfsLogicalFileName &lfn, IUserDescriptor *user, Access
     bool isForeign = false;
     try { isForeign = lfn.isForeign(); }
     catch(IException *e) { e->Release(); } // catch and ignore multi lfn case, will be checked later
-    if (isForeign)
-    {
-        // default denied in cloud, allowed in bare-metal
-        bool allow = isContainerized() ? false : true;
-        StringBuffer optValue;
-        // NB: component setting takes precedence over global
-        getComponentConfigSP()->getProp("expert/@allowForeign", optValue.clear());
-        if (optValue.length())
-            allow = strToBool(optValue);
-        else
-        {
-            getGlobalConfigSP()->getProp("expert/@allowForeign", optValue);
-            if (optValue.length())
-                allow = strToBool(optValue);
-        }
-        if (!allow)
-            throw makeStringExceptionV(0, "foreign access is not permitted from this system (file='%s')", lfn.get());
-    }
+    if (isForeign && !allowForeign())
+        throw makeStringExceptionV(0, "foreign access is not permitted from this system (file='%s')", lfn.get());
+
     // DFS service currently only supports remote files 
     if (isWrite(accessMode))
         viaDali = true;

diff --git a/esp/scm/ws_fs.ecm b/esp/scm/ws_fs.ecm
@@ -714,9 +714,19 @@ ESPresponse [exceptions_inline, nil_remove] GetDFUServerQueuesResponse
     ESParray<string> Names;
 };
 
+ESPrequest GetRemoteTargetsRequest
+{
+};
+
+ESPresponse [exceptions_inline, nil_remove] GetRemoteTargetsResponse
+{
+    ESParray<string> TargetNames;
+    bool             AllowForeign;
+};
+
 ESPservice [
     auth_feature("DEFERRED"),
-    version("1.25"),
+    version("1.26"),
     exceptions_inline("./smc_xslt/exceptions.xslt")] FileSpray
 {
     ESPmethod EchoDateTime(EchoDateTime, EchoDateTimeResponse);
@@ -751,6 +761,7 @@ ESPservice [
     ESPmethod [min_ver("1.13")] DropZoneFileSearch(DropZoneFileSearchRequest, DropZoneFileSearchResponse);
     ESPmethod GetSprayTargets(GetSprayTargetsRequest, GetSprayTargetsResponse);
     ESPmethod [min_ver("1.14")] GetDFUServerQueues(GetDFUServerQueuesRequest, GetDFUServerQueuesResponse);
+    ESPmethod [min_ver("1.26")] GetRemoteTargets(GetRemoteTargetsRequest, GetRemoteTargetsResponse);
 };
 
 SCMexportdef(FileSpray);

diff --git a/esp/services/ws_fs/ws_fsService.cpp b/esp/services/ws_fs/ws_fsService.cpp
@@ -3818,3 +3818,24 @@ bool CFileSprayEx::onGetDFUServerQueues(IEspContext &context, IEspGetDFUServerQu
 
     return true;
 }
+
+bool CFileSprayEx::onGetRemoteTargets(IEspContext &context, IEspGetRemoteTargetsRequest &req, IEspGetRemoteTargetsResponse &resp)
+{
+    try
+    {
+        context.ensureFeatureAccess(FILE_SPRAY_URL, SecAccess_Read, ECLWATCH_FILE_SPRAY_ACCESS_DENIED, "Permission denied.");
+
+        StringArray remoteTargetNames;
+        Owned<IPropertyTreeIterator> remoteItr = getRemoteStoragesIterator();
+        ForEach(*remoteItr)
+            remoteTargetNames.append(remoteItr->query().queryProp("@name"));
+        resp.setTargetNames(remoteTargetNames);
+        resp.setAllowForeign(allowForeign());
+    }
+    catch(IException* e)
+    {
+        FORWARDEXCEPTION(context, e,  ECLWATCH_INTERNAL_ERROR);
+    }
+
+    return true;
+}
diff --git a/esp/services/ws_fs/ws_fsService.hpp b/esp/services/ws_fs/ws_fsService.hpp
@@ -138,6 +138,7 @@ class CFileSprayEx : public CFileSpray
     virtual bool onDeleteDropZoneFiles(IEspContext &context, IEspDeleteDropZoneFilesRequest &req, IEspDFUWorkunitsActionResponse &resp);
     virtual bool onGetSprayTargets(IEspContext &context, IEspGetSprayTargetsRequest &req, IEspGetSprayTargetsResponse &resp);
     virtual bool onGetDFUServerQueues(IEspContext &context, IEspGetDFUServerQueuesRequest &req, IEspGetDFUServerQueuesResponse &resp);
+    virtual bool onGetRemoteTargets(IEspContext &context, IEspGetRemoteTargetsRequest &req, IEspGetRemoteTargetsResponse &resp);
 
 protected:
     StringBuffer m_QueueLabel;

diff --git a/system/jlib/jfile.cpp b/system/jlib/jfile.cpp
@@ -7657,6 +7657,11 @@ IPropertyTree * getRemoteStorage(const char * name)
     return global->getPropTree(xpath);
 }
 
+IPropertyTreeIterator * getRemoteStoragesIterator()
+{
+    return getGlobalConfigSP()->getElements("storage/remote");
+}
+
 IPropertyTreeIterator * getPlanesIterator(const char * category, const char *name)
 {
     StringBuffer xpath("storage/planes");

diff --git a/system/jlib/jfile.hpp b/system/jlib/jfile.hpp
@@ -782,6 +782,7 @@ interface IPropertyTreeIterator;
 extern jlib_decl IPropertyTree * getHostGroup(const char * name, bool required);
 extern jlib_decl IPropertyTree * getStoragePlane(const char * name);
 extern jlib_decl IPropertyTree * getRemoteStorage(const char * name);
+extern jlib_decl IPropertyTreeIterator * getRemoteStoragesIterator();
 extern jlib_decl IPropertyTreeIterator * getPlanesIterator(const char * category, const char *name);
 
 extern jlib_decl IFileIO *createBlockedIO(IFileIO *base, size32_t blockSize);