Merge pull request #17803 from jakesmith/HPCC-30331-endpoint-serializ…

…ation HPCC-30331 Avoid hostname serialization in some cases in k8s Reviewed-by: Gavin Halliday <[email protected]> Merged-by: Gavin Halliday <[email protected]>
hpcc-systems · Sep 22, 2023 · 250878c · 250878c
2 parents 625e53d + a8ce859
commit 250878c
Show file tree

Hide file tree

Showing 135 changed files with 531 additions and 504 deletions.
diff --git a/common/dllserver/dllserver.cpp b/common/dllserver/dllserver.cpp
@@ -503,7 +503,7 @@ void DllServer::doRegisterDll(const char * name, const char * kind, const char *
     RemoteFilename dllRemote;
     StringBuffer ipText, dllText;
     dllRemote.setRemotePath(dllPath);
-    dllRemote.queryIP().getIpText(ipText);
+    dllRemote.queryIP().getHostText(ipText);
     dllRemote.getLocalPath(dllText);
 
     Owned<IRemoteConnection> conn = getEntryConnection(name, RTM_LOCK_WRITE);

diff --git a/common/environment/dalienv.cpp b/common/environment/dalienv.cpp
@@ -113,7 +113,7 @@ EnvMachineOS queryOS(const IpAddress & ip)
         Owned<IEnvironmentFactory> factory = getEnvironmentFactory(true);
         Owned<IConstEnvironment> env = factory->openEnvironment();
         StringBuffer ipText;
-        ip.getIpText(ipText);
+        ip.getHostText(ipText);
         Owned<IConstMachineInfo> machine = env->getMachineByAddress(ipText.str());
         if (machine)
             ret = machine->getOS();
@@ -226,7 +226,7 @@ const char * querySlaveExecutable(const char * keyName, const char * exeName, co
     Owned<IEnvironmentFactory> factory = getEnvironmentFactory(true);
     Owned<IConstEnvironment> env = factory->openEnvironment();
     StringBuffer addr;
-    ip.getIpText(addr);
+    ip.getHostText(addr);
 
     StringBufferAdaptor spp(progpath);
     StringBufferAdaptor swd(workdir);
@@ -264,10 +264,10 @@ bool getRemoteRunInfo(const char * keyName, const char * exeName, const char * v
     // use dafilesrv to work out OS
     StringBuffer dalis;
     if (remotedali)
-        remotedali->endpoint().getUrlStr(dalis);
+        remotedali->endpoint().getEndpointHostText(dalis);
     // first get machine by IP
     StringBuffer ips;
-    ip.getIpText(ips);
+    ip.getHostText(ips);
 
     //Cannot use getEnvironmentFactory() since it is using a remotedali
     StringBuffer xpath;

diff --git a/common/environment/environment.cpp b/common/environment/environment.cpp
@@ -1972,7 +1972,7 @@ IConstDropZoneInfo * CLocalEnvironment::getDropZoneByAddressPath(const char * ne
 
 #ifdef _DEBUG
                 StringBuffer serverIpString;
-                serverIP.getIpText(serverIpString);
+                serverIP.getHostText(serverIpString);
                 LOG(MCdebugInfo, unknownJob, "Listed server: '%s', IP: '%s'", dropzoneServer.str(), serverIpString.str());
 #endif
                 if (strisame(netaddress, dropzoneServer) || targetIp.ipequals(serverIP))

diff --git a/common/pkgfiles/referencedfilelist.cpp b/common/pkgfiles/referencedfilelist.cpp
@@ -423,7 +423,7 @@ IPropertyTree *ReferencedFile::getSpecifiedOrRemoteFileTree(IUserDescriptor *use
     if (!fileTree)
         return NULL;
     StringAttrBuilder daliipText(daliip);
-    remote->endpoint().getUrlStr(daliipText);
+    remote->endpoint().getEndpointHostText(daliipText);
     filePrefix.set(remotePrefix);
     return fileTree.getClear();
 }
@@ -471,7 +471,7 @@ void ReferencedFile::resolveRemote(IUserDescriptor *user, INode *remote, const c
     flags |= RefFileNotFound;
 
     StringBuffer dest;
-    DBGLOG("Remote ReferencedFile not found %s [dali=%s, remote=%s, prefix=%s]", logicalName.str(), daliip.get(), remote ? remote->endpoint().getUrlStr(dest).str() : nullptr, remotePrefix);
+    DBGLOG("Remote ReferencedFile not found %s [dali=%s, remote=%s, prefix=%s]", logicalName.str(), daliip.get(), remote ? remote->endpoint().getEndpointHostText(dest).str() : nullptr, remotePrefix);
 }
 
 void ReferencedFile::resolve(const StringArray &locations, const char *srcCluster, IUserDescriptor *user, INode *remote, const char *remotePrefix, bool checkLocalFirst, StringArray *subfiles, bool _trackSubFiles, bool resolveForeign)
@@ -1025,7 +1025,7 @@ void ReferencedFileList::cloneRelationships()
         return;
 
     StringBuffer addr;
-    remote->endpoint().getUrlStr(addr);
+    remote->endpoint().getEndpointHostText(addr);
     IDistributedFileDirectory &dir = queryDistributedFileDirectory();
     ReferencedFileIterator files(this);
     ForEach(files)

diff --git a/common/remote/rmtspawn.cpp b/common/remote/rmtspawn.cpp
@@ -111,7 +111,7 @@ ISocket *spawnRemoteChild(SpawnKind kind, const char * exe, const SocketEndpoint
     unsigned port = SLAVE_CONNECT_PORT + ((unsigned)kind * NUM_SLAVE_CONNECT_PORT) + getRandom() % NUM_SLAVE_CONNECT_PORT;
     StringBuffer args;
 
-    myEP.getUrlStr(args);
+    myEP.getEndpointHostText(args);
     args.append(' ').append(replyTag).append(' ').append((unsigned)kind).append(" ").append(port);
     if (extra)
         args.append(' ').append(extra);
@@ -177,7 +177,7 @@ ISocket *spawnRemoteChild(SpawnKind kind, const char * exe, const SocketEndpoint
         try
         {
             StringBuffer tmp;
-            connectEP.getUrlStr(tmp);
+            connectEP.getEndpointHostText(tmp);
             LOG(MCdetailDebugInfo, unknownJob, "Try to connect to slave %s",tmp.str());
             Owned<ISocket> socket = ISocket::connect_wait(connectEP,MASTER_CONNECT_SLAVE_TIMEOUT);
             if (socket)
@@ -288,7 +288,7 @@ bool CRemoteParentInfo::processCommandLine(int argc, const char * * argv, String
 void CRemoteParentInfo::log()
 {
     StringBuffer temp;
-    LOG(MCdebugProgress, unknownJob, "Starting remote slave.  Master=%s reply=%d port=%d", parent.getUrlStr(temp).str(), replyTag, port);
+    LOG(MCdebugProgress, unknownJob, "Starting remote slave.  Master=%s reply=%d port=%d", parent.getEndpointHostText(temp).str(), replyTag, port);
 }
 
 bool CRemoteParentInfo::sendReply(unsigned version)
@@ -331,7 +331,7 @@ bool CRemoteParentInfo::sendReply(unsigned version)
                         if (version == connectVersion)
                         {
                             buffer.read(connectTag);
-                            masterIP.getIpText(masterIPtext.clear());
+                            masterIP.getHostText(masterIPtext.clear());
 
                             LOG(MCdetailDebugInfo, unknownJob, "Process incoming connection. reply=%d got(%d,%s)", replyTag,connectTag,masterIPtext.str());
 

diff --git a/common/remote/rmtssh.cpp b/common/remote/rmtssh.cpp
@@ -136,7 +136,7 @@ class CFRunSSH: public CInterface, implements IFRunSSH
                 if (sa.ordinality()) {
                     StringBuffer ns;
                     ForEachItemIn(i,sa) {
-                        sa.item(i).getIpText(ns.clear());
+                        sa.item(i).getHostText(ns.clear());
                         slaves.append(ns.str());
                     }
                     return;
@@ -548,7 +548,7 @@ class CFRunSSH: public CInterface, implements IFRunSSH
         strict = false;
         verbose = false;
         StringBuffer ips;
-        ip.getIpText(ips);
+        ip.getHostText(ips);
         slaves.kill();
         slaves.append(ips.str());
         numthreads = 1;

diff --git a/common/roxiecommlib/roxiecommunicationclient.cpp b/common/roxiecommlib/roxiecommunicationclient.cpp
@@ -232,7 +232,7 @@ class CRoxieCommunicationClient: implements IRoxieCommunicationClient, public CI
             if (remoteIP)
                 buf.append(remoteIP);
             else
-                ep.getUrlStr(buf);
+                ep.getEndpointHostText(buf);
 
             throw MakeStringException(ROXIECOMM_SOCKET_ERROR, "%s", buf.str());
         }

diff --git a/common/thorhelper/persistent.cpp b/common/thorhelper/persistent.cpp
@@ -56,7 +56,7 @@ class CPersistentInfo : implements IInterface, public CInterface
     {
         if(_ep)
         {
-            _ep->getUrlStr(epstr);
+            _ep->getEndpointHostText(epstr);
             keystr.set(epstr);
             addKeySuffix(proto, keystr);
         }
@@ -121,7 +121,7 @@ class CAvailKeeper
 private:
     inline StringBuffer& calcKey(SocketEndpoint& ep, PersistentProtocol proto, StringBuffer& keystr)
     {
-        ep.getUrlStr(keystr);
+        ep.getEndpointHostText(keystr);
         return addKeySuffix(proto, keystr);
     }
     SocketSet* findSet(CPersistentInfo* info, bool create = false)
@@ -214,7 +214,7 @@ class CPersistentHandler : implements IPersistentHandler, implements ISocketSele
         if (m_enableDoNotReuseList && ep != nullptr)
         {
             StringBuffer epstr;
-            ep->getUrlStr(epstr);
+            ep->getEndpointHostText(epstr);
             if(m_doNotReuseList.getValue(epstr.str()) != nullptr)
             {
                 PERSILOG(PersistentLogLevel::PLogNormal, "PERSISTENT: socket %d's target endpoint %s is in DoNotReuseList, will not add it.", sock->OShandle(), epstr.str());
@@ -434,7 +434,7 @@ class CPersistentHandler : implements IPersistentHandler, implements ISocketSele
         if(!ep)
             return false;
         StringBuffer epstr;
-        ep->getUrlStr(epstr);
+        ep->getEndpointHostText(epstr);
         if(epstr.length()> 0 && m_doNotReuseList.getValue(epstr.str()) != nullptr)
             return true;
         return false;

diff --git a/common/thorhelper/thorsoapcall.cpp b/common/thorhelper/thorsoapcall.cpp
@@ -340,7 +340,7 @@ class BlackLister : public CInterface, implements IThreadFactory
             if (soapTraceLevel > 3)
             {
                 StringBuffer s;
-                logctx.CTXLOG("endpoint %s is blacklisted", ep.getUrlStr(s).str());
+                logctx.CTXLOG("endpoint %s is blacklisted", ep.getEndpointHostText(s).str());
             }
             return true;
         }
@@ -364,15 +364,15 @@ class BlackLister : public CInterface, implements IThreadFactory
             if (soapTraceLevel > 0)
             {
                 StringBuffer s;
-                logctx.CTXLOG("Blacklisting endpoint %s", p.ep.getUrlStr(s).str());
+                logctx.CTXLOG("Blacklisting endpoint %s", p.ep.getEndpointHostText(s).str());
             }
         }
         else
         {
             if (soapTraceLevel > 3)
             {
                 StringBuffer s;
-                logctx.CTXLOG("Endpoint %s is already blacklisted", p.ep.getUrlStr(s).str());
+                logctx.CTXLOG("Endpoint %s is already blacklisted", p.ep.getEndpointHostText(s).str());
             }
 
         }
@@ -390,7 +390,7 @@ class BlackLister : public CInterface, implements IThreadFactory
         if ((match != NotFound) && (soapTraceLevel > 0))
         {
             StringBuffer s;
-            DBGLOG("De-blacklisting endpoint %s", ep.getUrlStr(s).str());
+            DBGLOG("De-blacklisting endpoint %s", ep.getEndpointHostText(s).str());
         }
     }
 
@@ -413,7 +413,7 @@ class BlackLister : public CInterface, implements IThreadFactory
         if (useBlacklister && lookup(ep, logctx))
         {
             StringBuffer s;
-            ep.getUrlStr(s);
+            ep.getEndpointHostText(s);
             throw MakeStringException(JSOCKERR_connection_failed, "%s %s", blOptions->getBLerror(), s.str());
         }
         Owned<IException> exc;
@@ -470,7 +470,7 @@ class BlackLister : public CInterface, implements IThreadFactory
         }
         if (exc->errorCode()==JSOCKERR_connection_failed) {
             StringBuffer s;
-            ep.getUrlStr(s);
+            ep.getEndpointHostText(s);
             throw MakeStringException(JSOCKERR_connection_failed, "connection failed %s", s.str());
         }
         throw exc.getClear();

diff --git a/common/workunit/workunit.cpp b/common/workunit/workunit.cpp
@@ -13885,7 +13885,7 @@ extern WORKUNIT_API void associateLocalFile(IWUQuery * query, WUFileType type, c
     else
     {
         StringBuffer hostname;
-        queryHostIP().getIpText(hostname);
+        queryHostIP().getHostText(hostname);
         query->addAssociatedFile(type, fullPathName, hostname, description, crc, minActivity, maxActivity);
     }
 }
@@ -14451,7 +14451,7 @@ void executeThorGraph(const char * graphName, IConstWorkUnit &workunit, const IP
         myep.serialize(msg);  // only used for tracing
         if (!conversation->send(msg)) {
             StringBuffer s("Failed to send query to Thor on ");
-            thorMaster.getUrlStr(s);
+            thorMaster.getEndpointHostText(s);
             throw MakeStringExceptionDirect(-1, s.str()); // maybe retry?
         }
         unsigned __int64 blockedTime = elapsedTimer.elapsedNs();
@@ -14461,21 +14461,21 @@ void executeThorGraph(const char * graphName, IConstWorkUnit &workunit, const IP
         }
 
         StringBuffer eps;
-        PROGLOG("Thor on %s running %s", thorMaster.getUrlStr(eps).str(), jobName.str());
+        PROGLOG("Thor on %s running %s", thorMaster.getEndpointHostText(eps).str(), jobName.str());
         MemoryBuffer reply;
         try
         {
             if (!conversation->recv(reply,INFINITE))
             {
                 StringBuffer s("Failed to receive reply from thor ");
-                thorMaster.getUrlStr(s);
+                thorMaster.getEndpointHostText(s);
                 throw MakeStringExceptionDirect(-1, s.str());
             }
         }
         catch (IException *e)
         {
             StringBuffer s("Failed to receive reply from thor ");
-            thorMaster.getUrlStr(s);
+            thorMaster.getEndpointHostText(s);
             s.append("; (").append(e->errorCode()).append(", ");
             e->errorMessage(s).append(")");
             e->Release();

diff --git a/common/workunit/wujobq.cpp b/common/workunit/wujobq.cpp
@@ -109,7 +109,7 @@ class CJobQueueItem: implements IJobQueueItem, public CInterface
         item->setProp("@wuid",qi->queryWUID());
         item->setProp("@owner",qi->queryOwner());
         StringBuffer eps;
-        qi->queryEndpoint().getUrlStr(eps);
+        getRemoteAccessibleHostText(eps, qi->queryEndpoint());
         item->setProp("@node",eps.str());
         StringBuffer dts;
         qi->queryEnqueuedTime().getString(dts);
@@ -1092,7 +1092,7 @@ class CJobQueue: public CJobQueueBase, implements IJobQueue
             ret = qd.root->addPropTree("Client");
             ret->setPropInt64("@session",sessionid);
             StringBuffer eps;
-            ret->setProp("@node",queryMyNode()->endpoint().getUrlStr(eps).str());
+            ret->setProp("@node",queryMyNode()->endpoint().getEndpointHostText(eps).str());
         }
         return ret;
     }

diff --git a/dali/base/daclient.cpp b/dali/base/daclient.cpp
@@ -231,7 +231,7 @@ CSDSServerStatus::CSDSServerStatus(const char *servername)
         IPropertyTree &root = *conn->queryRoot();
         root.setProp("@name",servername);
         StringBuffer node;
-        queryMyNode()->endpoint().getIpText(node);
+        queryMyNode()->endpoint().getHostText(node);
         root.setProp("@node",node.str());
         root.setPropInt("@mpport",queryMyNode()->endpoint().port);
         CDateTime dt;
@@ -336,7 +336,7 @@ bool updateDaliEnv(IPropertyTree *env, bool forceGroupUpdate, const char *daliIp
     if (querySDS().updateEnvironment(env, forceGroupUpdate, response))
     {
         StringBuffer tmp;
-        PROGLOG("Environment and node groups updated in dali at %s",daliep.getUrlStr(tmp).str());
+        PROGLOG("Environment and node groups updated in dali at %s",daliep.getEndpointHostText(tmp).str());
     }
     else
         ret = false;

diff --git a/dali/base/dacoven.cpp b/dali/base/dacoven.cpp
@@ -163,7 +163,7 @@ static void checkDaliVersionInfo(ICommunicator *comm, CDaliVersion &serverVersio
     mb.append(ClientVersion);
     mb.append(MinServerVersion);
     StringBuffer daliEpStr;
-    comm->queryGroup().queryNode(0).endpoint().getUrlStr(daliEpStr); // NB: there's always exactly 1 node
+    comm->queryGroup().queryNode(0).endpoint().getEndpointHostText(daliEpStr); // NB: there's always exactly 1 node
     if (!comm->sendRecv(mb, RANK_RANDOM, MPTAG_DALI_COVEN_REQUEST, VERSION_REQUEST_TIMEOUT))
         throw makeStringExceptionV(-1, "Failed retrieving version information from server [%s], legacy server?", daliEpStr.str());
     if (!mb.length())
@@ -206,7 +206,7 @@ static CriticalSection foreignDaliVersionCacheCrit;
 void checkForeignDaliVersionInfo(const INode *foreignDali, CDaliVersion &serverVersion, CDaliVersion &minClientVersion)
 {
     StringBuffer foreignDaliStr;
-    foreignDali->endpoint().getUrlStr(foreignDaliStr);
+    foreignDali->endpoint().getEndpointHostText(foreignDaliStr);
 
     CriticalBlock b(foreignDaliVersionCacheCrit);
     std::pair<CDaliVersion, CDaliVersion> result;
@@ -395,7 +395,7 @@ CDaliUidAllocator &CDaliUidAllocator::find(CIArrayOf<CDaliUidAllocator> &uidallo
         uidallocators.append(*ret);
     }
     StringBuffer eps;
-    DBGLOG("Added foreign UID allocator for %s", ret->node.getUrlStr(eps).str());
+    DBGLOG("Added foreign UID allocator for %s", ret->node.getEndpointHostText(eps).str());
     return *ret;
 }
 
@@ -498,7 +498,7 @@ class CCovenServer: public CCovenBase
                 store->serialize(mb.clear());
                 if (!sendRecv(mb,r,MPTAG_DALI_COVEN_REQUEST, COVEN_SERVER_TIMEOUT)) {
                     StringBuffer str;
-                    throw MakeStringException(-1,"Could not connect to %s",grp->queryNode(r).endpoint().getUrlStr(str).str());
+                    throw MakeStringException(-1,"Could not connect to %s",grp->queryNode(r).endpoint().getEndpointHostText(str).str());
                 }   
                 mergeStore(store,mb,true);
             }
@@ -508,7 +508,7 @@ class CCovenServer: public CCovenBase
                 for (;;)
                 {
                     if (!recv(mb,r,MPTAG_DALI_COVEN_REQUEST,&sender,COVEN_SERVER_TIMEOUT)) {
-                        throw MakeStringException(-1,"Could not connect to %s",grp->queryNode(r).endpoint().getUrlStr(str).str());
+                        throw MakeStringException(-1,"Could not connect to %s",grp->queryNode(r).endpoint().getEndpointHostText(str).str());
                     }
                     if (RANK_NULL==sender)
                         processMessage(mb);