diff --git a/common/thorhelper/thorsoapcall.cpp b/common/thorhelper/thorsoapcall.cpp index fbdd208d860..28f99263a69 100644 --- a/common/thorhelper/thorsoapcall.cpp +++ b/common/thorhelper/thorsoapcall.cpp @@ -106,6 +106,11 @@ class Url : public CInterface, implements IInterface return url.append(method).append("://").append(host).append(":").append(port).append(path); } + StringBuffer &getDynamicUrlSecretName(StringBuffer &secretName) const + { + return generateDynamicUrlSecretName(secretName, method, userPasswordPair, host, port, path); + } + IException *getUrlException(IException *e) const { StringBuffer url; @@ -552,23 +557,31 @@ class BlackLister : public CInterface, implements IThreadFactory } *blacklist; static IPersistentHandler* persistentHandler = nullptr; -static CriticalSection persistentCrit; -static std::atomic persistentInitDone{false}; +static CriticalSection globalFeatureCrit; +static std::atomic globalFeaturesInitDone{false}; +static std::atomic mapUrlsToSecrets{false}; +static std::atomic warnIfUrlNotMappedToSecret{false}; +static std::atomic requireUrlsMappedToSecrets{false}; -void initPersistentHandler() +void initGlobalFeatures() { - CriticalBlock block(persistentCrit); - if (!persistentInitDone) + CriticalBlock block(globalFeatureCrit); + if (!globalFeaturesInitDone) { -#ifndef _CONTAINERIZED - int maxPersistentRequests = queryEnvironmentConf().getPropInt("maxHttpCallPersistentRequests", 0); -#else + int maxPersistentRequests = 0; + if (!isContainerized()) + maxPersistentRequests = queryEnvironmentConf().getPropInt("maxHttpCallPersistentRequests", maxPersistentRequests); //global (backward compatible) + Owned conf = getComponentConfig(); - int maxPersistentRequests = conf->getPropInt("@maxHttpCallPersistentRequests", 0); -#endif + maxPersistentRequests = conf->getPropInt("@maxHttpCallPersistentRequests", maxPersistentRequests); //component config wins + mapUrlsToSecrets = conf->getPropBool("@mapHttpCallUrlsToSecrets", false); + warnIfUrlNotMappedToSecret = conf->getPropBool("@warnIfUrlNotMappedToSecret", mapUrlsToSecrets); + requireUrlsMappedToSecrets = conf->getPropBool("@requireUrlsMappedToSecrets", false); + if (maxPersistentRequests != 0) persistentHandler = createPersistentHandler(nullptr, DEFAULT_MAX_PERSISTENT_IDLE_TIME, maxPersistentRequests, PersistentLogLevel::PLogMin, true); - persistentInitDone = true; + + globalFeaturesInitDone = true; } } @@ -707,6 +720,8 @@ IColumnProvider * CreateColumnProvider(unsigned _callLatencyMs, bool _encoding) enum WSCType{STsoap, SThttp} ; //web service call type +static const char * getWsCallTypeName(WSCType wscType) { return wscType == STsoap ? "SOAPCALL" : "HTTPCALL"; } + //Web Services Call Asynchronous For interface IWSCAsyncFor: public IInterface { @@ -861,6 +876,41 @@ class CWSCHelperThread : public Thread } }; +bool loadConnectSecret(const char *vaultId, const char *secretName, UrlArray &urlArray, StringBuffer &issuer, StringBuffer &proxyAddress, bool required, WSCType wscType) +{ + Owned secret; + if (!isEmptyString(secretName)) + secret.setown(getSecret("ecl", secretName, vaultId, nullptr)); + if (!secret) + { + if (required) + throw MakeStringException(0, "%s %s SECRET not found", getWsCallTypeName(wscType), secretName); + return false; + } + + StringBuffer url; + getSecretKeyValue(url, secret, "url"); + if (url.isEmpty()) + throw MakeStringException(0, "%s %s HTTP SECRET must contain url", getWsCallTypeName(wscType), secretName); + UrlListParser urlListParser(url); + StringBuffer usernamePasswordPair; + getSecretKeyValue(usernamePasswordPair, secret, "username"); + if (usernamePasswordPair.length()) + { + if (strchr(usernamePasswordPair, ':')) + throw MakeStringException(0, "%s HTTP-CONNECT SECRET username contains illegal colon", getWsCallTypeName(wscType)); + StringBuffer password; + getSecretKeyValue(password, secret, "password"); + if (password.length()) + usernamePasswordPair.append(':').append(password); + } + urlListParser.getUrls(urlArray, usernamePasswordPair); + getSecretKeyValue(proxyAddress.clear(), secret, "proxy"); + getSecretKeyValue(issuer, secret, "issuer"); + return true; +} + + //================================================================================================= class CWSCHelper : implements IWSCHelper, public CInterface @@ -886,6 +936,7 @@ class CWSCHelper : implements IWSCHelper, public CInterface bool customClientCert = false; StringAttr clientCertIssuer; IRoxieAbortMonitor * roxieAbortMonitor; + StringBuffer issuer; //TBD sync up with other PR, it will benefit from this being able to come from the secret protected: IArrayOf threads; @@ -1019,7 +1070,7 @@ class CWSCHelper : implements IWSCHelper, public CInterface const char *hosts = hostsString.get(); if (isEmptyString(hosts)) - throw MakeStringException(0, "%sCALL specified no URLs",wscType == STsoap ? "SOAP" : "HTTP"); + throw MakeStringException(0, "%s specified no URLs", getWsCallTypeName(wscType)); if (0==strncmp(hosts, "mtls:", 5)) { clientCertIssuer.set("local"); @@ -1034,9 +1085,9 @@ class CWSCHelper : implements IWSCHelper, public CInterface { const char *finger = hosts+7; if (isEmptyString(finger)) - throw MakeStringException(0, "%sCALL HTTP-CONNECT SECRET specified with no name", wscType == STsoap ? "SOAP" : "HTTP"); + throw MakeStringException(0, "%s HTTP-CONNECT SECRET specified with no name", getWsCallTypeName(wscType)); if (!proxyAddress.isEmpty()) - throw MakeStringException(0, "%sCALL PROXYADDRESS can't be used with HTTP-CONNECT secrets", wscType == STsoap ? "SOAP" : "HTTP"); + throw MakeStringException(0, "%s PROXYADDRESS can't be used with HTTP-CONNECT secrets", getWsCallTypeName(wscType)); StringAttr vaultId; const char *thumb = strchr(finger, ':'); if (thumb) @@ -1046,43 +1097,45 @@ class CWSCHelper : implements IWSCHelper, public CInterface } StringBuffer secretName("http-connect-"); secretName.append(finger); - Owned secret = getSecret("ecl", secretName, vaultId, nullptr); - if (!secret) - throw MakeStringException(0, "%sCALL %s SECRET not found", wscType == STsoap ? "SOAP" : "HTTP", secretName.str()); - - StringBuffer url; - getSecretKeyValue(url, secret, "url"); - if (url.isEmpty()) - throw MakeStringException(0, "%sCALL %s HTTP SECRET must contain url", wscType == STsoap ? "SOAP" : "HTTP", secretName.str()); - UrlListParser urlListParser(url); - StringBuffer auth; - getSecretKeyValue(auth, secret, "username"); - if (auth.length()) - { - if (strchr(auth, ':')) - throw MakeStringException(0, "%sCALL HTTP-CONNECT SECRET username contains illegal colon", wscType == STsoap ? "SOAP" : "HTTP"); - auth.append(':'); - getSecretKeyValue(auth, secret, "password"); - } - urlListParser.getUrls(urlArray, auth); - proxyAddress.set(secret->queryProp("proxy")); - getSecretKeyValue(proxyAddress.clear(), secret, "proxy"); + loadConnectSecret(vaultId, secretName, urlArray, issuer, proxyAddress, true, wscType); } else { UrlListParser urlListParser(hosts); urlListParser.getUrls(urlArray); + if (mapUrlsToSecrets && urlArray.length()) + { + StringBuffer secretName; + UrlArray tempArray; + //TBD: If this is a list of URLs do we A. not check for a mapped secret, B. check the first one, C. Use long secret name including entire list + Url &url = urlArray.tos(); + url.getDynamicUrlSecretName(secretName); + if (secretName.length()) + { + if (loadConnectSecret(nullptr, secretName, tempArray, issuer, proxyAddress, requireUrlsMappedToSecrets, wscType)) + { + logctx.CTXLOG("Mapped %s URL!", wscCallTypeText()); + if (tempArray.length()) + urlArray.swapWith(tempArray); + } + else if (warnIfUrlNotMappedToSecret) + { + //should we warn even if the url doesn't have credentials embedded? If HTTPHEADER is being used to pass credentials, we still prefer connect secrets be used instead. + logctx.CTXLOG("Security Warning: %s not using a connection secret (auto secret = %s)", wscCallTypeText(), secretName.str()); + } + } + } } numUrls = urlArray.ordinality(); if (numUrls == 0) - throw MakeStringException(0, "%sCALL specified no URLs",wscType == STsoap ? "SOAP" : "HTTP"); + throw MakeStringException(0, "%s specified no URLs", getWsCallTypeName(wscType)); if (!proxyAddress.isEmpty()) { UrlListParser proxyUrlListParser(proxyAddress); if (0 == proxyUrlListParser.getUrls(proxyUrlArray)) - throw MakeStringException(0, "%sCALL proxy address specified no URLs",wscType == STsoap ? "SOAP" : "HTTP"); + throw MakeStringException(0, "%s proxy address specified no URLs", getWsCallTypeName(wscType)); } if (wscMode == SCrow) @@ -1259,7 +1312,7 @@ class CWSCHelper : implements IWSCHelper, public CInterface } } inline IXmlToRowTransformer * getRowTransformer() { return rowTransformer; } - inline const char * wscCallTypeText() const { return wscType == STsoap ? "SOAPCALL" : "HTTPCALL"; } + inline const char * wscCallTypeText() const { return getWsCallTypeName(wscType); } protected: friend class CWSCHelperThread; @@ -1606,11 +1659,15 @@ int CWSCHelperThread::run() IWSCHelper * createSoapCallHelper(IWSCRowProvider *r, IEngineRowAllocator * outputAllocator, const char *authToken, WSCMode wscMode, ClientCertificate *clientCert, const IContextLogger &logctx, IRoxieAbortMonitor * roxieAbortMonitor) { + if (!globalFeaturesInitDone) + initGlobalFeatures(); return new CWSCHelper(r, outputAllocator, authToken, wscMode, clientCert, logctx, roxieAbortMonitor, STsoap); } IWSCHelper * createHttpCallHelper(IWSCRowProvider *r, IEngineRowAllocator * outputAllocator, const char *authToken, WSCMode wscMode, ClientCertificate *clientCert, const IContextLogger &logctx, IRoxieAbortMonitor * roxieAbortMonitor) { + if (!globalFeaturesInitDone) + initGlobalFeatures(); return new CWSCHelper(r, outputAllocator, authToken, wscMode, clientCert, logctx, roxieAbortMonitor, SThttp); } @@ -2072,7 +2129,7 @@ class CWSCAsyncFor : implements IWSCAsyncFor, public CInterface, public CAsyncFo read += bytesRead; response.setLength(read); if (bytesRead==0) { - master->logctx.CTXLOG("%sCALL: Warning %sHTTP response terminated prematurely",master->wscType == STsoap ? "SOAP" : "HTTP",chunked?"CHUNKED ":""); + master->logctx.CTXLOG("%s: Warning %sHTTP response terminated prematurely", getWsCallTypeName(master->wscType),chunked?"CHUNKED ":""); break; // oops looks likesocket closed early } } @@ -2099,9 +2156,9 @@ class CWSCAsyncFor : implements IWSCAsyncFor, public CInterface, public CAsyncFo if (checkContentDecoding(dbgheader, response, contentEncoding)) decodeContent(contentEncoding.str(), response); if (soapTraceLevel > 6 || master->logXML) - master->logctx.mCTXLOG("%sCALL: LEN=%d %sresponse(%s%s)", master->wscType == STsoap ? "SOAP" : "HTTP",response.length(),chunked?"CHUNKED ":"", dbgheader.str(), response.str()); + master->logctx.mCTXLOG("%s: LEN=%d %sresponse(%s%s)", getWsCallTypeName(master->wscType),response.length(),chunked?"CHUNKED ":"", dbgheader.str(), response.str()); else if (soapTraceLevel > 8) - master->logctx.mCTXLOG("%sCALL: LEN=%d %sresponse(%s)", master->wscType == STsoap ? "SOAP" : "HTTP",response.length(),chunked?"CHUNKED ":"", response.str()); // not sure this is that useful but... + master->logctx.mCTXLOG("%s: LEN=%d %sresponse(%s)", getWsCallTypeName(master->wscType),response.length(),chunked?"CHUNKED ":"", response.str()); // not sure this is that useful but... return rval; } @@ -2248,7 +2305,7 @@ class CWSCAsyncFor : implements IWSCAsyncFor, public CInterface, public CAsyncFo inline void checkTimeLimitExceeded(unsigned * remainingMS) { if (master->isTimeLimitExceeded(remainingMS)) - throw MakeStringException(TIMELIMIT_EXCEEDED, "%sCALL TIMELIMIT(%ums) exceeded", master->wscType == STsoap ? "SOAP" : "HTTP", master->timeLimitMS); + throw MakeStringException(TIMELIMIT_EXCEEDED, "%s TIMELIMIT(%ums) exceeded", getWsCallTypeName(master->wscType), master->timeLimitMS); } inline bool checkKeepAlive(StringBuffer& headers) @@ -2369,7 +2426,7 @@ class CWSCAsyncFor : implements IWSCAsyncFor, public CInterface, public CAsyncFo { if (master->timeLimitExceeded) { - master->logctx.CTXLOG("%sCALL exiting: time limit (%ums) exceeded",master->wscType == STsoap ? "SOAP" : "HTTP", master->timeLimitMS); + master->logctx.CTXLOG("%s exiting: time limit (%ums) exceeded", getWsCallTypeName(master->wscType), master->timeLimitMS); processException(url, inputRows, e); return; } @@ -2377,7 +2434,7 @@ class CWSCAsyncFor : implements IWSCAsyncFor, public CInterface, public CAsyncFo if (e->errorCode() == ROXIE_ABORT_EVENT) { StringBuffer s; - master->logctx.CTXLOG("%sCALL exiting: Roxie Abort : %s",master->wscType == STsoap ? "SOAP" : "HTTP",e->errorMessage(s).str()); + master->logctx.CTXLOG("%s exiting: Roxie Abort : %s", getWsCallTypeName(master->wscType),e->errorMessage(s).str()); throw; } @@ -2402,7 +2459,7 @@ class CWSCAsyncFor : implements IWSCAsyncFor, public CInterface, public CAsyncFo checkRoxieAbortMonitor(master->roxieAbortMonitor); socket->write(request.str(), request.length()); if (soapTraceLevel > 4) - master->logctx.CTXLOG("%sCALL: sent request (%s) to %s:%d", master->wscType == STsoap ? "SOAP" : "HTTP",master->service.str(), url.host.str(), url.port); + master->logctx.CTXLOG("%s: sent request (%s) to %s:%d", getWsCallTypeName(master->wscType),master->service.str(), url.host.str(), url.port); checkTimeLimitExceeded(&remainingMS); checkRoxieAbortMonitor(master->roxieAbortMonitor); @@ -2412,7 +2469,7 @@ class CWSCAsyncFor : implements IWSCAsyncFor, public CInterface, public CAsyncFo keepAlive = keepAlive && keepAlive2; if (soapTraceLevel > 4) - master->logctx.CTXLOG("%sCALL: received response (%s) from %s:%d", master->wscType == STsoap ? "SOAP" : "HTTP",master->service.str(), url.host.str(), url.port); + master->logctx.CTXLOG("%s: received response (%s) from %s:%d", getWsCallTypeName(master->wscType),master->service.str(), url.host.str(), url.port); if (rval != 200) { @@ -2481,14 +2538,14 @@ class CWSCAsyncFor : implements IWSCAsyncFor, public CInterface, public CAsyncFo if (master->timeLimitExceeded) { processException(url, inputRows, e); - master->logctx.CTXLOG("%sCALL exiting: time limit (%ums) exceeded", master->wscType == STsoap ? "SOAP" : "HTTP", master->timeLimitMS); + master->logctx.CTXLOG("%s exiting: time limit (%ums) exceeded", getWsCallTypeName(master->wscType), master->timeLimitMS); break; } if (e->errorCode() == ROXIE_ABORT_EVENT) { StringBuffer s; - master->logctx.CTXLOG("%sCALL exiting: Roxie Abort : %s",master->wscType == STsoap ? "SOAP" : "HTTP",e->errorMessage(s).str()); + master->logctx.CTXLOG("%s exiting: Roxie Abort : %s", getWsCallTypeName(master->wscType),e->errorMessage(s).str()); throw; } @@ -2531,7 +2588,5 @@ class CWSCAsyncFor : implements IWSCAsyncFor, public CInterface, public CAsyncFo IWSCAsyncFor * createWSCAsyncFor(CWSCHelper * _master, IXmlWriterExt &_xmlWriter, ConstPointerArray &_inputRows, PTreeReaderOptions _options) { - if (!persistentInitDone) - initPersistentHandler(); return new CWSCAsyncFor(_master, _xmlWriter, _inputRows, _options); } diff --git a/ecl/eclcmd/eclcmd_core.cpp b/ecl/eclcmd/eclcmd_core.cpp index 8f3922dba42..62ad2dd01c5 100644 --- a/ecl/eclcmd/eclcmd_core.cpp +++ b/ecl/eclcmd/eclcmd_core.cpp @@ -18,6 +18,7 @@ #include #include "jlog.hpp" #include "jfile.hpp" +#include "jsecrets.hpp" #include "jargv.hpp" #include "jflz.hpp" #include "httpclient.hpp" @@ -2115,6 +2116,90 @@ class EclCmdZapGen : public EclCmdCommon }; + +class EclCmdUrlMapSecretName : public CInterfaceOf +{ +public: + EclCmdUrlMapSecretName() + { + + } + + virtual eclCmdOptionMatchIndicator parseCommandLineOptions(ArgvIterator &iter) override + { + eclCmdOptionMatchIndicator retVal = EclCmdOptionNoMatch; + if (iter.done()) + return EclCmdOptionNoMatch; + + for (; !iter.done(); iter.next()) + { + const char *arg = iter.query(); + if (*arg != '-') //parameters don't start with '-' + { + if (optUrl.length()) + { + fprintf(stderr, "\nunrecognized argument %s\n", arg); + return EclCmdOptionCompletion; + } + optUrl.set(arg); + retVal = EclCmdOptionMatch; + continue; + } + if (iter.matchOption(optUsername, ECLOPT_USERNAME)) + { + retVal = EclCmdOptionMatch; + continue; + } + } + return retVal; + } + virtual bool finalizeOptions(IProperties *globals) override + { + if (optUrl.isEmpty()) + { + fprintf(stdout, "\n URL parameter required.\n"); + return false; + } + return true; + } + virtual int processCMD() override + { + StringBuffer secretName; + generateDynamicUrlSecretName(secretName, optUrl, optUsername); + if (secretName.isEmpty()) + { + fputs("Error genenerating secret name.", stderr); + return 1; + } + fputs(secretName.str(), stdout); + fputs("\n", stdout); + return 0; + } + virtual void usage() override + { + fputs("\nUsage:\n" + "\n" + "The 'url-secret-name' command generates a secret name from a url that can be used to support\n" + " ECL SOAPCALL/HTTPCALL automated url to secret mapping.\n" + " Username can either be embedded in the url, such as https://username@example.com, or\n" + " Passed in as a parameter --username=username\n" + " Passwords embedded in the URL are not needed and will be ignored.\n" + "\n" + "When ECL SOAPCALL URL secret mapping is enabled SOAPCALL will convert the URL provided into a name of this format.\n" + " ECL will then attempt to lookup the secret, and if found will use the contents of the secret, rather then the original url.\n" + "\n" + "ecl url-secret-name [--username=]\n" + "\n" + " URL the URL to convert into a secret name\n" + " Options:\n" + " --username Username to associate with the URL. Will override any username embedded in the URL.\n", + stdout); + } +private: + StringAttr optUrl; + StringAttr optUsername; +}; + //========================================================================================= IEclCommand *createCoreEclCommand(const char *cmdname) @@ -2145,6 +2230,8 @@ IEclCommand *createCoreEclCommand(const char *cmdname) return new EclCmdStatus(); if (strieq(cmdname, "zapgen")) return new EclCmdZapGen(); + if (strieq(cmdname, "url-secret-name")) + return new EclCmdUrlMapSecretName(); if (strieq(cmdname, "sign")) return createSignEclCommand(); if (strieq(cmdname, "listkeyuid")) diff --git a/helm/hpcc/values.schema.json b/helm/hpcc/values.schema.json index 0677217299f..7b507f27451 100644 --- a/helm/hpcc/values.schema.json +++ b/helm/hpcc/values.schema.json @@ -1510,6 +1510,21 @@ }, "hpa": { "$ref": "#/definitions/hpa" + }, + "mapHttpCallUrlsToSecrets": { + "type": "boolean", + "default": false, + "description": "In SOAPCALL and HTTPCALL check if URLs have been mapped to secrets" + }, + "warnIfUrlNotMappedToSecret": { + "type": "boolean", + "default": false, + "description": "In SOAPCALL and HTTPCALL warn if URLs not mapped to secrets" + }, + "requireUrlsMappedToSecrets": { + "type": "boolean", + "default": false, + "description": "Require SOAPCALL and HTTPCALL URLs are secrets or mapped to secrets" } } }, @@ -2192,6 +2207,21 @@ "minimum": 0, "description": "Interval (in milliseconds) between checks that client socket is still open" }, + "mapHttpCallUrlsToSecrets": { + "type": "boolean", + "default": false, + "description": "In SOAPCALL and HTTPCALL check if URLs have been mapped to secrets" + }, + "warnIfUrlNotMappedToSecret": { + "type": "boolean", + "default": false, + "description": "In SOAPCALL and HTTPCALL warn if URLs not mapped to secrets" + }, + "requireUrlsMappedToSecrets": { + "type": "boolean", + "default": false, + "description": "Require SOAPCALL and HTTPCALL URLs are secrets or mapped to secrets" + }, "expert": { "description": "Custom internal options usually reserved for internal testing", "type": "object" @@ -2435,6 +2465,21 @@ }, "allowedPipePrograms": { "$ref" : "#/definitions/allowedPipePrograms" + }, + "mapHttpCallUrlsToSecrets": { + "type": "boolean", + "default": false, + "description": "In SOAPCALL and HTTPCALL check if URLs have been mapped to secrets" + }, + "warnIfUrlNotMappedToSecret": { + "type": "boolean", + "default": false, + "description": "In SOAPCALL and HTTPCALL warn if URLs not mapped to secrets" + }, + "requireUrlsMappedToSecrets": { + "type": "boolean", + "default": false, + "description": "Require SOAPCALL and HTTPCALL URLs are secrets or mapped to secrets" } } }, diff --git a/initfiles/componentfiles/configxml/agentexec.xsl b/initfiles/componentfiles/configxml/agentexec.xsl index 770832c2552..b2e323b9ebc 100644 --- a/initfiles/componentfiles/configxml/agentexec.xsl +++ b/initfiles/componentfiles/configxml/agentexec.xsl @@ -99,6 +99,15 @@ + + + + + + + + + diff --git a/initfiles/componentfiles/configxml/eclagent_config.xsd.in b/initfiles/componentfiles/configxml/eclagent_config.xsd.in index 6721cfa38a4..d4c6e0b1153 100644 --- a/initfiles/componentfiles/configxml/eclagent_config.xsd.in +++ b/initfiles/componentfiles/configxml/eclagent_config.xsd.in @@ -286,6 +286,27 @@ + + + + In SOAPCALL and HTTPCALL check if URLs have been mapped to secrets + + + + + + + In SOAPCALL and HTTPCALL warn if URLs not mapped to secrets + + + + + + + Require SOAPCALL and HTTPCALL URLs are secrets or mapped to secrets + + + diff --git a/initfiles/componentfiles/configxml/roxie.xsd.in b/initfiles/componentfiles/configxml/roxie.xsd.in index 9d339c30e5d..3175574f6d7 100644 --- a/initfiles/componentfiles/configxml/roxie.xsd.in +++ b/initfiles/componentfiles/configxml/roxie.xsd.in @@ -673,6 +673,27 @@ + + + + In SOAPCALL and HTTPCALL check if URLs have been mapped to secrets + + + + + + + In SOAPCALL and HTTPCALL warn if URLs not mapped to secrets + + + + + + + Require SOAPCALL and HTTPCALL URLs are secrets or mapped to secrets + + + diff --git a/initfiles/componentfiles/configxml/thor.xsd.in b/initfiles/componentfiles/configxml/thor.xsd.in index 84e8fcb1133..b3556aa3324 100644 --- a/initfiles/componentfiles/configxml/thor.xsd.in +++ b/initfiles/componentfiles/configxml/thor.xsd.in @@ -656,6 +656,27 @@ + + + + In SOAPCALL and HTTPCALL check if URLs have been mapped to secrets + + + + + + + In SOAPCALL and HTTPCALL warn if URLs not mapped to secrets + + + + + + + Require SOAPCALL and HTTPCALL URLs are secrets or mapped to secrets + + + diff --git a/system/jlib/jsecrets.cpp b/system/jlib/jsecrets.cpp index ce9887d2ee7..c5b30c42187 100644 --- a/system/jlib/jsecrets.cpp +++ b/system/jlib/jsecrets.cpp @@ -242,6 +242,61 @@ extern jlib_decl void splitUrlIsolateScheme(const char *url, StringBuffer &user, splitUrlAuthority(authority, authorityLen, user, password, hostPort, nullptr); } + +static StringBuffer &replaceExtraHostAndPortChars(StringBuffer &s) +{ + size_t l = s.length(); + for (size_t i = 0; i < l; i++) + { + if (s.charAt(i) == '.' || s.charAt(i) == ':') + s.setCharAt(i, '-'); + } + return s; +} + + +extern jlib_decl StringBuffer &generateDynamicUrlSecretName(StringBuffer &secretName, const char *scheme, const char *userPasswordPair, const char *host, unsigned port, const char *path) +{ + secretName.set("http-connect-"); + //Having the host and port visible will help with manageability wherever the secret is stored + if (scheme && !strnicmp("https", scheme, 5)) + secretName.append("ssl-"); + secretName.append(host); + //port is optionally already part of host + replaceExtraHostAndPortChars(secretName); + if (port) + secretName.append('-').append(port); + //Path and username are both sensitive and shouldn't be accessible in the name, include both in the hash to give us the uniqueness we need + unsigned hashvalue = 0; + if (!isEmptyString(path)) + hashvalue = hashcz((const unsigned char *)path, hashvalue); + if (!isEmptyString(userPasswordPair)) + { + const char *delim = strchr(userPasswordPair, ':'); + //Make unique for a given username, but not the current password. The pw provided could change but what's in the secret (if there is one) wins + if (delim) + hashvalue = hashc((const unsigned char *)userPasswordPair, delim-userPasswordPair, hashvalue); + else + hashvalue = hashcz((const unsigned char *)userPasswordPair, hashvalue); + } + if (hashvalue) + secretName.appendf("-%x", hashvalue); + return secretName; +} + +extern jlib_decl StringBuffer &generateDynamicUrlSecretName(StringBuffer &secretName, const char *url, const char *inputUsername) +{ + StringBuffer username; + StringBuffer urlPassword; + StringBuffer scheme; + StringBuffer hostPort; + StringBuffer path; + splitUrlIsolateScheme(url, username, urlPassword, scheme, hostPort, path); + if (!isEmptyString(inputUsername)) + username.set(inputUsername); + + return generateDynamicUrlSecretName(secretName, scheme, username, hostPort, 0, path); +} //--------------------------------------------------------------------------------------------------------------------- @@ -583,12 +638,10 @@ class CVault return false; } const char *s = envelope->queryProp(""); + rkind = kind; if (!isEmptyString(s)) - { - rkind = kind; content.append(s); - return true; - } + return true; } return false; } @@ -597,7 +650,8 @@ class CVault VStringBuffer vername("v.%s", isEmptyString(version) ? "latest" : version); Owned envelope = createPTree(vername); envelope->setPropInt("@created", (int) msTick()); - envelope->setProp("", content); + if (!isEmptyString(content)) + envelope->setProp("", content); { CriticalBlock block(vaultCS); IPropertyTree *parent = ensurePTree(cache, secret); @@ -656,6 +710,8 @@ class CVault } else OERRLOG("Error: Vault %s http error (%d) accessing secret %s.%s location %s", name.str(), res.error(), secretCacheKey, version ? version : "", location); + + addCachedSecret("", secretCacheKey, version); //cache misses so we don't keep calling the vault return false; } bool requestSecret(CVaultKind &rkind, StringBuffer &content, const char *secret, const char *version) @@ -807,7 +863,7 @@ IVaultManager *ensureVaultManager() return vaultManager; } -static IPropertyTree *getCachedLocalSecret(const char *category, const char *name) +static IPropertyTree *getCachedLocalSecret(const char *category, const char *name, bool &cachedMiss) { if (isEmptyString(name)) return nullptr; @@ -825,6 +881,11 @@ static IPropertyTree *getCachedLocalSecret(const char *category, const char *nam secretCache->removeProp(name); return nullptr; } + if (secret->hasProp("@miss")) + { + cachedMiss = true; + return nullptr; + } return secret.getClear(); } } @@ -889,7 +950,10 @@ static IPropertyTree *getLocalSecret(const char *category, const char * name) validateCategoryName(category); validateSecretName(name); - Owned tree = getCachedLocalSecret(category, name); + bool skipLocalFetch = false; + Owned tree = getCachedLocalSecret(category, name, skipLocalFetch); + if (skipLocalFetch) + return nullptr; if (tree) return tree.getClear(); return loadLocalSecret(category, name); @@ -915,7 +979,7 @@ static IPropertyTree *createPTreeFromVaultSecret(const char *content, CVaultKind } return tree.getClear(); } -static IPropertyTree *getCachedVaultSecret(const char *category, const char *vaultId, const char * name, const char *version) +static IPropertyTree *getCachedVaultSecret(const char *category, const char *vaultId, const char * name, const char *version, bool &cachedMiss) { CVaultKind kind; StringBuffer json; @@ -930,6 +994,11 @@ static IPropertyTree *getCachedVaultSecret(const char *category, const char *vau if (!vaultmgr->getCachedSecretFromVault(category, vaultId, kind, json, name, version)) return nullptr; } + if (json.isEmpty()) + { + cachedMiss = true; + return nullptr; + } return createPTreeFromVaultSecret(json.str(), kind); } @@ -956,16 +1025,25 @@ static IPropertyTree *getVaultSecret(const char *category, const char * name, co CVaultKind kind; StringBuffer json; IVaultManager *vaultmgr = ensureVaultManager(); + + bool cachedMiss = false; + if (isEmptyString(vaultId)) { - if (!vaultmgr->getCachedSecretByCategory(category, kind, json, name, version)) + if (vaultmgr->getCachedSecretByCategory(category, kind, json, name, version)) + cachedMiss = json.isEmpty(); + else vaultmgr->requestSecretByCategory(category, kind, json, name, version); } else { if (!vaultmgr->getCachedSecretFromVault(category, vaultId, kind, json, name, version)) + cachedMiss = json.isEmpty(); + else vaultmgr->requestSecretFromVault(category, vaultId, kind, json, name, version); } + if (cachedMiss) + return nullptr; return createPTreeFromVaultSecret(json.str(), kind); } @@ -974,14 +1052,18 @@ IPropertyTree *getSecretTree(const char *category, const char * name, const char if (!isEmptyString(optVaultId)) return getVaultSecret(category, name, optVaultId, optVersion); + //if we get back a null secret, it might be a cached miss, so don't go to the source if flag gets set + bool skipVaultFetch = false; + bool skipLocalFetch = false; + //check for any chached first - Owned secret = getCachedLocalSecret(category, name); + Owned secret = getCachedLocalSecret(category, name, skipLocalFetch); if (!secret) - secret.setown(getCachedVaultSecret(category, nullptr, name, nullptr)); + secret.setown(getCachedVaultSecret(category, nullptr, name, nullptr, skipVaultFetch)); //now check local, then vaults - if (!secret) + if (!secret && !skipLocalFetch) secret.setown(loadLocalSecret(category, name)); - if (!secret) + if (!secret && !skipVaultFetch) secret.setown(requestVaultSecret(category, nullptr, name, nullptr)); return secret.getClear(); } diff --git a/system/jlib/jsecrets.hpp b/system/jlib/jsecrets.hpp index c10cacd131a..81bada07763 100644 --- a/system/jlib/jsecrets.hpp +++ b/system/jlib/jsecrets.hpp @@ -57,6 +57,8 @@ extern jlib_decl IPropertyTree *createIssuerTlsClientConfig(const char *issuer, extern jlib_decl void splitFullUrl(const char *url, bool &https, StringBuffer &user, StringBuffer &password, StringBuffer &host, StringBuffer &port, StringBuffer &fullpath); extern jlib_decl void splitUrlSchemeHostPort(const char *url, StringBuffer &user, StringBuffer &password, StringBuffer &schemeHostPort, StringBuffer &path); extern jlib_decl void splitUrlIsolateScheme(const char *url, StringBuffer &user, StringBuffer &password, StringBuffer &scheme, StringBuffer &hostPort, StringBuffer &path); +extern jlib_decl StringBuffer &generateDynamicUrlSecretName(StringBuffer &secretName, const char *scheme, const char *userPasswordPair, const char *host, unsigned port, const char *path); +extern jlib_decl StringBuffer &generateDynamicUrlSecretName(StringBuffer &secretName, const char *url, const char *username); extern jlib_decl bool queryMtls();