Merge pull request #17719 from mckellyln/tls_buffer

HPCC-27255 TLS cert/key as buffers Reviewed-By: Anthony Fishbeck <[email protected]> Reviewed-by: Jake Smith <[email protected]> Reviewed-by: Gavin Halliday <[email protected]> Merged-by: Gavin Halliday <[email protected]>
hpcc-systems · Sep 19, 2023 · 5f388e5 · 5f388e5
2 parents 5a6a6cf + 6425bed
commit 5f388e5
Show file tree

Hide file tree

Showing 9 changed files with 288 additions and 177 deletions.
diff --git a/esp/bindings/SOAP/Platform/soapbind.cpp b/esp/bindings/SOAP/Platform/soapbind.cpp
@@ -271,24 +271,37 @@ int CHttpSoapBinding::HandleSoapRequest(CHttpRequest* request, CHttpResponse* re
     return 0;
 }
 
-
-static IPropertyTree *createSecClientConfig(const char *clientCertPath, const char *clientPrivateKey, const char *caCertsPath, bool acceptSelfSigned)
+static IPropertyTree *createSecClientConfig(const char *clientCertFileOrBuf, const char *clientPrivKeyFileOrBuf, const char *caCertsPathOrBuf, bool acceptSelfSigned)
 {
     Owned<IPropertyTree> info = createPTree();
 
-    if (!isEmptyString(clientCertPath))
+    if (!isEmptyString(clientCertFileOrBuf))
     {
-        info->setProp("certificate", clientCertPath);
-        if (!isEmptyString(clientPrivateKey))
-            info->setProp("privatekey", clientPrivateKey);
+        if (containsEmbeddedKey(clientCertFileOrBuf))
+            info->setProp("certificate_pem", clientCertFileOrBuf);
+        else
+            info->setProp("certificate", clientCertFileOrBuf);
+
+        if (!isEmptyString(clientPrivKeyFileOrBuf))
+        {
+            if (containsEmbeddedKey(clientPrivKeyFileOrBuf))
+                info->setProp("privatekey_pem", clientPrivKeyFileOrBuf);
+            else
+                info->setProp("privatekey", clientPrivKeyFileOrBuf);
+        }
     }
 
     IPropertyTree *verify = ensurePTree(info, "verify");
-    if (!isEmptyString(caCertsPath))
+
+    if (!isEmptyString(caCertsPathOrBuf))
     {
         IPropertyTree *ca = ensurePTree(verify, "ca_certificates");
-        ca->setProp("@path", caCertsPath);
+        if (containsEmbeddedKey(caCertsPathOrBuf))
+            ca->setProp("pem", caCertsPathOrBuf);
+        else
+            ca->setProp("@path", caCertsPathOrBuf);
     }
+
     verify->setPropBool("@enable", true);
     verify->setPropBool("@accept_selfsigned", acceptSelfSigned);
     verify->setProp("trusted_peers", "anyone");
@@ -313,6 +326,7 @@ void CSoapRequestBinding::post(const char *proxy, const char* url, IRpcResponseB
         soapclient.setReadTimeoutSecs(readTimeoutSecs_);
     if (mtls_secret_.length())
         soapclient.setMtlsSecretName(mtls_secret_);
+
     if (client_cert_.length() || ca_certs_.length() || accept_self_signed_)
         soapclient.setSecureSocketConfig(createSecClientConfig(client_cert_, client_priv_key_, ca_certs_, accept_self_signed_));
 

diff --git a/esp/bindings/SOAP/Platform/soapbind.hpp b/esp/bindings/SOAP/Platform/soapbind.hpp
@@ -181,12 +181,17 @@ class esp_http_decl CSoapRequestBinding : public CSoapComplexType,
     void setMtlsSecretName(const char *name){mtls_secret_.set(name);}
     const char *getMtlsSecretName(){return mtls_secret_.str();}
 
-    void setCACertificates(const char *path) override {ca_certs_.set(path);}
+    void setCACertificates(const char *path) override
+    {
+        ca_certs_.set(path);
+    }
+
     virtual void setClientCertificate(const char *certPath, const char *privateKeyPath) override
     {
         client_cert_.set(certPath);
         client_priv_key_.set(privateKeyPath);
     }
+
     virtual void setAcceptSelfSigned(bool acceptSelfSigned) override {accept_self_signed_=acceptSelfSigned;}
 
     void post(const char *proxy, const char* url, IRpcResponseBinding& response, const char *action=NULL);
@@ -208,20 +213,25 @@ inline void setRpcSSLOptions(IEspClientRpcSettings &rpc, bool useSSL, const char
     {
         if (!isEmptyString(clientCert))
         {
-            if (!checkFileExists(clientCert))
-                throw makeStringExceptionV(-1,"Client certificate not found %s.", clientCert);
             if (isEmptyString(clientPrivateKey))
                 throw makeStringException(-1,"Client private key not provided.");
-            if (!checkFileExists(clientPrivateKey))
+
+            if (!containsEmbeddedKey(clientCert) && !checkFileExists(clientCert))
+                throw makeStringExceptionV(-1,"Client certificate not found %s.", clientCert);
+            if (!containsEmbeddedKey(clientPrivateKey) && !checkFileExists(clientPrivateKey))
                 throw makeStringExceptionV(-1,"Client private key not found %s.", clientPrivateKey);
+
             rpc.setClientCertificate(clientCert, clientPrivateKey);
         }
+
         if (!isEmptyString(caCert))
         {
-            if (!checkFileExists(caCert))
+            if (!containsEmbeddedKey(caCert) && !checkFileExists(caCert))
                 throw makeStringExceptionV(-1,"CA certificate not found %s.", caCert);
+
             rpc.setCACertificates(caCert);
         }
+
         rpc.setAcceptSelfSigned(acceptSelfSigned);
     }
 }

diff --git a/esp/clients/ws_dfsclient/ws_dfsclient.cpp b/esp/clients/ws_dfsclient/ws_dfsclient.cpp
@@ -471,7 +471,7 @@ class CServiceSuperDistributedFile : public CServiceDistributedFileBase<IDistrib
         if (!legacyDFSSuperFile->existsPhysicalPartFiles(0))
         {
             const char * logicalName = queryLogicalName();
-            throw MakeStringException(-1, "Some physical parts do not exists, for logical file : %s",(isEmptyString(logicalName) ? "[unattached]" : logicalName));
+            throw makeStringExceptionV(-1, "Some physical parts do not exists, for logical file : %s",(isEmptyString(logicalName) ? "[unattached]" : logicalName));
         }
     }
 
@@ -573,68 +573,22 @@ IClientWsDfs *getDfsClient(const char *serviceUrl, IUserDescriptor *userDesc)
     return dfsClient.getClear();
 }
 
-static CriticalSection localSecretCrit;
-static constexpr unsigned cachedSecretTimeoutSecs = 120; // 2 mins
 static void configureClientSSL(IEspClientRpcSettings &rpc, const char *secretName)
 {
-    /*
-     * This is a bit of a kludge, it gets the certificates from secrets, and writes them to local temp strorage.
-     * It does this so that it can pass the filename paths to rpc ssl / secure socket layer, which currently only
-     * accepts filenames, not binary blobs from memory.
-     */
-    StringBuffer clientCertFilename, clientPrivateKeyFilename, caCertFilename;
-
-    StringBuffer tempDirStr;
-    verifyex(getConfigurationDirectory(getGlobalConfigSP()->queryPropTree("Directories"), "temp", "ssl", "ssl", tempDirStr));
-    addPathSepChar(tempDirStr);
-    tempDirStr.append(secretName);
-    addPathSepChar(tempDirStr);
-
-    clientCertFilename.append(tempDirStr).append("tls.crt");
-    clientPrivateKeyFilename.append(tempDirStr).append("tls.key");
-    caCertFilename.append(tempDirStr).append("ca.crt");
-
-    bool cacheEntryValid = false;
-    CriticalBlock b(localSecretCrit);
-    Owned<IFile> tempDir = createIFile(tempDirStr);
-    CDateTime timeOutTime, nowTime;
-    bool dirExists = false;
-    if (tempDir->getTime(&timeOutTime, nullptr, nullptr))
-    {
-        dirExists = true;
-        timeOutTime.adjustTimeSecs(cachedSecretTimeoutSecs);
-        nowTime.setNow();
-        if (nowTime.compare(timeOutTime, false) < 0)
-            cacheEntryValid = true;
-    }
-
-    if (!cacheEntryValid)
-    {
-        if (dirExists)
-            verifyex(tempDir->setTime(&nowTime, &nowTime, nullptr));
-        else
-            verifyex(tempDir->createDirectory());
-        StringBuffer secretValue;
-
-        Owned<IFile> file = createIFile(clientCertFilename);
-        Owned<IFileIO> io = file->open(IFOcreate);
-        getSecretValue(secretValue, "storage", secretName, "tls.crt", true);
-        io->write(0, secretValue.length(), secretValue.str());
-        io->close();
-
-        file.setown(createIFile(clientPrivateKeyFilename));
-        io.setown(file->open(IFOcreate));
-        getSecretValue(secretValue.clear(), "storage", secretName, "tls.key", true);
-        io->write(0, secretValue.length(), secretValue.str());
-        io->close();
-
-        file.setown(createIFile(caCertFilename));
-        io.setown(file->open(IFOcreate));
-        getSecretValue(secretValue.clear(), "storage", secretName, "ca.crt", true);
-        io->write(0, secretValue.length(), secretValue.str());
-        io->close();
-    }
-    setRpcSSLOptions(rpc, true, clientCertFilename, clientPrivateKeyFilename, caCertFilename, false);
+    Owned<IPropertyTree> secretPTree = getSecret("storage", secretName);
+    if (!secretPTree)
+        throw makeStringExceptionV(-1, "secret %s.%s not found", "storage", secretName);
+
+    StringBuffer certSecretBuf;
+    getSecretKeyValue(certSecretBuf, secretPTree, "tls.crt");
+
+    StringBuffer privKeySecretBuf;
+    getSecretKeyValue(privKeySecretBuf, secretPTree, "tls.key");
+
+    StringBuffer caCertFileBuf;
+    getSecretKeyValue(caCertFileBuf, secretPTree, "ca.crt");
+
+    setRpcSSLOptions(rpc, true, certSecretBuf.str(), privKeySecretBuf.str(), caCertFileBuf.str(), false);
 }
 
 static CriticalSection serviceLeaseMapCS;

diff --git a/system/jlib/jsecrets.cpp b/system/jlib/jsecrets.cpp
@@ -1066,6 +1066,25 @@ const MemoryAttr &getSecretUdpKey(bool required)
     return udpKey;
 }
 
+jlib_decl bool containsEmbeddedKey(const char *certificate)
+{
+    // look for any of:
+    // -----BEGIN PRIVATE KEY-----
+    // -----BEGIN RSA PRIVATE KEY-----
+    // -----BEGIN CERTIFICATE-----
+    // -----BEGIN PUBLIC KEY-----
+    // or maybe just:
+    // -----BEGIN
+
+    if ( (strstr(certificate, "-----BEGIN PRIVATE KEY-----")) ||
+         (strstr(certificate, "-----BEGIN RSA PRIVATE KEY-----")) ||
+         (strstr(certificate, "-----BEGIN PUBLIC KEY-----")) ||
+         (strstr(certificate, "-----BEGIN CERTIFICATE-----")) )
+        return true;
+
+    return false;
+}
+
 IPropertyTree *createTlsClientSecretInfo(const char *issuer, bool mutual, bool acceptSelfSigned, bool addCACert)
 {
     if (isEmptyString(issuer))

diff --git a/system/jlib/jsecrets.hpp b/system/jlib/jsecrets.hpp
@@ -36,6 +36,8 @@ extern jlib_decl bool getSecretValue(StringBuffer & result, const char *category
 extern jlib_decl void initSecretUdpKey();
 extern jlib_decl const MemoryAttr &getSecretUdpKey(bool required);
 
+extern jlib_decl bool containsEmbeddedKey(const char *certificate);
+
 extern jlib_decl IPropertyTree *queryTlsSecretInfo(const char *issuer);
 extern jlib_decl IPropertyTree *createTlsClientSecretInfo(const char *issuer, bool mutual, bool acceptSelfSigned, bool addCACert=true);
 

diff --git a/system/security/cryptohelper/cryptocommon.hpp b/system/security/cryptohelper/cryptocommon.hpp
@@ -21,10 +21,12 @@
 #if defined(_USE_OPENSSL)
 
 #include <opensslcommon.hpp>
+#include <openssl/ssl.h>
 #include <openssl/pem.h>
 #include <openssl/evp.h>
 #include <openssl/err.h>
 #include <openssl/rsa.h>
+#include <openssl/x509.h>
 
 #include "jiface.hpp"
 #include "jbuff.hpp"
@@ -39,7 +41,25 @@ jlib_decl void throwEVPExceptionV(int code, const char *format, ...) __attribute
 
 inline void voidBIOfree(BIO *bio) { BIO_free(bio); }
 inline void voidOpenSSLFree(void *m) { OPENSSL_free(m); }
+inline void voidSSLCTXfree(SSL_CTX *ctx)
+{
+    if (ctx)
+        SSL_CTX_free(ctx);
+}
+inline void voidX509StoreFree(X509_STORE *store)
+{
+    if (store)
+        X509_STORE_free(store);
+}
+inline void voidX509StkPopFree(STACK_OF(X509_INFO) *infoStk)
+{
+    if (infoStk)
+        sk_X509_INFO_pop_free(infoStk, X509_INFO_free);
+}
 
+typedef OwnedPtrCustomFree<X509_STORE, voidX509StoreFree> OwnedX509Store;
+typedef OwnedPtrCustomFree<STACK_OF(X509_INFO), voidX509StkPopFree> OwnedX509StkPtr;
+typedef OwnedPtrCustomFree<SSL_CTX, voidSSLCTXfree> OwnedSSLCTX;
 typedef OwnedPtrCustomFree<BIO, voidBIOfree> OwnedEVPBio;
 typedef OwnedPtrCustomFree<EVP_PKEY, EVP_PKEY_free> OwnedEVPPkey;
 typedef OwnedPtrCustomFree<EVP_PKEY_CTX, EVP_PKEY_CTX_free> OwnedEVPPkeyCtx;

diff --git a/system/security/securesocket/CMakeLists.txt b/system/security/securesocket/CMakeLists.txt
@@ -32,6 +32,7 @@ set (    SRCS
 include_directories ( 
          ./../../include 
          ./../../jlib 
+         ./../cryptohelper
          ./../../security/shared
          ${OPENSSL_INCLUDE_DIR}
     )