Skip to content

Commit

Permalink
Merge remote-tracking branch 'origin/candidate-8.12.x' into candidate…
Browse files Browse the repository at this point in the history 
…-9.0.x

Signed-off-by: Jake Smith <[email protected]>
  • Loading branch information
@jakesmith
jakesmith committed Aug 17, 2023
2 parents 5cb236f + 238ec7a commit ff7ee71
Show file tree
Hide file tree
Showing 6 changed files with 149 additions and 72 deletions.
15 changes: 15 additions & 0 deletions helm/hpcc/templates/_helpers.tpl
View file
Original file line number Diff line number Diff line change
Expand Up @@ -607,6 +607,21 @@ vaults:
{{- if index $vault "appRoleSecret" }}
appRoleSecret: {{ index $vault "appRoleSecret" }}
{{- end -}}
{{- if (hasKey $vault "retries") }}
retries: {{ $vault.retries }}
{{- end }}
{{- if (hasKey $vault "retryWait") }}
retryWait: {{ $vault.retryWait }}
{{- end }}
{{- if (hasKey $vault "connectTimeout") }}
connectTimeout: {{ $vault.connectTimeout }}
{{- end }}
{{- if (hasKey $vault "readTimeout") }}
readTimeout: {{ $vault.readTimeout }}
{{- end }}
{{- if (hasKey $vault "writeTimeout") }}
writeTimeout: {{ $vault.writeTimeout }}
{{- end }}
{{- end -}}
{{- end -}}
{{- end -}}
Expand Down
2 changes: 1 addition & 1 deletion helm/hpcc/templates/eclccserver.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -123,7 +123,7 @@ data:
{{- if not .disabled -}}
{{- $gitenv := get (fromYaml (include "hpcc.gatherGitEnvironment" (dict "root" $ "me" .))) "value" -}}
{{- $env := concat ($.Values.global.env | default list) (.env | default list) $gitenv -}}
{{- $secretsCategories := list "system" "codeVerify" "git" }}
{{- $secretsCategories := list "system" "codeVerify" "git" "storage" }}
{{- $commonCtx := dict "root" $ "me" . "includeCategories" (list "dll" "git" "debug") "secretsCategories" $secretsCategories "env" $env }}
{{- $configSHA := include "hpcc.getConfigSHA" ($commonCtx | merge (dict "configMapHelper" "hpcc.eclccServerConfigMap" "component" "eclccserver" "excludeKeys" "global,eclccserver.queues")) }}
apiVersion: apps/v1
Expand Down
20 changes: 20 additions & 0 deletions helm/hpcc/values.schema.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -845,6 +845,26 @@
"verify_server": {
"description": "optional relax server verification for trouble shooting",
"type": "boolean"
},
"retries": {
"description": "optional number of times to retry vault request in case of connect / socker error",
"type": "number"
},
"retryWait": {
"description": "optional wait time (in ms) between socket retries",
"type": "number"
},
"connectTimeout": {
"description": "optional timeout (in ms) for socket connect to vault",
"type": "number"
},
"readTimeout": {
"description": "optional timeout (in ms) for socket reading from vault",
"type": "number"
},
"writeTimeout": {
"description": "optional timeout (in ms) for socket writing to vault",
"type": "number"
}
},
"required": [ "name", "url" ],
Expand Down
15 changes: 8 additions & 7 deletions roxie/ccd/ccdqueue.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -305,15 +305,9 @@ void openMulticastSocket()
{
const char *desc = roxieMulticastEnabled ? "multicast" : "UDP";
multicastSocket.setown(ISocket::udp_create(ccdMulticastPort));
if (multicastTTL)
{
multicastSocket->set_ttl(multicastTTL);
DBGLOG("Roxie: %s TTL: %u", desc, multicastTTL);
}
else
DBGLOG("Roxie: %s TTL not set", desc);
multicastSocket->set_receive_buffer_size(udpMulticastBufferSize);
size32_t actualSize = multicastSocket->get_receive_buffer_size();

if (actualSize < udpMulticastBufferSize)
{
DBGLOG("Roxie: %s socket buffer size could not be set (requested=%d actual %d)", desc, udpMulticastBufferSize, actualSize);
Expand All @@ -323,6 +317,13 @@ void openMulticastSocket()
DBGLOG("Roxie: %s socket created port=%d sockbuffsize=%d actual %d", desc, ccdMulticastPort, udpMulticastBufferSize, actualSize);
if (roxieMulticastEnabled && !localAgent)
{
if (multicastTTL)
{
multicastSocket->set_ttl(multicastTTL);
DBGLOG("Roxie: %s TTL: %u", desc, multicastTTL);
}
else
DBGLOG("Roxie: %s TTL not set", desc);
Owned<const ITopologyServer> topology = getTopology();
for (unsigned channel : topology->queryChannels())
{
Expand Down
99 changes: 79 additions & 20 deletions system/jlib/jsecrets.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -282,6 +282,22 @@ static inline bool checkSecretExpired(unsigned created)

enum class VaultAuthType {unknown, k8s, appRole, token};

static void setTimevalMS(timeval &tv, time_t ms)
{
if (!ms)
tv = {0, 0};
else
{
tv.tv_sec = ms / 1000;
tv.tv_usec = (ms % 1000)*1000;
}
}

static bool isEmptyTimeval(const timeval &tv)
{
return (tv.tv_sec==0 && tv.tv_usec==0);
}

class CVault
{
private:
Expand All @@ -306,6 +322,11 @@ class CVault
time_t clientTokenExpiration = 0;
bool clientTokenRenewable = false;
bool verify_server = true;
unsigned retries = 3;
time_t retryWait = 1000;
timeval connectTimeout = {0, 0};
timeval readTimeout = {0, 0};
timeval writeTimeout = {0, 0};

public:
CVault(IPropertyTree *vault)
Expand All @@ -330,6 +351,13 @@ class CVault
PROGLOG("vault: namespace %s", vaultNamespace.str());
}
verify_server = vault->getPropBool("@verify_server", true);
retries = (unsigned) vault->getPropInt("@retries", 3);
retryWait = (time_t) vault->getPropInt("@retryWait", 1000);

setTimevalMS(connectTimeout, (time_t) vault->getPropInt("@connectTimeout"));
setTimevalMS(readTimeout, (time_t) vault->getPropInt("@readTimeout"));
setTimevalMS(writeTimeout, (time_t) vault->getPropInt("@writeTimeout"));

PROGLOG("Vault: httplib verify_server=%s", boolToStr(verify_server));

//set up vault client auth [appRole, clientToken (aka "token from the sky"), or kubernetes auth]
Expand Down Expand Up @@ -396,7 +424,7 @@ class CVault
void processClientTokenResponse(httplib::Result &res)
{
if (!res)
vaultAuthErrorV("missing login response, error %d", res.error());
vaultAuthErrorV("login communication error %d", res.error());
if (res.error()!=0)
OERRLOG("JSECRETS login calling HTTPLIB POST returned error %d", res.error());
if (res->status != 200)
Expand Down Expand Up @@ -438,6 +466,22 @@ class CVault

CVaultKind getVaultKind() const { return kind; }

void initClient(httplib::Client &cli, httplib::Headers &headers, unsigned &numRetries)
{
numRetries = retries;
cli.enable_server_certificate_verification(verify_server);
if (!isEmptyTimeval(connectTimeout))
cli.set_connection_timeout(connectTimeout.tv_sec, connectTimeout.tv_usec);
if (!isEmptyTimeval(readTimeout))
cli.set_read_timeout(readTimeout.tv_sec, readTimeout.tv_usec);
if (!isEmptyTimeval(writeTimeout))
cli.set_write_timeout(writeTimeout.tv_sec, writeTimeout.tv_usec);
if (username.length() && password.length())
cli.set_basic_auth(username, password);
if (vaultNamespace.length())
headers.emplace("X-Vault-Namespace", vaultNamespace.str());
}

//if we tried to use our token and it returned access denied it could be that we need to login again, or
// perhaps it could be specific permissions about the secret that was being accessed, I don't think we can tell the difference
void kubernetesLogin(bool permissionDenied)
Expand All @@ -454,14 +498,19 @@ class CVault
std::string json;
json.append("{\"jwt\": \"").append(login_token.str()).append("\", \"role\": \"").append(k8sAuthRole.str()).append("\"}");
httplib::Client cli(schemeHostPort.str());
cli.enable_server_certificate_verification(verify_server);

if (username.length() && password.length())
cli.set_basic_auth(username, password);
httplib::Headers headers;
if (vaultNamespace.length())
headers.emplace("X-Vault-Namespace", vaultNamespace.str());

unsigned numRetries = 0;
initClient(cli, headers, numRetries);
httplib::Result res = cli.Post("/v1/auth/kubernetes/login", headers, json, "application/json");
while (!res && numRetries--)
{
OERRLOG("Retrying vault %s kubernetes auth, communication error %d", name.str(), res.error());
if (retryWait)
Sleep(retryWait);
res = cli.Post("/v1/auth/kubernetes/login", headers, json, "application/json");
}

processClientTokenResponse(res);
}
//if we tried to use our token and it returned access denied it could be that we need to login again, or
Expand All @@ -485,15 +534,19 @@ class CVault
json.append("{\"role_id\": \"").append(appRoleId).append("\", \"secret_id\": \"").append(appRoleSecretId).append("\"}");

httplib::Client cli(schemeHostPort.str());
cli.enable_server_certificate_verification(verify_server);

if (username.length() && password.length())
cli.set_basic_auth(username, password);
httplib::Headers headers;
if (vaultNamespace.length())
headers.emplace("X-Vault-Namespace", vaultNamespace.str());

unsigned numRetries = 0;
initClient(cli, headers, numRetries);
httplib::Result res = cli.Post("/v1/auth/approle/login", headers, json, "application/json");
while (!res && numRetries--)
{
OERRLOG("Retrying vault %s appRole auth, communication error %d", name.str(), res.error());
if (retryWait)
Sleep(retryWait);
res = cli.Post("/v1/auth/approle/login", headers, json, "application/json");
}

processClientTokenResponse(res);
}
void checkAuthentication(bool permissionDenied)
Expand Down Expand Up @@ -549,18 +602,20 @@ class CVault
checkAuthentication(permissionDenied);

httplib::Client cli(schemeHostPort.str());
cli.enable_server_certificate_verification(verify_server);

if (username.length() && password.length())
cli.set_basic_auth(username.str(), password.str());

httplib::Headers headers = {
{ "X-Vault-Token", clientToken.str() }
};
if (vaultNamespace.length())
headers.emplace("X-Vault-Namespace", vaultNamespace.str());

unsigned numRetries = 0;
initClient(cli, headers, numRetries);
httplib::Result res = cli.Get(location, headers);
while (!res && numRetries--)
{
OERRLOG("Retrying vault %s get secret, communication error %d location %s", name.str(), res.error(), location ? location : "null");
if (retryWait)
Sleep(retryWait);
res = cli.Get(location, headers);
}

if (res)
{
Expand All @@ -578,6 +633,10 @@ class CVault
return requestSecretAtLocation(rkind, content, location, secret, version, true);
OERRLOG("Vault %s permission denied accessing secret (check namespace=%s?) %s.%s location %s [%d](%d) - response: %s", name.str(), vaultNamespace.str(), secret, version ? version : "", location ? location : "null", res->status, res.error(), res->body.c_str());
}
else if (res->status == 404)
{
OERRLOG("Vault %s secret not found %s.%s location %s", name.str(), secret, version ? version : "", location ? location : "null");
}
else
{
OERRLOG("Vault %s error accessing secret %s.%s location %s [%d](%d) - response: %s", name.str(), secret, version ? version : "", location ? location : "null", res->status, res.error(), res->body.c_str());
Expand Down
70 changes: 26 additions & 44 deletions system/security/LdapSecurity/ldapconnection.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -453,7 +453,6 @@ class CLdapConfig : implements ILdapConfig, public CInterface
const char * ldapDomain = cfg->queryProp(".//@ldapDomain");
for (int numHosts=0; numHosts < getHostCount(); numHosts++)
{
getLdapHost(hostbuf);
unsigned port = strieq("ldaps",m_protocol) ? m_ldap_secure_port : m_ldapport;

//Guesstimate system user baseDN based on config settings. It will be used if anonymous bind fails
Expand All @@ -465,16 +464,14 @@ class CLdapConfig : implements ILdapConfig, public CInterface

for(int retries = 0; retries <= LDAPSEC_MAX_RETRIES; retries++)
{
getLdapHost(hostbuf);//get next available AD, as it may have changed
rc = LdapUtils::getServerInfo(hostbuf.str(), sysUserDN.str(), m_sysuser_password.str(), m_protocol, port, m_cipherSuite, dcbuf, m_serverType, ldapDomain, m_timeout);
if(!LdapServerDown(rc) || retries >= LDAPSEC_MAX_RETRIES)
if(rc != LDAP_TIMEOUT || retries >= LDAPSEC_MAX_RETRIES)
break;
sleep(LDAPSEC_RETRY_WAIT);
if(retries < LDAPSEC_MAX_RETRIES)
{
DBGLOG("LDAP AD Server %s temporarily unreachable for user %s on port %d, retrying...", hostbuf.str(), sysUserDN.str(), port);
}
DBGLOG("LDAP AD Server %s temporarily unreachable for user %s on port %d, retrying...", hostbuf.str(), sysUserDN.str(), port);
}
if (rc != LDAP_SUCCESS)
if(LdapServerDown(rc))
{
rejectHost(hostbuf);
}
Expand Down Expand Up @@ -986,41 +983,21 @@ class CLdapConnection : implements ILdapConnection, public CInterface
StringBuffer hostbuf;
for (int numHosts=0; numHosts < m_ldapconfig->getHostCount(); numHosts++)
{
m_ldapconfig->getLdapHost(hostbuf);

for(int retries = 0; retries <= LDAPSEC_MAX_RETRIES; retries++)
{
m_ldapconfig->getLdapHost(hostbuf);//get next available AD, as it may have changed
rc = connect(hostbuf.str(), proto);
if(!LdapServerDown(rc) || retries > LDAPSEC_MAX_RETRIES)
if(rc == LDAP_SUCCESS)
return true;
if(rc != LDAP_TIMEOUT || retries >= LDAPSEC_MAX_RETRIES)
break;
sleep(LDAPSEC_RETRY_WAIT);
if(retries < LDAPSEC_MAX_RETRIES)
DBGLOG("Server temporarily unreachable, retrying ...");
}

if(rc == LDAP_SERVER_DOWN)
{
StringBuffer dc;
LdapUtils::getDcName(m_ldapconfig->getDomain(), dc);
if(dc.length() > 0)
{
WARNLOG("Using automatically obtained LDAP Server %s", dc.str());
rc = connect(dc.str(), proto);
}
}

if (rc != LDAP_SUCCESS)
{
m_ldapconfig->rejectHost(hostbuf);
DBGLOG("Server %s temporarily unreachable, retrying ...", hostbuf.str());
}
else
break;
m_ldapconfig->rejectHost(hostbuf);
}

if(rc == LDAP_SUCCESS)
return true;
else
return false;
return false;
}

virtual LDAP* getLd()
Expand Down Expand Up @@ -1953,13 +1930,12 @@ class CLdapClient : implements ILdapClient, public CInterface
ldap_memfree(userdn);

StringBuffer hostbuf;
m_ldapconfig->getLdapHost(hostbuf);
int rc = LDAP_SERVER_DOWN;
char *ldap_errstring=NULL;

for(int retries = 0; retries <= LDAPSEC_MAX_RETRIES; retries++)
{
DBGLOG("LdapBind for user %s (retries=%d).", username, retries);
m_ldapconfig->getLdapHost(hostbuf);//get next available AD, as it may have changed
DBGLOG("LdapBind for user %s (retries=%d) on host %s.", username, retries, hostbuf.str());
{
LDAP* user_ld = LdapUtils::LdapInit(m_ldapconfig->getProtocol(), hostbuf.str(), m_ldapconfig->getLdapPort(), m_ldapconfig->getLdapSecurePort(), m_ldapconfig->getCipherSuite());
rc = LdapUtils::LdapBind(user_ld, m_ldapconfig->getLdapTimeout(), m_ldapconfig->getDomain(), username, password, userdnbuf.str(), m_ldapconfig->getServerType(), m_ldapconfig->getAuthMethod());
Expand All @@ -1968,16 +1944,22 @@ class CLdapClient : implements ILdapClient, public CInterface
LDAP_UNBIND(user_ld);
}
DBGLOG("finished LdapBind for user %s, rc=%d", username, rc);
if(!LdapServerDown(rc) || retries > LDAPSEC_MAX_RETRIES)

if(rc==LDAP_SERVER_DOWN || rc==LDAP_UNAVAILABLE)
{
m_ldapconfig->rejectHost(hostbuf);
continue;//try again with next configured LDAP host
}
else if(rc==LDAP_TIMEOUT && retries < LDAPSEC_MAX_RETRIES)
{
sleep(LDAPSEC_RETRY_WAIT);
DBGLOG("Server %s temporarily unreachable, retrying ...", hostbuf.str());
}
else
break;
sleep(LDAPSEC_RETRY_WAIT);
if(retries < LDAPSEC_MAX_RETRIES)
DBGLOG("Server temporarily unreachable, retrying ...");
// Retrying next ldap sever, might be the same server
m_ldapconfig->getLdapHost(hostbuf);
}

if(rc == LDAP_SERVER_DOWN)
if(LdapServerDown(rc))
{
StringBuffer dc;
LdapUtils::getDcName(NULL, dc);
Expand Down

0 comments on commit ff7ee71

Please sign in to comment.