Skip to content

Commit

Permalink
HPCC-30390 move k8s functions into jcontainerized
Browse files Browse the repository at this point in the history 
Signed-off-by: Jake Smith <[email protected]>
  • Loading branch information
@jakesmith
jakesmith committed Sep 28, 2023
1 parent 0a7ee5b commit 42fbde6
Show file tree
Hide file tree
Showing 7 changed files with 110 additions and 106 deletions.
3 changes: 2 additions & 1 deletion dali/base/dautils.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@

#include "platform.h"
#include "jlib.hpp"
#include "jcontainerized.hpp"
#include "jstring.hpp"
#include "jfile.hpp"
#include "jmisc.hpp"
Expand Down Expand Up @@ -3628,7 +3629,7 @@ void remapGroupsToDafilesrv(IPropertyTree *file, INamedGroupStore *resolver)
CriticalBlock b(dafileSrvNodeCS);
if (nullptr == dafileSrvNode)
{
auto externalService = getDafileServiceFromConfig("directio");
auto externalService = k8s::getDafileServiceFromConfig("directio");
VStringBuffer dafilesrvEpStr("%s:%u", externalService.first.c_str(), externalService.second);
dafileSrvNode.setown(createINode(dafilesrvEpStr));
}
Expand Down
3 changes: 2 additions & 1 deletion esp/clients/ws_dfsclient/ws_dfsclient.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@
#include <vector>

#include "jliball.hpp"
#include "jcontainerized.hpp"
#include "jflz.hpp"
#include "jsecrets.hpp"
#include "seclib.hpp"
Expand Down Expand Up @@ -726,7 +727,7 @@ IDFSFile *lookupDFSFile(const char *logicalName, AccessMode accessMode, unsigned
const IPropertyTree &eclWatch = eclWatchServices->query();
StringBuffer eclWatchName;
eclWatch.getProp("@name", eclWatchName);
auto result = getExternalService(eclWatchName);
auto result = k8s::getExternalService(eclWatchName);
if (result.first.empty())
throw makeStringExceptionV(-1, "dfs '%s': service not found", eclWatchName.str());
if (0 == result.second)
Expand Down
5 changes: 3 additions & 2 deletions esp/services/ws_dfu/ws_dfuService.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@

#include <math.h>

#include "jcontainerized.hpp"
#include "daclient.hpp"
#include "daft.hpp"
#include "daftcfg.hpp"
Expand Down Expand Up @@ -6116,7 +6117,7 @@ void CWsDfuEx::dFUFileAccessCommon(IEspContext &context, const CDfsLogicalFileNa
if (!info)
throw makeStringExceptionV(-1, "dFUFileAccessCommon: file signing certificate ('%s') not defined in configuration.", keyPairName.str());

auto externalService = getDafileServiceFromConfig("stream");
auto externalService = k8s::getDafileServiceFromConfig("stream");
dafilesrvHost.set(externalService.first.c_str());
port = externalService.second;
secure = true;
Expand Down Expand Up @@ -6498,7 +6499,7 @@ bool CWsDfuEx::onDFUFileCreateV2(IEspContext &context, IEspDFUFileCreateV2Reques
fileDesc.setown(createFileDescriptor(tempFileName, planeName, numParts));
numParts = fileDesc->numParts();

auto externalService = getDafileServiceFromConfig("stream");
auto externalService = k8s::getDafileServiceFromConfig("stream");
dafilesrvHost.set(externalService.first.c_str());
port = externalService.second;
secure = true;
Expand Down
96 changes: 96 additions & 0 deletions system/jlib/jcontainerized.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@
limitations under the License.
############################################################################## */

#include "jerror.hpp"
#include "jexcept.hpp"
#include "jfile.hpp"
#include "jmisc.hpp"
Expand Down Expand Up @@ -292,6 +293,101 @@ std::vector<std::vector<std::string>> getPodNodes(const char *selector)
}
}

void runKubectlCommand(const char *title, const char *cmd, const char *input, StringBuffer *output)
{
#ifndef _CONTAINERIZED
UNIMPLEMENTED_X("runKubectlCommand");
#endif
// NB: will fire an exception if command fails (returns non-zero exit code)

StringBuffer _output, error;
if (!output)
output = &_output;
unsigned ret = runExternalCommand(title, *output, error, cmd, input, ".", nullptr);
if (output->length())
MLOG(MCdebugInfo, unknownJob, "%s: ret=%u, stdout=%s", cmd, ret, output->trimRight().str());
if (error.length())
MLOG(MCdebugError, unknownJob, "%s: ret=%u, stderr=%s", cmd, ret, error.trimRight().str());
if (ret)
{
if (input)
MLOG(MCdebugError, unknownJob, "Using input %s", input);
throw makeStringExceptionV(0, "Failed to run %s: error %u: %s", cmd, ret, error.str());
}
}

static CTimeLimitedCache<std::string, std::pair<std::string, unsigned>> externalServiceCache;
static CriticalSection externalServiceCacheCrit;
std::pair<std::string, unsigned> getExternalService(const char *serviceName)
{
#ifndef _CONTAINERIZED
UNIMPLEMENTED_X("getExternalService");
#endif
{
CriticalBlock b(externalServiceCacheCrit);
std::pair<std::string, unsigned> cachedExternalSevice;
if (externalServiceCache.get(serviceName, cachedExternalSevice))
return cachedExternalSevice;
}

StringBuffer output;
try
{
VStringBuffer getServiceCmd("kubectl get svc --selector=server=%s --output=jsonpath={.items[0].status.loadBalancer.ingress[0].hostname},{.items[0].status.loadBalancer.ingress[0].ip},{.items[0].spec.ports[0].port}", serviceName);
k8s::runKubectlCommand("get-external-service", getServiceCmd, nullptr, &output);
}
catch (IException *e)
{
EXCLOG(e);
VStringBuffer exceptionText("Failed to get external service for '%s'. Error: [%d, ", serviceName, e->errorCode());
e->errorMessage(exceptionText).append("]");
e->Release();
throw makeStringException(-1, exceptionText);
}
StringArray fields;
fields.appendList(output, ",");

// NB: add even if no result, want non-result to be cached too
std::string host, port;
if (fields.ordinality() == 3) // hostname,ip,port. NB: hostname may be missing, but still present as a blank field
{
host = fields.item(0); // hostname
if (0 == host.length())
host = fields.item(1); // ip
port = fields.item(2);
}
auto servicePair = std::make_pair(host, atoi(port.c_str()));
externalServiceCache.add(serviceName, servicePair);
return servicePair;
}

std::pair<std::string, unsigned> getDafileServiceFromConfig(const char *application)
{
#ifndef _CONTAINERIZED
UNIMPLEMENTED_X("getDafileServiceFromConfig");
#endif
/* NB: For now expect 1 dafilesrv in configuration only
* We could have multiple dafilesrv services with e.g. different specs./replicas etc. that
* serviced different planes. At the moment dafilesrv mounts all data planes.
*/
VStringBuffer serviceXPath("services[@type='%s']", application);
Owned<IPropertyTreeIterator> dafilesrvServices = getGlobalConfigSP()->getElements(serviceXPath);
if (!dafilesrvServices->first())
throw makeStringExceptionV(JLIBERR_K8sServiceError, "dafilesrv service '%s' not defined or disabled", application);
const IPropertyTree &dafilesrv = dafilesrvServices->query();
if (!dafilesrv.getPropBool("@public"))
throw makeStringExceptionV(JLIBERR_K8sServiceError, "dafilesrv service '%s' has no public service defined", application);
StringBuffer dafilesrvName;
dafilesrv.getProp("@name", dafilesrvName);
auto externalService = getExternalService(dafilesrvName);
if (externalService.first.empty())
throw makeStringExceptionV(JLIBERR_K8sServiceError, "dafilesrv service '%s' - external service '%s' not found", application, dafilesrvName.str());
if (0 == externalService.second)
throw makeStringExceptionV(JLIBERR_K8sServiceError, "dafilesrv service '%s' - external service '%s' port not defined", application, dafilesrvName.str());
return externalService;
}


static unsigned podInfoInitCBId = 0;
MODULE_INIT(INIT_PRIORITY_STANDARD)
{
Expand Down
7 changes: 7 additions & 0 deletions system/jlib/jcontainerized.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -37,5 +37,12 @@ jlib_decl void waitJob(const char *componentName, const char *job, unsigned pend
jlib_decl bool applyYaml(const char *componentName, const char *wuid, const char *job, const char *resourceType, const std::list<std::pair<std::string, std::string>> &extraParams, bool optional, bool autoCleanup);
jlib_decl void runJob(const char *componentName, const char *wuid, const char *job, const std::list<std::pair<std::string, std::string>> &extraParams={});

extern jlib_decl void runKubectlCommand(const char *title, const char *cmd, const char *input, StringBuffer *output);

// return the k8s external host and port for serviceName
extern jlib_decl std::pair<std::string, unsigned> getExternalService(const char *serviceName);

extern jlib_decl std::pair<std::string, unsigned> getDafileServiceFromConfig(const char *application);


}
94 changes: 0 additions & 94 deletions system/jlib/jmisc.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1004,100 +1004,6 @@ jlib_decl char **getSystemEnv()
}


void runKubectlCommand(const char *title, const char *cmd, const char *input, StringBuffer *output)
{
#ifndef _CONTAINERIZED
UNIMPLEMENTED_X("runKubectlCommand");
#endif
// NB: will fire an exception if command fails (returns non-zero exit code)

StringBuffer _output, error;
if (!output)
output = &_output;
unsigned ret = runExternalCommand(title, *output, error, cmd, input, ".", nullptr);
if (output->length())
MLOG(MCdebugInfo, unknownJob, "%s: ret=%u, stdout=%s", cmd, ret, output->trimRight().str());
if (error.length())
MLOG(MCdebugError, unknownJob, "%s: ret=%u, stderr=%s", cmd, ret, error.trimRight().str());
if (ret)
{
if (input)
MLOG(MCdebugError, unknownJob, "Using input %s", input);
throw makeStringExceptionV(0, "Failed to run %s: error %u: %s", cmd, ret, error.str());
}
}

static CTimeLimitedCache<std::string, std::pair<std::string, unsigned>> externalServiceCache;
static CriticalSection externalServiceCacheCrit;
std::pair<std::string, unsigned> getExternalService(const char *serviceName)
{
#ifndef _CONTAINERIZED
UNIMPLEMENTED_X("getExternalService");
#endif
{
CriticalBlock b(externalServiceCacheCrit);
std::pair<std::string, unsigned> cachedExternalSevice;
if (externalServiceCache.get(serviceName, cachedExternalSevice))
return cachedExternalSevice;
}

StringBuffer output;
try
{
VStringBuffer getServiceCmd("kubectl get svc --selector=server=%s --output=jsonpath={.items[0].status.loadBalancer.ingress[0].hostname},{.items[0].status.loadBalancer.ingress[0].ip},{.items[0].spec.ports[0].port}", serviceName);
runKubectlCommand("get-external-service", getServiceCmd, nullptr, &output);
}
catch (IException *e)
{
EXCLOG(e);
VStringBuffer exceptionText("Failed to get external service for '%s'. Error: [%d, ", serviceName, e->errorCode());
e->errorMessage(exceptionText).append("]");
e->Release();
throw makeStringException(-1, exceptionText);
}
StringArray fields;
fields.appendList(output, ",");

// NB: add even if no result, want non-result to be cached too
std::string host, port;
if (fields.ordinality() == 3) // hostname,ip,port. NB: hostname may be missing, but still present as a blank field
{
host = fields.item(0); // hostname
if (0 == host.length())
host = fields.item(1); // ip
port = fields.item(2);
}
auto servicePair = std::make_pair(host, atoi(port.c_str()));
externalServiceCache.add(serviceName, servicePair);
return servicePair;
}

std::pair<std::string, unsigned> getDafileServiceFromConfig(const char *application)
{
#ifndef _CONTAINERIZED
UNIMPLEMENTED_X("getDafileServiceFromConfig");
#endif
/* NB: For now expect 1 dafilesrv in configuration only
* We could have multiple dafilesrv services with e.g. different specs./replicas etc. that
* serviced different planes. At the moment dafilesrv mounts all data planes.
*/
VStringBuffer serviceXPath("services[@type='%s']", application);
Owned<IPropertyTreeIterator> dafilesrvServices = getGlobalConfigSP()->getElements(serviceXPath);
if (!dafilesrvServices->first())
throw makeStringExceptionV(JLIBERR_K8sServiceError, "dafilesrv service '%s' not defined or disabled", application);
const IPropertyTree &dafilesrv = dafilesrvServices->query();
if (!dafilesrv.getPropBool("@public"))
throw makeStringExceptionV(JLIBERR_K8sServiceError, "dafilesrv service '%s' has no public service defined", application);
StringBuffer dafilesrvName;
dafilesrv.getProp("@name", dafilesrvName);
auto externalService = getExternalService(dafilesrvName);
if (externalService.first.empty())
throw makeStringExceptionV(JLIBERR_K8sServiceError, "dafilesrv service '%s' - external service '%s' not found", application, dafilesrvName.str());
if (0 == externalService.second)
throw makeStringExceptionV(JLIBERR_K8sServiceError, "dafilesrv service '%s' - external service '%s' port not defined", application, dafilesrvName.str());
return externalService;
}

// checks if 'name' is an internal environment variable (prefixed with 'HPCC_')
// if !matches : returns null
// if matches : returns allocated copy of configured value or defaultValue if not set.
Expand Down
8 changes: 0 additions & 8 deletions system/jlib/jmisc.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -332,14 +332,6 @@ extern jlib_decl char *mkdtemp(char *_template);

extern jlib_decl char **getSystemEnv();


extern jlib_decl void runKubectlCommand(const char *title, const char *cmd, const char *input, StringBuffer *output);

// return the k8s external host and port for serviceName
extern jlib_decl std::pair<std::string, unsigned> getExternalService(const char *serviceName);

extern jlib_decl std::pair<std::string, unsigned> getDafileServiceFromConfig(const char *application);

extern jlib_decl char *getHPCCEnvVal(const char *name, const char *defaultValue);

#endif

0 comments on commit 42fbde6

Please sign in to comment.