From 7a351c0f8e1c4d31088c4d5fcbb62fea410362c3 Mon Sep 17 00:00:00 2001 From: Richard Chapman Date: Wed, 24 Jan 2024 17:07:47 +0000 Subject: [PATCH 01/24] HPCC-31171 Encrypt on worker threads rather than UDP send thread Signed-off-by: Richard Chapman --- roxie/ccd/ccdmain.cpp | 1 + roxie/udplib/udplib.hpp | 2 +- roxie/udplib/udpsha.cpp | 1 + roxie/udplib/udptrs.cpp | 14 ++++++++++++-- 4 files changed, 15 insertions(+), 3 deletions(-) diff --git a/roxie/ccd/ccdmain.cpp b/roxie/ccd/ccdmain.cpp index 40f01b73e6b..3545d613b85 100644 --- a/roxie/ccd/ccdmain.cpp +++ b/roxie/ccd/ccdmain.cpp @@ -1112,6 +1112,7 @@ int CCD_API roxie_main(int argc, const char *argv[], const char * defaultYaml) udpAllowAsyncPermits = topology->getPropBool("@udpAllowAsyncPermits", udpAllowAsyncPermits); udpMinSlotsPerSender = topology->getPropInt("@udpMinSlotsPerSender", udpMinSlotsPerSender); udpRemoveDuplicatePermits = topology->getPropBool("@udpRemoveDuplicatePermits", udpRemoveDuplicatePermits); + udpEncryptOnSendThread = topology->getPropBool("expert/@udpEncryptOnSendThread", udpEncryptOnSendThread); unsigned __int64 defaultNetworkSpeed = 10 * U64C(0x40000000); // 10Gb/s unsigned __int64 networkSpeed = topology->getPropInt64("@udpNetworkSpeed", defaultNetworkSpeed); // only used to sanity check the different udp options diff --git a/roxie/udplib/udplib.hpp b/roxie/udplib/udplib.hpp index 59c1d79d7a6..15bf8e1f569 100644 --- a/roxie/udplib/udplib.hpp +++ b/roxie/udplib/udplib.hpp @@ -230,7 +230,7 @@ extern UDPLIB_API unsigned udpStatsReportInterval; extern UDPLIB_API bool udpAdjustThreadPriorities; extern UDPLIB_API bool udpAllowAsyncPermits; extern UDPLIB_API bool udpRemoveDuplicatePermits; - +extern UDPLIB_API bool udpEncryptOnSendThread; //Should be in ccd extern UDPLIB_API unsigned multicastTTL; diff --git a/roxie/udplib/udpsha.cpp b/roxie/udplib/udpsha.cpp index de33bc6243e..a2ac2986ed8 100644 --- a/roxie/udplib/udpsha.cpp +++ b/roxie/udplib/udpsha.cpp @@ -76,6 +76,7 @@ bool udpAdjustThreadPriorities = false; // Adjust the priorities for the UDP rec // Enabling tends to cause a big rise in context switches from other threads, so disabled by default bool udpAllowAsyncPermits = false; // Allow requests to send more data to overtake the data packets that are being sent. bool udpRemoveDuplicatePermits = true; +bool udpEncryptOnSendThread = false; unsigned multicastTTL = 1; diff --git a/roxie/udplib/udptrs.cpp b/roxie/udplib/udptrs.cpp index 638826f81dc..dfcf205aa73 100644 --- a/roxie/udplib/udptrs.cpp +++ b/roxie/udplib/udptrs.cpp @@ -554,7 +554,7 @@ class UdpReceiverEntry : public IUdpReceiverEntry } try { - if (encrypted && !resending) + if (encrypted && !resending && udpEncryptOnSendThread) { length -= sizeof(UdpPacketHeader); const MemoryAttr &udpkey = getSecretUdpKey(true); @@ -1188,11 +1188,21 @@ class CSendManager : implements ISendManager, public CInterface // Interface ISendManager - virtual void writeOwn(IUdpReceiverEntry &receiver, DataBuffer *buffer, unsigned len, unsigned queue) override + virtual void writeOwn(IUdpReceiverEntry &receiver, DataBuffer *buffer, unsigned length, unsigned queue) override { // NOTE: takes ownership of the DataBuffer assert(queue < numQueues); assert(buffer); + if (encrypted && !udpEncryptOnSendThread) + { + UdpPacketHeader *header = (UdpPacketHeader*) buffer->data; + length -= sizeof(UdpPacketHeader); + const MemoryAttr &udpkey = getSecretUdpKey(true); + size_t encryptedLength = aesEncryptInPlace(udpkey.get(), udpkey.length(), buffer->data + sizeof(UdpPacketHeader), length, DATA_PAYLOAD - sizeof(UdpPacketHeader)); + length = encryptedLength + sizeof(UdpPacketHeader); + header->length = length; + assertex(length <= DATA_PAYLOAD); + } static_cast(receiver).pushData(queue, buffer); } From c310e390bbf1df5e9591409c3b4a55a352c66286 Mon Sep 17 00:00:00 2001 From: Gavin Halliday Date: Wed, 10 Jan 2024 15:15:14 +0000 Subject: [PATCH 02/24] HPCC-31095 Report an error if eclcc fails to update the local copy of a git repo Signed-off-by: Gavin Halliday --- ecl/eclcc/eclcc.cpp | 11 ++++++++++- ecl/hql/hqlerrors.hpp | 4 ++++ ecl/hql/hqlrepository.cpp | 25 +++++++++++++++++++++---- ecl/hql/hqlrepository.hpp | 6 +++++- 4 files changed, 40 insertions(+), 6 deletions(-) diff --git a/ecl/eclcc/eclcc.cpp b/ecl/eclcc/eclcc.cpp index 878b574c332..bfde52da235 100644 --- a/ecl/eclcc/eclcc.cpp +++ b/ecl/eclcc/eclcc.cpp @@ -1252,6 +1252,12 @@ void EclCC::processSingleQuery(const EclRepositoryManager & localRepositoryManag } IErrorReceiver & errorProcessor = *severityMapper; + //Associate the error handler - so that failures to fetch from git can be reported as errors, but also mapped + //to warnings to ensure that automated tasks do not fail because the could not connect to git. + localRepositoryManager.setErrorReceiver(&errorProcessor); + //Ensure the error processor is cleared up when we exit this function + COnScopeExit scoped([&]() { localRepositoryManager.setErrorReceiver(NULL); }); + //All dlls/exes are essentially cloneable because you may be running multiple instances at once //The only exception would be a dll created for a one-time query. (Currently handled by eclserver.) instance.wu->setCloneable(true); @@ -1507,7 +1513,10 @@ void EclCC::processSingleQuery(const EclRepositoryManager & localRepositoryManag { StringBuffer s; e->errorMessage(s); - errorProcessor.reportError(3, s.str(), defaultErrorPathname, 1, 0, 0); + unsigned errorCode = e->errorCode(); + if ((errorCode < HQL_ERROR_START) || (errorCode > HQL_ERROR_END)) + errorCode = ERR_UNKNOWN_EXCEPTION; + errorProcessor.reportError(errorCode, s.str(), defaultErrorPathname, 1, 0, 0); e->Release(); } diff --git a/ecl/hql/hqlerrors.hpp b/ecl/hql/hqlerrors.hpp index 6c9394b3e5a..3ae79a49b13 100644 --- a/ecl/hql/hqlerrors.hpp +++ b/ecl/hql/hqlerrors.hpp @@ -442,6 +442,10 @@ #define ERR_INVALID_PROBABILITY 2403 #define ERR_DEFAULT_VIRTUAL_CLASH 2404 #define ERR_EMBEDERROR 2405 +#define ERR_FAIL_CLONE_REPO 2406 +#define ERR_FAIL_UPDATE_REPO 2407 +#define ERR_CANNOT_RESOLVE_BRANCH 2408 +#define ERR_UNKNOWN_EXCEPTION 2409 #define ERR_CPP_COMPILE_ERROR 2999 diff --git a/ecl/hql/hqlrepository.cpp b/ecl/hql/hqlrepository.cpp index 75ca659e302..8d2c2d8aae1 100644 --- a/ecl/hql/hqlrepository.cpp +++ b/ecl/hql/hqlrepository.cpp @@ -821,6 +821,7 @@ IEclSourceCollection * EclRepositoryManager::resolveGitCollection(const char * r } bool ok = false; + Owned error; CCycleTimer gitDownloadTimer; if (alreadyExists) { @@ -828,7 +829,10 @@ IEclSourceCollection * EclRepositoryManager::resolveGitCollection(const char * r { unsigned retCode = runGitCommand(nullptr, "fetch origin", repoPath, true); if (retCode != 0) - DBGLOG("Failed to download the latest version of %s", defaultUrl); + { + VStringBuffer msg("Failed to download the latest version of '%s' error code (%u)", defaultUrl, retCode); + error.setown(createError(CategoryError, SeverityError, ERR_FAIL_UPDATE_REPO, msg.str(), nullptr, 0, 0, 0)); + } } ok = true; @@ -844,17 +848,30 @@ IEclSourceCollection * EclRepositoryManager::resolveGitCollection(const char * r VStringBuffer params("clone %s \"%s\" --no-checkout", repoUrn.str(), repo.str()); unsigned retCode = runGitCommand(nullptr, params, options.eclRepoPath, true); if (retCode != 0) - throw makeStringExceptionV(99, "Failed to clone dependency '%s'", defaultUrl); + { + VStringBuffer msg("Failed to clone dependency '%s' error code (%u)", defaultUrl, retCode); + error.setown(createError(CategoryError, SeverityError, ERR_FAIL_CLONE_REPO, msg.str(), nullptr, 0, 0, 0)); + } ok = true; } } gitDownloadCycles += gitDownloadTimer.elapsedCycles(); + if (error) + { + if (errorReceiver) + { + error.setown(errorReceiver->mapError(error)); // MORE: This mapping should really be done within reportError() + errorReceiver->report(error); + } + else + throw error.getClear(); + } if (!ok) throw makeStringExceptionV(99, "Cannot locate the source code for dependency '%s'. --fetchrepos not enabled", defaultUrl); if (startsWith(version, "semver:")) - throw makeStringExceptionV(99, "Semantic versioning not yet supported for dependency '%s'.", defaultUrl); + throw makeStringExceptionV(ERR_CANNOT_RESOLVE_BRANCH, "Semantic versioning not yet supported for dependency '%s'.", defaultUrl); // Really the version should be a SHA, but for flexibility version could be a sha, a tag or a branch (on origin). // Check for a remote branch first - because it appears that when git clones a repo, it creates a local branch for @@ -879,7 +896,7 @@ IEclSourceCollection * EclRepositoryManager::resolveGitCollection(const char * r DBGLOG("Version '%s' resolved to sha '%s'", version.str(), sha.str()); if (sha.isEmpty()) - throw makeStringExceptionV(99, "Branch/tag '%s' could not be found for dependency '%s'.", version.str(), defaultUrl); + throw makeStringExceptionV(ERR_CANNOT_RESOLVE_BRANCH, "Branch/tag '%s' could not be found for dependency '%s'.", version.str(), defaultUrl); path.append(repoPath).appendf("/.git/{%s", sha.str()); if (options.gitUser) diff --git a/ecl/hql/hqlrepository.hpp b/ecl/hql/hqlrepository.hpp index 34600e17dcc..a69febb6159 100644 --- a/ecl/hql/hqlrepository.hpp +++ b/ecl/hql/hqlrepository.hpp @@ -74,7 +74,10 @@ class HQL_API EclRepositoryManager } IEclSourceCollection * resolveGitCollection(const char * repoPath, const char * defaultUrl); - + void setErrorReceiver(IErrorReceiver * _errorReceiver) const + { + errorReceiver = _errorReceiver; + } protected: IEclRepository * createNewSourceFileEclRepository(IErrorReceiver *errs, const char * path, unsigned flags, unsigned trace, bool includeInArchive); IEclRepository * createGitRepository(IErrorReceiver *errs, const char * path, const char * urn, unsigned flags, unsigned trace, bool includeInArchive); @@ -85,6 +88,7 @@ class HQL_API EclRepositoryManager IEclPackage * queryRepository(IIdAtom * name, const char * defaultUrl, IEclSourceCollection * overrideSource, bool includeDefinitions); private: + mutable IErrorReceiver * errorReceiver = nullptr; // mutable to allow const methods to set it, it logically doesn't change the object using DependencyInfo = std::pair>; CIArrayOf repos; std::vector dependencies; From 1fa9a5ad4d2f05fc07cac2794cc62d0f9c27735f Mon Sep 17 00:00:00 2001 From: Jim DeFabia Date: Thu, 22 Feb 2024 11:09:08 -0500 Subject: [PATCH 03/24] HPCC-29889 Fix typo in storage plane name Signed-off-by: Jim DeFabia --- .../ContainerizedHPCC/ContainerizedMods/LocalDeployment.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/EN_US/ContainerizedHPCC/ContainerizedMods/LocalDeployment.xml b/docs/EN_US/ContainerizedHPCC/ContainerizedMods/LocalDeployment.xml index c7455252adb..fe3f3ef7419 100644 --- a/docs/EN_US/ContainerizedHPCC/ContainerizedMods/LocalDeployment.xml +++ b/docs/EN_US/ContainerizedHPCC/ContainerizedMods/LocalDeployment.xml @@ -453,7 +453,7 @@ OUTPUT(allPeople,,'MyData::allPeople',THOR,OVERWRITE); C:\hpccdata\hpcc-data\tutorial, you can then reference the file using this syntax: - '~plane::data::tutorial::originalperson' + '~plane::hpcc-data::tutorial::originalperson' Note: From a418c0b303515bd519c7e943ed5f06374e1538d1 Mon Sep 17 00:00:00 2001 From: Jake Smith Date: Thu, 22 Feb 2024 16:56:44 +0000 Subject: [PATCH 04/24] Split off 8.12.96 Signed-off-by: Jake Smith --- helm/hpcc/Chart.yaml | 4 ++-- helm/hpcc/templates/_helpers.tpl | 2 +- helm/hpcc/templates/dafilesrv.yaml | 2 +- helm/hpcc/templates/dali.yaml | 2 +- helm/hpcc/templates/dfuserver.yaml | 2 +- helm/hpcc/templates/eclagent.yaml | 4 ++-- helm/hpcc/templates/eclccserver.yaml | 4 ++-- helm/hpcc/templates/eclscheduler.yaml | 2 +- helm/hpcc/templates/esp.yaml | 2 +- helm/hpcc/templates/localroxie.yaml | 2 +- helm/hpcc/templates/roxie.yaml | 8 ++++---- helm/hpcc/templates/sasha.yaml | 2 +- helm/hpcc/templates/thor.yaml | 10 +++++----- version.cmake | 4 ++-- 14 files changed, 25 insertions(+), 25 deletions(-) diff --git a/helm/hpcc/Chart.yaml b/helm/hpcc/Chart.yaml index 53030300619..0200897e69a 100644 --- a/helm/hpcc/Chart.yaml +++ b/helm/hpcc/Chart.yaml @@ -6,9 +6,9 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. -version: 8.12.95-closedown0 +version: 8.12.97-closedown0 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. -appVersion: 8.12.95-closedown0 +appVersion: 8.12.97-closedown0 diff --git a/helm/hpcc/templates/_helpers.tpl b/helm/hpcc/templates/_helpers.tpl index 314f6d8e3d4..f500df450cd 100644 --- a/helm/hpcc/templates/_helpers.tpl +++ b/helm/hpcc/templates/_helpers.tpl @@ -1240,7 +1240,7 @@ kind: Service metadata: name: {{ $lvars.serviceName | quote }} labels: - helmVersion: 8.12.95-closedown0 + helmVersion: 8.12.97-closedown0 {{- include "hpcc.addStandardLabels" (dict "root" $.root "instance" $lvars.serviceName ) | indent 4 }} {{- if $lvars.labels }} {{ toYaml $lvars.labels | indent 4 }} diff --git a/helm/hpcc/templates/dafilesrv.yaml b/helm/hpcc/templates/dafilesrv.yaml index c71a0bdcda5..a64789d390a 100644 --- a/helm/hpcc/templates/dafilesrv.yaml +++ b/helm/hpcc/templates/dafilesrv.yaml @@ -50,7 +50,7 @@ spec: labels: {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "dafilesrv" "name" "dafilesrv" "instance" .name) | indent 8 }} server: {{ .name | quote }} - helmVersion: 8.12.95-closedown0 + helmVersion: 8.12.97-closedown0 annotations: checksum/config: {{ $configSHA }} spec: diff --git a/helm/hpcc/templates/dali.yaml b/helm/hpcc/templates/dali.yaml index 7afa7597e7e..5aac3f8781c 100644 --- a/helm/hpcc/templates/dali.yaml +++ b/helm/hpcc/templates/dali.yaml @@ -82,7 +82,7 @@ spec: run: {{ $dali.name | quote }} server: {{ $dali.name | quote }} app: dali - helmVersion: 8.12.95-closedown0 + helmVersion: 8.12.97-closedown0 {{- if hasKey $.Values.global "metrics" }} {{- include "hpcc.generateMetricsReporterLabel" $.Values.global.metrics | nindent 8 }} {{- end }} diff --git a/helm/hpcc/templates/dfuserver.yaml b/helm/hpcc/templates/dfuserver.yaml index b7c9f3c829b..62884409cb0 100644 --- a/helm/hpcc/templates/dfuserver.yaml +++ b/helm/hpcc/templates/dfuserver.yaml @@ -56,7 +56,7 @@ spec: {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "dfuserver" "name" "dfuserver" "instance" .name) | indent 8 }} run: {{ .name | quote }} accessDali: "yes" - helmVersion: 8.12.95-closedown0 + helmVersion: 8.12.97-closedown0 {{- if hasKey . "labels" }} {{ toYaml .labels | indent 8 }} {{- end }} diff --git a/helm/hpcc/templates/eclagent.yaml b/helm/hpcc/templates/eclagent.yaml index 36890b12c55..0a12a7d9d89 100644 --- a/helm/hpcc/templates/eclagent.yaml +++ b/helm/hpcc/templates/eclagent.yaml @@ -58,7 +58,7 @@ data: {{- include "hpcc.addStandardLabels" (dict "root" $ "component" $apptype "name" "eclagent" "instance" $appJobName "instanceOf" (printf "%s-job" .me.name)) | indent 12 }} accessDali: "yes" accessEsp: "yes" - helmVersion: 8.12.95-closedown0 + helmVersion: 8.12.97-closedown0 {{- if hasKey .me "labels" }} {{ toYaml .me.labels | indent 12 }} {{- end }} @@ -137,7 +137,7 @@ spec: run: {{ .name | quote }} accessDali: "yes" accessEsp: {{ .useChildProcesses | default false | ternary "yes" "no" | quote }} - helmVersion: 8.12.95-closedown0 + helmVersion: 8.12.97-closedown0 {{- if hasKey . "labels" }} {{ toYaml .labels | indent 8 }} {{- end }} diff --git a/helm/hpcc/templates/eclccserver.yaml b/helm/hpcc/templates/eclccserver.yaml index 9eb4401c0ca..f3ba5b56951 100644 --- a/helm/hpcc/templates/eclccserver.yaml +++ b/helm/hpcc/templates/eclccserver.yaml @@ -57,7 +57,7 @@ data: {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "eclccserver" "name" "eclccserver" "instance" $compileJobName "instanceOf" (printf "%s-job" .me.name)) | indent 12 }} accessDali: "yes" accessEsp: "yes" - helmVersion: 8.12.95-closedown0 + helmVersion: 8.12.97-closedown0 {{- if hasKey .me "labels" }} {{ toYaml .me.labels | indent 12 }} {{- end }} @@ -142,7 +142,7 @@ spec: run: {{ .name | quote }} accessDali: "yes" accessEsp: {{ .useChildProcesses | default false | ternary "yes" "no" | quote }} - helmVersion: 8.12.95-closedown0 + helmVersion: 8.12.97-closedown0 {{- if hasKey . "labels" }} {{ toYaml .labels | indent 8 }} {{- end }} diff --git a/helm/hpcc/templates/eclscheduler.yaml b/helm/hpcc/templates/eclscheduler.yaml index f957d63b870..5824a7e0ded 100644 --- a/helm/hpcc/templates/eclscheduler.yaml +++ b/helm/hpcc/templates/eclscheduler.yaml @@ -64,7 +64,7 @@ spec: run: {{ .name | quote }} accessDali: "yes" accessEsp: "no" - helmVersion: 8.12.95-closedown0 + helmVersion: 8.12.97-closedown0 {{- if hasKey . "labels" }} {{ toYaml .labels | indent 8 }} {{- end }} diff --git a/helm/hpcc/templates/esp.yaml b/helm/hpcc/templates/esp.yaml index e29df29bec4..cf1c9a528b2 100644 --- a/helm/hpcc/templates/esp.yaml +++ b/helm/hpcc/templates/esp.yaml @@ -117,7 +117,7 @@ spec: server: {{ .name | quote }} accessDali: "yes" app: {{ $application }} - helmVersion: 8.12.95-closedown0 + helmVersion: 8.12.97-closedown0 {{- include "hpcc.addStandardLabels" (dict "root" $ "name" $application "component" "esp" "instance" .name) | indent 8 }} {{- if hasKey $.Values.global "metrics" }} {{- include "hpcc.generateMetricsReporterLabel" $.Values.global.metrics | nindent 8 }} diff --git a/helm/hpcc/templates/localroxie.yaml b/helm/hpcc/templates/localroxie.yaml index d6723d86bfe..fc3a53693d8 100644 --- a/helm/hpcc/templates/localroxie.yaml +++ b/helm/hpcc/templates/localroxie.yaml @@ -70,7 +70,7 @@ spec: server: {{ $servername | quote }} accessDali: "yes" accessEsp: "yes" - helmVersion: 8.12.95-closedown0 + helmVersion: 8.12.97-closedown0 {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "roxie-server" "name" "roxie" "instance" $roxie.name) | indent 8 }} {{- if hasKey . "labels" }} {{ toYaml .labels | indent 8 }} diff --git a/helm/hpcc/templates/roxie.yaml b/helm/hpcc/templates/roxie.yaml index 10e17500f42..74739ccbc89 100644 --- a/helm/hpcc/templates/roxie.yaml +++ b/helm/hpcc/templates/roxie.yaml @@ -120,7 +120,7 @@ spec: {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "topology-server" "name" "roxie" "instance" $commonCtx.toponame) | indent 8 }} run: {{ $commonCtx.toponame | quote }} roxie-cluster: {{ $roxie.name | quote }} - helmVersion: 8.12.95-closedown0 + helmVersion: 8.12.97-closedown0 {{- if hasKey $.Values.global "metrics" }} {{- include "hpcc.generateMetricsReporterLabel" $.Values.global.metrics | nindent 8}} {{- end }} @@ -180,7 +180,7 @@ kind: Service metadata: name: {{ $commonCtx.toponame | quote }} labels: - helmVersion: 8.12.95-closedown0 + helmVersion: 8.12.97-closedown0 {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "topology-server" "name" "roxie" "instance" $commonCtx.toponame) | indent 4 }} spec: ports: @@ -242,7 +242,7 @@ spec: roxie-cluster: {{ $roxie.name | quote }} accessDali: "yes" accessEsp: "yes" - helmVersion: 8.12.95-closedown0 + helmVersion: 8.12.97-closedown0 {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "roxie-server" "name" "roxie" "instance" $servername) | indent 8 }} {{- if hasKey $.Values.global "metrics" }} {{- include "hpcc.generateMetricsReporterLabel" $.Values.global.metrics | nindent 8}} @@ -347,7 +347,7 @@ spec: roxie-cluster: {{ $roxie.name | quote }} accessDali: "yes" accessEsp: "yes" - helmVersion: 8.12.95-closedown0 + helmVersion: 8.12.97-closedown0 {{- if hasKey $.Values.global "metrics" }} {{- include "hpcc.generateMetricsReporterLabel" $.Values.global.metrics | nindent 8}} {{- end }} diff --git a/helm/hpcc/templates/sasha.yaml b/helm/hpcc/templates/sasha.yaml index ed72c4afab3..c0505ebe848 100644 --- a/helm/hpcc/templates/sasha.yaml +++ b/helm/hpcc/templates/sasha.yaml @@ -52,7 +52,7 @@ spec: run: {{ $serviceName | quote }} server: {{ $serviceName | quote }} accessDali: {{ (has "dali" $sasha.access) | ternary "yes" "no" | quote }} - helmVersion: 8.12.95-closedown0 + helmVersion: 8.12.97-closedown0 {{- if hasKey $sasha "labels" }} {{ toYaml $sasha.labels | indent 8 }} {{- end }} diff --git a/helm/hpcc/templates/thor.yaml b/helm/hpcc/templates/thor.yaml index abea9da64f6..26af599425f 100644 --- a/helm/hpcc/templates/thor.yaml +++ b/helm/hpcc/templates/thor.yaml @@ -82,7 +82,7 @@ data: labels: accessDali: "yes" accessEsp: "yes" - helmVersion: 8.12.95-closedown0 + helmVersion: 8.12.97-closedown0 {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "eclagent" "name" "thor" "instance" $eclAgentJobName "instanceOf" (printf "%s-job" .eclAgentName)) | indent 8 }} {{- if hasKey .me "labels" }} {{ toYaml .me.labels | indent 12 }} @@ -149,7 +149,7 @@ data: accessEsp: "yes" app: "thor" component: "thormanager" - helmVersion: 8.12.95-closedown0 + helmVersion: 8.12.97-closedown0 instance: "_HPCC_JOBNAME_" job: "_HPCC_JOBNAME_" {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "thormanager" "name" "thor" "instance" $thorManagerJobName "instanceOf" (printf "%s-thormanager-job" .me.name)) | indent 12 }} @@ -218,7 +218,7 @@ data: accessEsp: "yes" app: "thor" component: "thorworker" - helmVersion: 8.12.95-closedown0 + helmVersion: 8.12.97-closedown0 instance: "_HPCC_JOBNAME_" job: "_HPCC_JOBNAME_" {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "thorworker" "name" "thor" "instance" $thorWorkerJobName "instanceOf" (printf "%s-thorworker-job" .me.name)) | indent 12 }} @@ -353,7 +353,7 @@ spec: accessEsp: {{ $commonCtx.eclAgentUseChildProcesses | ternary "yes" "no" | quote }} app: "thor" component: "thor-eclagent" - helmVersion: 8.12.95-closedown0 + helmVersion: 8.12.97-closedown0 instance: {{ $commonCtx.eclAgentName | quote }} {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "eclagent" "name" "thor" "instance" $commonCtx.eclAgentName ) | indent 8 }} {{- if hasKey $commonCtx.me "labels" }} @@ -418,7 +418,7 @@ spec: accessEsp: "no" app: "thor" component: "thor-thoragent" - helmVersion: 8.12.95-closedown0 + helmVersion: 8.12.97-closedown0 instance: {{ $commonCtx.thorAgentName | quote }} {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "eclagent" "name" "thor" "instance" $commonCtx.thorAgentName ) | indent 8 }} {{- if hasKey $commonCtx.me "labels" }} diff --git a/version.cmake b/version.cmake index ee9abc760c7..5c76bd58503 100644 --- a/version.cmake +++ b/version.cmake @@ -5,8 +5,8 @@ set ( HPCC_NAME "Community Edition" ) set ( HPCC_PROJECT "community" ) set ( HPCC_MAJOR 8 ) set ( HPCC_MINOR 12 ) -set ( HPCC_POINT 95 ) +set ( HPCC_POINT 97 ) set ( HPCC_MATURITY "closedown" ) set ( HPCC_SEQUENCE 0 ) -set ( HPCC_TAG_TIMESTAMP "2024-02-15T16:51:05Z" ) +set ( HPCC_TAG_TIMESTAMP "2024-02-22T16:56:43Z" ) ### From dc5f057b05e4985518886364724bd1af9117f12f Mon Sep 17 00:00:00 2001 From: Jake Smith Date: Thu, 22 Feb 2024 16:59:14 +0000 Subject: [PATCH 05/24] Split off 9.0.86 Signed-off-by: Jake Smith --- helm/hpcc/Chart.yaml | 4 ++-- helm/hpcc/templates/_helpers.tpl | 2 +- helm/hpcc/templates/dafilesrv.yaml | 2 +- helm/hpcc/templates/dali.yaml | 2 +- helm/hpcc/templates/dfuserver.yaml | 2 +- helm/hpcc/templates/eclagent.yaml | 4 ++-- helm/hpcc/templates/eclccserver.yaml | 4 ++-- helm/hpcc/templates/eclscheduler.yaml | 2 +- helm/hpcc/templates/esp.yaml | 2 +- helm/hpcc/templates/localroxie.yaml | 2 +- helm/hpcc/templates/roxie.yaml | 8 ++++---- helm/hpcc/templates/sasha.yaml | 2 +- helm/hpcc/templates/thor.yaml | 10 +++++----- version.cmake | 4 ++-- 14 files changed, 25 insertions(+), 25 deletions(-) diff --git a/helm/hpcc/Chart.yaml b/helm/hpcc/Chart.yaml index f6657b679e9..a938937ac31 100644 --- a/helm/hpcc/Chart.yaml +++ b/helm/hpcc/Chart.yaml @@ -6,9 +6,9 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. -version: 9.0.85-closedown0 +version: 9.0.87-closedown0 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. -appVersion: 9.0.85-closedown0 +appVersion: 9.0.87-closedown0 diff --git a/helm/hpcc/templates/_helpers.tpl b/helm/hpcc/templates/_helpers.tpl index fbc8857f6b4..16ee6364ce2 100644 --- a/helm/hpcc/templates/_helpers.tpl +++ b/helm/hpcc/templates/_helpers.tpl @@ -1314,7 +1314,7 @@ kind: Service metadata: name: {{ $lvars.serviceName | quote }} labels: - helmVersion: 9.0.85-closedown0 + helmVersion: 9.0.87-closedown0 {{- include "hpcc.addStandardLabels" (dict "root" $.root "instance" $lvars.serviceName ) | indent 4 }} {{- if $lvars.labels }} {{ toYaml $lvars.labels | indent 4 }} diff --git a/helm/hpcc/templates/dafilesrv.yaml b/helm/hpcc/templates/dafilesrv.yaml index d67dcaabce2..cf6318da855 100644 --- a/helm/hpcc/templates/dafilesrv.yaml +++ b/helm/hpcc/templates/dafilesrv.yaml @@ -50,7 +50,7 @@ spec: labels: {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "dafilesrv" "name" "dafilesrv" "instance" .name) | indent 8 }} server: {{ .name | quote }} - helmVersion: 9.0.85-closedown0 + helmVersion: 9.0.87-closedown0 annotations: checksum/config: {{ $configSHA }} spec: diff --git a/helm/hpcc/templates/dali.yaml b/helm/hpcc/templates/dali.yaml index 25d0903df79..b8f497dbb9d 100644 --- a/helm/hpcc/templates/dali.yaml +++ b/helm/hpcc/templates/dali.yaml @@ -82,7 +82,7 @@ spec: run: {{ $dali.name | quote }} server: {{ $dali.name | quote }} app: dali - helmVersion: 9.0.85-closedown0 + helmVersion: 9.0.87-closedown0 {{- if hasKey $.Values.global "metrics" }} {{- include "hpcc.generateMetricsReporterLabel" $.Values.global.metrics | nindent 8 }} {{- end }} diff --git a/helm/hpcc/templates/dfuserver.yaml b/helm/hpcc/templates/dfuserver.yaml index fe8d5d35a1a..5b21e4627a7 100644 --- a/helm/hpcc/templates/dfuserver.yaml +++ b/helm/hpcc/templates/dfuserver.yaml @@ -56,7 +56,7 @@ spec: {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "dfuserver" "name" "dfuserver" "instance" .name) | indent 8 }} run: {{ .name | quote }} accessDali: "yes" - helmVersion: 9.0.85-closedown0 + helmVersion: 9.0.87-closedown0 {{- if hasKey . "labels" }} {{ toYaml .labels | indent 8 }} {{- end }} diff --git a/helm/hpcc/templates/eclagent.yaml b/helm/hpcc/templates/eclagent.yaml index 8b7f0f307c5..c6411f74b13 100644 --- a/helm/hpcc/templates/eclagent.yaml +++ b/helm/hpcc/templates/eclagent.yaml @@ -58,7 +58,7 @@ data: {{- include "hpcc.addStandardLabels" (dict "root" $ "component" $apptype "name" "eclagent" "instance" $appJobName "instanceOf" (printf "%s-job" .me.name)) | indent 12 }} accessDali: "yes" accessEsp: "yes" - helmVersion: 9.0.85-closedown0 + helmVersion: 9.0.87-closedown0 {{- if hasKey .me "labels" }} {{ toYaml .me.labels | indent 12 }} {{- end }} @@ -135,7 +135,7 @@ spec: run: {{ .name | quote }} accessDali: "yes" accessEsp: {{ .useChildProcesses | default false | ternary "yes" "no" | quote }} - helmVersion: 9.0.85-closedown0 + helmVersion: 9.0.87-closedown0 {{- if hasKey . "labels" }} {{ toYaml .labels | indent 8 }} {{- end }} diff --git a/helm/hpcc/templates/eclccserver.yaml b/helm/hpcc/templates/eclccserver.yaml index bb91567a599..1efafebf798 100644 --- a/helm/hpcc/templates/eclccserver.yaml +++ b/helm/hpcc/templates/eclccserver.yaml @@ -57,7 +57,7 @@ data: {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "eclccserver" "name" "eclccserver" "instance" $compileJobName "instanceOf" (printf "%s-job" .me.name)) | indent 12 }} accessDali: "yes" accessEsp: "yes" - helmVersion: 9.0.85-closedown0 + helmVersion: 9.0.87-closedown0 {{- if hasKey .me "labels" }} {{ toYaml .me.labels | indent 12 }} {{- end }} @@ -142,7 +142,7 @@ spec: run: {{ .name | quote }} accessDali: "yes" accessEsp: {{ .useChildProcesses | default false | ternary "yes" "no" | quote }} - helmVersion: 9.0.85-closedown0 + helmVersion: 9.0.87-closedown0 {{- if hasKey . "labels" }} {{ toYaml .labels | indent 8 }} {{- end }} diff --git a/helm/hpcc/templates/eclscheduler.yaml b/helm/hpcc/templates/eclscheduler.yaml index 083a12e2156..9e2c185bbb5 100644 --- a/helm/hpcc/templates/eclscheduler.yaml +++ b/helm/hpcc/templates/eclscheduler.yaml @@ -64,7 +64,7 @@ spec: run: {{ .name | quote }} accessDali: "yes" accessEsp: "no" - helmVersion: 9.0.85-closedown0 + helmVersion: 9.0.87-closedown0 {{- if hasKey . "labels" }} {{ toYaml .labels | indent 8 }} {{- end }} diff --git a/helm/hpcc/templates/esp.yaml b/helm/hpcc/templates/esp.yaml index 6fa4efcf441..b9fc87b366d 100644 --- a/helm/hpcc/templates/esp.yaml +++ b/helm/hpcc/templates/esp.yaml @@ -117,7 +117,7 @@ spec: server: {{ .name | quote }} accessDali: "yes" app: {{ $application }} - helmVersion: 9.0.85-closedown0 + helmVersion: 9.0.87-closedown0 {{- include "hpcc.addStandardLabels" (dict "root" $ "name" $application "component" "esp" "instance" .name) | indent 8 }} {{- if hasKey $.Values.global "metrics" }} {{- include "hpcc.generateMetricsReporterLabel" $.Values.global.metrics | nindent 8 }} diff --git a/helm/hpcc/templates/localroxie.yaml b/helm/hpcc/templates/localroxie.yaml index 9ae1e6356d3..f27b8dcf4ae 100644 --- a/helm/hpcc/templates/localroxie.yaml +++ b/helm/hpcc/templates/localroxie.yaml @@ -70,7 +70,7 @@ spec: server: {{ $servername | quote }} accessDali: "yes" accessEsp: "yes" - helmVersion: 9.0.85-closedown0 + helmVersion: 9.0.87-closedown0 {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "roxie-server" "name" "roxie" "instance" $roxie.name) | indent 8 }} {{- if hasKey . "labels" }} {{ toYaml .labels | indent 8 }} diff --git a/helm/hpcc/templates/roxie.yaml b/helm/hpcc/templates/roxie.yaml index 2787b2b16ad..1e2d9134a25 100644 --- a/helm/hpcc/templates/roxie.yaml +++ b/helm/hpcc/templates/roxie.yaml @@ -120,7 +120,7 @@ spec: {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "topology-server" "name" "roxie" "instance" $commonCtx.toponame) | indent 8 }} run: {{ $commonCtx.toponame | quote }} roxie-cluster: {{ $roxie.name | quote }} - helmVersion: 9.0.85-closedown0 + helmVersion: 9.0.87-closedown0 {{- if hasKey $.Values.global "metrics" }} {{- include "hpcc.generateMetricsReporterLabel" $.Values.global.metrics | nindent 8}} {{- end }} @@ -180,7 +180,7 @@ kind: Service metadata: name: {{ $commonCtx.toponame | quote }} labels: - helmVersion: 9.0.85-closedown0 + helmVersion: 9.0.87-closedown0 {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "topology-server" "name" "roxie" "instance" $commonCtx.toponame) | indent 4 }} spec: ports: @@ -242,7 +242,7 @@ spec: roxie-cluster: {{ $roxie.name | quote }} accessDali: "yes" accessEsp: "yes" - helmVersion: 9.0.85-closedown0 + helmVersion: 9.0.87-closedown0 {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "roxie-server" "name" "roxie" "instance" $servername) | indent 8 }} {{- if hasKey $.Values.global "metrics" }} {{- include "hpcc.generateMetricsReporterLabel" $.Values.global.metrics | nindent 8}} @@ -347,7 +347,7 @@ spec: roxie-cluster: {{ $roxie.name | quote }} accessDali: "yes" accessEsp: "yes" - helmVersion: 9.0.85-closedown0 + helmVersion: 9.0.87-closedown0 {{- if hasKey $.Values.global "metrics" }} {{- include "hpcc.generateMetricsReporterLabel" $.Values.global.metrics | nindent 8}} {{- end }} diff --git a/helm/hpcc/templates/sasha.yaml b/helm/hpcc/templates/sasha.yaml index 3c668cc1eb6..a7c865680a4 100644 --- a/helm/hpcc/templates/sasha.yaml +++ b/helm/hpcc/templates/sasha.yaml @@ -52,7 +52,7 @@ spec: run: {{ $serviceName | quote }} server: {{ $serviceName | quote }} accessDali: {{ (has "dali" $sasha.access) | ternary "yes" "no" | quote }} - helmVersion: 9.0.85-closedown0 + helmVersion: 9.0.87-closedown0 {{- if hasKey $sasha "labels" }} {{ toYaml $sasha.labels | indent 8 }} {{- end }} diff --git a/helm/hpcc/templates/thor.yaml b/helm/hpcc/templates/thor.yaml index 4a7a532d03c..b323ec845a6 100644 --- a/helm/hpcc/templates/thor.yaml +++ b/helm/hpcc/templates/thor.yaml @@ -82,7 +82,7 @@ data: labels: accessDali: "yes" accessEsp: "yes" - helmVersion: 9.0.85-closedown0 + helmVersion: 9.0.87-closedown0 {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "eclagent" "name" "thor" "instance" $eclAgentJobName "instanceOf" (printf "%s-job" .eclAgentName)) | indent 8 }} {{- if hasKey .me "labels" }} {{ toYaml .me.labels | indent 12 }} @@ -147,7 +147,7 @@ data: accessEsp: "yes" app: "thor" component: "thormanager" - helmVersion: 9.0.85-closedown0 + helmVersion: 9.0.87-closedown0 instance: "_HPCC_JOBNAME_" job: "_HPCC_JOBNAME_" {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "thormanager" "name" "thor" "instance" $thorManagerJobName "instanceOf" (printf "%s-thormanager-job" .me.name)) | indent 12 }} @@ -214,7 +214,7 @@ data: accessEsp: "yes" app: "thor" component: "thorworker" - helmVersion: 9.0.85-closedown0 + helmVersion: 9.0.87-closedown0 instance: "_HPCC_JOBNAME_" job: "_HPCC_JOBNAME_" {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "thorworker" "name" "thor" "instance" $thorWorkerJobName "instanceOf" (printf "%s-thorworker-job" .me.name)) | indent 12 }} @@ -347,7 +347,7 @@ spec: accessEsp: {{ $commonCtx.eclAgentUseChildProcesses | ternary "yes" "no" | quote }} app: "thor" component: "thor-eclagent" - helmVersion: 9.0.85-closedown0 + helmVersion: 9.0.87-closedown0 instance: {{ $commonCtx.eclAgentName | quote }} {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "eclagent" "name" "thor" "instance" $commonCtx.eclAgentName ) | indent 8 }} {{- if hasKey $commonCtx.me "labels" }} @@ -412,7 +412,7 @@ spec: accessEsp: "no" app: "thor" component: "thor-thoragent" - helmVersion: 9.0.85-closedown0 + helmVersion: 9.0.87-closedown0 instance: {{ $commonCtx.thorAgentName | quote }} {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "eclagent" "name" "thor" "instance" $commonCtx.thorAgentName ) | indent 8 }} {{- if hasKey $commonCtx.me "labels" }} diff --git a/version.cmake b/version.cmake index a7ea7927ac3..861315c51b5 100644 --- a/version.cmake +++ b/version.cmake @@ -5,8 +5,8 @@ set ( HPCC_NAME "Community Edition" ) set ( HPCC_PROJECT "community" ) set ( HPCC_MAJOR 9 ) set ( HPCC_MINOR 0 ) -set ( HPCC_POINT 85 ) +set ( HPCC_POINT 87 ) set ( HPCC_MATURITY "closedown" ) set ( HPCC_SEQUENCE 0 ) -set ( HPCC_TAG_TIMESTAMP "2024-02-15T16:52:21Z" ) +set ( HPCC_TAG_TIMESTAMP "2024-02-22T16:59:14Z" ) ### From 0d918dd14177d03140d8ee863d67a826f9d55acd Mon Sep 17 00:00:00 2001 From: Jake Smith Date: Thu, 22 Feb 2024 17:02:01 +0000 Subject: [PATCH 06/24] Split off 9.2.64 Signed-off-by: Jake Smith --- helm/hpcc/Chart.yaml | 4 ++-- helm/hpcc/templates/_helpers.tpl | 2 +- helm/hpcc/templates/dafilesrv.yaml | 2 +- helm/hpcc/templates/dali.yaml | 2 +- helm/hpcc/templates/dfuserver.yaml | 2 +- helm/hpcc/templates/eclagent.yaml | 4 ++-- helm/hpcc/templates/eclccserver.yaml | 4 ++-- helm/hpcc/templates/eclscheduler.yaml | 2 +- helm/hpcc/templates/esp.yaml | 2 +- helm/hpcc/templates/localroxie.yaml | 2 +- helm/hpcc/templates/roxie.yaml | 8 ++++---- helm/hpcc/templates/sasha.yaml | 2 +- helm/hpcc/templates/thor.yaml | 10 +++++----- version.cmake | 4 ++-- 14 files changed, 25 insertions(+), 25 deletions(-) diff --git a/helm/hpcc/Chart.yaml b/helm/hpcc/Chart.yaml index 2098400579d..49f31f92590 100644 --- a/helm/hpcc/Chart.yaml +++ b/helm/hpcc/Chart.yaml @@ -6,9 +6,9 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. -version: 9.2.63-closedown0 +version: 9.2.65-closedown0 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. -appVersion: 9.2.63-closedown0 +appVersion: 9.2.65-closedown0 diff --git a/helm/hpcc/templates/_helpers.tpl b/helm/hpcc/templates/_helpers.tpl index eba570d0012..160701f2df3 100644 --- a/helm/hpcc/templates/_helpers.tpl +++ b/helm/hpcc/templates/_helpers.tpl @@ -1405,7 +1405,7 @@ kind: Service metadata: name: {{ $lvars.serviceName | quote }} labels: - helmVersion: 9.2.63-closedown0 + helmVersion: 9.2.65-closedown0 {{- include "hpcc.addStandardLabels" (dict "root" $.root "instance" $lvars.serviceName ) | indent 4 }} {{- if $lvars.labels }} {{ toYaml $lvars.labels | indent 4 }} diff --git a/helm/hpcc/templates/dafilesrv.yaml b/helm/hpcc/templates/dafilesrv.yaml index a33dd250344..f0850fdb836 100644 --- a/helm/hpcc/templates/dafilesrv.yaml +++ b/helm/hpcc/templates/dafilesrv.yaml @@ -50,7 +50,7 @@ spec: labels: {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "dafilesrv" "name" "dafilesrv" "instance" .name) | indent 8 }} server: {{ .name | quote }} - helmVersion: 9.2.63-closedown0 + helmVersion: 9.2.65-closedown0 annotations: checksum/config: {{ $configSHA }} {{- include "hpcc.generateAnnotations" $commonCtx | indent 8 }} diff --git a/helm/hpcc/templates/dali.yaml b/helm/hpcc/templates/dali.yaml index f87cc292c2c..dbea2014a08 100644 --- a/helm/hpcc/templates/dali.yaml +++ b/helm/hpcc/templates/dali.yaml @@ -82,7 +82,7 @@ spec: run: {{ $dali.name | quote }} server: {{ $dali.name | quote }} app: dali - helmVersion: 9.2.63-closedown0 + helmVersion: 9.2.65-closedown0 {{- if hasKey $.Values.global "metrics" }} {{- include "hpcc.generateMetricsReporterLabel" $.Values.global.metrics | nindent 8 }} {{- end }} diff --git a/helm/hpcc/templates/dfuserver.yaml b/helm/hpcc/templates/dfuserver.yaml index 6d7e57c18bb..d61acb8b72e 100644 --- a/helm/hpcc/templates/dfuserver.yaml +++ b/helm/hpcc/templates/dfuserver.yaml @@ -56,7 +56,7 @@ spec: {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "dfuserver" "name" "dfuserver" "instance" .name) | indent 8 }} run: {{ .name | quote }} accessDali: "yes" - helmVersion: 9.2.63-closedown0 + helmVersion: 9.2.65-closedown0 {{- if hasKey . "labels" }} {{ toYaml .labels | indent 8 }} {{- end }} diff --git a/helm/hpcc/templates/eclagent.yaml b/helm/hpcc/templates/eclagent.yaml index 60854fb304c..bccbfd74dd2 100644 --- a/helm/hpcc/templates/eclagent.yaml +++ b/helm/hpcc/templates/eclagent.yaml @@ -58,7 +58,7 @@ data: {{- include "hpcc.addStandardLabels" (dict "root" $ "component" $apptype "name" "eclagent" "instance" $appJobName "instanceOf" (printf "%s-job" .me.name)) | indent 12 }} accessDali: "yes" accessEsp: "yes" - helmVersion: 9.2.63-closedown0 + helmVersion: 9.2.65-closedown0 {{- if hasKey .me "labels" }} {{ toYaml .me.labels | indent 12 }} {{- end }} @@ -133,7 +133,7 @@ spec: run: {{ .name | quote }} accessDali: "yes" accessEsp: {{ .useChildProcesses | default false | ternary "yes" "no" | quote }} - helmVersion: 9.2.63-closedown0 + helmVersion: 9.2.65-closedown0 {{- if hasKey . "labels" }} {{ toYaml .labels | indent 8 }} {{- end }} diff --git a/helm/hpcc/templates/eclccserver.yaml b/helm/hpcc/templates/eclccserver.yaml index 47ed839a4f1..25a0c0893b9 100644 --- a/helm/hpcc/templates/eclccserver.yaml +++ b/helm/hpcc/templates/eclccserver.yaml @@ -57,7 +57,7 @@ data: {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "eclccserver" "name" "eclccserver" "instance" $compileJobName "instanceOf" (printf "%s-job" .me.name)) | indent 12 }} accessDali: "yes" accessEsp: "yes" - helmVersion: 9.2.63-closedown0 + helmVersion: 9.2.65-closedown0 {{- if hasKey .me "labels" }} {{ toYaml .me.labels | indent 12 }} {{- end }} @@ -140,7 +140,7 @@ spec: run: {{ .name | quote }} accessDali: "yes" accessEsp: {{ .useChildProcesses | default false | ternary "yes" "no" | quote }} - helmVersion: 9.2.63-closedown0 + helmVersion: 9.2.65-closedown0 {{- if hasKey . "labels" }} {{ toYaml .labels | indent 8 }} {{- end }} diff --git a/helm/hpcc/templates/eclscheduler.yaml b/helm/hpcc/templates/eclscheduler.yaml index 78cc5f1ce3d..33cc9b9f38d 100644 --- a/helm/hpcc/templates/eclscheduler.yaml +++ b/helm/hpcc/templates/eclscheduler.yaml @@ -64,7 +64,7 @@ spec: run: {{ .name | quote }} accessDali: "yes" accessEsp: "no" - helmVersion: 9.2.63-closedown0 + helmVersion: 9.2.65-closedown0 {{- if hasKey . "labels" }} {{ toYaml .labels | indent 8 }} {{- end }} diff --git a/helm/hpcc/templates/esp.yaml b/helm/hpcc/templates/esp.yaml index 4ec77abb658..c39803e432c 100644 --- a/helm/hpcc/templates/esp.yaml +++ b/helm/hpcc/templates/esp.yaml @@ -120,7 +120,7 @@ spec: accessSasha: "yes" {{- end }} app: {{ $application }} - helmVersion: 9.2.63-closedown0 + helmVersion: 9.2.65-closedown0 {{- include "hpcc.addStandardLabels" (dict "root" $ "name" $application "component" "esp" "instance" .name) | indent 8 }} {{- if hasKey $.Values.global "metrics" }} {{- include "hpcc.generateMetricsReporterLabel" $.Values.global.metrics | nindent 8 }} diff --git a/helm/hpcc/templates/localroxie.yaml b/helm/hpcc/templates/localroxie.yaml index 1bf0fc7b199..56210d11d8d 100644 --- a/helm/hpcc/templates/localroxie.yaml +++ b/helm/hpcc/templates/localroxie.yaml @@ -70,7 +70,7 @@ spec: server: {{ $servername | quote }} accessDali: "yes" accessEsp: "yes" - helmVersion: 9.2.63-closedown0 + helmVersion: 9.2.65-closedown0 {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "roxie-server" "name" "roxie" "instance" $roxie.name) | indent 8 }} {{- if hasKey . "labels" }} {{ toYaml .labels | indent 8 }} diff --git a/helm/hpcc/templates/roxie.yaml b/helm/hpcc/templates/roxie.yaml index b478dc62252..8b06d3408bc 100644 --- a/helm/hpcc/templates/roxie.yaml +++ b/helm/hpcc/templates/roxie.yaml @@ -120,7 +120,7 @@ spec: {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "topology-server" "name" "roxie" "instance" $commonCtx.toponame) | indent 8 }} run: {{ $commonCtx.toponame | quote }} roxie-cluster: {{ $roxie.name | quote }} - helmVersion: 9.2.63-closedown0 + helmVersion: 9.2.65-closedown0 {{- if hasKey $.Values.global "metrics" }} {{- include "hpcc.generateMetricsReporterLabel" $.Values.global.metrics | nindent 8}} {{- end }} @@ -178,7 +178,7 @@ kind: Service metadata: name: {{ $commonCtx.toponame | quote }} labels: - helmVersion: 9.2.63-closedown0 + helmVersion: 9.2.65-closedown0 {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "topology-server" "name" "roxie" "instance" $commonCtx.toponame) | indent 4 }} spec: ports: @@ -240,7 +240,7 @@ spec: roxie-cluster: {{ $roxie.name | quote }} accessDali: "yes" accessEsp: "yes" - helmVersion: 9.2.63-closedown0 + helmVersion: 9.2.65-closedown0 {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "roxie-server" "name" "roxie" "instance" $servername) | indent 8 }} {{- if hasKey $.Values.global "metrics" }} {{- include "hpcc.generateMetricsReporterLabel" $.Values.global.metrics | nindent 8}} @@ -346,7 +346,7 @@ spec: roxie-cluster: {{ $roxie.name | quote }} accessDali: "yes" accessEsp: "yes" - helmVersion: 9.2.63-closedown0 + helmVersion: 9.2.65-closedown0 {{- if hasKey $.Values.global "metrics" }} {{- include "hpcc.generateMetricsReporterLabel" $.Values.global.metrics | nindent 8}} {{- end }} diff --git a/helm/hpcc/templates/sasha.yaml b/helm/hpcc/templates/sasha.yaml index 018e27c261e..ae7c47d2baa 100644 --- a/helm/hpcc/templates/sasha.yaml +++ b/helm/hpcc/templates/sasha.yaml @@ -53,7 +53,7 @@ spec: server: {{ $serviceName | quote }} app: sasha accessDali: {{ (has "dali" $sasha.access) | ternary "yes" "no" | quote }} - helmVersion: 9.2.63-closedown0 + helmVersion: 9.2.65-closedown0 {{- if hasKey $sasha "labels" }} {{ toYaml $sasha.labels | indent 8 }} {{- end }} diff --git a/helm/hpcc/templates/thor.yaml b/helm/hpcc/templates/thor.yaml index 9038753761c..b5547330d3e 100644 --- a/helm/hpcc/templates/thor.yaml +++ b/helm/hpcc/templates/thor.yaml @@ -82,7 +82,7 @@ data: labels: accessDali: "yes" accessEsp: "yes" - helmVersion: 9.2.63-closedown0 + helmVersion: 9.2.65-closedown0 {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "eclagent" "name" "thor" "instance" $eclAgentJobName "instanceOf" (printf "%s-job" .eclAgentName)) | indent 8 }} {{- if hasKey .me "labels" }} {{ toYaml .me.labels | indent 12 }} @@ -145,7 +145,7 @@ data: accessEsp: "yes" app: "thor" component: "thormanager" - helmVersion: 9.2.63-closedown0 + helmVersion: 9.2.65-closedown0 instance: "_HPCC_JOBNAME_" job: "_HPCC_JOBNAME_" {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "thormanager" "name" "thor" "instance" $thorManagerJobName "instanceOf" (printf "%s-thormanager-job" .me.name)) | indent 12 }} @@ -210,7 +210,7 @@ data: accessEsp: "yes" app: "thor" component: "thorworker" - helmVersion: 9.2.63-closedown0 + helmVersion: 9.2.65-closedown0 instance: "_HPCC_JOBNAME_" job: "_HPCC_JOBNAME_" {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "thorworker" "name" "thor" "instance" $thorWorkerJobName "instanceOf" (printf "%s-thorworker-job" .me.name)) | indent 12 }} @@ -341,7 +341,7 @@ spec: accessEsp: {{ $commonCtx.eclAgentUseChildProcesses | ternary "yes" "no" | quote }} app: "thor" component: "thor-eclagent" - helmVersion: 9.2.63-closedown0 + helmVersion: 9.2.65-closedown0 instance: {{ $commonCtx.eclAgentName | quote }} {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "eclagent" "name" "thor" "instance" $commonCtx.eclAgentName ) | indent 8 }} {{- if hasKey $commonCtx.me "labels" }} @@ -404,7 +404,7 @@ spec: accessEsp: "no" app: "thor" component: "thor-thoragent" - helmVersion: 9.2.63-closedown0 + helmVersion: 9.2.65-closedown0 instance: {{ $commonCtx.thorAgentName | quote }} {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "eclagent" "name" "thor" "instance" $commonCtx.thorAgentName ) | indent 8 }} {{- if hasKey $commonCtx.me "labels" }} diff --git a/version.cmake b/version.cmake index ae408843f56..7833a82e358 100644 --- a/version.cmake +++ b/version.cmake @@ -5,8 +5,8 @@ set ( HPCC_NAME "Community Edition" ) set ( HPCC_PROJECT "community" ) set ( HPCC_MAJOR 9 ) set ( HPCC_MINOR 2 ) -set ( HPCC_POINT 63 ) +set ( HPCC_POINT 65 ) set ( HPCC_MATURITY "closedown" ) set ( HPCC_SEQUENCE 0 ) -set ( HPCC_TAG_TIMESTAMP "2024-02-15T16:53:37Z" ) +set ( HPCC_TAG_TIMESTAMP "2024-02-22T17:02:01Z" ) ### From 01b8001a3e81e96648370ef50cee43f5ae68a85e Mon Sep 17 00:00:00 2001 From: g-pan Date: Thu, 22 Feb 2024 13:08:39 -0500 Subject: [PATCH 07/24] HPCC-24527 Document Single User Security Manager Signed-off-by: g-pan --- .../Inst-Mods/hpcc_ldap.xml | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/docs/EN_US/Installing_and_RunningTheHPCCPlatform/Inst-Mods/hpcc_ldap.xml b/docs/EN_US/Installing_and_RunningTheHPCCPlatform/Inst-Mods/hpcc_ldap.xml index cdf7a1f33a6..c77822cd9e5 100644 --- a/docs/EN_US/Installing_and_RunningTheHPCCPlatform/Inst-Mods/hpcc_ldap.xml +++ b/docs/EN_US/Installing_and_RunningTheHPCCPlatform/Inst-Mods/hpcc_ldap.xml @@ -287,6 +287,25 @@ url="http://httpd.apache.org/docs/2.2/programs/htpasswd.html">http://httpd.apache.org/docs/2.2/programs/htpasswd.html. + + Single User Security Manager + + The Single User security manager is a specialized security manager + that allows a username/password combination to be specified on the ESP + startup command line. At runtime, when you attempt to access any + authenticating ESP feature, such as ECL Watch, you must specify a + username/password combination. + + A single user security manager could be useful for a custom + deployment where you do not want to configure an entire LDAP server or + create a Linux HTPASSWD file, such as a classroom environment or a custom + HPCC Systems Virtual Machine. + + See the Security + Manager Plugin Framework document for more information + on configuring and deploying Security Manager plugins. + Using LDAP Authentication From f5d012dd8644eacede7c7594cc243d73443c7d1b Mon Sep 17 00:00:00 2001 From: Shamser Ahmed Date: Thu, 22 Feb 2024 16:32:30 +0000 Subject: [PATCH 08/24] HPCC-31330 FileAccessCost for dfu workunit incorrectly set Something went wrong when rebasing HPCC-31028 or HPCC-30974. CDFUprogress::setFileAccessCost requires cost_type argument. Signed-off-by: Shamser Ahmed --- dali/ft/filecopy.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dali/ft/filecopy.cpp b/dali/ft/filecopy.cpp index 5802a8bf1c5..d93ade88777 100644 --- a/dali/ft/filecopy.cpp +++ b/dali/ft/filecopy.cpp @@ -3826,7 +3826,7 @@ void FileSprayer::updateTargetProperties() curProps.setPropInt64(getDFUQResultFieldName(DFUQRFnumDiskReads), prevNumReads + totalNumReads); curProps.setPropInt64(getDFUQResultFieldName(DFUQRFreadCost), legacyReadCost + prevReadCost + totalReadCost); } - progressReport->setFileAccessCost(cost_type2money(totalReadCost+totalWriteCost)); + progressReport->setFileAccessCost(totalReadCost+totalWriteCost); if (error) throw error.getClear(); } From 6b8e68b750931725e3ba9636758c15bb346b4fb6 Mon Sep 17 00:00:00 2001 From: Jake Smith Date: Fri, 23 Feb 2024 17:44:53 +0000 Subject: [PATCH 09/24] HPCC-31341 Fix DFU key copy issue if copying to different sized cluster If copying a key, wrap will be used, and the target part size must match the source. The target number of parts was being set into numPartsOverride, which was then picked up by the dfu job to ensure the target file was created with the same number of parts. However, the same setting is used as a request via spray/copy services and is not permitted to be used on a key copy. If the DFU workunit was resubmitted, the persisted numPartsOverride was detected and a 'DestinationNumPartOverride is provided but getWrap is true' error was fired. Avoid setting numPartsOverride, instead set numParts, thus avoiding the error if the DFU workunit is resubmitted. Also allow NumPartsOverride as long as it matches source # parts. And improve the error message. Signed-off-by: Jake Smith --- dali/dfu/dfurun.cpp | 12 ++++++++---- dali/dfu/dfuwu.cpp | 2 +- dali/dfu/dfuwu.hpp | 1 + 3 files changed, 10 insertions(+), 5 deletions(-) diff --git a/dali/dfu/dfurun.cpp b/dali/dfu/dfurun.cpp index 74a795261e8..1e7d69f5e53 100644 --- a/dali/dfu/dfurun.cpp +++ b/dali/dfu/dfurun.cpp @@ -1427,15 +1427,19 @@ class CDFUengine: public CInterface, implements IDFUengine // keys default wrap for copy if (destination->getWrap()||(iskey&&(cmd==DFUcmd_copy))) { - if (destination->getNumPartsOverride()) - throw makeStringExceptionV(-1, "DestinationNumPartOverride is provided but %s", destination->getWrap()?"getWrap is true":"is copying a key"); - dst->setNumPartsOverride(srcFile->numParts()); + unsigned numOverrideParts = destination->getNumPartsOverride(); + if (numOverrideParts) + { + if (srcFile->numParts() != numOverrideParts) + throw makeStringExceptionV(-1, "Destination NumPartsOverride is provided but %s", (iskey&&(cmd==DFUcmd_copy))?"not supported when copying a key":"getWrap is true"); + } + dst->setNumParts(srcFile->numParts()); } else if (plane) { // use destination defaultSprayParts if requestor doesn't provide num parts if (plane->hasProp("@defaultSprayParts") && destination->getNumPartsOverride()==0) - dst->setNumPartsOverride(plane->getPropInt("@defaultSprayParts")); + dst->setNumParts(plane->getPropInt("@defaultSprayParts")); } } break; diff --git a/dali/dfu/dfuwu.cpp b/dali/dfu/dfuwu.cpp index fcd3403e6a6..f90ebbefea2 100644 --- a/dali/dfu/dfuwu.cpp +++ b/dali/dfu/dfuwu.cpp @@ -1376,7 +1376,7 @@ class CDFUfileSpec: public CLinkedDFUWUchild, implements IDFUfileSpec { queryRoot()->setProp("@partmask",val); } - void setNumParts(unsigned val) + virtual void setNumParts(unsigned val) override { queryRoot()->setPropInt("@numparts",val); diff --git a/dali/dfu/dfuwu.hpp b/dali/dfu/dfuwu.hpp index 59c15a49e9d..7e56085a5f5 100644 --- a/dali/dfu/dfuwu.hpp +++ b/dali/dfu/dfuwu.hpp @@ -279,6 +279,7 @@ interface IDFUfileSpec: extends IConstDFUfileSpec virtual void setFromXML(const char *xml) = 0; virtual void setCompressed(bool set) = 0; virtual void setWrap(bool val) = 0; + virtual void setNumParts(unsigned val) = 0; virtual void setNumPartsOverride(unsigned num) = 0; virtual void setReplicateOffset(int val) = 0; // sets for all clusters virtual void setDiffKey(const char *keyname) = 0; From 270bd025f59ed3364f9cb559192b8abdbd3fdad6 Mon Sep 17 00:00:00 2001 From: Gavin Halliday Date: Fri, 23 Feb 2024 11:51:10 +0000 Subject: [PATCH 10/24] HPCC-31332 Allow the SOAPCALL PERSIST() parameter to be non-constant Signed-off-by: Gavin Halliday --- ecl/hql/hqlgram.y | 2 +- ecl/hqlcpp/hqlhtcpp.cpp | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/ecl/hql/hqlgram.y b/ecl/hql/hqlgram.y index 9e3e190beb7..87dc61c7484 100644 --- a/ecl/hql/hqlgram.y +++ b/ecl/hql/hqlgram.y @@ -3832,7 +3832,7 @@ soapFlag if ($3.queryExprType()->isBoolean()) parser->normalizeExpression($3, type_boolean, true); else - parser->normalizeExpression($3, type_int, true); + parser->normalizeExpression($3, type_int, false); $$.setExpr(createExprAttribute(persistAtom, $3.getExpr()), $1); } ; diff --git a/ecl/hqlcpp/hqlhtcpp.cpp b/ecl/hqlcpp/hqlhtcpp.cpp index f2939a87576..aa425cc8089 100644 --- a/ecl/hqlcpp/hqlhtcpp.cpp +++ b/ecl/hqlcpp/hqlhtcpp.cpp @@ -18242,7 +18242,7 @@ ABoundActivity * HqlCppTranslator::doBuildActivitySOAP(BuildCtx & ctx, IHqlExpre else if (matchesBoolean(persistArg, true)) persistArg = nullptr; else if (!matchesConstantValue(persistArg, 0)) // Avoid generating 0 since that is the default implementation - doBuildUnsignedFunction(instance->createctx, "getPersistMaxRequests", persistArg); + doBuildUnsignedFunction(instance->startctx, "getPersistMaxRequests", persistArg); } //virtual unsigned getFlags() From eebc907852fe7d2701067109413768d7d3cd221c Mon Sep 17 00:00:00 2001 From: Gordon Smith Date: Mon, 26 Feb 2024 07:40:55 +0000 Subject: [PATCH 11/24] HPCC-31334 Prevent double call for custom page sizes Page loaded lists would make 2 calls to fetch the first page (first for default value and a second for overriden user size) Signed-off-by: Gordon Smith --- esp/src/src-react/components/controls/Grid.tsx | 4 ++-- esp/src/src-react/hooks/store.ts | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/esp/src/src-react/components/controls/Grid.tsx b/esp/src/src-react/components/controls/Grid.tsx index 3bad2bf3368..28cba6d84fc 100644 --- a/esp/src/src-react/components/controls/Grid.tsx +++ b/esp/src/src-react/components/controls/Grid.tsx @@ -158,13 +158,13 @@ export interface FluentStoreStateResponse { setPageSize: (pageSize: number) => void; total: number; setTotal: (total: number) => void; - refreshTable: RefreshTable + refreshTable: RefreshTable; } export function useFluentStoreState({ page }: FluentStoreStateProps): FluentStoreStateResponse { const [selection, setSelection] = React.useState([]); const [pageNum, setPageNum] = React.useState(page); - const [pageSize, setPageSize] = React.useState(25); + const [pageSize, setPageSize] = React.useState(); const [total, setTotal] = React.useState(0); const refreshTable = useRefreshTable(); diff --git a/esp/src/src-react/hooks/store.ts b/esp/src/src-react/hooks/store.ts index e8f598ca566..95f48517129 100644 --- a/esp/src/src-react/hooks/store.ts +++ b/esp/src/src-react/hooks/store.ts @@ -40,7 +40,7 @@ function fromString(value: string, defaultValue: T): T { function useStore(store: IKeyValStore, key: string, defaultValue: T, monitor: boolean = false): [value: T, setValue: (value: T) => void, reset: () => void] { - const [value, setValue] = React.useState(defaultValue); + const [value, setValue] = React.useState(); React.useEffect(() => { if (!store) return; From 49baa140129731497e3588918597971f2624c2f5 Mon Sep 17 00:00:00 2001 From: Jake Smith Date: Mon, 26 Feb 2024 17:42:19 +0000 Subject: [PATCH 12/24] HPCC-31348 Default sprayed files to compressed in containerized To match engine behaviour in containerized environments. Engine default change introduced by HPCC-30280. Signed-off-by: Jake Smith --- ecllibrary/std/File.ecl | 20 ++++++++++---------- plugins/proxies/lib_fileservices.ecllib | 16 ++++++++-------- 2 files changed, 18 insertions(+), 18 deletions(-) diff --git a/ecllibrary/std/File.ecl b/ecllibrary/std/File.ecl index 4371f36dfa7..08cad3fc7b8 100644 --- a/ecllibrary/std/File.ecl +++ b/ecllibrary/std/File.ecl @@ -437,7 +437,7 @@ EXPORT DfuPlusExec(varstring cmdline) := * @param noCommon Set to FALSE to enable commoning up of puller or pusher processes on same host. Default is TRUE. * @return The DFU workunit id for the job. */ -EXPORT varstring fSprayFixed(varstring sourceIP='', varstring sourcePath, integer4 recordSize, varstring destinationGroup, varstring destinationLogicalName, integer4 timeOut=-1, varstring espServerIpPort=GETENV('ws_fs_server'), integer4 maxConnections=-1, boolean allowOverwrite=FALSE, boolean replicate=FALSE, boolean compress=FALSE, boolean failIfNoSourceFile=FALSE, integer4 expireDays=-1, const varstring dfuServerQueue='', boolean noSplit=FALSE, varstring sourcePlane='', unsigned4 destinationNumParts=0) := +EXPORT varstring fSprayFixed(varstring sourceIP='', varstring sourcePath, integer4 recordSize, varstring destinationGroup, varstring destinationLogicalName, integer4 timeOut=-1, varstring espServerIpPort=GETENV('ws_fs_server'), integer4 maxConnections=-1, boolean allowOverwrite=FALSE, boolean replicate=FALSE, boolean compress=__CONTAINERIZED__, boolean failIfNoSourceFile=FALSE, integer4 expireDays=-1, const varstring dfuServerQueue='', boolean noSplit=FALSE, varstring sourcePlane='', unsigned4 destinationNumParts=0) := lib_fileservices.FileServices.fSprayFixed(sourceIP, sourcePath, recordSize, destinationGroup, destinationLogicalName, timeOut, espServerIpPort, maxConnections, allowOverwrite, replicate, compress, failIfNoSourceFile, expireDays, dfuServerQueue, noSplit, sourcePlane, destinationNumParts); /** @@ -446,15 +446,15 @@ EXPORT varstring fSprayFixed(varstring sourceIP='', varstring sourcePath, intege * @see fSprayFixed */ -EXPORT SprayFixed(varstring sourceIP='', varstring sourcePath, integer4 recordSize, varstring destinationGroup, varstring destinationLogicalName, integer4 timeOut=-1, varstring espServerIpPort=GETENV('ws_fs_server'), integer4 maxConnections=-1, boolean allowOverwrite=FALSE, boolean replicate=FALSE, boolean compress=FALSE, boolean failIfNoSourceFile=FALSE, integer4 expireDays=-1, const varstring dfuServerQueue='', boolean noSplit=FALSE, varstring sourcePlane='', unsigned4 destinationNumParts=0, boolean noCommon=GetNoCommonDefault()) := +EXPORT SprayFixed(varstring sourceIP='', varstring sourcePath, integer4 recordSize, varstring destinationGroup, varstring destinationLogicalName, integer4 timeOut=-1, varstring espServerIpPort=GETENV('ws_fs_server'), integer4 maxConnections=-1, boolean allowOverwrite=FALSE, boolean replicate=FALSE, boolean compress=__CONTAINERIZED__, boolean failIfNoSourceFile=FALSE, integer4 expireDays=-1, const varstring dfuServerQueue='', boolean noSplit=FALSE, varstring sourcePlane='', unsigned4 destinationNumParts=0, boolean noCommon=GetNoCommonDefault()) := lib_fileservices.FileServices.SprayFixed(sourceIP, sourcePath, recordSize, destinationGroup, destinationLogicalName, timeOut, espServerIpPort, maxConnections, allowOverwrite, replicate, compress, failIfNoSourceFile, expireDays, dfuServerQueue, noSplit, sourcePlane, destinationNumParts, noCommon); // fSprayVariable is now called fSprayDelimited (but the old name is available for backward compatibility) -EXPORT varstring fSprayVariable(varstring sourceIP='', varstring sourcePath, integer4 sourceMaxRecordSize=8192, varstring sourceCsvSeparate='\\,', varstring sourceCsvTerminate='\\n,\\r\\n', varstring sourceCsvQuote='\"', varstring destinationGroup, varstring destinationLogicalName, integer4 timeOut=-1, varstring espServerIpPort=GETENV('ws_fs_server'), integer4 maxConnections=-1, boolean allowOverwrite=FALSE, boolean replicate=FALSE, boolean compress=FALSE, varstring sourceCsvEscape='', boolean failIfNoSourceFile=FALSE, boolean recordStructurePresent=FALSE, boolean quotedTerminator=TRUE, varstring encoding='ascii', integer4 expireDays=-1, varstring dfuServerQueue='', boolean noSplit=FALSE, varstring sourcePlane='', unsigned4 destinationNumParts=0, boolean noCommon=GetNoCommonDefault()) := +EXPORT varstring fSprayVariable(varstring sourceIP='', varstring sourcePath, integer4 sourceMaxRecordSize=8192, varstring sourceCsvSeparate='\\,', varstring sourceCsvTerminate='\\n,\\r\\n', varstring sourceCsvQuote='\"', varstring destinationGroup, varstring destinationLogicalName, integer4 timeOut=-1, varstring espServerIpPort=GETENV('ws_fs_server'), integer4 maxConnections=-1, boolean allowOverwrite=FALSE, boolean replicate=FALSE, boolean compress=__CONTAINERIZED__, varstring sourceCsvEscape='', boolean failIfNoSourceFile=FALSE, boolean recordStructurePresent=FALSE, boolean quotedTerminator=TRUE, varstring encoding='ascii', integer4 expireDays=-1, varstring dfuServerQueue='', boolean noSplit=FALSE, varstring sourcePlane='', unsigned4 destinationNumParts=0, boolean noCommon=GetNoCommonDefault()) := lib_fileservices.FileServices.fSprayVariable(sourceIP, sourcePath, sourceMaxRecordSize, sourceCsvSeparate, sourceCsvTerminate, sourceCsvQuote, destinationGroup, destinationLogicalName, timeOut, espServerIpPort, maxConnections, allowOverwrite, replicate, compress, sourceCsvEscape, failIfNoSourceFile, recordStructurePresent, quotedTerminator, encoding, expireDays, dfuServerQueue, noSplit, sourcePlane, destinationNumParts, noCommon); // SprayVariable is now called SprayDelimited (but the old name is available for backward compatibility) -EXPORT SprayVariable(varstring sourceIP='', varstring sourcePath, integer4 sourceMaxRecordSize=8192, varstring sourceCsvSeparate='\\,', varstring sourceCsvTerminate='\\n,\\r\\n', varstring sourceCsvQuote='\"', varstring destinationGroup, varstring destinationLogicalName, integer4 timeOut=-1, varstring espServerIpPort=GETENV('ws_fs_server'), integer4 maxConnections=-1, boolean allowOverwrite=FALSE, boolean replicate=FALSE, boolean compress=FALSE, varstring sourceCsvEscape='', boolean failIfNoSourceFile=FALSE, boolean recordStructurePresent=FALSE, boolean quotedTerminator=TRUE, varstring encoding='ascii', integer4 expireDays=-1, const varstring dfuServerQueue='', boolean noSplit=FALSE, varstring sourcePlane='', unsigned4 destinationNumParts=0, boolean noCommon=GetNoCommonDefault()) := +EXPORT SprayVariable(varstring sourceIP='', varstring sourcePath, integer4 sourceMaxRecordSize=8192, varstring sourceCsvSeparate='\\,', varstring sourceCsvTerminate='\\n,\\r\\n', varstring sourceCsvQuote='\"', varstring destinationGroup, varstring destinationLogicalName, integer4 timeOut=-1, varstring espServerIpPort=GETENV('ws_fs_server'), integer4 maxConnections=-1, boolean allowOverwrite=FALSE, boolean replicate=FALSE, boolean compress=__CONTAINERIZED__, varstring sourceCsvEscape='', boolean failIfNoSourceFile=FALSE, boolean recordStructurePresent=FALSE, boolean quotedTerminator=TRUE, varstring encoding='ascii', integer4 expireDays=-1, const varstring dfuServerQueue='', boolean noSplit=FALSE, varstring sourcePlane='', unsigned4 destinationNumParts=0, boolean noCommon=GetNoCommonDefault()) := lib_fileservices.FileServices.SprayVariable(sourceIP, sourcePath, sourceMaxRecordSize, sourceCsvSeparate, sourceCsvTerminate, sourceCsvQuote, destinationGroup, destinationLogicalName, timeOut, espServerIpPort, maxConnections, allowOverwrite, replicate, compress, sourceCsvEscape, failIfNoSourceFile, recordStructurePresent, quotedTerminator, encoding, expireDays, dfuServerQueue, noSplit, sourcePlane, destinationNumParts, noCommon); /** @@ -492,7 +492,7 @@ EXPORT SprayVariable(varstring sourceIP='', varstring sourcePath, integer4 sourc * @param noCommon Set to FALSE to enable commoning up of puller or pusher processes on same host. Default is TRUE. * @return The DFU workunit id for the job. */ -EXPORT varstring fSprayDelimited(varstring sourceIP='', varstring sourcePath, integer4 sourceMaxRecordSize=8192, varstring sourceCsvSeparate='\\,', varstring sourceCsvTerminate='\\n,\\r\\n', varstring sourceCsvQuote='\"', varstring destinationGroup, varstring destinationLogicalName, integer4 timeOut=-1, varstring espServerIpPort=GETENV('ws_fs_server'), integer4 maxConnections=-1, boolean allowOverwrite=FALSE, boolean replicate=FALSE, boolean compress=FALSE, varstring sourceCsvEscape='', boolean failIfNoSourceFile=FALSE, boolean recordStructurePresent=FALSE, boolean quotedTerminator=TRUE, varstring encoding='ascii', integer4 expireDays=-1, const varstring dfuServerQueue='', boolean noSplit=FALSE, varstring sourcePlane='', unsigned4 destinationNumParts=0, boolean noCommon=GetNoCommonDefault()) := +EXPORT varstring fSprayDelimited(varstring sourceIP='', varstring sourcePath, integer4 sourceMaxRecordSize=8192, varstring sourceCsvSeparate='\\,', varstring sourceCsvTerminate='\\n,\\r\\n', varstring sourceCsvQuote='\"', varstring destinationGroup, varstring destinationLogicalName, integer4 timeOut=-1, varstring espServerIpPort=GETENV('ws_fs_server'), integer4 maxConnections=-1, boolean allowOverwrite=FALSE, boolean replicate=FALSE, boolean compress=__CONTAINERIZED__, varstring sourceCsvEscape='', boolean failIfNoSourceFile=FALSE, boolean recordStructurePresent=FALSE, boolean quotedTerminator=TRUE, varstring encoding='ascii', integer4 expireDays=-1, const varstring dfuServerQueue='', boolean noSplit=FALSE, varstring sourcePlane='', unsigned4 destinationNumParts=0, boolean noCommon=GetNoCommonDefault()) := lib_fileservices.FileServices.fSprayVariable(sourceIP, sourcePath, sourceMaxRecordSize, sourceCsvSeparate, sourceCsvTerminate, sourceCsvQuote, destinationGroup, destinationLogicalName, timeOut, espServerIpPort, maxConnections, allowOverwrite, replicate, compress, sourceCsvEscape, failIfNoSourceFile, recordStructurePresent, quotedTerminator, encoding, expireDays, dfuServerQueue, noSplit, sourcePlane, destinationNumParts, noCommon); /** @@ -501,7 +501,7 @@ EXPORT varstring fSprayDelimited(varstring sourceIP='', varstring sourcePath, in * @see fSprayDelimited */ -EXPORT SprayDelimited(varstring sourceIP='', varstring sourcePath, integer4 sourceMaxRecordSize=8192, varstring sourceCsvSeparate='\\,', varstring sourceCsvTerminate='\\n,\\r\\n', varstring sourceCsvQuote='\"', varstring destinationGroup, varstring destinationLogicalName, integer4 timeOut=-1, varstring espServerIpPort=GETENV('ws_fs_server'), integer4 maxConnections=-1, boolean allowOverwrite=FALSE, boolean replicate=FALSE, boolean compress=FALSE, varstring sourceCsvEscape='', boolean failIfNoSourceFile=FALSE, boolean recordStructurePresent=FALSE, boolean quotedTerminator=TRUE, const varstring encoding='ascii', integer4 expireDays=-1, const varstring dfuServerQueue='', boolean noSplit=FALSE, varstring sourcePlane='', unsigned4 destinationNumParts=0, boolean noCommon=GetNoCommonDefault()) := +EXPORT SprayDelimited(varstring sourceIP='', varstring sourcePath, integer4 sourceMaxRecordSize=8192, varstring sourceCsvSeparate='\\,', varstring sourceCsvTerminate='\\n,\\r\\n', varstring sourceCsvQuote='\"', varstring destinationGroup, varstring destinationLogicalName, integer4 timeOut=-1, varstring espServerIpPort=GETENV('ws_fs_server'), integer4 maxConnections=-1, boolean allowOverwrite=FALSE, boolean replicate=FALSE, boolean compress=__CONTAINERIZED__, varstring sourceCsvEscape='', boolean failIfNoSourceFile=FALSE, boolean recordStructurePresent=FALSE, boolean quotedTerminator=TRUE, const varstring encoding='ascii', integer4 expireDays=-1, const varstring dfuServerQueue='', boolean noSplit=FALSE, varstring sourcePlane='', unsigned4 destinationNumParts=0, boolean noCommon=GetNoCommonDefault()) := lib_fileservices.FileServices.SprayVariable(sourceIP, sourcePath, sourceMaxRecordSize, sourceCsvSeparate, sourceCsvTerminate, sourceCsvQuote, destinationGroup, destinationLogicalName, timeOut, espServerIpPort, maxConnections, allowOverwrite, replicate, compress, sourceCsvEscape, failIfNoSourceFile, recordStructurePresent, quotedTerminator, encoding, expireDays, dfuServerQueue, noSplit, sourcePlane, destinationNumParts, noCommon); @@ -533,7 +533,7 @@ EXPORT SprayDelimited(varstring sourceIP='', varstring sourcePath, integer4 sour * @return The DFU workunit id for the job. */ -EXPORT varstring fSprayXml(varstring sourceIP='', varstring sourcePath, integer4 sourceMaxRecordSize=8192, varstring sourceRowTag, varstring sourceEncoding='utf8', varstring destinationGroup, varstring destinationLogicalName, integer4 timeOut=-1, varstring espServerIpPort=GETENV('ws_fs_server'), integer4 maxConnections=-1, boolean allowOverwrite=FALSE, boolean replicate=FALSE, boolean compress=FALSE, boolean failIfNoSourceFile=FALSE, integer4 expireDays=-1, const varstring dfuServerQueue='', boolean noSplit=FALSE, varstring sourcePlane='', unsigned4 destinationNumParts=0, boolean noCommon=GetNoCommonDefault()) := +EXPORT varstring fSprayXml(varstring sourceIP='', varstring sourcePath, integer4 sourceMaxRecordSize=8192, varstring sourceRowTag, varstring sourceEncoding='utf8', varstring destinationGroup, varstring destinationLogicalName, integer4 timeOut=-1, varstring espServerIpPort=GETENV('ws_fs_server'), integer4 maxConnections=-1, boolean allowOverwrite=FALSE, boolean replicate=FALSE, boolean compress=__CONTAINERIZED__, boolean failIfNoSourceFile=FALSE, integer4 expireDays=-1, const varstring dfuServerQueue='', boolean noSplit=FALSE, varstring sourcePlane='', unsigned4 destinationNumParts=0, boolean noCommon=GetNoCommonDefault()) := lib_fileservices.FileServices.fSprayXml(sourceIP, sourcePath, sourceMaxRecordSize, sourceRowTag, sourceEncoding, destinationGroup, destinationLogicalName, timeOut, espServerIpPort, maxConnections, allowOverwrite, replicate, compress, failIfNoSourceFile, expireDays, dfuServerQueue, noSplit, sourcePlane, destinationNumParts, noCommon); /** @@ -542,7 +542,7 @@ EXPORT varstring fSprayXml(varstring sourceIP='', varstring sourcePath, integer4 * @see fSprayXml */ -EXPORT SprayXml(varstring sourceIP='', varstring sourcePath, integer4 sourceMaxRecordSize=8192, varstring sourceRowTag, varstring sourceEncoding='utf8', varstring destinationGroup, varstring destinationLogicalName, integer4 timeOut=-1, varstring espServerIpPort=GETENV('ws_fs_server'), integer4 maxConnections=-1, boolean allowOverwrite=FALSE, boolean replicate=FALSE, boolean compress=FALSE, boolean failIfNoSourceFile=FALSE, integer4 expireDays=-1, const varstring dfuServerQueue='', boolean noSplit=FALSE, varstring sourcePlane='', unsigned4 destinationNumParts=0, boolean noCommon=GetNoCommonDefault()) := +EXPORT SprayXml(varstring sourceIP='', varstring sourcePath, integer4 sourceMaxRecordSize=8192, varstring sourceRowTag, varstring sourceEncoding='utf8', varstring destinationGroup, varstring destinationLogicalName, integer4 timeOut=-1, varstring espServerIpPort=GETENV('ws_fs_server'), integer4 maxConnections=-1, boolean allowOverwrite=FALSE, boolean replicate=FALSE, boolean compress=__CONTAINERIZED__, boolean failIfNoSourceFile=FALSE, integer4 expireDays=-1, const varstring dfuServerQueue='', boolean noSplit=FALSE, varstring sourcePlane='', unsigned4 destinationNumParts=0, boolean noCommon=GetNoCommonDefault()) := lib_fileservices.FileServices.SprayXml(sourceIP, sourcePath, sourceMaxRecordSize, sourceRowTag, sourceEncoding, destinationGroup, destinationLogicalName, timeOut, espServerIpPort, maxConnections, allowOverwrite, replicate, compress, failIfNoSourceFile, expireDays, dfuServerQueue, noSplit, sourcePlane, destinationNumParts, noCommon); @@ -578,7 +578,7 @@ EXPORT SprayXml(varstring sourceIP='', varstring sourcePath, integer4 sourceMaxR * @return The DFU workunit id for the job. */ -EXPORT varstring fSprayJson(varstring sourceIP='', varstring sourcePath, integer4 sourceMaxRecordSize=8192, varstring sourceRowPath='/', varstring sourceEncoding='utf8', varstring destinationGroup, varstring destinationLogicalName, integer4 timeOut=-1, varstring espServerIpPort=GETENV('ws_fs_server'), integer4 maxConnections=-1, boolean allowOverwrite=FALSE, boolean replicate=FALSE, boolean compress=FALSE, boolean failIfNoSourceFile=FALSE, integer4 expireDays=-1, const varstring dfuServerQueue='', boolean noSplit=FALSE, const varstring username = '', const varstring userPw = '', varstring sourcePlane='', unsigned4 destinationNumParts=0, boolean noCommon=GetNoCommonDefault()) := +EXPORT varstring fSprayJson(varstring sourceIP='', varstring sourcePath, integer4 sourceMaxRecordSize=8192, varstring sourceRowPath='/', varstring sourceEncoding='utf8', varstring destinationGroup, varstring destinationLogicalName, integer4 timeOut=-1, varstring espServerIpPort=GETENV('ws_fs_server'), integer4 maxConnections=-1, boolean allowOverwrite=FALSE, boolean replicate=FALSE, boolean compress=__CONTAINERIZED__, boolean failIfNoSourceFile=FALSE, integer4 expireDays=-1, const varstring dfuServerQueue='', boolean noSplit=FALSE, const varstring username = '', const varstring userPw = '', varstring sourcePlane='', unsigned4 destinationNumParts=0, boolean noCommon=GetNoCommonDefault()) := lib_fileservices.FileServices.fSprayJson(sourceIP, sourcePath, sourceMaxRecordSize, sourceRowPath, sourceEncoding, destinationGroup, destinationLogicalName, timeOut, espServerIpPort, maxConnections, allowOverwrite, replicate, compress, failIfNoSourceFile, expireDays, dfuServerQueue, noSplit, username, userPw, sourcePlane, destinationNumParts, noCommon); @@ -588,7 +588,7 @@ EXPORT varstring fSprayJson(varstring sourceIP='', varstring sourcePath, integer * @see fSprayJson */ -EXPORT SprayJson(varstring sourceIP='', varstring sourcePath, integer4 sourceMaxRecordSize=8192, varstring sourceRowPath='/', varstring sourceEncoding='utf8', varstring destinationGroup, varstring destinationLogicalName, integer4 timeOut=-1, varstring espServerIpPort=GETENV('ws_fs_server'), integer4 maxConnections=-1, boolean allowOverwrite=FALSE, boolean replicate=FALSE, boolean compress=FALSE, boolean failIfNoSourceFile=FALSE, integer4 expireDays=-1, const varstring dfuServerQueue='', boolean noSplit=FALSE, const varstring username = '', const varstring userPw = '', varstring sourcePlane='', unsigned4 destinationNumParts=0, boolean noCommon=GetNoCommonDefault()) := +EXPORT SprayJson(varstring sourceIP='', varstring sourcePath, integer4 sourceMaxRecordSize=8192, varstring sourceRowPath='/', varstring sourceEncoding='utf8', varstring destinationGroup, varstring destinationLogicalName, integer4 timeOut=-1, varstring espServerIpPort=GETENV('ws_fs_server'), integer4 maxConnections=-1, boolean allowOverwrite=FALSE, boolean replicate=FALSE, boolean compress=__CONTAINERIZED__, boolean failIfNoSourceFile=FALSE, integer4 expireDays=-1, const varstring dfuServerQueue='', boolean noSplit=FALSE, const varstring username = '', const varstring userPw = '', varstring sourcePlane='', unsigned4 destinationNumParts=0, boolean noCommon=GetNoCommonDefault()) := lib_fileservices.FileServices.SprayJson(sourceIP, sourcePath, sourceMaxRecordSize, sourceRowPath, sourceEncoding, destinationGroup, destinationLogicalName, timeOut, espServerIpPort, maxConnections, allowOverwrite, replicate, compress, failIfNoSourceFile, expireDays, dfuServerQueue, noSplit, username, userPw, sourcePlane, destinationNumParts, noCommon); diff --git a/plugins/proxies/lib_fileservices.ecllib b/plugins/proxies/lib_fileservices.ecllib index 33e8412ce2e..bdc82429e8e 100644 --- a/plugins/proxies/lib_fileservices.ecllib +++ b/plugins/proxies/lib_fileservices.ecllib @@ -39,10 +39,10 @@ export FileServices := SERVICE : plugin('fileservices'), time SendEmail(const varstring to, const varstring subject, const varstring body, const varstring mailServer=GETENV('SMTPserver'), unsigned4 port=(unsigned4) GETENV('SMTPport', '25'), const varstring sender=GETENV('emailSenderAddress'), const varstring cc='', const varstring bcc='', boolean highPriority=false, boolean termJobOnFail=true) : c,action,context,entrypoint='fsSendEmail_v3'; SendEmailAttachText(const varstring to, const varstring subject, const varstring body, const varstring attachment, const varstring mimeType, const varstring attachmentName, const varstring mailServer=GETENV('SMTPserver'), unsigned4 port=(unsigned4) GETENV('SMTPport', '25'), const varstring sender=GETENV('emailSenderAddress'), const varstring cc='', const varstring bcc='', boolean highPriority=false, boolean termJobOnFail=true) : c,action,context,entrypoint='fsSendEmailAttachText_v3'; SendEmailAttachData(const varstring to, const varstring subject, const varstring body, const data attachment, const varstring mimeType, const varstring attachmentName, const varstring mailServer=GETENV('SMTPserver'), unsigned4 port=(unsigned4) GETENV('SMTPport', '25'), const varstring sender=GETENV('emailSenderAddress'), const varstring cc='', const varstring bcc='', boolean highPriority=false, boolean termJobOnFail=true) : c,action,context,entrypoint='fsSendEmailAttachData_v3'; - SprayFixed(const varstring sourceIP='', const varstring sourcePath, integer4 recordSize, const varstring destinationGroup, const varstring destinationLogicalName, integer4 timeOut=-1, const varstring espServerIpPort=GETENV('ws_fs_server'), integer4 maxConnections=-1, boolean allowoverwrite=false, boolean replicate=false,boolean compress=false, boolean failIfNoSourceFile=false, integer4 expireDays=-1, const varstring dfuServerQueue='', boolean noSplit=false, const varstring sourcePlane='', unsigned4 destinationNumParts=0, boolean noCommon=true) : c,action,context,entrypoint='fsSprayFixed_v6'; - SprayVariable(const varstring sourceIP='', const varstring sourcePath, integer4 sourceMaxRecordSize=8192, const varstring sourceCsvSeparate='\\,', const varstring sourceCsvTerminate='\\n,\\r\\n', const varstring sourceCsvQuote='\"', const varstring destinationGroup, const varstring destinationLogicalName, integer4 timeOut=-1, const varstring espServerIpPort=GETENV('ws_fs_server'), integer4 maxConnections=-1, boolean allowoverwrite=false, boolean replicate=false,boolean compress=false,const varstring sourceCsvEscape='', boolean failIfNoSourceFile=false, boolean recordStructurePresent=false, boolean quotedTerminator=true, const varstring encoding='ascii', integer4 expireDays=-1, const varstring dfuServerQueue='', boolean noSplit=false, const varstring sourcePlane='', unsigned4 destinationNumParts=0, boolean noCommon=true) : c,action,context,entrypoint='fsSprayVariable_v10'; - SprayXml(const varstring sourceIP='', const varstring sourcePath, integer4 sourceMaxRecordSize=8192, const varstring sourceRowTag, const varstring sourceEncoding='utf8', const varstring destinationGroup, const varstring destinationLogicalName, integer4 timeOut=-1, const varstring espServerIpPort=GETENV('ws_fs_server'), integer4 maxConnections=-1, boolean allowoverwrite=false, boolean replicate=false,boolean compress=false, boolean failIfNoSourceFile=false, integer4 expireDays=-1, const varstring dfuServerQueue='', boolean noSplit=false, const varstring sourcePlane='', unsigned4 destinationNumParts=0, boolean noCommon=true) : c,action,context,entrypoint='fsSprayXml_v6'; - SprayJson(const varstring sourceIP='', const varstring sourcePath, integer4 sourceMaxRecordSize=8192, const varstring sourceRowPath='/', const varstring sourceEncoding='utf8', const varstring destinationGroup, const varstring destinationLogicalName, integer4 timeOut=-1, const varstring espServerIpPort=GETENV('ws_fs_server'), integer4 maxConnections=-1, boolean allowoverwrite=false, boolean replicate=false,boolean compress=false, boolean failIfNoSourceFile=false, integer4 expireDays=-1, const varstring dfuServerQueue='', boolean noSplit=false, const varstring username = '', const varstring userPw = '', const varstring sourcePlane='', unsigned4 destinationNumParts=0, boolean noCommon=true) : c,action,context,entrypoint='fsSprayJson_v3'; + SprayFixed(const varstring sourceIP='', const varstring sourcePath, integer4 recordSize, const varstring destinationGroup, const varstring destinationLogicalName, integer4 timeOut=-1, const varstring espServerIpPort=GETENV('ws_fs_server'), integer4 maxConnections=-1, boolean allowoverwrite=false, boolean replicate=false,boolean compress=__CONTAINERIZED__, boolean failIfNoSourceFile=false, integer4 expireDays=-1, const varstring dfuServerQueue='', boolean noSplit=false, const varstring sourcePlane='', unsigned4 destinationNumParts=0, boolean noCommon=true) : c,action,context,entrypoint='fsSprayFixed_v6'; + SprayVariable(const varstring sourceIP='', const varstring sourcePath, integer4 sourceMaxRecordSize=8192, const varstring sourceCsvSeparate='\\,', const varstring sourceCsvTerminate='\\n,\\r\\n', const varstring sourceCsvQuote='\"', const varstring destinationGroup, const varstring destinationLogicalName, integer4 timeOut=-1, const varstring espServerIpPort=GETENV('ws_fs_server'), integer4 maxConnections=-1, boolean allowoverwrite=false, boolean replicate=false,boolean compress=__CONTAINERIZED__,const varstring sourceCsvEscape='', boolean failIfNoSourceFile=false, boolean recordStructurePresent=false, boolean quotedTerminator=true, const varstring encoding='ascii', integer4 expireDays=-1, const varstring dfuServerQueue='', boolean noSplit=false, const varstring sourcePlane='', unsigned4 destinationNumParts=0, boolean noCommon=true) : c,action,context,entrypoint='fsSprayVariable_v10'; + SprayXml(const varstring sourceIP='', const varstring sourcePath, integer4 sourceMaxRecordSize=8192, const varstring sourceRowTag, const varstring sourceEncoding='utf8', const varstring destinationGroup, const varstring destinationLogicalName, integer4 timeOut=-1, const varstring espServerIpPort=GETENV('ws_fs_server'), integer4 maxConnections=-1, boolean allowoverwrite=false, boolean replicate=false,boolean compress=__CONTAINERIZED__, boolean failIfNoSourceFile=false, integer4 expireDays=-1, const varstring dfuServerQueue='', boolean noSplit=false, const varstring sourcePlane='', unsigned4 destinationNumParts=0, boolean noCommon=true) : c,action,context,entrypoint='fsSprayXml_v6'; + SprayJson(const varstring sourceIP='', const varstring sourcePath, integer4 sourceMaxRecordSize=8192, const varstring sourceRowPath='/', const varstring sourceEncoding='utf8', const varstring destinationGroup, const varstring destinationLogicalName, integer4 timeOut=-1, const varstring espServerIpPort=GETENV('ws_fs_server'), integer4 maxConnections=-1, boolean allowoverwrite=false, boolean replicate=false,boolean compress=__CONTAINERIZED__, boolean failIfNoSourceFile=false, integer4 expireDays=-1, const varstring dfuServerQueue='', boolean noSplit=false, const varstring username = '', const varstring userPw = '', const varstring sourcePlane='', unsigned4 destinationNumParts=0, boolean noCommon=true) : c,action,context,entrypoint='fsSprayJson_v3'; Despray(const varstring logicalName, const varstring destinationIP='', const varstring destinationPath, integer4 timeOut=-1, const varstring espServerIpPort=GETENV('ws_fs_server'), integer4 maxConnections=-1, boolean allowoverwrite=false, const varstring destinationPlane='') : c,action,context,entrypoint='fsDespray2'; Copy(const varstring sourceLogicalName, const varstring destinationGroup, const varstring destinationLogicalName, const varstring sourceDali='', integer4 timeOut=-1, const varstring espServerIpPort=GETENV('ws_fs_server'), integer4 maxConnections=-1, boolean allowoverwrite=false, boolean replicate=false, boolean asSuperfile=false, boolean compress=false, boolean forcePush=false, integer4 transferBufferSize=0, boolean preserveCompression=true, boolean noSplit=false, integer4 expireDays=-1, boolean ensure=false) : c,action,context,entrypoint='fsCopy_v4'; Replicate(const varstring logicalName, integer4 timeOut=-1, const varstring espServerIpPort=GETENV('ws_fs_server')) : c,action,context,entrypoint='fsReplicate'; @@ -66,10 +66,10 @@ export FileServices := SERVICE : plugin('fileservices'), time AbortDfuWorkunit(const varstring wuid,const varstring espServerIpPort=GETENV('ws_fs_server')) : c,action,globalcontext,entrypoint='fsAbortDfuWorkunit'; MonitorLogicalFileName(const varstring event_name, const varstring name, integer4 shotcount=1,const varstring espServerIpPort=GETENV('ws_fs_server')) : c,action,context,entrypoint='fsMonitorLogicalFileName'; MonitorFile(const varstring event_name, const varstring ip, const varstring filename, boolean subdirs=false, integer4 shotcount=1,const varstring espServerIpPort=GETENV('ws_fs_server')) : c,action,context,entrypoint='fsMonitorFile'; - varstring fSprayFixed(const varstring sourceIP='', const varstring sourcePath, integer4 recordSize, const varstring destinationGroup, const varstring destinationLogicalName, integer4 timeOut=-1, const varstring espServerIpPort=GETENV('ws_fs_server'), integer4 maxConnections=-1, boolean allowoverwrite=false, boolean replicate=false, boolean compress=false, boolean failIfNoSourceFile=false, integer4 expireDays=-1, const varstring dfuServerQueue='', boolean noSplit=false, const varstring sourcePlane='', unsigned4 destinationNumParts=0, boolean noCommon=true) : c,action,context,entrypoint='fsfSprayFixed_v6'; - varstring fSprayVariable(const varstring sourceIP='', const varstring sourcePath, integer4 sourceMaxRecordSize=8192, const varstring sourceCsvSeparate='\\,', const varstring sourceCsvTerminate='\\n,\\r\\n', const varstring sourceCsvQuote='\"', const varstring destinationGroup, const varstring destinationLogicalName, integer4 timeOut=-1, const varstring espServerIpPort=GETENV('ws_fs_server'), integer4 maxConnections=-1, boolean allowoverwrite=false, boolean replicate=false, boolean compress=false,const varstring sourceCsvEscape='', boolean failIfNoSourceFile=false, boolean recordStructurePresent=false, boolean quotedTerminator=true, varstring encoding='ascii', integer4 expireDays=-1, const varstring dfuServerQueue='', boolean noSplit=false, const varstring sourcePlane='', unsigned4 destinationNumParts=0, boolean noCommon=true) : c,action,context,entrypoint='fsfSprayVariable_v10'; - varstring fSprayXml(const varstring sourceIP='', const varstring sourcePath, integer4 sourceMaxRecordSize=8192, const varstring sourceRowTag, const varstring sourceEncoding='utf8', const varstring destinationGroup, const varstring destinationLogicalName, integer4 timeOut=-1, const varstring espServerIpPort=GETENV('ws_fs_server'), integer4 maxConnections=-1, boolean allowoverwrite=false, boolean replicate=false, boolean compress=false, boolean failIfNoSourceFile=false, integer4 expireDays=-1, const varstring dfuServerQueue='', boolean noSplit=false, const varstring sourcePlane='', unsigned4 destinationNumParts=0, boolean noCommon=true) : c,action,context,entrypoint='fsfSprayXml_v6'; - varstring fSprayJson(const varstring sourceIP='', const varstring sourcePath, integer4 sourceMaxRecordSize=8192, const varstring sourceRowPath='/', const varstring sourceEncoding='utf8', const varstring destinationGroup, const varstring destinationLogicalName, integer4 timeOut=-1, const varstring espServerIpPort=GETENV('ws_fs_server'), integer4 maxConnections=-1, boolean allowoverwrite=false, boolean replicate=false, boolean compress=false, boolean failIfNoSourceFile=false, integer4 expireDays=-1, const varstring dfuServerQueue='', boolean noSplit=false, const varstring username = '', const varstring userPw = '', const varstring sourcePlane='', unsigned4 destinationNumParts=0, boolean noCommon=true) : c,action,context,entrypoint='fsfSprayJson_v3'; + varstring fSprayFixed(const varstring sourceIP='', const varstring sourcePath, integer4 recordSize, const varstring destinationGroup, const varstring destinationLogicalName, integer4 timeOut=-1, const varstring espServerIpPort=GETENV('ws_fs_server'), integer4 maxConnections=-1, boolean allowoverwrite=false, boolean replicate=false, boolean compress=__CONTAINERIZED__, boolean failIfNoSourceFile=false, integer4 expireDays=-1, const varstring dfuServerQueue='', boolean noSplit=false, const varstring sourcePlane='', unsigned4 destinationNumParts=0, boolean noCommon=true) : c,action,context,entrypoint='fsfSprayFixed_v6'; + varstring fSprayVariable(const varstring sourceIP='', const varstring sourcePath, integer4 sourceMaxRecordSize=8192, const varstring sourceCsvSeparate='\\,', const varstring sourceCsvTerminate='\\n,\\r\\n', const varstring sourceCsvQuote='\"', const varstring destinationGroup, const varstring destinationLogicalName, integer4 timeOut=-1, const varstring espServerIpPort=GETENV('ws_fs_server'), integer4 maxConnections=-1, boolean allowoverwrite=false, boolean replicate=false, boolean compress=__CONTAINERIZED__,const varstring sourceCsvEscape='', boolean failIfNoSourceFile=false, boolean recordStructurePresent=false, boolean quotedTerminator=true, varstring encoding='ascii', integer4 expireDays=-1, const varstring dfuServerQueue='', boolean noSplit=false, const varstring sourcePlane='', unsigned4 destinationNumParts=0, boolean noCommon=true) : c,action,context,entrypoint='fsfSprayVariable_v10'; + varstring fSprayXml(const varstring sourceIP='', const varstring sourcePath, integer4 sourceMaxRecordSize=8192, const varstring sourceRowTag, const varstring sourceEncoding='utf8', const varstring destinationGroup, const varstring destinationLogicalName, integer4 timeOut=-1, const varstring espServerIpPort=GETENV('ws_fs_server'), integer4 maxConnections=-1, boolean allowoverwrite=false, boolean replicate=false, boolean compress=__CONTAINERIZED__, boolean failIfNoSourceFile=false, integer4 expireDays=-1, const varstring dfuServerQueue='', boolean noSplit=false, const varstring sourcePlane='', unsigned4 destinationNumParts=0, boolean noCommon=true) : c,action,context,entrypoint='fsfSprayXml_v6'; + varstring fSprayJson(const varstring sourceIP='', const varstring sourcePath, integer4 sourceMaxRecordSize=8192, const varstring sourceRowPath='/', const varstring sourceEncoding='utf8', const varstring destinationGroup, const varstring destinationLogicalName, integer4 timeOut=-1, const varstring espServerIpPort=GETENV('ws_fs_server'), integer4 maxConnections=-1, boolean allowoverwrite=false, boolean replicate=false, boolean compress=__CONTAINERIZED__, boolean failIfNoSourceFile=false, integer4 expireDays=-1, const varstring dfuServerQueue='', boolean noSplit=false, const varstring username = '', const varstring userPw = '', const varstring sourcePlane='', unsigned4 destinationNumParts=0, boolean noCommon=true) : c,action,context,entrypoint='fsfSprayJson_v3'; varstring fDespray(const varstring logicalName, const varstring destinationIP='', const varstring destinationPath, integer4 timeOut=-1, const varstring espServerIpPort=GETENV('ws_fs_server'), integer4 maxConnections=-1, boolean allowoverwrite=false, const varstring destinationPlane='') : c,action,context,entrypoint='fsfDespray2'; varstring fCopy(const varstring sourceLogicalName, const varstring destinationGroup, const varstring destinationLogicalName, const varstring sourceDali='', integer4 timeOut=-1, const varstring espServerIpPort=GETENV('ws_fs_server'), integer4 maxConnections=-1, boolean allowoverwrite=false, boolean replicate=false, boolean asSuperfile=false, boolean compress=false, boolean forcePush=false, integer4 transferBufferSize=0, boolean preserveCompression=true, boolean noSplit=false, integer4 expireDays=-1, boolean ensure=false) : c,action,context,entrypoint='fsfCopy_v4'; varstring fMonitorLogicalFileName(const varstring event_name, const varstring name, integer4 shotcount=1,const varstring espServerIpPort=GETENV('ws_fs_server')) : c,action,context,entrypoint='fsfMonitorLogicalFileName'; From ae125c4db0b98951fcd3d7377f3eaccc8438f3ee Mon Sep 17 00:00:00 2001 From: Gordon Smith Date: Mon, 26 Feb 2024 20:18:24 +0000 Subject: [PATCH 13/24] HPCC-31355 Windows build failing in GH Actions Older versions of MS STL is missing an implementation for "_Thrd_sleep_for" Signed-off-by: Gordon Smith --- system/jlib/jtrace.cpp | 7 ++++++- vcpkg.json.in | 17 +++++++++-------- 2 files changed, 15 insertions(+), 9 deletions(-) diff --git a/system/jlib/jtrace.cpp b/system/jlib/jtrace.cpp index 4c3c811b201..9014c7bf282 100644 --- a/system/jlib/jtrace.cpp +++ b/system/jlib/jtrace.cpp @@ -15,7 +15,6 @@ limitations under the License. ############################################################################## */ - #include "opentelemetry/trace/semantic_conventions.h" //known span defines #include "opentelemetry/context/propagation/global_propagator.h" // context::propagation::GlobalTextMapPropagator::GetGlobalPropagator #include "opentelemetry/sdk/trace/tracer_provider_factory.h" //opentelemetry::sdk::trace::TracerProviderFactory::Create(context) @@ -1425,3 +1424,9 @@ ITraceManager & queryTraceManager() { return *theTraceManager.query([] () { return new CTraceManager; }); //throws if not initialized } + +#if WIN32 && _MSC_VER < 1939 // _MSC_VER < VS 2022 17.9 +extern "C" void __stdcall _Thrd_sleep_for(const unsigned long ms) noexcept { // suspend current thread for `ms` milliseconds + Sleep(ms); +} +#endif diff --git a/vcpkg.json.in b/vcpkg.json.in index 199c2b42bde..535e9bb3dbe 100644 --- a/vcpkg.json.in +++ b/vcpkg.json.in @@ -168,6 +168,15 @@ "name": "openssl", "platform": "@VCPKG_OPENSSL@" }, + { + "name": "opentelemetry-cpp", + "default-features": false, + "features": [ + "otlp", + "otlp-http", + "otlp-grpc" + ] + }, { "name": "python3", "platform": "@VCPKG_PYTHON3@ & windows" @@ -192,14 +201,6 @@ { "name": "zlib", "platform": "@VCPKG_ZLIB@" - }, - { - "name": "opentelemetry-cpp", - "features": [ - "otlp", - "otlp-http", - "otlp-grpc" - ] } ] } \ No newline at end of file From 513ceb7eec292af178f5548354a9f9480e304683 Mon Sep 17 00:00:00 2001 From: Gavin Halliday Date: Tue, 27 Feb 2024 13:40:45 +0000 Subject: [PATCH 14/24] HPCC-31342 Improve the accuracy of the roxie span elapsed time to ns Signed-off-by: Gavin Halliday --- roxie/ccd/ccd.hpp | 2 +- roxie/ccd/ccdlistener.cpp | 40 ++++++++++++++++++++------------------ roxie/ccd/ccdprotocol.cpp | 17 ++++++++-------- roxie/ccd/hpccprotocol.hpp | 2 +- 4 files changed, 32 insertions(+), 29 deletions(-) diff --git a/roxie/ccd/ccd.hpp b/roxie/ccd/ccd.hpp index 1bae04e0101..dfaa5a22fd2 100644 --- a/roxie/ccd/ccd.hpp +++ b/roxie/ccd/ccd.hpp @@ -786,7 +786,7 @@ class ContextLogger : implements IRoxieContextLogger, public CInterface { stats.recordStatistics(progress, false); } - void exportStatsToSpan(bool failed, unsigned elapsed, unsigned memused, unsigned agentsDuplicates, unsigned agentsResends); + void exportStatsToSpan(bool failed, stat_type elapsedNs, unsigned memused, unsigned agentsDuplicates, unsigned agentsResends); }; class StringContextLogger : public ContextLogger diff --git a/roxie/ccd/ccdlistener.cpp b/roxie/ccd/ccdlistener.cpp index f8c0838c1b1..7badb16cc46 100644 --- a/roxie/ccd/ccdlistener.cpp +++ b/roxie/ccd/ccdlistener.cpp @@ -945,11 +945,11 @@ extern void updateAffinity(unsigned __int64 affinity) //-------------------------------------------------------------------------------------------------------------------- -void ContextLogger::exportStatsToSpan(bool failed, unsigned elapsed, unsigned memused, unsigned agentsDuplicates, unsigned agentsResends) +void ContextLogger::exportStatsToSpan(bool failed, stat_type elapsedNs, unsigned memused, unsigned agentsDuplicates, unsigned agentsResends) { if (activeSpan->isRecording()) { - setSpanAttribute("time_elapsed", elapsed*1000000ULL); + setSpanAttribute("time_elapsed", elapsedNs); if (failed) setSpanAttribute("num_failures", 1); if (memused) @@ -1001,7 +1001,7 @@ class RoxieWorkUnitListener : public RoxieListener worker->threadmain(); } - virtual void noteQuery(IHpccProtocolMsgContext *msgctx, const char *peer, bool failed, unsigned bytesOut, unsigned elapsed, unsigned memused, unsigned agentsReplyLen, unsigned agentsDuplicates, unsigned agentsResends, bool continuationNeeded, unsigned requestArraySize) + virtual void noteQuery(IHpccProtocolMsgContext *msgctx, const char *peer, bool failed, unsigned bytesOut, stat_type elapsedNs, unsigned memused, unsigned agentsReplyLen, unsigned agentsDuplicates, unsigned agentsResends, bool continuationNeeded, unsigned requestArraySize) { } @@ -1110,14 +1110,14 @@ class RoxieQueryWorker : implements IPooledThread, public CInterface RoxieQueryWorker(RoxieListener *_pool) { pool = _pool; - qstart = msTick(); + startNs = nsTick(); time(&startTime); } // interface IPooledThread virtual void init(void *) override { - qstart = msTick(); + startNs = nsTick(); time(&startTime); } @@ -1135,7 +1135,7 @@ class RoxieQueryWorker : implements IPooledThread, public CInterface protected: RoxieListener *pool; - unsigned qstart; + stat_type startNs; time_t startTime; }; @@ -1316,9 +1316,10 @@ class RoxieWorkUnitWorker : public RoxieQueryWorker reportUnknownException(wu, logctx); } #endif - unsigned elapsed = msTick() - qstart; - noteQuery(failed, elapsed, priority); - queryFactory->noteQuery(startTime, failed, elapsed, memused, agentsReplyLen, 0); + unsigned elapsedNs = nsTick() - startNs; + unsigned elapsedMs = nanoToMilli(elapsedNs); + noteQuery(failed, elapsedMs, priority); + queryFactory->noteQuery(startTime, failed, elapsedMs, memused, agentsReplyLen, 0); if (logctx.queryTraceLevel() && (logFullQueries || logctx.intercept)) { StringBuffer s; @@ -1340,9 +1341,9 @@ class RoxieWorkUnitWorker : public RoxieQueryWorker txidInfo.append(']'); } - logctx.CTXLOG("COMPLETE: %s%s complete in %u msecs memory=%u Mb priority=%d agentsreply=%u duplicatePackets=%u resentPackets=%u%s", wuid.get(), txidInfo.str(), elapsed, memused, priority, agentsReplyLen, agentsDuplicates, agentsResends, s.str()); + logctx.CTXLOG("COMPLETE: %s%s complete in %u msecs memory=%u Mb priority=%d agentsreply=%u duplicatePackets=%u resentPackets=%u%s", wuid.get(), txidInfo.str(), elapsedMs, memused, priority, agentsReplyLen, agentsDuplicates, agentsResends, s.str()); } - logctx.exportStatsToSpan(failed, elapsed, memused, agentsDuplicates, agentsResends); + logctx.exportStatsToSpan(failed, elapsedNs, memused, agentsDuplicates, agentsResends); } private: @@ -1603,12 +1604,13 @@ class RoxieProtocolMsgContext : implements IHpccProtocolMsgContext, public CInte combinedQueryStats.noteQuery(failed, elapsedTime); } } - void noteQuery(const char *peer, bool failed, unsigned elapsed, unsigned memused, unsigned agentsReplyLen, unsigned agentsDuplicates, unsigned agentsResends, unsigned bytesOut, bool continuationNeeded, unsigned requestArraySize) + void noteQuery(const char *peer, bool failed, stat_type elapsedNs, unsigned memused, unsigned agentsReplyLen, unsigned agentsDuplicates, unsigned agentsResends, unsigned bytesOut, bool continuationNeeded, unsigned requestArraySize) { - noteQueryStats(failed, elapsed); + unsigned elapsedMs = nanoToMilli(elapsedNs); + noteQueryStats(failed, elapsedMs); if (queryFactory) { - queryFactory->noteQuery(startTime, failed, elapsed, memused, agentsReplyLen, bytesOut); + queryFactory->noteQuery(startTime, failed, elapsedMs, memused, agentsReplyLen, bytesOut); queryFactory.clear(); } if (logctx) @@ -1634,14 +1636,14 @@ class RoxieProtocolMsgContext : implements IHpccProtocolMsgContext, public CInte if (txIds.length()) txIds.insert(0, '[').append(']'); if (requestArraySize > 1) - logctx->CTXLOG("COMPLETE: %s(x%u) %s%s from %s complete in %u msecs memory=%u Mb priority=%d agentsreply=%u duplicatePackets=%u resentPackets=%u resultsize=%u continue=%d%s", queryName.get(), requestArraySize, uid.get(), txIds.str(), peer, elapsed, memused, getQueryPriority(), agentsReplyLen, agentsDuplicates, agentsResends, bytesOut, continuationNeeded, s.str()); + logctx->CTXLOG("COMPLETE: %s(x%u) %s%s from %s complete in %u msecs memory=%u Mb priority=%d agentsreply=%u duplicatePackets=%u resentPackets=%u resultsize=%u continue=%d%s", queryName.get(), requestArraySize, uid.get(), txIds.str(), peer, elapsedMs, memused, getQueryPriority(), agentsReplyLen, agentsDuplicates, agentsResends, bytesOut, continuationNeeded, s.str()); else - logctx->CTXLOG("COMPLETE: %s %s%s from %s complete in %u msecs memory=%u Mb priority=%d agentsreply=%u duplicatePackets=%u resentPackets=%u resultsize=%u continue=%d%s", queryName.get(), uid.get(), txIds.str(), peer, elapsed, memused, getQueryPriority(), agentsReplyLen, agentsDuplicates, agentsResends, bytesOut, continuationNeeded, s.str()); + logctx->CTXLOG("COMPLETE: %s %s%s from %s complete in %u msecs memory=%u Mb priority=%d agentsreply=%u duplicatePackets=%u resentPackets=%u resultsize=%u continue=%d%s", queryName.get(), uid.get(), txIds.str(), peer, elapsedMs, memused, getQueryPriority(), agentsReplyLen, agentsDuplicates, agentsResends, bytesOut, continuationNeeded, s.str()); } } - logctx->exportStatsToSpan(failed, elapsed, memused, agentsDuplicates, agentsResends); + logctx->exportStatsToSpan(failed, elapsedNs, memused, agentsDuplicates, agentsResends); } } }; @@ -1922,10 +1924,10 @@ class RoxieProtocolMsgSink : implements IHpccNativeProtocolMsgSink, public CInte roxieMsgCtx->ensureDebugCommandHandler().doDebugCommand(msg, &roxieMsgCtx->ensureDebuggerContext(uid), out); } - virtual void noteQuery(IHpccProtocolMsgContext *msgctx, const char *peer, bool failed, unsigned bytesOut, unsigned elapsed, unsigned memused, unsigned agentsReplyLen, unsigned agentsDuplicates, unsigned agentsResends, bool continuationNeeded, unsigned requestArraySize) + virtual void noteQuery(IHpccProtocolMsgContext *msgctx, const char *peer, bool failed, unsigned bytesOut, stat_type elapsedNs, unsigned memused, unsigned agentsReplyLen, unsigned agentsDuplicates, unsigned agentsResends, bool continuationNeeded, unsigned requestArraySize) { RoxieProtocolMsgContext *roxieMsgCtx = checkGetRoxieMsgContext(msgctx); - roxieMsgCtx->noteQuery(peer, failed, elapsed, memused, agentsReplyLen, agentsDuplicates, agentsResends, bytesOut, continuationNeeded, requestArraySize); + roxieMsgCtx->noteQuery(peer, failed, elapsedNs, memused, agentsReplyLen, agentsDuplicates, agentsResends, bytesOut, continuationNeeded, requestArraySize); } }; diff --git a/roxie/ccd/ccdprotocol.cpp b/roxie/ccd/ccdprotocol.cpp index c7d8841bc57..d5a4fa01e84 100644 --- a/roxie/ccd/ccdprotocol.cpp +++ b/roxie/ccd/ccdprotocol.cpp @@ -366,14 +366,14 @@ class ProtocolQueryWorker : public CInterface, implements IPooledThread ProtocolQueryWorker(ProtocolListener *_listener) : listener(_listener) { - qstart = msTick(); + startNs = nsTick(); time(&startTime); } // interface IPooledThread virtual void init(void *) override { - qstart = msTick(); + startNs = nsTick(); time(&startTime); } @@ -391,7 +391,7 @@ class ProtocolQueryWorker : public CInterface, implements IPooledThread protected: ProtocolListener *listener; - unsigned qstart; + stat_type startNs; time_t startTime; }; @@ -1733,7 +1733,7 @@ class RoxieSocketWorker : public ProtocolQueryWorker if (resetQstart) { resetQstart = false; - qstart = msTick(); + startNs = nsTick(); time(&startTime); msgctx.setown(sink->createMsgContext(startTime)); } @@ -1846,7 +1846,7 @@ class RoxieSocketWorker : public ProtocolQueryWorker Owned protocol = createProtocolResponse(queryPT->queryName(), client, httpHelper, logctx, protocolFlags | HPCC_PROTOCOL_CONTROL, global->defaultXmlReadFlags); sink->onControlMsg(msgctx, queryPT, protocol); protocol->finalize(0); - if (streq(queryName, "lock") || streq(queryName, "childlock")) //don't reset qstart, lock time should be included + if (streq(queryName, "lock") || streq(queryName, "childlock")) //don't reset startNs, lock time should be included goto readAnother; } else if (isStatus) @@ -2107,7 +2107,8 @@ class RoxieSocketWorker : public ProtocolQueryWorker } } unsigned bytesOut = client? client->bytesOut() : 0; - unsigned elapsed = msTick() - qstart; + stat_type elapsedNs = nsTick() - startNs; + unsigned elapsedMs = nanoToMilli(elapsedNs); if (client) { logctx.noteStatistic(StTimeSocketReadIO, client->getStatistic(StTimeSocketReadIO)); @@ -2118,7 +2119,7 @@ class RoxieSocketWorker : public ProtocolQueryWorker logctx.noteStatistic(StNumSocketWrites, client->getStatistic(StNumSocketWrites)); } - sink->noteQuery(msgctx.get(), peerStr, failed, bytesOut, elapsed, memused, agentsReplyLen, agentsDuplicates, agentsResends, continuationNeeded, requestArraySize); + sink->noteQuery(msgctx.get(), peerStr, failed, bytesOut, elapsedNs, memused, agentsReplyLen, agentsDuplicates, agentsResends, continuationNeeded, requestArraySize); if (continuationNeeded) { rawText.clear(); @@ -2158,7 +2159,7 @@ class RoxieSocketWorker : public ProtocolQueryWorker { FlushingStringBuffer response(client, (protocolFlags & HPCC_PROTOCOL_BLOCKED), mlResponseFmt, (protocolFlags & HPCC_PROTOCOL_NATIVE_RAW), false, logctx); response.startDataset("SummaryStats", NULL, (unsigned) -1); - VStringBuffer s(" COMPLETE: %s %s complete in %u msecs memory=%u Mb agentsreply=%u duplicatePackets=%u resentPackets=%u resultsize=%u continue=%d", queryName.get(), uid, elapsed, memused, agentsReplyLen, agentsDuplicates, agentsResends, bytesOut, continuationNeeded); + VStringBuffer s(" COMPLETE: %s %s complete in %u msecs memory=%u Mb agentsreply=%u duplicatePackets=%u resentPackets=%u resultsize=%u continue=%d", queryName.get(), uid, elapsedMs, memused, agentsReplyLen, agentsDuplicates, agentsResends, bytesOut, continuationNeeded); logctx.getStats(s).newline(); response.flushXML(s, true); } diff --git a/roxie/ccd/hpccprotocol.hpp b/roxie/ccd/hpccprotocol.hpp index 8b7114f9e63..08fd7a3f952 100644 --- a/roxie/ccd/hpccprotocol.hpp +++ b/roxie/ccd/hpccprotocol.hpp @@ -96,7 +96,7 @@ interface IHpccProtocolMsgSink : extends IInterface virtual IHpccProtocolMsgContext *createMsgContext(time_t startTime) = 0; virtual StringArray &getTargetNames(StringArray &targets) = 0; - virtual void noteQuery(IHpccProtocolMsgContext *msgctx, const char *peer, bool failed, unsigned bytesOut, unsigned elapsed, unsigned memused, unsigned agentsReplyLen, unsigned agentDuplicates, unsigned agentResends, bool continuationNeeded, unsigned requestArraySize) = 0; + virtual void noteQuery(IHpccProtocolMsgContext *msgctx, const char *peer, bool failed, unsigned bytesOut, stat_type elapsedNs, unsigned memused, unsigned agentsReplyLen, unsigned agentDuplicates, unsigned agentResends, bool continuationNeeded, unsigned requestArraySize) = 0; virtual void onQueryMsg(IHpccProtocolMsgContext *msgctx, IPropertyTree *msg, IHpccProtocolResponse *protocol, unsigned flags, PTreeReaderOptions readFlags, const char *target, unsigned idx, unsigned &memused, unsigned &agentReplyLen, unsigned &agentDuplicates, unsigned &agentResends, StringAttr &statsWuid) = 0; }; From adfdfdfa9a5a5ba901353f38869d29a015ccd50c Mon Sep 17 00:00:00 2001 From: Gordon Smith Date: Tue, 27 Feb 2024 18:12:33 +0000 Subject: [PATCH 15/24] HPCC-31355 Potential issue with WIN32 use Signed-off-by: Gordon Smith --- system/jlib/jtrace.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/system/jlib/jtrace.cpp b/system/jlib/jtrace.cpp index 9014c7bf282..bc213c6c197 100644 --- a/system/jlib/jtrace.cpp +++ b/system/jlib/jtrace.cpp @@ -1425,7 +1425,7 @@ ITraceManager & queryTraceManager() return *theTraceManager.query([] () { return new CTraceManager; }); //throws if not initialized } -#if WIN32 && _MSC_VER < 1939 // _MSC_VER < VS 2022 17.9 +#if defined(_MSC_VER) && _MSC_VER < 1939 // _MSC_VER < VS 2022 17.9 extern "C" void __stdcall _Thrd_sleep_for(const unsigned long ms) noexcept { // suspend current thread for `ms` milliseconds Sleep(ms); } From 85859750c2f7ecd9a45eb25b503ebba89fac981e Mon Sep 17 00:00:00 2001 From: Jeremy Clements <79224539+jeclrsg@users.noreply.github.com> Date: Tue, 27 Feb 2024 15:03:38 -0500 Subject: [PATCH 16/24] HPCC-31360 ECL Watch fix Lock links Signed-off-by: Jeremy Clements <79224539+jeclrsg@users.noreply.github.com> --- esp/src/eclwatch/LockDialogWidget.js | 2 +- esp/src/src-react/components/Title.tsx | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/esp/src/eclwatch/LockDialogWidget.js b/esp/src/eclwatch/LockDialogWidget.js index 3028c6cd3b5..badf1e2f795 100644 --- a/esp/src/eclwatch/LockDialogWidget.js +++ b/esp/src/eclwatch/LockDialogWidget.js @@ -153,7 +153,7 @@ define([ cookie("Status", "Locked"); context.storage.setItem("Status", "Locked"); } else if (cookie("Status") === "Unlocked") { - xhr("esp/lock", { + xhr("/esp/lock", { method: "post" }).then(function (response) { if (response) { diff --git a/esp/src/src-react/components/Title.tsx b/esp/src/src-react/components/Title.tsx index ec472bbc373..9af34dfa076 100644 --- a/esp/src/src-react/components/Title.tsx +++ b/esp/src/src-react/components/Title.tsx @@ -67,7 +67,7 @@ export const DevTitle: React.FunctionComponent = ({ const titlebarColorSet = React.useMemo(() => { return titlebarColor && titlebarColor !== theme.palette.themeLight; - }, [theme.palette, titlebarColor]); + }, [theme.palette, titlebarColor]); const personaProps: IPersonaSharedProps = React.useMemo(() => { return { @@ -139,7 +139,7 @@ export const DevTitle: React.FunctionComponent = ({ { key: "divider_2", itemType: ContextualMenuItemType.Divider }, { key: "lock", text: nlsHPCC.Lock, disabled: !currentUser?.username, onClick: () => { - fetch("esp/lock", { + fetch("/esp/lock", { method: "post" }).then(() => { setUserSession({ ...userSession, Status: "Locked" }); @@ -287,7 +287,7 @@ export const DevTitle: React.FunctionComponent = ({ - + Date: Wed, 14 Feb 2024 12:29:38 +0000 Subject: [PATCH 17/24] HPCC-31281 Warn if configuration may lead to excessive tracing Some configuration settings are designed for use only while temporarily diagnosing an issue, and should not be left on permanently due to excessive logging impact. Generate a warning if such options are set. Signed-off-by: Richard Chapman --- helm/hpcc/templates/_warnings.tpl | 36 +++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/helm/hpcc/templates/_warnings.tpl b/helm/hpcc/templates/_warnings.tpl index 70e590c7b4e..e4aa25e2def 100644 --- a/helm/hpcc/templates/_warnings.tpl +++ b/helm/hpcc/templates/_warnings.tpl @@ -73,6 +73,42 @@ Pass in dict with root and warnings {{- $_ := set $ctx "warnings" (append $ctx.warnings $warning) -}} {{- end -}} {{- end -}} + {{- /* Warn if any tracing flags are set in Roxie */ -}} + {{- range $roxie := .root.Values.roxie -}} + {{- if not $roxie.disabled -}} + {{- range $key, $value := $roxie -}} + {{- if or (eq "traceLevel" $key) (eq "udpTraceLevel" $key) -}} + {{- if gt (int $value) 1 -}} + {{- $warning := dict "source" "helm" "severity" "warning" -}} + {{- $_ := set $warning "msg" (printf "Roxie %s tracing option %s: %s may result in excessive logging" $roxie.name $key (toString $value)) -}} + {{- $_ := set $ctx "warnings" (append $ctx.warnings $warning) -}} + {{- end -}} + {{- else if or (hasPrefix "trace" $key) (hasPrefix "udpTrace" $key) -}} + {{- if $value -}} + {{- $warning := dict "source" "helm" "severity" "warning" -}} + {{- $_ := set $warning "msg" (printf "Roxie %s tracing option %s: %s may result in excessive logging" $roxie.name $key (toString $value)) -}} + {{- $_ := set $ctx "warnings" (append $ctx.warnings $warning) -}} + {{- end -}} + {{- end -}} + {{- end -}} + {{- end -}} + {{- end -}} + {{- /* Warn if any expert flags are set */ -}} + {{- range $cname, $ctypes := .root.Values -}} + {{- range $id, $component := $ctypes -}} + {{- if kindIs "map" $component -}} + {{- if not $component.disabled -}} + {{- range $key, $value := $component.expert -}} + {{- if $value -}} + {{- $warning := dict "source" "helm" "severity" "warning" -}} + {{- $_ := set $warning "msg" (printf "%s %s expert option %s: %s is set" (title $cname) $component.name $key (toString $value)) -}} + {{- $_ := set $ctx "warnings" (append $ctx.warnings $warning) -}} + {{- end -}} + {{- end -}} + {{- end -}} + {{- end -}} + {{- end -}} + {{- end -}} {{- /* Warn when resources not provided, default cpu rate used and components requiring resources for cost calcs */ -}} {{- if eq .root.Values.global.cost.perCpu 0.0565000000001 -}} {{- $_ := set $ctx "usingDefaultCpuCost" "true" -}} From 003d45098a33b8bd6a17331fabb283dc5a77a3d0 Mon Sep 17 00:00:00 2001 From: Gavin Halliday Date: Tue, 12 Dec 2023 08:55:11 +0000 Subject: [PATCH 18/24] HPCC-31344 Extend roxie end-to-end testing Signed-off-by: Gavin Halliday --- testing/regress/ecl/common/SoapTextTest.ecl | 98 +++++++++++++++++---- testing/regress/ecl/soaptext1.ecl | 55 ++++++++++-- 2 files changed, 133 insertions(+), 20 deletions(-) diff --git a/testing/regress/ecl/common/SoapTextTest.ecl b/testing/regress/ecl/common/SoapTextTest.ecl index 26b1d0fbbf9..25d91cd69a0 100644 --- a/testing/regress/ecl/common/SoapTextTest.ecl +++ b/testing/regress/ecl/common/SoapTextTest.ecl @@ -28,6 +28,28 @@ files := setup.files(multiPart, false); EXPORT SoapTextTest := MODULE + EXPORT ConnectionType := ENUM(Plain, Encrypted, Trusted); + + EXPORT CallOptionsRecord := RECORD + ConnectionType connectionToRoxie := ConnectionType.Encrypted; + ConnectionType connectionToEsp := ConnectionType.Encrypted; + boolean persistConnectToRoxie := FALSE; + boolean persistConnectToEsp := FALSE; + boolean connectDirectToRoxie := FALSE; + boolean embedServiceCalls := FALSE; + END; + + EXPORT ConfigOptionsRecord := RECORD + string remoteEspUrl := ''; + string remoteRoxieUrl := ''; + string RoxieUrl_Plain := ''; + string RoxieUrl_Encrypted := ''; + string RoxieUrl_Trusted := ''; + string EspUrl_Plain := ''; + string EspUrl_Encrypted := ''; + string EspUrl_Trusted := ''; + END; + import Std, Setup.Ts; EXPORT wordRec := { string word; }; @@ -43,6 +65,7 @@ EXPORT SoapTextTest := MODULE EXPORT countServiceRequest := RECORD STRING search; + CallOptionsRecord callOptions; END; EXPORT countServiceResponse := RECORD UNSIGNED cnt; @@ -54,6 +77,11 @@ EXPORT SoapTextTest := MODULE END; //External searvice2: Given a list of words, find the documents they all occur in, and return the number of counts of each word + EXPORT JoinServiceRequestRecord := RECORD + DATASET(wordRec) search; + CallOptionsRecord callOptions; + END; + EXPORT joinServiceResponseRecord := RECORD Ts.DocumentId doc; STRING word; @@ -84,13 +112,33 @@ EXPORT SoapTextTest := MODULE RETURN OUTPUT(doSearchWords(searchWords)); END; - EXPORT doMain(string serviceUrl, string searchWords, unsigned documentLimit) := FUNCTION + EXPORT MainServiceRequestRecord := RECORD + string searchWords := ''; + unsigned documentLimit := 3; + string serviceUrl := ''; + unsigned maxResults := 50; + + CallOptionsRecord callOptions; + ConfigOptionsRecord configOptions; + END; + + //The response from the main service + EXPORT MainServiceResponseRecord := RECORD + Ts.DocumentId doc; + SET OF STRING words; + SET OF UNSIGNED counts; + END; + + EXPORT doMain(string serviceUrl, string searchWords, unsigned documentLimit, CallOptionsRecord callOptions, ConfigOptionsRecord configOptions) := FUNCTION - soapcallDocumentCount(string searchWord) := SOAPCALL(serviceUrl, 'soaptest_getdocumentcount', countServiceRequest, transform(countServiceRequest, SELF.search := searchWord), countServiceResponse, PERSIST).cnt; - callDocumentCount(string search) := IF((serviceUrl != ''), soapcallDocumentCount(search), doDocumentCount(search)); + boolean usePersistConnection := IF(callOptions.connectDirectToRoxie, callOptions.persistConnectToRoxie, callOptions.persistConnectToEsp); + unsigned persistConnectCount := IF(usePersistConnection, 100, 0); - soapcallSearchWords(DATASET(wordRec) searchWords) := SOAPCALL(serviceUrl, 'soaptest_getsearchwords', { DATASET(wordRec) search := searchWords }, DATASET(joinServiceResponseRecord), PERSIST); - callSearchWords(DATASET(wordRec) searchWords) := IF((serviceUrl != ''), soapcallSearchWords(searchWords), doSearchWords(searchWords)); + soapcallDocumentCount(string searchWord) := SOAPCALL(serviceUrl, 'soaptest_getdocumentcount', countServiceRequest, transform(countServiceRequest, SELF.search := searchWord, SELF.callOptions := callOptions), countServiceResponse, PERSIST(persistConnectCount),timeout(1)).cnt; + callDocumentCount(string search) := IF(NOT callOptions.embedServiceCalls, soapcallDocumentCount(search), doDocumentCount(search)); + + soapcallSearchWords(DATASET(wordRec) searchWords) := SOAPCALL(serviceUrl, 'soaptest_getsearchwords', JoinServiceRequestRecord, transform(JoinServiceRequestRecord, SELF.search := searchWords, SELF.callOptions := callOptions), DATASET(joinServiceResponseRecord), PERSIST(persistConnectCount),timeout(1)); + callSearchWords(DATASET(wordRec) searchWords) := IF(NOT callOptions.embedServiceCalls, soapcallSearchWords(searchWords), doSearchWords(searchWords)); splitWords := Std.Str.SplitWords(searchWords, ',', false); splitWordsDs := DATASET(splitwords, wordRec); @@ -102,13 +150,7 @@ EXPORT SoapTextTest := MODULE searchAgain := PROJECT(leastCommon, TRANSFORM(wordRec, SELF.word := LEFT.word)); joinLeastCommon := callSearchWords(searchAgain); - rollupRecord := RECORD - Ts.DocumentId doc; - SET OF STRING words; - SET OF UNSIGNED counts; - END; - - rollupRecord rollupWords(joinServiceResponseRecord l, DATASET(joinServiceResponseRecord) matches) := TRANSFORM + MainServiceResponseRecord rollupWords(joinServiceResponseRecord l, DATASET(joinServiceResponseRecord) matches) := TRANSFORM SELF.doc := l.doc; sortedWords := SORT(matches, -cnt); SELF.words := SET(sortedWords, word); @@ -119,14 +161,40 @@ EXPORT SoapTextTest := MODULE RETURN rolledUp; END; + // The published search service take a list of words, and a maximum number of significant documents + EXPORT runMainService(string searchWords, unsigned documentLimit, unsigned maxResults, CallOptionsRecord callOptions, ConfigOptionsRecord configOptions) := FUNCTION + + defaultRoxieUrl := IF(configOptions.remoteRoxieUrl != '', configOptions.remoteRoxieUrl, '.'); + defaultEspUrl := IF(configOptions.remoteEspUrl != '', configOptions.remoteEspUrl, '.'); + plainRoxieUrl := IF(configOptions.RoxieUrl_Plain != '', configOptions.RoxieUrl_Plain, defaultRoxieUrl + ':9876'); + encryptedRoxieUrl := IF(configOptions.RoxieUrl_Encrypted != '', configOptions.RoxieUrl_Encrypted, defaultRoxieUrl + ':19876'); + trustedRoxieUrl := IF(configOptions.RoxieUrl_Trusted != '', configOptions.RoxieUrl_Trusted, defaultRoxieUrl + ':29876'); + plainEspUrl := IF(configOptions.EspUrl_Plain != '', configOptions.EspUrl_Plain, defaultEspUrl + ':8002//WsEcl/soap/query/roxie'); + encryptedEspUrl := IF(configOptions.EspUrl_Encrypted != '', configOptions.EspUrl_Encrypted, defaultEspUrl + ':18002//WsEcl/soap/query/roxie'); + trustedEspUrl := IF(configOptions.EspUrl_Trusted != '', configOptions.EspUrl_Trusted, defaultEspUrl + ':28002//WsEcl/soap/query/roxie '); + + roxieUrl := CASE(callOptions.connectionToRoxie, + ConnectionType.Plain => 'http://' + plainRoxieUrl, + ConnectionType.Encrypted => 'https://' + encryptedRoxieUrl, + 'https://' + trustedRoxieUrl); + espUrl := CASE(callOptions.connectionToEsp, + ConnectionType.Plain => 'http://' + plainEspUrl, + ConnectionType.Encrypted => 'https://' + encryptedEspUrl, + 'https://' + trustedEspUrl); + serviceUrl := IF(callOptions.connectDirectToRoxie, roxieUrl, espUrl); + + RETURN OUTPUT(CHOOSEN(doMain(serviceUrl, searchWords, documentLimit, callOptions, configOptions), maxResults)); + END; + EXPORT mainService() := FUNCTION // The published search service take a list of words, and a maximum number of significant documents string searchWords := '' : stored('searchWords'); unsigned documentLimit := 3 : stored('documentLimit'); - serviceUrl := '' : stored('url'); unsigned maxResults := 50; - RETURN OUTPUT(CHOOSEN(doMain(serviceUrl, searchWords, documentLimit), maxResults)); - END; + callOptions := ROW(CallOptionsRecord) : stored('callOptions'); + configOptions := ROW(ConfigOptionsRecord) : stored('configOptions'); + RETURN runMainService(searchWords, documentLimit, maxResults, callOptions, configOptions); + END; END; diff --git a/testing/regress/ecl/soaptext1.ecl b/testing/regress/ecl/soaptext1.ecl index a54fd2241aa..97bfd53c212 100644 --- a/testing/regress/ecl/soaptext1.ecl +++ b/testing/regress/ecl/soaptext1.ecl @@ -15,18 +15,63 @@ limitations under the License. ############################################################################## */ -//xversion url='http://.:9876' //nothor //nohthor -TargetIP := '.' : STORED('TargetIP'); -serviceUrl := 'http://'+TargetIP+':9876'; +//version callDirect=true +//version callViaEsp=false,persistConnection=false,encryptConnection=false +//version callViaEsp=false,persistConnection=true,encryptConnection=false +//xversion callViaEsp=false,persistConnection=false,encryptConnection=true +//xversion callViaEsp=false,persistConnection=true,encryptConnection=true +//version callViaEsp=true,persistConnection=false,encryptConnection=false +//version callViaEsp=true,persistConnection=true,encryptConnection=false +//xversion callViaEsp=true,persistConnection=false,encryptConnection=true +//xversion callViaEsp=true,persistConnection=true,encryptConnection=true + +import ^ as root; + +//Simplified configuration +roxieIP := '.' : STORED('TargetIP'); +espIP := '.' : STORED('TargetIP'); + +//Simplified call options - doesn't provide all combinations that should be tested eventually +boolean encryptConnection := #IFDEFINED(root.encryptConnection, false); +boolean persistConnection := #IFDEFINED(root.persistConnection, false); +boolean callDirect := #IFDEFINED(root.callDirect, false); +boolean callViaEsp := #IFDEFINED(root.callViaEsp, false); //--- end of version configuration --- import common.SoapTextTest; #stored ('searchWords', 'one,and,sheep,when,richard,king'); -#stored ('url', serviceUrl); -SoapTextTest.mainService(); +string searchWords := '' : stored('searchWords'); +unsigned documentLimit := 3 : stored('documentLimit'); +unsigned maxResults := 50; + +//---------------- + +ConnectionType := SoapTextTest.ConnectionType; + +SoapTextTest.CallOptionsRecord initCallOptions() := TRANSFORM + SELF.connectionToRoxie := IF(encryptConnection, ConnectionType.Encrypted, ConnectionType.Plain); + SELF.connectionToEsp := IF(encryptConnection, ConnectionType.Encrypted, ConnectionType.Plain); + SELF.persistConnectToRoxie := persistConnection; + SELF.persistConnectToEsp := persistConnection; + SELF.connectDirectToRoxie := NOT callViaEsp; + SELF.embedServiceCalls := callDirect; + SELF := []; +END; + +SoapTextTest.ConfigOptionsRecord initConfigOptions() := TRANSFORM + SELF.remoteEspUrl := espIP; + SELF.remoteRoxieUrl := roxieIP; + SELF := []; +END; + + +callOptions := ROW(initCallOptions()); +configOptions := ROW(initConfigOptions()); + +SoapTextTest.runMainService(searchWords, documentLimit, maxResults, callOptions, configOptions); From b204e609691ad2b305fb8b8f842465d1e293de9f Mon Sep 17 00:00:00 2001 From: Richard Chapman Date: Wed, 14 Feb 2024 11:47:59 +0000 Subject: [PATCH 19/24] HPCC-31253 Roxie excessive logging info for waiting for ok_to_send Report periodically how many times it has happened, rather than reporting every time Signed-off-by: Richard Chapman --- roxie/udplib/udptrs.cpp | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/roxie/udplib/udptrs.cpp b/roxie/udplib/udptrs.cpp index 638826f81dc..e3333646446 100644 --- a/roxie/udplib/udptrs.cpp +++ b/roxie/udplib/udptrs.cpp @@ -74,12 +74,14 @@ using roxiemem::DataBuffer; */ // UdpResentList keeps a copy of up to TRACKER_BITS previously sent packets so we can send them again +RelaxedAtomic okToSendTimeouts; RelaxedAtomic packetsResent; RelaxedAtomic flowRequestsSent; RelaxedAtomic flowPermitsReceived; RelaxedAtomic dataPacketsSent; static unsigned lastResentReport = 0; +static unsigned lastOkToSendTimeouts = 0; static unsigned lastPacketsResent = 0; static unsigned lastFlowRequestsSent = 0; static unsigned lastFlowPermitsReceived = 0; @@ -384,11 +386,12 @@ class UdpReceiverEntry : public IUdpReceiverEntry void resendRequestToSend() { // This is called from the timeout thread when a previously-send request has had no response + okToSendTimeouts.fastInc(); timeouts++; if (udpTraceLevel || udpTraceFlow || udpTraceTimeouts) { //Avoid tracing too many times - otherwise might get large number 5000+ messages - if (timeouts == 1 || maxRequestDeadTimeouts < 10 || (timeouts % (maxRequestDeadTimeouts / 10) == 0)) + if (maxRequestDeadTimeouts < 10 || (timeouts % (maxRequestDeadTimeouts / 10) == 0)) { int timeExpired = msTick()-requestExpiryTime; StringBuffer s; @@ -894,7 +897,12 @@ class CSendManager : implements ISendManager, public CInterface if (udpStatsReportInterval && (now-lastResentReport > udpStatsReportInterval)) { // MORE - some of these should really be tracked per destination - lastResentReport = now; + if (okToSendTimeouts > lastOkToSendTimeouts) + { + EXCLOG(MCoperatorError,"ERROR: UdpSender: timed out %u more times waiting for ok_to_send msg in last %u seconds", + okToSendTimeouts - lastOkToSendTimeouts, (now-lastResentReport)/1000); + lastOkToSendTimeouts = okToSendTimeouts; + } if (packetsResent > lastPacketsResent) { DBGLOG("Sender: %u more packets resent by this agent (%u total)", packetsResent-lastPacketsResent, packetsResent-0); @@ -915,6 +923,7 @@ class CSendManager : implements ISendManager, public CInterface DBGLOG("Sender: %u more data packets sent by this agent (%u total)", dataPacketsSent - lastDataPacketsSent, dataPacketsSent-0); lastDataPacketsSent = dataPacketsSent; } + lastResentReport = now; } } return 0; From 4827474105013220d57ad3fe4b3f585cccda02d8 Mon Sep 17 00:00:00 2001 From: Gavin Halliday Date: Wed, 28 Feb 2024 10:28:23 +0000 Subject: [PATCH 20/24] HPCC-31361 Improve the error message when failing to read git file contents Signed-off-by: Gavin Halliday --- common/remote/hooks/git/gitfile.cpp | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/common/remote/hooks/git/gitfile.cpp b/common/remote/hooks/git/gitfile.cpp index 1a3e7d14ebe..a5c53b6e1f9 100644 --- a/common/remote/hooks/git/gitfile.cpp +++ b/common/remote/hooks/git/gitfile.cpp @@ -174,19 +174,19 @@ class GitRepositoryFileIO : implements CSimpleInterfaceOf return buf.length() > LFSsiglen && memcmp(buf.toByteArray(), LFSsig, LFSsiglen)==0; } public: - GitRepositoryFileIO(GitCommitTree * commitTree, const char *gitDirectory, const git_oid * oid, const char * gitUser) + GitRepositoryFileIO(const char * filename, GitCommitTree * commitTree, const char *gitDirectory, const git_oid * oid, const char * gitUser) { git_blob *blob = nullptr; int error = git_blob_lookup(&blob, git_tree_owner(commitTree->queryTree()), oid); if (error) - throw MakeStringException(0, "git git_blob_lookup returned exit status %d", error); + throw MakeStringException(0, "git git_blob_lookup for '%s' returned exit status %d", filename, error); git_object_size_t blobsize = git_blob_rawsize(blob); const void * data = git_blob_rawcontent(blob); buf.append(blobsize, data); git_blob_free(blob); if (isLFSfile()) - readLfsContents(gitDirectory, gitUser); + readLfsContents(filename, gitDirectory, gitUser); } virtual size32_t read(offset_t pos, size32_t len, void * data) { @@ -229,7 +229,7 @@ class GitRepositoryFileIO : implements CSimpleInterfaceOf } protected: - void readLfsContents(const char *gitDirectory, const char * gitUser) + void readLfsContents(const char * filename, const char *gitDirectory, const char * gitUser) { EnvironmentVector env; Owned extractedKey; @@ -254,8 +254,10 @@ class GitRepositoryFileIO : implements CSimpleInterfaceOf env.emplace_back("HPCC_GIT_PASSPATH", extractedKey->queryFilename()); } else - DBGLOG("Secret doesn't contain password for git user %s", gitUser); + OWARNLOG("Secret doesn't contain password for git user %s", gitUser); } + else + OWARNLOG("No secret found for git user %s", gitUser); } Owned pipe = createPipeProcess(); for (const auto & cur : env) @@ -275,7 +277,7 @@ class GitRepositoryFileIO : implements CSimpleInterfaceOf if (retcode) { buf.clear(); // Can't rely on destructor to clean this for me - throw MakeStringException(0, "git-lfs returned exit status %d", retcode); + throw MakeStringException(0, "git-lfs for '%s' (user %s) returned exit status %d", filename, gitUser ? gitUser : "", retcode); } } @@ -388,12 +390,12 @@ class GitRepositoryFile : implements IFile, public CInterface virtual IFileIO * open(IFOmode mode, IFEflags extraFlags) override { assertex(mode==IFOread && isExisting && !isDir); - return new GitRepositoryFileIO(commitTree, gitDirectory, &oid, gitUser); + return new GitRepositoryFileIO(fullName, commitTree, gitDirectory, &oid, gitUser); } virtual IFileIO * openShared(IFOmode mode, IFSHmode shmode, IFEflags extraFlags) override { assertex(mode==IFOread && isExisting && !isDir); - return new GitRepositoryFileIO(commitTree, gitDirectory, &oid, gitUser); + return new GitRepositoryFileIO(fullName, commitTree, gitDirectory, &oid, gitUser); } From cae7c7af4dfa82d5ef50343ed47bc11e7a8dcb00 Mon Sep 17 00:00:00 2001 From: Gavin Halliday Date: Wed, 28 Feb 2024 13:04:10 +0000 Subject: [PATCH 21/24] HPCC-31362 Ensure the git username is exported to the environment Signed-off-by: Gavin Halliday --- common/remote/hooks/git/gitfile.cpp | 5 +++++ ecl/hql/hqlrepository.cpp | 1 + 2 files changed, 6 insertions(+) diff --git a/common/remote/hooks/git/gitfile.cpp b/common/remote/hooks/git/gitfile.cpp index ccebaddb28b..702a0ab2f3b 100644 --- a/common/remote/hooks/git/gitfile.cpp +++ b/common/remote/hooks/git/gitfile.cpp @@ -236,9 +236,14 @@ class GitRepositoryFileIO : implements CSimpleInterfaceOf //If fetching from git and the username is specified then use the script file to provide the username/password //Only support retrieving the password as a secret - not as a filename + //NB: This code should be kept in sync with runGitCommand() in hqlrepository.cpp (and ideally combined) if (!isEmptyString(gitUser)) { + env.emplace_back("HPCC_GIT_USERNAME", gitUser); + + // If gituser is specified never prompt for credentials, otherwise the server can hang. env.emplace_back("GIT_TERMINAL_PROMPT", "0"); + StringBuffer scriptPath; getPackageFolder(scriptPath); addPathSepChar(scriptPath).append("bin/hpccaskpass.sh"); diff --git a/ecl/hql/hqlrepository.cpp b/ecl/hql/hqlrepository.cpp index 4a6ad7e5b75..7b7bf395085 100644 --- a/ecl/hql/hqlrepository.cpp +++ b/ecl/hql/hqlrepository.cpp @@ -937,6 +937,7 @@ unsigned EclRepositoryManager::runGitCommand(StringBuffer * output, const char * Owned extractedKey; EnvironmentVector env; //If fetching from git and the username is specified then use the script file to provide the username/password + //NB: This code should be kept in sync with readLfsContents() in gitfile.cpp (and ideally combined) if (needCredentials) { //If the username is supplied, then get the secret and write it to a temporary location. From 737b8a04eab672ec5ad79c5fd903bedfa23b6a18 Mon Sep 17 00:00:00 2001 From: Gordon Smith Date: Tue, 20 Feb 2024 10:59:30 +0000 Subject: [PATCH 22/24] HPCC-31305 Auto refresh main pages on focus change Mirror v5 behaviours --- esp/src/src-react/components/Files.tsx | 11 +++++++ esp/src/src-react/components/Queries.tsx | 11 +++++++ esp/src/src-react/components/Workunits.tsx | 13 +++++++- esp/src/src-react/hooks/util.ts | 37 ++++++++++++++++++++++ 4 files changed, 71 insertions(+), 1 deletion(-) diff --git a/esp/src/src-react/components/Files.tsx b/esp/src/src-react/components/Files.tsx index 6d8de3172cc..70fb194cfb0 100644 --- a/esp/src/src-react/components/Files.tsx +++ b/esp/src/src-react/components/Files.tsx @@ -9,6 +9,7 @@ import { QuerySortItem } from "src/store/Store"; import nlsHPCC from "src/nlsHPCC"; import { useConfirm } from "../hooks/confirm"; import { useMyAccount } from "../hooks/user"; +import { useHasFocus, useIsMounted } from "../hooks/util"; import { HolyGrail } from "../layouts/HolyGrail"; import { pushParams } from "../util/history"; import { FluentPagedGrid, FluentPagedFooter, useCopyButtons, useFluentStoreState, FluentColumns } from "./controls/Grid"; @@ -116,6 +117,16 @@ export const Files: React.FunctionComponent = ({ total, setTotal, refreshTable } = useFluentStoreState({ page }); + // Refresh on focus --- + const isMounted = useIsMounted(); + const hasFocus = useHasFocus(); + React.useEffect(() => { + if (isMounted && hasFocus) { + refreshTable.call(); + } + // eslint-disable-next-line react-hooks/exhaustive-deps + }, [hasFocus]); + // Grid --- const gridStore = React.useMemo(() => { return store ? store : CreateDFUQueryStore(); diff --git a/esp/src/src-react/components/Queries.tsx b/esp/src/src-react/components/Queries.tsx index b30922e3f49..a0da7144844 100644 --- a/esp/src/src-react/components/Queries.tsx +++ b/esp/src/src-react/components/Queries.tsx @@ -6,6 +6,7 @@ import nlsHPCC from "src/nlsHPCC"; import { QuerySortItem } from "src/store/Store"; import { useConfirm } from "../hooks/confirm"; import { useMyAccount } from "../hooks/user"; +import { useHasFocus, useIsMounted } from "../hooks/util"; import { HolyGrail } from "../layouts/HolyGrail"; import { pushParams } from "../util/history"; import { FluentPagedGrid, FluentPagedFooter, useCopyButtons, useFluentStoreState, FluentColumns } from "./controls/Grid"; @@ -76,6 +77,16 @@ export const Queries: React.FunctionComponent = ({ const hasFilter = React.useMemo(() => Object.keys(filter).length > 0, [filter]); + // Refresh on focus --- + const isMounted = useIsMounted(); + const hasFocus = useHasFocus(); + React.useEffect(() => { + if (isMounted && hasFocus) { + refreshTable.call(); + } + // eslint-disable-next-line react-hooks/exhaustive-deps + }, [hasFocus]); + // Grid --- const gridStore = React.useMemo(() => { return store || ESPQuery.CreateQueryStore({}); diff --git a/esp/src/src-react/components/Workunits.tsx b/esp/src/src-react/components/Workunits.tsx index 855cb6eedc2..6c17aa6c368 100644 --- a/esp/src/src-react/components/Workunits.tsx +++ b/esp/src/src-react/components/Workunits.tsx @@ -8,8 +8,9 @@ import { formatCost } from "src/Session"; import nlsHPCC from "src/nlsHPCC"; import { useConfirm } from "../hooks/confirm"; import { useMyAccount } from "../hooks/user"; -import { HolyGrail } from "../layouts/HolyGrail"; import { pushParams } from "../util/history"; +import { useHasFocus, useIsMounted } from "../hooks/util"; +import { HolyGrail } from "../layouts/HolyGrail"; import { FluentPagedGrid, FluentPagedFooter, useCopyButtons, useFluentStoreState, FluentColumns } from "./controls/Grid"; import { Fields } from "./forms/Fields"; import { Filter } from "./forms/Filter"; @@ -103,6 +104,16 @@ export const Workunits: React.FunctionComponent = ({ total, setTotal, refreshTable } = useFluentStoreState({ page }); + // Refresh on focus --- + const isMounted = useIsMounted(); + const hasFocus = useHasFocus(); + React.useEffect(() => { + if (isMounted && hasFocus) { + refreshTable.call(); + } + // eslint-disable-next-line react-hooks/exhaustive-deps + }, [hasFocus]); + // Grid --- const query = React.useMemo(() => { return formatQuery(filter); diff --git a/esp/src/src-react/hooks/util.ts b/esp/src/src-react/hooks/util.ts index f609f77e49c..4bac374ea70 100644 --- a/esp/src/src-react/hooks/util.ts +++ b/esp/src/src-react/hooks/util.ts @@ -7,3 +7,40 @@ export function useCounter(): [number, () => void] { return [counter, () => setCounter(counter + 1)]; } +export function useIsMounted(): boolean { + const isMountedRef = React.useRef(false); + + React.useEffect(() => { + isMountedRef.current = true; + return () => { + isMountedRef.current = false; + }; + }, []); + + return isMountedRef.current; +} + +export function useHasFocus(): boolean { + + const [hasFocus, setHasFocus] = React.useState(true); + + const onFocus = () => { + setHasFocus(true); + }; + + const onBlur = () => { + setHasFocus(false); + }; + + React.useEffect(() => { + window.addEventListener("focus", onFocus); + window.addEventListener("blur", onBlur); + + return () => { + window.removeEventListener("focus", onFocus); + window.removeEventListener("blur", onBlur); + }; + }, []); + + return hasFocus; +} From 6f8aa5695f915209fd35bbf7cf631577d2f88db4 Mon Sep 17 00:00:00 2001 From: Jeremy Clements <79224539+jeclrsg@users.noreply.github.com> Date: Fri, 23 Feb 2024 15:25:02 -0500 Subject: [PATCH 23/24] HPCC-31325 ECL Watch WU publish add RemoteStorage Signed-off-by: Jeremy Clements <79224539+jeclrsg@users.noreply.github.com> --- esp/src/eclwatch/WUDetailsWidget.js | 1 + .../eclwatch/templates/WUDetailsWidget.html | 1 + esp/src/package-lock.json | 22 +++++++------- esp/src/package.json | 2 +- .../components/forms/PublishQuery.tsx | 30 ++++++++++++++++++- esp/src/src/ESPWorkunit.ts | 3 +- esp/src/src/nls/hpcc.ts | 1 + 7 files changed, 46 insertions(+), 14 deletions(-) diff --git a/esp/src/eclwatch/WUDetailsWidget.js b/esp/src/eclwatch/WUDetailsWidget.js index e69bd044d3e..c7d2c331af0 100644 --- a/esp/src/eclwatch/WUDetailsWidget.js +++ b/esp/src/eclwatch/WUDetailsWidget.js @@ -237,6 +237,7 @@ define([ this.wu.publish( dom.byId(this.id + "Jobname2").value, dom.byId(this.id + "RemoteDali").value, + dom.byId(this.id + "RemoteStorage").value, dom.byId(this.id + "SourceProcess").value, registry.byId(this.id + "Priority").value, dom.byId(this.id + "Comment").value, diff --git a/esp/src/eclwatch/templates/WUDetailsWidget.html b/esp/src/eclwatch/templates/WUDetailsWidget.html index 16758f5fb0c..a56d36ec15c 100644 --- a/esp/src/eclwatch/templates/WUDetailsWidget.html +++ b/esp/src/eclwatch/templates/WUDetailsWidget.html @@ -30,6 +30,7 @@
+