From fd170ff51e5d75d3b5ac76d78761122ffad4856d Mon Sep 17 00:00:00 2001 From: Gavin Halliday Date: Mon, 17 Jun 2024 09:49:08 +0100 Subject: [PATCH 01/29] HPCC-32068 Fix some (false postive) coverity warnings Signed-off-by: Gavin Halliday --- system/jlib/jsmartsock.cpp | 9 +++------ testing/unittests/jstreamtests.cpp | 2 +- 2 files changed, 4 insertions(+), 7 deletions(-) diff --git a/system/jlib/jsmartsock.cpp b/system/jlib/jsmartsock.cpp index e2a080e7a51..cc11e40a370 100644 --- a/system/jlib/jsmartsock.cpp +++ b/system/jlib/jsmartsock.cpp @@ -30,6 +30,7 @@ class SmartSocketListParser public: SmartSocketListParser(const char * text) { + assertex(text); fullText = strdup(text); } @@ -235,11 +236,9 @@ CSmartSocketFactory::CSmartSocketFactory(IPropertyTree &service, bool _retry, un dnsInterval=_dnsInterval; retry = _retry; + retryInterval = _retryInterval; if (retry) - { - retryInterval = _retryInterval; this->start(false); - } } CSmartSocketFactory::CSmartSocketFactory(const char *_socklist, IPropertyTree* _tlsConfig, bool _retry, unsigned _retryInterval, unsigned _dnsInterval) @@ -261,11 +260,9 @@ CSmartSocketFactory::CSmartSocketFactory(const char *_socklist, IPropertyTree* _ dnsInterval=_dnsInterval; retry = _retry; + retryInterval = _retryInterval; if (retry) - { - retryInterval = _retryInterval; this->start(false); - } } CSmartSocketFactory::~CSmartSocketFactory() diff --git a/testing/unittests/jstreamtests.cpp b/testing/unittests/jstreamtests.cpp index 98303578ccc..7cd8625a5f2 100644 --- a/testing/unittests/jstreamtests.cpp +++ b/testing/unittests/jstreamtests.cpp @@ -509,7 +509,7 @@ class JlibStreamStressTest : public CppUnit::TestFixture public: CDataProvider & dataProvider; IBufferedSerialOutputStream * out; - offset_t totalSent; + offset_t totalSent = 0; unsigned numRows; }; From 6ca7abcac7235b33c0898784a3233341ff09c9fe Mon Sep 17 00:00:00 2001 From: Shamser Ahmed Date: Tue, 2 Jul 2024 12:03:15 +0100 Subject: [PATCH 02/29] HPCC-32191 Update mappings for Hash Join activity with new join stats Signed-off-by: Shamser Ahmed --- thorlcr/thorutil/thormisc.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/thorlcr/thorutil/thormisc.cpp b/thorlcr/thorutil/thormisc.cpp index 09b3fc233af..4928363a266 100644 --- a/thorlcr/thorutil/thormisc.cpp +++ b/thorlcr/thorutil/thormisc.cpp @@ -77,13 +77,13 @@ const StatisticsMapping spillStatistics({StTimeSpillElapsed, StTimeSortElapsed, const StatisticsMapping soapcallStatistics({StTimeSoapcall}); const StatisticsMapping basicActivityStatistics({StTimeTotalExecute, StTimeLocalExecute, StTimeBlocked}); const StatisticsMapping groupActivityStatistics({StNumGroups, StNumGroupMax}, basicActivityStatistics); -const StatisticsMapping hashJoinActivityStatistics({StNumLeftRows, StNumRightRows}, basicActivityStatistics); const StatisticsMapping indexReadFileStatistics({}, diskReadRemoteStatistics, jhtreeCacheStatistics); const StatisticsMapping indexReadActivityStatistics({StNumRowsProcessed}, indexReadFileStatistics, basicActivityStatistics); const StatisticsMapping indexWriteActivityStatistics({StPerReplicated, StNumLeafCacheAdds, StNumNodeCacheAdds, StNumBlobCacheAdds }, basicActivityStatistics, diskWriteRemoteStatistics); const StatisticsMapping keyedJoinActivityStatistics({ StNumIndexAccepted, StNumPreFiltered, StNumDiskSeeks, StNumDiskAccepted, StNumDiskRejected}, basicActivityStatistics, indexReadFileStatistics); const StatisticsMapping loopActivityStatistics({StNumIterations}, basicActivityStatistics); const StatisticsMapping commonJoinActivityStatistics({StNumMatchLeftRowsMax, StNumMatchRightRowsMax, StNumMatchCandidates, StNumMatchCandidatesMax}, basicActivityStatistics); +const StatisticsMapping hashJoinActivityStatistics({StNumLeftRows, StNumRightRows}, commonJoinActivityStatistics); const StatisticsMapping allJoinActivityStatistics({}, commonJoinActivityStatistics); const StatisticsMapping lookupJoinActivityStatistics({StNumSmartJoinSlavesDegradedToStd, StNumSmartJoinDegradedToLocal}, spillStatistics, commonJoinActivityStatistics); const StatisticsMapping joinActivityStatistics({StNumLeftRows, StNumRightRows}, commonJoinActivityStatistics, spillStatistics); From 9824afebc7581674898baea7870bf99afdfe2528 Mon Sep 17 00:00:00 2001 From: "Dan S. Camper" Date: Tue, 2 Jul 2024 09:39:11 -0500 Subject: [PATCH 03/29] HPCC-32195 Document regex cache expert setting in helm chart --- helm/hpcc/docs/expert.md | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/helm/hpcc/docs/expert.md b/helm/hpcc/docs/expert.md index a42fc346ba5..5d2249072e2 100644 --- a/helm/hpcc/docs/expert.md +++ b/helm/hpcc/docs/expert.md @@ -13,6 +13,8 @@ global: time: 200 interval: 75 probes: 9 + regex: + cacheSize: 500 ``` NB: Some components (e.g. DfuServer and Thor) also have an 'expert' settings area (see values schema) that can be used for relavent settings @@ -73,6 +75,13 @@ service via remote file reads with the ~remote:: syntax. Setting expert.allowForeign to true, enables foreign access for compatibility with legacy bare-metal environments that have their Dali and Dafilesrv's open. +## regex (cacheSize: unsigned) + +See the regex example above. If set, this should be added at the global level. +The default value is 500. Set to zero to disable the cache. +This value is applied at the process level: Each Thor worker, Roxie process, and hthor worker receives +its own cache. Threads/channels within a process share that process's cache. + # Plane Expert Settings From 862e826861a640f959d3e3fac0feee5f077d8016 Mon Sep 17 00:00:00 2001 From: Rodrigo Pastrana Date: Tue, 2 Jul 2024 15:43:01 -0400 Subject: [PATCH 04/29] HPCC-31971 SoapCall Span Enhancement - Renames socket write span - Provides target url and service in name - Provides target service and request type Signed-off-by: Rodrigo Pastrana --- common/thorhelper/thorsoapcall.cpp | 24 +++++++++++++++--------- 1 file changed, 15 insertions(+), 9 deletions(-) diff --git a/common/thorhelper/thorsoapcall.cpp b/common/thorhelper/thorsoapcall.cpp index bee4d69b47d..99c5241ecc7 100644 --- a/common/thorhelper/thorsoapcall.cpp +++ b/common/thorhelper/thorsoapcall.cpp @@ -2559,10 +2559,16 @@ class CWSCAsyncFor : implements IWSCAsyncFor, public CInterface, public CAsyncFo { checkTimeLimitExceeded(&remainingMS); checkRoxieAbortMonitor(master->roxieAbortMonitor); - OwnedSpanScope socketOperationSpan = master->activitySpanScope->createClientSpan("Socket Write"); - setSpanURLAttributes(socketOperationSpan, url); - Owned traceHeaders = ::getClientHeaders(socketOperationSpan); + StringBuffer spanName; + spanName.appendf("%s %s %s:%d", getWsCallTypeName(master->wscType), master->service.str(), url.host.str(), url.port); + OwnedSpanScope requestSpan = master->activitySpanScope->createClientSpan(spanName.str()); + + setSpanURLAttributes(requestSpan, url); + requestSpan->setSpanAttribute("request.type", getWsCallTypeName(master->wscType)); + requestSpan->setSpanAttribute("service.name", master->service.str()); + + Owned traceHeaders = ::getClientHeaders(requestSpan); createHttpRequest(request, url, traceHeaders); socket->write(request.str(), request.length()); @@ -2575,7 +2581,7 @@ class CWSCAsyncFor : implements IWSCAsyncFor, public CInterface, public CAsyncFo bool keepAlive2; StringBuffer contentType; int rval = readHttpResponse(response, socket, keepAlive2, contentType); - socketOperationSpan->setSpanAttribute("http.response.status_code", (int64_t)rval); + requestSpan->setSpanAttribute("http.response.status_code", (int64_t)rval); keepAlive = keepAlive && keepAlive2; if (soapTraceLevel > 4) @@ -2583,22 +2589,22 @@ class CWSCAsyncFor : implements IWSCAsyncFor, public CInterface, public CAsyncFo if (rval != 200) { - socketOperationSpan->setSpanStatusSuccess(false); + requestSpan->setSpanStatusSuccess(false); if (rval == 503) { - socketOperationSpan->recordError(SpanError("Server Too Busy", 1001, true, true)); + requestSpan->recordError(SpanError("Server Too Busy", 1001, true, true)); throw new ReceivedRoxieException(1001, "Server Too Busy"); } StringBuffer text; text.appendf("HTTP error (%d) in processQuery",rval); rtlAddExceptionTag(text, "soapresponse", response.str()); - socketOperationSpan->recordError(SpanError(text.str(), -1, true, true)); + requestSpan->recordError(SpanError(text.str(), -1, true, true)); throw MakeStringExceptionDirect(-1, text.str()); } if (response.length() == 0) { - socketOperationSpan->recordError(SpanError("Zero length response in processQuery", -1, true, true)); + requestSpan->recordError(SpanError("Zero length response in processQuery", -1, true, true)); throw MakeStringException(-1, "Zero length response in processQuery"); } checkTimeLimitExceeded(&remainingMS); @@ -2614,7 +2620,7 @@ class CWSCAsyncFor : implements IWSCAsyncFor, public CInterface, public CAsyncFo persistentHandler->add(socket, &ep, proto); } - socketOperationSpan->setSpanStatusSuccess(true); + requestSpan->setSpanStatusSuccess(true); break; } catch (IReceivedRoxieException *e) From b55d5379447842e9e108c9b027d29d11b85cb217 Mon Sep 17 00:00:00 2001 From: Jake Smith Date: Wed, 10 Jul 2024 17:47:27 +0100 Subject: [PATCH 05/29] HPCC-32229 Avoid connecting to parent when deleting path Change daliadmin delete to connect to tail node directly, this avoids an unnecessary fetch of siblings and potential contention of parent. Signed-off-by: Jake Smith --- dali/daliadmin/daadmin.cpp | 32 +++++++++++--------------------- 1 file changed, 11 insertions(+), 21 deletions(-) diff --git a/dali/daliadmin/daadmin.cpp b/dali/daliadmin/daadmin.cpp index f3f25fad57b..d1ea8bd9b02 100644 --- a/dali/daliadmin/daadmin.cpp +++ b/dali/daliadmin/daadmin.cpp @@ -265,34 +265,24 @@ bool importFromFile(const char *path,const char *filename,bool add,StringBuffer //============================================================================= -bool erase(const char *path,bool backup,StringBuffer &out) +bool erase(const char *path, bool backup, StringBuffer &out) { - StringBuffer head; - StringBuffer tmp; - const char *tail=splitpath(path,head,tmp); - Owned conn = querySDS().connect(head.str(),myProcessSession(),RTM_LOCK_WRITE, daliConnectTimeoutMs); - if (!conn) { + Owned conn = querySDS().connect(path, myProcessSession(), RTM_LOCK_WRITE, daliConnectTimeoutMs); + if (!conn) + { out.appendf("Could not connect to %s",path); return false; } - Owned root = conn->getRoot(); - Owned child = root->getPropTree(tail); - if (!child) { - out.appendf("Couldn't find %s/%s",head.str(),tail); - return false; - } - if (backup) { + IPropertyTree *root = conn->queryRoot(); + if (backup) + { StringBuffer bakname; - Owned io = createUniqueFile(NULL,"daliadmin", "bak", bakname); - out.appendf("Saving backup of %s/%s to %s",head.str(),tail,bakname.str()); + Owned io = createUniqueFile(NULL, "daliadmin", "bak", bakname); + out.appendf("Saving backup of %s to %s", path, bakname.str()); Owned fstream = createBufferedIOStream(io); - toXML(child, *fstream); // formatted (default) + toXML(root, *fstream); // formatted (default) } - root->removeTree(child); - child.clear(); - root.clear(); - conn->commit(); - conn->close(); + conn->close(true); return true; } From 3225279104edd0ace7dba384e2f77c6480e27b68 Mon Sep 17 00:00:00 2001 From: M Kelly Date: Thu, 11 Jul 2024 09:37:13 -0400 Subject: [PATCH 06/29] HPCC-32234 SYS: swap and mem util stats sometimes invalid on containers Signed-off-by: M Kelly --- system/jlib/jdebug.cpp | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/system/jlib/jdebug.cpp b/system/jlib/jdebug.cpp index 78b9813c4a0..17154d4a99c 100644 --- a/system/jlib/jdebug.cpp +++ b/system/jlib/jdebug.cpp @@ -1453,7 +1453,13 @@ static void getMemUsage(unsigned &inuse,unsigned &active,unsigned &total,unsigne bufptr = strchr(bufptr, '\n'); } inuse = total-free-cached; - swapinuse = swaptotal-swapfree-swapcached; + + // not sure if a bug in kernel or container or ... + // but sometimes we see swapfree > 0 when swaptotal == 0 + if ((swapfree + swapcached) >= swaptotal) + swapinuse = 0; + else + swapinuse = swaptotal-swapfree-swapcached; #endif } From 9a6c03be69990b3cef867afa44e3b9dbbbe05913 Mon Sep 17 00:00:00 2001 From: Jake Smith Date: Thu, 11 Jul 2024 17:13:46 +0100 Subject: [PATCH 07/29] Split off 9.2.104 Signed-off-by: Jake Smith --- helm/hpcc/Chart.yaml | 4 ++-- helm/hpcc/templates/_helpers.tpl | 2 +- helm/hpcc/templates/dafilesrv.yaml | 2 +- helm/hpcc/templates/dali.yaml | 2 +- helm/hpcc/templates/dfuserver.yaml | 2 +- helm/hpcc/templates/eclagent.yaml | 4 ++-- helm/hpcc/templates/eclccserver.yaml | 4 ++-- helm/hpcc/templates/eclscheduler.yaml | 2 +- helm/hpcc/templates/esp.yaml | 2 +- helm/hpcc/templates/localroxie.yaml | 2 +- helm/hpcc/templates/roxie.yaml | 8 ++++---- helm/hpcc/templates/sasha.yaml | 2 +- helm/hpcc/templates/thor.yaml | 10 +++++----- version.cmake | 4 ++-- 14 files changed, 25 insertions(+), 25 deletions(-) diff --git a/helm/hpcc/Chart.yaml b/helm/hpcc/Chart.yaml index 37ad52a2e92..adf0ac1742e 100644 --- a/helm/hpcc/Chart.yaml +++ b/helm/hpcc/Chart.yaml @@ -6,9 +6,9 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. -version: 9.2.103-closedown0 +version: 9.2.105-closedown0 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. -appVersion: 9.2.103-closedown0 +appVersion: 9.2.105-closedown0 diff --git a/helm/hpcc/templates/_helpers.tpl b/helm/hpcc/templates/_helpers.tpl index cac26347e1e..cd2297894f8 100644 --- a/helm/hpcc/templates/_helpers.tpl +++ b/helm/hpcc/templates/_helpers.tpl @@ -1405,7 +1405,7 @@ kind: Service metadata: name: {{ $lvars.serviceName | quote }} labels: - helmVersion: 9.2.103-closedown0 + helmVersion: 9.2.105-closedown0 {{- include "hpcc.addStandardLabels" (dict "root" $.root "instance" $lvars.serviceName ) | indent 4 }} {{- if $lvars.labels }} {{ toYaml $lvars.labels | indent 4 }} diff --git a/helm/hpcc/templates/dafilesrv.yaml b/helm/hpcc/templates/dafilesrv.yaml index 030b400c674..bf0ad62a753 100644 --- a/helm/hpcc/templates/dafilesrv.yaml +++ b/helm/hpcc/templates/dafilesrv.yaml @@ -50,7 +50,7 @@ spec: labels: {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "dafilesrv" "name" "dafilesrv" "instance" .name) | indent 8 }} server: {{ .name | quote }} - helmVersion: 9.2.103-closedown0 + helmVersion: 9.2.105-closedown0 annotations: checksum/config: {{ $configSHA }} {{- include "hpcc.generateAnnotations" $commonCtx | indent 8 }} diff --git a/helm/hpcc/templates/dali.yaml b/helm/hpcc/templates/dali.yaml index 25babfb26eb..523ddb016df 100644 --- a/helm/hpcc/templates/dali.yaml +++ b/helm/hpcc/templates/dali.yaml @@ -82,7 +82,7 @@ spec: run: {{ $dali.name | quote }} server: {{ $dali.name | quote }} app: dali - helmVersion: 9.2.103-closedown0 + helmVersion: 9.2.105-closedown0 {{- if hasKey $.Values.global "metrics" }} {{- include "hpcc.generateMetricsReporterLabel" $.Values.global.metrics | nindent 8 }} {{- end }} diff --git a/helm/hpcc/templates/dfuserver.yaml b/helm/hpcc/templates/dfuserver.yaml index 98861431c67..46cfa108bcf 100644 --- a/helm/hpcc/templates/dfuserver.yaml +++ b/helm/hpcc/templates/dfuserver.yaml @@ -56,7 +56,7 @@ spec: {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "dfuserver" "name" "dfuserver" "instance" .name) | indent 8 }} run: {{ .name | quote }} accessDali: "yes" - helmVersion: 9.2.103-closedown0 + helmVersion: 9.2.105-closedown0 {{- if hasKey . "labels" }} {{ toYaml .labels | indent 8 }} {{- end }} diff --git a/helm/hpcc/templates/eclagent.yaml b/helm/hpcc/templates/eclagent.yaml index 8f14a31df62..a5590499670 100644 --- a/helm/hpcc/templates/eclagent.yaml +++ b/helm/hpcc/templates/eclagent.yaml @@ -58,7 +58,7 @@ data: {{- include "hpcc.addStandardLabels" (dict "root" $ "component" $apptype "name" "eclagent" "instance" $appJobName "instanceOf" (printf "%s-job" .me.name)) | indent 12 }} accessDali: "yes" accessEsp: "yes" - helmVersion: 9.2.103-closedown0 + helmVersion: 9.2.105-closedown0 {{- if hasKey .me "labels" }} {{ toYaml .me.labels | indent 12 }} {{- end }} @@ -133,7 +133,7 @@ spec: run: {{ .name | quote }} accessDali: "yes" accessEsp: {{ .useChildProcesses | default false | ternary "yes" "no" | quote }} - helmVersion: 9.2.103-closedown0 + helmVersion: 9.2.105-closedown0 {{- if hasKey . "labels" }} {{ toYaml .labels | indent 8 }} {{- end }} diff --git a/helm/hpcc/templates/eclccserver.yaml b/helm/hpcc/templates/eclccserver.yaml index ea659f47a04..f0ad7f5cdf8 100644 --- a/helm/hpcc/templates/eclccserver.yaml +++ b/helm/hpcc/templates/eclccserver.yaml @@ -57,7 +57,7 @@ data: {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "eclccserver" "name" "eclccserver" "instance" $compileJobName "instanceOf" (printf "%s-job" .me.name)) | indent 12 }} accessDali: "yes" accessEsp: "yes" - helmVersion: 9.2.103-closedown0 + helmVersion: 9.2.105-closedown0 {{- if hasKey .me "labels" }} {{ toYaml .me.labels | indent 12 }} {{- end }} @@ -140,7 +140,7 @@ spec: run: {{ .name | quote }} accessDali: "yes" accessEsp: {{ .useChildProcesses | default false | ternary "yes" "no" | quote }} - helmVersion: 9.2.103-closedown0 + helmVersion: 9.2.105-closedown0 {{- if hasKey . "labels" }} {{ toYaml .labels | indent 8 }} {{- end }} diff --git a/helm/hpcc/templates/eclscheduler.yaml b/helm/hpcc/templates/eclscheduler.yaml index 59e9b95d443..b1a860ec8ab 100644 --- a/helm/hpcc/templates/eclscheduler.yaml +++ b/helm/hpcc/templates/eclscheduler.yaml @@ -64,7 +64,7 @@ spec: run: {{ .name | quote }} accessDali: "yes" accessEsp: "no" - helmVersion: 9.2.103-closedown0 + helmVersion: 9.2.105-closedown0 {{- if hasKey . "labels" }} {{ toYaml .labels | indent 8 }} {{- end }} diff --git a/helm/hpcc/templates/esp.yaml b/helm/hpcc/templates/esp.yaml index c8bdfdf744e..15e17ba26e0 100644 --- a/helm/hpcc/templates/esp.yaml +++ b/helm/hpcc/templates/esp.yaml @@ -120,7 +120,7 @@ spec: accessSasha: "yes" {{- end }} app: {{ $application }} - helmVersion: 9.2.103-closedown0 + helmVersion: 9.2.105-closedown0 {{- include "hpcc.addStandardLabels" (dict "root" $ "name" $application "component" "esp" "instance" .name) | indent 8 }} {{- if hasKey $.Values.global "metrics" }} {{- include "hpcc.generateMetricsReporterLabel" $.Values.global.metrics | nindent 8 }} diff --git a/helm/hpcc/templates/localroxie.yaml b/helm/hpcc/templates/localroxie.yaml index 7a115c72bd8..f113cf4ef06 100644 --- a/helm/hpcc/templates/localroxie.yaml +++ b/helm/hpcc/templates/localroxie.yaml @@ -70,7 +70,7 @@ spec: server: {{ $servername | quote }} accessDali: "yes" accessEsp: "yes" - helmVersion: 9.2.103-closedown0 + helmVersion: 9.2.105-closedown0 {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "roxie-server" "name" "roxie" "instance" $roxie.name) | indent 8 }} {{- if hasKey . "labels" }} {{ toYaml .labels | indent 8 }} diff --git a/helm/hpcc/templates/roxie.yaml b/helm/hpcc/templates/roxie.yaml index ac234501338..814b06bc95d 100644 --- a/helm/hpcc/templates/roxie.yaml +++ b/helm/hpcc/templates/roxie.yaml @@ -120,7 +120,7 @@ spec: {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "topology-server" "name" "roxie" "instance" $commonCtx.toponame) | indent 8 }} run: {{ $commonCtx.toponame | quote }} roxie-cluster: {{ $roxie.name | quote }} - helmVersion: 9.2.103-closedown0 + helmVersion: 9.2.105-closedown0 {{- if hasKey $.Values.global "metrics" }} {{- include "hpcc.generateMetricsReporterLabel" $.Values.global.metrics | nindent 8}} {{- end }} @@ -178,7 +178,7 @@ kind: Service metadata: name: {{ $commonCtx.toponame | quote }} labels: - helmVersion: 9.2.103-closedown0 + helmVersion: 9.2.105-closedown0 {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "topology-server" "name" "roxie" "instance" $commonCtx.toponame) | indent 4 }} spec: ports: @@ -240,7 +240,7 @@ spec: roxie-cluster: {{ $roxie.name | quote }} accessDali: "yes" accessEsp: "yes" - helmVersion: 9.2.103-closedown0 + helmVersion: 9.2.105-closedown0 {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "roxie-server" "name" "roxie" "instance" $servername) | indent 8 }} {{- if hasKey $.Values.global "metrics" }} {{- include "hpcc.generateMetricsReporterLabel" $.Values.global.metrics | nindent 8}} @@ -346,7 +346,7 @@ spec: roxie-cluster: {{ $roxie.name | quote }} accessDali: "yes" accessEsp: "yes" - helmVersion: 9.2.103-closedown0 + helmVersion: 9.2.105-closedown0 {{- if hasKey $.Values.global "metrics" }} {{- include "hpcc.generateMetricsReporterLabel" $.Values.global.metrics | nindent 8}} {{- end }} diff --git a/helm/hpcc/templates/sasha.yaml b/helm/hpcc/templates/sasha.yaml index 0f2d89d163e..8093e5e2a03 100644 --- a/helm/hpcc/templates/sasha.yaml +++ b/helm/hpcc/templates/sasha.yaml @@ -53,7 +53,7 @@ spec: server: {{ $serviceName | quote }} app: sasha accessDali: {{ (has "dali" $sasha.access) | ternary "yes" "no" | quote }} - helmVersion: 9.2.103-closedown0 + helmVersion: 9.2.105-closedown0 {{- if hasKey $sasha "labels" }} {{ toYaml $sasha.labels | indent 8 }} {{- end }} diff --git a/helm/hpcc/templates/thor.yaml b/helm/hpcc/templates/thor.yaml index 5efebe03a8b..4ecd4a377b9 100644 --- a/helm/hpcc/templates/thor.yaml +++ b/helm/hpcc/templates/thor.yaml @@ -82,7 +82,7 @@ data: labels: accessDali: "yes" accessEsp: "yes" - helmVersion: 9.2.103-closedown0 + helmVersion: 9.2.105-closedown0 {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "eclagent" "name" "thor" "instance" $eclAgentJobName "instanceOf" (printf "%s-job" .eclAgentName)) | indent 8 }} {{- if hasKey .me "labels" }} {{ toYaml .me.labels | indent 12 }} @@ -145,7 +145,7 @@ data: accessEsp: "yes" app: "thor" component: "thormanager" - helmVersion: 9.2.103-closedown0 + helmVersion: 9.2.105-closedown0 instance: "_HPCC_JOBNAME_" job: "_HPCC_JOBNAME_" {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "thormanager" "name" "thor" "instance" $thorManagerJobName "instanceOf" (printf "%s-thormanager-job" .me.name)) | indent 12 }} @@ -210,7 +210,7 @@ data: accessEsp: "yes" app: "thor" component: "thorworker" - helmVersion: 9.2.103-closedown0 + helmVersion: 9.2.105-closedown0 instance: "_HPCC_JOBNAME_" job: "_HPCC_JOBNAME_" {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "thorworker" "name" "thor" "instance" $thorWorkerJobName "instanceOf" (printf "%s-thorworker-job" .me.name)) | indent 12 }} @@ -341,7 +341,7 @@ spec: accessEsp: {{ $commonCtx.eclAgentUseChildProcesses | ternary "yes" "no" | quote }} app: "thor" component: "thor-eclagent" - helmVersion: 9.2.103-closedown0 + helmVersion: 9.2.105-closedown0 instance: {{ $commonCtx.eclAgentName | quote }} {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "eclagent" "name" "thor" "instance" $commonCtx.eclAgentName ) | indent 8 }} {{- if hasKey $commonCtx.me "labels" }} @@ -404,7 +404,7 @@ spec: accessEsp: "no" app: "thor" component: "thor-thoragent" - helmVersion: 9.2.103-closedown0 + helmVersion: 9.2.105-closedown0 instance: {{ $commonCtx.thorAgentName | quote }} {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "eclagent" "name" "thor" "instance" $commonCtx.thorAgentName ) | indent 8 }} {{- if hasKey $commonCtx.me "labels" }} diff --git a/version.cmake b/version.cmake index 316946fa4d1..4db0c573463 100644 --- a/version.cmake +++ b/version.cmake @@ -5,8 +5,8 @@ set ( HPCC_NAME "Community Edition" ) set ( HPCC_PROJECT "community" ) set ( HPCC_MAJOR 9 ) set ( HPCC_MINOR 2 ) -set ( HPCC_POINT 103 ) +set ( HPCC_POINT 105 ) set ( HPCC_MATURITY "closedown" ) set ( HPCC_SEQUENCE 0 ) -set ( HPCC_TAG_TIMESTAMP "2024-07-05T08:41:44Z" ) +set ( HPCC_TAG_TIMESTAMP "2024-07-11T16:13:46Z" ) ### From ed8b3516bc8e7018af31020287c634bd7fea5cd4 Mon Sep 17 00:00:00 2001 From: Jake Smith Date: Thu, 11 Jul 2024 17:14:39 +0100 Subject: [PATCH 08/29] Split off 9.4.78 Signed-off-by: Jake Smith --- helm/hpcc/Chart.yaml | 4 ++-- helm/hpcc/templates/_helpers.tpl | 2 +- helm/hpcc/templates/dafilesrv.yaml | 2 +- helm/hpcc/templates/dali.yaml | 2 +- helm/hpcc/templates/dfuserver.yaml | 2 +- helm/hpcc/templates/eclagent.yaml | 4 ++-- helm/hpcc/templates/eclccserver.yaml | 4 ++-- helm/hpcc/templates/eclscheduler.yaml | 2 +- helm/hpcc/templates/esp.yaml | 2 +- helm/hpcc/templates/localroxie.yaml | 2 +- helm/hpcc/templates/roxie.yaml | 8 ++++---- helm/hpcc/templates/sasha.yaml | 2 +- helm/hpcc/templates/thor.yaml | 10 +++++----- version.cmake | 4 ++-- 14 files changed, 25 insertions(+), 25 deletions(-) diff --git a/helm/hpcc/Chart.yaml b/helm/hpcc/Chart.yaml index bd512664537..dd579b95240 100644 --- a/helm/hpcc/Chart.yaml +++ b/helm/hpcc/Chart.yaml @@ -6,9 +6,9 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. -version: 9.4.77-closedown0 +version: 9.4.79-closedown0 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. -appVersion: 9.4.77-closedown0 +appVersion: 9.4.79-closedown0 diff --git a/helm/hpcc/templates/_helpers.tpl b/helm/hpcc/templates/_helpers.tpl index 44b5450b6d3..149a98180ce 100644 --- a/helm/hpcc/templates/_helpers.tpl +++ b/helm/hpcc/templates/_helpers.tpl @@ -1519,7 +1519,7 @@ kind: Service metadata: name: {{ $lvars.serviceName | quote }} labels: - helmVersion: 9.4.77-closedown0 + helmVersion: 9.4.79-closedown0 {{- include "hpcc.addStandardLabels" (dict "root" $.root "instance" $lvars.serviceName ) | indent 4 }} {{- if $lvars.labels }} {{ toYaml $lvars.labels | indent 4 }} diff --git a/helm/hpcc/templates/dafilesrv.yaml b/helm/hpcc/templates/dafilesrv.yaml index 4c100577984..550874d326b 100644 --- a/helm/hpcc/templates/dafilesrv.yaml +++ b/helm/hpcc/templates/dafilesrv.yaml @@ -51,7 +51,7 @@ spec: labels: {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "dafilesrv" "name" "dafilesrv" "instance" .name) | indent 8 }} server: {{ .name | quote }} - helmVersion: 9.4.77-closedown0 + helmVersion: 9.4.79-closedown0 annotations: checksum/config: {{ $configSHA }} {{- include "hpcc.generateAnnotations" $commonCtx | indent 8 }} diff --git a/helm/hpcc/templates/dali.yaml b/helm/hpcc/templates/dali.yaml index b10898b7fe9..35a8cc3f52e 100644 --- a/helm/hpcc/templates/dali.yaml +++ b/helm/hpcc/templates/dali.yaml @@ -88,7 +88,7 @@ spec: run: {{ $dali.name | quote }} server: {{ $dali.name | quote }} app: dali - helmVersion: 9.4.77-closedown0 + helmVersion: 9.4.79-closedown0 {{- if hasKey $.Values.global "metrics" }} {{- include "hpcc.generateMetricsReporterLabel" $.Values.global.metrics | nindent 8 }} {{- end }} diff --git a/helm/hpcc/templates/dfuserver.yaml b/helm/hpcc/templates/dfuserver.yaml index acb1b44ab0b..97f3cc85f11 100644 --- a/helm/hpcc/templates/dfuserver.yaml +++ b/helm/hpcc/templates/dfuserver.yaml @@ -57,7 +57,7 @@ spec: {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "dfuserver" "name" "dfuserver" "instance" .name) | indent 8 }} run: {{ .name | quote }} accessDali: "yes" - helmVersion: 9.4.77-closedown0 + helmVersion: 9.4.79-closedown0 {{- if hasKey . "labels" }} {{ toYaml .labels | indent 8 }} {{- end }} diff --git a/helm/hpcc/templates/eclagent.yaml b/helm/hpcc/templates/eclagent.yaml index 56ca2e1f78e..b2a010dfc52 100644 --- a/helm/hpcc/templates/eclagent.yaml +++ b/helm/hpcc/templates/eclagent.yaml @@ -60,7 +60,7 @@ data: {{- include "hpcc.addStandardLabels" (dict "root" $ "component" $apptype "name" "eclagent" "instance" $appJobName "instanceOf" (printf "%s-job" .me.name)) | indent 12 }} accessDali: "yes" accessEsp: "yes" - helmVersion: 9.4.77-closedown0 + helmVersion: 9.4.79-closedown0 {{- if hasKey .me "labels" }} {{ toYaml .me.labels | indent 12 }} {{- end }} @@ -137,7 +137,7 @@ spec: run: {{ .name | quote }} accessDali: "yes" accessEsp: {{ .useChildProcesses | default false | ternary "yes" "no" | quote }} - helmVersion: 9.4.77-closedown0 + helmVersion: 9.4.79-closedown0 {{- if hasKey . "labels" }} {{ toYaml .labels | indent 8 }} {{- end }} diff --git a/helm/hpcc/templates/eclccserver.yaml b/helm/hpcc/templates/eclccserver.yaml index 7aaf05e1d7e..5c87776b868 100644 --- a/helm/hpcc/templates/eclccserver.yaml +++ b/helm/hpcc/templates/eclccserver.yaml @@ -58,7 +58,7 @@ data: {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "eclccserver" "name" "eclccserver" "instance" $compileJobName "instanceOf" (printf "%s-job" .me.name)) | indent 12 }} accessDali: "yes" accessEsp: "yes" - helmVersion: 9.4.77-closedown0 + helmVersion: 9.4.79-closedown0 {{- if hasKey .me "labels" }} {{ toYaml .me.labels | indent 12 }} {{- end }} @@ -143,7 +143,7 @@ spec: run: {{ .name | quote }} accessDali: "yes" accessEsp: {{ .useChildProcesses | default false | ternary "yes" "no" | quote }} - helmVersion: 9.4.77-closedown0 + helmVersion: 9.4.79-closedown0 {{- if hasKey . "labels" }} {{ toYaml .labels | indent 8 }} {{- end }} diff --git a/helm/hpcc/templates/eclscheduler.yaml b/helm/hpcc/templates/eclscheduler.yaml index 439327aab07..f9632b6cc7e 100644 --- a/helm/hpcc/templates/eclscheduler.yaml +++ b/helm/hpcc/templates/eclscheduler.yaml @@ -65,7 +65,7 @@ spec: run: {{ .name | quote }} accessDali: "yes" accessEsp: "no" - helmVersion: 9.4.77-closedown0 + helmVersion: 9.4.79-closedown0 {{- if hasKey . "labels" }} {{ toYaml .labels | indent 8 }} {{- end }} diff --git a/helm/hpcc/templates/esp.yaml b/helm/hpcc/templates/esp.yaml index 3622e7f306c..9eb822b5498 100644 --- a/helm/hpcc/templates/esp.yaml +++ b/helm/hpcc/templates/esp.yaml @@ -122,7 +122,7 @@ spec: accessSasha: "yes" {{- end }} app: {{ $application }} - helmVersion: 9.4.77-closedown0 + helmVersion: 9.4.79-closedown0 {{- include "hpcc.addStandardLabels" (dict "root" $ "name" $application "component" "esp" "instance" .name) | indent 8 }} {{- if hasKey $.Values.global "metrics" }} {{- include "hpcc.generateMetricsReporterLabel" $.Values.global.metrics | nindent 8 }} diff --git a/helm/hpcc/templates/localroxie.yaml b/helm/hpcc/templates/localroxie.yaml index 87e6681f876..4c9d2176cd6 100644 --- a/helm/hpcc/templates/localroxie.yaml +++ b/helm/hpcc/templates/localroxie.yaml @@ -73,7 +73,7 @@ spec: server: {{ $servername | quote }} accessDali: "yes" accessEsp: "yes" - helmVersion: 9.4.77-closedown0 + helmVersion: 9.4.79-closedown0 {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "roxie-server" "name" "roxie" "instance" $roxie.name) | indent 8 }} {{- if hasKey . "labels" }} {{ toYaml .labels | indent 8 }} diff --git a/helm/hpcc/templates/roxie.yaml b/helm/hpcc/templates/roxie.yaml index 43e3c3429c6..2895339abcf 100644 --- a/helm/hpcc/templates/roxie.yaml +++ b/helm/hpcc/templates/roxie.yaml @@ -125,7 +125,7 @@ spec: {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "topology-server" "name" "roxie" "instance" $commonCtx.toponame) | indent 8 }} run: {{ $commonCtx.toponame | quote }} roxie-cluster: {{ $roxie.name | quote }} - helmVersion: 9.4.77-closedown0 + helmVersion: 9.4.79-closedown0 {{- if hasKey $.Values.global "metrics" }} {{- include "hpcc.generateMetricsReporterLabel" $.Values.global.metrics | nindent 8}} {{- end }} @@ -182,7 +182,7 @@ kind: Service metadata: name: {{ $commonCtx.toponame | quote }} labels: - helmVersion: 9.4.77-closedown0 + helmVersion: 9.4.79-closedown0 {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "topology-server" "name" "roxie" "instance" $commonCtx.toponame) | indent 4 }} spec: ports: @@ -244,7 +244,7 @@ spec: roxie-cluster: {{ $roxie.name | quote }} accessDali: "yes" accessEsp: "yes" - helmVersion: 9.4.77-closedown0 + helmVersion: 9.4.79-closedown0 {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "roxie-server" "name" "roxie" "instance" $servername) | indent 8 }} {{- if hasKey $.Values.global "metrics" }} {{- include "hpcc.generateMetricsReporterLabel" $.Values.global.metrics | nindent 8}} @@ -352,7 +352,7 @@ spec: roxie-cluster: {{ $roxie.name | quote }} accessDali: "yes" accessEsp: "yes" - helmVersion: 9.4.77-closedown0 + helmVersion: 9.4.79-closedown0 {{- if hasKey $.Values.global "metrics" }} {{- include "hpcc.generateMetricsReporterLabel" $.Values.global.metrics | nindent 8}} {{- end }} diff --git a/helm/hpcc/templates/sasha.yaml b/helm/hpcc/templates/sasha.yaml index a8808e4bf46..9afb257dbeb 100644 --- a/helm/hpcc/templates/sasha.yaml +++ b/helm/hpcc/templates/sasha.yaml @@ -53,7 +53,7 @@ spec: server: {{ $serviceName | quote }} app: sasha accessDali: {{ (has "dali" $sasha.access) | ternary "yes" "no" | quote }} - helmVersion: 9.4.77-closedown0 + helmVersion: 9.4.79-closedown0 {{- if hasKey $sasha "labels" }} {{ toYaml $sasha.labels | indent 8 }} {{- end }} diff --git a/helm/hpcc/templates/thor.yaml b/helm/hpcc/templates/thor.yaml index ee145b5d751..ee4c592b6a7 100644 --- a/helm/hpcc/templates/thor.yaml +++ b/helm/hpcc/templates/thor.yaml @@ -86,7 +86,7 @@ data: labels: accessDali: "yes" accessEsp: "yes" - helmVersion: 9.4.77-closedown0 + helmVersion: 9.4.79-closedown0 {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "eclagent" "name" "thor" "instance" $eclAgentJobName "instanceOf" (printf "%s-job" .eclAgentName)) | indent 8 }} {{- if hasKey .me "labels" }} {{ toYaml .me.labels | indent 12 }} @@ -151,7 +151,7 @@ data: accessEsp: "yes" app: "thor" component: "thormanager" - helmVersion: 9.4.77-closedown0 + helmVersion: 9.4.79-closedown0 instance: "_HPCC_JOBNAME_" job: "_HPCC_JOBNAME_" {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "thormanager" "name" "thor" "instance" $thorManagerJobName "instanceOf" (printf "%s-thormanager-job" .me.name)) | indent 12 }} @@ -218,7 +218,7 @@ data: accessEsp: "yes" app: "thor" component: "thorworker" - helmVersion: 9.4.77-closedown0 + helmVersion: 9.4.79-closedown0 instance: "_HPCC_JOBNAME_" job: "_HPCC_JOBNAME_" {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "thorworker" "name" "thor" "instance" $thorWorkerJobName "instanceOf" (printf "%s-thorworker-job" .me.name)) | indent 12 }} @@ -351,7 +351,7 @@ spec: accessEsp: {{ $commonCtx.eclAgentUseChildProcesses | ternary "yes" "no" | quote }} app: "thor" component: "thor-eclagent" - helmVersion: 9.4.77-closedown0 + helmVersion: 9.4.79-closedown0 instance: {{ $commonCtx.eclAgentName | quote }} {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "eclagent" "name" "thor" "instance" $commonCtx.eclAgentName ) | indent 8 }} {{- if hasKey $commonCtx.me "labels" }} @@ -416,7 +416,7 @@ spec: accessEsp: "no" app: "thor" component: "thor-thoragent" - helmVersion: 9.4.77-closedown0 + helmVersion: 9.4.79-closedown0 instance: {{ $commonCtx.thorAgentName | quote }} {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "eclagent" "name" "thor" "instance" $commonCtx.thorAgentName ) | indent 8 }} {{- if hasKey $commonCtx.me "labels" }} diff --git a/version.cmake b/version.cmake index 9fbe6320864..531c0d80b09 100644 --- a/version.cmake +++ b/version.cmake @@ -5,8 +5,8 @@ set ( HPCC_NAME "Community Edition" ) set ( HPCC_PROJECT "community" ) set ( HPCC_MAJOR 9 ) set ( HPCC_MINOR 4 ) -set ( HPCC_POINT 77 ) +set ( HPCC_POINT 79 ) set ( HPCC_MATURITY "closedown" ) set ( HPCC_SEQUENCE 0 ) -set ( HPCC_TAG_TIMESTAMP "2024-07-05T08:40:25Z" ) +set ( HPCC_TAG_TIMESTAMP "2024-07-11T16:14:39Z" ) ### From 58924f39f357fe5cb93af0c01d551c85d9a1027b Mon Sep 17 00:00:00 2001 From: Jake Smith Date: Thu, 11 Jul 2024 17:15:36 +0100 Subject: [PATCH 09/29] Split off 9.6.30 Signed-off-by: Jake Smith --- helm/hpcc/Chart.yaml | 4 ++-- helm/hpcc/templates/_helpers.tpl | 2 +- helm/hpcc/templates/dafilesrv.yaml | 2 +- helm/hpcc/templates/dali.yaml | 2 +- helm/hpcc/templates/dfuserver.yaml | 2 +- helm/hpcc/templates/eclagent.yaml | 4 ++-- helm/hpcc/templates/eclccserver.yaml | 4 ++-- helm/hpcc/templates/eclscheduler.yaml | 2 +- helm/hpcc/templates/esp.yaml | 2 +- helm/hpcc/templates/localroxie.yaml | 2 +- helm/hpcc/templates/roxie.yaml | 8 ++++---- helm/hpcc/templates/sasha.yaml | 2 +- helm/hpcc/templates/thor.yaml | 10 +++++----- version.cmake | 4 ++-- 14 files changed, 25 insertions(+), 25 deletions(-) diff --git a/helm/hpcc/Chart.yaml b/helm/hpcc/Chart.yaml index e7a985dca35..5087fbd1363 100644 --- a/helm/hpcc/Chart.yaml +++ b/helm/hpcc/Chart.yaml @@ -6,9 +6,9 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. -version: 9.6.29-closedown0 +version: 9.6.31-closedown0 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. -appVersion: 9.6.29-closedown0 +appVersion: 9.6.31-closedown0 diff --git a/helm/hpcc/templates/_helpers.tpl b/helm/hpcc/templates/_helpers.tpl index e6e11796b7b..607d36b54ae 100644 --- a/helm/hpcc/templates/_helpers.tpl +++ b/helm/hpcc/templates/_helpers.tpl @@ -1523,7 +1523,7 @@ kind: Service metadata: name: {{ $lvars.serviceName | quote }} labels: - helmVersion: 9.6.29-closedown0 + helmVersion: 9.6.31-closedown0 {{- include "hpcc.addStandardLabels" (dict "root" $.root "instance" $lvars.serviceName ) | indent 4 }} {{- if $lvars.labels }} {{ toYaml $lvars.labels | indent 4 }} diff --git a/helm/hpcc/templates/dafilesrv.yaml b/helm/hpcc/templates/dafilesrv.yaml index 3ed084449ad..b0fa40357de 100644 --- a/helm/hpcc/templates/dafilesrv.yaml +++ b/helm/hpcc/templates/dafilesrv.yaml @@ -51,7 +51,7 @@ spec: labels: {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "dafilesrv" "name" "dafilesrv" "instance" .name) | indent 8 }} server: {{ .name | quote }} - helmVersion: 9.6.29-closedown0 + helmVersion: 9.6.31-closedown0 annotations: checksum/config: {{ $configSHA }} {{- include "hpcc.generateAnnotations" $commonCtx | indent 8 }} diff --git a/helm/hpcc/templates/dali.yaml b/helm/hpcc/templates/dali.yaml index 3fbc0a46c9d..d07f8677a5c 100644 --- a/helm/hpcc/templates/dali.yaml +++ b/helm/hpcc/templates/dali.yaml @@ -88,7 +88,7 @@ spec: run: {{ $dali.name | quote }} server: {{ $dali.name | quote }} app: dali - helmVersion: 9.6.29-closedown0 + helmVersion: 9.6.31-closedown0 {{- if hasKey $.Values.global "metrics" }} {{- include "hpcc.generateMetricsReporterLabel" $.Values.global.metrics | nindent 8 }} {{- end }} diff --git a/helm/hpcc/templates/dfuserver.yaml b/helm/hpcc/templates/dfuserver.yaml index 3630e102027..71cc69d8d17 100644 --- a/helm/hpcc/templates/dfuserver.yaml +++ b/helm/hpcc/templates/dfuserver.yaml @@ -57,7 +57,7 @@ spec: {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "dfuserver" "name" "dfuserver" "instance" .name) | indent 8 }} run: {{ .name | quote }} accessDali: "yes" - helmVersion: 9.6.29-closedown0 + helmVersion: 9.6.31-closedown0 {{- if hasKey . "labels" }} {{ toYaml .labels | indent 8 }} {{- end }} diff --git a/helm/hpcc/templates/eclagent.yaml b/helm/hpcc/templates/eclagent.yaml index 1ecf5855156..2164783e7e4 100644 --- a/helm/hpcc/templates/eclagent.yaml +++ b/helm/hpcc/templates/eclagent.yaml @@ -62,7 +62,7 @@ data: {{- include "hpcc.addStandardLabels" (dict "root" $ "component" $apptype "name" "eclagent" "instance" $appJobName "instanceOf" (printf "%s-job" .me.name)) | indent 12 }} accessDali: "yes" accessEsp: "yes" - helmVersion: 9.6.29-closedown0 + helmVersion: 9.6.31-closedown0 {{- if hasKey .me "labels" }} {{ toYaml .me.labels | indent 12 }} {{- end }} @@ -139,7 +139,7 @@ spec: run: {{ .name | quote }} accessDali: "yes" accessEsp: {{ .useChildProcesses | default false | ternary "yes" "no" | quote }} - helmVersion: 9.6.29-closedown0 + helmVersion: 9.6.31-closedown0 {{- if hasKey . "labels" }} {{ toYaml .labels | indent 8 }} {{- end }} diff --git a/helm/hpcc/templates/eclccserver.yaml b/helm/hpcc/templates/eclccserver.yaml index ae5f590aca7..ac8d45843eb 100644 --- a/helm/hpcc/templates/eclccserver.yaml +++ b/helm/hpcc/templates/eclccserver.yaml @@ -62,7 +62,7 @@ data: {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "eclccserver" "name" "eclccserver" "instance" $compileJobName "instanceOf" (printf "%s-job" .me.name)) | indent 12 }} accessDali: "yes" accessEsp: "yes" - helmVersion: 9.6.29-closedown0 + helmVersion: 9.6.31-closedown0 {{- if hasKey .me "labels" }} {{ toYaml .me.labels | indent 12 }} {{- end }} @@ -147,7 +147,7 @@ spec: run: {{ .name | quote }} accessDali: "yes" accessEsp: {{ .useChildProcesses | default false | ternary "yes" "no" | quote }} - helmVersion: 9.6.29-closedown0 + helmVersion: 9.6.31-closedown0 {{- if hasKey . "labels" }} {{ toYaml .labels | indent 8 }} {{- end }} diff --git a/helm/hpcc/templates/eclscheduler.yaml b/helm/hpcc/templates/eclscheduler.yaml index bb5fb36b087..4c0b76c320d 100644 --- a/helm/hpcc/templates/eclscheduler.yaml +++ b/helm/hpcc/templates/eclscheduler.yaml @@ -65,7 +65,7 @@ spec: run: {{ .name | quote }} accessDali: "yes" accessEsp: "no" - helmVersion: 9.6.29-closedown0 + helmVersion: 9.6.31-closedown0 {{- if hasKey . "labels" }} {{ toYaml .labels | indent 8 }} {{- end }} diff --git a/helm/hpcc/templates/esp.yaml b/helm/hpcc/templates/esp.yaml index 4d9711a69c8..59e9a6cd9c1 100644 --- a/helm/hpcc/templates/esp.yaml +++ b/helm/hpcc/templates/esp.yaml @@ -125,7 +125,7 @@ spec: accessSasha: "yes" {{- end }} app: {{ $application }} - helmVersion: 9.6.29-closedown0 + helmVersion: 9.6.31-closedown0 {{- include "hpcc.addStandardLabels" (dict "root" $ "name" $application "component" "esp" "instance" .name) | indent 8 }} {{- if hasKey $.Values.global "metrics" }} {{- include "hpcc.generateMetricsReporterLabel" $.Values.global.metrics | nindent 8 }} diff --git a/helm/hpcc/templates/localroxie.yaml b/helm/hpcc/templates/localroxie.yaml index d8b351c43ef..7dc9a1cde5b 100644 --- a/helm/hpcc/templates/localroxie.yaml +++ b/helm/hpcc/templates/localroxie.yaml @@ -73,7 +73,7 @@ spec: server: {{ $servername | quote }} accessDali: "yes" accessEsp: "yes" - helmVersion: 9.6.29-closedown0 + helmVersion: 9.6.31-closedown0 {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "roxie-server" "name" "roxie" "instance" $roxie.name) | indent 8 }} {{- if hasKey . "labels" }} {{ toYaml .labels | indent 8 }} diff --git a/helm/hpcc/templates/roxie.yaml b/helm/hpcc/templates/roxie.yaml index 07d177220e3..f87b3d73196 100644 --- a/helm/hpcc/templates/roxie.yaml +++ b/helm/hpcc/templates/roxie.yaml @@ -125,7 +125,7 @@ spec: {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "topology-server" "name" "roxie" "instance" $commonCtx.toponame) | indent 8 }} run: {{ $commonCtx.toponame | quote }} roxie-cluster: {{ $roxie.name | quote }} - helmVersion: 9.6.29-closedown0 + helmVersion: 9.6.31-closedown0 {{- if hasKey $.Values.global "metrics" }} {{- include "hpcc.generateMetricsReporterLabel" $.Values.global.metrics | nindent 8}} {{- end }} @@ -182,7 +182,7 @@ kind: Service metadata: name: {{ $commonCtx.toponame | quote }} labels: - helmVersion: 9.6.29-closedown0 + helmVersion: 9.6.31-closedown0 {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "topology-server" "name" "roxie" "instance" $commonCtx.toponame) | indent 4 }} spec: ports: @@ -244,7 +244,7 @@ spec: roxie-cluster: {{ $roxie.name | quote }} accessDali: "yes" accessEsp: "yes" - helmVersion: 9.6.29-closedown0 + helmVersion: 9.6.31-closedown0 {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "roxie-server" "name" "roxie" "instance" $servername) | indent 8 }} {{- if hasKey $.Values.global "metrics" }} {{- include "hpcc.generateMetricsReporterLabel" $.Values.global.metrics | nindent 8}} @@ -352,7 +352,7 @@ spec: roxie-cluster: {{ $roxie.name | quote }} accessDali: "yes" accessEsp: "yes" - helmVersion: 9.6.29-closedown0 + helmVersion: 9.6.31-closedown0 {{- if hasKey $.Values.global "metrics" }} {{- include "hpcc.generateMetricsReporterLabel" $.Values.global.metrics | nindent 8}} {{- end }} diff --git a/helm/hpcc/templates/sasha.yaml b/helm/hpcc/templates/sasha.yaml index aa022562c67..0781a27acdc 100644 --- a/helm/hpcc/templates/sasha.yaml +++ b/helm/hpcc/templates/sasha.yaml @@ -53,7 +53,7 @@ spec: server: {{ $serviceName | quote }} app: sasha accessDali: {{ (has "dali" $sasha.access) | ternary "yes" "no" | quote }} - helmVersion: 9.6.29-closedown0 + helmVersion: 9.6.31-closedown0 {{- if hasKey $sasha "labels" }} {{ toYaml $sasha.labels | indent 8 }} {{- end }} diff --git a/helm/hpcc/templates/thor.yaml b/helm/hpcc/templates/thor.yaml index 09b19c12ed8..87a3015a52c 100644 --- a/helm/hpcc/templates/thor.yaml +++ b/helm/hpcc/templates/thor.yaml @@ -88,7 +88,7 @@ data: labels: accessDali: "yes" accessEsp: "yes" - helmVersion: 9.6.29-closedown0 + helmVersion: 9.6.31-closedown0 {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "eclagent" "name" "thor" "instance" $eclAgentJobName "instanceOf" (printf "%s-job" .eclAgentName)) | indent 8 }} {{- if hasKey .me "labels" }} {{ toYaml .me.labels | indent 12 }} @@ -153,7 +153,7 @@ data: accessEsp: "yes" app: "thor" component: "thormanager" - helmVersion: 9.6.29-closedown0 + helmVersion: 9.6.31-closedown0 instance: "_HPCC_JOBNAME_" job: "_HPCC_JOBNAME_" {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "thormanager" "name" "thor" "instance" $thorManagerJobName "instanceOf" (printf "%s-thormanager-job" .me.name)) | indent 12 }} @@ -220,7 +220,7 @@ data: accessEsp: "yes" app: "thor" component: "thorworker" - helmVersion: 9.6.29-closedown0 + helmVersion: 9.6.31-closedown0 instance: "_HPCC_JOBNAME_" job: "_HPCC_JOBNAME_" {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "thorworker" "name" "thor" "instance" $thorWorkerJobName "instanceOf" (printf "%s-thorworker-job" .me.name)) | indent 12 }} @@ -353,7 +353,7 @@ spec: accessEsp: {{ $commonCtx.eclAgentUseChildProcesses | ternary "yes" "no" | quote }} app: "thor" component: "thor-eclagent" - helmVersion: 9.6.29-closedown0 + helmVersion: 9.6.31-closedown0 instance: {{ $commonCtx.eclAgentName | quote }} {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "eclagent" "name" "thor" "instance" $commonCtx.eclAgentName ) | indent 8 }} {{- if hasKey $commonCtx.me "labels" }} @@ -418,7 +418,7 @@ spec: accessEsp: "no" app: "thor" component: "thor-thoragent" - helmVersion: 9.6.29-closedown0 + helmVersion: 9.6.31-closedown0 instance: {{ $commonCtx.thorAgentName | quote }} {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "eclagent" "name" "thor" "instance" $commonCtx.thorAgentName ) | indent 8 }} {{- if hasKey $commonCtx.me "labels" }} diff --git a/version.cmake b/version.cmake index ce1a22a1538..5a69bba8dff 100644 --- a/version.cmake +++ b/version.cmake @@ -5,8 +5,8 @@ set ( HPCC_NAME "Community Edition" ) set ( HPCC_PROJECT "community" ) set ( HPCC_MAJOR 9 ) set ( HPCC_MINOR 6 ) -set ( HPCC_POINT 29 ) +set ( HPCC_POINT 31 ) set ( HPCC_MATURITY "closedown" ) set ( HPCC_SEQUENCE 0 ) -set ( HPCC_TAG_TIMESTAMP "2024-07-05T08:39:00Z" ) +set ( HPCC_TAG_TIMESTAMP "2024-07-11T16:15:36Z" ) ### From 52757b0dbf24d0f01522f38afa8ea9f38099c33a Mon Sep 17 00:00:00 2001 From: Jake Smith Date: Thu, 11 Jul 2024 17:16:39 +0100 Subject: [PATCH 10/29] Split off 9.8.4 Signed-off-by: Jake Smith --- helm/hpcc/Chart.yaml | 4 ++-- helm/hpcc/templates/_helpers.tpl | 2 +- helm/hpcc/templates/dafilesrv.yaml | 2 +- helm/hpcc/templates/dali.yaml | 2 +- helm/hpcc/templates/dfuserver.yaml | 2 +- helm/hpcc/templates/eclagent.yaml | 4 ++-- helm/hpcc/templates/eclccserver.yaml | 4 ++-- helm/hpcc/templates/eclscheduler.yaml | 2 +- helm/hpcc/templates/esp.yaml | 2 +- helm/hpcc/templates/localroxie.yaml | 2 +- helm/hpcc/templates/roxie.yaml | 8 ++++---- helm/hpcc/templates/sasha.yaml | 2 +- helm/hpcc/templates/thor.yaml | 10 +++++----- version.cmake | 4 ++-- 14 files changed, 25 insertions(+), 25 deletions(-) diff --git a/helm/hpcc/Chart.yaml b/helm/hpcc/Chart.yaml index 2ea44e4ddff..2d18419c99d 100644 --- a/helm/hpcc/Chart.yaml +++ b/helm/hpcc/Chart.yaml @@ -6,9 +6,9 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. -version: 9.8.3-closedown0 +version: 9.8.5-closedown0 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. -appVersion: 9.8.3-closedown0 +appVersion: 9.8.5-closedown0 diff --git a/helm/hpcc/templates/_helpers.tpl b/helm/hpcc/templates/_helpers.tpl index 9dfe15ff52c..7307367562b 100644 --- a/helm/hpcc/templates/_helpers.tpl +++ b/helm/hpcc/templates/_helpers.tpl @@ -1523,7 +1523,7 @@ kind: Service metadata: name: {{ $lvars.serviceName | quote }} labels: - helmVersion: 9.8.3-closedown0 + helmVersion: 9.8.5-closedown0 {{- include "hpcc.addStandardLabels" (dict "root" $.root "instance" $lvars.serviceName ) | indent 4 }} {{- if $lvars.labels }} {{ toYaml $lvars.labels | indent 4 }} diff --git a/helm/hpcc/templates/dafilesrv.yaml b/helm/hpcc/templates/dafilesrv.yaml index e107953ff67..e378c759bf5 100644 --- a/helm/hpcc/templates/dafilesrv.yaml +++ b/helm/hpcc/templates/dafilesrv.yaml @@ -51,7 +51,7 @@ spec: labels: {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "dafilesrv" "name" "dafilesrv" "instance" .name) | indent 8 }} server: {{ .name | quote }} - helmVersion: 9.8.3-closedown0 + helmVersion: 9.8.5-closedown0 annotations: checksum/config: {{ $configSHA }} {{- include "hpcc.generateAnnotations" $commonCtx | indent 8 }} diff --git a/helm/hpcc/templates/dali.yaml b/helm/hpcc/templates/dali.yaml index 4d6c0d58a2f..2adfbbd328d 100644 --- a/helm/hpcc/templates/dali.yaml +++ b/helm/hpcc/templates/dali.yaml @@ -88,7 +88,7 @@ spec: run: {{ $dali.name | quote }} server: {{ $dali.name | quote }} app: dali - helmVersion: 9.8.3-closedown0 + helmVersion: 9.8.5-closedown0 {{- if hasKey $.Values.global "metrics" }} {{- include "hpcc.generateMetricsReporterLabel" $.Values.global.metrics | nindent 8 }} {{- end }} diff --git a/helm/hpcc/templates/dfuserver.yaml b/helm/hpcc/templates/dfuserver.yaml index 3641037b562..4622e253e30 100644 --- a/helm/hpcc/templates/dfuserver.yaml +++ b/helm/hpcc/templates/dfuserver.yaml @@ -57,7 +57,7 @@ spec: {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "dfuserver" "name" "dfuserver" "instance" .name) | indent 8 }} run: {{ .name | quote }} accessDali: "yes" - helmVersion: 9.8.3-closedown0 + helmVersion: 9.8.5-closedown0 {{- if hasKey . "labels" }} {{ toYaml .labels | indent 8 }} {{- end }} diff --git a/helm/hpcc/templates/eclagent.yaml b/helm/hpcc/templates/eclagent.yaml index a8bebc0ec99..954184a51dc 100644 --- a/helm/hpcc/templates/eclagent.yaml +++ b/helm/hpcc/templates/eclagent.yaml @@ -62,7 +62,7 @@ data: {{- include "hpcc.addStandardLabels" (dict "root" $ "component" $apptype "name" "eclagent" "instance" $appJobName "instanceOf" (printf "%s-job" .me.name)) | indent 12 }} accessDali: "yes" accessEsp: "yes" - helmVersion: 9.8.3-closedown0 + helmVersion: 9.8.5-closedown0 {{- if hasKey .me "labels" }} {{ toYaml .me.labels | indent 12 }} {{- end }} @@ -139,7 +139,7 @@ spec: run: {{ .name | quote }} accessDali: "yes" accessEsp: {{ .useChildProcesses | default false | ternary "yes" "no" | quote }} - helmVersion: 9.8.3-closedown0 + helmVersion: 9.8.5-closedown0 {{- if hasKey . "labels" }} {{ toYaml .labels | indent 8 }} {{- end }} diff --git a/helm/hpcc/templates/eclccserver.yaml b/helm/hpcc/templates/eclccserver.yaml index c1d8221f96f..ccfbd73a288 100644 --- a/helm/hpcc/templates/eclccserver.yaml +++ b/helm/hpcc/templates/eclccserver.yaml @@ -62,7 +62,7 @@ data: {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "eclccserver" "name" "eclccserver" "instance" $compileJobName "instanceOf" (printf "%s-job" .me.name)) | indent 12 }} accessDali: "yes" accessEsp: "yes" - helmVersion: 9.8.3-closedown0 + helmVersion: 9.8.5-closedown0 {{- if hasKey .me "labels" }} {{ toYaml .me.labels | indent 12 }} {{- end }} @@ -147,7 +147,7 @@ spec: run: {{ .name | quote }} accessDali: "yes" accessEsp: {{ .useChildProcesses | default false | ternary "yes" "no" | quote }} - helmVersion: 9.8.3-closedown0 + helmVersion: 9.8.5-closedown0 {{- if hasKey . "labels" }} {{ toYaml .labels | indent 8 }} {{- end }} diff --git a/helm/hpcc/templates/eclscheduler.yaml b/helm/hpcc/templates/eclscheduler.yaml index 0d5b83fc66b..9738de3223d 100644 --- a/helm/hpcc/templates/eclscheduler.yaml +++ b/helm/hpcc/templates/eclscheduler.yaml @@ -65,7 +65,7 @@ spec: run: {{ .name | quote }} accessDali: "yes" accessEsp: "no" - helmVersion: 9.8.3-closedown0 + helmVersion: 9.8.5-closedown0 {{- if hasKey . "labels" }} {{ toYaml .labels | indent 8 }} {{- end }} diff --git a/helm/hpcc/templates/esp.yaml b/helm/hpcc/templates/esp.yaml index 476d7c4917b..6b9fc36695a 100644 --- a/helm/hpcc/templates/esp.yaml +++ b/helm/hpcc/templates/esp.yaml @@ -125,7 +125,7 @@ spec: accessSasha: "yes" {{- end }} app: {{ $application }} - helmVersion: 9.8.3-closedown0 + helmVersion: 9.8.5-closedown0 {{- include "hpcc.addStandardLabels" (dict "root" $ "name" $application "component" "esp" "instance" .name) | indent 8 }} {{- if hasKey $.Values.global "metrics" }} {{- include "hpcc.generateMetricsReporterLabel" $.Values.global.metrics | nindent 8 }} diff --git a/helm/hpcc/templates/localroxie.yaml b/helm/hpcc/templates/localroxie.yaml index 989712bf0e7..c764ab1f8f3 100644 --- a/helm/hpcc/templates/localroxie.yaml +++ b/helm/hpcc/templates/localroxie.yaml @@ -73,7 +73,7 @@ spec: server: {{ $servername | quote }} accessDali: "yes" accessEsp: "yes" - helmVersion: 9.8.3-closedown0 + helmVersion: 9.8.5-closedown0 {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "roxie-server" "name" "roxie" "instance" $roxie.name) | indent 8 }} {{- if hasKey . "labels" }} {{ toYaml .labels | indent 8 }} diff --git a/helm/hpcc/templates/roxie.yaml b/helm/hpcc/templates/roxie.yaml index aac49e8b770..3fb3c33f137 100644 --- a/helm/hpcc/templates/roxie.yaml +++ b/helm/hpcc/templates/roxie.yaml @@ -125,7 +125,7 @@ spec: {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "topology-server" "name" "roxie" "instance" $commonCtx.toponame) | indent 8 }} run: {{ $commonCtx.toponame | quote }} roxie-cluster: {{ $roxie.name | quote }} - helmVersion: 9.8.3-closedown0 + helmVersion: 9.8.5-closedown0 {{- if hasKey $.Values.global "metrics" }} {{- include "hpcc.generateMetricsReporterLabel" $.Values.global.metrics | nindent 8}} {{- end }} @@ -182,7 +182,7 @@ kind: Service metadata: name: {{ $commonCtx.toponame | quote }} labels: - helmVersion: 9.8.3-closedown0 + helmVersion: 9.8.5-closedown0 {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "topology-server" "name" "roxie" "instance" $commonCtx.toponame) | indent 4 }} spec: ports: @@ -244,7 +244,7 @@ spec: roxie-cluster: {{ $roxie.name | quote }} accessDali: "yes" accessEsp: "yes" - helmVersion: 9.8.3-closedown0 + helmVersion: 9.8.5-closedown0 {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "roxie-server" "name" "roxie" "instance" $servername) | indent 8 }} {{- if hasKey $.Values.global "metrics" }} {{- include "hpcc.generateMetricsReporterLabel" $.Values.global.metrics | nindent 8}} @@ -352,7 +352,7 @@ spec: roxie-cluster: {{ $roxie.name | quote }} accessDali: "yes" accessEsp: "yes" - helmVersion: 9.8.3-closedown0 + helmVersion: 9.8.5-closedown0 {{- if hasKey $.Values.global "metrics" }} {{- include "hpcc.generateMetricsReporterLabel" $.Values.global.metrics | nindent 8}} {{- end }} diff --git a/helm/hpcc/templates/sasha.yaml b/helm/hpcc/templates/sasha.yaml index bd9dae275b8..ef7b5862862 100644 --- a/helm/hpcc/templates/sasha.yaml +++ b/helm/hpcc/templates/sasha.yaml @@ -53,7 +53,7 @@ spec: server: {{ $serviceName | quote }} app: sasha accessDali: {{ (has "dali" $sasha.access) | ternary "yes" "no" | quote }} - helmVersion: 9.8.3-closedown0 + helmVersion: 9.8.5-closedown0 {{- if hasKey $sasha "labels" }} {{ toYaml $sasha.labels | indent 8 }} {{- end }} diff --git a/helm/hpcc/templates/thor.yaml b/helm/hpcc/templates/thor.yaml index e7432127b82..0f65429e5ee 100644 --- a/helm/hpcc/templates/thor.yaml +++ b/helm/hpcc/templates/thor.yaml @@ -88,7 +88,7 @@ data: labels: accessDali: "yes" accessEsp: "yes" - helmVersion: 9.8.3-closedown0 + helmVersion: 9.8.5-closedown0 {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "eclagent" "name" "thor" "instance" $eclAgentJobName "instanceOf" (printf "%s-job" .eclAgentName)) | indent 8 }} {{- if hasKey .me "labels" }} {{ toYaml .me.labels | indent 12 }} @@ -153,7 +153,7 @@ data: accessEsp: "yes" app: "thor" component: "thormanager" - helmVersion: 9.8.3-closedown0 + helmVersion: 9.8.5-closedown0 instance: "_HPCC_JOBNAME_" job: "_HPCC_JOBNAME_" {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "thormanager" "name" "thor" "instance" $thorManagerJobName "instanceOf" (printf "%s-thormanager-job" .me.name)) | indent 12 }} @@ -220,7 +220,7 @@ data: accessEsp: "yes" app: "thor" component: "thorworker" - helmVersion: 9.8.3-closedown0 + helmVersion: 9.8.5-closedown0 instance: "_HPCC_JOBNAME_" job: "_HPCC_JOBNAME_" {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "thorworker" "name" "thor" "instance" $thorWorkerJobName "instanceOf" (printf "%s-thorworker-job" .me.name)) | indent 12 }} @@ -353,7 +353,7 @@ spec: accessEsp: {{ $commonCtx.eclAgentUseChildProcesses | ternary "yes" "no" | quote }} app: "thor" component: "thor-eclagent" - helmVersion: 9.8.3-closedown0 + helmVersion: 9.8.5-closedown0 instance: {{ $commonCtx.eclAgentName | quote }} {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "eclagent" "name" "thor" "instance" $commonCtx.eclAgentName ) | indent 8 }} {{- if hasKey $commonCtx.me "labels" }} @@ -418,7 +418,7 @@ spec: accessEsp: "no" app: "thor" component: "thor-thoragent" - helmVersion: 9.8.3-closedown0 + helmVersion: 9.8.5-closedown0 instance: {{ $commonCtx.thorAgentName | quote }} {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "eclagent" "name" "thor" "instance" $commonCtx.thorAgentName ) | indent 8 }} {{- if hasKey $commonCtx.me "labels" }} diff --git a/version.cmake b/version.cmake index b831e0d1bb4..4d5e94db6d0 100644 --- a/version.cmake +++ b/version.cmake @@ -5,8 +5,8 @@ set ( HPCC_NAME "Community Edition" ) set ( HPCC_PROJECT "community" ) set ( HPCC_MAJOR 9 ) set ( HPCC_MINOR 8 ) -set ( HPCC_POINT 3 ) +set ( HPCC_POINT 5 ) set ( HPCC_MATURITY "closedown" ) set ( HPCC_SEQUENCE 0 ) -set ( HPCC_TAG_TIMESTAMP "2024-07-05T08:37:32Z" ) +set ( HPCC_TAG_TIMESTAMP "2024-07-11T16:16:39Z" ) ### From 4245fbef5f910c70bd71d09f906ea4c1d33c25e1 Mon Sep 17 00:00:00 2001 From: Michael Gardner Date: Thu, 11 Jul 2024 13:48:40 -0400 Subject: [PATCH 11/29] HPCC-27361 Fix std/DataPatterns ecllib signing Signed-off-by: Michael Gardner --- ecllibrary/std/DataPatterns/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ecllibrary/std/DataPatterns/CMakeLists.txt b/ecllibrary/std/DataPatterns/CMakeLists.txt index f7474bec76c..7de94fa1a68 100644 --- a/ecllibrary/std/DataPatterns/CMakeLists.txt +++ b/ecllibrary/std/DataPatterns/CMakeLists.txt @@ -22,6 +22,6 @@ set(SRCS foreach(module ${SRCS}) SIGN_MODULE(${module}) - INSTALL(FILES ${CMAKE_CURRENT_SOURCE_DIR}/${module} DESTINATION share/ecllibrary/std/DataPatterns COMPONENT Runtime) + INSTALL(FILES ${CMAKE_CURRENT_BINARY_DIR}/${module} DESTINATION share/ecllibrary/std/DataPatterns COMPONENT Runtime) endforeach() From ae7fd884fd5768bcd994011f18b8022456681cbc Mon Sep 17 00:00:00 2001 From: Jake Smith Date: Thu, 11 Jul 2024 19:41:58 +0100 Subject: [PATCH 12/29] HPCC-32237 Fix getPlaneAttributeValue default value bug Signed-off-by: Jake Smith --- system/jlib/jfile.cpp | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/system/jlib/jfile.cpp b/system/jlib/jfile.cpp index 2b8d3be1ca0..803fb045565 100644 --- a/system/jlib/jfile.cpp +++ b/system/jlib/jfile.cpp @@ -7866,9 +7866,12 @@ unsigned __int64 getPlaneAttributeValue(const char *planeName, PlaneAttributeTyp CriticalBlock b(planeAttriubuteMapCrit); auto it = planeAttributesMap.find(planeName); if (it != planeAttributesMap.end()) - return it->second[planeAttrType]; - else - return defaultValue; + { + unsigned v = it->second[planeAttrType]; + if (v) // a plane attribute value of 0 is considered as not set + return v; + } + return defaultValue; } size32_t getBlockedFileIOSize(const char *planeName, size32_t defaultSize) From 97eda0c4e5c05f249fc548ebd1d1903257df99b9 Mon Sep 17 00:00:00 2001 From: Gordon Smith Date: Fri, 12 Jul 2024 09:12:42 +0100 Subject: [PATCH 13/29] HPCC-32232 Remove LN release actions Removes the creation of a release in the LN repo Removes the uploading of assets to the LN repo Signed-off-by: Gordon Smith --- .github/workflows/build-assets.yml | 16 +--------------- 1 file changed, 1 insertion(+), 15 deletions(-) diff --git a/.github/workflows/build-assets.yml b/.github/workflows/build-assets.yml index b3444e37433..04ca89aaeec 100644 --- a/.github/workflows/build-assets.yml +++ b/.github/workflows/build-assets.yml @@ -420,20 +420,6 @@ jobs: cmake -S /hpcc-dev/LN -B /hpcc-dev/build -DHPCC_SOURCE_DIR=/hpcc-dev/HPCC-Platform ${{ needs.preamble.outputs.cmake_docker_config }} -DBUILD_LEVEL=ENTERPRISE -DSIGN_MODULES_PASSPHRASE=${{ secrets.SIGN_MODULES_PASSPHRASE }} -DSIGN_MODULES_KEYID=${{ secrets.SIGN_MODULES_KEYID }} -DPLATFORM=ON -DINCLUDE_PLUGINS=ON -DCONTAINERIZED=OFF -DSUPPRESS_REMBED=ON -DSUPPRESS_V8EMBED=ON -DSUPPRESS_SPARK=ON -DCPACK_STRIP_FILES=OFF && \ cmake --build /hpcc-dev/build --parallel $(nproc) --target package" - - name: Upload Assets (enterprise) - if: ${{ matrix.ee }} - uses: ncipollo/release-action@v1.12.0 - with: - allowUpdates: true - generateReleaseNotes: false - prerelease: ${{ contains(github.ref, '-rc') }} - owner: ${{ secrets.LNB_ACTOR }} - repo: LN - token: ${{ secrets.LNB_TOKEN }} - tag: ${{ needs.preamble.outputs.internal_tag }} - artifacts: "${{ needs.preamble.outputs.folder_build }}/hpccsystems-platform-enterprise*.deb,${{ needs.preamble.outputs.folder_build }}/hpccsystems-platform-enterprise*.rpm" - - # Common --- - name: Cleanup Environment if: always() @@ -699,7 +685,7 @@ jobs: C:\"Program Files (x86)"\"Microsoft SDKs"\ClickOnce\SignTool\signtool.exe sign /debug /f ../../sign/hpcc_code_signing.pfx /p ${{ secrets.SIGNING_CERTIFICATE_PASSPHRASE}} /t http://timestamp.digicert.com /fd SHA256 hpccsystems-eclide*.exe - name: Upload Assets - uses: ncipollo/release-action@v1.12.0 + uses: ncipollo/release-action@v1.14.0 with: allowUpdates: true generateReleaseNotes: false From 49ea8dadf4be330a745a958a07f724cc974f1024 Mon Sep 17 00:00:00 2001 From: Rodrigo Pastrana Date: Fri, 12 Jul 2024 12:35:45 -0400 Subject: [PATCH 14/29] HPCC-32240 Grafana/Loki logaccess client secret improvements - Removes checked-in secrets templates - Adds secret manifest - Adds detailed steps in readme Signed-off-by: Rodrigo Pastrana --- helm/managed/logging/loki-stack/README.md | 70 +++++++++++++++++-- .../create-grafana-logaccess-secret.sh | 36 ++++++---- .../loki-stack/grafana-logaccess-secret.yaml | 12 ++++ .../loki-stack/secrets-templates/password | 1 - .../loki-stack/secrets-templates/username | 1 - 5 files changed, 99 insertions(+), 21 deletions(-) create mode 100644 helm/managed/logging/loki-stack/grafana-logaccess-secret.yaml delete mode 100644 helm/managed/logging/loki-stack/secrets-templates/password delete mode 100644 helm/managed/logging/loki-stack/secrets-templates/username diff --git a/helm/managed/logging/loki-stack/README.md b/helm/managed/logging/loki-stack/README.md index 6c4714e898c..e50a0437444 100644 --- a/helm/managed/logging/loki-stack/README.md +++ b/helm/managed/logging/loki-stack/README.md @@ -85,27 +85,87 @@ The logAccess feature allows HPCC to query and package relevant logs for various ### Provide target Grafana/Loki access information -HPCC logAccess requires access to the Grafana username/password. Those values must be provided via a secure secret object. +HPCC logAccess requires access to the Grafana username/password credentials. Those values must be provided via a secure secret object. The secret is expected to be in the 'esp' category, and be named 'grafana-logaccess'. The following key-value pairs are required (key names must be spelled exactly as shown here) username - This should contain the Grafana username password - This should contain the Grafana password +#### Create secret using script The included 'create-grafana-logaccess-secret.sh' helper can be used to create the necessary secret. -Example scripted secret creation command (assuming ./secrets-templates contains a file named exactly as the above keys): +Example scripted secret creation command: ``` - create-grafana-logaccess-secret.sh -d HPCC-Platform/helm/managed/logging/loki-stack/secrets-templates/ -n hpcc + create-grafana-logaccess-secret.sh -u admin -p somepass -n hpcc ``` +#### Create secret manually from file Otherwise, users can create the secret manually. -Example manual secret creation command (assuming ./secrets-templates contains a file named exactly as the above keys): +Example manual secret creation command (assuming ./secrets-templates contains files named exactly as the above keys): ``` - kubectl create secret generic grafana-logaccess --from-file=HPCC-Platform/helm/managed//logging/loki-stack/secrets-templates/ -n hpcc + kubectl create secret generic grafana-logaccess --from-file=HPCC-Platform/helm/managed/logging/loki-stack/secrets-templates/ -n hpcc +``` + +#### Create secret manually from manifest +Otherwise, users can create the secret through a manifest file. + +First, base64 encode the credentials: + +``` +echo -n 'admin' | base64 +echo -n 'whatevergrafanapassword' | base64 +``` + +Add the encoded values to the provided manifest file 'grafana-logaccess-secret.yaml' + +``` +apiVersion: v1 +kind: Secret +metadata: + name: grafana-logaccess +type: Opaque +data: + #Base64 encoded username and password for Grafana + #can be encoded using the following command: + # echo -n 'admin' | base64 + username: YWRtaW4= + # echo -n 'whatevergrafanapassword' | base64 + password: d2hhdGV2ZXJncmFmYW5hcGFzc3dvcmQ= +``` + +Then apply the manifest values: + +``` +kubectl apply -f ./grafana-logaccess-secret.yaml --namespace hpcc --server-side +``` + +#### Verify secret + +At this point, confirm the secret has been created with the expected key values: + +``` +kubectl describe secret grafana-logaccess -n hpcc +``` + +The output should be something like this: + +``` +kubectl describe secret grafana-logaccess -n hpcc +Name: grafana-logaccess +Namespace: hpcc +Labels: +Annotations: + +Type: Opaque + +Data +==== +password: 40 bytes +username: 5 bytes ``` ### Configure HPCC logAccess diff --git a/helm/managed/logging/loki-stack/create-grafana-logaccess-secret.sh b/helm/managed/logging/loki-stack/create-grafana-logaccess-secret.sh index f4c7efbed09..e1246b1f62e 100755 --- a/helm/managed/logging/loki-stack/create-grafana-logaccess-secret.sh +++ b/helm/managed/logging/loki-stack/create-grafana-logaccess-secret.sh @@ -1,30 +1,24 @@ #!/bin/bash WORK_DIR=$(dirname $0) -source ${WORK_DIR}/env-loganalytics k8scommand="kubectl" secretname="grafana-logaccess" -secretsdir="${WORK_DIR}/secrets-templates" namespace="default" +username="admin" +password="" usage() { echo "Creates necessary k8s secret used by HPCC's logAccess to access Loki data source through Grafana" echo "> create-grafana-logaccess-secret.sh [Options]" echo "" + echo "Example: create-grafana-logaccess-secret.sh -u admin -p mypassword -n mynamespace" + echo "" echo "Options:" - echo "-d Specifies directory containing required secret values in self named files." - echo " Defaults to /<${secretssubdir}>" + echo "-u Grafana user name (default: admin)" + echo "-p Grafana password (required)" echo "-h Print Usage message" - echo "-n Specifies namespace for secret" - echo "" - echo "Requires directory containing secret values in dedicated files." - echo "Defaults to ${secretssubdir} if not specified via -d option." - echo "" - echo "Expected directory structure:" - echo "${secretsdir}/" - echo " password - Should contain Grafana user name" - echo " username - Should contain Grafana password" + echo "-n Specifies namespace for secret (default: default)" } while [ "$#" -gt 0 ]; do @@ -37,6 +31,12 @@ while [ "$#" -gt 0 ]; do -d) shift secretsdir=$1 ;; + -u) shift + username=$1 + ;; + -p) shift + password=$1 + ;; -n) shift namespace=$1 ;; @@ -44,6 +44,13 @@ while [ "$#" -gt 0 ]; do shift done +if [ -z "${password}" ]; +then + echo "Error: Missing required password!" + echo >&2 + usage + exit 1 +fi echo "Creating '${namespace}/${secretname}' secret." command -v ${k8scommand} >/dev/null 2>&1 || { echo >&2 "Aborting - '${k8scommand}' not found!"; exit 1; } @@ -53,10 +60,11 @@ if [[ $? -eq 0 ]] then echo "WARNING: Target secret '${namespace}/${secretname}' already exists! Delete it and re-run if secret update desired." echo "${errormessage}" + echo "use this command: '${k8scommand} delete secret ${secretname} -n ${namespace}'" exit 1 fi -errormessage=$(${k8scommand} create secret generic ${secretname} --from-file=${secretsdir} -n ${namespace} ) +errormessage=$(${k8scommand} create secret generic ${secretname} --from-literal=username=${username} --from-literal=password=${password} -n ${namespace}) if [[ $? -ne 0 ]] then echo "Error creating: Target secret '${namespace}/${secretname}'!" diff --git a/helm/managed/logging/loki-stack/grafana-logaccess-secret.yaml b/helm/managed/logging/loki-stack/grafana-logaccess-secret.yaml new file mode 100644 index 00000000000..2e1cec96eae --- /dev/null +++ b/helm/managed/logging/loki-stack/grafana-logaccess-secret.yaml @@ -0,0 +1,12 @@ +apiVersion: v1 +kind: Secret +metadata: + name: grafana-logaccess +type: Opaque +data: + #Base64 encoded username and password for Grafana + #can be encoded using the following command: + # echo -n 'admin' | base64 + username: YWRtaW4= + # echo -n 'whatevergrafanapassword' | base64 + password: d2hhdGV2ZXJncmFmYW5hcGFzc3dvcmQ= \ No newline at end of file diff --git a/helm/managed/logging/loki-stack/secrets-templates/password b/helm/managed/logging/loki-stack/secrets-templates/password deleted file mode 100644 index 6b3a9a39380..00000000000 --- a/helm/managed/logging/loki-stack/secrets-templates/password +++ /dev/null @@ -1 +0,0 @@ - \ No newline at end of file diff --git a/helm/managed/logging/loki-stack/secrets-templates/username b/helm/managed/logging/loki-stack/secrets-templates/username deleted file mode 100644 index f77b00407e0..00000000000 --- a/helm/managed/logging/loki-stack/secrets-templates/username +++ /dev/null @@ -1 +0,0 @@ -admin \ No newline at end of file From 05ba3120eb7c90da903c389f3159964df38d571a Mon Sep 17 00:00:00 2001 From: Gavin Halliday Date: Mon, 15 Jul 2024 14:08:36 +0100 Subject: [PATCH 15/29] HPCC-32090 Export members of RtlFieldInfo from eclrtl Signed-off-by: Gavin Halliday --- rtl/include/eclhelper.hpp | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/rtl/include/eclhelper.hpp b/rtl/include/eclhelper.hpp index ef442890c8c..cc0c0c843ba 100644 --- a/rtl/include/eclhelper.hpp +++ b/rtl/include/eclhelper.hpp @@ -46,6 +46,17 @@ typedef unsigned short UChar; #endif #include "rtlconst.hpp" + +#if !defined(ECLRTL_LOCAL) +#ifdef ECLRTL_EXPORTS +#define ECLRTL_API DECL_EXPORT +#else +#define ECLRTL_API DECL_IMPORT +#endif +#else +#define ECLRTL_API +#endif + //Should be incremented whenever the virtuals in the context or a helper are changed, so //that a work unit can't be rerun. Try as hard as possible to retain compatibility. #define ACTIVITY_INTERFACE_VERSION 654 @@ -484,7 +495,7 @@ inline byte getVirtualInitializer(const void * initializer) { return (byte)(mems typedef IThorDiskCallback IVirtualFieldCallback; //Core struct used for representing meta for a field. Effectively used as an interface. -struct RtlFieldInfo +struct ECLRTL_API RtlFieldInfo { constexpr inline RtlFieldInfo(const char * _name, const char * _xpath, const RtlTypeInfo * _type, unsigned _flags = 0, const char *_initializer = NULL) : name(_name), xpath(_xpath), type(_type), initializer(_initializer), flags(_type->fieldType | _flags) {} From acb21b9a39d24c5406b4ede7657c8750c8c29b03 Mon Sep 17 00:00:00 2001 From: Michael Gardner Date: Mon, 15 Jul 2024 11:23:58 -0400 Subject: [PATCH 16/29] HPCC-32247 Add ECLIDE Deployment to JFrog internal repositories Signed-off-by: Michael Gardner --- .github/workflows/build-assets.yml | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/.github/workflows/build-assets.yml b/.github/workflows/build-assets.yml index 04ca89aaeec..58f9ffcea43 100644 --- a/.github/workflows/build-assets.yml +++ b/.github/workflows/build-assets.yml @@ -691,6 +691,16 @@ jobs: generateReleaseNotes: false prerelease: ${{ contains(github.ref, '-rc') }} artifacts: "./ECLIDE/build/*.exe" + + - name: Upload Assets to Jfrog (windows) + if: ${{ github.repository_owner == 'hpcc-systems' }} + shell: bash + run: | + cd ./ECLIDE/build + packages=($(ls -1 hpccsystems-*.exe )) + for _package in ${packages[@]}; do + curl -u${{ secrets.JFROG_USERNAME }}:${{ secrets.JFROG_PASSWORD }} "https://${{ secrets.JFROG_REGISTRY }}/hpccpl-windows-local/LN/windows/x86_64/${_package}" -T ${_package} + done - name: Upload error logs if: ${{ failure() || cancelled() }} From a55eb1cfb923ac8bf89313a3b93966323b945405 Mon Sep 17 00:00:00 2001 From: Rodrigo Pastrana Date: Thu, 13 Jun 2024 00:14:06 -0400 Subject: [PATCH 17/29] HPCC-32041 WsStore support invalid IPT paths - Encodes all user provided IPT path components - Decodes all IPT paths provided to user Signed-off-by: Rodrigo Pastrana --- .../ws_store/espstorelib/daliKVStore.cpp | 176 +++++++++++++----- esp/services/ws_store/ws_storeService.cpp | 35 ++-- 2 files changed, 148 insertions(+), 63 deletions(-) diff --git a/esp/services/ws_store/espstorelib/daliKVStore.cpp b/esp/services/ws_store/espstorelib/daliKVStore.cpp index b3f1a40419d..9e7774b578c 100644 --- a/esp/services/ws_store/espstorelib/daliKVStore.cpp +++ b/esp/services/ws_store/espstorelib/daliKVStore.cpp @@ -17,18 +17,29 @@ #include "daliKVStore.hpp" +void getEncodedLowerCaseUserName(StringBuffer & out, ISecUser * username) +{ + StringBuffer userlowercased; + userlowercased.set(username->getName()).toLowerCase(); + + encodePTreeName(out, userlowercased.str()); +} + bool CDALIKVStore::createStore(const char * apptype, const char * storename, const char * description, ISecUser * owner, unsigned int maxvalsize=DALI_KVSTORE_MAXVALSIZE_DEFAULT) { if (!storename || !*storename) throw MakeStringException(-1, "DALI Keystore createStore(): Store name not provided"); + StringBuffer encodedStoreName; + encodePTreeName(encodedStoreName, storename); + ensureAttachedToDali(); //throws if in offline mode Owned conn = querySDS().connect(DALI_KVSTORE_PATH, myProcessSession(), RTM_LOCK_READ, SDS_LOCK_TIMEOUT_KVSTORE); if (!conn) throw MakeStringException(-1, "Unable to connect to DALI KeyValue store root: '%s'", DALI_KVSTORE_PATH); - VStringBuffer xpath("Store[%s='%s'][1]", DALI_KVSTORE_NAME_ATT, storename); + VStringBuffer xpath("Store[%s='%s'][1]", DALI_KVSTORE_NAME_ATT, encodedStoreName.str()); { Owned root = conn->getRoot(); if (root->hasProp(xpath.str())) @@ -53,7 +64,7 @@ bool CDALIKVStore::createStore(const char * apptype, const char * storename, con } Owned apptree = createPTree(); - apptree->setProp(DALI_KVSTORE_NAME_ATT, storename); + apptree->setProp(DALI_KVSTORE_NAME_ATT, encodedStoreName.str()); CDateTime dt; dt.setNow(); StringBuffer str; @@ -89,15 +100,21 @@ bool CDALIKVStore::set(const char * storename, const char * thenamespace, const if (isEmptyString(storename)) throw MakeStringException(-1, "DALI Keystore set(): Store name not provided"); + StringBuffer encodedStoreName; + encodePTreeName(encodedStoreName, storename); + if (!global && (!owner || isEmptyString(owner->getName()))) throw MakeStringException(-1, "DALI Keystore set(): Attempting to set non-global entry but owner name not provided"); if (isEmptyString(thenamespace)) throw MakeStringException(-1, "DALI Keystore set(): namespace not provided"); + StringBuffer encodedNameSpace; + encodePTreeName(encodedNameSpace, thenamespace); + ensureAttachedToDali(); //throws if in offline mode - VStringBuffer xpath("%s/Store[%s='%s'][1]", DALI_KVSTORE_PATH, DALI_KVSTORE_NAME_ATT, storename); + VStringBuffer xpath("%s/Store[%s='%s'][1]", DALI_KVSTORE_PATH, DALI_KVSTORE_NAME_ATT, encodedStoreName.str()); Owned conn = querySDS().connect(xpath.str(), myProcessSession(), RTM_LOCK_WRITE, SDS_LOCK_TIMEOUT_KVSTORE); if (!conn) throw MakeStringException(-1, "DALI Keystore set(): Unable to connect to DALI KeyValue store path '%s'", xpath.str()); //rodrigo, not sure if this is too much info @@ -111,7 +128,7 @@ bool CDALIKVStore::set(const char * storename, const char * thenamespace, const if (global) xpath.set(DALI_KVSTORE_GLOBAL); else - xpath.set(owner->getName()).toLowerCase(); + getEncodedLowerCaseUserName(xpath.clear(), owner); Owned ownertree = storetree->getPropTree(xpath.str()); if (!ownertree) @@ -121,18 +138,20 @@ bool CDALIKVStore::set(const char * storename, const char * thenamespace, const dt.setNow(); StringBuffer str; - Owned nstree = ownertree->getPropTree(thenamespace); + Owned nstree = ownertree->getPropTree(encodedNameSpace.str()); if (!nstree) { - nstree.setown(createPTree(thenamespace)); + nstree.setown(createPTree(encodedNameSpace.str())); nstree->setProp(DALI_KVSTORE_CREATEDTIME_ATT,dt.getString(str).str()); } - Owned valuetree = nstree->getPropTree(key); + StringBuffer encodedKey; + encodePTreeName(encodedKey, key); + Owned valuetree = nstree->getPropTree(encodedKey.str()); if (!valuetree) { - nstree->setProp(key, value); - valuetree.setown(nstree->getPropTree(key)); + nstree->setProp(encodedKey.str(), value); + valuetree.setown(nstree->getPropTree(encodedKey.str())); valuetree->setProp(DALI_KVSTORE_CREATEDTIME_ATT,dt.getString(str).str()); valuetree->setProp(DALI_KVSTORE_CREATEDBY_ATT, owner ? owner->getName(): ""); } @@ -143,7 +162,7 @@ bool CDALIKVStore::set(const char * storename, const char * thenamespace, const valuetree->setProp(".", value); } - ownertree->setPropTree(thenamespace, LINK(nstree)); + ownertree->setPropTree(encodedNameSpace.str(), LINK(nstree)); storetree->setPropTree(xpath.str(), LINK(ownertree)); conn->commit(); @@ -156,26 +175,31 @@ IPropertyTree * CDALIKVStore::getAllKeyProperties(const char * storename, const if (isEmptyString(storename)) throw MakeStringException(-1, "DALI Keystore fetchKeyProperties(): Store name not provided"); + StringBuffer encodedStoreName; + encodePTreeName(encodedStoreName, storename); + if (!global && (!username || isEmptyString(username->getName()))) throw MakeStringException(-1, "DALI Keystore fetchKeyProperties(): Attempting to set non-global entry but owner name not provided"); if (isEmptyString(ns)) throw MakeStringException(-1, "DALI Keystore fetchKeyProperties(): namespace not provided"); + StringBuffer encodedNS; + encodePTreeName(encodedNS, ns); + ensureAttachedToDali(); //throws if in offline mode - VStringBuffer xpath("%s/Store[%s='%s'][1]/", DALI_KVSTORE_PATH, DALI_KVSTORE_NAME_ATT, storename); + VStringBuffer xpath("%s/Store[%s='%s'][1]/", DALI_KVSTORE_PATH, DALI_KVSTORE_NAME_ATT, encodedStoreName.str()); if (global) xpath.append(DALI_KVSTORE_GLOBAL); else - { - StringBuffer userlowercased; - userlowercased.set(username->getName()).toLowerCase(); - xpath.append(userlowercased); - } + getEncodedLowerCaseUserName(xpath, username); + + StringBuffer encodedKey; + encodePTreeName(encodedKey, key); - xpath.appendf("/%s/%s", ns, key); + xpath.appendf("/%s/%s", encodedNS.str(), encodedKey.str()); Owned conn = querySDS().connect(xpath.str(), myProcessSession(), RTM_LOCK_READ, SDS_LOCK_TIMEOUT_KVSTORE); if (!conn) @@ -189,26 +213,31 @@ bool CDALIKVStore::fetchKeyProperty(StringBuffer & propval , const char * storen if (isEmptyString(storename)) throw MakeStringException(-1, "DALI Keystore fetchKeyProperty(): Store name not provided"); + StringBuffer encodedStoreName; + encodePTreeName(encodedStoreName, storename); + if (!global && (!username || isEmptyString(username->getName()))) throw MakeStringException(-1, "DALI Keystore fetchKeyProperty(): Attempting to set non-global entry but owner name not provided"); if (isEmptyString(ns)) throw MakeStringException(-1, "DALI Keystore fetchKeyProperty(): namespace not provided"); + StringBuffer encodedNamespace; + encodePTreeName(encodedNamespace, ns); + ensureAttachedToDali(); //throws if in offline mode - VStringBuffer xpath("%s/Store[%s='%s'][1]/", DALI_KVSTORE_PATH, DALI_KVSTORE_NAME_ATT, storename); + VStringBuffer xpath("%s/Store[%s='%s'][1]/", DALI_KVSTORE_PATH, DALI_KVSTORE_NAME_ATT, encodedStoreName.str()); if (global) xpath.append(DALI_KVSTORE_GLOBAL); else - { - StringBuffer userlowercased; - userlowercased.set(username->getName()).toLowerCase(); - xpath.append(userlowercased); - } + getEncodedLowerCaseUserName(xpath, username); + + StringBuffer encodedKey; + encodePTreeName(encodedKey, key); - xpath.appendf("/%s/%s", ns, key); + xpath.appendf("/%s/%s", encodedNamespace.str(), encodedKey.str()); Owned conn = querySDS().connect(xpath.str(), myProcessSession(), RTM_LOCK_READ, SDS_LOCK_TIMEOUT_KVSTORE); if (!conn) @@ -216,7 +245,10 @@ bool CDALIKVStore::fetchKeyProperty(StringBuffer & propval , const char * storen Owned keytree = conn->getRoot(); - keytree->getProp(property,propval.clear()); + StringBuffer encodedPropName; + encodePTreeName(encodedPropName, property); + + keytree->getProp(encodedPropName.str(),propval.clear()); return true; } @@ -225,18 +257,27 @@ bool CDALIKVStore::deletekey(const char * storename, const char * thenamespace, if (!storename || !*storename) throw MakeStringException(-1, "DALI Keystore deletekey(): Store name not provided"); + StringBuffer encodedStoreName; + encodePTreeName(encodedStoreName, storename); + if (!thenamespace || !*thenamespace) throw MakeStringException(-1, "DALI KV Store deletekey(): target namespace not provided!"); + StringBuffer encodedNS; + encodePTreeName(encodedNS, thenamespace); + if (!key || !*key) throw MakeStringException(-1, "DALI KV Store deletekey(): target key not provided!"); + StringBuffer encodedKey; + encodePTreeName(encodedKey, key); + if (!global && (!user || isEmptyString(user->getName()))) throw MakeStringException(-1, "DALI Keystore set(): Attempting to set non-global entry but user not provided"); ensureAttachedToDali(); //throws if in offline mode - VStringBuffer xpath("%s/Store[%s='%s'][1]", DALI_KVSTORE_PATH, DALI_KVSTORE_NAME_ATT, storename); + VStringBuffer xpath("%s/Store[%s='%s'][1]", DALI_KVSTORE_PATH, DALI_KVSTORE_NAME_ATT, encodedStoreName.str()); Owned conn = querySDS().connect(xpath.str(), myProcessSession(), RTM_LOCK_WRITE, SDS_LOCK_TIMEOUT_KVSTORE); if (!conn) throw MakeStringException(-1, "DALI Keystore deletekey(): Unable to connect to DALI KeyValue store root path '%s'", DALI_KVSTORE_PATH); @@ -246,9 +287,9 @@ bool CDALIKVStore::deletekey(const char * storename, const char * thenamespace, if (global) xpath.set(DALI_KVSTORE_GLOBAL); else - xpath.set(user->getName()).toLowerCase(); + getEncodedLowerCaseUserName(xpath.clear(), user); - xpath.appendf("/%s/%s", thenamespace,key); + xpath.appendf("/%s/%s", encodedNS.str(), encodedKey.str()); if(!storetree->hasProp(xpath.str())) throw MakeStringException(-1, "DALI KV Store deletekey(): Could not find '%s/%s/%s' for user '%s'", storename, thenamespace, key, global ? "GLOBAL USER" : user->getName()); @@ -264,15 +305,21 @@ bool CDALIKVStore::deleteNamespace(const char * storename, const char * thenames if (!storename || !*storename) throw MakeStringException(-1, "DALI Keystore deletekey(): Store name not provided"); + StringBuffer encodedStoreName; + encodePTreeName(encodedStoreName, storename); + if (!global && (!user || isEmptyString(user->getName()))) throw MakeStringException(-1, "DALI Keystore deleteNamespace(): Attempting to fetch non-global keys but user not provided"); if (isEmptyString(thenamespace)) throw MakeStringException(-1, "DALI KV Store deleteNamespace(): target namespace not provided!"); + StringBuffer encodedNS; + encodePTreeName(encodedNS, thenamespace); + ensureAttachedToDali(); //throws if in offline mode - VStringBuffer xpath("%s/Store[%s='%s']", DALI_KVSTORE_PATH, DALI_KVSTORE_NAME_ATT, storename); + VStringBuffer xpath("%s/Store[%s='%s']", DALI_KVSTORE_PATH, DALI_KVSTORE_NAME_ATT, encodedStoreName.str()); Owned conn = querySDS().connect(xpath.str(), myProcessSession(), RTM_LOCK_WRITE, SDS_LOCK_TIMEOUT_KVSTORE); if (!conn) throw MakeStringException(-1, "DALI Keystore deleteNamespace(): Unable to connect to DALI KeyValue store path '%s'", xpath.str()); @@ -282,9 +329,9 @@ bool CDALIKVStore::deleteNamespace(const char * storename, const char * thenames if (global) xpath.set(DALI_KVSTORE_GLOBAL); else - xpath.set(user->getName()).toLowerCase(); + getEncodedLowerCaseUserName(xpath.clear(), user); - xpath.appendf("/%s", thenamespace); //we're interested in the children of the namespace + xpath.appendf("/%s", encodedNS.str()); //we're interested in the children of the namespace if(!storetree->hasProp(xpath.str())) throw MakeStringException(-1, "DALI KV Store deleteNamespace(): invalid namespace detected '%s/%s' for user '%s'", storename, thenamespace, global ? "GLOBAL USER" : user->getName()); @@ -300,12 +347,15 @@ bool CDALIKVStore::fetchAllNamespaces(StringArray & namespaces, const char * sto if (!storename || !*storename) throw MakeStringException(-1, "DALI Keystore fetchAllNamespaces(): Store name not provided"); + StringBuffer encodedStoreName; + encodePTreeName(encodedStoreName, storename); + if (!global && (!user || isEmptyString(user->getName()))) throw MakeStringException(-1, "DALI Keystore fetchAllNamespaces(): Attempting to fetch non-global keys but requester name not provided"); ensureAttachedToDali(); //throws if in offline mode - VStringBuffer xpath("%s/Store[%s='%s']", DALI_KVSTORE_PATH, DALI_KVSTORE_NAME_ATT, storename); + VStringBuffer xpath("%s/Store[%s='%s']", DALI_KVSTORE_PATH, DALI_KVSTORE_NAME_ATT, encodedStoreName.str()); Owned conn = querySDS().connect(xpath.str(), myProcessSession(), RTM_LOCK_READ, SDS_LOCK_TIMEOUT_KVSTORE); if (!conn) throw MakeStringException(-1, "DALI Keystore fetchAllNamespaces: Unable to connect to DALI KeyValue store path '%s'", xpath.str()); @@ -313,16 +363,21 @@ bool CDALIKVStore::fetchAllNamespaces(StringArray & namespaces, const char * sto Owned storetree = conn->getRoot(); if (global) - xpath.setf("%s/*", DALI_KVSTORE_GLOBAL); //we're interested in the children of the namespace + xpath.set(DALI_KVSTORE_GLOBAL); else - xpath.setf("%s/*", user->getName()).toLowerCase(); //we're interested in the children of the namespace + getEncodedLowerCaseUserName(xpath.clear(), user); + + xpath.append("/*"); //we're interested in the children of the namespace StringBuffer name; Owned iter = storetree->getElements(xpath.str()); ForEach(*iter) { iter->query().getName(name.clear()); - namespaces.append(name.str()); + StringBuffer decodedName; + decodePtreeName(decodedName, name.str()); + + namespaces.append(decodedName.str()); } return true; @@ -333,15 +388,21 @@ bool CDALIKVStore::fetchKeySet(StringArray & keyset, const char * storename, con if (!storename || !*storename) throw MakeStringException(-1, "DALI Keystore fetchKeySet(): Store name not provided"); + StringBuffer encodedStoreName; + encodePTreeName(encodedStoreName, storename); + if (!global && (!user || isEmptyString(user->getName()))) throw MakeStringException(-1, "DALI Keystore fetchKeySet(): Attempting to fetch non-global keys but requester name not provided"); if (isEmptyString(ns)) throw MakeStringException(-1, "DALI Keystore fetchKeySet: Namespace not provided!"); + StringBuffer encodedNS; + encodePTreeName(encodedNS, ns); + ensureAttachedToDali(); //throws if in offline mode - VStringBuffer xpath("%s/Store[%s='%s']", DALI_KVSTORE_PATH, DALI_KVSTORE_NAME_ATT, storename); + VStringBuffer xpath("%s/Store[%s='%s']", DALI_KVSTORE_PATH, DALI_KVSTORE_NAME_ATT, encodedStoreName.str()); Owned conn = querySDS().connect(xpath.str(), myProcessSession(), RTM_LOCK_READ, SDS_LOCK_TIMEOUT_KVSTORE); if (!conn) throw MakeStringException(-1, "DALI Keystore fetchKeySet: Unable to connect to DALI KeyValue store path '%s'", DALI_KVSTORE_PATH); @@ -351,9 +412,10 @@ bool CDALIKVStore::fetchKeySet(StringArray & keyset, const char * storename, con if (global) xpath.set(DALI_KVSTORE_GLOBAL); else - xpath.set(user->getName()).toLowerCase(); + getEncodedLowerCaseUserName(xpath.clear(), user); + + xpath.appendf("/%s/*", encodedNS.str()); //we're interested in the children of the namespace - xpath.appendf("/%s/*", ns); //we're interested in the children of the namespace if(!storetree->hasProp(xpath.str())) throw MakeStringException(-1, "DALI Keystore fetchKeySet: invalid namespace '%s' detected!", ns); @@ -362,7 +424,9 @@ bool CDALIKVStore::fetchKeySet(StringArray & keyset, const char * storename, con ForEach(*iter) { iter->query().getName(name.clear()); - keyset.append(name.str()); + StringBuffer decodedName; + decodePtreeName(decodedName, name.str()); + keyset.append(decodedName.str()); } return true; @@ -373,15 +437,21 @@ bool CDALIKVStore::fetch(const char * storename, const char * ns, const char * k if (!storename || !*storename) throw MakeStringException(-1, "DALI Keystore fetch(): Store name not provided"); - if (!global && (!user || isEmptyString(user->getName()))) + StringBuffer encodedStoreName; + encodePTreeName(encodedStoreName, storename); + + if (!global && (!user || isEmptyString(user->getName()))) throw MakeStringException(-1, "DALI Keystore fetch(): Attempting to fetch non-global entry but requester name not provided"); - if (isEmptyString(ns)) - throw MakeStringException(-1, "DALI Keystore fetch: key not provided!"); + if (isEmptyString(ns)) + throw MakeStringException(-1, "DALI Keystore fetch: key not provided!"); + + StringBuffer encodedNS; + encodePTreeName(encodedNS, ns); ensureAttachedToDali(); //throws if in offline mode - VStringBuffer xpath("%s/Store[%s='%s']", DALI_KVSTORE_PATH, DALI_KVSTORE_NAME_ATT, storename); + VStringBuffer xpath("%s/Store[%s='%s']", DALI_KVSTORE_PATH, DALI_KVSTORE_NAME_ATT, encodedStoreName.str()); Owned conn = querySDS().connect(xpath.str(), myProcessSession(), RTM_LOCK_READ, SDS_LOCK_TIMEOUT_KVSTORE); if (!conn) throw MakeStringException(-1, "DALI Keystore fetch: Unable to connect to DALI KeyValue store path '%s'", xpath.str()); @@ -391,15 +461,17 @@ bool CDALIKVStore::fetch(const char * storename, const char * ns, const char * k if (global) xpath.set(DALI_KVSTORE_GLOBAL); else - xpath.set(user->getName()).toLowerCase(); + getEncodedLowerCaseUserName(xpath.clear(), user); - xpath.appendf("/%s", ns); + xpath.appendf("/%s", encodedNS.str()); if(!storetree->hasProp(xpath.str())) throw MakeStringException(-1, "DALI Keystore fetch: invalid namespace '%s' detected!", ns); if (key && *key) { - xpath.appendf("/%s", key); + StringBuffer encodedKey; + encodePTreeName(encodedKey, key); + xpath.appendf("/%s", encodedKey.str()); if(!storetree->hasProp(xpath.str())) { throw makeStringExceptionV(ECLWATCH_INVALID_QUERY_KEY, "DALI Keystore fetch: invalid key '%s' detected!", key); @@ -422,15 +494,21 @@ IPropertyTree * CDALIKVStore::getAllPairs(const char * storename, const char * n if (!storename || !*storename) throw MakeStringException(-1, "DALI Keystore fetchAll(): Store name not provided"); + StringBuffer encodedStoreName; + encodePTreeName(encodedStoreName, storename); + if (!global && (!user || isEmptyString(user->getName()))) throw MakeStringException(-1, "DALI Keystore fetchAll(): Attempting to fetch non-global entries but requester name not provided"); if (isEmptyString(ns)) throw MakeStringException(-1, "DALI Keystore fetchAll: Namespace not provided!"); + StringBuffer encodedNS; + encodePTreeName(encodedNS, ns); + ensureAttachedToDali(); //throws if in offline mode - VStringBuffer xpath("%s/Store[%s='%s']", DALI_KVSTORE_PATH, DALI_KVSTORE_NAME_ATT, storename); + VStringBuffer xpath("%s/Store[%s='%s']", DALI_KVSTORE_PATH, DALI_KVSTORE_NAME_ATT, encodedStoreName.str()); Owned conn = querySDS().connect(xpath.str(), myProcessSession(), RTM_LOCK_READ, SDS_LOCK_TIMEOUT_KVSTORE); if (!conn) throw MakeStringException(-1, "DALI Keystore fetchAll: Unable to connect to DALI KeyValue store path '%s'", xpath.str()); @@ -440,9 +518,9 @@ IPropertyTree * CDALIKVStore::getAllPairs(const char * storename, const char * n if (global) xpath.set(DALI_KVSTORE_GLOBAL); else - xpath.set(user->getName()).toLowerCase(); + getEncodedLowerCaseUserName(xpath.clear(), user); - xpath.appendf("/%s", ns); + xpath.appendf("/%s", encodedNS.str()); if(!storetree->hasProp(xpath.str())) throw MakeStringException(-1, "DALI Keystore fetchAll: invalid namespace '%s' detected!", ns); diff --git a/esp/services/ws_store/ws_storeService.cpp b/esp/services/ws_store/ws_storeService.cpp index 75ddde7d6e5..c9fd66c20ed 100644 --- a/esp/services/ws_store/ws_storeService.cpp +++ b/esp/services/ws_store/ws_storeService.cpp @@ -271,6 +271,7 @@ bool CwsstoreEx::onListKeys(IEspContext &context, IEspListKeysRequest &req, IEsp StringArray keys; m_storeProvider->fetchKeySet(keys, storename, ns, secuser.get(), !req.getUserSpecific()); + resp.setKeySet(keys); resp.setNamespace(ns); resp.setStoreName(storename); @@ -385,22 +386,28 @@ bool CwsstoreEx::onFetchAll(IEspContext &context, IEspFetchAllRequest &req, IEsp storename = m_defaultStore.get(); } - Owned nstree = m_storeProvider->getAllPairs(storename, ns, secuser.get(), !req.getUserSpecific()); - IArrayOf pairs; - - Owned iter = nstree->getElements("*"); - ForEach(*iter) { - StringBuffer name; - StringBuffer value; - iter->query().getName(name); - nstree->getProp(name.str(), value); - - Owned kvpair = createKVPair("",""); - kvpair->setKey(name.str()); - kvpair->setValue(value.str()); - pairs.append(*kvpair.getClear()); + Owned nstree = m_storeProvider->getAllPairs(storename, ns, secuser.get(), !req.getUserSpecific()); + + Owned iter = nstree->getElements("*"); + ForEach(*iter) + { + StringBuffer name; + StringBuffer value; + iter->query().getName(name); + nstree->getProp(name.str(), value); + + Owned kvpair = createKVPair("",""); + + //it's possible this has been encoded, so decode it + StringBuffer decoded; + decodePtreeName(decoded, name.str()); + + kvpair->setKey(decoded.str()); + kvpair->setValue(value.str()); + pairs.append(*kvpair.getClear()); + } } resp.setPairs(pairs); From b5d2a39fd1f01f67a61eba457f29f477eeee0fe7 Mon Sep 17 00:00:00 2001 From: Rodrigo Pastrana Date: Wed, 3 Jul 2024 16:59:27 -0400 Subject: [PATCH 18/29] HPCC-32208 Properly report Grafana LogAccess Config problem - Tracks and reports invalid config issue - Routes config err messages to operators - Cleans up error message wording Signed-off-by: Rodrigo Pastrana --- .../Grafana/CurlClient/GrafanaCurlClient.cpp | 31 ++++++++++++++++--- 1 file changed, 26 insertions(+), 5 deletions(-) diff --git a/system/logaccess/Grafana/CurlClient/GrafanaCurlClient.cpp b/system/logaccess/Grafana/CurlClient/GrafanaCurlClient.cpp index 5ada0237838..4cbd356f096 100644 --- a/system/logaccess/Grafana/CurlClient/GrafanaCurlClient.cpp +++ b/system/logaccess/Grafana/CurlClient/GrafanaCurlClient.cpp @@ -547,6 +547,9 @@ const char * sortByDirection(SortByDirection direction) */ bool GrafanaLogAccessCurlClient::fetchLog(LogQueryResultDetails & resultDetails, const LogAccessConditions & options, StringBuffer & returnbuf, LogAccessLogFormat format) { + if (m_dataSourcesAPIURI.isEmpty()) + throw makeStringExceptionV(-1, "%s: Cannot query because Grafana datasource was not established, check logaccess configuration!", COMPONENT_NAME); + try { resultDetails.totalReceived = 0; @@ -729,12 +732,30 @@ GrafanaLogAccessCurlClient::GrafanaLogAccessCurlClient(IPropertyTree & logAccess m_grafanaPassword.set(logAccessPluginConfig.queryProp("connection/@password")); } - //this is very important, without this, we can't target the correct datasource - fetchDatasourceByName(m_targetDataSource.name.get()); + try + { + //this is very important, without this, we can't target the correct datasource + fetchDatasourceByName(m_targetDataSource.name.get()); + } + catch(IException * e) + { + StringBuffer description; + OERRLOG("%s: Exception fetching Loki/Grafana datasource!!: (%d) - %s", COMPONENT_NAME, e->errorCode(), e->errorMessage(description).str()); + e->Release(); + } - std::string availableLabels; - fetchLabels(availableLabels); - DBGLOG("%s: Available labels on target loki/grafana: %s", COMPONENT_NAME, availableLabels.c_str()); + try + { + std::string availableLabels; + fetchLabels(availableLabels); + DBGLOG("%s: Available labels on target loki/grafana: %s", COMPONENT_NAME, availableLabels.c_str()); + } + catch(IException * e) + { + StringBuffer description; + OERRLOG("%s: Exception fetching available labels: (%d) - %s", COMPONENT_NAME, e->errorCode(), e->errorMessage(description).str()); + e->Release(); + } m_expectedLogFormat = defaultExpectedLogFormat; if (logAccessPluginConfig.hasProp("logFormat/@type")) From 8d2b0677d4c14694baa9253e249ecbbdfdda0fc4 Mon Sep 17 00:00:00 2001 From: Rodrigo Pastrana Date: Fri, 28 Jun 2024 17:59:51 -0400 Subject: [PATCH 19/29] HPCC-32052 Improve WsStore fetch miss message - Changes log message to clarify simple fetch miss - Removes dependancy on exception on misses - Removes dbg logging of misses - Fixes invalid response type in CDALIKVStore::fetch Signed-off-by: Rodrigo Pastrana --- .../ws_store/espstorelib/daliKVStore.cpp | 8 ++------ esp/services/ws_store/ws_storeService.cpp | 20 +++---------------- 2 files changed, 5 insertions(+), 23 deletions(-) diff --git a/esp/services/ws_store/espstorelib/daliKVStore.cpp b/esp/services/ws_store/espstorelib/daliKVStore.cpp index b3f1a40419d..fbda761a692 100644 --- a/esp/services/ws_store/espstorelib/daliKVStore.cpp +++ b/esp/services/ws_store/espstorelib/daliKVStore.cpp @@ -401,15 +401,11 @@ bool CDALIKVStore::fetch(const char * storename, const char * ns, const char * k { xpath.appendf("/%s", key); if(!storetree->hasProp(xpath.str())) - { - throw makeStringExceptionV(ECLWATCH_INVALID_QUERY_KEY, "DALI Keystore fetch: invalid key '%s' detected!", key); - } + return false; else - { value.set(storetree->queryProp(xpath.str())); - } - return value.str(); + return true; } else throw makeStringException(-1, "DALI Keystore fetch: Key not provided!"); diff --git a/esp/services/ws_store/ws_storeService.cpp b/esp/services/ws_store/ws_storeService.cpp index 75ddde7d6e5..e63cb3db7b8 100644 --- a/esp/services/ws_store/ws_storeService.cpp +++ b/esp/services/ws_store/ws_storeService.cpp @@ -312,25 +312,11 @@ bool CwsstoreEx::onFetch(IEspContext &context, IEspFetchRequest &req, IEspFetchR storename = m_defaultStore.get(); } - try - { - m_storeProvider->fetch(storename, req.getNamespace(), req.getKey(), value, secuser.get(), !req.getUserSpecific()); + bool success = m_storeProvider->fetch(storename, req.getNamespace(), req.getKey(), value, secuser.get(), !req.getUserSpecific()); + if (success) resp.setValue(value.str()); - } - catch(IException * e) - { - if (e->errorCode() == ECLWATCH_INVALID_QUERY_KEY) - { - StringBuffer msg; - LOG(MCuserInfo, "WsStore: %s", e->errorMessage(msg).str()); - e->Release(); - return false; - } - else - throw e; - } - return true; + return success; } bool CwsstoreEx::onFetchKeyMetadata(IEspContext &context, IEspFetchKeyMDRequest &req, IEspFetchKeyMDResponse &resp) From a31f7a96460537b1cd5fe0d7296c50108b68aa40 Mon Sep 17 00:00:00 2001 From: Shamser Ahmed Date: Thu, 11 Jul 2024 11:31:40 +0100 Subject: [PATCH 20/29] HPCC-32218 Implement StNumParallelExecute stat for activities Signed-off-by: Shamser Ahmed --- system/jlib/jstatcodes.h | 1 + system/jlib/jstats.cpp | 1 + thorlcr/graph/thgraphslave.hpp | 8 +++++++- thorlcr/thorutil/thormisc.cpp | 2 +- 4 files changed, 10 insertions(+), 2 deletions(-) diff --git a/system/jlib/jstatcodes.h b/system/jlib/jstatcodes.h index aa133ad5ba2..9e034906601 100644 --- a/system/jlib/jstatcodes.h +++ b/system/jlib/jstatcodes.h @@ -311,6 +311,7 @@ enum StatisticKind StNumMatchRightRowsMax, StNumMatchCandidates, StNumMatchCandidatesMax, + StNumParallelExecute, StMax, //For any quantity there is potentially the following variants. diff --git a/system/jlib/jstats.cpp b/system/jlib/jstats.cpp index 92f02d1bdec..ac6352ddc80 100644 --- a/system/jlib/jstats.cpp +++ b/system/jlib/jstats.cpp @@ -983,6 +983,7 @@ static const constexpr StatisticMeta statsMetaData[StMax] = { { NUMSTAT(MatchRightRowsMax), "The largest number of right rows in a join group" }, { NUMSTAT(MatchCandidates), "The number of candidate combinations of left and right rows forming join groups" }, { NUMSTAT(MatchCandidatesMax), "The largest number of candidate combinations of left and right rows in a single group" }, + { NUMSTAT(ParallelExecute), "The number of parallel execution paths for this activity" }, }; static MapStringTo statisticNameMap(true); diff --git a/thorlcr/graph/thgraphslave.hpp b/thorlcr/graph/thgraphslave.hpp index f6b73c912c1..3d46ecf29d0 100644 --- a/thorlcr/graph/thgraphslave.hpp +++ b/thorlcr/graph/thgraphslave.hpp @@ -471,6 +471,7 @@ class graphslave_decl CThorStrandProcessor : public CInterfaceOf strands; @@ -496,7 +497,12 @@ class graphslave_decl CThorStrandedActivity : public CSlaveActivity virtual CThorStrandProcessor *createStrandSourceProcessor(bool inputOrdered) = 0; inline unsigned numStrands() const { return strands.ordinality(); } - +// CSlaveActivity + virtual void gatherActiveStats(CRuntimeStatisticCollection &activeStats) const override + { + PARENT::gatherActiveStats(activeStats); + activeStats.addStatistic(StNumParallelExecute, numStrands()); + } // IThorDataLink virtual IStrandJunction *getOutputStreams(CActivityBase &_ctx, unsigned idx, PointerArrayOf &streams, const CThorStrandOptions * consumerOptions, bool consumerOrdered, IOrderedCallbackCollection * orderedCallbacks) override; virtual unsigned __int64 queryTotalCycles() const override; diff --git a/thorlcr/thorutil/thormisc.cpp b/thorlcr/thorutil/thormisc.cpp index 09b3fc233af..973c8c9af18 100644 --- a/thorlcr/thorutil/thormisc.cpp +++ b/thorlcr/thorutil/thormisc.cpp @@ -75,7 +75,7 @@ static Owned ClusterMPAllocator; // stat. mappings shared between master and slave activities const StatisticsMapping spillStatistics({StTimeSpillElapsed, StTimeSortElapsed, StNumSpills, StSizeSpillFile, StSizePeakTempDisk}); const StatisticsMapping soapcallStatistics({StTimeSoapcall}); -const StatisticsMapping basicActivityStatistics({StTimeTotalExecute, StTimeLocalExecute, StTimeBlocked}); +const StatisticsMapping basicActivityStatistics({StTimeTotalExecute, StTimeLocalExecute, StTimeBlocked, StNumParallelExecute}); const StatisticsMapping groupActivityStatistics({StNumGroups, StNumGroupMax}, basicActivityStatistics); const StatisticsMapping hashJoinActivityStatistics({StNumLeftRows, StNumRightRows}, basicActivityStatistics); const StatisticsMapping indexReadFileStatistics({}, diskReadRemoteStatistics, jhtreeCacheStatistics); From 68771edc9e74a7bd5e2ecc368f75f5b38f586c09 Mon Sep 17 00:00:00 2001 From: Richard Chapman Date: Wed, 3 Jul 2024 09:56:22 +0100 Subject: [PATCH 21/29] HPCC-32201 Fix compilation error on Clang/Mac Signed-off-by: Richard Chapman --- system/logaccess/Grafana/CurlClient/GrafanaCurlClient.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/system/logaccess/Grafana/CurlClient/GrafanaCurlClient.cpp b/system/logaccess/Grafana/CurlClient/GrafanaCurlClient.cpp index 5ada0237838..355c047e37a 100644 --- a/system/logaccess/Grafana/CurlClient/GrafanaCurlClient.cpp +++ b/system/logaccess/Grafana/CurlClient/GrafanaCurlClient.cpp @@ -651,7 +651,7 @@ bool GrafanaLogAccessCurlClient::fetchLog(LogQueryResultDetails & resultDetails, encodeURL(fullQuery, logLineParser.str()); fullQuery.appendf("&start=%s000000000", std::to_string(trange.getStartt().getSimple()).c_str()); - if (trange.getEndt().isNull() != -1) //aka 'to' has been initialized + if (!trange.getEndt().isNull()) //aka 'to' has been initialized { fullQuery.appendf("&end=%s000000000", std::to_string(trange.getEndt().getSimple()).c_str()); } From dd1c3466eee1b477ad2d066e03ad4c239575c929 Mon Sep 17 00:00:00 2001 From: Richard Chapman Date: Wed, 17 Jul 2024 15:44:05 +0100 Subject: [PATCH 22/29] HPCC-32260 Can't compile generated code on latest clang (mac) Signed-off-by: Richard Chapman --- ecl/hqlcpp/hqlres.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ecl/hqlcpp/hqlres.cpp b/ecl/hqlcpp/hqlres.cpp index f3445589360..87c485a3dca 100644 --- a/ecl/hqlcpp/hqlres.cpp +++ b/ecl/hqlcpp/hqlres.cpp @@ -648,7 +648,8 @@ bool ResourceManager::flush(StringBuffer &filename, const char *basename, bool f } fwrite(s.data.get(), 1, s.data.length(), bin); fclose(bin); - fprintf(f, " .size %s,%u\n", label.str(), (unsigned)s.data.length()); + if (!generateClang) + fprintf(f, " .size %s,%u\n", label.str(), (unsigned)s.data.length()); } fclose(f); #endif From c6413d4a5424eefa54eb00707f73959b033f1805 Mon Sep 17 00:00:00 2001 From: Jake Smith Date: Wed, 17 Jul 2024 17:36:03 +0100 Subject: [PATCH 23/29] HPCC-32267 Suppress buffer io if compressed in write stream Signed-off-by: Jake Smith --- common/thorhelper/thorcommon.cpp | 3 +++ 1 file changed, 3 insertions(+) diff --git a/common/thorhelper/thorcommon.cpp b/common/thorhelper/thorcommon.cpp index 38f29328615..82bfa6b785e 100644 --- a/common/thorhelper/thorcommon.cpp +++ b/common/thorhelper/thorcommon.cpp @@ -1647,7 +1647,10 @@ IExtRowWriter *createRowWriter(IFile *iFile, IRowInterfaces *rowIf, unsigned fla { OwnedIFileIO iFileIO; if (TestRwFlag(flags, rw_compress)) + { + flags &= ~rw_buffered; // if compressed, do not want buffered stream as well iFileIO.setown(createCompressedFileWriter(iFile, rowIf, flags, compressor, compressorBlkSz)); + } else iFileIO.setown(iFile->open((flags & rw_extend)?IFOwrite:IFOcreate)); if (!iFileIO) From a6719721def1b5da40501338063334c805b3a77d Mon Sep 17 00:00:00 2001 From: Jeremy Clements <79224539+jeclrsg@users.noreply.github.com> Date: Wed, 17 Jul 2024 17:30:45 -0400 Subject: [PATCH 24/29] HPCC-32257 ECL Watch v9 Files list reduce Logical Name width Signed-off-by: Jeremy Clements <79224539+jeclrsg@users.noreply.github.com> --- esp/src/src-react/components/Files.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/esp/src/src-react/components/Files.tsx b/esp/src/src-react/components/Files.tsx index 14939bd6164..08f5f953c34 100644 --- a/esp/src/src-react/components/Files.tsx +++ b/esp/src/src-react/components/Files.tsx @@ -171,7 +171,7 @@ export const Files: React.FunctionComponent = ({ }, Name: { label: nlsHPCC.LogicalName, - width: 360, + width: 180, formatter: (name, row) => { const file = Get(row.NodeGroup, name, row); if (row.__hpcc_isDir) { From 805eba3dac8a52377a302f08a62c1c858de127fe Mon Sep 17 00:00:00 2001 From: Jake Smith Date: Wed, 17 Jul 2024 11:20:25 +0100 Subject: [PATCH 25/29] HPCC-32258 Fix missing loop result spilling stats Signed-off-by: Jake Smith --- thorlcr/activities/loop/thloop.cpp | 2 +- thorlcr/activities/loop/thloopslave.cpp | 4 ++-- thorlcr/activities/nsplitter/thnsplitterslave.cpp | 2 +- thorlcr/master/thactivitymaster.cpp | 2 +- thorlcr/thorutil/thormisc.cpp | 4 ++-- thorlcr/thorutil/thormisc.hpp | 2 +- 6 files changed, 8 insertions(+), 8 deletions(-) diff --git a/thorlcr/activities/loop/thloop.cpp b/thorlcr/activities/loop/thloop.cpp index ebdde73be06..b32da6e09ab 100644 --- a/thorlcr/activities/loop/thloop.cpp +++ b/thorlcr/activities/loop/thloop.cpp @@ -334,7 +334,7 @@ class CLocalResultActivityMasterBase : public CMasterActivity Owned inputRowIf; public: - CLocalResultActivityMasterBase(CMasterGraphElement *info) : CMasterActivity(info) + CLocalResultActivityMasterBase(CMasterGraphElement *info) : CMasterActivity(info, spillingActivityStatistics) { } virtual void init() override diff --git a/thorlcr/activities/loop/thloopslave.cpp b/thorlcr/activities/loop/thloopslave.cpp index 722358f0e4d..ea7d149ff89 100644 --- a/thorlcr/activities/loop/thloopslave.cpp +++ b/thorlcr/activities/loop/thloopslave.cpp @@ -643,7 +643,7 @@ class CLocalResultSpillActivity : public CSlaveActivity } public: - CLocalResultSpillActivity(CGraphElementBase *_container) : CSlaveActivity(_container) + CLocalResultSpillActivity(CGraphElementBase *_container) : CSlaveActivity(_container, spillingActivityStatistics) { helper = (IHThorLocalResultSpillArg *)queryHelper(); appendOutputLinked(this); @@ -708,7 +708,7 @@ class CLocalResultSpillActivity : public CSlaveActivity class CLocalResultWriteActivityBase : public ProcessSlaveActivity { public: - CLocalResultWriteActivityBase(CGraphElementBase *_container) : ProcessSlaveActivity(_container) + CLocalResultWriteActivityBase(CGraphElementBase *_container) : ProcessSlaveActivity(_container, spillingActivityStatistics) { } virtual IThorResult *createResult() = 0; diff --git a/thorlcr/activities/nsplitter/thnsplitterslave.cpp b/thorlcr/activities/nsplitter/thnsplitterslave.cpp index ca9ccd0d6df..5331fedf845 100644 --- a/thorlcr/activities/nsplitter/thnsplitterslave.cpp +++ b/thorlcr/activities/nsplitter/thnsplitterslave.cpp @@ -152,7 +152,7 @@ class NSplitterSlaveActivity : public CSlaveActivity, implements ISharedSmartBuf } } public: - NSplitterSlaveActivity(CGraphElementBase *_container) : CSlaveActivity(_container, nsplitterActivityStatistics), writer(*this) + NSplitterSlaveActivity(CGraphElementBase *_container) : CSlaveActivity(_container, spillingActivityStatistics), writer(*this) { numOutputs = container.getOutputs(); connectedOutputSet.setown(createBitSet()); diff --git a/thorlcr/master/thactivitymaster.cpp b/thorlcr/master/thactivitymaster.cpp index f80ee0bd258..86210a47c87 100644 --- a/thorlcr/master/thactivitymaster.cpp +++ b/thorlcr/master/thactivitymaster.cpp @@ -210,7 +210,7 @@ class CGenericMasterGraphElement : public CMasterGraphElement ret = new CMasterActivity(this); break; case TAKsplit: - ret = new CMasterActivity(this, nsplitterActivityStatistics); + ret = new CMasterActivity(this, spillingActivityStatistics); break; case TAKsoap_rowdataset: case TAKsoap_rowaction: diff --git a/thorlcr/thorutil/thormisc.cpp b/thorlcr/thorutil/thormisc.cpp index 09b3fc233af..b6fa56231d6 100644 --- a/thorlcr/thorutil/thormisc.cpp +++ b/thorlcr/thorutil/thormisc.cpp @@ -82,7 +82,6 @@ const StatisticsMapping indexReadFileStatistics({}, diskReadRemoteStatistics, jh const StatisticsMapping indexReadActivityStatistics({StNumRowsProcessed}, indexReadFileStatistics, basicActivityStatistics); const StatisticsMapping indexWriteActivityStatistics({StPerReplicated, StNumLeafCacheAdds, StNumNodeCacheAdds, StNumBlobCacheAdds }, basicActivityStatistics, diskWriteRemoteStatistics); const StatisticsMapping keyedJoinActivityStatistics({ StNumIndexAccepted, StNumPreFiltered, StNumDiskSeeks, StNumDiskAccepted, StNumDiskRejected}, basicActivityStatistics, indexReadFileStatistics); -const StatisticsMapping loopActivityStatistics({StNumIterations}, basicActivityStatistics); const StatisticsMapping commonJoinActivityStatistics({StNumMatchLeftRowsMax, StNumMatchRightRowsMax, StNumMatchCandidates, StNumMatchCandidatesMax}, basicActivityStatistics); const StatisticsMapping allJoinActivityStatistics({}, commonJoinActivityStatistics); const StatisticsMapping lookupJoinActivityStatistics({StNumSmartJoinSlavesDegradedToStd, StNumSmartJoinDegradedToLocal}, spillStatistics, commonJoinActivityStatistics); @@ -96,8 +95,9 @@ const StatisticsMapping indexDistribActivityStatistics({}, basicActivityStatisti const StatisticsMapping soapcallActivityStatistics({}, basicActivityStatistics, soapcallStatistics); const StatisticsMapping hashDedupActivityStatistics({}, spillStatistics, diskWriteRemoteStatistics, basicActivityStatistics); const StatisticsMapping hashDistribActivityStatistics({StNumLocalRows, StNumRemoteRows, StSizeRemoteWrite}, basicActivityStatistics); -const StatisticsMapping nsplitterActivityStatistics({}, spillStatistics, basicActivityStatistics); +const StatisticsMapping spillingActivityStatistics({}, spillStatistics, basicActivityStatistics); const StatisticsMapping spillingWriteAheadStatistics({}, spillStatistics); +const StatisticsMapping loopActivityStatistics({StNumIterations}, spillingActivityStatistics); MODULE_INIT(INIT_PRIORITY_STANDARD) { diff --git a/thorlcr/thorutil/thormisc.hpp b/thorlcr/thorutil/thormisc.hpp index cf0b92bb4b0..a9d4da09c92 100644 --- a/thorlcr/thorutil/thormisc.hpp +++ b/thorlcr/thorutil/thormisc.hpp @@ -167,7 +167,7 @@ extern graph_decl const StatisticsMapping soapcallActivityStatistics; extern graph_decl const StatisticsMapping indexReadFileStatistics; extern graph_decl const StatisticsMapping hashDedupActivityStatistics; extern graph_decl const StatisticsMapping hashDistribActivityStatistics; -extern graph_decl const StatisticsMapping nsplitterActivityStatistics; +extern graph_decl const StatisticsMapping spillingActivityStatistics; extern graph_decl const StatisticsMapping spillingWriteAheadStatistics; class BooleanOnOff From 582c0ebd29be3cc6adbffea8a511fb489818c72b Mon Sep 17 00:00:00 2001 From: Shamser Ahmed Date: Tue, 16 Jul 2024 14:15:46 +0100 Subject: [PATCH 26/29] HPCC-32241 Temp size for sort to use disk size and track actual graph temp disk usage SizePeakTempDisk for graphs was previous calculated by summing the each activity's active temp file size. However, as this was done at internals, spikes in temp file sizes between intervals were not recorded in the SizePeakTempDisk stats. This change uses a TempFileSize tracker at the graph level to track the actual peak/active temp file size for the graph. Sort activities was using the size of the uncompressed data as the size of the temp file. This change uses the actual size of temp file for sort activity's peak/active temp file size Signed-off-by: Shamser Ahmed --- thorlcr/graph/thgraph.hpp | 23 ++++++++++++++++++++--- thorlcr/graph/thgraphslave.cpp | 15 +-------------- thorlcr/graph/thgraphslave.hpp | 1 - thorlcr/msort/tsorts.cpp | 2 +- thorlcr/thorutil/thormisc.hpp | 10 ++++++++++ 5 files changed, 32 insertions(+), 19 deletions(-) diff --git a/thorlcr/graph/thgraph.hpp b/thorlcr/graph/thgraph.hpp index b0adfbade47..b1045fda17a 100644 --- a/thorlcr/graph/thgraph.hpp +++ b/thorlcr/graph/thgraph.hpp @@ -630,7 +630,7 @@ class graph_decl CGraphBase : public CGraphStub, implements IEclGraphResults CChildGraphTable childGraphsTable; CGraphStubArrayCopy orderedChildGraphs; Owned tmpHandler; - + Owned tempFileSizeTracker; void clean(); protected: @@ -805,7 +805,24 @@ class graph_decl CGraphBase : public CGraphStub, implements IEclGraphResults virtual void end(); virtual void abort(IException *e) override; virtual IThorGraphResults *createThorGraphResults(unsigned num); - + CFileSizeTracker * queryTempFileSizeTracker() + { + if (!tempFileSizeTracker) + tempFileSizeTracker.setown(new CFileSizeTracker); + return tempFileSizeTracker; + } + offset_t queryPeakTempSize() + { + if (tempFileSizeTracker) + return tempFileSizeTracker->queryPeakSize(); + return 0; + } + offset_t queryActiveTempSize() + { + if (tempFileSizeTracker) + return tempFileSizeTracker->queryActiveSize(); + return 0; + } // IExceptionHandler virtual bool fireException(IException *e); @@ -1175,7 +1192,7 @@ class graph_decl CActivityBase : implements CInterfaceOf, im CFileSizeTracker * queryTempFileSizeTracker() { if (!tempFileSizeTracker) - tempFileSizeTracker.setown(new CFileSizeTracker); + tempFileSizeTracker.setown(new CFileSizeTracker(queryGraph().queryParent()->queryTempFileSizeTracker())); return tempFileSizeTracker; } offset_t queryActiveTempSize() const diff --git a/thorlcr/graph/thgraphslave.cpp b/thorlcr/graph/thgraphslave.cpp index 14c8d7ea6d0..e39c45909d5 100644 --- a/thorlcr/graph/thgraphslave.cpp +++ b/thorlcr/graph/thgraphslave.cpp @@ -1323,20 +1323,7 @@ bool CSlaveGraph::serializeStats(MemoryBuffer &mb) if (tempHandler) stats.mergeStatistic(StSizeGraphSpill, sizeGraphSpill); - // calculate peak spill size - if (started&&initialized) - { - offset_t activeTempSize = 0; - Owned iter = getConnectedIterator(); - ForEach (*iter) - { - CGraphElementBase &element = iter->query(); - CSlaveActivity &activity = (CSlaveActivity &)*element.queryActivity(); - activeTempSize += activity.queryActiveTempSize(); - } - if (activeTempSize > peakTempSize) - peakTempSize = activeTempSize; - } + offset_t peakTempSize = queryPeakTempSize(); if (peakTempSize) stats.mergeStatistic(StSizePeakTempDisk, peakTempSize); if (peakTempSize + sizeGraphSpill) diff --git a/thorlcr/graph/thgraphslave.hpp b/thorlcr/graph/thgraphslave.hpp index f6b73c912c1..1c9595348f2 100644 --- a/thorlcr/graph/thgraphslave.hpp +++ b/thorlcr/graph/thgraphslave.hpp @@ -522,7 +522,6 @@ class graphslave_decl CSlaveGraph : public CGraphBase bool doneInit = false; std::atomic_bool progressActive; ProcessInfo processStartInfo; - offset_t peakTempSize = 0; public: diff --git a/thorlcr/msort/tsorts.cpp b/thorlcr/msort/tsorts.cpp index 55afb16704a..7e58fa50ea1 100644 --- a/thorlcr/msort/tsorts.cpp +++ b/thorlcr/msort/tsorts.cpp @@ -257,8 +257,8 @@ class CWriteIntercept : public CSimpleInterface } output->flush(); offset_t end = output->getPosition(); + dataFile->noteSize(output->getStatistic(StSizeDiskWrite)); output.clear(); - dataFile->noteSize(end); writeidxofs(end); if (idxFileIO) { diff --git a/thorlcr/thorutil/thormisc.hpp b/thorlcr/thorutil/thormisc.hpp index cf0b92bb4b0..5dec5ee43bc 100644 --- a/thorlcr/thorutil/thormisc.hpp +++ b/thorlcr/thorutil/thormisc.hpp @@ -326,19 +326,29 @@ class CFileSizeTracker: public CInterface { RelaxedAtomic activeSize{0}; RelaxedAtomic peakSize{0}; + CFileSizeTracker * parentFileSizeTracker; public: + CFileSizeTracker(CFileSizeTracker *parent=nullptr): parentFileSizeTracker(parent) + { + } void growSize(offset_t size) { if (size) { offset_t newActiveSize = activeSize.add_fetch(size); peakSize.store_max(newActiveSize); + if (parentFileSizeTracker) + parentFileSizeTracker->growSize(size); } } void shrinkSize(offset_t size) { if (size) + { activeSize.fetch_sub(size); + if (parentFileSizeTracker) + parentFileSizeTracker->shrinkSize(size); + } } offset_t queryActiveSize() const { From 1e48efb6654703df2f0f3aa0c88491bc86c05bcf Mon Sep 17 00:00:00 2001 From: xwang2713 Date: Wed, 17 Jul 2024 21:56:23 -0400 Subject: [PATCH 27/29] HPCC-32242 add serviceName as required by Kubernetes StatefulSet spec --- helm/hpcc/templates/dali.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/helm/hpcc/templates/dali.yaml b/helm/hpcc/templates/dali.yaml index 2adfbbd328d..fb89d7b4e8f 100644 --- a/helm/hpcc/templates/dali.yaml +++ b/helm/hpcc/templates/dali.yaml @@ -77,6 +77,7 @@ spec: run: {{ $dali.name | quote }} server: {{ $dali.name | quote }} app: dali + serviceName: {{ $dali.name | quote }} updateStrategy: type: RollingUpdate rollingUpdate: From afefda85f743f13ff748731365947a5951f68291 Mon Sep 17 00:00:00 2001 From: Jake Smith Date: Wed, 17 Jul 2024 16:58:29 +0100 Subject: [PATCH 28/29] HPCC-32266 Ensure LOOP result streams are compressed. Signed-off-by: Jake Smith --- thorlcr/thorutil/thmem.cpp | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/thorlcr/thorutil/thmem.cpp b/thorlcr/thorutil/thmem.cpp index 299f1e3118a..6f2c7e4fcfd 100644 --- a/thorlcr/thorutil/thmem.cpp +++ b/thorlcr/thorutil/thmem.cpp @@ -336,6 +336,11 @@ class CSharedSpillableRowSet : public CSpillableStreamBase block.clearCB = true; assertex(((offset_t)-1) != outputOffset); unsigned rwFlags = DEFAULT_RWFLAGS | mapESRToRWFlags(owner->emptyRowSemantics); + if (owner->spillCompInfo) + { + rwFlags |= rw_compress; + rwFlags |= owner->spillCompInfo; + } spillStream.setown(::createRowStreamEx(&(owner->spillFile->queryIFile()), owner->rowIf, outputOffset, (offset_t)-1, (unsigned __int64)-1, rwFlags)); owner->rows.unregisterWriteCallback(*this); // no longer needed ret = spillStream->nextRow(); @@ -376,9 +381,10 @@ class CSharedSpillableRowSet : public CSpillableStreamBase }; public: - CSharedSpillableRowSet(CActivityBase &_activity, CThorSpillableRowArray &inRows, IThorRowInterfaces *_rowIf, EmptyRowSemantics _emptyRowSemantics, unsigned _spillPriority) + CSharedSpillableRowSet(CActivityBase &_activity, CThorSpillableRowArray &inRows, IThorRowInterfaces *_rowIf, EmptyRowSemantics _emptyRowSemantics, unsigned _spillCompInfo, unsigned _spillPriority) : CSpillableStreamBase(_activity, inRows, _rowIf, _emptyRowSemantics, _spillPriority) { + spillCompInfo = _spillCompInfo; } IRowStream *createRowStream() { @@ -387,6 +393,11 @@ class CSharedSpillableRowSet : public CSpillableStreamBase { block.clearCB = true; unsigned rwFlags = DEFAULT_RWFLAGS | mapESRToRWFlags(emptyRowSemantics); + if (spillCompInfo) + { + rwFlags |= rw_compress; + rwFlags |= spillCompInfo; + } return ::createRowStream(&spillFile->queryIFile(), rowIf, rwFlags); } rowidx_t toRead = rows.numCommitted(); @@ -1380,10 +1391,7 @@ rowidx_t CThorSpillableRowArray::save(CFileOwner &iFileOwner, unsigned _spillCom rowidx_t n = numCommitted(); if (0 == n) return 0; - ActPrintLog(&activity, "%s: CThorSpillableRowArray::save (skipNulls=%s, emptyRowSemantics=%u) max rows = %" RIPF "u", _tracingPrefix, boolToStr(skipNulls), emptyRowSemantics, n); - - if (_spillCompInfo) - assertex(0 == writeCallbacks.ordinality()); // incompatible + ActPrintLog(&activity, "%s: CThorSpillableRowArray::save (skipNulls=%s, emptyRowSemantics=%u) max rows = %" RIPF "u", _tracingPrefix, boolToStr(skipNulls), emptyRowSemantics, n); unsigned rwFlags = DEFAULT_RWFLAGS; if (_spillCompInfo) @@ -1773,7 +1781,7 @@ class CThorRowCollectorBase : public CSpillable * because roxiemem's background thread may be blocked on that lock, and calling roxiemem::addRowBuffer here would cause deadlock */ activateSharedCallback = true; - spillableRowSet.setown(new CSharedSpillableRowSet(activity, spillableRows, rowIf, emptyRowSemantics, spillPriority)); + spillableRowSet.setown(new CSharedSpillableRowSet(activity, spillableRows, rowIf, emptyRowSemantics, spillCompInfo, spillPriority)); spillableRowSet->setTracingPrefix(tracingPrefix); } } From 0cb7a3c93e8b9b6724f1c13568fe2bfce0d1c21c Mon Sep 17 00:00:00 2001 From: xwang2713 Date: Wed, 17 Jul 2024 21:56:23 -0400 Subject: [PATCH 29/29] HPCC-32242 add serviceName as required by Kubernetes StatefulSet spec --- helm/hpcc/templates/dali.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/helm/hpcc/templates/dali.yaml b/helm/hpcc/templates/dali.yaml index 35a8cc3f52e..ce1362da5b6 100644 --- a/helm/hpcc/templates/dali.yaml +++ b/helm/hpcc/templates/dali.yaml @@ -77,6 +77,7 @@ spec: run: {{ $dali.name | quote }} server: {{ $dali.name | quote }} app: dali + serviceName: {{ $dali.name | quote }} updateStrategy: type: RollingUpdate rollingUpdate: