From 595364edb218e054989da6315da619af5f993bfc Mon Sep 17 00:00:00 2001 From: Jake Smith Date: Fri, 26 Jul 2024 15:25:04 +0100 Subject: [PATCH 1/9] Split off 9.0.128 Signed-off-by: Jake Smith --- helm/hpcc/Chart.yaml | 4 ++-- helm/hpcc/templates/_helpers.tpl | 2 +- helm/hpcc/templates/dafilesrv.yaml | 2 +- helm/hpcc/templates/dali.yaml | 2 +- helm/hpcc/templates/dfuserver.yaml | 2 +- helm/hpcc/templates/eclagent.yaml | 4 ++-- helm/hpcc/templates/eclccserver.yaml | 4 ++-- helm/hpcc/templates/eclscheduler.yaml | 2 +- helm/hpcc/templates/esp.yaml | 2 +- helm/hpcc/templates/localroxie.yaml | 2 +- helm/hpcc/templates/roxie.yaml | 8 ++++---- helm/hpcc/templates/sasha.yaml | 2 +- helm/hpcc/templates/thor.yaml | 10 +++++----- version.cmake | 4 ++-- 14 files changed, 25 insertions(+), 25 deletions(-) diff --git a/helm/hpcc/Chart.yaml b/helm/hpcc/Chart.yaml index 0ab6e45fa80..5255ef6d030 100644 --- a/helm/hpcc/Chart.yaml +++ b/helm/hpcc/Chart.yaml @@ -6,9 +6,9 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. -version: 9.0.127-closedown0 +version: 9.0.129-closedown0 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. -appVersion: 9.0.127-closedown0 +appVersion: 9.0.129-closedown0 diff --git a/helm/hpcc/templates/_helpers.tpl b/helm/hpcc/templates/_helpers.tpl index 71faa4082b8..9cb6d98801e 100644 --- a/helm/hpcc/templates/_helpers.tpl +++ b/helm/hpcc/templates/_helpers.tpl @@ -1314,7 +1314,7 @@ kind: Service metadata: name: {{ $lvars.serviceName | quote }} labels: - helmVersion: 9.0.127-closedown0 + helmVersion: 9.0.129-closedown0 {{- include "hpcc.addStandardLabels" (dict "root" $.root "instance" $lvars.serviceName ) | indent 4 }} {{- if $lvars.labels }} {{ toYaml $lvars.labels | indent 4 }} diff --git a/helm/hpcc/templates/dafilesrv.yaml b/helm/hpcc/templates/dafilesrv.yaml index ba830ec770c..f1f6fae94a5 100644 --- a/helm/hpcc/templates/dafilesrv.yaml +++ b/helm/hpcc/templates/dafilesrv.yaml @@ -50,7 +50,7 @@ spec: labels: {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "dafilesrv" "name" "dafilesrv" "instance" .name) | indent 8 }} server: {{ .name | quote }} - helmVersion: 9.0.127-closedown0 + helmVersion: 9.0.129-closedown0 annotations: checksum/config: {{ $configSHA }} spec: diff --git a/helm/hpcc/templates/dali.yaml b/helm/hpcc/templates/dali.yaml index 5574603c45d..ef05fc36c13 100644 --- a/helm/hpcc/templates/dali.yaml +++ b/helm/hpcc/templates/dali.yaml @@ -82,7 +82,7 @@ spec: run: {{ $dali.name | quote }} server: {{ $dali.name | quote }} app: dali - helmVersion: 9.0.127-closedown0 + helmVersion: 9.0.129-closedown0 {{- if hasKey $.Values.global "metrics" }} {{- include "hpcc.generateMetricsReporterLabel" $.Values.global.metrics | nindent 8 }} {{- end }} diff --git a/helm/hpcc/templates/dfuserver.yaml b/helm/hpcc/templates/dfuserver.yaml index c7984644219..2f4d65ff071 100644 --- a/helm/hpcc/templates/dfuserver.yaml +++ b/helm/hpcc/templates/dfuserver.yaml @@ -56,7 +56,7 @@ spec: {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "dfuserver" "name" "dfuserver" "instance" .name) | indent 8 }} run: {{ .name | quote }} accessDali: "yes" - helmVersion: 9.0.127-closedown0 + helmVersion: 9.0.129-closedown0 {{- if hasKey . "labels" }} {{ toYaml .labels | indent 8 }} {{- end }} diff --git a/helm/hpcc/templates/eclagent.yaml b/helm/hpcc/templates/eclagent.yaml index da05108b72b..1cfcaffb116 100644 --- a/helm/hpcc/templates/eclagent.yaml +++ b/helm/hpcc/templates/eclagent.yaml @@ -58,7 +58,7 @@ data: {{- include "hpcc.addStandardLabels" (dict "root" $ "component" $apptype "name" "eclagent" "instance" $appJobName "instanceOf" (printf "%s-job" .me.name)) | indent 12 }} accessDali: "yes" accessEsp: "yes" - helmVersion: 9.0.127-closedown0 + helmVersion: 9.0.129-closedown0 {{- if hasKey .me "labels" }} {{ toYaml .me.labels | indent 12 }} {{- end }} @@ -135,7 +135,7 @@ spec: run: {{ .name | quote }} accessDali: "yes" accessEsp: {{ .useChildProcesses | default false | ternary "yes" "no" | quote }} - helmVersion: 9.0.127-closedown0 + helmVersion: 9.0.129-closedown0 {{- if hasKey . "labels" }} {{ toYaml .labels | indent 8 }} {{- end }} diff --git a/helm/hpcc/templates/eclccserver.yaml b/helm/hpcc/templates/eclccserver.yaml index ba60a34b0f0..f8f1d287dc0 100644 --- a/helm/hpcc/templates/eclccserver.yaml +++ b/helm/hpcc/templates/eclccserver.yaml @@ -57,7 +57,7 @@ data: {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "eclccserver" "name" "eclccserver" "instance" $compileJobName "instanceOf" (printf "%s-job" .me.name)) | indent 12 }} accessDali: "yes" accessEsp: "yes" - helmVersion: 9.0.127-closedown0 + helmVersion: 9.0.129-closedown0 {{- if hasKey .me "labels" }} {{ toYaml .me.labels | indent 12 }} {{- end }} @@ -142,7 +142,7 @@ spec: run: {{ .name | quote }} accessDali: "yes" accessEsp: {{ .useChildProcesses | default false | ternary "yes" "no" | quote }} - helmVersion: 9.0.127-closedown0 + helmVersion: 9.0.129-closedown0 {{- if hasKey . "labels" }} {{ toYaml .labels | indent 8 }} {{- end }} diff --git a/helm/hpcc/templates/eclscheduler.yaml b/helm/hpcc/templates/eclscheduler.yaml index 1b5d0018262..bf7a7459964 100644 --- a/helm/hpcc/templates/eclscheduler.yaml +++ b/helm/hpcc/templates/eclscheduler.yaml @@ -64,7 +64,7 @@ spec: run: {{ .name | quote }} accessDali: "yes" accessEsp: "no" - helmVersion: 9.0.127-closedown0 + helmVersion: 9.0.129-closedown0 {{- if hasKey . "labels" }} {{ toYaml .labels | indent 8 }} {{- end }} diff --git a/helm/hpcc/templates/esp.yaml b/helm/hpcc/templates/esp.yaml index 6756930094f..30b9481c481 100644 --- a/helm/hpcc/templates/esp.yaml +++ b/helm/hpcc/templates/esp.yaml @@ -117,7 +117,7 @@ spec: server: {{ .name | quote }} accessDali: "yes" app: {{ $application }} - helmVersion: 9.0.127-closedown0 + helmVersion: 9.0.129-closedown0 {{- include "hpcc.addStandardLabels" (dict "root" $ "name" $application "component" "esp" "instance" .name) | indent 8 }} {{- if hasKey $.Values.global "metrics" }} {{- include "hpcc.generateMetricsReporterLabel" $.Values.global.metrics | nindent 8 }} diff --git a/helm/hpcc/templates/localroxie.yaml b/helm/hpcc/templates/localroxie.yaml index 519596cc908..e24e601a867 100644 --- a/helm/hpcc/templates/localroxie.yaml +++ b/helm/hpcc/templates/localroxie.yaml @@ -70,7 +70,7 @@ spec: server: {{ $servername | quote }} accessDali: "yes" accessEsp: "yes" - helmVersion: 9.0.127-closedown0 + helmVersion: 9.0.129-closedown0 {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "roxie-server" "name" "roxie" "instance" $roxie.name) | indent 8 }} {{- if hasKey . "labels" }} {{ toYaml .labels | indent 8 }} diff --git a/helm/hpcc/templates/roxie.yaml b/helm/hpcc/templates/roxie.yaml index be28f790cbc..39749b268b5 100644 --- a/helm/hpcc/templates/roxie.yaml +++ b/helm/hpcc/templates/roxie.yaml @@ -120,7 +120,7 @@ spec: {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "topology-server" "name" "roxie" "instance" $commonCtx.toponame) | indent 8 }} run: {{ $commonCtx.toponame | quote }} roxie-cluster: {{ $roxie.name | quote }} - helmVersion: 9.0.127-closedown0 + helmVersion: 9.0.129-closedown0 {{- if hasKey $.Values.global "metrics" }} {{- include "hpcc.generateMetricsReporterLabel" $.Values.global.metrics | nindent 8}} {{- end }} @@ -180,7 +180,7 @@ kind: Service metadata: name: {{ $commonCtx.toponame | quote }} labels: - helmVersion: 9.0.127-closedown0 + helmVersion: 9.0.129-closedown0 {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "topology-server" "name" "roxie" "instance" $commonCtx.toponame) | indent 4 }} spec: ports: @@ -242,7 +242,7 @@ spec: roxie-cluster: {{ $roxie.name | quote }} accessDali: "yes" accessEsp: "yes" - helmVersion: 9.0.127-closedown0 + helmVersion: 9.0.129-closedown0 {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "roxie-server" "name" "roxie" "instance" $servername) | indent 8 }} {{- if hasKey $.Values.global "metrics" }} {{- include "hpcc.generateMetricsReporterLabel" $.Values.global.metrics | nindent 8}} @@ -347,7 +347,7 @@ spec: roxie-cluster: {{ $roxie.name | quote }} accessDali: "yes" accessEsp: "yes" - helmVersion: 9.0.127-closedown0 + helmVersion: 9.0.129-closedown0 {{- if hasKey $.Values.global "metrics" }} {{- include "hpcc.generateMetricsReporterLabel" $.Values.global.metrics | nindent 8}} {{- end }} diff --git a/helm/hpcc/templates/sasha.yaml b/helm/hpcc/templates/sasha.yaml index 5af4b0631bb..d1910b6e34a 100644 --- a/helm/hpcc/templates/sasha.yaml +++ b/helm/hpcc/templates/sasha.yaml @@ -52,7 +52,7 @@ spec: run: {{ $serviceName | quote }} server: {{ $serviceName | quote }} accessDali: {{ (has "dali" $sasha.access) | ternary "yes" "no" | quote }} - helmVersion: 9.0.127-closedown0 + helmVersion: 9.0.129-closedown0 {{- if hasKey $sasha "labels" }} {{ toYaml $sasha.labels | indent 8 }} {{- end }} diff --git a/helm/hpcc/templates/thor.yaml b/helm/hpcc/templates/thor.yaml index 32a862f51cf..bcda109fa22 100644 --- a/helm/hpcc/templates/thor.yaml +++ b/helm/hpcc/templates/thor.yaml @@ -82,7 +82,7 @@ data: labels: accessDali: "yes" accessEsp: "yes" - helmVersion: 9.0.127-closedown0 + helmVersion: 9.0.129-closedown0 {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "eclagent" "name" "thor" "instance" $eclAgentJobName "instanceOf" (printf "%s-job" .eclAgentName)) | indent 8 }} {{- if hasKey .me "labels" }} {{ toYaml .me.labels | indent 12 }} @@ -147,7 +147,7 @@ data: accessEsp: "yes" app: "thor" component: "thormanager" - helmVersion: 9.0.127-closedown0 + helmVersion: 9.0.129-closedown0 instance: "_HPCC_JOBNAME_" job: "_HPCC_JOBNAME_" {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "thormanager" "name" "thor" "instance" $thorManagerJobName "instanceOf" (printf "%s-thormanager-job" .me.name)) | indent 12 }} @@ -214,7 +214,7 @@ data: accessEsp: "yes" app: "thor" component: "thorworker" - helmVersion: 9.0.127-closedown0 + helmVersion: 9.0.129-closedown0 instance: "_HPCC_JOBNAME_" job: "_HPCC_JOBNAME_" {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "thorworker" "name" "thor" "instance" $thorWorkerJobName "instanceOf" (printf "%s-thorworker-job" .me.name)) | indent 12 }} @@ -347,7 +347,7 @@ spec: accessEsp: {{ $commonCtx.eclAgentUseChildProcesses | ternary "yes" "no" | quote }} app: "thor" component: "thor-eclagent" - helmVersion: 9.0.127-closedown0 + helmVersion: 9.0.129-closedown0 instance: {{ $commonCtx.eclAgentName | quote }} {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "eclagent" "name" "thor" "instance" $commonCtx.eclAgentName ) | indent 8 }} {{- if hasKey $commonCtx.me "labels" }} @@ -412,7 +412,7 @@ spec: accessEsp: "no" app: "thor" component: "thor-thoragent" - helmVersion: 9.0.127-closedown0 + helmVersion: 9.0.129-closedown0 instance: {{ $commonCtx.thorAgentName | quote }} {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "eclagent" "name" "thor" "instance" $commonCtx.thorAgentName ) | indent 8 }} {{- if hasKey $commonCtx.me "labels" }} diff --git a/version.cmake b/version.cmake index 8ee2e5dfa63..11de13f18a4 100644 --- a/version.cmake +++ b/version.cmake @@ -5,8 +5,8 @@ set ( HPCC_NAME "Community Edition" ) set ( HPCC_PROJECT "community" ) set ( HPCC_MAJOR 9 ) set ( HPCC_MINOR 0 ) -set ( HPCC_POINT 127 ) +set ( HPCC_POINT 129 ) set ( HPCC_MATURITY "closedown" ) set ( HPCC_SEQUENCE 0 ) -set ( HPCC_TAG_TIMESTAMP "2024-07-11T16:12:06Z" ) +set ( HPCC_TAG_TIMESTAMP "2024-07-26T14:25:03Z" ) ### From aec1585f22e660ea337602a9dd627cb60cea1167 Mon Sep 17 00:00:00 2001 From: Gavin Halliday Date: Mon, 29 Jul 2024 11:36:03 +0100 Subject: [PATCH 2/9] HPCC-32314 Fix false positive warnings generated by Coverity Signed-off-by: Gavin Halliday --- system/jlib/jmisc.hpp | 2 +- system/jlib/jstats.cpp | 7 +++++++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/system/jlib/jmisc.hpp b/system/jlib/jmisc.hpp index 6e6fe9acafe..60c4a95ec13 100644 --- a/system/jlib/jmisc.hpp +++ b/system/jlib/jmisc.hpp @@ -342,7 +342,7 @@ template class TimeDivisionTracker protected: unsigned __int64 totals[NUMSTATES] = {0}; unsigned counts[NUMSTATES] = {0}; - const char *stateNames[NUMSTATES]; + const char *stateNames[NUMSTATES] = {}; unsigned __int64 lastTick = get_cycles_now(); unsigned currentState = 0; StringAttr name; diff --git a/system/jlib/jstats.cpp b/system/jlib/jstats.cpp index af280d1685c..eed74316f27 100644 --- a/system/jlib/jstats.cpp +++ b/system/jlib/jstats.cpp @@ -1301,6 +1301,13 @@ class CComponentStatistics //-------------------------------------------------------------------------------------------------------------------- +//This object is accessed by all StatisticsMapping constructors. It is important that it is initialised before any of +//them are called. Coverity routinely complains about possible GLOBAL_INIT_ORDER problems. These are false positives +//as long as +// * this definition of allStatsMappings comes first in this file before any StatisticsMappings +// * there must be no other files in jlib that declare a StatisticMapping. +// StatisticsMappings in other dlls are ok because it is guaranteed that the dependent dll/so is initialised first + static std::unordered_map allStatsMappings; static int compareUnsigned(unsigned const * left, unsigned const * right) From 0c030368fa6709f1d1502014a8fa7ea9d272a682 Mon Sep 17 00:00:00 2001 From: Gavin Halliday Date: Mon, 29 Jul 2024 17:42:32 +0100 Subject: [PATCH 3/9] HPCC-32327 Fix minor leak deserializing jobids from logging Signed-off-by: Gavin Halliday --- system/jlib/jlog.cpp | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/system/jlib/jlog.cpp b/system/jlib/jlog.cpp index 2d8bf718e3e..4adf5801a43 100644 --- a/system/jlib/jlog.cpp +++ b/system/jlib/jlog.cpp @@ -275,6 +275,11 @@ void LogMsgJobInfo::deserialize(MemoryBuffer & in) { // kludge for backward compatibility of pre 8.0 clients that send a LogMsgJobId: (_uint64), not a string // NB: jobID pre 8.0 was redundant as always equal to UnknownJob + if (isDeserialized) + { + free((void *) jobIDStr); + jobIDStr = nullptr; + } dbgassertex(in.remaining() >= sizeof(LogMsgJobId)); // should always be at least this amount, because userID follows the jobID if (0 == memcmp(in.toByteArray()+in.getPos(), &UnknownJob, sizeof(LogMsgJobId))) // pre 8.0 client { From c3b3361b63339bb25b53c31554ea95876ee52fbc Mon Sep 17 00:00:00 2001 From: Gavin Halliday Date: Mon, 29 Jul 2024 17:45:41 +0100 Subject: [PATCH 4/9] HPCC-32322 Ensure dali delta header is 0 terminated Signed-off-by: Gavin Halliday --- dali/base/dasds.cpp | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/dali/base/dasds.cpp b/dali/base/dasds.cpp index 3ad59c4fa26..a712cd93230 100644 --- a/dali/base/dasds.cpp +++ b/dali/base/dasds.cpp @@ -5351,13 +5351,15 @@ class CStoreHelper : implements IStoreHelper, public CInterface return true; offset_t fSize = iFileIO->size(); PROGLOG("Loading delta: %s (size=%.2f MB)", filename, ((double)fSize) / 0x100000); + size32_t lenDeltaHeader = strlen(deltaHeader); MemoryBuffer tmp; - char *ptr = (char *) tmp.reserveTruncate(strlen(deltaHeader)); + char *ptr = (char *) tmp.reserveTruncate(lenDeltaHeader+1); unsigned embeddedCrc = 0; offset_t pos = 0; bool hasCrcHeader = false; // check really only needed for deltas proceeding CRC header - if (strlen(deltaHeader) == iFileIO->read(0, strlen(deltaHeader), ptr)) + if (lenDeltaHeader == iFileIO->read(0, lenDeltaHeader, ptr)) { + ptr[lenDeltaHeader] = '\0'; if (0 == memicmp(deltaHeader, ptr, 5)) { pos = deltaHeaderSizeStart; From 4aa2f85660f219fc310e52211570f688af70bcd3 Mon Sep 17 00:00:00 2001 From: Ilhan Gelle Date: Thu, 25 Jul 2024 16:55:25 -0500 Subject: [PATCH 5/9] HPCC-32238 Add large_list data type to Parquet Plugin --- plugins/parquet/parquetembed.cpp | 30 ++++++++++++++++++++++++------ plugins/parquet/parquetembed.hpp | 8 ++++++++ 2 files changed, 32 insertions(+), 6 deletions(-) diff --git a/plugins/parquet/parquetembed.cpp b/plugins/parquet/parquetembed.cpp index 2349ea70530..88cb29bd3f9 100644 --- a/plugins/parquet/parquetembed.cpp +++ b/plugins/parquet/parquetembed.cpp @@ -714,7 +714,7 @@ std::shared_ptr ParquetWriter::makeChildRecord(const RtlField const RtlFieldInfo childFieldInfo = RtlFieldInfo("", "", child); std::vector> childField; reportIfFailure(fieldToNode(&childFieldInfo, childField)); - return std::make_shared(childField[0]); + return std::make_shared(childField[0]); } } @@ -845,8 +845,8 @@ void ParquetWriter::beginSet(const RtlFieldInfo *field) arrow::FieldPath match = getNestedFieldBuilder(field, childBuilder); fieldBuilderStack.push_back(std::make_shared(field, childBuilder, CPNTSet, std::move(match))); - arrow::ListBuilder *listBuilder = static_cast(childBuilder); - reportIfFailure(listBuilder->Append()); + arrow::LargeListBuilder *largeListBuilder = static_cast(childBuilder); + reportIfFailure(largeListBuilder->Append()); } /** @@ -946,7 +946,7 @@ arrow::ArrayBuilder *ParquetWriter::getFieldBuilder(const RtlFieldInfo *field) return recordBatchBuilder->GetField(schema->GetFieldIndex(fieldName.str())); } else if (fieldBuilderStack.back()->nodeType == CPNTSet) - return static_cast(fieldBuilderStack.back()->structPtr)->value_builder(); + return static_cast(fieldBuilderStack.back()->structPtr)->value_builder(); else return fieldBuilderStack.back()->structPtr->child(fieldBuilderStack.back()->childrenProcessed++); } @@ -1463,6 +1463,12 @@ void ParquetRowBuilder::processBeginSet(const RtlFieldInfo *field, bool &isAll) newPathNode.childCount = arrayVisitor->listArr->value_slice(currentRow)->length(); pathStack.push_back(newPathNode); } + else if (arrayVisitor->type == LargeListType) + { + ParquetColumnTracker newPathNode(field, arrayVisitor->largeListArr, CPNTSet); + newPathNode.childCount = arrayVisitor->largeListArr->value_slice(currentRow)->length(); + pathStack.push_back(newPathNode); + } else { failx("Error reading nested set with name %s.", field->name); @@ -1585,8 +1591,20 @@ void ParquetRowBuilder::nextFromStruct(const RtlFieldInfo *field) } else if (pathStack.back().nodeType == CPNTSet) { - auto child = arrayVisitor->listArr->value_slice(currentRow); - reportIfFailure(child->Accept(arrayVisitor.get())); + if (arrayVisitor->type == ListType) + { + auto child = arrayVisitor->listArr->value_slice(currentRow); + reportIfFailure(child->Accept(arrayVisitor.get())); + } + else if (arrayVisitor->type == LargeListType) + { + auto child = arrayVisitor->largeListArr->value_slice(currentRow); + reportIfFailure(child->Accept(arrayVisitor.get())); + } + else + { + failx("Unexpected type in CPNTSet: neither ListType nor LargeListType"); + } } } diff --git a/plugins/parquet/parquetembed.hpp b/plugins/parquet/parquetembed.hpp index 877a35d0ab9..f7545026cae 100644 --- a/plugins/parquet/parquetembed.hpp +++ b/plugins/parquet/parquetembed.hpp @@ -131,6 +131,7 @@ enum ParquetArrayType LargeBinaryType, DecimalType, ListType, + LargeListType, StructType, RealType }; @@ -314,6 +315,12 @@ class ParquetArrayVisitor : public arrow::ArrayVisitor type = ListType; return arrow::Status::OK(); } + arrow::Status Visit(const arrow::LargeListArray &array) + { + largeListArr = &array; + type = LargeListType; + return arrow::Status::OK(); + } arrow::Status Visit(const arrow::StructArray &array) { structArr = &array; @@ -348,6 +355,7 @@ class ParquetArrayVisitor : public arrow::ArrayVisitor const arrow::Decimal128Array *decArr = nullptr; const arrow::Decimal256Array *largeDecArr = nullptr; const arrow::ListArray *listArr = nullptr; + const arrow::LargeListArray *largeListArr = nullptr; const arrow::StructArray *structArr = nullptr; }; From 47762e8a726551f13e295e9657597224b04424aa Mon Sep 17 00:00:00 2001 From: Gavin Halliday Date: Tue, 30 Jul 2024 06:44:26 +0100 Subject: [PATCH 6/9] HPCC-32336 Copy @leakStore option from environment to daliconf.xml Signed-off-by: Gavin Halliday --- initfiles/componentfiles/configxml/dali.xsl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/initfiles/componentfiles/configxml/dali.xsl b/initfiles/componentfiles/configxml/dali.xsl index da7aec45291..b751bf3c863 100644 --- a/initfiles/componentfiles/configxml/dali.xsl +++ b/initfiles/componentfiles/configxml/dali.xsl @@ -148,7 +148,7 @@ dalisds.xml 0 - + From b36a0a6f1b31048e2888389e1f43cd3d4177a62c Mon Sep 17 00:00:00 2001 From: Gavin Halliday Date: Mon, 29 Jul 2024 17:41:02 +0100 Subject: [PATCH 7/9] HPCC-32325 Fix socket object leak from new MP connection Signed-off-by: Gavin Halliday --- system/mp/mpcomm.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/system/mp/mpcomm.cpp b/system/mp/mpcomm.cpp index 7154b61e6e1..72f45bdc5f2 100644 --- a/system/mp/mpcomm.cpp +++ b/system/mp/mpcomm.cpp @@ -2498,7 +2498,7 @@ bool CMPConnectThread::handleAcceptedSocket(ISocket *_sock, unsigned timeoutMs, #endif checkSelfDestruct(&connectHdr.id[0],sizeof(connectHdr.id)); Owned channel = parent->lookup(_remoteep); - if (!channel->attachSocket(sock.getClear(),_remoteep,hostep,false,&rd,addrval)) + if (!channel->attachSocket(sock,_remoteep,hostep,false,&rd,addrval)) { #ifdef _FULLTRACE PROGLOG("MP Connect Thread: lookup failed"); From af1584987f61212cdcae75e3a0ff3c33a01c6d5c Mon Sep 17 00:00:00 2001 From: Jeremy Clements <79224539+jeclrsg@users.noreply.github.com> Date: Tue, 30 Jul 2024 12:32:03 -0400 Subject: [PATCH 8/9] HPCC-32338 ECL Watch v9 LogViewer Message column default width set a reasonable default width for the Message column in LogViewer Signed-off-by: Jeremy Clements <79224539+jeclrsg@users.noreply.github.com> --- esp/src/src-react/components/LogViewer.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/esp/src/src-react/components/LogViewer.tsx b/esp/src/src-react/components/LogViewer.tsx index 677ae267779..82e027a55b4 100644 --- a/esp/src/src-react/components/LogViewer.tsx +++ b/esp/src/src-react/components/LogViewer.tsx @@ -49,7 +49,7 @@ export const LogViewer: React.FunctionComponent = ({ csvFormatter: level => Level[level].toUpperCase() }, id: { label: nlsHPCC.Source, width: 212, sortable: false }, - message: { label: nlsHPCC.Message, sortable: false } + message: { label: nlsHPCC.Message, width: 720, sortable: false } }; }, []); From 076c9835df0149957d6dc662d98557280cf69fb4 Mon Sep 17 00:00:00 2001 From: Gavin Halliday Date: Tue, 30 Jul 2024 12:28:18 +0100 Subject: [PATCH 9/9] HPCC-32300 Move helm version generation into a function Signed-off-by: Gavin Halliday --- helm/hpcc/templates/_helpers.tpl | 6 +++++- helm/hpcc/templates/dafilesrv.yaml | 2 +- helm/hpcc/templates/dali.yaml | 2 +- helm/hpcc/templates/dfuserver.yaml | 2 +- helm/hpcc/templates/eclagent.yaml | 4 ++-- helm/hpcc/templates/eclccserver.yaml | 4 ++-- helm/hpcc/templates/eclscheduler.yaml | 2 +- helm/hpcc/templates/esp.yaml | 2 +- helm/hpcc/templates/localroxie.yaml | 2 +- helm/hpcc/templates/roxie.yaml | 8 ++++---- helm/hpcc/templates/sasha.yaml | 2 +- helm/hpcc/templates/thor.yaml | 10 +++++----- 12 files changed, 25 insertions(+), 21 deletions(-) diff --git a/helm/hpcc/templates/_helpers.tpl b/helm/hpcc/templates/_helpers.tpl index 9cb6d98801e..5e435251a61 100644 --- a/helm/hpcc/templates/_helpers.tpl +++ b/helm/hpcc/templates/_helpers.tpl @@ -1269,6 +1269,10 @@ Pass in dict with .root, .visibility defined {{- end -}} {{- end -}} +{{- define "hpcc.generateHelmVersion" -}} +helmVersion: 9.0.129-closedown0 +{{- end -}} + {{/* A template to generate a service Pass in dict with .root, .name, .service, .defaultPort, .selector defined @@ -1314,7 +1318,7 @@ kind: Service metadata: name: {{ $lvars.serviceName | quote }} labels: - helmVersion: 9.0.129-closedown0 + {{- include "hpcc.generateHelmVersion" . | nindent 4 }} {{- include "hpcc.addStandardLabels" (dict "root" $.root "instance" $lvars.serviceName ) | indent 4 }} {{- if $lvars.labels }} {{ toYaml $lvars.labels | indent 4 }} diff --git a/helm/hpcc/templates/dafilesrv.yaml b/helm/hpcc/templates/dafilesrv.yaml index f1f6fae94a5..94bbb12e759 100644 --- a/helm/hpcc/templates/dafilesrv.yaml +++ b/helm/hpcc/templates/dafilesrv.yaml @@ -50,7 +50,7 @@ spec: labels: {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "dafilesrv" "name" "dafilesrv" "instance" .name) | indent 8 }} server: {{ .name | quote }} - helmVersion: 9.0.129-closedown0 + {{- include "hpcc.generateHelmVersion" . | nindent 8 }} annotations: checksum/config: {{ $configSHA }} spec: diff --git a/helm/hpcc/templates/dali.yaml b/helm/hpcc/templates/dali.yaml index ef05fc36c13..0ad88836567 100644 --- a/helm/hpcc/templates/dali.yaml +++ b/helm/hpcc/templates/dali.yaml @@ -82,7 +82,7 @@ spec: run: {{ $dali.name | quote }} server: {{ $dali.name | quote }} app: dali - helmVersion: 9.0.129-closedown0 + {{- include "hpcc.generateHelmVersion" . | nindent 8 }} {{- if hasKey $.Values.global "metrics" }} {{- include "hpcc.generateMetricsReporterLabel" $.Values.global.metrics | nindent 8 }} {{- end }} diff --git a/helm/hpcc/templates/dfuserver.yaml b/helm/hpcc/templates/dfuserver.yaml index 2f4d65ff071..cb49047653a 100644 --- a/helm/hpcc/templates/dfuserver.yaml +++ b/helm/hpcc/templates/dfuserver.yaml @@ -56,7 +56,7 @@ spec: {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "dfuserver" "name" "dfuserver" "instance" .name) | indent 8 }} run: {{ .name | quote }} accessDali: "yes" - helmVersion: 9.0.129-closedown0 + {{- include "hpcc.generateHelmVersion" . | nindent 8 }} {{- if hasKey . "labels" }} {{ toYaml .labels | indent 8 }} {{- end }} diff --git a/helm/hpcc/templates/eclagent.yaml b/helm/hpcc/templates/eclagent.yaml index 1cfcaffb116..50015189236 100644 --- a/helm/hpcc/templates/eclagent.yaml +++ b/helm/hpcc/templates/eclagent.yaml @@ -58,7 +58,7 @@ data: {{- include "hpcc.addStandardLabels" (dict "root" $ "component" $apptype "name" "eclagent" "instance" $appJobName "instanceOf" (printf "%s-job" .me.name)) | indent 12 }} accessDali: "yes" accessEsp: "yes" - helmVersion: 9.0.129-closedown0 + {{- include "hpcc.generateHelmVersion" . | nindent 12 }} {{- if hasKey .me "labels" }} {{ toYaml .me.labels | indent 12 }} {{- end }} @@ -135,7 +135,7 @@ spec: run: {{ .name | quote }} accessDali: "yes" accessEsp: {{ .useChildProcesses | default false | ternary "yes" "no" | quote }} - helmVersion: 9.0.129-closedown0 + {{- include "hpcc.generateHelmVersion" . | nindent 8 }} {{- if hasKey . "labels" }} {{ toYaml .labels | indent 8 }} {{- end }} diff --git a/helm/hpcc/templates/eclccserver.yaml b/helm/hpcc/templates/eclccserver.yaml index f8f1d287dc0..6a0c4f1ed7b 100644 --- a/helm/hpcc/templates/eclccserver.yaml +++ b/helm/hpcc/templates/eclccserver.yaml @@ -57,7 +57,7 @@ data: {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "eclccserver" "name" "eclccserver" "instance" $compileJobName "instanceOf" (printf "%s-job" .me.name)) | indent 12 }} accessDali: "yes" accessEsp: "yes" - helmVersion: 9.0.129-closedown0 + {{- include "hpcc.generateHelmVersion" . | nindent 12 }} {{- if hasKey .me "labels" }} {{ toYaml .me.labels | indent 12 }} {{- end }} @@ -142,7 +142,7 @@ spec: run: {{ .name | quote }} accessDali: "yes" accessEsp: {{ .useChildProcesses | default false | ternary "yes" "no" | quote }} - helmVersion: 9.0.129-closedown0 + {{- include "hpcc.generateHelmVersion" . | nindent 8 }} {{- if hasKey . "labels" }} {{ toYaml .labels | indent 8 }} {{- end }} diff --git a/helm/hpcc/templates/eclscheduler.yaml b/helm/hpcc/templates/eclscheduler.yaml index bf7a7459964..445c07a5eb4 100644 --- a/helm/hpcc/templates/eclscheduler.yaml +++ b/helm/hpcc/templates/eclscheduler.yaml @@ -64,7 +64,7 @@ spec: run: {{ .name | quote }} accessDali: "yes" accessEsp: "no" - helmVersion: 9.0.129-closedown0 + {{- include "hpcc.generateHelmVersion" . | nindent 8 }} {{- if hasKey . "labels" }} {{ toYaml .labels | indent 8 }} {{- end }} diff --git a/helm/hpcc/templates/esp.yaml b/helm/hpcc/templates/esp.yaml index 30b9481c481..456b21724f7 100644 --- a/helm/hpcc/templates/esp.yaml +++ b/helm/hpcc/templates/esp.yaml @@ -117,7 +117,7 @@ spec: server: {{ .name | quote }} accessDali: "yes" app: {{ $application }} - helmVersion: 9.0.129-closedown0 + {{- include "hpcc.generateHelmVersion" . | nindent 8 }} {{- include "hpcc.addStandardLabels" (dict "root" $ "name" $application "component" "esp" "instance" .name) | indent 8 }} {{- if hasKey $.Values.global "metrics" }} {{- include "hpcc.generateMetricsReporterLabel" $.Values.global.metrics | nindent 8 }} diff --git a/helm/hpcc/templates/localroxie.yaml b/helm/hpcc/templates/localroxie.yaml index e24e601a867..26e43df0a06 100644 --- a/helm/hpcc/templates/localroxie.yaml +++ b/helm/hpcc/templates/localroxie.yaml @@ -70,7 +70,7 @@ spec: server: {{ $servername | quote }} accessDali: "yes" accessEsp: "yes" - helmVersion: 9.0.129-closedown0 + {{- include "hpcc.generateHelmVersion" . | nindent 8 }} {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "roxie-server" "name" "roxie" "instance" $roxie.name) | indent 8 }} {{- if hasKey . "labels" }} {{ toYaml .labels | indent 8 }} diff --git a/helm/hpcc/templates/roxie.yaml b/helm/hpcc/templates/roxie.yaml index 39749b268b5..196f29c0293 100644 --- a/helm/hpcc/templates/roxie.yaml +++ b/helm/hpcc/templates/roxie.yaml @@ -120,7 +120,7 @@ spec: {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "topology-server" "name" "roxie" "instance" $commonCtx.toponame) | indent 8 }} run: {{ $commonCtx.toponame | quote }} roxie-cluster: {{ $roxie.name | quote }} - helmVersion: 9.0.129-closedown0 + {{- include "hpcc.generateHelmVersion" . | nindent 8 }} {{- if hasKey $.Values.global "metrics" }} {{- include "hpcc.generateMetricsReporterLabel" $.Values.global.metrics | nindent 8}} {{- end }} @@ -180,7 +180,7 @@ kind: Service metadata: name: {{ $commonCtx.toponame | quote }} labels: - helmVersion: 9.0.129-closedown0 + {{- include "hpcc.generateHelmVersion" . | nindent 4 }} {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "topology-server" "name" "roxie" "instance" $commonCtx.toponame) | indent 4 }} spec: ports: @@ -242,7 +242,7 @@ spec: roxie-cluster: {{ $roxie.name | quote }} accessDali: "yes" accessEsp: "yes" - helmVersion: 9.0.129-closedown0 + {{- include "hpcc.generateHelmVersion" . | nindent 8 }} {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "roxie-server" "name" "roxie" "instance" $servername) | indent 8 }} {{- if hasKey $.Values.global "metrics" }} {{- include "hpcc.generateMetricsReporterLabel" $.Values.global.metrics | nindent 8}} @@ -347,7 +347,7 @@ spec: roxie-cluster: {{ $roxie.name | quote }} accessDali: "yes" accessEsp: "yes" - helmVersion: 9.0.129-closedown0 + {{- include "hpcc.generateHelmVersion" . | nindent 8 }} {{- if hasKey $.Values.global "metrics" }} {{- include "hpcc.generateMetricsReporterLabel" $.Values.global.metrics | nindent 8}} {{- end }} diff --git a/helm/hpcc/templates/sasha.yaml b/helm/hpcc/templates/sasha.yaml index d1910b6e34a..a7cb5603007 100644 --- a/helm/hpcc/templates/sasha.yaml +++ b/helm/hpcc/templates/sasha.yaml @@ -52,7 +52,7 @@ spec: run: {{ $serviceName | quote }} server: {{ $serviceName | quote }} accessDali: {{ (has "dali" $sasha.access) | ternary "yes" "no" | quote }} - helmVersion: 9.0.129-closedown0 + {{- include "hpcc.generateHelmVersion" . | nindent 8 }} {{- if hasKey $sasha "labels" }} {{ toYaml $sasha.labels | indent 8 }} {{- end }} diff --git a/helm/hpcc/templates/thor.yaml b/helm/hpcc/templates/thor.yaml index bcda109fa22..4584fc4e674 100644 --- a/helm/hpcc/templates/thor.yaml +++ b/helm/hpcc/templates/thor.yaml @@ -82,7 +82,7 @@ data: labels: accessDali: "yes" accessEsp: "yes" - helmVersion: 9.0.129-closedown0 + {{- include "hpcc.generateHelmVersion" . | nindent 12 }} {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "eclagent" "name" "thor" "instance" $eclAgentJobName "instanceOf" (printf "%s-job" .eclAgentName)) | indent 8 }} {{- if hasKey .me "labels" }} {{ toYaml .me.labels | indent 12 }} @@ -147,7 +147,7 @@ data: accessEsp: "yes" app: "thor" component: "thormanager" - helmVersion: 9.0.129-closedown0 + {{- include "hpcc.generateHelmVersion" . | nindent 12 }} instance: "_HPCC_JOBNAME_" job: "_HPCC_JOBNAME_" {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "thormanager" "name" "thor" "instance" $thorManagerJobName "instanceOf" (printf "%s-thormanager-job" .me.name)) | indent 12 }} @@ -214,7 +214,7 @@ data: accessEsp: "yes" app: "thor" component: "thorworker" - helmVersion: 9.0.129-closedown0 + {{- include "hpcc.generateHelmVersion" . | nindent 12 }} instance: "_HPCC_JOBNAME_" job: "_HPCC_JOBNAME_" {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "thorworker" "name" "thor" "instance" $thorWorkerJobName "instanceOf" (printf "%s-thorworker-job" .me.name)) | indent 12 }} @@ -347,7 +347,7 @@ spec: accessEsp: {{ $commonCtx.eclAgentUseChildProcesses | ternary "yes" "no" | quote }} app: "thor" component: "thor-eclagent" - helmVersion: 9.0.129-closedown0 + {{- include "hpcc.generateHelmVersion" . | nindent 8 }} instance: {{ $commonCtx.eclAgentName | quote }} {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "eclagent" "name" "thor" "instance" $commonCtx.eclAgentName ) | indent 8 }} {{- if hasKey $commonCtx.me "labels" }} @@ -412,7 +412,7 @@ spec: accessEsp: "no" app: "thor" component: "thor-thoragent" - helmVersion: 9.0.129-closedown0 + {{- include "hpcc.generateHelmVersion" . | nindent 8 }} instance: {{ $commonCtx.thorAgentName | quote }} {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "eclagent" "name" "thor" "instance" $commonCtx.thorAgentName ) | indent 8 }} {{- if hasKey $commonCtx.me "labels" }}