Merge remote-tracking branch 'origin/candidate-9.8.x'

Signed-off-by: Gavin Halliday <[email protected]>

dali/base/dasds.cpp

@@ -5507,13 +5507,15 @@ class CStoreHelper : implements IStoreHelper, public CInterface
             return true;
         offset_t fSize = iFileIO->size();
         PROGLOG("Loading delta: %s (size=%.2f MB)", filename, ((double)fSize) / 0x100000);
+        size32_t lenDeltaHeader = strlen(deltaHeader);
         MemoryBuffer tmp;
-        char *ptr = (char *) tmp.reserveTruncate(strlen(deltaHeader));
+        char *ptr = (char *) tmp.reserveTruncate(lenDeltaHeader+1);
         unsigned embeddedCrc = 0;
         offset_t pos = 0;
         bool hasCrcHeader = false; // check really only needed for deltas proceeding CRC header
-        if (strlen(deltaHeader) == iFileIO->read(0, strlen(deltaHeader), ptr))
+        if (lenDeltaHeader == iFileIO->read(0, lenDeltaHeader, ptr))
         {
+            ptr[lenDeltaHeader] = '\0';
             if (0 == memicmp(deltaHeader, ptr, 5))
             {
                 pos = deltaHeaderSizeStart;

esp/src/src-react/components/LogViewer.tsx

@@ -49,7 +49,7 @@ export const LogViewer: React.FunctionComponent<LogViewerProps> = ({
                 csvFormatter: level => Level[level].toUpperCase()
             },
             id: { label: nlsHPCC.Source, width: 212, sortable: false },
-            message: { label: nlsHPCC.Message, sortable: false }
+            message: { label: nlsHPCC.Message, width: 720, sortable: false }
         };
     }, []);
 

helm/hpcc/templates/_helpers.tpl

@@ -1476,6 +1476,10 @@ Pass in dict with .root, .visibility defined
 {{- end -}}
 {{- end -}}
 
+{{- define "hpcc.generateHelmVersion" -}}
+helmVersion: 9.9.0-trunk0
+{{- end -}}
+
 {{/*
 A template to generate a service
 Pass in dict with .root, .name, .service, .defaultPort, .selector defined
@@ -1523,7 +1527,7 @@ kind: Service
 metadata:
   name: {{ $lvars.serviceName | quote }}
   labels:
-    helmVersion: 9.9.0-trunk0
+    {{- include "hpcc.generateHelmVersion" . | nindent 4 }}
     {{- include "hpcc.addStandardLabels" (dict "root" $.root "instance" $lvars.serviceName ) | indent 4 }}
 {{- if $lvars.labels }}
 {{ toYaml $lvars.labels | indent 4 }}

helm/hpcc/templates/dafilesrv.yaml

@@ -51,7 +51,7 @@ spec:
       labels:
         {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "dafilesrv" "name" "dafilesrv" "instance" .name) | indent 8 }}
         server: {{ .name | quote }}
-        helmVersion: 9.9.0-trunk0
+        {{- include "hpcc.generateHelmVersion" . | nindent 8 }}
       annotations:
         checksum/config: {{ $configSHA }}
         {{- include "hpcc.generateAnnotations" $commonCtx | indent 8 }}

helm/hpcc/templates/dali.yaml

@@ -89,7 +89,7 @@ spec:
         run: {{ $dali.name | quote }}
         server: {{ $dali.name | quote }}
         app: dali
-        helmVersion: 9.9.0-trunk0
+        {{- include "hpcc.generateHelmVersion" . | nindent 8 }}
 {{- if hasKey $.Values.global "metrics" }}
  {{- include "hpcc.generateMetricsReporterLabel" $.Values.global.metrics | nindent 8 }}
 {{- end }}

helm/hpcc/templates/dfuserver.yaml

@@ -57,7 +57,7 @@ spec:
         {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "dfuserver" "name" "dfuserver" "instance" .name) | indent 8 }}
         run: {{ .name | quote }}
         accessDali: "yes"
-        helmVersion: 9.9.0-trunk0
+        {{- include "hpcc.generateHelmVersion" . | nindent 8 }}
 {{- if hasKey . "labels" }}
 {{ toYaml .labels | indent 8 }}
 {{- end }}

helm/hpcc/templates/eclagent.yaml

@@ -62,7 +62,7 @@ data:
             {{- include "hpcc.addStandardLabels" (dict "root" $ "component" $apptype "name" "eclagent" "instance" $appJobName "instanceOf" (printf "%s-job" .me.name)) | indent 12 }}
             accessDali: "yes"
             accessEsp: "yes"
-            helmVersion: 9.9.0-trunk0
+            {{- include "hpcc.generateHelmVersion" . | nindent 12 }}
 {{- if hasKey .me "labels" }}
 {{ toYaml .me.labels | indent 12 }}
 {{- end }}
@@ -139,7 +139,7 @@ spec:
         run: {{ .name | quote }}
         accessDali: "yes"
         accessEsp: {{ .useChildProcesses | default false | ternary "yes" "no" | quote }}
-        helmVersion: 9.9.0-trunk0
+        {{- include "hpcc.generateHelmVersion" . | nindent 8 }}
 {{- if hasKey . "labels" }}
 {{ toYaml .labels | indent 8 }}
 {{- end }}

helm/hpcc/templates/eclccserver.yaml

@@ -62,7 +62,7 @@ data:
             {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "eclccserver" "name" "eclccserver" "instance" $compileJobName "instanceOf" (printf "%s-job" .me.name)) | indent 12 }}
             accessDali: "yes"
             accessEsp: "yes"
-            helmVersion: 9.9.0-trunk0
+            {{- include "hpcc.generateHelmVersion" . | nindent 12 }}
 {{- if hasKey .me "labels" }}
 {{ toYaml .me.labels | indent 12 }}
 {{- end }}
@@ -147,7 +147,7 @@ spec:
         run: {{ .name | quote }}
         accessDali: "yes"
         accessEsp: {{ .useChildProcesses | default false | ternary "yes" "no" | quote }}
-        helmVersion: 9.9.0-trunk0
+        {{- include "hpcc.generateHelmVersion" . | nindent 8 }}
 {{- if hasKey . "labels" }}
 {{ toYaml .labels | indent 8 }}
 {{- end }}

helm/hpcc/templates/eclscheduler.yaml

@@ -65,7 +65,7 @@ spec:
         run: {{ .name | quote }}
         accessDali: "yes"
         accessEsp: "no"
-        helmVersion: 9.9.0-trunk0
+        {{- include "hpcc.generateHelmVersion" . | nindent 8 }}
 {{- if hasKey . "labels" }}
 {{ toYaml .labels | indent 8 }}
 {{- end }}

helm/hpcc/templates/esp.yaml

@@ -125,7 +125,7 @@ spec:
         accessSasha: "yes"
 {{- end }}
         app: {{ $application }}
-        helmVersion: 9.9.0-trunk0
+        {{- include "hpcc.generateHelmVersion" . | nindent 8 }}
         {{- include "hpcc.addStandardLabels" (dict "root" $ "name" $application "component" "esp" "instance" .name) | indent 8 }}
 {{- if hasKey $.Values.global "metrics" }}
  {{- include "hpcc.generateMetricsReporterLabel" $.Values.global.metrics | nindent 8 }}

helm/hpcc/templates/localroxie.yaml

@@ -73,7 +73,7 @@ spec:
         server: {{ $servername | quote }}
         accessDali: "yes"
         accessEsp: "yes"
-        helmVersion: 9.9.0-trunk0
+        {{- include "hpcc.generateHelmVersion" . | nindent 8 }}
         {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "roxie-server" "name" "roxie" "instance" $roxie.name) | indent 8 }}
 {{-    if hasKey . "labels" }}
 {{      toYaml .labels | indent 8 }}

helm/hpcc/templates/roxie.yaml

@@ -125,7 +125,7 @@ spec:
 {{-    include "hpcc.addStandardLabels" (dict "root" $ "component" "topology-server" "name" "roxie" "instance" $commonCtx.toponame) | indent 8 }}
         run: {{ $commonCtx.toponame | quote }}
         roxie-cluster: {{ $roxie.name | quote }}
-        helmVersion: 9.9.0-trunk0
+        {{- include "hpcc.generateHelmVersion" . | nindent 8 }}
 {{- if hasKey $.Values.global "metrics" }}
 {{-     include "hpcc.generateMetricsReporterLabel"  $.Values.global.metrics | nindent 8}}
 {{-    end }}
@@ -182,7 +182,7 @@ kind: Service
 metadata:
   name: {{ $commonCtx.toponame | quote }}
   labels:
-    helmVersion: 9.9.0-trunk0
+    {{- include "hpcc.generateHelmVersion" . | nindent 4 }}
     {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "topology-server" "name" "roxie" "instance" $commonCtx.toponame) | indent 4 }}
 spec:
   ports:
@@ -244,7 +244,7 @@ spec:
         roxie-cluster: {{ $roxie.name | quote }}
         accessDali: "yes"
         accessEsp: "yes"
-        helmVersion: 9.9.0-trunk0
+        {{- include "hpcc.generateHelmVersion" . | nindent 8 }}
         {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "roxie-server" "name" "roxie" "instance" $servername) | indent 8 }}
 {{-     if hasKey $.Values.global "metrics" }}
 {{-      include "hpcc.generateMetricsReporterLabel" $.Values.global.metrics | nindent 8}}
@@ -352,7 +352,7 @@ spec:
         roxie-cluster: {{ $roxie.name | quote }}
         accessDali: "yes"
         accessEsp: "yes"
-        helmVersion: 9.9.0-trunk0
+        {{- include "hpcc.generateHelmVersion" . | nindent 8 }}
 {{- if hasKey $.Values.global "metrics" }}
 {{-      include "hpcc.generateMetricsReporterLabel" $.Values.global.metrics | nindent 8}}
 {{-     end }}

helm/hpcc/templates/sasha.yaml

@@ -53,7 +53,7 @@ spec:
         server: {{ $serviceName | quote }}
         app: sasha
         accessDali: {{ (has "dali" $sasha.access) | ternary "yes" "no" | quote }}
-        helmVersion: 9.9.0-trunk0
+        {{- include "hpcc.generateHelmVersion" . | nindent 8 }}
 {{- if hasKey $sasha "labels" }}
 {{ toYaml $sasha.labels | indent 8 }}
 {{- end }}

helm/hpcc/templates/thor.yaml

@@ -88,7 +88,7 @@ data:
           labels:
             accessDali: "yes"
             accessEsp: "yes"
-            helmVersion: 9.9.0-trunk0
+            {{- include "hpcc.generateHelmVersion" . | nindent 12 }}
             {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "eclagent" "name" "thor" "instance" $eclAgentJobName "instanceOf" (printf "%s-job" .eclAgentName)) | indent 8 }}
 {{- if hasKey .me "labels" }}
 {{ toYaml .me.labels | indent 12 }}
@@ -153,7 +153,7 @@ data:
             accessEsp: "yes"
             app: "thor"
             component: "thormanager"
-            helmVersion: 9.9.0-trunk0
+            {{- include "hpcc.generateHelmVersion" . | nindent 12 }}
             instance: "_HPCC_JOBNAME_"
             job: "_HPCC_JOBNAME_"
             {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "thormanager" "name" "thor" "instance" $thorManagerJobName "instanceOf" (printf "%s-thormanager-job" .me.name)) | indent 12 }}
@@ -220,7 +220,7 @@ data:
             accessEsp: "yes"
             app: "thor"
             component: "thorworker"
-            helmVersion: 9.9.0-trunk0
+            {{- include "hpcc.generateHelmVersion" . | nindent 12 }}
             instance: "_HPCC_JOBNAME_"
             job: "_HPCC_JOBNAME_"
             {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "thorworker" "name" "thor" "instance" $thorWorkerJobName "instanceOf" (printf "%s-thorworker-job" .me.name)) | indent 12 }}
@@ -353,7 +353,7 @@ spec:
         accessEsp: {{ $commonCtx.eclAgentUseChildProcesses | ternary "yes" "no" | quote }}
         app: "thor"
         component: "thor-eclagent"
-        helmVersion: 9.9.0-trunk0
+        {{- include "hpcc.generateHelmVersion" . | nindent 8 }}
         instance: {{ $commonCtx.eclAgentName | quote }}
         {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "eclagent" "name" "thor" "instance" $commonCtx.eclAgentName ) | indent 8 }}
 {{- if hasKey $commonCtx.me "labels" }}
@@ -418,7 +418,7 @@ spec:
         accessEsp: "no"
         app: "thor"
         component: "thor-thoragent"
-        helmVersion: 9.9.0-trunk0
+        {{- include "hpcc.generateHelmVersion" . | nindent 8 }}
         instance: {{ $commonCtx.thorAgentName | quote }}
         {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "eclagent" "name" "thor" "instance" $commonCtx.thorAgentName ) | indent 8 }}
 {{- if hasKey $commonCtx.me "labels" }}

initfiles/componentfiles/configxml/dali.xsl

@@ -148,7 +148,7 @@
       <xsl:element name="SDS">
         <xsl:attribute name="store">dalisds.xml</xsl:attribute>
         <xsl:attribute name="caseInsensitive">0</xsl:attribute>
-        <xsl:copy-of select="@nobackup | @recoverFromIncErrors | @snmpSendWarnings | @enableSNMP | @enableSysLog | @snmpErrorMsgLevel | @msgLevel | @lightweightCoalesce | @keepStores | @deltaSaveThresholdSecs | @deltaTransactionQueueLimit | @deltaTransactionMaxMemMB"/>
+        <xsl:copy-of select="@nobackup | @recoverFromIncErrors | @snmpSendWarnings | @enableSNMP | @enableSysLog | @snmpErrorMsgLevel | @msgLevel | @lightweightCoalesce | @keepStores | @deltaSaveThresholdSecs | @deltaTransactionQueueLimit | @deltaTransactionMaxMemMB | @leakStore"/>
         <xsl:if test="string(@IdlePeriod) != ''">
             <xsl:attribute name="lCIdlePeriod">
                 <xsl:value-of select="@IdlePeriod"/>

plugins/parquet/parquetembed.cpp

@@ -714,7 +714,7 @@ std::shared_ptr<arrow::NestedType> ParquetWriter::makeChildRecord(const RtlField
         const RtlFieldInfo childFieldInfo = RtlFieldInfo("", "", child);
         std::vector<std::shared_ptr<arrow::Field>> childField;
         reportIfFailure(fieldToNode(&childFieldInfo, childField));
-        return std::make_shared<arrow::ListType>(childField[0]);
+        return std::make_shared<arrow::LargeListType>(childField[0]);
     }
 }
 
@@ -845,8 +845,8 @@ void ParquetWriter::beginSet(const RtlFieldInfo *field)
     arrow::FieldPath match = getNestedFieldBuilder(field, childBuilder);
     fieldBuilderStack.push_back(std::make_shared<ArrayBuilderTracker>(field, childBuilder, CPNTSet, std::move(match)));
 
-    arrow::ListBuilder *listBuilder = static_cast<arrow::ListBuilder *>(childBuilder);
-    reportIfFailure(listBuilder->Append());
+    arrow::LargeListBuilder *largeListBuilder = static_cast<arrow::LargeListBuilder *>(childBuilder);
+    reportIfFailure(largeListBuilder->Append());
 }
 
 /**
@@ -946,7 +946,7 @@ arrow::ArrayBuilder *ParquetWriter::getFieldBuilder(const RtlFieldInfo *field)
         return recordBatchBuilder->GetField(schema->GetFieldIndex(fieldName.str()));
     }
     else if (fieldBuilderStack.back()->nodeType == CPNTSet)
-        return static_cast<arrow::ListBuilder *>(fieldBuilderStack.back()->structPtr)->value_builder();
+        return static_cast<arrow::LargeListBuilder *>(fieldBuilderStack.back()->structPtr)->value_builder();
     else
         return fieldBuilderStack.back()->structPtr->child(fieldBuilderStack.back()->childrenProcessed++);
 }
@@ -1463,6 +1463,12 @@ void ParquetRowBuilder::processBeginSet(const RtlFieldInfo *field, bool &isAll)
         newPathNode.childCount = arrayVisitor->listArr->value_slice(currentRow)->length();
         pathStack.push_back(newPathNode);
     }
+    else if (arrayVisitor->type == LargeListType)
+    {
+        ParquetColumnTracker newPathNode(field, arrayVisitor->largeListArr, CPNTSet);
+        newPathNode.childCount = arrayVisitor->largeListArr->value_slice(currentRow)->length();
+        pathStack.push_back(newPathNode);
+    }
     else
     {
         failx("Error reading nested set with name %s.", field->name);
@@ -1585,8 +1591,20 @@ void ParquetRowBuilder::nextFromStruct(const RtlFieldInfo *field)
     }
     else if (pathStack.back().nodeType == CPNTSet)
     {
-        auto child = arrayVisitor->listArr->value_slice(currentRow);
-        reportIfFailure(child->Accept(arrayVisitor.get()));
+        if (arrayVisitor->type == ListType)
+        {
+            auto child = arrayVisitor->listArr->value_slice(currentRow);
+            reportIfFailure(child->Accept(arrayVisitor.get()));
+        }
+        else if (arrayVisitor->type == LargeListType)
+        {
+            auto child = arrayVisitor->largeListArr->value_slice(currentRow);
+            reportIfFailure(child->Accept(arrayVisitor.get()));
+        }
+        else
+        {
+            failx("Unexpected type in CPNTSet: neither ListType nor LargeListType");
+        }
     }
 }
 

plugins/parquet/parquetembed.hpp

@@ -131,6 +131,7 @@ enum ParquetArrayType
     LargeBinaryType,
     DecimalType,
     ListType,
+    LargeListType,
     StructType,
     RealType
 };
@@ -314,6 +315,12 @@ class ParquetArrayVisitor : public arrow::ArrayVisitor
         type = ListType;
         return arrow::Status::OK();
     }
+    arrow::Status Visit(const arrow::LargeListArray &array)
+    {
+        largeListArr = &array;
+        type = LargeListType;
+        return arrow::Status::OK();
+    }
     arrow::Status Visit(const arrow::StructArray &array)
     {
         structArr = &array;
@@ -348,6 +355,7 @@ class ParquetArrayVisitor : public arrow::ArrayVisitor
     const arrow::Decimal128Array *decArr = nullptr;
     const arrow::Decimal256Array *largeDecArr = nullptr;
     const arrow::ListArray *listArr = nullptr;
+    const arrow::LargeListArray *largeListArr = nullptr;
     const arrow::StructArray *structArr = nullptr;
 };
 

system/jlib/jlog.cpp

@@ -275,6 +275,11 @@ void LogMsgJobInfo::deserialize(MemoryBuffer & in)
 {
 // kludge for backward compatibility of pre 8.0 clients that send a LogMsgJobId: (_uint64), not a string
 // NB: jobID pre 8.0 was redundant as always equal to UnknownJob
+    if (isDeserialized)
+    {
+        free((void *) jobIDStr);
+        jobIDStr = nullptr;
+    }
     dbgassertex(in.remaining() >= sizeof(LogMsgJobId)); // should always be at least this amount, because userID follows the jobID
     if (0 == memcmp(in.toByteArray()+in.getPos(), &UnknownJob, sizeof(LogMsgJobId))) // pre 8.0 client
     {

system/jlib/jmisc.hpp

@@ -342,7 +342,7 @@ template<unsigned NUMSTATES, bool reportOther> class TimeDivisionTracker
 protected:
     unsigned __int64 totals[NUMSTATES] = {0};
     unsigned counts[NUMSTATES] = {0};
-    const char *stateNames[NUMSTATES];
+    const char *stateNames[NUMSTATES] = {};
     unsigned __int64 lastTick = get_cycles_now();
     unsigned currentState = 0;
     StringAttr name;

system/jlib/jstats.cpp

@@ -1302,6 +1302,13 @@ class CComponentStatistics
 
 //--------------------------------------------------------------------------------------------------------------------
 
+//This object is accessed by all StatisticsMapping constructors.  It is important that it is initialised before any of
+//them are called.  Coverity routinely complains about possible GLOBAL_INIT_ORDER problems.  These are false positives
+//as long as
+//   * this definition of allStatsMappings comes first in this file before any StatisticsMappings
+//   * there must be no other files in jlib that declare a StatisticMapping.
+// StatisticsMappings in other dlls are ok because it is guaranteed that the dependent dll/so is initialised first
+
 static std::unordered_map<unsigned, const StatisticsMapping *> allStatsMappings;
 
 static int compareUnsigned(unsigned const * left, unsigned const * right)

system/mp/mpcomm.cpp

@@ -2508,7 +2508,7 @@ bool CMPConnectThread::handleAcceptedSocket(CConnectSelectHandler::CSocketHandle
         checkSelfDestruct(&connectHdr.id[0],sizeof(connectHdr.id));
         Owned<CMPChannel> channel = parent->lookup(_remoteep);
         size32_t rd = sizeof(connectHdr);
-        if (!channel->attachSocket(LINK(sock),_remoteep,hostep,false,&rd,addrval))
+        if (!channel->attachSocket(sock,_remoteep,hostep,false,&rd,addrval))
         {
 #ifdef _FULLTRACE       
             PROGLOG("MP Connect Thread: lookup failed");