Merge remote-tracking branch 'origin/candidate-9.4.x'

Signed-off-by: Gordon Smith <[email protected]> # Conflicts: # helm/hpcc/Chart.yaml # helm/hpcc/templates/_helpers.tpl # helm/hpcc/templates/dafilesrv.yaml # helm/hpcc/templates/dali.yaml # helm/hpcc/templates/dfuserver.yaml # helm/hpcc/templates/eclagent.yaml # helm/hpcc/templates/eclccserver.yaml # helm/hpcc/templates/eclscheduler.yaml # helm/hpcc/templates/esp.yaml # helm/hpcc/templates/localroxie.yaml # helm/hpcc/templates/roxie.yaml # helm/hpcc/templates/sasha.yaml # helm/hpcc/templates/thor.yaml # version.cmake
hpcc-systems · Mar 7, 2024 · 91ea460 · 91ea460
2 parents 80adbd9 + 5f47101
commit 91ea460
Show file tree

Hide file tree

Showing 15 changed files with 305 additions and 117 deletions.
diff --git a/dali/ft/filecopy.cpp b/dali/ft/filecopy.cpp
@@ -3573,17 +3573,30 @@ void FileSprayer::updateTargetProperties()
             }
 
             // Update @writeCost and @numWrites in subfile properties and update totalWriteCost
-            if (superTgt)
+            if (superTgt && superTgt->numSubFiles() > 0)
             {
                 if (cur.whichOutput != (unsigned)-1)
                 {
-                    unsigned targetPartNum = targets.item(cur.whichOutput).partNum;
-                    IDistributedFile &subfile = superTgt->querySubFile(targetPartNum, true);
-                    DistributedFilePropertyLock lock(&subfile);
+                    IDistributedFile *subfile;
+                    if (superTgt->numSubFiles() > 1)
+                    {
+                        Owned<IFileDescriptor> fDesc = superTgt->getFileDescriptor();
+                        ISuperFileDescriptor *superFDesc = fDesc->querySuperFileDescriptor();
+                        unsigned subfileNum, subFilePartNum;
+                        superFDesc->mapSubPart(targets.item(cur.whichOutput).partNum, subfileNum, subFilePartNum);
+                        subfile = superTgt->querySubPart(subfileNum, subFilePartNum);
+                    }
+                    else
+                    {
+                        // If there is a single subfile, it is not necessary to map part to subfile
+                        // (also, querySuperFileDescriptor return nullptr if num subfile == 1)
+                        subfile = &superTgt->querySubFile(0);
+                    }
+                    DistributedFilePropertyLock lock(subfile);
                     IPropertyTree &subFileProps = lock.queryAttributes();
                     cost_type prevNumWrites = subFileProps.getPropInt64(getDFUQResultFieldName(DFUQRFnumDiskWrites));
                     cost_type prevWriteCost = subFileProps.getPropInt64(getDFUQResultFieldName(DFUQRFwriteCost));
-                    cost_type curWriteCost = calcFileAccessCost(&subfile, curProgress.numWrites, 0);
+                    cost_type curWriteCost = calcFileAccessCost(subfile, curProgress.numWrites, 0);
                     subFileProps.setPropInt64(getDFUQResultFieldName(DFUQRFwriteCost), prevWriteCost + curWriteCost);
                     subFileProps.setPropInt64(getDFUQResultFieldName(DFUQRFnumDiskWrites), prevNumWrites + curProgress.numWrites);
                     totalWriteCost += curWriteCost;
@@ -3786,23 +3799,35 @@ void FileSprayer::updateTargetProperties()
     if (distributedSource)
     {
         IDistributedSuperFile * superSrc = distributedSource->querySuperFile();
-        if (superSrc)
+        if (superSrc && superSrc->numSubFiles() > 0)
         {
+            Owned<IFileDescriptor> fDesc = superSrc->getFileDescriptor();
+            ISuperFileDescriptor *superFDesc = fDesc->querySuperFileDescriptor();
             ForEachItemIn(idx, partition)
             {
                 PartitionPoint & cur = partition.item(idx);
                 OutputProgress & curProgress = progress.item(idx);
-
                 if (cur.whichInput != (unsigned)-1)
                 {
-                    unsigned sourcePartNum = sources.item(cur.whichInput).partNum;
-                    IDistributedFile &subfile = superSrc->querySubFile(sourcePartNum, true);
-                    DistributedFilePropertyLock lock(&subfile);
+                    IDistributedFile *subfile;
+                    if (superFDesc)
+                    {
+                        unsigned subfileNum, subFilePartNum;
+                        superFDesc->mapSubPart(sources.item(cur.whichInput).partNum, subfileNum, subFilePartNum);
+                        subfile = superSrc->querySubPart(subfileNum, subFilePartNum);
+                    }
+                    else
+                    {
+                        // superFDesc==nullptr if there is a single file
+                        // so query the first (and only) subfile
+                        subfile = &superSrc->querySubFile(0);
+                    }
+                    DistributedFilePropertyLock lock(subfile);
                     IPropertyTree &subFileProps = lock.queryAttributes();
                     stat_type prevNumReads = subFileProps.getPropInt64(getDFUQResultFieldName(DFUQRFnumDiskReads), 0);
-                    cost_type legacyReadCost = getLegacyReadCost(subfile.queryAttributes(), &subfile);
+                    cost_type legacyReadCost = getLegacyReadCost(subfile->queryAttributes(), subfile);
                     cost_type prevReadCost = subFileProps.getPropInt64(getDFUQResultFieldName(DFUQRFreadCost), 0);
-                    cost_type curReadCost = calcFileAccessCost(&subfile, 0, curProgress.numReads);
+                    cost_type curReadCost = calcFileAccessCost(subfile, 0, curProgress.numReads);
                     subFileProps.setPropInt64(getDFUQResultFieldName(DFUQRFnumDiskReads), prevNumReads + curProgress.numReads);
                     subFileProps.setPropInt64(getDFUQResultFieldName(DFUQRFreadCost), legacyReadCost + prevReadCost + curReadCost);
                     totalReadCost += curReadCost;

diff --git a/docs/EN_US/HPCCClientTools/CT_Mods/CT_Overview.xml b/docs/EN_US/HPCCClientTools/CT_Mods/CT_Overview.xml
@@ -129,22 +129,6 @@
 
         <listitem>
           <para>Install the client tools software to your machine.</para>
-
-          <variablelist>
-            <varlistentry>
-              <term>Note:</term>
-
-              <listitem>
-                <para>For large ECL workunits, the 32-bit compiler (eclcc)
-                could run out of memory quicker than the 64-bit version.
-                Therefore in most cases, you should install the 64-bit
-                version. However, for machines with 4GB memory or less, you
-                should use the 32-bit client tools. If you have both versions
-                on your local machine, the IDE will give preference to the
-                64-bit version.</para>
-              </listitem>
-            </varlistentry>
-          </variablelist>
         </listitem>
       </orderedlist>
 

diff --git a/docs/EN_US/HPCCClientTools/CT_Mods/CT_Overview_withoutIDE.xml b/docs/EN_US/HPCCClientTools/CT_Mods/CT_Overview_withoutIDE.xml
@@ -112,20 +112,6 @@
 
         <listitem>
           <para>Install the client tools software to your machine.</para>
-
-          <para><variablelist>
-              <varlistentry>
-                <term>Note:</term>
-
-                <listitem>
-                  <para>For large ECL workunits, the 32-bit compiler (eclcc)
-                  could run out of memory quicker than the 64-bit version.
-                  Therefore in most cases, you should install the 64-bit
-                  version. However, for machines with 4GB memory or less, you
-                  should use the 32-bit client tools.</para>
-                </listitem>
-              </varlistentry>
-            </variablelist></para>
         </listitem>
       </orderedlist>
 

diff --git a/ecl/hql/hqlrepository.cpp b/ecl/hql/hqlrepository.cpp
@@ -827,11 +827,11 @@ IEclSourceCollection * EclRepositoryManager::resolveGitCollection(const char * r
     {
         if (options.updateRepos)
         {
-            unsigned retCode = runGitCommand(nullptr, "fetch origin", repoPath, true);
+            unsigned retCode = runGitCommand(nullptr, "fetch origin --prune", repoPath, true);
             if (retCode != 0)
             {
                 VStringBuffer msg("Failed to download the latest version of '%s' error code (%u)", defaultUrl, retCode);
-                error.setown(createError(CategoryError, SeverityError, ERR_FAIL_UPDATE_REPO, msg.str(), nullptr, 0, 0, 0));
+                error.setown(createError(CategoryError, SeverityWarning, ERR_FAIL_UPDATE_REPO, msg.str(), nullptr, 0, 0, 0));
             }
         }
 

diff --git a/esp/src/src-react/components/Frame.tsx b/esp/src/src-react/components/Frame.tsx
@@ -91,7 +91,7 @@ export const Frame: React.FunctionComponent<FrameProps> = () => {
     }, []);
 
     React.useEffect(() => {
-        document.title = `${showEnvironmentTitle && environmentTitle.length ? environmentTitle : "ECL Watch "}${locationPathname.split("/").join(" | ")}`;
+        document.title = `${(showEnvironmentTitle && environmentTitle) ? environmentTitle : "ECL Watch "}${locationPathname.split("/").join(" | ")}`;
     }, [environmentTitle, locationPathname, showEnvironmentTitle]);
 
     React.useEffect(() => {

diff --git a/esp/src/src-react/components/Title.tsx b/esp/src/src-react/components/Title.tsx
@@ -247,11 +247,6 @@ export const DevTitle: React.FunctionComponent<DevTitleProps> = ({
         }
     }, [currentUser]);
 
-    React.useEffect(() => {
-        if (!environmentTitle) return;
-        document.title = environmentTitle;
-    }, [environmentTitle]);
-
     return <div style={{ backgroundColor: titlebarColorSet ? titlebarColor : theme.palette.themeLight }}>
         <BannerMessageBar />
         <Stack horizontal verticalAlign="center" horizontalAlign="space-between">
@@ -264,7 +259,7 @@ export const DevTitle: React.FunctionComponent<DevTitleProps> = ({
                         <Link href="#/activities">
                             <Text variant="large" nowrap block >
                                 <b title="ECL Watch" style={{ color: titlebarColorSet ? Utility.textColor(titlebarColor) : theme.palette.themeDarker }}>
-                                    {showEnvironmentTitle && environmentTitle.length ? environmentTitle : "ECL Watch"}
+                                    {(showEnvironmentTitle && environmentTitle) ? environmentTitle : "ECL Watch"}
                                 </b>
                             </Text>
                         </Link>

diff --git a/helm/hpcc/templates/_helpers.tpl b/helm/hpcc/templates/_helpers.tpl
@@ -1175,7 +1175,9 @@ Generate instance queue names
 {{- range $auxQueue := $auxThorQueues }}
  {{- $combinedList = append $combinedList $auxQueue -}}
 {{- end -}}
-{{- toYaml $combinedList -}}
+{{- if $combinedList -}}
+ {{- toYaml $combinedList -}}
+{{- end -}}
 {{- end -}}
 
 {{- define "hpcc.usesRemoteIssuer" -}}

diff --git a/plugins/eeproxies/lib_addrclean.ecllib b/plugins/eeproxies/lib_addrclean.ecllib
@@ -30,4 +30,5 @@ export AddrCleanLib := SERVICE : plugin('addrcleanlib'),time
   string34 Zip9ToGeo34(const string zip9, const string server = '', unsigned2 port = 0) : c, pure, context, entrypoint='aclZip9ToGeo34Ctx';
   boolean SetTracing(boolean trace=true) : c, action, context, entrypoint='aclSetTracingCtx';
   boolean SetTiming(boolean timing=true) : c, action, context, entrypoint='aclSetTimingCtx';
+  CleanUtil(unsigned4 flags = 0) : c, pure, context, entrypoint='aclCleanUtilCtx';
 END;
diff --git a/roxie/ccd/ccdlistener.cpp b/roxie/ccd/ccdlistener.cpp
@@ -949,9 +949,9 @@ void ContextLogger::exportStatsToSpan(bool failed, stat_type elapsedNs, unsigned
 {
     if (activeSpan->isRecording())
     {
+        activeSpan->setSpanStatus(failed);
         setSpanAttribute("time_elapsed", elapsedNs);
-        if (failed)
-            setSpanAttribute("num_failures", 1);
+
         if (memused)
             setSpanAttribute("size_peak_row_memory", memused * 0x100000);
 
@@ -1316,7 +1316,7 @@ class RoxieWorkUnitWorker : public RoxieQueryWorker
             reportUnknownException(wu, logctx);
         }
 #endif
-        unsigned elapsedNs = nsTick() - startNs;
+        stat_type elapsedNs = nsTick() - startNs;
         unsigned elapsedMs = nanoToMilli(elapsedNs);
         noteQuery(failed, elapsedMs, priority);
         queryFactory->noteQuery(startTime, failed, elapsedMs, memused, agentsReplyLen, 0);

diff --git a/roxie/ccd/ccdmain.cpp b/roxie/ccd/ccdmain.cpp
@@ -1560,7 +1560,7 @@ int CCD_API roxie_main(int argc, const char *argv[], const char * defaultYaml)
 
                                 const char *passPhrase = roxieFarm.queryProp("@passphrase");
                                 if (!isEmptyString(passPhrase))
-                                    decrypt(passPhraseStr, passPhrase);
+                                    passPhraseStr.append(passPhrase); // NB passphrase is decrypted in CSecureSocketContext::createNewContext()
 
                                 const char *certFile = roxieFarm.queryProp("@certificateFileName");
                                 if (!certFile)

diff --git a/rtl/eclrtl/rtltype.cpp b/rtl/eclrtl/rtltype.cpp
@@ -62,7 +62,7 @@ void rtlPadTruncString(unsigned & tgtLen, char * & tgt, unsigned newLen, unsigne
 {
     if (len > newLen)
         len = newLen;
-    char * buff = (char *)malloc(len);
+    char * buff = (char *)malloc(newLen);
     memcpy(buff, src, len);
     if (len < newLen)
         memset(buff+len, ' ', newLen - len);

diff --git a/system/jhtree/jhtree.cpp b/system/jhtree/jhtree.cpp
@@ -626,7 +626,7 @@ class CNodeCacheEntry : public CInterface
 public:
     CriticalSection cs;
 private:
-    std::atomic<const CJHTreeNode *> node = nullptr;
+    std::atomic<const CJHTreeNode *> node{nullptr};
 public:
     ~CNodeCacheEntry()
     {