Merge remote-tracking branch 'origin/candidate-9.6.x' into candidate-…

…9.6.0

Signed-off-by: Gordon Smith <[email protected]>

.github/workflows/build-assets.yml

@@ -248,7 +248,7 @@ jobs:
           sudo rm -f ${{ needs.preamble.outputs.folder_build }}/CMakeCache.txt
           sudo rm -rf ${{ needs.preamble.outputs.folder_build }}/CMakeFiles
           docker run --rm --mount ${{ needs.preamble.outputs.mount_platform }} --mount ${{ needs.preamble.outputs.mount_build }} ${{ steps.vars.outputs.docker_tag_candidate_base }} "\
-            cmake -S /hpcc-dev/HPCC-Platform -B /hpcc-dev/build -DMAKE_DOCS_ONLY=ON -DUSE_NATIVE_LIBRARIES=ON -DDOCS_AUTO=ON -DDOC_LANGS=ALL && \
+            cmake -S /hpcc-dev/HPCC-Platform -B /hpcc-dev/build ${{ needs.preamble.outputs.cmake_docker_config }} -DSIGN_MODULES_PASSPHRASE=${{ secrets.SIGN_MODULES_PASSPHRASE }} -DSIGN_MODULES_KEYID=${{ secrets.SIGN_MODULES_KEYID }} -DMAKE_DOCS_ONLY=ON -DUSE_NATIVE_LIBRARIES=ON -DDOCS_AUTO=ON -DDOC_LANGS=ALL && \
             cmake --build /hpcc-dev/build --parallel $(nproc) --target all"
           docker run --rm --mount ${{ needs.preamble.outputs.mount_platform }} --mount ${{ needs.preamble.outputs.mount_build }} ${{ steps.vars.outputs.docker_tag_candidate_base }} "cd /hpcc-dev/build/Release/docs/EN_US && zip ALL_HPCC_DOCS_EN_US-${{ needs.preamble.outputs.community_tag }}.zip *.pdf"
           docker run --rm --mount ${{ needs.preamble.outputs.mount_platform }} --mount ${{ needs.preamble.outputs.mount_build }} ${{ steps.vars.outputs.docker_tag_candidate_base }} "cd /hpcc-dev/build/Release/docs/PT_BR && zip ALL_HPCC_DOCS_PT_BR-${{ needs.preamble.outputs.community_tag }}.zip *.pdf"

dali/server/daldap.cpp

@@ -124,15 +124,14 @@ class CDaliLdapConnection: implements IDaliLdapConnection, public CInterface
         }
         else
         {
-            DBGLOG("NULL UserDescriptor in daldap.cpp getPermissions('%s')", key ? key : "NULL");
+            DBGLOG("NULL UserDescriptor in daldap.cpp getPermissions('%s')", key);
         }
 
         if (0 == username.length())
         {
             username.append(filesdefaultuser);
             decrypt(password, filesdefaultpassword);
-            OWARNLOG("Missing credentials, injecting deprecated filesdefaultuser for request %s %s", nullText(key),
-                     nullText(obj));
+            OWARNLOG("Missing credentials, injecting deprecated filesdefaultuser for request %s %s", key, nullText(obj));
             logNullUser(nullptr);
         }
 
@@ -148,15 +147,15 @@ class CDaliLdapConnection: implements IDaliLdapConnection, public CInterface
         if (perm == SecAccess_Unavailable)
         {
             OWARNLOG("LDAP: getPermissions(%s) Unable to get perms for scope=%s user=%s, setting 'SecAccess_None'",
-                     nullText(key), nullText(obj), username.str());
+                     key, nullText(obj), username.str());
             perm = SecAccess_None;
         }
         unsigned taken = msTick() - start;
 #ifndef _DEBUG
         if (taken>100)
 #endif
         {
-            PROGLOG("LDAP: getPermissions(%s) scope=%s user=%s returns %d in %d ms", nullText(key), nullText(obj),
+            PROGLOG("LDAP: getPermissions(%s) scope=%s user=%s returns %d in %d ms", key, nullText(obj),
                     username.str(), perm, taken);
         }
         if (auditflags & DALI_LDAP_AUDIT_REPORT)
@@ -176,6 +175,7 @@ class CDaliLdapConnection: implements IDaliLdapConnection, public CInterface
         }
         return perm;
     }
+
     bool clearPermissionsCache(IUserDescriptor *udesc)
     {
         if (!ldapsecurity || ((getLDAPflags() & DLF_ENABLED) == 0))

docs/EN_US/ContainerizedHPCC/ContainerizedMods/ContainerLogging.xml

@@ -527,8 +527,8 @@ myelk-kibana LoadBalancer 10.110.129.199 localhost 5601:31465/TCP 68m
         <para>Example use:</para>
 
         <programlisting>helm install myhpcc hpcc/hpcc \
-  -f HPCC-Platform/helm/examples/azure/log-analytics/loganalytics-hpcc-logaccess.yaml \
-  -f HPCC-Platform/helm/examples/az</programlisting>
+  -f HPCC-Platform/helm/examples/azure/log-analytics/loganalytics-hpcc-logaccess.yaml
+</programlisting>
       </sect3>
     </sect2>
 

docs/EN_US/ContainerizedHPCC/ContainerizedMods/CustomConfig.xml

@@ -619,6 +619,158 @@ eclagent:
         </listitem>
       </orderedlist>
     </sect2>
+
+    <sect2 id="threeiThorsOneQueue" role="nobrk">
+      <title>Create a Custom Configuration Chart for Multiple Thors Listening
+      to a Common Queue</title>
+
+      <para>In this section, we will create three Thors that listen to a
+      common queue (in addition to their own queue). This provides the ability
+      to define distinct Thor cluster configurations but allow them to form a
+      single target behind a single queue. These clusters can be bound to
+      certain node pools in different availability zones, if desired. You can
+      use this example as a starting point and adjust the number of Thor
+      clusters you want.</para>
+
+      <para>This is accomplished by defining additional auxiliary target
+      queues for each Thor definition and using a common name as an auxiliary
+      queue.</para>
+
+      <orderedlist>
+        <listitem>
+          <para>If you have not already added the HPCC Systems repository to
+          the helm repository list, add it now.</para>
+
+          <para><programlisting>helm repo add hpcc https://hpcc-systems.github.io/helm-chart/</programlisting></para>
+
+          <para>If you have added it, update to the latest charts:</para>
+
+          <para><programlisting>helm repo update</programlisting></para>
+        </listitem>
+
+        <listitem>
+          <para>Create a new text file and name it <emphasis
+          role="bold">threethorsonequeue.yaml</emphasis> and open it in a text
+          editor.</para>
+
+          <para>You can use any text editor.</para>
+        </listitem>
+
+        <listitem>
+          <para>Open the default values file that you saved earlier
+          (myvalues.yaml) in a text editor.</para>
+        </listitem>
+
+        <listitem>
+          <para>Copy the entire <emphasis role="bold">thor:</emphasis> section
+          and paste it into the new threethorsonequeue.yaml file.</para>
+        </listitem>
+
+        <listitem>
+          <para>Copy the entire contents of the new yaml file, except the
+          first line (thor:), and paste it at the end of the file <emphasis
+          role="bold">twice</emphasis>.</para>
+
+          <para>This creates three - name: sections.</para>
+        </listitem>
+
+        <listitem>
+          <para>Edit the file in the following manner:</para>
+
+          <para><orderedlist>
+              <listitem>
+                <para>Give each Thor a unique value for <emphasis
+                role="bold">name:</emphasis>.</para>
+
+                <para>In this example, we use <emphasis role="bold">thor1,
+                thor2, </emphasis>and <emphasis
+                role="bold">thor3</emphasis>.</para>
+              </listitem>
+
+              <listitem>
+                <para>Add an <emphasis role="bold">auxQueues:</emphasis> entry
+                to each Thor block using a common name</para>
+
+                <para>In this example, we are using</para>
+
+                <para><emphasis role="bold">auxQueues: [ thorQ
+                ]</emphasis></para>
+              </listitem>
+
+              <listitem>
+                <para>Make sure the <emphasis role="bold">prefix:</emphasis>
+                is the same in each Thor block.</para>
+              </listitem>
+            </orderedlist></para>
+        </listitem>
+
+        <listitem>
+          <para>Save the file and close the text editor.</para>
+
+          <para>The resulting threethorsonequeue.yaml file should look like
+          this:</para>
+
+          <para><emphasis role="bold">Note:</emphasis> The comments have been
+          removed to simplify the example:</para>
+
+          <para><programlisting>thor:
+- name: thor1
+  auxQueues: [ thorQ ]
+  maxGraphs: 2
+  maxJobs: 2
+  numWorkers: 4
+  numWorkersPerPod: 2
+  prefix: thor
+- name: thor2
+  maxGraphs: 2
+  maxJobs: 2
+  numWorkers: 4
+  numWorkersPerPod: 2
+  prefix: thor
+  auxQueues: [ thorQ ]
+- name: thor3
+  maxGraphs: 2
+  maxJobs: 2
+  numWorkers: 4
+  numWorkersPerPod: 2
+  prefix: thor
+  auxQueues: [ thorQ ]</programlisting></para>
+        </listitem>
+      </orderedlist>
+
+      <para><emphasis role="bold">Deploy using the new custom configuration
+      chart.</emphasis></para>
+
+      <orderedlist>
+        <listitem>
+          <para>Open a terminal and navigate to the folder where you saved the
+          threethorsonequeue.yaml file.</para>
+        </listitem>
+
+        <listitem>
+          <para>Deploy your HPCC Systems Platform, adding the new
+          configuration to your command:</para>
+
+          <para><programlisting># If you have previously stopped your cluster
+
+helm install mycluster hpcc/hpcc -f threethorsonequeue.yaml
+
+# To upgrade without stopping
+
+helm upgrade mycluster hpcc/hpcc -f threethorsonequeue.yaml
+</programlisting></para>
+        </listitem>
+
+        <listitem>
+          <para>After you confirm that your deployment is running, open ECL
+          Watch.</para>
+
+          <para>You should see four Thor clusters available as Targets --
+          thor1, thor2, thor3, and a fourth queue that all three Thors listen
+          to-- thorQ.</para>
+        </listitem>
+      </orderedlist>
+    </sect2>
   </sect1>
 
   <sect1 id="CostTracking1">

esp/scm/ws_topology.ecm

@@ -24,6 +24,20 @@ ESPenum RoxieQueueFilter : string
     WorkunitsOnly("WorkunitsOnly")
 };
 
+ESPenum TpMachineType : string
+{
+    All("ALLNODES"),
+    Thor("THORMACHINES"),
+    Hole("HOLEMACHINES"),
+    Roxie("ROXIEMACHINES"),
+    Machines("MACHINES"),
+    Available("AVAILABLEMACHINES"),
+    DropZone("DROPZONE"),
+    StandBy("STANDBYNNODE"),
+    ThorSpare("THORSPARENODES"),
+    HoleStandby("HOLESTANDBYNODES")
+};
+
 //  ===========================================================================
 
 ESPStruct TpMachine
@@ -418,7 +432,7 @@ ESPresponse [exceptions_inline,encode(0)] TpClusterInfoResponse
 ESPrequest [] TpMachineQueryRequest
 {
     //Type of Node the parent is....
-    string        Type;
+    ESPenum TpMachineType Type;
     string        Cluster;
     string        OldIP;
     //Path to Parent within Dali(Encoded to prevent back-engineering)
@@ -433,7 +447,7 @@ ESPresponse [exceptions_inline] TpMachineQueryResponse
 {
     bool   EnablePreflightInfo;
     [min_ver("1.15")] bool   HasThorSpareProcess;
-    string        Type;
+    ESPenum TpMachineType Type;
     string        Cluster;
     string        OldIP;
     string       LogDirectory;

esp/services/ws_topology/ws_topologyService.cpp

@@ -1536,12 +1536,13 @@ bool CWsTopologyEx::onTpMachineQuery(IEspContext &context, IEspTpMachineQueryReq
         const char* directory = req.getDirectory();
 
         bool hasThorSpareProcess = false;
-        const char* type = req.getType();
-        if (!type || !*type || (strcmp(eqAllNodes,type) == 0))
+        const CTpMachineType type = req.getType();
+//        if (!type || !*type || (strcmp(eqAllNodes,type) == 0))
+        if (type == TpMachineType_Undefined || type == CTpMachineType_All )
         {
-            m_TpWrapper.getClusterMachineList(version, eqTHORMACHINES, path, directory, MachineList, hasThorSpareProcess);
-            m_TpWrapper.getClusterMachineList(version, eqHOLEMACHINES, path, directory, MachineList, hasThorSpareProcess);
-            m_TpWrapper.getClusterMachineList(version, eqROXIEMACHINES,path, directory, MachineList, hasThorSpareProcess);
+            m_TpWrapper.getClusterMachineList(version, CTpMachineType_Thor, path, directory, MachineList, hasThorSpareProcess);
+            m_TpWrapper.getClusterMachineList(version, CTpMachineType_Hole, path, directory, MachineList, hasThorSpareProcess);
+            m_TpWrapper.getClusterMachineList(version, CTpMachineType_Roxie,path, directory, MachineList, hasThorSpareProcess);
         }
         else
         {

esp/smc/SMCLib/TpContainer.cpp

@@ -41,7 +41,7 @@ const char* MSG_FAILED_GET_ENVIRONMENT_INFO = "Failed to get environment informa
 //////////////////////////////////////////////////////////////////////
 
 void CTpWrapper::getClusterMachineList(double clientVersion,
-                                       const char* ClusterType,
+                                       CTpMachineType ClusterType,
                                        const char* ClusterPath,
                                        const char* ClusterDirectory,
                                        IArrayOf<IEspTpMachine> &MachineList,

esp/smc/SMCLib/TpWrapper.cpp

@@ -55,7 +55,7 @@ IPropertyTree* CTpWrapper::getEnvironment(const char* xpath)
 }
 
 void CTpWrapper::getClusterMachineList(double clientVersion,
-                                       const char* ClusterType,
+                                       CTpMachineType ClusterType,
                                        const char* ClusterPath,
                                        const char* ClusterDirectory,
                                        IArrayOf<IEspTpMachine> &MachineList,
@@ -68,7 +68,7 @@ void CTpWrapper::getClusterMachineList(double clientVersion,
         getAttPath(ClusterPath,path);
         set<string> machineNames; //used for checking duplicates
 
-        if (strcmp(eqTHORMACHINES,ClusterType) == 0)
+        if (ClusterType == CTpMachineType_Thor)
         {
             bool multiSlaves = false;
             getMachineList(clientVersion, eqThorMasterProcess, path.str(), "", ClusterDirectory, MachineList);
@@ -81,41 +81,41 @@ void CTpWrapper::getClusterMachineList(double clientVersion,
             if (!checkMultiSlavesFlag(ClusterName) &&(count < MachineList.length()))
                 hasThorSpareProcess = true;
         }
-        else if (strcmp(eqHOLEMACHINES,ClusterType) == 0)
+        else if (ClusterType == CTpMachineType_Hole)
         {
             getMachineList(clientVersion, eqHoleSocketProcess, path.str(), "", ClusterDirectory, MachineList);
             getMachineList(clientVersion, eqHoleProcessorProcess, path.str(), "", ClusterDirectory, MachineList);
             getMachineList(clientVersion, eqHoleControlProcess, path.str(), "", ClusterDirectory, MachineList);
             getMachineList(clientVersion, eqHoleCollatorProcess, path.str(), "", ClusterDirectory, MachineList);
             getMachineList(clientVersion, eqHoleStandbyProcess, path.str(), "", ClusterDirectory, MachineList);
         }
-        else if (strcmp(eqROXIEMACHINES,ClusterType) == 0)
+        else if (ClusterType == CTpMachineType_Roxie)
         {
             getMachineList(clientVersion, "RoxieServerProcess", path.str(), "", ClusterDirectory, MachineList, &machineNames);
         }
-        else if (strcmp(eqMACHINES,ClusterType) == 0)
+        else if (ClusterType == CTpMachineType_Machines)
         {
             //load a list of available machines.......
             getMachineList(clientVersion, "Computer", "/Environment/Hardware", "", ClusterDirectory, MachineList);
         }
-        else if (strcmp("AVAILABLEMACHINES",ClusterType) == 0)
+        else if (ClusterType == CTpMachineType_Available)
         {
             getMachineList(clientVersion, "Computer", "/Environment/Hardware", eqMachineAvailablability, ClusterDirectory, MachineList);
         }
-        else if (strcmp("DROPZONE",ClusterType) == 0)
+        else if (ClusterType == CTpMachineType_DropZone)
         {
             getDropZoneMachineList(clientVersion, false, MachineList);
         }
-        else if (strcmp("STANDBYNNODE",ClusterType) == 0)
+        else if (ClusterType == CTpMachineType_StandBy)
         {
             getThorSpareMachineList(clientVersion, ClusterName, ClusterDirectory, MachineList);
             getMachineList(clientVersion, eqHoleStandbyProcess, path.str(), "", ClusterDirectory, MachineList);
         }
-        else if (strcmp("THORSPARENODES",ClusterType) == 0)
+        else if (ClusterType == CTpMachineType_ThorSpare)
         {
             getThorSpareMachineList(clientVersion, ClusterName, ClusterDirectory, MachineList);
         }
-        else if (strcmp("HOLESTANDBYNODES",ClusterType) == 0)
+        else if (ClusterType == CTpMachineType_HoleStandby)
         {
             getMachineList(clientVersion, eqHoleStandbyProcess, path.str(), "", ClusterDirectory, MachineList);
         }
@@ -756,23 +756,23 @@ void CTpWrapper::queryTargetClusterProcess(double version, const char* processNa
     OS_TYPE os = OS_WINDOWS;
     unsigned int clusterTypeLen = strlen(clusterType);
     const char* childType = NULL;
-    const char* clusterType0 = NULL;
+    CTpMachineType clusterType0 = TpMachineType_Undefined;
     if (clusterTypeLen > 4)
     {
         if (!strnicmp(clusterType, "roxie", 4))
         {
             childType = "RoxieServerProcess[1]";
-            clusterType0 = eqROXIEMACHINES;
+            clusterType0 = CTpMachineType_Roxie;
         }
         else if (!strnicmp(clusterType, "thor", 4))
         {
             childType = "ThorMasterProcess";
-            clusterType0 = eqTHORMACHINES;
+            clusterType0 = CTpMachineType_Thor;
         }
         else
         {
             childType = "HoleControlProcess";
-            clusterType0 = eqHOLEMACHINES;
+            clusterType0 = CTpMachineType_Hole;
         }
     }
 
@@ -800,7 +800,7 @@ void CTpWrapper::queryTargetClusterProcess(double version, const char* processNa
     }
     clusterInfo->setOS(os);
 
-    if (clusterType0 && *clusterType0)
+    if (clusterType0 != TpMachineType_Undefined)
     {
         bool hasThorSpareProcess = false;
         IArrayOf<IEspTpMachine> machineList;