Merge remote-tracking branch 'origin/candidate-9.4.x'

Signed-off-by: Jake Smith <[email protected]>
hpcc-systems · Feb 22, 2024 · 2371a21 · 2371a21
2 parents fb740b6 + 97d8396
commit 2371a21
Show file tree

Hide file tree

Showing 43 changed files with 781 additions and 286 deletions.
diff --git a/.github/workflows/build-assets.yml b/.github/workflows/build-assets.yml
@@ -157,7 +157,7 @@ jobs:
           path: ${{ needs.preamble.outputs.folder_platform }}
 
       - name: Checkout LN
-        if: ${{ matrix.ln }}
+        if: ${{ matrix.ln || matrix.ee }} 
         uses: actions/checkout@v4
         with:
           repository: ${{ github.repository_owner }}/LN

diff --git a/common/thorhelper/thorcommon.hpp b/common/thorhelper/thorcommon.hpp
@@ -457,6 +457,10 @@ class CStatsContextLogger : public CSimpleInterfaceOf<IContextLogger>
     {
         activeSpan->setSpanAttribute(name, value);
     }
+    virtual void setSpanAttribute(const char *name, __uint64 value) const override
+    {
+        activeSpan->setSpanAttribute(name, value);
+    }
     virtual const char *queryGlobalId() const override
     {
         return activeSpan->queryGlobalId();

diff --git a/dali/base/dadfs.cpp b/dali/base/dadfs.cpp
@@ -9934,6 +9934,7 @@ class CInitGroups
     StringArray clusternames;
     unsigned defaultTimeout;
     bool machinesLoaded;
+    bool writeLock;
 
     GroupType getGroupType(const char *type)
     {
@@ -10009,6 +10010,8 @@ class CInitGroups
 
     void addClusterGroup(const char *name, IPropertyTree *newClusterGroup, bool realCluster)
     {
+        if (!writeLock)
+            throw makeStringException(0, "CInitGroups::addClusterGroup called in read-only mode");
         VStringBuffer prop("Group[@name=\"%s\"]", name);
         IPropertyTree *root = groupsconnlock.conn->queryRoot();
         IPropertyTree *old = root->queryPropTree(prop.str());
@@ -10366,11 +10369,12 @@ class CInitGroups
         return root->queryPropTree(xpath.str());
     }
 public:
-    CInitGroups(unsigned _defaultTimeout)
-        : groupsconnlock("constructGroup",SDS_GROUPSTORE_ROOT,true,false,false,_defaultTimeout)
+    CInitGroups(unsigned _defaultTimeout, bool _writeLock)
+        : groupsconnlock("constructGroup",SDS_GROUPSTORE_ROOT,_writeLock,false,false,_defaultTimeout)
     {
         defaultTimeout = _defaultTimeout;
         machinesLoaded = false;
+        writeLock = _writeLock;
     }
 
     IPropertyTree *queryCluster(const IPropertyTree *env, const char *_clusterName, const char *type, const char *msg, StringBuffer &messages)
@@ -10707,13 +10711,13 @@ class CInitGroups
 
 void initClusterGroups(bool force, StringBuffer &response, IPropertyTree *oldEnvironment, unsigned timems)
 {
-    CInitGroups init(timems);
+    CInitGroups init(timems, true);
     init.constructGroups(force, response, oldEnvironment);
 }
 
 void initClusterAndStoragePlaneGroups(bool force, IPropertyTree *oldEnvironment, unsigned timems)
 {
-    CInitGroups init(timems);
+    CInitGroups init(timems, true);
 
     StringBuffer response;
     init.constructGroups(force, response, oldEnvironment);
@@ -10728,19 +10732,19 @@ void initClusterAndStoragePlaneGroups(bool force, IPropertyTree *oldEnvironment,
 
 bool resetClusterGroup(const char *clusterName, const char *type, bool spares, StringBuffer &response, unsigned timems)
 {
-    CInitGroups init(timems);
+    CInitGroups init(timems, true);
     return init.resetClusterGroup(clusterName, type, spares, response);
 }
 
 bool addClusterSpares(const char *clusterName, const char *type, const std::vector<std::string> &hosts, StringBuffer &response, unsigned timems)
 {
-    CInitGroups init(timems);
+    CInitGroups init(timems, true);
     return init.addSpares(clusterName, type, hosts, response);
 }
 
 bool removeClusterSpares(const char *clusterName, const char *type, const std::vector<std::string> &hosts, StringBuffer &response, unsigned timems)
 {
-    CInitGroups init(timems);
+    CInitGroups init(timems, true);
     return init.removeSpares(clusterName, type, hosts, response);
 }
 
@@ -10764,7 +10768,7 @@ static IGroup *getClusterNodeGroup(const char *clusterName, const char *type, bo
      * to DFS and elsewhere.
      */
     Owned<IGroup> nodeGroup = queryNamedGroupStore().lookup(nodeGroupName);
-    CInitGroups init(timems);
+    CInitGroups init(timems, false);
     Owned<IGroup> expandedClusterGroup = init.getGroupFromCluster(type, cluster, true);
     if (!expandedClusterGroup)
         throwStringExceptionV(0, "Failed to get group for '%s' cluster '%s'", type, clusterName);

diff --git a/docs/EN_US/ContainerizedHPCC/ContainerizedMods/ContainerLogging.xml b/docs/EN_US/ContainerizedHPCC/ContainerizedMods/ContainerLogging.xml
@@ -630,7 +630,48 @@ ContainerLog
       (1) up to "report all messages" (100). The default log level is rather
       high (80) and should be adjusted accordingly.</para>
 
-      <para>For example, verbosity should be medium for all components:</para>
+      <para>These are the available log levels:</para>
+
+      <para><itemizedlist>
+          <listitem>
+            <para>CriticalMsgThreshold = 1;</para>
+          </listitem>
+
+          <listitem>
+            <para>FatalMsgThreshold = 1;</para>
+          </listitem>
+
+          <listitem>
+            <para>ErrMsgThreshold = 10;</para>
+          </listitem>
+
+          <listitem>
+            <para>WarnMsgThreshold = 20;</para>
+          </listitem>
+
+          <listitem>
+            <para>AudMsgThreshold = 30;</para>
+          </listitem>
+
+          <listitem>
+            <para>ProgressMsgThreshold = 50;</para>
+          </listitem>
+
+          <listitem>
+            <para>InfoMsgThreshold = 60;</para>
+          </listitem>
+
+          <listitem>
+            <para>DebugMsgThreshold = 80;</para>
+          </listitem>
+
+          <listitem>
+            <para>ExtraneousMsgThreshold = 90;</para>
+          </listitem>
+        </itemizedlist></para>
+
+      <para>For example, to show only progress and lower level (more critical)
+      messages set the verbosity to 50:</para>
 
       <programlisting>helm install myhpcc ./hpcc --set global.logging.detail="50" </programlisting>
     </sect2>

diff --git a/docs/EN_US/ECLLanguageReference/ECLR_mods/BltInFunc-DISTRIBUTE.xml b/docs/EN_US/ECLLanguageReference/ECLR_mods/BltInFunc-DISTRIBUTE.xml
@@ -104,16 +104,19 @@
           <row>
             <entry><emphasis>maxskew</emphasis></entry>
 
-            <entry>A floating point number in the range of zero (0.0) to one
-            (1.0) specifying the minimum skew to allow (0.1=10%).</entry>
+            <entry>A value between zero (0) and one (1.0 = 100%) indicating
+            the maximum percentage of skew to allow before the job fails (the
+            default skew is 1.0 / &lt;number of worker nodes on
+            cluster&gt;).</entry>
           </row>
 
           <row>
             <entry><emphasis>skewlimit</emphasis></entry>
 
-            <entry>Optional. A floating point number in the range of zero
-            (0.0) to one (1.0) specifying the maximum skew to allow
-            (0.1=10%).</entry>
+            <entry>Optional. A value between zero (0) and one (1.0 = 100%)
+            indicating the desired maximum percentage of skew to allow (the
+            default skew is 1.0 / &lt;number of worker nodes on
+            cluster&gt;).</entry>
           </row>
 
           <row>

diff --git a/docs/EN_US/ECLLanguageReference/ECLR_mods/BltInFunc-JOIN.xml b/docs/EN_US/ECLLanguageReference/ECLR_mods/BltInFunc-JOIN.xml
@@ -534,15 +534,17 @@
 
             <entry>A value between zero (0) and one (1.0 = 100%) indicating
             the maximum percentage of skew to allow before the job fails (the
-            default is 0.1 = 10%).</entry>
+            default skew is 1.0 / &lt;number of worker nodes on
+            cluster&gt;).</entry>
           </row>
 
           <row>
             <entry><emphasis>target</emphasis></entry>
 
             <entry>Optional. A value between zero (0) and one (1.0 = 100%)
             indicating the desired maximum percentage of skew to allow (the
-            default is 0.1 = 10%).</entry>
+            default skew is 1.0 / &lt;number of worker nodes on
+            cluster&gt;).</entry>
           </row>
 
           <row>

diff --git a/docs/EN_US/ECLLanguageReference/ECLR_mods/BltInFunc-SORT.xml b/docs/EN_US/ECLLanguageReference/ECLR_mods/BltInFunc-SORT.xml
@@ -102,15 +102,17 @@
 
             <entry>A value between zero (0) and one (1.0 = 100%) indicating
             the maximum percentage of skew to allow before the job fails (the
-            default is 0.1 = 10%).</entry>
+            default skew is 1.0 / &lt;number of worker nodes on
+            cluster&gt;).</entry>
           </row>
 
           <row>
             <entry><emphasis>target</emphasis></entry>
 
             <entry>Optional. A value between zero (0) and one (1.0 = 100%)
             indicating the desired maximum percentage of skew to allow (the
-            default is 0.1 = 10%).</entry>
+            default skew is 1.0 / &lt;number of worker nodes on
+            cluster&gt;).</entry>
           </row>
 
           <row>

diff --git a/ecl/eclcc/eclcc.cpp b/ecl/eclcc/eclcc.cpp
@@ -1602,9 +1602,9 @@ void EclCC::processSingleQuery(const EclRepositoryManager & localRepositoryManag
     {
         const BlockIoStats summaryIo = systemIoFinishInfo->querySummaryStats() - systemIoStartInfo->querySummaryStats();
         if (summaryIo.rd_sectors)
-            updateWorkunitStat(instance.wu, SSToperation, scopeName, StSizeOsDiskRead, NULL, summaryIo.rd_sectors * summaryIo.getSectorSize());
+            updateWorkunitStat(instance.wu, SSToperation, scopeName, StSizeDiskRead, NULL, summaryIo.rd_sectors * summaryIo.getSectorSize());
         if (summaryIo.wr_sectors)
-            updateWorkunitStat(instance.wu, SSToperation, scopeName, StSizeOsDiskWrite, NULL, summaryIo.wr_sectors * summaryIo.getSectorSize());
+            updateWorkunitStat(instance.wu, SSToperation, scopeName, StSizeDiskWrite, NULL, summaryIo.wr_sectors * summaryIo.getSectorSize());
     }
 }
 

diff --git a/esp/applications/eclservices/ldap_authorization_map.yaml b/esp/applications/eclservices/ldap_authorization_map.yaml
@@ -84,6 +84,9 @@ ldap:
          -  path: SashaAccess
             resource: SashaAccess
             description: Access to WS Sasha service
+         -  path: PackageMapAccess
+            resource: PackageMapAccess
+            description: Access to packagemaps
       WsWorkunits:
          Feature:
          -  path: OwnWorkunitsAccess
@@ -143,7 +146,11 @@ ldap:
          -  path: ConfigAccess
             resource: ConfigAccess
             description: Access to super computer environment
-      WsPackageProcess: []
+      WsPackageProcess:
+         Feature:
+         -  path: PackageMapAccess
+            resource: PackageMapAccess
+            description: Access to packagemaps
       ws_account: []
       ws_access: []
       ws_esdlconfig: []

diff --git a/esp/applications/eclwatch/ldap_authorization_map.yaml b/esp/applications/eclwatch/ldap_authorization_map.yaml
@@ -85,6 +85,9 @@ ldap:
          -  path: SashaAccess
             resource: SashaAccess
             description: Access to WS Sasha service
+         -  path: PackageMapAccess
+            resource: PackageMapAccess
+            description: Access to packagemaps
       WsWorkunits:
          Feature:
          -  path: OwnWorkunitsAccess
@@ -154,7 +157,11 @@ ldap:
          -  path: ResourceQueryAccess
             resource: ResourceQueryAccess
             description: Access to resources
-      WsPackageProcess: []
+      WsPackageProcess:
+         Feature:
+         -  path: PackageMapAccess
+            resource: PackageMapAccess
+            description: Access to packagemaps
       ws_account: []
       ws_access: []
       ws_esdlconfig: []

diff --git a/esp/bindings/http/platform/httpservice.cpp b/esp/bindings/http/platform/httpservice.cpp
@@ -153,6 +153,33 @@ static bool authenticateOptionalFailed(IEspContext& ctx, IEspHttpBinding* bindin
     return false;
 }
 
+static bool checkHttpPathStaysWithinBounds(const char *path)
+{
+    if (isEmptyString(path))
+        return true;
+    //The path that follows /esp/files should be relative, not absolute - reject immediately if it is.
+    if (isAbsolutePath(path))
+        return false;
+    int depth = 0;
+    StringArray nodes;
+    nodes.appendList(path, "/");
+    ForEachItemIn(i, nodes)
+    {
+        const char *node = nodes.item(i);
+        if (!*node || streq(node, ".")) //empty or "." doesn't advance
+            continue;
+        if (!streq(node, ".."))
+            depth++;
+        else
+        {
+            depth--;
+            if (depth<0)  //only really care that the relative http path doesn't position itself above its own root node
+                return false;
+        }
+    }
+    return true;
+}
+
 EspHttpBinding* CEspHttpServer::getBinding()
 {
     EspHttpBinding* thebinding=NULL;
@@ -417,7 +444,16 @@ int CEspHttpServer::processRequest()
                         m_response->redirect(*m_request.get(),url);
                     }
                     else
+                    {
+                        if (strieq(methodName.str(), "files_") && !checkHttpPathStaysWithinBounds(pathEx))
+                        {
+                            AERRLOG("Get File %s: attempted access outside of %sfiles/", pathEx.str(), getCFD());
+                            m_response->setStatus(HTTP_STATUS_NOT_FOUND);
+                            m_response->send();
+                            return 0;
+                        }
                         thebinding->onGet(m_request.get(), m_response.get());
+                    }
                 }
                 else
                     unsupported();
@@ -733,33 +769,6 @@ static void httpGetDirectory(CHttpRequest* request, CHttpResponse* response, con
     response->send();
 }
 
-static bool checkHttpPathStaysWithinBounds(const char *path)
-{
-    if (!path || !*path)
-        return true;
-    //The path that follows /esp/files should be relative, not absolute - reject immediately if it is.
-    if (isAbsolutePath(path))
-        return false;
-    int depth = 0;
-    StringArray nodes;
-    nodes.appendList(path, "/");
-    ForEachItemIn(i, nodes)
-    {
-        const char *node = nodes.item(i);
-        if (!*node || streq(node, ".")) //empty or "." doesn't advance
-            continue;
-        if (!streq(node, ".."))
-            depth++;
-        else
-        {
-            depth--;
-            if (depth<0)  //only really care that the relative http path doesn't position itself above its own root node
-                return false;
-        }
-    }
-    return true;
-}
-
 int CEspHttpServer::onGetFile(CHttpRequest* request, CHttpResponse* response, const char *urlpath)
 {
         if (!request || !response || !urlpath)
@@ -810,6 +819,14 @@ int CEspHttpServer::onGetXslt(CHttpRequest* request, CHttpResponse* response, co
         if (!request || !response || !path)
             return -1;
 
+        if (!checkHttpPathStaysWithinBounds(path))
+        {
+            AERRLOG("Get File %s: attempted access outside of %sxslt/", path, getCFD());
+            response->setStatus(HTTP_STATUS_NOT_FOUND);
+            response->send();
+            return 0;
+        }
+
         StringBuffer mimetype, etag, lastModified;
         MemoryBuffer content;
         bool modified = true;

diff --git a/esp/platform/espp.cpp b/esp/platform/espp.cpp
@@ -364,6 +364,20 @@ void initializeMetrics(CEspConfig* config)
         pMetricsTree.setown(getComponentConfigSP()->getPropTree("metrics"));
     }
 
+#ifdef _DEBUG
+    if (!pMetricsTree)
+    {
+        const char * defaultMetricsYaml = R"!!(
+sinks:
+  - type: log
+    name: loggingsink
+    settings:
+      period: 60
+)!!";
+        pMetricsTree.setown(createPTreeFromYAMLString(defaultMetricsYaml));
+    }
+#endif
+
     if (pMetricsTree != nullptr)
     {
         PROGLOG("Metrics initializing...");