Merge remote-tracking branch 'origin/candidate-9.4.x'

Signed-off-by: Gordon Smith <[email protected]> # Conflicts: # helm/hpcc/Chart.yaml # helm/hpcc/templates/_helpers.tpl # helm/hpcc/templates/dafilesrv.yaml # helm/hpcc/templates/dali.yaml # helm/hpcc/templates/dfuserver.yaml # helm/hpcc/templates/eclagent.yaml # helm/hpcc/templates/eclccserver.yaml # helm/hpcc/templates/eclscheduler.yaml # helm/hpcc/templates/esp.yaml # helm/hpcc/templates/localroxie.yaml # helm/hpcc/templates/roxie.yaml # helm/hpcc/templates/sasha.yaml # helm/hpcc/templates/thor.yaml # version.cmake
hpcc-systems · Oct 25, 2023 · 910338f · 910338f
2 parents e22c97b + ef9fd44
commit 910338f
Show file tree

Hide file tree

Showing 32 changed files with 769 additions and 95 deletions.
diff --git a/.github/workflows/build-assets.yml b/.github/workflows/build-assets.yml
@@ -34,6 +34,7 @@ jobs:
       internal_ref: ${{ steps.vars.outputs.internal_ref }}
       community_tag: ${{ steps.vars.outputs.community_tag }}
       internal_tag: ${{ steps.vars.outputs.internal_tag }}
+      hpcc_version: ${{ steps.vars.outputs.hpcc_version }}
       candidate_base_branch: ${{ steps.vars.outputs.candidate_base_branch }}
       candidate_branch: ${{ steps.vars.outputs.candidate_branch }}
       cmake_docker_config: ${{ steps.vars.outputs.cmake_docker_config }}
@@ -54,6 +55,7 @@ jobs:
           community_tag=$(echo $community_ref | cut -d'/' -f3)
           echo "community_tag=$community_tag" >> $GITHUB_OUTPUT
           echo "internal_tag=$(echo $community_tag | sed 's/community/internal/')" >> $GITHUB_OUTPUT
+          echo "hpcc_version=$(echo $community_tag | sed 's/community_//' | sed 's/-[0-9]$//')" >> $GITHUB_OUTPUT
           community_base_ref=${{ github.event.base_ref || github.ref }}
           candidate_branch=$(echo $community_base_ref | cut -d'/' -f3)
           echo "candidate_branch=$candidate_branch" >> $GITHUB_OUTPUT
@@ -90,7 +92,6 @@ jobs:
     strategy:
       matrix:
         include:
-          - os: ubuntu-23.04
           - os: ubuntu-22.04
           - os: ubuntu-22.04
             name: k8s
@@ -269,6 +270,14 @@ jobs:
           username: ${{ secrets.JFROG_USERNAME }}
           password: ${{ secrets.JFROG_PASSWORD }}
 
+      - name: Setup JFrog CLI (internal)
+        if: ${{ matrix.ln && matrix.container && github.repository_owner == 'hpcc-systems' }}
+        uses: jfrog/setup-jfrog-cli@v3
+        env:
+          JF_URL: https://${{ secrets.JFROG_REGISTRY }}
+          JF_USER: ${{ secrets.JFROG_USERNAME }}
+          JF_PASSWORD: ${{ secrets.JFROG_PASSWORD }}
+
       - name: CMake Packages (internal)
         if: ${{ matrix.ln && !matrix.container && !matrix.documentation }}
         run: |
@@ -323,21 +332,27 @@ jobs:
           echo "$k8s_pkg_file"
 
       - name: Create Docker Image (internal)
-        uses: docker/build-push-action@v4
+        uses: docker/build-push-action@v5
         if: ${{ matrix.ln && matrix.container && !matrix.documentation }}
         with:
           builder: ${{ steps.buildx.outputs.name }}
           file: ${{ needs.preamble.outputs.folder_platform }}/dockerfiles/vcpkg/platform-core-${{ matrix.os }}/Dockerfile
           context: ${{ needs.preamble.outputs.folder_build }}
-          push: ${{ github.repository_owner == 'hpcc-systems' }}
+          push: false
+          load: true
           build-args: |
             PKG_FILE=${{ steps.ln-container.outputs.k8s_pkg_file }}
           tags: |
-            ${{ secrets.JFROG_REGISTRY || 'dummy.io' }}/ln/platform-core-${{ matrix.os }}:${{ needs.preamble.outputs.internal_tag }}
-            ${{ secrets.JFROG_REGISTRY || 'dummy.io' }}/ln/platform-core-${{ matrix.os }}:${{ needs.preamble.outputs.candidate_base_branch }}
+            ${{ secrets.JFROG_REGISTRY || 'dummy.io' }}/hpccpl-docker-local/platform-core-ln:${{ needs.preamble.outputs.hpcc_version }}
           cache-from: |
             type=registry,ref=hpccsystems/platform-core-${{ matrix.os }}:${{ needs.preamble.outputs.candidate_base_branch }}
 
+      - name: JFrog Docker Push and Publish
+        if: ${{ matrix.ln && matrix.container && !matrix.documentation && github.repository_owner == 'hpcc-systems' }}
+        run: |
+          jf docker push ${{ secrets.JFROG_REGISTRY || 'dummy.io' }}/hpccpl-docker-local/platform-core-ln:${{ needs.preamble.outputs.hpcc_version }} --build-name=platform-core-ln --build-number=${{ needs.preamble.outputs.hpcc_version }} --project=hpccpl
+          jf rt bp platform-core-ln ${{ needs.preamble.outputs.hpcc_version }} --project=hpccpl
+
       #  Common  ---
       - name: Cleanup Environment
         if: always()

diff --git a/dali/base/dautils.cpp b/dali/base/dautils.cpp
@@ -855,6 +855,8 @@ bool CDfsLogicalFileName::normalizeExternal(const char * name, StringAttr &res,
             normalizeScope(s1, s1, ns1-s1, planeName, strict, false);
 
             str.append("::").append(planeName);
+            //Allow wildcards in plane path
+            allowWild = true;
             break;
         }
         case lfntype_remote:

diff --git a/dockerfiles/vcpkg/platform-core-ubuntu-22.04/Dockerfile b/dockerfiles/vcpkg/platform-core-ubuntu-22.04/Dockerfile
@@ -33,7 +33,6 @@ RUN apt-get clean -y && \
     expect \
     g++ \
     git \
-    git-lfs \
     locales \
     jq \
     openssh-client \
@@ -59,10 +58,14 @@ RUN apt-get install -y \
     gdb \
     nano 
 
-RUN curl -LO https://storage.googleapis.com/kubernetes-release/release/v1.27.6/bin/linux/amd64/kubectl && \
+RUN curl -LO https://storage.googleapis.com/kubernetes-release/release/v1.28.3/bin/linux/amd64/kubectl && \
     chmod +x ./kubectl && \
     mv ./kubectl /usr/local/bin
 
+RUN curl -LO https://packagecloud.io/github/git-lfs/packages/debian/bullseye/git-lfs_3.4.0_amd64.deb/download && \
+    dpkg -i download && \
+    rm download
+
 # Set the locale
 RUN locale-gen en_US.UTF-8
 ENV LANG en_US.UTF-8  

diff --git a/docs/EN_US/HPCCClientTools/CT_Mods/CT_ECL_CLI.xml b/docs/EN_US/HPCCClientTools/CT_Mods/CT_ECL_CLI.xml
@@ -165,6 +165,13 @@
                 <entry>returns a list of all the key user IDs that can be used
                 by the sign command.</entry>
               </row>
+
+              <row>
+                <entry>url-secret-name</entry>
+
+                <entry>generate a secret name from a URL for automatic URL
+                mapping</entry>
+              </row>
             </tbody>
           </tgroup>
         </informaltable>
@@ -8609,6 +8616,64 @@ ecl zapgen W20171018-091399 --path ~ --inc-thor-slave-logs --description "Unexpe
           </informaltable></para>
       </sect2>
 
+      <sect2 id="CT-url-secret-name" role="brk">
+        <title>ecl url-secret-name</title>
+
+        <para><emphasis role="bold">ecl url-secret-name url</emphasis>
+        [--<emphasis>username</emphasis>=&lt;username&gt;]</para>
+
+        <para><informaltable colsep="1" frame="all" rowsep="1">
+            <tgroup cols="2">
+              <colspec align="left" colwidth="125.55pt" />
+
+              <colspec />
+
+              <tbody>
+                <row>
+                  <entry>ecl url-secret-name</entry>
+
+                  <entry>Generates a secret name from a URL for automatic URL
+                  mapping.</entry>
+                </row>
+
+                <row>
+                  <entry>url</entry>
+
+                  <entry>The URL to convert into a secret name.</entry>
+                </row>
+
+                <row>
+                  <entry><emphasis>--username</emphasis></entry>
+
+                  <entry>Optional. The username to associate with the URL.
+                  This will override any username embedded in the URL.</entry>
+                </row>
+              </tbody>
+            </tgroup>
+          </informaltable>The <emphasis>ecl url-secret-name </emphasis>command
+        generates a secret name from a URL that can be used to support ECL
+        SOAPCALL/HTTPCALL automated URL to Secret mapping. </para>
+
+        <para>A username can either be embedded in the URL, such as
+        https://[email protected], or passed in as a parameter using the
+        --username=username option. If a username is passed in as a parameter
+        it overrides a username in the URL. </para>
+
+        <para>Passwords embedded in the URL are not needed and will be
+        ignored. </para>
+
+        <para>When ECL SOAPCALL URL secret mapping is enabled SOAPCALL will
+        convert the URL provided into a name of this format. ECL will then
+        attempt to lookup the secret, and if found, will use the contents of
+        the secret, rather then the original URL.</para>
+
+        <para>Examples:</para>
+
+        <programlisting lang="bash">ecl url-secret-name https://example.com --username jimi
+ecl url-secret-name http://example.com --username jimi
+</programlisting>
+      </sect2>
+
       <sect2 id="CT-RCTL-MEMLOCK" role="brk">
         <title>ecl roxie memlock</title>
 

diff --git a/docs/EN_US/HPCCSystemAdmin/SA-Mods/SecMgrMod.xml b/docs/EN_US/HPCCSystemAdmin/SA-Mods/SecMgrMod.xml
@@ -7,7 +7,7 @@
   <para>The Security Manager Plugin framework provides a mechanism for the
   creation and deployment of custom security manager plugins.</para>
 
-  <sect1 id="how-to-create-a-security-manager-plugin">
+  <sect1 id="how-to-create-a-security-manager-plugin" role="nobrk">
     <title>Plugin Development</title>
 
     <para>A custom Security Manager Plugin consists of a <emphasis
@@ -44,9 +44,9 @@
       </itemizedlist></para>
 
     <para><emphasis role="bold">Buildset definition</emphasis> - The plugin
-    declares itself as an HPCC Systems<superscript>®</superscript> Security Manager Plugin component, and declares
-    the location of the plugin files and the configuration definition
-    schema.</para>
+    declares itself as an HPCC Systems<superscript>®</superscript> Security
+    Manager Plugin component, and declares the location of the plugin files
+    and the configuration definition schema.</para>
 
     <para><emphasis role="bold">EXAMPLE:</emphasis></para>
 
@@ -67,8 +67,8 @@
     <para><emphasis role="bold">Configuration Definition</emphasis> - The
     plugin must provide a definition of the configuration elements and the
     structure it expects to receive at the time it is instantiated. The XSD
-    file is consumed by the HPCC Systems Configuration Manager component and is
-    rendered as a GUI form. The configuration definition is defined as an
+    file is consumed by the HPCC Systems Configuration Manager component and
+    is rendered as a GUI form. The configuration definition is defined as an
     element of the component name (as declared in the buildset) followed by
     attributes and/or complex elements.</para>
 
@@ -157,8 +157,6 @@
   &lt;/xs:element&gt;
 &lt;/xs:schema&gt;</programlisting>
 
-    <?hard-pagebreak ?>
-
     <para><emphasis role="bold">genenvrules.conf</emphasis> - (optional) This
     file allows the plugin to add itself to the
     "do_not(automatically)_generate" list. While this is an optional file,
@@ -170,8 +168,8 @@
 
     <para><emphasis role="bold">Configuration transformation rules</emphasis>
     - (optional) specified as an xsl template, this set of rules can be
-    applied to the configuration XML. Refer to XSL templates in the HPCC Systems
-    source tree.</para>
+    applied to the configuration XML. Refer to XSL templates in the HPCC
+    Systems source tree.</para>
 
     <sect2 id="concrete-example">
       <title>Concrete Example</title>

diff --git a/docs/EN_US/HPCCSystemAdmin/SA-Mods/SecMgrModConfDeploy.xml b/docs/EN_US/HPCCSystemAdmin/SA-Mods/SecMgrModConfDeploy.xml
@@ -5,9 +5,11 @@
   <title role="nbrk">Configure and Deploy the Security Manager Plugin</title>
 
   <para>The following sections detail the process of configuring your HPCC
-  Systems<superscript>®</superscript> platform to use the Security Manager Plugin.</para>
+  Systems<superscript>®</superscript> platform to use the Security Manager
+  Plugin.</para>
 
-  <sect1 id="how-to-configure-an-existing-security-manager-plugin">
+  <sect1 id="how-to-configure-an-existing-security-manager-plugin"
+         role="nobrk">
     <title>How to Configure a Security Manager Plugin</title>
 
     <para>Once the plugin has been installed, the plugin can be configured

diff --git a/docs/PT_BR/ContainerizedHPCC/ContainerizedMods/LocalDeployment.xml b/docs/PT_BR/ContainerizedHPCC/ContainerizedMods/LocalDeployment.xml
@@ -11,7 +11,7 @@
   <sect1 id="prereq" role="nobrk">
     <title>Pré-requisitos</title>
 
-    <para><graphic fileref="../../images/WhatYouNeed.jpg"/></para>
+    <para><graphic fileref="../../images/WhatYouNeed.jpg" /></para>
 
     <para>Todas ferramentas de terceiros devem ser 64-bits.</para>
   </sect1>
@@ -269,7 +269,11 @@ mkdir c:\hpccdata\dropzone</programlisting></para>
           </varlistentry>
         </variablelist>
       </listitem>
+    </orderedlist>
+
+    <?hard-pagebreak ?>
 
+    <orderedlist continuation="continues">
       <listitem>
         <para>Instale o hpcc-localfile Helm chart.</para>
 
@@ -367,9 +371,9 @@ sasha:
         <para>Isto <emphasis role="bold">não</emphasis> é necessário em
         ambientes MacOS ou WSL 2.</para>
 
-        <graphic fileref="../../images/dockerFileShare.jpg"/>
+        <graphic fileref="../../images/dockerFileShare.jpg" />
 
-        <para/>
+        <para></para>
       </listitem>
 
       <listitem>
@@ -422,7 +426,7 @@ OUTPUT(allPeople,,'MyData::allPeople',THOR,OVERWRITE);
       </listitem>
     </orderedlist>
 
-    <para/>
+    <para></para>
   </sect1>
 
   <sect1 id="StoragePlanes">
@@ -481,10 +485,10 @@ OUTPUT(allPeople,,'MyData::allPeople',THOR,OVERWRITE);
         </varlistentry>
       </variablelist></para>
 
-    <para/>
+    <para></para>
 
-    <para/>
+    <para></para>
 
-    <para/>
+    <para></para>
   </sect1>
 </chapter>
diff --git a/esp/bindings/http/client/httpclient.cpp b/esp/bindings/http/client/httpclient.cpp
@@ -516,6 +516,7 @@ void copyHeaders(CHttpMessage &copyTo, CHttpMessage &copyFrom, bool resetForward
                 }
                 break;
             case 'X':
+            case 'x':
                 if (strieq(name, "X-Forwarded-For"))
                 {
                     if (resetForwardedFor)

diff --git a/esp/bindings/http/platform/httptransport.cpp b/esp/bindings/http/platform/httptransport.cpp
@@ -798,6 +798,49 @@ void CHttpMessage::logSOAPMessage(const char* message, const char* prefix)
     return;
 }
 
+static const char* POST_METHOD_STR = "POST ";
+static bool skipLogContent(const char* httpHeader)
+{
+    if (!startsWith(httpHeader, POST_METHOD_STR))
+        return false;
+
+    const char* servicePtr = httpHeader + 5;
+    if (servicePtr[0] != '/')
+        return false;
+
+    const char* methodPtr = strchr(++servicePtr, '/');
+    if (!methodPtr)
+        return false;
+
+    unsigned serviceType = 0;
+    if (startsWithIgnoreCase(servicePtr, "ws_access/"))
+        serviceType = 1;
+    else if (startsWithIgnoreCase(servicePtr, "ws_account/"))
+        serviceType = 2;
+    if (serviceType == 0)
+        return false;
+
+    StringBuffer espMethod;
+    const char* tail = strchr(++methodPtr, '.');
+    if (tail && (startsWithIgnoreCase(tail, ".xml") || startsWithIgnoreCase(tail, ".json")))
+        espMethod.append(tail - methodPtr, methodPtr);
+    else
+    {
+        tail = strchr(methodPtr, '?');
+        if (!tail)
+            tail = strchr(methodPtr, ' ');
+        if (tail)
+            espMethod.append(tail - methodPtr, methodPtr);
+        else
+            espMethod.append(methodPtr);
+    }
+
+    if (serviceType == 1)
+        return (strieq(espMethod, "AddUser") || strieq(espMethod, "UserResetPass"));
+
+    return strieq(espMethod, "UpdateUser");
+}
+
 void CHttpMessage::logMessage(MessageLogFlag messageLogFlag, const char *prefix)
 {
     logMessage(messageLogFlag, m_content, prefix);
@@ -812,8 +855,7 @@ void CHttpMessage::logMessage(MessageLogFlag messageLogFlag, StringBuffer& conte
 
         if (((messageLogFlag == LOGCONTENT) || (messageLogFlag == LOGALL)) && (content.length() > 0))
         {//log content
-            if ((m_header.length() > 0) && (startsWith(m_header.str(), "POST /ws_access/AddUser")
-                || startsWith(m_header.str(), "POST /ws_access/UserResetPass") || startsWith(m_header.str(), "POST /ws_account/UpdateUser")))
+            if (skipLogContent(m_header))
                 DBGLOG("%s<For security, ESP does not log the content of this request.>", prefix);
             else if (isSoapMessage())
                 logSOAPMessage(content.str(), prefix);

diff --git a/esp/espcommon/espcommon.cpp b/esp/espcommon/espcommon.cpp
@@ -59,8 +59,13 @@ ESPCOMMON_API std::shared_ptr<hpccMetrics::ScaledHistogramMetric> registerServic
 {
     std::string metricName(processName);
     metricName.append(".").append(serviceName).append(".").append(methodName);
-    auto no_ = std::remove(metricName.begin(), metricName.end(), '_');
-    metricName.erase(no_, metricName.end());
+
+    // Remove unwanted characters from new metric name
+    constexpr char removeChars[] = "_-* ";
+    for (unsigned i=0; i<strlen(removeChars); ++i)
+    {
+        metricName.erase(std::remove(metricName.begin(), metricName.end(), removeChars[i]), metricName.end());
+    }
     return registerProfilingMetric(metricName.c_str(), desc, profilingOptions);
 }