Merge remote-tracking branch 'origin/candidate-9.4.x'

Signed-off-by: Gavin Halliday <[email protected]>

.github/workflows/build-assets.yml

@@ -189,7 +189,7 @@ jobs:
         run: |
           mkdir -p ${{ needs.preamble.outputs.folder_build }}
           echo "${{ secrets.SIGNING_SECRET }}" > ${{ needs.preamble.outputs.folder_build }}/private.key
-          plugins=("CASSANDRAEMBED" "COUCHBASEEMBED" "ECLBLAS" "H3" "JAVAEMBED" "KAFKA" "MEMCACHED" "MONGODBEMBED" "MYSQLEMBED" "NLP" "PARQUETEMBED" "REDIS" "REMBED" "SQLITE3EMBED" "SQS" "PLATFORM" "CLIENTTOOLS_ONLY")
+          plugins=("CASSANDRAEMBED" "COUCHBASEEMBED" "ECLBLAS" "H3" "JAVAEMBED" "KAFKA" "MEMCACHED" "MONGODBEMBED" "MYSQLEMBED" "NLP" "PARQUETEMBED" "REDIS" "REMBED" "SQLITE3EMBED" "SQS" "WASMEMBED" "PLATFORM" "CLIENTTOOLS_ONLY")
           for plugin in "${plugins[@]}"; do
             sudo rm -f ${{ needs.preamble.outputs.folder_build }}/CMakeCache.txt
             sudo rm -rf ${{ needs.preamble.outputs.folder_build }}/CMakeFiles

.github/workflows/build-docker.yml

@@ -128,7 +128,7 @@ jobs:
           mkdir -p ${{ github.workspace }}/build
           mkdir -p ${{ github.workspace }}/.ccache
           declare -a plugins
-          plugins=(${{ inputs.single-package == true && '"PLATFORM"' || '"PLATFORM" "CASSANDRAEMBED" "COUCHBASEEMBED" "ECLBLAS" "H3" "JAVAEMBED" "KAFKA" "MEMCACHED" "MONGODBEMBED" "MYSQLEMBED" "NLP" "PARQUETEMBED" "REDIS" "REMBED" "SQLITE3EMBED" "SQS"' }})
+          plugins=(${{ inputs.single-package == true && '"PLATFORM"' || '"PLATFORM" "CASSANDRAEMBED" "COUCHBASEEMBED" "ECLBLAS" "H3" "JAVAEMBED" "KAFKA" "MEMCACHED" "MONGODBEMBED" "MYSQLEMBED" "NLP" "PARQUETEMBED" "REDIS" "REMBED" "SQLITE3EMBED" "SQS" "WASMEMBED"' }})
           for plugin in "${plugins[@]}"; do
             sudo rm -f ${{ github.workspace }}/build/CMakeCache.txt
             sudo rm -rf ${{ github.workspace }}/build/CMakeFiles

.github/workflows/build-gh_runner.yml

@@ -205,7 +205,7 @@ jobs:
           mkdir -p ${{ github.workspace }}/LN
           mkdir -p ${{ github.workspace }}/build
           declare -a plugins
-          plugins=(${{ inputs.single-package == true && '"PLATFORM"' || '"PLATFORM" "CASSANDRAEMBED" "COUCHBASEEMBED" "ECLBLAS" "H3" "JAVAEMBED" "KAFKA" "MEMCACHED" "MONGODBEMBED" "MYSQLEMBED" "NLP" "PARQUETEMBED" "REDIS" "REMBED" "SQLITE3EMBED" "SQS"' }})
+          plugins=(${{ inputs.single-package == true && '"PLATFORM"' || '"PLATFORM" "CASSANDRAEMBED" "COUCHBASEEMBED" "ECLBLAS" "H3" "JAVAEMBED" "KAFKA" "MEMCACHED" "MONGODBEMBED" "MYSQLEMBED" "NLP" "PARQUETEMBED" "REDIS" "REMBED" "SQLITE3EMBED" "SQS" "WASMEMBED"' }})
           for plugin in "${plugins[@]}"; do
             rm -f ./build/CMakeCache.txt
             rm -rf ./build/CMakeFiles

.github/workflows/test-smoke-gh_runner.yml

@@ -139,3 +139,13 @@ jobs:
               /var/log/HPCCSystems
               /home/runner/HPCCSystems-regression
           if-no-files-found: error
+
+  succeeded:
+    runs-on: ${{ inputs.os }}
+    needs: main
+    steps:
+      - shell: "bash"
+        run: |
+          echo "...all tests passed..."
+
+

.github/workflows/test-ui-gh_runner.yml

@@ -70,20 +70,6 @@ jobs:
             gdb \
             ${{ inputs.dependencies }}
 
-      - name: Install UI Dependencies
-        if: steps.check.outputs.runtests
-        run: |
-          wget https://dl.google.com/linux/direct/google-chrome-stable_current_amd64.deb
-          sudo apt-get install -y ./google-chrome-stable_current_amd64.deb
-          wget https://chromedriver.storage.googleapis.com/2.41/chromedriver_linux64.zip
-          unzip chromedriver_linux64.zip
-          sudo mv chromedriver /usr/bin/chromedriver
-          sudo chown root:root /usr/bin/chromedriver
-          sudo chmod +x /usr/bin/chromedriver
-          wget https://selenium-release.storage.googleapis.com/3.141/selenium-server-standalone-3.141.59.jar
-          wget http://www.java2s.com/Code/JarDownload/testng/testng-6.8.7.jar.zip
-          unzip testng-6.8.7.jar.zip
-
       - name: Download Package
         if: steps.check.outputs.runtests
         uses: actions/download-artifact@v3
@@ -109,13 +95,29 @@ jobs:
           sudo update-locale
           sudo /etc/init.d/hpcc-init start
 
+      - name: Install UI Dependencies
+        if: steps.check.outputs.runtests
+        shell: "bash"
+        run: |
+          wget https://dl.google.com/linux/direct/google-chrome-stable_current_amd64.deb
+          sudo apt-get install -y ./google-chrome-stable_current_amd64.deb
+          wget https://chromedriver.storage.googleapis.com/2.41/chromedriver_linux64.zip
+          unzip chromedriver_linux64.zip
+          sudo mv chromedriver /usr/bin/chromedriver
+          sudo chown root:root /usr/bin/chromedriver
+          sudo chmod +x /usr/bin/chromedriver
+          wget https://selenium-release.storage.googleapis.com/3.141/selenium-server-standalone-3.141.59.jar
+          wget http://www.java2s.com/Code/JarDownload/testng/testng-6.8.7.jar.zip
+          unzip testng-6.8.7.jar.zip
+
       - name: Run Tests
         timeout-minutes: 10 # generous, current runtime is ~1min, this should be increased if new tests are added
         if: steps.check.outputs.runtests
+        shell: "bash"
         run: |
-          export CLASSPATH=".:$(realpath selenium-server-standalone-3.141.59.jar):$(realpath testng-6.8.7.jar)"
+          export CLASSPATH=".:${{ github.workspace }}/selenium-server-standalone-3.141.59.jar:${{ github.workspace }}/testng-6.8.7.jar"
           pushd ${{ inputs.asset-name }}-ui_test-files
-          sudo ./run.sh tests http://localhost:8010 > eclWatchUiTest.log 2>&1
+          ./run.sh tests http://localhost:8010 > eclWatchUiTest.log 2>&1
           retCode=$?
           echo "UI test done"
           [[ $retCode -ne 0 ]] && exit 1

CMakeLists.txt

@@ -169,6 +169,7 @@ if ( PLUGIN )
     HPCC_ADD_SUBDIRECTORY (dali/base)
     HPCC_ADD_SUBDIRECTORY (plugins/Rembed "REMBED")
     HPCC_ADD_SUBDIRECTORY (plugins/v8embed "V8EMBED")
+    HPCC_ADD_SUBDIRECTORY (plugins/wasmembed "WASMEMBED")
     HPCC_ADD_SUBDIRECTORY (plugins/memcached "MEMCACHED")
     HPCC_ADD_SUBDIRECTORY (plugins/redis "REDIS")
     HPCC_ADD_SUBDIRECTORY (plugins/javaembed "JAVAEMBED")

cmake_modules/plugins.cmake

@@ -42,6 +42,7 @@ set(PLUGINS_LIST
     SQLITE3EMBED
     SQS
     V8EMBED
+    WASMEMBED
     EXAMPLEPLUGIN
 )
 

common/remote/hooks/git/gitfile.cpp

@@ -244,7 +244,7 @@ class GitRepositoryFileIO : implements CSimpleInterfaceOf<IFileIO>
             addPathSepChar(scriptPath).append("bin/hpccaskpass.sh");
             env.emplace_back("GIT_ASKPASS", scriptPath);
 
-            Owned<IPropertyTree> secret = getSecret("git", gitUser);
+            Owned<const IPropertyTree> secret = getSecret("git", gitUser);
             if (secret)
             {
                 MemoryBuffer gitKey;

common/thorhelper/thorsoapcall.cpp

@@ -878,7 +878,7 @@ class CWSCHelperThread : public Thread
 
 bool loadConnectSecret(const char *vaultId, const char *secretName, UrlArray &urlArray, StringBuffer &issuer, StringBuffer &proxyAddress, bool required, WSCType wscType)
 {
-    Owned<IPropertyTree> secret;
+    Owned<const IPropertyTree> secret;
     if (!isEmptyString(secretName))
         secret.setown(getSecret("ecl", secretName, vaultId, nullptr));
     if (!secret)

dali/base/dadfs.cpp

@@ -11168,6 +11168,7 @@ class CDaliDFSServer: public Thread, public CTransactionLogTracker, implements I
             CScopeConnectLock sconnlock("getFileTree", *logicalname, false, false, false, defaultTimeout);
             IPropertyTree* sroot = sconnlock.conn()?sconnlock.conn()->queryRoot():NULL;
             logicalname->getTail(tail);
+            INamedGroupStore *groupResolver = &queryNamedGroupStore();
             if (version >= 2)
             {
                 Owned<IPropertyTree> tree = getNamedPropTree(sroot,queryDfsXmlBranchName(DXB_File),"@name",tail.str(),false);
@@ -11179,7 +11180,9 @@ class CDaliDFSServer: public Thread, public CTransactionLogTracker, implements I
                         // asking for the returned meta data to be remapped to point to the dafilesrv service.
                         if (hasMask(opts, GetFileTreeOpts::remapToService))
                         {
+                            tree.setown(createPTreeFromIPT(tree)); // copy live Dali tree, because it is about to be altered by remapGroupsToDafilesrv
                             remapGroupsToDafilesrv(tree, &queryNamedGroupStore());
+                            groupResolver = nullptr; // do not attempt to resolve remapped group (it will not exist and cause addUnique to create a new anon one)
 
                             const char *remotePlaneName = tree->queryProp("@group");
                             Owned<IPropertyTree> filePlane = getStoragePlane(remotePlaneName);
@@ -11189,7 +11192,7 @@ class CDaliDFSServer: public Thread, public CTransactionLogTracker, implements I
                         }
                     }
 
-                    Owned<IFileDescriptor> fdesc = deserializeFileDescriptorTree(tree,&queryNamedGroupStore(),IFDSF_EXCLUDE_CLUSTERNAMES);
+                    Owned<IFileDescriptor> fdesc = deserializeFileDescriptorTree(tree,groupResolver,IFDSF_EXCLUDE_CLUSTERNAMES);
                     mb.append((int)1); // 1 == standard file
                     fdesc->serialize(mb);
 
@@ -11220,15 +11223,20 @@ class CDaliDFSServer: public Thread, public CTransactionLogTracker, implements I
                 {
                     if (version == MDFS_GET_FILE_TREE_V2)
                     {
-#ifdef _CONTAINERIZED
-                        // NB: to be here, the client is by definition legacy, and should be foreign.
-                        // foreignAccess must also have been configured in this environment
-
-                        if (getComponentConfigSP()->getPropBool("@foreignAccess"))
-                            remapGroupsToDafilesrv(tree, &queryNamedGroupStore());
-#endif
+                        if (isContainerized())
+                        {
+                            // NB: to be here, the client is by definition legacy, and should only be via ~foreign.
+                            // NB: foreignAccess is a auto-generated template setting, that is set to true if Dali and directio,
+                            // have been exposed in the helm chart for foreign access.
+                            if (getComponentConfigSP()->getPropBool("@foreignAccess"))
+                            {
+                                tree.setown(createPTreeFromIPT(tree)); // copy live Dali tree, because it is about to be altered by remapGroupsToDafilesrv
+                                remapGroupsToDafilesrv(tree, &queryNamedGroupStore());
+                                groupResolver = nullptr; // do not attempt to resolve remapped group (it will not exist and cause addUnique to create a new anon one)
+                            }
+                        }
 
-                        Owned<IFileDescriptor> fdesc = deserializeFileDescriptorTree(tree,&queryNamedGroupStore(),IFDSF_EXCLUDE_CLUSTERNAMES);
+                        Owned<IFileDescriptor> fdesc = deserializeFileDescriptorTree(tree,groupResolver,IFDSF_EXCLUDE_CLUSTERNAMES);
 
                         mb.append((int)-2).append(version);
 

dockerfiles/vcpkg/amazonlinux.dockerfile

@@ -1,16 +1,6 @@
 ARG VCPKG_REF=latest
 FROM hpccsystems/platform-build-base-amazonlinux:$VCPKG_REF
 
-RUN amazon-linux-extras install java-openjdk11 && yum install -y \
-    java-11-openjdk-devel \
-    python3-devel \
-    epel-release
-RUN yum install -y \
-    ccache \
-    R-core-devel \
-    R-Rcpp-devel \
-    R-RInside-devel
-
-WORKDIR /hpcc-dev
-
 ENTRYPOINT ["/bin/bash", "--login", "-c"]
+
+CMD ["/bin/bash"]

dockerfiles/vcpkg/centos-7.dockerfile

@@ -0,0 +1,6 @@
+ARG VCPKG_REF=latest
+FROM hpccsystems/platform-build-base-centos-7:$VCPKG_REF
+
+ENTRYPOINT ["/bin/bash", "--login", "-c"]
+
+CMD ["/bin/bash"]

dockerfiles/vcpkg/centos-8.dockerfile

@@ -0,0 +1,6 @@
+ARG VCPKG_REF=latest
+FROM hpccsystems/platform-build-base-centos-8:$VCPKG_REF
+
+ENTRYPOINT ["/bin/bash", "--login", "-c"]
+
+CMD ["/bin/bash"]

dockerfiles/vcpkg/ubuntu-20.04.dockerfile

@@ -0,0 +1,6 @@
+ARG VCPKG_REF=latest
+FROM hpccsystems/platform-build-base-ubuntu-20.04:$VCPKG_REF
+
+ENTRYPOINT ["/bin/bash", "--login", "-c"]
+
+CMD ["/bin/bash"]

dockerfiles/vcpkg/ubuntu-22.04.dockerfile

@@ -0,0 +1,6 @@
+ARG VCPKG_REF=latest
+FROM hpccsystems/platform-build-base-ubuntu-22.04:$VCPKG_REF
+
+ENTRYPOINT ["/bin/bash", "--login", "-c"]
+
+CMD ["/bin/bash"]

dockerfiles/vcpkg/ubuntu-23.10.dockerfile

@@ -0,0 +1,6 @@
+ARG VCPKG_REF=latest
+FROM hpccsystems/platform-build-base-ubuntu-23.10:$VCPKG_REF
+
+ENTRYPOINT ["/bin/bash", "--login", "-c"]
+
+CMD ["/bin/bash"]

ecl/hql/hqlgram2.cpp

@@ -9700,11 +9700,11 @@ void HqlGram::checkDerivedCompatible(IIdAtom * name, IHqlExpression * scope, IHq
             if (match)
             {
                 if (!canBeVirtual(match))
-                    reportError(ERR_MISMATCH_PROTO, errpos, "Definition %s, cannot override this kind of definition", str(name));
+                    reportError(ERR_MISMATCH_PROTO, errpos, "Definition %s, cannot override this kind of definition", nullText(str(name)));
                 else
                 {
                     if (!areSymbolsCompatible(expr, isParametered, parameters, match))
-                        reportError(ERR_MISMATCH_PROTO, errpos, "Prototypes for %s in base and derived modules must match", str(name));
+                        reportError(ERR_MISMATCH_PROTO, errpos, "Prototypes for %s in base and derived modules must match", nullText(str(name)));
                 }
             }
         }

ecl/hql/hqlrepository.cpp

@@ -959,7 +959,7 @@ unsigned EclRepositoryManager::runGitCommand(StringBuffer * output, const char *
             }
             else
             {
-                Owned<IPropertyTree> secret = getSecret("git", options.gitUser.str());
+                Owned<const IPropertyTree> secret = getSecret("git", options.gitUser.str());
                 if (secret)
                 {
                     MemoryBuffer gitKey;

esp/bindings/http/platform/httpprot.cpp

@@ -216,32 +216,38 @@ CSecureHttpProtocol::CSecureHttpProtocol(IPropertyTree* cfg)
     {
         m_config.setown(cfg);
 
-        //ensure keys are specified. Passphrase is optional
-        StringBuffer sb;
-        cfg->getProp("certificate", sb);
-        if(sb.length() == 0)
-        {
-            throw MakeStringException(-1, "certificate file not specified in config file");
-        }
+        IEspPlugin *pplg = loadPlugin(SSLIB);
+        if (!pplg)
+            throw MakeStringException(-1, "dll/shared-object %s can't be loaded", SSLIB);
 
-        cfg->getProp("privatekey", sb.clear());
-        if(sb.length() == 0)
+        const char *issuer = cfg->queryProp("issuer");
+        if (!isEmptyString(issuer))
         {
-            throw MakeStringException(-1, "private key file not specified in config file");
+            const char *trustedPeers = nullptr;
+            if (cfg->hasProp("verify"))
+                trustedPeers = cfg->queryProp("verify/trusted_peers");
+            createSecureSocketContextSecretSrv_t xproc = (createSecureSocketContextSecretSrv_t) pplg->getProcAddress("createSecureSocketContextSecretSrv");
+            if (!xproc)
+                throw MakeStringException(-1, "procedure createSecureSocketContextSecretSrv can't be loaded");
+            m_ssctx.setown(xproc(issuer, trustedPeers, false));
         }
-
-        createSecureSocketContextEx2_t xproc = NULL;
-        IEspPlugin *pplg = loadPlugin(SSLIB);
-        if (pplg)
-            xproc = (createSecureSocketContextEx2_t) pplg->getProcAddress("createSecureSocketContextEx2");
         else
-            throw MakeStringException(-1, "dll/shared-object %s can't be loaded", SSLIB);
-
-
-        if (xproc)
+        {
+            //ensure keys are specified. Passphrase is optional
+            StringBuffer sb;
+            cfg->getProp("certificate", sb);
+            if(sb.isEmpty())
+                throw MakeStringException(-1, "certificate file not specified in config file");
+
+            cfg->getProp("privatekey", sb.clear());
+            if(sb.isEmpty())
+                throw MakeStringException(-1, "private key file not specified in config file");
+
+            createSecureSocketContextEx2_t xproc = (createSecureSocketContextEx2_t) pplg->getProcAddress("createSecureSocketContextEx2");
+            if (!xproc)
+                throw MakeStringException(-1, "procedure createSecureSocketContextEx2 can't be loaded");
             m_ssctx.setown(xproc(cfg, ServerSocket));
-        else
-            throw MakeStringException(-1, "procedure createSecureSocketContextEx2 can't be loaded");
+        }
     }
 }
 

esp/clients/ws_dfsclient/ws_dfsclient.cpp

@@ -599,7 +599,7 @@ IClientWsDfs *getDfsClient(const char *serviceUrl, IUserDescriptor *userDesc)
 
 static void configureClientSSL(IEspClientRpcSettings &rpc, const char *secretName)
 {
-    Owned<IPropertyTree> secretPTree = getSecret("storage", secretName);
+    Owned<const IPropertyTree> secretPTree = getSecret("storage", secretName);
     if (!secretPTree)
         throw makeStringExceptionV(-1, "secret %s.%s not found", "storage", secretName);
 

esp/esdlscriptlib/esdl_script.cpp

@@ -911,7 +911,7 @@ class CEsdlTransformOperationMySqlCall : public CEsdlTransformOperationBase
             recordException(ESDL_SCRIPT_MissingOperationAttr, msg.append(name));
         }
     }
-    IPropertyTree *getSecretInfo(IXpathContext * sourceContext)
+    const IPropertyTree *getSecretInfo(IXpathContext * sourceContext)
     {
         //leaving flexibility for the secret to be configured multiple ways
         //  the most secure option in my opinion is to at least have the server, name, and password all in the secret
@@ -943,7 +943,7 @@ class CEsdlTransformOperationMySqlCall : public CEsdlTransformOperationBase
         options.append(name).append('=').append(value);
 
     }
-    void appendOption(StringBuffer &options, const char *name, IXpathContext * sourceContext, ICompiledXpath *cx, IPropertyTree *secret, bool required)
+    void appendOption(StringBuffer &options, const char *name, IXpathContext * sourceContext, ICompiledXpath *cx, const IPropertyTree *secret, bool required)
     {
         if (secret && secret->hasProp(name))
         {
@@ -971,7 +971,7 @@ class CEsdlTransformOperationMySqlCall : public CEsdlTransformOperationBase
     }
     IEmbedFunctionContext *createFunctionContext(IXpathContext * sourceContext)
     {
-        Owned<IPropertyTree> secret = getSecretInfo(sourceContext);
+        Owned<const IPropertyTree> secret = getSecretInfo(sourceContext);
         StringBuffer options;
         appendOption(options, "server", sourceContext, m_server, secret, true);
         appendOption(options, "user", sourceContext, m_user, secret, true);