diff --git a/.github/workflows/build-assets.yml b/.github/workflows/build-assets.yml index 9aeb67f532c..7d3ec45658d 100644 --- a/.github/workflows/build-assets.yml +++ b/.github/workflows/build-assets.yml @@ -189,7 +189,7 @@ jobs: run: | mkdir -p ${{ needs.preamble.outputs.folder_build }} echo "${{ secrets.SIGNING_SECRET }}" > ${{ needs.preamble.outputs.folder_build }}/private.key - plugins=("CASSANDRAEMBED" "COUCHBASEEMBED" "ECLBLAS" "H3" "JAVAEMBED" "KAFKA" "MEMCACHED" "MONGODBEMBED" "MYSQLEMBED" "NLP" "PARQUETEMBED" "REDIS" "REMBED" "SQLITE3EMBED" "SQS" "PLATFORM" "CLIENTTOOLS_ONLY") + plugins=("CASSANDRAEMBED" "COUCHBASEEMBED" "ECLBLAS" "H3" "JAVAEMBED" "KAFKA" "MEMCACHED" "MONGODBEMBED" "MYSQLEMBED" "NLP" "PARQUETEMBED" "REDIS" "REMBED" "SQLITE3EMBED" "SQS" "WASMEMBED" "PLATFORM" "CLIENTTOOLS_ONLY") for plugin in "${plugins[@]}"; do sudo rm -f ${{ needs.preamble.outputs.folder_build }}/CMakeCache.txt sudo rm -rf ${{ needs.preamble.outputs.folder_build }}/CMakeFiles diff --git a/.github/workflows/build-docker.yml b/.github/workflows/build-docker.yml index e7bfab4a02f..24560aa6331 100644 --- a/.github/workflows/build-docker.yml +++ b/.github/workflows/build-docker.yml @@ -128,7 +128,7 @@ jobs: mkdir -p ${{ github.workspace }}/build mkdir -p ${{ github.workspace }}/.ccache declare -a plugins - plugins=(${{ inputs.single-package == true && '"PLATFORM"' || '"PLATFORM" "CASSANDRAEMBED" "COUCHBASEEMBED" "ECLBLAS" "H3" "JAVAEMBED" "KAFKA" "MEMCACHED" "MONGODBEMBED" "MYSQLEMBED" "NLP" "PARQUETEMBED" "REDIS" "REMBED" "SQLITE3EMBED" "SQS"' }}) + plugins=(${{ inputs.single-package == true && '"PLATFORM"' || '"PLATFORM" "CASSANDRAEMBED" "COUCHBASEEMBED" "ECLBLAS" "H3" "JAVAEMBED" "KAFKA" "MEMCACHED" "MONGODBEMBED" "MYSQLEMBED" "NLP" "PARQUETEMBED" "REDIS" "REMBED" "SQLITE3EMBED" "SQS" "WASMEMBED"' }}) for plugin in "${plugins[@]}"; do sudo rm -f ${{ github.workspace }}/build/CMakeCache.txt sudo rm -rf ${{ github.workspace }}/build/CMakeFiles diff --git a/.github/workflows/build-gh_runner.yml b/.github/workflows/build-gh_runner.yml index 48823c004c5..2b352e2cb99 100644 --- a/.github/workflows/build-gh_runner.yml +++ b/.github/workflows/build-gh_runner.yml @@ -205,7 +205,7 @@ jobs: mkdir -p ${{ github.workspace }}/LN mkdir -p ${{ github.workspace }}/build declare -a plugins - plugins=(${{ inputs.single-package == true && '"PLATFORM"' || '"PLATFORM" "CASSANDRAEMBED" "COUCHBASEEMBED" "ECLBLAS" "H3" "JAVAEMBED" "KAFKA" "MEMCACHED" "MONGODBEMBED" "MYSQLEMBED" "NLP" "PARQUETEMBED" "REDIS" "REMBED" "SQLITE3EMBED" "SQS"' }}) + plugins=(${{ inputs.single-package == true && '"PLATFORM"' || '"PLATFORM" "CASSANDRAEMBED" "COUCHBASEEMBED" "ECLBLAS" "H3" "JAVAEMBED" "KAFKA" "MEMCACHED" "MONGODBEMBED" "MYSQLEMBED" "NLP" "PARQUETEMBED" "REDIS" "REMBED" "SQLITE3EMBED" "SQS" "WASMEMBED"' }}) for plugin in "${plugins[@]}"; do rm -f ./build/CMakeCache.txt rm -rf ./build/CMakeFiles diff --git a/.github/workflows/test-smoke-gh_runner.yml b/.github/workflows/test-smoke-gh_runner.yml index b01fdd3ebbc..f0fdfbaeab5 100644 --- a/.github/workflows/test-smoke-gh_runner.yml +++ b/.github/workflows/test-smoke-gh_runner.yml @@ -139,3 +139,13 @@ jobs: /var/log/HPCCSystems /home/runner/HPCCSystems-regression if-no-files-found: error + + succeeded: + runs-on: ${{ inputs.os }} + needs: main + steps: + - shell: "bash" + run: | + echo "...all tests passed..." + + diff --git a/.github/workflows/test-ui-gh_runner.yml b/.github/workflows/test-ui-gh_runner.yml index 0126f2c6e39..16530443396 100644 --- a/.github/workflows/test-ui-gh_runner.yml +++ b/.github/workflows/test-ui-gh_runner.yml @@ -70,20 +70,6 @@ jobs: gdb \ ${{ inputs.dependencies }} - - name: Install UI Dependencies - if: steps.check.outputs.runtests - run: | - wget https://dl.google.com/linux/direct/google-chrome-stable_current_amd64.deb - sudo apt-get install -y ./google-chrome-stable_current_amd64.deb - wget https://chromedriver.storage.googleapis.com/2.41/chromedriver_linux64.zip - unzip chromedriver_linux64.zip - sudo mv chromedriver /usr/bin/chromedriver - sudo chown root:root /usr/bin/chromedriver - sudo chmod +x /usr/bin/chromedriver - wget https://selenium-release.storage.googleapis.com/3.141/selenium-server-standalone-3.141.59.jar - wget http://www.java2s.com/Code/JarDownload/testng/testng-6.8.7.jar.zip - unzip testng-6.8.7.jar.zip - - name: Download Package if: steps.check.outputs.runtests uses: actions/download-artifact@v3 @@ -109,13 +95,29 @@ jobs: sudo update-locale sudo /etc/init.d/hpcc-init start + - name: Install UI Dependencies + if: steps.check.outputs.runtests + shell: "bash" + run: | + wget https://dl.google.com/linux/direct/google-chrome-stable_current_amd64.deb + sudo apt-get install -y ./google-chrome-stable_current_amd64.deb + wget https://chromedriver.storage.googleapis.com/2.41/chromedriver_linux64.zip + unzip chromedriver_linux64.zip + sudo mv chromedriver /usr/bin/chromedriver + sudo chown root:root /usr/bin/chromedriver + sudo chmod +x /usr/bin/chromedriver + wget https://selenium-release.storage.googleapis.com/3.141/selenium-server-standalone-3.141.59.jar + wget http://www.java2s.com/Code/JarDownload/testng/testng-6.8.7.jar.zip + unzip testng-6.8.7.jar.zip + - name: Run Tests timeout-minutes: 10 # generous, current runtime is ~1min, this should be increased if new tests are added if: steps.check.outputs.runtests + shell: "bash" run: | - export CLASSPATH=".:$(realpath selenium-server-standalone-3.141.59.jar):$(realpath testng-6.8.7.jar)" + export CLASSPATH=".:${{ github.workspace }}/selenium-server-standalone-3.141.59.jar:${{ github.workspace }}/testng-6.8.7.jar" pushd ${{ inputs.asset-name }}-ui_test-files - sudo ./run.sh tests http://localhost:8010 > eclWatchUiTest.log 2>&1 + ./run.sh tests http://localhost:8010 > eclWatchUiTest.log 2>&1 retCode=$? echo "UI test done" [[ $retCode -ne 0 ]] && exit 1 diff --git a/CMakeLists.txt b/CMakeLists.txt index b26f33ec7e0..621f77deaaa 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -169,6 +169,7 @@ if ( PLUGIN ) HPCC_ADD_SUBDIRECTORY (dali/base) HPCC_ADD_SUBDIRECTORY (plugins/Rembed "REMBED") HPCC_ADD_SUBDIRECTORY (plugins/v8embed "V8EMBED") + HPCC_ADD_SUBDIRECTORY (plugins/wasmembed "WASMEMBED") HPCC_ADD_SUBDIRECTORY (plugins/memcached "MEMCACHED") HPCC_ADD_SUBDIRECTORY (plugins/redis "REDIS") HPCC_ADD_SUBDIRECTORY (plugins/javaembed "JAVAEMBED") diff --git a/cmake_modules/plugins.cmake b/cmake_modules/plugins.cmake index b96a7e7cd09..f87df647507 100644 --- a/cmake_modules/plugins.cmake +++ b/cmake_modules/plugins.cmake @@ -42,6 +42,7 @@ set(PLUGINS_LIST SQLITE3EMBED SQS V8EMBED + WASMEMBED EXAMPLEPLUGIN ) diff --git a/common/remote/hooks/git/gitfile.cpp b/common/remote/hooks/git/gitfile.cpp index ccebaddb28b..1a3e7d14ebe 100644 --- a/common/remote/hooks/git/gitfile.cpp +++ b/common/remote/hooks/git/gitfile.cpp @@ -244,7 +244,7 @@ class GitRepositoryFileIO : implements CSimpleInterfaceOf addPathSepChar(scriptPath).append("bin/hpccaskpass.sh"); env.emplace_back("GIT_ASKPASS", scriptPath); - Owned secret = getSecret("git", gitUser); + Owned secret = getSecret("git", gitUser); if (secret) { MemoryBuffer gitKey; diff --git a/common/thorhelper/thorsoapcall.cpp b/common/thorhelper/thorsoapcall.cpp index 98df33c7a32..b4a0d56976e 100644 --- a/common/thorhelper/thorsoapcall.cpp +++ b/common/thorhelper/thorsoapcall.cpp @@ -878,7 +878,7 @@ class CWSCHelperThread : public Thread bool loadConnectSecret(const char *vaultId, const char *secretName, UrlArray &urlArray, StringBuffer &issuer, StringBuffer &proxyAddress, bool required, WSCType wscType) { - Owned secret; + Owned secret; if (!isEmptyString(secretName)) secret.setown(getSecret("ecl", secretName, vaultId, nullptr)); if (!secret) diff --git a/dali/base/dadfs.cpp b/dali/base/dadfs.cpp index c6c1fdb2442..00fea1d1ddb 100644 --- a/dali/base/dadfs.cpp +++ b/dali/base/dadfs.cpp @@ -11168,6 +11168,7 @@ class CDaliDFSServer: public Thread, public CTransactionLogTracker, implements I CScopeConnectLock sconnlock("getFileTree", *logicalname, false, false, false, defaultTimeout); IPropertyTree* sroot = sconnlock.conn()?sconnlock.conn()->queryRoot():NULL; logicalname->getTail(tail); + INamedGroupStore *groupResolver = &queryNamedGroupStore(); if (version >= 2) { Owned tree = getNamedPropTree(sroot,queryDfsXmlBranchName(DXB_File),"@name",tail.str(),false); @@ -11179,7 +11180,9 @@ class CDaliDFSServer: public Thread, public CTransactionLogTracker, implements I // asking for the returned meta data to be remapped to point to the dafilesrv service. if (hasMask(opts, GetFileTreeOpts::remapToService)) { + tree.setown(createPTreeFromIPT(tree)); // copy live Dali tree, because it is about to be altered by remapGroupsToDafilesrv remapGroupsToDafilesrv(tree, &queryNamedGroupStore()); + groupResolver = nullptr; // do not attempt to resolve remapped group (it will not exist and cause addUnique to create a new anon one) const char *remotePlaneName = tree->queryProp("@group"); Owned filePlane = getStoragePlane(remotePlaneName); @@ -11189,7 +11192,7 @@ class CDaliDFSServer: public Thread, public CTransactionLogTracker, implements I } } - Owned fdesc = deserializeFileDescriptorTree(tree,&queryNamedGroupStore(),IFDSF_EXCLUDE_CLUSTERNAMES); + Owned fdesc = deserializeFileDescriptorTree(tree,groupResolver,IFDSF_EXCLUDE_CLUSTERNAMES); mb.append((int)1); // 1 == standard file fdesc->serialize(mb); @@ -11220,15 +11223,20 @@ class CDaliDFSServer: public Thread, public CTransactionLogTracker, implements I { if (version == MDFS_GET_FILE_TREE_V2) { -#ifdef _CONTAINERIZED - // NB: to be here, the client is by definition legacy, and should be foreign. - // foreignAccess must also have been configured in this environment - - if (getComponentConfigSP()->getPropBool("@foreignAccess")) - remapGroupsToDafilesrv(tree, &queryNamedGroupStore()); -#endif + if (isContainerized()) + { + // NB: to be here, the client is by definition legacy, and should only be via ~foreign. + // NB: foreignAccess is a auto-generated template setting, that is set to true if Dali and directio, + // have been exposed in the helm chart for foreign access. + if (getComponentConfigSP()->getPropBool("@foreignAccess")) + { + tree.setown(createPTreeFromIPT(tree)); // copy live Dali tree, because it is about to be altered by remapGroupsToDafilesrv + remapGroupsToDafilesrv(tree, &queryNamedGroupStore()); + groupResolver = nullptr; // do not attempt to resolve remapped group (it will not exist and cause addUnique to create a new anon one) + } + } - Owned fdesc = deserializeFileDescriptorTree(tree,&queryNamedGroupStore(),IFDSF_EXCLUDE_CLUSTERNAMES); + Owned fdesc = deserializeFileDescriptorTree(tree,groupResolver,IFDSF_EXCLUDE_CLUSTERNAMES); mb.append((int)-2).append(version); diff --git a/dockerfiles/vcpkg/amazonlinux.dockerfile b/dockerfiles/vcpkg/amazonlinux.dockerfile index 190badf60f4..c7ea3c4e138 100644 --- a/dockerfiles/vcpkg/amazonlinux.dockerfile +++ b/dockerfiles/vcpkg/amazonlinux.dockerfile @@ -1,16 +1,6 @@ ARG VCPKG_REF=latest FROM hpccsystems/platform-build-base-amazonlinux:$VCPKG_REF -RUN amazon-linux-extras install java-openjdk11 && yum install -y \ - java-11-openjdk-devel \ - python3-devel \ - epel-release -RUN yum install -y \ - ccache \ - R-core-devel \ - R-Rcpp-devel \ - R-RInside-devel - -WORKDIR /hpcc-dev - ENTRYPOINT ["/bin/bash", "--login", "-c"] + +CMD ["/bin/bash"] diff --git a/dockerfiles/vcpkg/centos-7.dockerfile b/dockerfiles/vcpkg/centos-7.dockerfile new file mode 100644 index 00000000000..04d1ba29af5 --- /dev/null +++ b/dockerfiles/vcpkg/centos-7.dockerfile @@ -0,0 +1,6 @@ +ARG VCPKG_REF=latest +FROM hpccsystems/platform-build-base-centos-7:$VCPKG_REF + +ENTRYPOINT ["/bin/bash", "--login", "-c"] + +CMD ["/bin/bash"] diff --git a/dockerfiles/vcpkg/centos-8.dockerfile b/dockerfiles/vcpkg/centos-8.dockerfile new file mode 100644 index 00000000000..108398a2711 --- /dev/null +++ b/dockerfiles/vcpkg/centos-8.dockerfile @@ -0,0 +1,6 @@ +ARG VCPKG_REF=latest +FROM hpccsystems/platform-build-base-centos-8:$VCPKG_REF + +ENTRYPOINT ["/bin/bash", "--login", "-c"] + +CMD ["/bin/bash"] \ No newline at end of file diff --git a/dockerfiles/vcpkg/ubuntu-20.04.dockerfile b/dockerfiles/vcpkg/ubuntu-20.04.dockerfile new file mode 100644 index 00000000000..af0e1dc41b2 --- /dev/null +++ b/dockerfiles/vcpkg/ubuntu-20.04.dockerfile @@ -0,0 +1,6 @@ +ARG VCPKG_REF=latest +FROM hpccsystems/platform-build-base-ubuntu-20.04:$VCPKG_REF + +ENTRYPOINT ["/bin/bash", "--login", "-c"] + +CMD ["/bin/bash"] \ No newline at end of file diff --git a/dockerfiles/vcpkg/ubuntu-22.04.dockerfile b/dockerfiles/vcpkg/ubuntu-22.04.dockerfile new file mode 100644 index 00000000000..4c7fe928b5c --- /dev/null +++ b/dockerfiles/vcpkg/ubuntu-22.04.dockerfile @@ -0,0 +1,6 @@ +ARG VCPKG_REF=latest +FROM hpccsystems/platform-build-base-ubuntu-22.04:$VCPKG_REF + +ENTRYPOINT ["/bin/bash", "--login", "-c"] + +CMD ["/bin/bash"] diff --git a/dockerfiles/vcpkg/ubuntu-23.10.dockerfile b/dockerfiles/vcpkg/ubuntu-23.10.dockerfile new file mode 100644 index 00000000000..652e1e1a42e --- /dev/null +++ b/dockerfiles/vcpkg/ubuntu-23.10.dockerfile @@ -0,0 +1,6 @@ +ARG VCPKG_REF=latest +FROM hpccsystems/platform-build-base-ubuntu-23.10:$VCPKG_REF + +ENTRYPOINT ["/bin/bash", "--login", "-c"] + +CMD ["/bin/bash"] \ No newline at end of file diff --git a/ecl/hql/hqlgram2.cpp b/ecl/hql/hqlgram2.cpp index 9a8cb03c58d..999af21edaf 100644 --- a/ecl/hql/hqlgram2.cpp +++ b/ecl/hql/hqlgram2.cpp @@ -9700,11 +9700,11 @@ void HqlGram::checkDerivedCompatible(IIdAtom * name, IHqlExpression * scope, IHq if (match) { if (!canBeVirtual(match)) - reportError(ERR_MISMATCH_PROTO, errpos, "Definition %s, cannot override this kind of definition", str(name)); + reportError(ERR_MISMATCH_PROTO, errpos, "Definition %s, cannot override this kind of definition", nullText(str(name))); else { if (!areSymbolsCompatible(expr, isParametered, parameters, match)) - reportError(ERR_MISMATCH_PROTO, errpos, "Prototypes for %s in base and derived modules must match", str(name)); + reportError(ERR_MISMATCH_PROTO, errpos, "Prototypes for %s in base and derived modules must match", nullText(str(name))); } } } diff --git a/ecl/hql/hqlrepository.cpp b/ecl/hql/hqlrepository.cpp index 4a6ad7e5b75..75ca659e302 100644 --- a/ecl/hql/hqlrepository.cpp +++ b/ecl/hql/hqlrepository.cpp @@ -959,7 +959,7 @@ unsigned EclRepositoryManager::runGitCommand(StringBuffer * output, const char * } else { - Owned secret = getSecret("git", options.gitUser.str()); + Owned secret = getSecret("git", options.gitUser.str()); if (secret) { MemoryBuffer gitKey; diff --git a/esp/bindings/http/platform/httpprot.cpp b/esp/bindings/http/platform/httpprot.cpp index d4b2b69285e..e4e6af64ec1 100644 --- a/esp/bindings/http/platform/httpprot.cpp +++ b/esp/bindings/http/platform/httpprot.cpp @@ -216,32 +216,38 @@ CSecureHttpProtocol::CSecureHttpProtocol(IPropertyTree* cfg) { m_config.setown(cfg); - //ensure keys are specified. Passphrase is optional - StringBuffer sb; - cfg->getProp("certificate", sb); - if(sb.length() == 0) - { - throw MakeStringException(-1, "certificate file not specified in config file"); - } + IEspPlugin *pplg = loadPlugin(SSLIB); + if (!pplg) + throw MakeStringException(-1, "dll/shared-object %s can't be loaded", SSLIB); - cfg->getProp("privatekey", sb.clear()); - if(sb.length() == 0) + const char *issuer = cfg->queryProp("issuer"); + if (!isEmptyString(issuer)) { - throw MakeStringException(-1, "private key file not specified in config file"); + const char *trustedPeers = nullptr; + if (cfg->hasProp("verify")) + trustedPeers = cfg->queryProp("verify/trusted_peers"); + createSecureSocketContextSecretSrv_t xproc = (createSecureSocketContextSecretSrv_t) pplg->getProcAddress("createSecureSocketContextSecretSrv"); + if (!xproc) + throw MakeStringException(-1, "procedure createSecureSocketContextSecretSrv can't be loaded"); + m_ssctx.setown(xproc(issuer, trustedPeers, false)); } - - createSecureSocketContextEx2_t xproc = NULL; - IEspPlugin *pplg = loadPlugin(SSLIB); - if (pplg) - xproc = (createSecureSocketContextEx2_t) pplg->getProcAddress("createSecureSocketContextEx2"); else - throw MakeStringException(-1, "dll/shared-object %s can't be loaded", SSLIB); - - - if (xproc) + { + //ensure keys are specified. Passphrase is optional + StringBuffer sb; + cfg->getProp("certificate", sb); + if(sb.isEmpty()) + throw MakeStringException(-1, "certificate file not specified in config file"); + + cfg->getProp("privatekey", sb.clear()); + if(sb.isEmpty()) + throw MakeStringException(-1, "private key file not specified in config file"); + + createSecureSocketContextEx2_t xproc = (createSecureSocketContextEx2_t) pplg->getProcAddress("createSecureSocketContextEx2"); + if (!xproc) + throw MakeStringException(-1, "procedure createSecureSocketContextEx2 can't be loaded"); m_ssctx.setown(xproc(cfg, ServerSocket)); - else - throw MakeStringException(-1, "procedure createSecureSocketContextEx2 can't be loaded"); + } } } diff --git a/esp/clients/ws_dfsclient/ws_dfsclient.cpp b/esp/clients/ws_dfsclient/ws_dfsclient.cpp index 6a9045952f0..8270b912a8b 100644 --- a/esp/clients/ws_dfsclient/ws_dfsclient.cpp +++ b/esp/clients/ws_dfsclient/ws_dfsclient.cpp @@ -599,7 +599,7 @@ IClientWsDfs *getDfsClient(const char *serviceUrl, IUserDescriptor *userDesc) static void configureClientSSL(IEspClientRpcSettings &rpc, const char *secretName) { - Owned secretPTree = getSecret("storage", secretName); + Owned secretPTree = getSecret("storage", secretName); if (!secretPTree) throw makeStringExceptionV(-1, "secret %s.%s not found", "storage", secretName); diff --git a/esp/esdlscriptlib/esdl_script.cpp b/esp/esdlscriptlib/esdl_script.cpp index 4c4c24b7f5c..6d6427387f8 100644 --- a/esp/esdlscriptlib/esdl_script.cpp +++ b/esp/esdlscriptlib/esdl_script.cpp @@ -911,7 +911,7 @@ class CEsdlTransformOperationMySqlCall : public CEsdlTransformOperationBase recordException(ESDL_SCRIPT_MissingOperationAttr, msg.append(name)); } } - IPropertyTree *getSecretInfo(IXpathContext * sourceContext) + const IPropertyTree *getSecretInfo(IXpathContext * sourceContext) { //leaving flexibility for the secret to be configured multiple ways // the most secure option in my opinion is to at least have the server, name, and password all in the secret @@ -943,7 +943,7 @@ class CEsdlTransformOperationMySqlCall : public CEsdlTransformOperationBase options.append(name).append('=').append(value); } - void appendOption(StringBuffer &options, const char *name, IXpathContext * sourceContext, ICompiledXpath *cx, IPropertyTree *secret, bool required) + void appendOption(StringBuffer &options, const char *name, IXpathContext * sourceContext, ICompiledXpath *cx, const IPropertyTree *secret, bool required) { if (secret && secret->hasProp(name)) { @@ -971,7 +971,7 @@ class CEsdlTransformOperationMySqlCall : public CEsdlTransformOperationBase } IEmbedFunctionContext *createFunctionContext(IXpathContext * sourceContext) { - Owned secret = getSecretInfo(sourceContext); + Owned secret = getSecretInfo(sourceContext); StringBuffer options; appendOption(options, "server", sourceContext, m_server, secret, true); appendOption(options, "user", sourceContext, m_user, secret, true); diff --git a/esp/services/ws_workunits/ws_workunitsService.cpp b/esp/services/ws_workunits/ws_workunitsService.cpp index b912dc0caa9..ffbdf933181 100644 --- a/esp/services/ws_workunits/ws_workunitsService.cpp +++ b/esp/services/ws_workunits/ws_workunitsService.cpp @@ -3638,6 +3638,16 @@ void getScheduledWUs(IEspContext &context, WUShowScheduledFilters *filters, cons { jobName.set(cw->queryJobName()); owner.set(cw->queryUser()); + if ((cw->getState() == WUStateScheduled) && cw->aborting()) + { + stateID = WUStateAborting; + state.set("aborting"); + } + else + { + stateID = cw->getState(); + state.set(cw->queryStateDesc()); + } } if (!filters->jobName.isEmpty() && (jobName.isEmpty() || !WildMatch(jobName.str(), filters->jobName, true))) @@ -3646,26 +3656,8 @@ void getScheduledWUs(IEspContext &context, WUShowScheduledFilters *filters, cons match = false; else if (!filters->eventText.isEmpty() && (ieventText.isEmpty() || !WildMatch(ieventText, filters->eventText, true))) match = false; - else if (!filters->state.isEmpty()) - { - if (!cw) - match = false; - else - { - if ((cw->getState() == WUStateScheduled) && cw->aborting()) - { - stateID = WUStateAborting; - state.set("aborting"); - } - else - { - stateID = cw->getState(); - state.set(cw->queryStateDesc()); - } - if (!strieq(filters->state, state.str())) - match = false; - } - } + else if (!filters->state.isEmpty() && !strisame(filters->state, state.str())) + match = false; } catch (IException *e) { diff --git a/esp/src/eclwatch/PermissionsWidget.js b/esp/src/eclwatch/PermissionsWidget.js index 5a0bf49fe7f..77d7a166570 100644 --- a/esp/src/eclwatch/PermissionsWidget.js +++ b/esp/src/eclwatch/PermissionsWidget.js @@ -11,7 +11,12 @@ define([ "hpcc/GridDetailsWidget", "src/ws_access", "src/ESPUtil", - "src/Utility" + "src/Utility", + + "dijit/Dialog", + "dijit/form/TextBox", + + "hpcc/TableContainer" ], function (declare, nlsHPCCMod, registry, CheckBox, diff --git a/esp/src/eclwatch/ShowAccountPermissionsWidget.js b/esp/src/eclwatch/ShowAccountPermissionsWidget.js index cd87980db35..05fe65e894c 100644 --- a/esp/src/eclwatch/ShowAccountPermissionsWidget.js +++ b/esp/src/eclwatch/ShowAccountPermissionsWidget.js @@ -13,7 +13,12 @@ define([ "src/ws_access", "src/ESPUtil", "src/Utility", - "hpcc/ShowInheritedPermissionsWidget" + "hpcc/ShowInheritedPermissionsWidget", + + "dijit/Dialog", + "dijit/form/TextBox", + + "hpcc/TableContainer" ], function (declare, lang, nlsHPCCMod, arrayUtil, registry, CheckBox, diff --git a/esp/src/package-lock.json b/esp/src/package-lock.json index 7b0b6bb4a6f..c982cc67955 100644 --- a/esp/src/package-lock.json +++ b/esp/src/package-lock.json @@ -14,10 +14,10 @@ "@fluentui/react-experiments": "8.14.95", "@fluentui/react-hooks": "8.6.29", "@fluentui/react-icons-mdl2": "1.3.47", - "@hpcc-js/chart": "2.81.8", - "@hpcc-js/codemirror": "2.60.13", - "@hpcc-js/common": "2.71.13", - "@hpcc-js/comms": "2.84.5", + "@hpcc-js/chart": "2.81.7", + "@hpcc-js/codemirror": "2.60.12", + "@hpcc-js/common": "2.71.12", + "@hpcc-js/comms": "2.85.0", "@hpcc-js/dataflow": "8.1.6", "@hpcc-js/eclwatch": "2.73.29", "@hpcc-js/graph": "2.85.9", @@ -1577,9 +1577,9 @@ } }, "node_modules/@hpcc-js/comms": { - "version": "2.84.5", - "resolved": "https://registry.npmjs.org/@hpcc-js/comms/-/comms-2.84.5.tgz", - "integrity": "sha512-anm1u1ssaQq/cVi9/EGhXW6zcZCMbdQ7wvL3YVcWsxPG7Wyis3gvEZ99pzuRGdqtN5Kmuhm1l4fsolqI4n/F8w==", + "version": "2.85.0", + "resolved": "https://registry.npmjs.org/@hpcc-js/comms/-/comms-2.85.0.tgz", + "integrity": "sha512-ZnvI7T35qyj6Dm1uT7f+j1Xgq0QGPyMAypGq8jacq2VAo2Q24eg/AL57E8PBLOM1Q0UAV/WYJqz4eTk9n+UwgQ==", "dependencies": { "@hpcc-js/ddl-shim": "^2.20.6", "@hpcc-js/util": "^2.50.6", diff --git a/esp/src/package.json b/esp/src/package.json index bbecfb3da94..895bbcbf876 100644 --- a/esp/src/package.json +++ b/esp/src/package.json @@ -39,10 +39,10 @@ "@fluentui/react-experiments": "8.14.95", "@fluentui/react-hooks": "8.6.29", "@fluentui/react-icons-mdl2": "1.3.47", - "@hpcc-js/chart": "2.81.8", - "@hpcc-js/codemirror": "2.60.13", - "@hpcc-js/common": "2.71.13", - "@hpcc-js/comms": "2.84.5", + "@hpcc-js/chart": "2.81.7", + "@hpcc-js/codemirror": "2.60.12", + "@hpcc-js/common": "2.71.12", + "@hpcc-js/comms": "2.85.0", "@hpcc-js/dataflow": "8.1.6", "@hpcc-js/eclwatch": "2.73.29", "@hpcc-js/graph": "2.85.9", diff --git a/esp/src/src-react/hooks/workunit.ts b/esp/src/src-react/hooks/workunit.ts index 783061de755..9b331369fbc 100644 --- a/esp/src/src-react/hooks/workunit.ts +++ b/esp/src/src-react/hooks/workunit.ts @@ -23,7 +23,7 @@ export function useWorkunit(wuid: string, full: boolean = false): [Workunit, WUS let active = true; let handle; const refresh = singletonDebounce(wu, "refresh"); - refresh(full) + refresh(full, { IncludeTotalClusterTime: true }) .then(() => { if (active) { setRetVal({ workunit: wu, state: wu.StateID, lastUpdate: Date.now(), isComplete: wu.isComplete(), refresh }); diff --git a/esp/src/src-react/layouts/DojoAdapter.tsx b/esp/src/src-react/layouts/DojoAdapter.tsx index f48e3a5455c..1861875a3a6 100644 --- a/esp/src/src-react/layouts/DojoAdapter.tsx +++ b/esp/src/src-react/layouts/DojoAdapter.tsx @@ -60,8 +60,12 @@ export const DojoAdapter: React.FunctionComponent = ({ }, ...delayProps }, elem); - widget.startup(); - widget.resize(); + if (widget.startup) { + widget.startup(); + } + if (widget.resize) { + widget.resize(); + } if (widget.init) { widget.init(params || {}); } diff --git a/helm/examples/tracing/README.md b/helm/examples/tracing/README.md index bd1778a4aa5..5caba096c10 100644 --- a/helm/examples/tracing/README.md +++ b/helm/examples/tracing/README.md @@ -10,6 +10,9 @@ All configuration options detailed here are part of the HPCC Systems Helm chart, ### Tracing cofiguration options - disabled - (default: false) disables tracking and reporting of internal traces and spans - alwaysCreateGlobalIds - If true, assign newly created global ID to any requests that do not supply one. +- optAlwaysCreateTraceIds - If true components generate trace/span ids if none are provided by the remote caller. +- logSpanStart - If true, generate a log entry whenever a span is started (default: false) +- logSpanFinish - If true, generate a log entry whenever a span is finished (default: true) - exporter - Defines The type of exporter in charge of forwarding span data to target back-end - type - (defalt: NONE) "OTLP-HTTP" | "OTLP-GRCP" | "OS" | "NONE" - OTLP-HTTP diff --git a/helm/hpcc/templates/esp.yaml b/helm/hpcc/templates/esp.yaml index b4e04617713..8120ef58369 100644 --- a/helm/hpcc/templates/esp.yaml +++ b/helm/hpcc/templates/esp.yaml @@ -45,6 +45,7 @@ data: tls: {{ include "hpcc.isIssuerEnabled" (dict "root" .root "issuerKeyName" $issuerKeyName) }} {{- end }} tls_config: + issuer: {{ $issuerKeyName }} {{- if $externalCert }} certificate: /opt/HPCCSystems/secrets/certificates/{{ $issuerKeyName }}/tls.crt privatekey: /opt/HPCCSystems/secrets/certificates/{{ $issuerKeyName }}/tls.key diff --git a/helm/hpcc/values.schema.json b/helm/hpcc/values.schema.json index c57bd13ae45..d73687e2039 100644 --- a/helm/hpcc/values.schema.json +++ b/helm/hpcc/values.schema.json @@ -1114,7 +1114,7 @@ }, "alwaysCreateTraceIds": { "type": "boolean", - "description": "If true, components generate trace/span ids to aid in unit of worktracing" + "description": "If true, components generate trace/span ids if none are provided by the remote caller" }, "exporter": { "type": "object", @@ -1135,6 +1135,16 @@ "description": "Defines the manner in which trace data is processed - in batches, or simple as available" } } + }, + "logSpanStart": { + "type": "boolean", + "description": "If true, generate a log entry whenever a span is started", + "default": false + }, + "logSpanFinish": { + "type": "boolean", + "description": "If true, generate a log entry whenever a span is finished", + "default": true } }, "additionalProperties": { "type": ["integer", "string", "boolean"] } diff --git a/plugins/CMakeLists.txt b/plugins/CMakeLists.txt index 109e74b7abb..63de3d1cc68 100644 --- a/plugins/CMakeLists.txt +++ b/plugins/CMakeLists.txt @@ -30,6 +30,7 @@ add_subdirectory (proxies) add_subdirectory (sqlite3) add_subdirectory (mysql) add_subdirectory (v8embed) +add_subdirectory (wasmembed) HPCC_ADD_SUBDIRECTORY (py3embed "USE_PYTHON3") HPCC_ADD_SUBDIRECTORY (pyembed "USE_PYTHON2") add_subdirectory (javaembed) diff --git a/plugins/wasmembed/CMakeLists.txt b/plugins/wasmembed/CMakeLists.txt new file mode 100644 index 00000000000..7dff7328253 --- /dev/null +++ b/plugins/wasmembed/CMakeLists.txt @@ -0,0 +1,66 @@ +project(wasmembed) + +if(WASMEMBED) + ADD_PLUGIN(wasmembed) + if(MAKE_WASMEMBED) + + set(CMAKE_CXX_STANDARD 20) + find_path(WASMTIME_CPP_API_INCLUDE_DIRS "wasmtime-cpp-api/wasmtime.hh" + PATHS ${VCPKG_INSTALLED_DIR}/${VCPKG_TARGET_TRIPLET} + ) + if (WIN32) + find_library(WASMTIME_LIB NAMES wasmtime.dll + PATHS ${VCPKG_INSTALLED_DIR}/${VCPKG_TARGET_TRIPLET} + ) + else() + find_library(WASMTIME_LIB NAMES wasmtime + PATHS ${VCPKG_INSTALLED_DIR}/${VCPKG_TARGET_TRIPLET} + ) + endif() + + include_directories( + ${WASMTIME_CPP_API_INCLUDE_DIRS}/wasmtime-c-api + ${WASMTIME_CPP_API_INCLUDE_DIRS}/wasmtime-cpp-api + ./../../system/include + ./../../system/jlib + ./../../rtl/eclrtl + ./../../rtl/include + + # Following includes are needed for "enginecontext.hpp" + ./../../common/thorhelper + ./../../dali/base + ./../../system/mp + ) + + add_definitions(-D_USRDLL -DWASMEMBED_EXPORTS) + + add_library(wasmembed SHARED + wasmembed.cpp + secure-enclave.cpp + abi.cpp + util.cpp + ) + + target_link_libraries(wasmembed + ${WASMTIME_LIB} + eclrtl + jlib + ) + + install( + TARGETS wasmembed + DESTINATION plugins + CALC_DEPS + ) + + else() + message(WARNING "Cannot build wasmembed plugin") + endif() +endif() + +if(PLATFORM OR CLIENTTOOLS_ONLY) + install( + FILES ${CMAKE_CURRENT_SOURCE_DIR}/wasm.ecllib + DESTINATION plugins + COMPONENT Runtime) +endif() diff --git a/plugins/wasmembed/abi.cpp b/plugins/wasmembed/abi.cpp new file mode 100644 index 00000000000..43b91339e4b --- /dev/null +++ b/plugins/wasmembed/abi.cpp @@ -0,0 +1,269 @@ +/* + See: https://github.com/WebAssembly/component-model/blob/main/design/mvp/CanonicalABI.md + https://github.com/WebAssembly/component-model/blob/main/design/mvp/canonical-abi/definitions.py +*/ + +#include "abi.hpp" + +#include "jexcept.hpp" + +auto UTF16_TAG = 1U << 31; + +// +/* canonical despecialize (python) ------------------------------------------------------------- + +def despecialize(t): + match t: + case Tuple(ts) : return Record([ Field(str(i), t) for i,t in enumerate(ts) ]) + case Union(ts) : return Variant([ Case(str(i), t) for i,t in enumerate(ts) ]) + case Enum(labels) : return Variant([ Case(l, None) for l in labels ]) + case Option(t) : return Variant([ Case("none", None), Case("some", t) ]) + case Result(ok, error) : return Variant([ Case("ok", ok), Case("error", error) ]) + case _ : return t + +*/ + +// template +// wasmtime::ValType despecialize(const T &t) +// { +// switch (t.kind()) +// { +// case wasmtime::ValKind::I32: +// case wasmtime::ValKind::I64: +// case wasmtime::ValKind::F32: +// case wasmtime::ValKind::F64: +// case wasmtime::ValKind::V128: +// return t.kind(); +// default: +// return wasmtime::ValType::i32(); +// } +// } + +/* canonical alignment (python) ------------------------------------------------------------- + +def alignment(t): + match despecialize(t): + case Bool() : return 1 + case S8() | U8() : return 1 + case S16() | U16() : return 2 + case S32() | U32() : return 4 + case S64() | U64() : return 8 + case Float32() : return 4 + case Float64() : return 8 + case Char() : return 4 + case String() | List(_) : return 4 + case Record(fields) : return alignment_record(fields) + case Variant(cases) : return alignment_variant(cases) + case Flags(labels) : return alignment_flags(labels) + case Own(_) | Borrow(_) : return 4 + +*/ + +// int alignment(const wasmtime::ValType &t) +// { +// switch (t.kind()) +// { +// case wasmtime::ValKind::I32: +// case wasmtime::ValKind::F32: +// return 4; +// case wasmtime::ValKind::I64: +// case wasmtime::ValKind::F64: +// return 8; +// case wasmtime::ValKind::V128: +// return 16; +// default: +// return 1; +// } +// } + +/* canonical align_to (python) ------------------------------------------------------------- + +def align_to(ptr, alignment): + return math.ceil(ptr / alignment) * alignment + +*/ + +uint32_t align_to(uint32_t ptr, uint32_t alignment) +{ + return (ptr + alignment - 1) & ~(alignment - 1); +} + +bool isAligned(uint32_t ptr, uint32_t alignment) +{ + return (ptr & (alignment - 1)) == 0; +} + +// loading --- + +/* canonical load_int (python) ------------------------------------------------------------- + +def load_int(cx, ptr, nbytes, signed = False): + return int.from_bytes(cx.opts.memory[ptr : ptr+nbytes], 'little', signed=signed) + +*/ + +template +T load_int(const wasmtime::Span &data, uint32_t ptr) +{ + T retVal = 0; + if constexpr (sizeof(T) == 1) + { + retVal = static_cast(data[ptr]); + } + else if constexpr (sizeof(T) == 2) + { + retVal = static_cast((static_cast(data[ptr + 1]) << 8) | + static_cast(data[ptr])); + } + else if constexpr (sizeof(T) == 4) + { + retVal = static_cast((static_cast(data[ptr + 3]) << 24) | + (static_cast(data[ptr + 2]) << 16) | + (static_cast(data[ptr + 1]) << 8) | + static_cast(data[ptr])); + } + else if constexpr (sizeof(T) == 8) + { + retVal = static_cast((static_cast(data[ptr + 7]) << 56) | + (static_cast(data[ptr + 6]) << 48) | + (static_cast(data[ptr + 5]) << 40) | + (static_cast(data[ptr + 4]) << 32) | + (static_cast(data[ptr + 3]) << 24) | + (static_cast(data[ptr + 2]) << 16) | + (static_cast(data[ptr + 1]) << 8) | + static_cast(data[ptr])); + } + return retVal; +} +/* canonical load_string_from_range (python) ------------------------------------------------------------- + +def load_string_from_range(cx, ptr, tagged_code_units): + match cx.opts.string_encoding: + case 'utf8': + alignment = 1 + byte_length = tagged_code_units + encoding = 'utf-8' + case 'utf16': + alignment = 2 + byte_length = 2 * tagged_code_units + encoding = 'utf-16-le' + case 'latin1+utf16': + alignment = 2 + if bool(tagged_code_units & UTF16_TAG): + byte_length = 2 * (tagged_code_units ^ UTF16_TAG) + encoding = 'utf-16-le' + else: + byte_length = tagged_code_units + encoding = 'latin-1' + + trap_if(ptr != align_to(ptr, alignment)) + trap_if(ptr + byte_length > len(cx.opts.memory)) + try: + s = cx.opts.memory[ptr : ptr+byte_length].decode(encoding) + except UnicodeError: + trap() + + return (s, cx.opts.string_encoding, tagged_code_units) + +*/ + +// More: Not currently available from the wasmtime::context object, see https://github.com/bytecodealliance/wasmtime/issues/6719 +static const std::string global_encoding = "utf8"; + +std::tuple load_string_from_range(const wasmtime::Span &data, uint32_t ptr, uint32_t tagged_code_units) +{ + std::string encoding = "utf-8"; + uint32_t byte_length = tagged_code_units; + uint32_t alignment = 1; + if (global_encoding.compare("utf8") == 0) + { + alignment = 1; + byte_length = tagged_code_units; + encoding = "utf-8"; + } + else if (global_encoding.compare("utf16") == 0) + { + alignment = 2; + byte_length = 2 * tagged_code_units; + encoding = "utf-16-le"; + } + else if (global_encoding.compare("latin1+utf16") == 0) + { + alignment = 2; + if (tagged_code_units & UTF16_TAG) + { + byte_length = 2 * (tagged_code_units ^ UTF16_TAG); + encoding = "utf-16-le"; + } + else + { + byte_length = tagged_code_units; + encoding = "latin-1"; + } + } + + if (!isAligned(ptr, alignment)) + { + throw makeStringException(3, "Invalid alignment"); + } + + if (ptr + byte_length > data.size()) + { + throw makeStringException(1, "Out of bounds"); + } + + return std::make_tuple(ptr, encoding, byte_length); +} + +/* canonical load_string (python) ------------------------------------------------------------- + +def load_string(cx, ptr): + begin = load_int(cx, ptr, 4) + tagged_code_units = load_int(cx, ptr + 4, 4) + return load_string_from_range(cx, begin, tagged_code_units) + +*/ +std::tuple load_string(const wasmtime::Span &data, uint32_t ptr) +{ + uint32_t begin = load_int(data, ptr); + uint32_t tagged_code_units = load_int(data, ptr + 4); + return load_string_from_range(data, begin, tagged_code_units); +} + +/* canonical load_list_from_range (python) ------------------------------------------------------------- + +def load_list_from_range(cx, ptr, length, elem_type): + trap_if(ptr != align_to(ptr, alignment(elem_type))) + trap_if(ptr + length * size(elem_type) > len(cx.opts.memory)) + a = [] + for i in range(length): + a.append(load(cx, ptr + i * size(elem_type), elem_type)) + return a + +*/ + +template +std::vector load_list_from_range(const wasmtime::Span &data, uint32_t ptr, uint32_t length) +{ + if (!isAligned(ptr, alignment(T{}))) + throw makeStringException(2, "Pointer is not aligned"); + if (ptr + length * sizeof(T) > data.size()) + throw makeStringException(1, "Out of bounds access"); + std::vector a; + for (uint32_t i = 0; i < length; i++) + { + a.push_back(load(data, ptr + i * sizeof(T))); + } + return a; +} + +/* canonical load_list (python) ------------------------------------------------------------- + +def load_list(cx, ptr, elem_type): + begin = load_int(cx, ptr, 4) + length = load_int(cx, ptr + 4, 4) + return load_list_from_range(cx, begin, length, elem_type) + +*/ + +// Storing --- diff --git a/plugins/wasmembed/abi.hpp b/plugins/wasmembed/abi.hpp new file mode 100644 index 00000000000..38f09fe1020 --- /dev/null +++ b/plugins/wasmembed/abi.hpp @@ -0,0 +1,3 @@ +#include + +std::tuple load_string(const wasmtime::Span &data, uint32_t ptr); diff --git a/plugins/wasmembed/hpcc-platform.wit b/plugins/wasmembed/hpcc-platform.wit new file mode 100644 index 00000000000..9ff9693c540 --- /dev/null +++ b/plugins/wasmembed/hpcc-platform.wit @@ -0,0 +1,25 @@ +/* WebAssembly Component Interface + + See: https://component-model.bytecodealliance.org/design/wit.html + + HPCC-Platform is the "host". + The WebAssembly module is the "guest". + + Memory management rules: + * Guests calling host functions: + - Guests must dispose all params as needed + - Guests must dispose "results" as needed + * Hosts calling guest functions: + - Guests must dispose all params as needed + - Hosts must dispose "results" as needed by + calling cabi_post_XXX for each function call + */ + +package hpcc-systems:hpcc-platform + +world wasmembed { + + import dbglog: func(msg: string) + + // export myfunc(params: string) -> string +} diff --git a/plugins/wasmembed/secure-enclave.cpp b/plugins/wasmembed/secure-enclave.cpp new file mode 100644 index 00000000000..07a76f7d656 --- /dev/null +++ b/plugins/wasmembed/secure-enclave.cpp @@ -0,0 +1,626 @@ +#include "secure-enclave.hpp" + +#include "eclrtl_imp.hpp" +#include "jexcept.hpp" +#include "jiface.hpp" +#include "eclhelper.hpp" +#include "enginecontext.hpp" + +#include "abi.hpp" +#include "util.hpp" + +#include +#include + +// From deftype.hpp in common +#define UNKNOWN_LENGTH 0xFFFFFFF1 + +// #define ENABLE_TRACE +#ifdef ENABLE_TRACE +#define TRACE(format, ...) DBGLOG(format __VA_OPT__(, ) __VA_ARGS__) +#else +#define TRACE(format, ...) \ + do \ + { \ + } while (0) +#endif +class WasmEngine +{ +private: + std::once_flag wasmLoadedFlag; + std::unordered_map wasmModules; + + wasmtime::Module createModule(const std::string &wasmName, const wasmtime::Span &wasm) + { + TRACE("WasmEngine createModule %s", wasmName.c_str()); + try + { + wasmtime::Store store(engine); + wasmtime::WasiConfig wasi; + wasi.inherit_argv(); + wasi.inherit_env(); + wasi.inherit_stdin(); + wasi.inherit_stdout(); + wasi.inherit_stderr(); + store.context().set_wasi(std::move(wasi)).unwrap(); + return wasmtime::Module::compile(engine, wasm).unwrap(); + } + catch (const wasmtime::Error &e) + { + throw makeStringExceptionV(100, "WasmEngine createModule failed: %s", e.message().c_str()); + } + } + + void loadWasmFiles(ICodeContext *codeCtx) + { + TRACE("WasmEngine loadWasmFiles"); + IEngineContext *engine = codeCtx->queryEngineContext(); + if (!engine) + throw makeStringException(100, "Failed to get engine context"); + + StringArray manifestModules; + engine->getManifestFiles("wasm", manifestModules); + + ForEachItemIn(idx, manifestModules) + { + const char *path = manifestModules.item(idx); + TRACE("WasmEngine loadWasmFiles %s", path); + std::vector contents = readWasmBinaryToBuffer(path); + auto module = createModule(path, contents); + std::filesystem::path p(path); + wasmModules.insert(std::make_pair(p.stem(), module)); + } + } + +public: + wasmtime::Engine engine; + + WasmEngine() + { + TRACE("WASM SE WasmEngine"); + } + + ~WasmEngine() + { + TRACE("WASM SE ~WasmEngine"); + } + + void setCodeContext(ICodeContext *codeCtx) + { + TRACE("WASM SE setCodeContext"); + std::call_once(wasmLoadedFlag, &WasmEngine::loadWasmFiles, this, codeCtx); + } + + bool hasModule(const std::string &wasmName) const + { + TRACE("WASM SE hasModule"); + return wasmModules.find(wasmName) != wasmModules.end(); + } + + wasmtime::Module getModule(const std::string &wasmName) const + { + TRACE("WASM SE getModule"); + auto found = wasmModules.find(wasmName); + if (found == wasmModules.end()) + throw makeStringExceptionV(100, "Wasm module not found: %s", wasmName.c_str()); + return found->second; + } +}; +static std::unique_ptr wasmEngine = std::make_unique(); + +class WasmStore +{ +private: + wasmtime::Store store; + + std::unordered_map wasmInstances; + std::unordered_map wasmMems; + std::unordered_map wasmFuncs; + +public: + WasmStore() : store(wasmEngine->engine) + { + TRACE("WASM SE WasmStore"); + } + + ~WasmStore() + { + TRACE("WASM SE ~WasmStore"); + } + + bool hasInstance(const std::string &wasmName) const + { + TRACE("WASM SE hasInstance"); + return wasmInstances.find(wasmName) != wasmInstances.end(); + } + + wasmtime::Instance getInstance(const std::string &wasmName) const + { + TRACE("WASM SE getInstance"); + auto found = wasmInstances.find(wasmName); + if (found == wasmInstances.end()) + throw makeStringExceptionV(100, "Wasm instance not found: %s", wasmName.c_str()); + return found->second; + } + + void registerInstance(const std::string &wasmName) + { + TRACE("WASM SE registerInstance %s", wasmName.c_str()); + if (hasInstance(wasmName)) + { + throw makeStringExceptionV(100, "Wasm instance already registered: %s", wasmName.c_str()); + } + TRACE("WASM SE createInstance %s", wasmName.c_str()); + auto module = wasmEngine->getModule(wasmName); + try + { + wasmtime::Linker linker(wasmEngine->engine); + linker.define_wasi().unwrap(); + + auto callback = [this, wasmName](wasmtime::Caller caller, uint32_t msg, uint32_t msg_len) + { + auto data = this->getData(wasmName); + auto msg_ptr = (char *)&data[msg]; + std::string str(msg_ptr, msg_len); + DBGLOG("from wasm: %s", str.c_str()); + }; + auto host_func = linker.func_wrap("$root", "dbglog", callback).unwrap(); + + auto newInstance = linker.instantiate(store, module).unwrap(); + linker.define_instance(store, "linking2", newInstance).unwrap(); + + for (auto exportItem : module.exports()) + { + auto externType = wasmtime::ExternType::from_export(exportItem); + std::string name(exportItem.name()); + if (std::holds_alternative(externType)) + { + TRACE("WASM SE Exported function: %s", name.c_str()); + auto func = std::get(*newInstance.get(store, name)); + wasmFuncs.insert(std::make_pair(wasmName + "." + name, func)); + } + else if (std::holds_alternative(externType)) + { + TRACE("WASM SE Exported memory: %s", name.c_str()); + auto memory = std::get(*newInstance.get(store, name)); + wasmMems.insert(std::make_pair(wasmName + "." + name, memory)); + } + else if (std::holds_alternative(externType)) + { + TRACE("WASM SE Exported table: %s", name.c_str()); + } + else if (std::holds_alternative(externType)) + { + TRACE("WASM SE Exported global: %s", name.c_str()); + } + else + { + TRACE("WASM SE Unknown export type"); + } + } + wasmInstances.insert(std::make_pair(wasmName, newInstance)); + } + catch (const wasmtime::Error &e) + { + throw makeStringExceptionV(100, "WASM SE createInstance: %s", e.message().c_str()); + } + } + + bool hasFunc(const std::string &qualifiedID) const + { + TRACE("WASM SE hasFunc"); + return wasmFuncs.find(qualifiedID) != wasmFuncs.end(); + } + + wasmtime::Func getFunc(const std::string &qualifiedID) const + { + TRACE("WASM SE getFunc"); + auto found = wasmFuncs.find(qualifiedID); + if (found == wasmFuncs.end()) + throw makeStringExceptionV(100, "Wasm function not found: %s", qualifiedID.c_str()); + return found->second; + } + + wasmtime::ValType::ListRef getFuncParams(const std::string &qualifiedID) + { + TRACE("WASM SE getFuncParams"); + auto func = getFunc(qualifiedID); + wasmtime::FuncType funcType = func.type(store.context()); + return funcType->params(); + } + + wasmtime::ValType::ListRef getFuncResults(const std::string &qualifiedID) + { + TRACE("WASM SE getFuncResults"); + auto func = getFunc(qualifiedID); + wasmtime::FuncType funcType = func.type(store.context()); + return funcType->results(); + } + + std::vector call(const std::string &qualifiedID, const std::vector ¶ms) + { + TRACE("WASM SE call"); + auto func = getFunc(qualifiedID); + try + { + auto retVal = func.call(store, params).unwrap(); + return retVal; + } + catch (const wasmtime::Trap &e) + { + throw makeStringExceptionV(100, "WASM SE call: %s", e.message().c_str()); + } + } + + std::vector callRealloc(const std::string &wasmName, const std::vector ¶ms) + { + TRACE("WASM SE callRealloc"); + return call(createQualifiedID(wasmName, "cabi_realloc"), params); + } + + wasmtime::Span getData(const std::string &wasmName) + { + TRACE("WASM SE getData"); + auto found = wasmMems.find(createQualifiedID(wasmName, "memory")); + if (found == wasmMems.end()) + throw makeStringExceptionV(100, "Wasm memory not found: %s", wasmName.c_str()); + return found->second.data(store.context()); + } +}; +thread_local std::unique_ptr wasmStore = std::make_unique(); + +class SecureFunction : public CInterfaceOf +{ + std::string wasmName; + std::string funcName; + std::string qualifiedID; + + std::vector args; + std::vector wasmResults; + +public: + SecureFunction(ICodeContext *codeCtx) + { + TRACE("WASM SE se:constructor"); + wasmEngine->setCodeContext(codeCtx); + } + + virtual ~SecureFunction() + { + TRACE("WASM SE se:destructor"); + + // Garbage Collection --- + // Function results --- + auto gc_func_name = createQualifiedID(wasmName, "cabi_post_" + funcName); + if (wasmStore->hasFunc(gc_func_name)) + { + for (auto &result : wasmResults) + { + wasmStore->call(gc_func_name, {result}); + } + } + } + + // IEmbedFunctionContext --- + void setActivityContext(const IThorActivityContext *activityCtx) + { + } + + virtual IInterface *bindParamWriter(IInterface *esdl, const char *esdlservice, const char *esdltype, const char *name) + { + TRACE("WASM SE paramWriterCommit"); + return NULL; + } + virtual void paramWriterCommit(IInterface *writer) + { + TRACE("WASM SE paramWriterCommit"); + } + virtual void writeResult(IInterface *esdl, const char *esdlservice, const char *esdltype, IInterface *writer) + { + TRACE("WASM SE writeResult"); + } + virtual void bindBooleanParam(const char *name, bool val) + { + TRACE("WASM SE bindBooleanParam %s %i", name, val); + args.push_back(val); + } + virtual void bindDataParam(const char *name, size32_t len, const void *val) + { + TRACE("WASM SE bindDataParam %s %d", name, len); + } + virtual void bindFloatParam(const char *name, float val) + { + TRACE("WASM SE bindFloatParam %s %f", name, val); + args.push_back(val); + } + virtual void bindRealParam(const char *name, double val) + { + TRACE("WASM SE bindRealParam %s %f", name, val); + args.push_back(val); + } + virtual void bindSignedSizeParam(const char *name, int size, __int64 val) + { + TRACE("WASM SE bindSignedSizeParam %s %i %lld", name, size, val); + if (size <= 4) + args.push_back(static_cast(val)); + else + args.push_back(static_cast(val)); + } + virtual void bindSignedParam(const char *name, __int64 val) + { + TRACE("WASM SE bindSignedParam %s %lld", name, val); + args.push_back(static_cast(val)); + } + virtual void bindUnsignedSizeParam(const char *name, int size, unsigned __int64 val) + { + TRACE("WASM SE bindUnsignedSizeParam %s %i %llu", name, size, val); + if (size <= 4) + args.push_back(static_cast(val)); + else + args.push_back(static_cast(val)); + } + virtual void bindUnsignedParam(const char *name, unsigned __int64 val) + { + TRACE("WASM SE bindUnsignedParam %s %llu", name, val); + args.push_back(static_cast(val)); + } + virtual void bindStringParam(const char *name, size32_t bytes, const char *val) + { + TRACE("WASM SE bindStringParam %s %d %s", name, bytes, val); + size32_t utfCharCount; + rtlDataAttr utfText; + rtlStrToUtf8X(utfCharCount, utfText.refstr(), bytes, val); + bindUTF8Param(name, utfCharCount, utfText.getstr()); + } + virtual void bindVStringParam(const char *name, const char *val) + { + TRACE("WASM SE bindVStringParam %s %s", name, val); + bindStringParam(name, strlen(val), val); + } + virtual void bindUTF8Param(const char *name, size32_t chars, const char *val) + { + TRACE("WASM SE bindUTF8Param %s %d %s", name, chars, val); + auto bytes = rtlUtf8Size(chars, val); + auto memIdxVar = wasmStore->callRealloc(wasmName, {0, 0, 1, (int32_t)bytes}); + auto memIdx = memIdxVar[0].i32(); + auto mem = wasmStore->getData(wasmName); + memcpy(&mem[memIdx], val, bytes); + args.push_back(memIdx); + args.push_back((int32_t)bytes); + } + virtual void bindUnicodeParam(const char *name, size32_t chars, const UChar *val) + { + TRACE("WASM SE bindUnicodeParam %s %d", name, chars); + size32_t utfCharCount; + rtlDataAttr utfText; + rtlUnicodeToUtf8X(utfCharCount, utfText.refstr(), chars, val); + bindUTF8Param(name, utfCharCount, utfText.getstr()); + } + + virtual void bindSetParam(const char *name, int elemType, size32_t elemSize, bool isAll, size32_t totalBytes, const void *setData) + { + TRACE("WASM SE bindSetParam %s %d %d %d %d %p", name, elemType, elemSize, isAll, totalBytes, setData); + throw makeStringException(200, "bindSetParam not implemented"); + + type_vals typecode = (type_vals)elemType; + const byte *inData = (const byte *)setData; + const byte *endData = inData + totalBytes; + int numElems; + if (elemSize == UNKNOWN_LENGTH) + { + numElems = 0; + // Will need 2 passes to work out how many elements there are in the set :( + while (inData < endData) + { + int thisSize; + switch (elemType) + { + case type_varstring: + thisSize = strlen((const char *)inData) + 1; + break; + case type_string: + thisSize = *(size32_t *)inData + sizeof(size32_t); + break; + case type_unicode: + thisSize = (*(size32_t *)inData) * sizeof(UChar) + sizeof(size32_t); + break; + case type_utf8: + thisSize = rtlUtf8Size(*(size32_t *)inData, inData + sizeof(size32_t)) + sizeof(size32_t); + break; + default: + rtlFail(0, "wasmembed: Unsupported parameter type"); + break; + } + inData += thisSize; + numElems++; + } + inData = (const byte *)setData; + } + else + numElems = totalBytes / elemSize; + + std::vector memIdxVar; + int32_t memIdx; + + switch (typecode) + { + case type_boolean: + memIdxVar = wasmStore->callRealloc(wasmName, {0, 0, 1, (int32_t)numElems}); + memIdx = memIdxVar[0].i32(); + break; + default: + rtlFail(0, "wasmembed: Unsupported parameter type"); + break; + } + + auto mem = wasmStore->getData(wasmName); + size32_t thisSize = elemSize; + for (int idx = 0; idx < numElems; idx++) + { + switch (typecode) + { + case type_boolean: + mem[memIdx + idx] = *(bool *)inData; + break; + default: + rtlFail(0, "v8embed: Unsupported parameter type"); + break; + } + inData += thisSize; + } + args.push_back(memIdx); + args.push_back(numElems); + } + + virtual void bindRowParam(const char *name, IOutputMetaData &metaVal, const byte *val) override + { + TRACE("WASM SE bindRowParam %s %p", name, val); + throw makeStringException(200, "bindRowParam not implemented"); + } + virtual void bindDatasetParam(const char *name, IOutputMetaData &metaVal, IRowStream *val) + { + TRACE("WASM SE bindDatasetParam %s %p", name, val); + throw makeStringException(200, "bindDatasetParam not implemented"); + } + virtual bool getBooleanResult() + { + TRACE("WASM SE getBooleanResult"); + return wasmResults[0].i32(); + } + virtual void getDataResult(size32_t &len, void *&result) + { + TRACE("WASM SE getDataResult"); + throw makeStringException(200, "getDataResult not implemented"); + } + virtual double getRealResult() + { + TRACE("WASM SE getRealResult"); + if (wasmResults[0].kind() == wasmtime::ValKind::F64) + return wasmResults[0].f64(); + return wasmResults[0].f32(); + } + virtual __int64 getSignedResult() + { + TRACE("WASM SE getSignedResult1 %i", (uint8_t)wasmResults[0].kind()); + if (wasmResults[0].kind() == wasmtime::ValKind::I64) + { + return wasmResults[0].i64(); + } + return static_cast<__int64>(wasmResults[0].i32()); + } + virtual unsigned __int64 getUnsignedResult() + { + TRACE("WASM SE getUnsignedResult"); + if (wasmResults[0].kind() == wasmtime::ValKind::I64) + return wasmResults[0].i64(); + return static_cast(wasmResults[0].i32()); + } + virtual void getStringResult(size32_t &chars, char *&result) + { + TRACE("WASM SE getStringResult %zu", wasmResults.size()); + auto ptr = wasmResults[0].i32(); + auto data = wasmStore->getData(wasmName); + uint32_t strPtr; + std::string encoding; + uint32_t bytes; + std::tie(strPtr, encoding, bytes) = load_string(data, ptr); + size32_t codepoints = rtlUtf8Length(bytes, &data[strPtr]); + rtlUtf8ToStrX(chars, result, codepoints, reinterpret_cast(&data[strPtr])); + } + virtual void getUTF8Result(size32_t &chars, char *&result) + { + TRACE("WASM SE getUTF8Result"); + auto ptr = wasmResults[0].i32(); + auto data = wasmStore->getData(wasmName); + uint32_t strPtr; + std::string encoding; + uint32_t bytes; + std::tie(strPtr, encoding, bytes) = load_string(data, ptr); + chars = rtlUtf8Length(bytes, &data[strPtr]); + TRACE("WASM SE getUTF8Result %d %d", bytes, chars); + result = (char *)rtlMalloc(bytes); + memcpy(result, &data[strPtr], bytes); + } + virtual void getUnicodeResult(size32_t &chars, UChar *&result) + { + TRACE("WASM SE getUnicodeResult"); + auto ptr = wasmResults[0].i32(); + auto data = wasmStore->getData(wasmName); + uint32_t strPtr; + std::string encoding; + uint32_t bytes; + std::tie(strPtr, encoding, bytes) = load_string(data, ptr); + unsigned numchars = rtlUtf8Length(bytes, &data[strPtr]); + rtlUtf8ToUnicodeX(chars, result, numchars, reinterpret_cast(&data[strPtr])); + } + virtual void getSetResult(bool &__isAllResult, size32_t &resultBytes, void *&result, int elemType, size32_t elemSize) + { + TRACE("WASM SE getSetResult %d %d %zu", elemType, elemSize, wasmResults.size()); + auto ptr = wasmResults[0].i32(); + auto data = wasmStore->getData(wasmName); + + throw makeStringException(200, "getSetResult not implemented"); + } + virtual IRowStream *getDatasetResult(IEngineRowAllocator *_resultAllocator) + { + TRACE("WASM SE getDatasetResult"); + throw makeStringException(200, "getDatasetResult not implemented"); + return NULL; + } + virtual byte *getRowResult(IEngineRowAllocator *_resultAllocator) + { + TRACE("WASM SE getRowResult"); + throw makeStringException(200, "getRowResult not implemented"); + return NULL; + } + virtual size32_t getTransformResult(ARowBuilder &builder) + { + TRACE("WASM SE getTransformResult"); + throw makeStringException(200, "getTransformResult not implemented"); + return 0; + } + virtual void loadCompiledScript(size32_t chars, const void *_script) override + { + TRACE("WASM SE loadCompiledScript %p", _script); + throw makeStringException(200, "loadCompiledScript not implemented"); + } + virtual void enter() override + { + TRACE("WASM SE enter"); + } + virtual void reenter(ICodeContext *codeCtx) override + { + TRACE("WASM SE reenter"); + } + virtual void exit() override + { + TRACE("WASM SE exit"); + } + virtual void compileEmbeddedScript(size32_t lenChars, const char *_utf) override + { + TRACE("WASM SE compileEmbeddedScript"); + throw makeStringException(200, "compileEmbeddedScript not supported"); + } + virtual void importFunction(size32_t lenChars, const char *qualifiedName) override + { + TRACE("WASM SE importFunction: %s", qualifiedName); + + qualifiedID = std::string(qualifiedName, lenChars); + std::tie(wasmName, funcName) = splitQualifiedID(qualifiedID); + + if (!wasmStore->hasInstance(wasmName)) + { + wasmStore->registerInstance(wasmName); + } + } + virtual void callFunction() + { + TRACE("WASM SE callFunction %s", qualifiedID.c_str()); + wasmResults = wasmStore->call(qualifiedID, args); + } +}; + +IEmbedFunctionContext *createISecureEnclave(ICodeContext *codeCtx) +{ + return new SecureFunction(codeCtx); +} diff --git a/plugins/wasmembed/secure-enclave.hpp b/plugins/wasmembed/secure-enclave.hpp new file mode 100644 index 00000000000..fe81927e191 --- /dev/null +++ b/plugins/wasmembed/secure-enclave.hpp @@ -0,0 +1,4 @@ +#include "platform.h" +#include "eclrtl.hpp" + +IEmbedFunctionContext *createISecureEnclave(ICodeContext *codeCtx); diff --git a/plugins/wasmembed/test/CMakeLists.txt b/plugins/wasmembed/test/CMakeLists.txt new file mode 100644 index 00000000000..14079ec94ec --- /dev/null +++ b/plugins/wasmembed/test/CMakeLists.txt @@ -0,0 +1,33 @@ +cmake_minimum_required(VERSION 3.22) + +project(wasmembed) + +set(WASM_PATH "${CMAKE_CURRENT_BINARY_DIR}/bin/${PROJECT_NAME}.wasm") + +add_custom_command( + MAIN_DEPENDENCY ${CMAKE_CURRENT_SOURCE_DIR}/hpcc-scalar-test.wit + OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/hpcc_scalar_test.c ${CMAKE_CURRENT_BINARY_DIR}/hpcc_scalar_test.h ${CMAKE_CURRENT_BINARY_DIR}/hpcc_scalar_test_component_type.o + COMMAND wit-bindgen c --out-dir ${CMAKE_CURRENT_BINARY_DIR} ${CMAKE_CURRENT_SOURCE_DIR}/hpcc-scalar-test.wit +) +add_custom_target(wit-generate ALL DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/hpcc_scalar_test.c) + +set(CMAKE_EXECUTABLE_SUFFIX ".wasm") +set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -nostartfiles -fno-exceptions --sysroot=/${WASI_SDK_PREFIX}/share/wasi-sysroot -Wl,--no-entry") + +include_directories( + ${CMAKE_CURRENT_BINARY_DIR} +) + +add_executable(wasmembed + main.cpp + ${CMAKE_CURRENT_BINARY_DIR}/hpcc_scalar_test.c + ${CMAKE_CURRENT_BINARY_DIR}/hpcc_scalar_test.h +) + +target_link_libraries(wasmembed + ${CMAKE_CURRENT_BINARY_DIR}/hpcc_scalar_test_component_type.o +) + +install(TARGETS wasmembed + RUNTIME DESTINATION ${CMAKE_CURRENT_BINARY_DIR}/../install-target +) diff --git a/plugins/wasmembed/test/build.sh b/plugins/wasmembed/test/build.sh new file mode 100755 index 00000000000..7f91603130c --- /dev/null +++ b/plugins/wasmembed/test/build.sh @@ -0,0 +1,23 @@ +#!/bin/bash + +SCRIPT_DIR="$( cd -- "$( dirname -- "${BASH_SOURCE[0]:-$0}"; )" &> /dev/null && pwd 2> /dev/null; )" +ROOT_DIR="${SCRIPT_DIR}/../../.." + +echo "SCRIPT_DIR: ${SCRIPT_DIR}" +echo "ROOT_DIR: $ROOT_DIR" + +docker build --progress plain -f "${SCRIPT_DIR}/wasm32-wasi/Dockerfile" \ + -t wasm32-wasi:latest \ + "${SCRIPT_DIR}/." + +CMAKE_OPTIONS="-G Ninja -DCMAKE_BUILD_TYPE=MinSizeRel -DCMAKE_TOOLCHAIN_FILE=/hpcc-dev/wasi-sdk/share/cmake/wasi-sdk.cmake -DWASI_SDK_PREFIX=/hpcc-dev/wasi-sdk" + +docker run --rm \ + --mount source="${SCRIPT_DIR}",target=/hpcc-dev/wasmembed,type=bind,consistency=cached \ + --mount source="${ROOT_DIR}/testing/regress/ecl",target=/hpcc-dev/install-target,type=bind,consistency=cached \ + wasm32-wasi:latest \ + "rm -rf ./build && \ + cmake -S . -B /hpcc-dev/build ${CMAKE_OPTIONS} && \ + cmake --build /hpcc-dev/build --target install" + +echo "docker run -it --mount source=\"${SCRIPT_DIR}\",target=/hpcc-dev/wasmembed,type=bind,consistency=cached wasm32-wasi:latest bash" diff --git a/plugins/wasmembed/test/hpcc-scalar-test.wit b/plugins/wasmembed/test/hpcc-scalar-test.wit new file mode 100644 index 00000000000..3b632d46ea1 --- /dev/null +++ b/plugins/wasmembed/test/hpcc-scalar-test.wit @@ -0,0 +1,28 @@ +package hpcc-systems:hpcc-platform + +world hpcc-scalar-test { +/* imports --- + + guests dispose all params as needed + guests should dispose "results" as needed +*/ + import dbglog: func(msg: string) + +/* exports --- + + guests dispose all params as needed + hosts call cabi_post_XXX to dispose "results" as needed +*/ + export bool-test: func(a: bool, b: bool) -> bool + export float32-test: func(a: float32, b: float32) -> float32 + export float64-test: func(a: float64, b: float64) -> float64 + export u8-test: func(a: u8, b: u8) -> u8 + export u16-test: func(a: u16, b: u16) -> u16 + export u32-test: func(a: u32, b: u32) -> u32 + export u64-test: func(a: u64, b: u64) -> u64 + export s8-test: func(a: s8, b: s8) -> s8 + export s16-test: func(a: s16, b: s16) -> s16 + export s32-test: func(a: s32, b: s32) -> s32 + export s64-test: func(a: s64, b: s64) -> s64 + export utf8-string-test: func(a: string, b: string) -> string +} diff --git a/plugins/wasmembed/test/main.cpp b/plugins/wasmembed/test/main.cpp new file mode 100644 index 00000000000..c0d7420e5f1 --- /dev/null +++ b/plugins/wasmembed/test/main.cpp @@ -0,0 +1,70 @@ +#include "hpcc_scalar_test.h" + +#include + +void dbglog(const std::string str) +{ + hpcc_scalar_test_string_t msg; + hpcc_scalar_test_string_set(&msg, str.c_str()); + hpcc_scalar_test_dbglog(&msg); +} + +bool hpcc_scalar_test_bool_test(bool a, bool b) +{ + return a && b; +} +float hpcc_scalar_test_float32_test(float a, float b) +{ + return a + b; +} +double hpcc_scalar_test_float64_test(double a, double b) +{ + return a + b; +} +uint8_t hpcc_scalar_test_u8_test(uint8_t a, uint8_t b) +{ + return a + b; +} +uint16_t hpcc_scalar_test_u16_test(uint16_t a, uint16_t b) +{ + return a + b; +} +uint32_t hpcc_scalar_test_u32_test(uint32_t a, uint32_t b) +{ + return a + b; +} +uint64_t hpcc_scalar_test_u64_test(uint64_t a, uint64_t b) +{ + return a + b; +} +int8_t hpcc_scalar_test_s8_test(int8_t a, int8_t b) +{ + return a + b; +} +int16_t hpcc_scalar_test_s16_test(int16_t a, int16_t b) +{ + return a + b; +} +int32_t hpcc_scalar_test_s32_test(int32_t a, int32_t b) +{ + return a + b; +} +int64_t hpcc_scalar_test_s64_test(int64_t a, int64_t b) +{ + return a + b; +} +uint32_t hpcc_scalar_test_char_test(uint32_t a, uint32_t b) +{ + return a + b; +} +static uint32_t tally = 0; +void hpcc_scalar_test_utf8_string_test(hpcc_scalar_test_string_t *a, hpcc_scalar_test_string_t *b, hpcc_scalar_test_string_t *ret) +{ + std::string s1(a->ptr, a->len); + hpcc_scalar_test_string_free(a); + std::string s2(b->ptr, b->len); + hpcc_scalar_test_string_free(b); + std::string r = s1 + s2; + dbglog(std::to_string(++tally) + ": " + r); + hpcc_scalar_test_string_dup(ret, r.c_str()); +} diff --git a/plugins/wasmembed/test/wasm32-wasi/Dockerfile b/plugins/wasmembed/test/wasm32-wasi/Dockerfile new file mode 100644 index 00000000000..5746ac1eb34 --- /dev/null +++ b/plugins/wasmembed/test/wasm32-wasi/Dockerfile @@ -0,0 +1,41 @@ +FROM ubuntu:22.04 + +RUN apt-get update && \ + apt-get install -y \ + autoconf \ + autogen \ + automake \ + clang \ + cmake \ + curl \ + libtool \ + lld \ + llvm \ + make \ + ninja-build \ + wget + +RUN curl --proto '=https' --tlsv1.3 https://sh.rustup.rs -sSf | sh -s -- -y + +SHELL ["/bin/bash", "--login", "-c"] + +WORKDIR /hpcc-dev + +ARG WIT_VERSION=0.9.0 +RUN cargo install wasm-tools && \ + cargo install --git https://github.com/bytecodealliance/wit-bindgen --tag wit-bindgen-cli-${WIT_VERSION} wit-bindgen-cli && \ + curl https://wasmtime.dev/install.sh -sSf | bash + +# List of current vertsion can be found in https://github.com/bytecodealliance/wit-bindgen/releases --- +ARG WASI_VERSION=20 +ARG WASI_MINOR_VERSION=0 +ARG WASI_VERSION_FULL=${WASI_VERSION}.${WASI_MINOR_VERSION} +RUN wget https://github.com/WebAssembly/wasi-sdk/releases/download/wasi-sdk-${WASI_VERSION}/wasi-sdk-${WASI_VERSION_FULL}-linux.tar.gz +RUN tar xvf wasi-sdk-${WASI_VERSION_FULL}-linux.tar.gz && rm wasi-sdk-${WASI_VERSION_FULL}-linux.tar.gz +RUN mv wasi-sdk-${WASI_VERSION_FULL} wasi-sdk + +WORKDIR /hpcc-dev/wasmembed + +ENTRYPOINT ["/bin/bash", "--login", "-c"] + +CMD ["bash"] diff --git a/plugins/wasmembed/util.cpp b/plugins/wasmembed/util.cpp new file mode 100644 index 00000000000..7f65691c7b4 --- /dev/null +++ b/plugins/wasmembed/util.cpp @@ -0,0 +1,46 @@ +#include "util.hpp" + +#include "platform.h" +#include "jexcept.hpp" +#include "jfile.hpp" + +std::vector readWasmBinaryToBuffer(const char *filename) +{ + Owned file = createIFile(filename); + Owned fileIO = file->open(IFOread); + if (!fileIO) + throw makeStringExceptionV(0, "Failed to open %s", filename); + + MemoryBuffer mb; + size32_t count = read(fileIO, 0, (size32_t)-1, mb); + uint8_t *ptr = (uint8_t *)mb.detach(); + return std::vector(ptr, ptr + count); +} + +std::string extractContentInDoubleQuotes(const std::string &input) +{ + + std::size_t firstQuote = input.find_first_of('"'); + if (firstQuote == std::string::npos) + return ""; + + std::size_t secondQuote = input.find('"', firstQuote + 1); + if (secondQuote == std::string::npos) + return ""; + + return input.substr(firstQuote + 1, secondQuote - firstQuote - 1); +} + +std::pair splitQualifiedID(const std::string &qualifiedName) +{ + std::size_t firstDot = qualifiedName.find_first_of('.'); + if (firstDot == std::string::npos || firstDot == 0 || firstDot == qualifiedName.size() - 1) + throw makeStringExceptionV(3, "Invalid import function '%s', expected format: .", qualifiedName.c_str()); + + return std::make_pair(qualifiedName.substr(0, firstDot), qualifiedName.substr(firstDot + 1)); +} + +std::string createQualifiedID(const std::string &wasmName, const std::string &funcName) +{ + return wasmName + "." + funcName; +} \ No newline at end of file diff --git a/plugins/wasmembed/util.hpp b/plugins/wasmembed/util.hpp new file mode 100644 index 00000000000..22282eb0579 --- /dev/null +++ b/plugins/wasmembed/util.hpp @@ -0,0 +1,7 @@ +#include +#include + +std::vector readWasmBinaryToBuffer(const char *filename); +std::string extractContentInDoubleQuotes(const std::string &input); +std::pair splitQualifiedID(const std::string &qualifiedName); +std::string createQualifiedID(const std::string &wasmName, const std::string &funcName); diff --git a/plugins/wasmembed/wasm.ecllib b/plugins/wasmembed/wasm.ecllib new file mode 100644 index 00000000000..eb5d8313c02 --- /dev/null +++ b/plugins/wasmembed/wasm.ecllib @@ -0,0 +1,10 @@ +EXPORT Language := SERVICE : plugin('wasmembed') + integer getEmbedContext() : cpp, pure, fold, namespace='wasmLanguageHelper', entrypoint='getEmbedContext', prototype='IEmbedContext* getEmbedContext()'; + string syntaxCheck(const varstring funcname, UTF8 body, const varstring argnames, const varstring compileOptions, const varstring persistOptions) : cpp, pure, fold, namespace='wasmLanguageHelper', entrypoint='syntaxCheck'; +END; +EXPORT getEmbedContext := Language.getEmbedContext; +EXPORT syntaxCheck := Language.syntaxCheck; +EXPORT boolean supportsImport := true; +EXPORT boolean supportsScript := true; +EXPORT boolean prebind := false; +EXPORT boolean singletonEmbedContext := false; \ No newline at end of file diff --git a/plugins/wasmembed/wasmembed.cpp b/plugins/wasmembed/wasmembed.cpp new file mode 100644 index 00000000000..a2ce1d35d43 --- /dev/null +++ b/plugins/wasmembed/wasmembed.cpp @@ -0,0 +1,65 @@ +#include "secure-enclave.hpp" + +#include "jexcept.hpp" +#include "hqlplugins.hpp" + +static const char *compatibleVersions[] = { + "WASM Embed Helper 1.0.0", + NULL}; + +static const char *version = "WASM Embed Helper 1.0.0"; + +extern "C" DECL_EXPORT bool getECLPluginDefinition(ECLPluginDefinitionBlock *pb) +{ + if (pb->size == sizeof(ECLPluginDefinitionBlockEx)) + { + ECLPluginDefinitionBlockEx *pbx = (ECLPluginDefinitionBlockEx *)pb; + pbx->compatibleVersions = compatibleVersions; + } + else if (pb->size != sizeof(ECLPluginDefinitionBlock)) + return false; + pb->magicVersion = PLUGIN_VERSION; + pb->version = version; + pb->moduleName = "wasm"; + pb->ECL = NULL; + pb->flags = PLUGIN_MULTIPLE_VERSIONS; + pb->description = "WASM Embed Helper"; + return true; +} + +namespace wasmLanguageHelper +{ + class WasmEmbedContext : public CInterfaceOf + { + public: + virtual IEmbedFunctionContext *createFunctionContext(unsigned flags, const char *options) override + { + return createFunctionContextEx(nullptr, nullptr, flags, options); + } + + virtual IEmbedFunctionContext *createFunctionContextEx(ICodeContext *codeCtx, const IThorActivityContext *activityContext, unsigned flags, const char *options) override + { + return createISecureEnclave(codeCtx); + } + + virtual IEmbedServiceContext *createServiceContext(const char *service, unsigned flags, const char *options) override + { + throwUnexpected(); + return nullptr; + } + } theEmbedContext; + + extern DECL_EXPORT IEmbedContext *getEmbedContext() + { + return LINK(&theEmbedContext); + } + + extern DECL_EXPORT void syntaxCheck(size32_t &__lenResult, char *&__result, const char *funcname, size32_t charsBody, const char *body, const char *argNames, const char *compilerOptions, const char *persistOptions) + { + StringBuffer result; + // MORE - ::syntaxCheck(__lenResult, __result, funcname, charsBody, body, argNames, compilerOptions, persistOptions); + __lenResult = result.length(); + __result = result.detach(); + } + +} // namespace diff --git a/rtl/eclrtl/eclrtl.cpp b/rtl/eclrtl/eclrtl.cpp index 7fb01efd1cf..980ebf2198e 100644 --- a/rtl/eclrtl/eclrtl.cpp +++ b/rtl/eclrtl/eclrtl.cpp @@ -6279,7 +6279,7 @@ void rtlBase64Decode(size32_t & tlen, void * & tgt, size32_t slen, const char * void rtlGetEclUserSecret(size32_t & outlen, void * & out, const char *name, const char *key) { - Owned secret = getSecret("eclUser", name); + Owned secret = getSecret("eclUser", name); if (secret) { MemoryBuffer data; diff --git a/system/codesigner/gpgcodesigner.cpp b/system/codesigner/gpgcodesigner.cpp index 8ed528fd21b..3e3c124a470 100644 --- a/system/codesigner/gpgcodesigner.cpp +++ b/system/codesigner/gpgcodesigner.cpp @@ -138,7 +138,7 @@ void GpgCodeSigner::importKeysFromSecret(const char * cat, const char *keytype) for (int keyentry = 1; ; keyentry++) { VStringBuffer keysecretname("gpg-%s-key-%d", keytype, keyentry); - Owned secretKey = getSecret(cat, keysecretname.str()); + Owned secretKey = getSecret(cat, keysecretname.str()); if (secretKey) { StringBuffer gpgKey; diff --git a/system/jlib/CMakeLists.txt b/system/jlib/CMakeLists.txt index 0b5a9a0490f..711d8958b90 100644 --- a/system/jlib/CMakeLists.txt +++ b/system/jlib/CMakeLists.txt @@ -42,10 +42,10 @@ endif(NOT TARGET libbase58) find_package(yaml CONFIG REQUIRED) -#For OpenTel exporter find_package(Protobuf REQUIRED) -#For http exporter -#find_package(CURL REQUIRED) +find_package(CURL CONFIG REQUIRED) +find_package(gRPC CONFIG REQUIRED) +find_package(nlohmann_json CONFIG REQUIRED) find_package(opentelemetry-cpp CONFIG REQUIRED) SET (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${STRICT_CXX_FLAGS}") @@ -239,8 +239,6 @@ if ( ${HAVE_LIBCRYPT} ) target_link_libraries ( jlib crypt) endif ( ${HAVE_LIBCRYPT} ) -find_package(gRPC CONFIG REQUIRED) - target_link_libraries ( jlib opentelemetry-cpp::ostream_span_exporter diff --git a/system/jlib/jarray.hpp b/system/jlib/jarray.hpp index a2c1240bb6d..9870c8d9c31 100644 --- a/system/jlib/jarray.hpp +++ b/system/jlib/jarray.hpp @@ -153,7 +153,7 @@ class ArrayOf : public AllocatorOf typedef int (*CompareFunc)(const MEMBER *, const MEMBER *); // Should really be const, as should the original array functions public: - ~ArrayOf() { kill(); } + ~ArrayOf() { kill(); } MEMBER & operator[](size_t pos) { return element((aindex_t)pos); } const MEMBER & operator[](size_t pos) const { return element((aindex_t)pos); } diff --git a/system/jlib/jbuff.hpp b/system/jlib/jbuff.hpp index 5d6392dbec6..010e9dcc66d 100644 --- a/system/jlib/jbuff.hpp +++ b/system/jlib/jbuff.hpp @@ -66,10 +66,10 @@ template class OwnedPtrCustomFree CLASS *ptr = nullptr; public: - OwnedPtrCustomFree() { } - OwnedPtrCustomFree(CLASS *_ptr) : ptr(_ptr) { } - OwnedPtrCustomFree(SELF &&_ptr) { ptr = _ptr.getClear(); } - ~OwnedPtrCustomFree() { safeFree(ptr); } + OwnedPtrCustomFree() { } + OwnedPtrCustomFree(CLASS *_ptr) : ptr(_ptr) { } + OwnedPtrCustomFree(SELF &&_ptr) { ptr = _ptr.getClear(); } + ~OwnedPtrCustomFree() { safeFree(ptr); } void operator = (CLASS * _ptr) { diff --git a/system/jlib/jhash.hpp b/system/jlib/jhash.hpp index c8790fe2e4c..3653abd4c21 100644 --- a/system/jlib/jhash.hpp +++ b/system/jlib/jhash.hpp @@ -468,13 +468,13 @@ template class CMinHashTable } public: - CMinHashTable(unsigned _initialSize = 7) + CMinHashTable(unsigned _initialSize = 7) { htn = _initialSize; n = 0; table = (C **)calloc(sizeof(C *),htn); } - ~CMinHashTable() + ~CMinHashTable() { C **t = table+htn; while (t--!=table) @@ -625,7 +625,7 @@ template class CTimeLimitedCache { public: - CTimeLimitedCache(unsigned timeoutMs=defaultCacheTimeoutMs) + CTimeLimitedCache(unsigned timeoutMs=defaultCacheTimeoutMs) { timeoutPeriodCycles = ((cycle_t)timeoutMs) * queryOneSecCycles() / 1000; } diff --git a/system/jlib/jiface.hpp b/system/jlib/jiface.hpp index 4f3d6b2a744..ca502ab010d 100644 --- a/system/jlib/jiface.hpp +++ b/system/jlib/jiface.hpp @@ -88,8 +88,8 @@ class CSimpleInterfaceOf : public INTERFACE } private: - CSimpleInterfaceOf(const CSimpleInterfaceOf&) = delete; - CSimpleInterfaceOf(CSimpleInterfaceOf &&) = delete; + CSimpleInterfaceOf(const CSimpleInterfaceOf&) = delete; + CSimpleInterfaceOf(CSimpleInterfaceOf &&) = delete; CSimpleInterfaceOf & operator = (const CSimpleInterfaceOf &) = delete; mutable std::atomic xxcount; }; diff --git a/system/jlib/jptree.hpp b/system/jlib/jptree.hpp index 80888c8218d..e52db5eec1a 100644 --- a/system/jlib/jptree.hpp +++ b/system/jlib/jptree.hpp @@ -438,11 +438,14 @@ extern jlib_decl unsigned getPropertyTreeHash(const IPropertyTree & source, unsi //to not be modified and to remain valid and consistent until it is released. interface ISyncedPropertyTree : extends IInterface { +//The following functions check whether something is up to date before returning their values. + //Return a version-hash which changes whenever the property tree changes - so that a caller can determine whether it needs to update + virtual unsigned getVersion() const = 0; virtual const IPropertyTree * getTree() const = 0; virtual bool getProp(MemoryBuffer & result, const char * xpath) const = 0; virtual bool getProp(StringBuffer & result, const char * xpath) const = 0; - //Return a version-hash which changes whenever the property tree changes - so that a caller can determine whether it needs to update - virtual unsigned getVersion() const = 0; + +// The following functions return the current cached state - they do not force a check to see if the value is up to date virtual bool isStale() const = 0; // An indication that the property tree may be out of date because it couldn't be resynchronized. virtual bool isValid() const = 0; // Is the property tree non-null? Typically called at startup to check configuration is provided. }; diff --git a/system/jlib/jsecrets.cpp b/system/jlib/jsecrets.cpp index 5951ad6fbb7..9fa5e60d7c6 100644 --- a/system/jlib/jsecrets.cpp +++ b/system/jlib/jsecrets.cpp @@ -31,6 +31,9 @@ #if defined(__clang__) || defined(__GNUC__) #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Wformat-nonliteral" + +//httplib also generates warning about access outside of array bounds in gcc +#pragma GCC diagnostic ignored "-Warray-bounds" #endif #ifdef _USE_OPENSSL @@ -64,36 +67,19 @@ CVaultKind getSecretType(const char *s) } interface IVaultManager : extends IInterface { - virtual bool getCachedSecretFromVault(const char *category, const char *vaultId, CVaultKind &kind, StringBuffer &content, const char *secret, const char *version) = 0; virtual bool requestSecretFromVault(const char *category, const char *vaultId, CVaultKind &kind, StringBuffer &content, const char *secret, const char *version) = 0; - virtual bool getCachedSecretByCategory(const char *category, CVaultKind &kind, StringBuffer &content, const char *secret, const char *version) = 0; virtual bool requestSecretByCategory(const char *category, CVaultKind &kind, StringBuffer &content, const char *secret, const char *version) = 0; }; -static CriticalSection secretCacheCS; -static Owned secretCache; -static CriticalSection mtlsInfoCacheCS; -static std::unordered_map> mtlsInfoCache; static Owned vaultManager; static MemoryAttr udpKey; static bool udpKeyInitialized = false; -MODULE_INIT(INIT_PRIORITY_SYSTEM) +static const IPropertyTree *getLocalSecret(const char *category, const char * name) { - secretCache.setown(createPTree()); - return true; + return getSecret(category, name, "k8s", nullptr); } -MODULE_EXIT() -{ - vaultManager.clear(); - secretCache.clear(); - mtlsInfoCache.clear(); - udpKey.clear(); -} - -static IPropertyTree *getLocalSecret(const char *category, const char * name); - //based on kubernetes secret / key names. Even if some vault backends support additional characters we'll restrict to this subset for now static const char *validSecretNameChrs = ".-"; @@ -319,8 +305,8 @@ extern jlib_decl StringBuffer &generateDynamicUrlSecretName(StringBuffer &secret unsigned portNum = port.length() ? atoi(port) : 0; return generateDynamicUrlSecretName(secretName, scheme, username, host, portNum, path); } -//--------------------------------------------------------------------------------------------------------------------- +//--------------------------------------------------------------------------------------------------------------------- static StringBuffer secretDirectory; static CriticalSection secretCS; @@ -364,18 +350,6 @@ static StringBuffer &buildSecretPath(StringBuffer &path, const char *category, c return addPathSepChar(path.append(ensureSecretDirectory())).append(category).append(PATHSEPCHAR).append(name).append(PATHSEPCHAR); } -static bool checkSecretExpired(unsigned created) -{ - if (!created) - return false; - unsigned age = msTick() - created; - return age > getSecretTimeout(); -} - -static bool hasCacheExpired(const IPropertyTree * secret) -{ - return checkSecretExpired((unsigned)secret->getPropInt("@created")); -} enum class VaultAuthType {unknown, k8s, appRole, token, clientcert}; @@ -395,6 +369,134 @@ static bool isEmptyTimeval(const timeval &tv) return (tv.tv_sec==0 && tv.tv_usec==0); } +//--------------------------------------------------------------------------------------------------------------------- + +//Represents an entry in the secret cache. Once created it is always used for the secret. +using cache_timestamp = unsigned; +class SecretCacheEntry : public CInterface +{ + friend class SecretCache; + +public: + //A cache entry is initally created that has a create and access time of now, but the checkTimestamp + //is set so that needsRefresh() will return true. + SecretCacheEntry(cache_timestamp _now) + : contentTimestamp(_now), accessedTimestamp(_now), checkedTimestamp(_now - 2 * secretTimeoutMs) + { + } + + unsigned getHash() const + { + return contentHash; + } + + // We should never replace known contents for unknown contents + // so once this returns true it should always return true + bool hasContents() const + { + return contents != nullptr; + } + + //Is the secret potentially out of date? + bool isStale() const + { + cache_timestamp now = msTick(); + cache_timestamp elapsed = (now - contentTimestamp); + return (elapsed > secretTimeoutMs); + } + + // Is it time to check if there is a new value for this secret? + bool needsRefresh(cache_timestamp now) const + { + cache_timestamp elapsed = (now - checkedTimestamp); + return (elapsed > secretTimeoutMs); + } + + bool needsRefresh() const + { + return needsRefresh(msTick()); + } + + void noteFailedUpdate(cache_timestamp now) + { + //Update the checked timestamp - so that we do not continually check for updates to secrets which + //are stale because the vault or other source of values in inaccessible. + //Keep using the last good value + checkedTimestamp = now; + } + + //The following functions can only be called from member functions of SecretCache +private: + void updateContents(IPropertyTree * _contents, cache_timestamp now) + { + contents.set(_contents); + updateHash(); + contentTimestamp = now; + accessedTimestamp = now; + checkedTimestamp = now; + } + + void updateHash() + { + if (contents) + contentHash = getPropertyTreeHash(*contents, 0x811C9DC5); + else + contentHash = 0; + } +private: + Linked contents;// Can only be accessed when SecretCache::cs is held + cache_timestamp contentTimestamp = 0; // When was this secret read from disk/vault + cache_timestamp accessedTimestamp = 0; // When was this secret last accessed? + cache_timestamp checkedTimestamp = 0; // When was this last checked for updates? + unsigned contentHash = 0; +}; + +// A cache of (secret[:version] to a secret cache entry) +// Once a hash table entry has been created for a secret it is never removed and the associated +// value is never replaced. This means it is safe to keep a pointer to the entry in another class. +class SecretCache +{ +public: + const IPropertyTree * getContents(SecretCacheEntry * match) + { + //Return contents within the critical section so no other thread can modify it + CriticalBlock block(cs); + return LINK(match->contents); + } + + //Check to see if a secret exists, and if not add a null entry that has expired. + SecretCacheEntry * resolveSecret(const std::string & secretKey, cache_timestamp now) + { + SecretCacheEntry * result; + CriticalBlock block(cs); + auto match = secrets.find(secretKey); + if (match != secrets.cend()) + { + result = match->second.get(); + result->accessedTimestamp = now; + } + else + { + //Insert an entry with a null value that is marked as out of date + result = new SecretCacheEntry(now); + secrets.emplace(secretKey, result); + } + return result; + } + + void updateSecret(SecretCacheEntry * match, IPropertyTree * value, cache_timestamp now) + { + CriticalBlock block(cs); + match->updateContents(value, now); + } + +private: + CriticalSection cs; + std::unordered_map> secrets; +}; + +//--------------------------------------------------------------------------------------------------------------------- + class CVault { private: @@ -402,7 +504,6 @@ class CVault CVaultKind kind; CriticalSection vaultCS; - Owned cache; std::string clientCertPath; std::string clientKeyPath; @@ -445,7 +546,6 @@ class CVault if (!checkFileExists(clientKeyPath.c_str())) WARNLOG("vault: client key not found, %s", clientKeyPath.c_str()); - cache.setown(createPTree()); StringBuffer url; replaceEnvVariables(url, vault->queryProp("@url"), false); PROGLOG("vault url %s", url.str()); @@ -486,7 +586,7 @@ class CVault } else if (vault->hasProp("@client-secret")) { - Owned clientSecret = getLocalSecret("system", vault->queryProp("@client-secret")); + Owned clientSecret = getLocalSecret("system", vault->queryProp("@client-secret")); if (clientSecret) { StringBuffer tokenText; @@ -672,7 +772,7 @@ class CVault return; DBGLOG("appRoleLogin%s", permissionDenied ? " because existing token permission denied" : ""); StringBuffer appRoleSecretId; - Owned appRoleSecret = getLocalSecret("system", appRoleSecretName); + Owned appRoleSecret = getLocalSecret("system", appRoleSecretName); if (!appRoleSecret) vaultAuthErrorV("appRole secret %s not found", appRoleSecretName.str()); else if (!getSecretKeyValue(appRoleSecretId, appRoleSecret, "secret-id")) @@ -712,42 +812,6 @@ class CVault if (clientToken.isEmpty()) vaultAuthError("no vault access token"); } - bool getCachedSecret(CVaultKind &rkind, StringBuffer &content, const char *secret, const char *version) - { - CriticalBlock block(vaultCS); - IPropertyTree *tree = cache->queryPropTree(secret); - if (tree) - { - VStringBuffer vername("v.%s", isEmptyString(version) ? "latest" : version); - IPropertyTree *envelope = tree->queryPropTree(vername); - if (!envelope) - return false; - if (hasCacheExpired(envelope)) - { - tree->removeTree(envelope); - return false; - } - const char *s = envelope->queryProp(""); - rkind = kind; - if (!isEmptyString(s)) - content.append(s); - return true; - } - return false; - } - void addCachedSecret(const char *content, const char *secret, const char *version) - { - VStringBuffer vername("v.%s", isEmptyString(version) ? "latest" : version); - Owned envelope = createPTree(vername); - envelope->setPropInt("@created", (int) msTick()); - if (!isEmptyString(content)) - envelope->setProp("", content); - { - CriticalBlock block(vaultCS); - IPropertyTree *parent = ensurePTree(cache, secret); - parent->setPropTree(vername, envelope.getClear()); - } - } bool requestSecretAtLocation(CVaultKind &rkind, StringBuffer &content, const char *location, const char *secretCacheKey, const char *version, bool permissionDenied) { checkAuthentication(permissionDenied); @@ -779,7 +843,6 @@ class CVault { rkind = kind; content.append(res->body.c_str()); - addCachedSecret(content.str(), secretCacheKey, version); return true; } else if (res->status == 403) @@ -801,7 +864,6 @@ class CVault else OERRLOG("Error: Vault %s http error (%d) accessing secret %s.%s location %s", name.str(), res.error(), secretCacheKey, version ? version : "", location); - addCachedSecret("", secretCacheKey, version); //cache misses so we don't keep calling the vault return false; } bool requestSecret(CVaultKind &rkind, StringBuffer &content, const char *secret, const char *version) @@ -831,16 +893,6 @@ class CVaultSet if (!isEmptyString(name)) vaults.emplace(name, std::unique_ptr(new CVault(vault))); } - bool getCachedSecret(CVaultKind &kind, StringBuffer &content, const char *secret, const char *version) - { - auto it = vaults.begin(); - for (; it != vaults.end(); it++) - { - if (it->second->getCachedSecret(kind, content, secret, version)) - return true; - } - return false; - } bool requestSecret(CVaultKind &kind, StringBuffer &content, const char *secret, const char *version) { auto it = vaults.begin(); @@ -851,15 +903,6 @@ class CVaultSet } return false; } - bool getCachedSecretFromVault(const char *vaultId, CVaultKind &kind, StringBuffer &content, const char *secret, const char *version) - { - if (isEmptyString(vaultId)) - return false; - auto it = vaults.find(vaultId); - if (it == vaults.end()) - return false; - return it->second->getCachedSecret(kind, content, secret, version); - } bool requestSecretFromVault(const char *vaultId, CVaultKind &kind, StringBuffer &content, const char *secret, const char *version) { if (isEmptyString(vaultId)) @@ -906,15 +949,6 @@ class CVaultManager : public CInterfaceOf it->second->addVault(&vault); } } - bool getCachedSecretFromVault(const char *category, const char *vaultId, CVaultKind &kind, StringBuffer &content, const char *secret, const char *version) override - { - if (isEmptyString(category)) - return false; - auto it = categories.find(category); - if (it == categories.end()) - return false; - return it->second->getCachedSecretFromVault(vaultId, kind, content, secret, version); - } bool requestSecretFromVault(const char *category, const char *vaultId, CVaultKind &kind, StringBuffer &content, const char *secret, const char *version) override { if (isEmptyString(category)) @@ -925,15 +959,6 @@ class CVaultManager : public CInterfaceOf return it->second->requestSecretFromVault(vaultId, kind, content, secret, version); } - bool getCachedSecretByCategory(const char *category, CVaultKind &kind, StringBuffer &content, const char *secret, const char *version) override - { - if (isEmptyString(category)) - return false; - auto it = categories.find(category); - if (it == categories.end()) - return false; - return it->second->getCachedSecret(kind, content, secret, version); - } bool requestSecretByCategory(const char *category, CVaultKind &kind, StringBuffer &content, const char *secret, const char *version) override { if (isEmptyString(category)) @@ -953,56 +978,34 @@ IVaultManager *ensureVaultManager() return vaultManager; } -static IPropertyTree *getCachedLocalSecret(const char *category, const char *name, bool &cachedMiss) +//--------------------------------------------------------------------------------------------------------------------- + +static SecretCache globalSecretCache; +static CriticalSection mtlsInfoCacheCS; +static std::unordered_map> mtlsInfoCache; + +MODULE_INIT(INIT_PRIORITY_SYSTEM) { - if (isEmptyString(name)) - return nullptr; - Owned secret; - { - CriticalBlock block(secretCacheCS); - IPropertyTree *tree = secretCache->queryPropTree(category); - if (!tree) - return nullptr; - secret.setown(tree->getPropTree(name)); - if (secret) - { - if (hasCacheExpired(secret)) - { - secretCache->removeProp(name); - return nullptr; - } - if (secret->hasProp("@miss")) - { - cachedMiss = true; - return nullptr; - } - return secret.getClear(); - } - } - return nullptr; + return true; } -static void addCachedLocalSecret(const char *category, const char *name, IPropertyTree *secret) +MODULE_EXIT() { - if (!secret || isEmptyString(name) || isEmptyString(category)) - return; - secret->setPropInt("@created", (int)msTick()); - { - CriticalBlock block(secretCacheCS); - IPropertyTree *tree = ensurePTree(secretCache, category); - tree->setPropTree(name, LINK(secret)); - } + vaultManager.clear(); + udpKey.clear(); } -static IPropertyTree *loadLocalSecret(const char *category, const char * name) + +static IPropertyTree * resolveLocalSecret(const char *category, const char * name) { StringBuffer path; buildSecretPath(path, category, name); + Owned entries = createDirectoryIterator(path); if (!entries || !entries->first()) return nullptr; - Owned tree = createPTree(name); - tree->setPropInt("@created", (int) msTick()); + + Owned tree(createPTree(name)); ForEach(*entries) { if (entries->isDir()) @@ -1018,22 +1021,8 @@ static IPropertyTree *loadLocalSecret(const char *category, const char * name) continue; tree->setPropBin(name, content.length(), content.bufferBase()); } - addCachedLocalSecret(category, name, tree); - return tree.getClear(); -} - -static IPropertyTree *getLocalSecret(const char *category, const char * name) -{ - validateCategoryName(category); - validateSecretName(name); - bool skipLocalFetch = false; - Owned tree = getCachedLocalSecret(category, name, skipLocalFetch); - if (skipLocalFetch) - return nullptr; - if (tree) - return tree.getClear(); - return loadLocalSecret(category, name); + return tree.getClear(); } static IPropertyTree *createPTreeFromVaultSecret(const char *content, CVaultKind kind) @@ -1056,30 +1045,8 @@ static IPropertyTree *createPTreeFromVaultSecret(const char *content, CVaultKind } return tree.getClear(); } -static IPropertyTree *getCachedVaultSecret(const char *category, const char *vaultId, const char * name, const char *version, bool &cachedMiss) -{ - CVaultKind kind; - StringBuffer json; - IVaultManager *vaultmgr = ensureVaultManager(); - if (isEmptyString(vaultId)) - { - if (!vaultmgr->getCachedSecretByCategory(category, kind, json, name, version)) - return nullptr; - } - else - { - if (!vaultmgr->getCachedSecretFromVault(category, vaultId, kind, json, name, version)) - return nullptr; - } - if (json.isEmpty()) - { - cachedMiss = true; - return nullptr; - } - return createPTreeFromVaultSecret(json.str(), kind); -} -static IPropertyTree *requestVaultSecret(const char *category, const char *vaultId, const char * name, const char *version) +static IPropertyTree *resolveVaultSecret(const char *category, const char * name, const char *vaultId, const char *version) { CVaultKind kind; StringBuffer json; @@ -1097,55 +1064,58 @@ static IPropertyTree *requestVaultSecret(const char *category, const char *vault return createPTreeFromVaultSecret(json.str(), kind); } -static IPropertyTree *getVaultSecret(const char *category, const char * name, const char *vaultId, const char *version) + +static SecretCacheEntry * getSecretEntry(const char *category, const char * name, const char * optVaultId, const char * optVersion) { - CVaultKind kind; - StringBuffer json; - IVaultManager *vaultmgr = ensureVaultManager(); + cache_timestamp now = msTick(); - bool cachedMiss = false; + std::string key; + key.append(category).append("/").append(name); + if (optVaultId) + key.append("@").append(optVaultId); + if (optVersion) + key.append("#").append(optVersion); - if (isEmptyString(vaultId)) + SecretCacheEntry * match = globalSecretCache.resolveSecret(key, now); + if (!match->needsRefresh(now)) + return match; + + Owned resolved; + if (!isEmptyString(optVaultId)) { - if (vaultmgr->getCachedSecretByCategory(category, kind, json, name, version)) - cachedMiss = json.isEmpty(); + if (strieq(optVaultId, "k8s")) + resolved.setown(resolveLocalSecret(category, name)); else - vaultmgr->requestSecretByCategory(category, kind, json, name, version); + resolved.setown(resolveVaultSecret(category, name, optVaultId, optVersion)); } else { - if (!vaultmgr->getCachedSecretFromVault(category, vaultId, kind, json, name, version)) - cachedMiss = json.isEmpty(); - else - vaultmgr->requestSecretFromVault(category, vaultId, kind, json, name, version); + resolved.setown(resolveLocalSecret(category, name)); + if (!resolved) + resolved.setown(resolveVaultSecret(category, name, nullptr, optVersion)); } - if (cachedMiss) - return nullptr; - return createPTreeFromVaultSecret(json.str(), kind); + + //If the secret could no longer be resolved (e.g. a vault has gone down) then keep the old one + if (resolved) + globalSecretCache.updateSecret(match, resolved, now); + else + match->noteFailedUpdate(now); + + return match; } -IPropertyTree *getSecretTree(const char *category, const char * name, const char * optVaultId, const char * optVersion) +static const IPropertyTree *getSecretTree(const char *category, const char * name, const char * optVaultId, const char * optVersion) { - if (!isEmptyString(optVaultId)) - return getVaultSecret(category, name, optVaultId, optVersion); + SecretCacheEntry * secret = getSecretEntry(category, name, optVaultId, optVersion); + if (secret) + return globalSecretCache.getContents(secret); + return nullptr; +} - //if we get back a null secret, it might be a cached miss, so don't go to the source if flag gets set - bool skipVaultFetch = false; - bool skipLocalFetch = false; - //check for any chached first - Owned secret = getCachedLocalSecret(category, name, skipLocalFetch); - if (!secret) - secret.setown(getCachedVaultSecret(category, nullptr, name, nullptr, skipVaultFetch)); - //now check local, then vaults - if (!secret && !skipLocalFetch) - secret.setown(loadLocalSecret(category, name)); - if (!secret && !skipVaultFetch) - secret.setown(requestVaultSecret(category, nullptr, name, nullptr)); - return secret.getClear(); -} +//Public interface to the secrets -IPropertyTree *getSecret(const char *category, const char * name, const char * optVaultId, const char * optVersion) +const IPropertyTree *getSecret(const char *category, const char * name, const char * optVaultId, const char * optVersion) { validateCategoryName(category); validateSecretName(name); @@ -1172,7 +1142,7 @@ bool getSecretKeyValue(StringBuffer & result, const IPropertyTree *secret, const extern jlib_decl bool getSecretValue(StringBuffer & result, const char *category, const char * name, const char * key, bool required) { - Owned secret = getSecret(category, name); + Owned secret = getSecret(category, name); if (required && !secret) throw MakeStringException(-1, "secret %s.%s not found", category, name); bool found = getSecretKeyValue(result, secret, key); @@ -1186,10 +1156,9 @@ extern jlib_decl bool getSecretValue(StringBuffer & result, const char *category class CSecret final : public CInterfaceOf { public: - CSecret(const char *_category, const char * _name, const char * _vaultId, const char * _version, const IPropertyTree * _secret) + CSecret(const char *_category, const char * _name, const char * _vaultId, const char * _version, SecretCacheEntry * _secret) : category(_category), name(_name), vaultId(_vaultId), version(_version), secret(_secret) { - updateHash(); } virtual const IPropertyTree * getTree() const override; @@ -1197,34 +1166,34 @@ class CSecret final : public CInterfaceOf virtual bool getProp(MemoryBuffer & result, const char * key) const override { CriticalBlock block(secretCs); - checkStale(); - return getSecretKeyValue(result, secret, key); + checkUptoDate(); + Owned contents = globalSecretCache.getContents(secret); + return getSecretKeyValue(result, contents, key); } virtual bool getProp(StringBuffer & result, const char * key) const override { CriticalBlock block(secretCs); - checkStale(); - return getSecretKeyValue(result, secret, key); + checkUptoDate(); + Owned contents = globalSecretCache.getContents(secret); + return getSecretKeyValue(result, contents, key); } virtual bool isStale() const override { - return secret && hasCacheExpired(secret); + return secret->isStale(); } virtual unsigned getVersion() const override { CriticalBlock block(secretCs); - checkStale(); - return secretHash; + checkUptoDate(); + return secret->getHash(); } virtual bool isValid() const override { - CriticalBlock block(secretCs); - return secret != nullptr; + return secret->hasContents(); } protected: - void checkStale() const; - void updateHash() const; + void checkUptoDate() const; protected: StringAttr category; @@ -1232,21 +1201,20 @@ class CSecret final : public CInterfaceOf StringAttr vaultId; StringAttr version; mutable CriticalSection secretCs; - mutable Linked secret; - mutable unsigned secretHash = 0; + mutable SecretCacheEntry * secret; }; const IPropertyTree * CSecret::getTree() const { CriticalBlock block(secretCs); - checkStale(); - return LINK(secret); + checkUptoDate(); + return globalSecretCache.getContents(secret); } -void CSecret::checkStale() const +void CSecret::checkUptoDate() const { - if (isStale()) + if (secret->needsRefresh()) { #ifdef TRACE_SECRETS DBGLOG("Secret %s/%s is stale updating from %u...", category.str(), name.str(), secretHash); @@ -1254,8 +1222,10 @@ void CSecret::checkStale() const //MORE: This could block or fail - in roxie especially it would be better to return the old value try { - secret.setown(getSecretTree(category, name, vaultId, version)); - updateHash(); + SecretCacheEntry * newSecret = getSecretEntry(category, name, vaultId, version); + //Check the secret is always returned consistently. It would be possible to call a slightly + //more optimal function to refresh a secret, but this is simplest. + assertex(secret == newSecret); } catch (IException * e) { @@ -1266,17 +1236,12 @@ void CSecret::checkStale() const } } -void CSecret::updateHash() const -{ - if (secret) - secretHash = getPropertyTreeHash(*secret.get(), 0x811C9DC5); - else - secretHash = 0; -} - ISyncedPropertyTree * resolveSecret(const char *category, const char * name, const char * optVaultId, const char * optVersion) { - Owned resolved = getSecret(category, name, optVaultId, optVersion); + validateCategoryName(category); + validateSecretName(name); + + SecretCacheEntry * resolved = getSecretEntry(category, name, optVaultId, optVersion); return new CSecret(category, name, optVaultId, optVersion, resolved); } @@ -1354,13 +1319,13 @@ class CSyncedCertificateBase : public CInterfaceOf virtual bool getProp(MemoryBuffer & result, const char * key) const override final { CriticalBlock block(secretCs); - checkStale(); + checkUptoDate(); return getSecretKeyValue(result, config, key); } virtual bool getProp(StringBuffer & result, const char * key) const override final { CriticalBlock block(secretCs); - checkStale(); + checkUptoDate(); return getSecretKeyValue(result, config, key); } virtual bool isStale() const override final @@ -1375,7 +1340,7 @@ class CSyncedCertificateBase : public CInterfaceOf virtual unsigned getVersion() const override final { CriticalBlock block(secretCs); - checkStale(); + checkUptoDate(); //If information that is combined with the secret (e.g. trusted peers) can also change dynamically this would //need to be a separate hash calculated from the config tree return secretHash; @@ -1385,7 +1350,7 @@ class CSyncedCertificateBase : public CInterfaceOf virtual void updateConfigFromSecret(const IPropertyTree * secretInfo) const = 0; protected: - void checkStale() const; + void checkUptoDate() const; void createConfig() const; void createDefaultConfigFromSecret(const IPropertyTree * secretInfo, bool addCertificates, bool addCertificateAuthority) const; void updateCertificateFromSecret(const IPropertyTree * secretInfo) const; @@ -1403,11 +1368,11 @@ class CSyncedCertificateBase : public CInterfaceOf const IPropertyTree * CSyncedCertificateBase::getTree() const { CriticalBlock block(secretCs); - checkStale(); + checkUptoDate(); return LINK(config); } -void CSyncedCertificateBase::checkStale() const +void CSyncedCertificateBase::checkUptoDate() const { if (secretHash != secret->getVersion()) createConfig(); diff --git a/system/jlib/jsecrets.hpp b/system/jlib/jsecrets.hpp index 9601deb4d7d..2d7f1c286be 100644 --- a/system/jlib/jsecrets.hpp +++ b/system/jlib/jsecrets.hpp @@ -28,9 +28,9 @@ extern jlib_decl void setSecretMount(const char * path); extern jlib_decl void setSecretTimeout(unsigned timeoutMs); //Return the current (cached) value of a secret. If the secret is not defined, return nullptr. -extern jlib_decl IPropertyTree *getSecret(const char *category, const char * name, const char * optVaultId = nullptr, const char * optVersion = nullptr); +extern jlib_decl const IPropertyTree *getSecret(const char *category, const char * name, const char * optVaultId = nullptr, const char * optVersion = nullptr); // resolveSecret() always returns an object, which will potentially be updated behind the scenes. If no secret is originally -// defined, but it then configured in a vault or Kubernetes secret, it will be bicked up when the cache entry is +// defined, but it then added to a vault or Kubernetes secret, it will then be picked up when the cache entry is // refreshed - allowing missing configuration to be updated for a live system. extern jlib_decl ISyncedPropertyTree * resolveSecret(const char *category, const char * name, const char * optRequiredVault, const char* optVersion); diff --git a/system/jlib/jsuperhash.hpp b/system/jlib/jsuperhash.hpp index 8806ad52993..2cd6216f412 100644 --- a/system/jlib/jsuperhash.hpp +++ b/system/jlib/jsuperhash.hpp @@ -178,9 +178,9 @@ class SimpleHashTableOf : public SuperHashTableOf { typedef SimpleHashTableOf SELF; public: - SimpleHashTableOf(void) : SuperHashTableOf() { } - SimpleHashTableOf(unsigned initsize) : SuperHashTableOf(initsize) { } - ~SimpleHashTableOf() { SELF::_releaseAll(); } + SimpleHashTableOf(void) : SuperHashTableOf() { } + SimpleHashTableOf(unsigned initsize) : SuperHashTableOf(initsize) { } + ~SimpleHashTableOf() { SELF::_releaseAll(); } IMPLEMENT_SUPERHASHTABLEOF_REF_FIND(ET, FP); @@ -209,9 +209,9 @@ class OwningSimpleHashTableOf : public SimpleHashTableOf { typedef OwningSimpleHashTableOf SELF; public: - OwningSimpleHashTableOf(void) : SimpleHashTableOf() { } - OwningSimpleHashTableOf(unsigned initsize) : SimpleHashTableOf(initsize) { } - ~OwningSimpleHashTableOf() { SELF::_releaseAll(); } + OwningSimpleHashTableOf(void) : SimpleHashTableOf() { } + OwningSimpleHashTableOf(unsigned initsize) : SimpleHashTableOf(initsize) { } + ~OwningSimpleHashTableOf() { SELF::_releaseAll(); } virtual void onRemove(void *et) { ((ET *)et)->Release(); } }; @@ -288,9 +288,9 @@ class StringSuperHashTableOf : public SuperHashTableOf { typedef StringSuperHashTableOf SELF; public: - StringSuperHashTableOf(void) : SuperHashTableOf() { } - StringSuperHashTableOf(unsigned initsize) : SuperHashTableOf(initsize) { } - ~StringSuperHashTableOf() { SELF::_releaseAll(); } + StringSuperHashTableOf(void) : SuperHashTableOf() { } + StringSuperHashTableOf(unsigned initsize) : SuperHashTableOf(initsize) { } + ~StringSuperHashTableOf() { SELF::_releaseAll(); } virtual void onAdd(void *et __attribute__((unused))) { } virtual void onRemove(void *et __attribute__((unused))) { } @@ -318,9 +318,9 @@ class OwningStringSuperHashTableOf : public StringSuperHashTableOf { typedef OwningStringSuperHashTableOf SELF; public: - OwningStringSuperHashTableOf(void) : StringSuperHashTableOf() { } - OwningStringSuperHashTableOf(unsigned initsize) : StringSuperHashTableOf(initsize) { } - ~OwningStringSuperHashTableOf() { SELF::_releaseAll(); } + OwningStringSuperHashTableOf(void) : StringSuperHashTableOf() { } + OwningStringSuperHashTableOf(unsigned initsize) : StringSuperHashTableOf(initsize) { } + ~OwningStringSuperHashTableOf() { SELF::_releaseAll(); } virtual void onRemove(void *et) { ((ET *)et)->Release(); } }; @@ -422,7 +422,7 @@ class ThreadSafeOwningSimpleHashTableOf : public ThreadSafeSimpleHashTableOf SELF; public: - ~ThreadSafeOwningSimpleHashTableOf() { SELF::_releaseAll(); } + ~ThreadSafeOwningSimpleHashTableOf() { SELF::_releaseAll(); } virtual void onRemove(void *et) { ((ET *)et)->Release(); } }; diff --git a/system/jlib/jtrace.cpp b/system/jlib/jtrace.cpp index 3914417ce8e..a776a5236dd 100644 --- a/system/jlib/jtrace.cpp +++ b/system/jlib/jtrace.cpp @@ -161,9 +161,12 @@ class CSpan : public CInterfaceOf virtual void beforeDispose() override { - StringBuffer out; - toLog(out); - LOG(MCmonitorEvent, "Span end: {%s}", out.str()); + if (queryTraceManager().logSpanFinish()) + { + StringBuffer out; + toLog(out); + LOG(MCmonitorEvent, "SpanFinish: {%s}", out.str()); + } } const char * getSpanID() const @@ -453,9 +456,12 @@ class CSpan : public CInterfaceOf { storeSpanContext(); - StringBuffer out; - toLog(out); - LOG(MCmonitorEvent, "Span start: {%s}", out.str()); + if (queryTraceManager().logSpanStart()) + { + StringBuffer out; + toLog(out); + LOG(MCmonitorEvent, "SpanStart: {%s}", out.str()); + } } } } @@ -755,6 +761,8 @@ class CTraceManager : implements ITraceManager, public CInterface bool enabled = true; bool optAlwaysCreateGlobalIds = false; bool optAlwaysCreateTraceIds = true; + bool optLogSpanStart = false; + bool optLogSpanFinish = true; StringAttr moduleName; //Initializes the global trace provider which is required for all Otel based tracing operations. @@ -942,6 +950,8 @@ class CTraceManager : implements ITraceManager, public CInterface //Non open-telemetry tracing configuration if (traceConfig) { + optLogSpanStart = traceConfig->getPropBool("@logSpanStart", optLogSpanStart); + optLogSpanFinish = traceConfig->getPropBool("@logSpanFinish", optLogSpanFinish); optAlwaysCreateGlobalIds = traceConfig->getPropBool("@alwaysCreateGlobalIds", optAlwaysCreateGlobalIds); optAlwaysCreateTraceIds = traceConfig->getPropBool("@alwaysCreateTraceIds", optAlwaysCreateTraceIds); } @@ -991,28 +1001,31 @@ class CTraceManager : implements ITraceManager, public CInterface throw makeStringExceptionV(-1, "TraceManager must be intialized!"); } - ISpan * createServerSpan(const char * name, StringArray & httpHeaders, SpanFlags flags) const override + virtual ISpan * createServerSpan(const char * name, StringArray & httpHeaders, SpanFlags flags) const override { Owned headerProperties = getHeadersAsProperties(httpHeaders); return new CServerSpan(name, moduleName.get(), headerProperties, flags); } - ISpan * createServerSpan(const char * name, const IProperties * httpHeaders, SpanFlags flags) const override + virtual ISpan * createServerSpan(const char * name, const IProperties * httpHeaders, SpanFlags flags) const override { return new CServerSpan(name, moduleName.get(), httpHeaders, flags); } - const char * getTracedComponentName() const override + virtual const char * getTracedComponentName() const override { return moduleName.get(); } - bool isTracingEnabled() const override + virtual bool isTracingEnabled() const override { return enabled; } - virtual bool alwaysCreateGlobalIds() const + //These functions are only accessed within this module, and should not really be in the public interface + //Something to possibly revisit later - either by passing as flags to the spans, or having a static + //function to provide the result of queryTraceManager() as a CTraceManager. + virtual bool alwaysCreateGlobalIds() const override { return optAlwaysCreateGlobalIds; } @@ -1021,6 +1034,16 @@ class CTraceManager : implements ITraceManager, public CInterface { return optAlwaysCreateTraceIds; } + + virtual bool logSpanStart() const override + { + return optLogSpanStart; + } + + virtual bool logSpanFinish() const override + { + return optLogSpanFinish; + } }; static Singleton theTraceManager; diff --git a/system/jlib/jtrace.hpp b/system/jlib/jtrace.hpp index f305fe49829..2c1234b0622 100644 --- a/system/jlib/jtrace.hpp +++ b/system/jlib/jtrace.hpp @@ -32,9 +32,9 @@ static constexpr const char *kCallerIdOtelAttributeName = "hpcc.callerid"; enum class SpanFlags : unsigned { - None = 0x000000000, - EnsureGlobalId = 0x000000001, - EnsureTraceId = 0x000000010, + None = 0x00000000, + EnsureGlobalId = 0x00000001, + EnsureTraceId = 0x00000002, }; BITMASK_ENUM(SpanFlags); @@ -66,6 +66,8 @@ interface ITraceManager : extends IInterface virtual bool isTracingEnabled() const = 0; virtual bool alwaysCreateGlobalIds() const = 0; virtual bool alwaysCreateTraceIds() const = 0; + virtual bool logSpanStart() const = 0; + virtual bool logSpanFinish() const = 0; virtual const char * getTracedComponentName() const = 0; }; diff --git a/system/logaccess/Azure/LogAnalytics/CurlClient/AzureLogAnalyticsCurlClient.cpp b/system/logaccess/Azure/LogAnalytics/CurlClient/AzureLogAnalyticsCurlClient.cpp index fad1077b23d..cd02bcceb9a 100644 --- a/system/logaccess/Azure/LogAnalytics/CurlClient/AzureLogAnalyticsCurlClient.cpp +++ b/system/logaccess/Azure/LogAnalytics/CurlClient/AzureLogAnalyticsCurlClient.cpp @@ -286,7 +286,7 @@ AzureLogAnalyticsCurlClient::AzureLogAnalyticsCurlClient(IPropertyTree & logAcce { PROGLOG("%s: Resolving all required configuration values...", COMPONENT_NAME); - Owned secretTree = getSecret(azureLogAccessSecretCategory, azureLogAccessSecretName); + Owned secretTree = getSecret(azureLogAccessSecretCategory, azureLogAccessSecretName); if (!secretTree) throw makeStringExceptionV(-1, "%s: Could not fetch %s information!", COMPONENT_NAME, azureLogAccessSecretName); diff --git a/system/mp/mpcomm.cpp b/system/mp/mpcomm.cpp index 483f79765e5..299f674e1fb 100644 --- a/system/mp/mpcomm.cpp +++ b/system/mp/mpcomm.cpp @@ -2132,7 +2132,7 @@ CMPConnectThread::CMPConnectThread(CMPServer *_parent, unsigned port, bool _list #if defined(_USE_OPENSSL) if (parent->useTLS) - secureContextServer.setown(createSecureSocketContextSecretSrv("local", true)); + secureContextServer.setown(createSecureSocketContextSecretSrv("local", nullptr, true)); #endif } diff --git a/system/security/LdapSecurity/ldapconnection.cpp b/system/security/LdapSecurity/ldapconnection.cpp index 3b52cf7ea4e..3c55315f9a6 100644 --- a/system/security/LdapSecurity/ldapconnection.cpp +++ b/system/security/LdapSecurity/ldapconnection.cpp @@ -409,7 +409,7 @@ class CLdapConfig : implements ILdapConfig, public CInterface DBGLOG("Retrieving LDAP Admin username/password from secrets repo: %s %s", !vaultId.isEmpty() ? vaultId.str() : "", adminUserSecretKey.str()); - Owned secretTree(getSecret("authn", adminUserSecretKey.str(), vaultId, nullptr)); + Owned secretTree(getSecret("authn", adminUserSecretKey.str(), vaultId, nullptr)); if (!secretTree) throw MakeStringException(-1, "Error retrieving LDAP Admin username/password"); @@ -494,7 +494,7 @@ class CLdapConfig : implements ILdapConfig, public CInterface cfg->getProp(".//@hpccAdminVaultId", vaultId);//optional HashiCorp vault ID DBGLOG("Retrieving optional HPCC Admin username/password from secrets repo: %s %s", !vaultId.isEmpty() ? vaultId.str() : "", adminUserSecretKey.str()); - Owned secretTree(getSecret("authn", adminUserSecretKey.str(), vaultId, nullptr)); + Owned secretTree(getSecret("authn", adminUserSecretKey.str(), vaultId, nullptr)); if (secretTree) { getSecretKeyValue(m_HPCCAdminUser_username, secretTree, "username"); diff --git a/system/security/plugins/testauthSecurity/testauthSecurity.cpp b/system/security/plugins/testauthSecurity/testauthSecurity.cpp index 7da09257991..feb800cbf8f 100644 --- a/system/security/plugins/testauthSecurity/testauthSecurity.cpp +++ b/system/security/plugins/testauthSecurity/testauthSecurity.cpp @@ -230,7 +230,7 @@ class CTestAuthSecurityManager : public CBaseSecurityManager const char* secretKey = userSettings.queryProp("@secretKey"); if (!isEmptyString(secretKey)) { - Owned secretTree = getSecret("authn", secretKey); + Owned secretTree = getSecret("authn", secretKey); if (!secretTree) throw makeStringExceptionV(-1, "Error retrieving the secret for %s.", secretKey); diff --git a/system/security/securesocket/securesocket.cpp b/system/security/securesocket/securesocket.cpp index 7c61a6c9a5f..cf35bd3ef40 100644 --- a/system/security/securesocket/securesocket.cpp +++ b/system/security/securesocket/securesocket.cpp @@ -2011,14 +2011,15 @@ SECURESOCKET_API ISecureSocketContext* createSecureSocketContextSecret(const cha } -SECURESOCKET_API ISecureSocketContext* createSecureSocketContextSecretSrv(const char *issuer, bool requireMtlsFlag) +SECURESOCKET_API ISecureSocketContext* createSecureSocketContextSecretSrv(const char *issuer, const char *optTrustedPeers, bool requireMtlsFlag) { if (requireMtlsFlag && !queryMtls()) - throw makeStringException(-100, "TLS secure communication requested but not configured"); + throw makeStringException(-100, "MTLS secure context required but not configured"); + + Owned info = getIssuerTlsSyncedConfig(issuer, optTrustedPeers, false); - Owned info = getIssuerTlsSyncedConfig(issuer); if (!info->isValid()) - throw makeStringException(-101, "TLS secure communication requested but not configured (2)"); + throw makeStringExceptionV(-101, "TLS issuer %s secure context requested but not configured (2)", issuer); return createSecureSocketContextSynced(info, ServerSocket); } @@ -2216,7 +2217,7 @@ class CSingletonSecureSocketConnection: public CSingletonSocketConnection state = Snone; cancelling = false; secureContextClient.setown(createSecureSocketContextSecret("local", ClientSocket)); - secureContextServer.setown(createSecureSocketContextSecretSrv("local", true)); + secureContextServer.setown(createSecureSocketContextSecretSrv("local", nullptr, true)); #ifdef _CONTAINERIZED tlsLogLevel = getComponentConfigSP()->getPropInt("logging/@detail", SSLogMin); if (tlsLogLevel >= ExtraneousMsgThreshold) // or InfoMsgThreshold ? diff --git a/system/security/securesocket/securesocket.hpp b/system/security/securesocket/securesocket.hpp index cc790676f7d..4d75bb56e75 100644 --- a/system/security/securesocket/securesocket.hpp +++ b/system/security/securesocket/securesocket.hpp @@ -84,13 +84,14 @@ typedef ISecureSocketContext* (*createSecureSocketContext_t)(SecureSocketType); typedef ISecureSocketContext* (*createSecureSocketContextEx_t)(const char* certFileOrBuf, const char* privKeyFileOrBuf, const char* passphrase, SecureSocketType); typedef ISecureSocketContext* (*createSecureSocketContextEx2_t)(IPropertyTree* config, SecureSocketType); typedef ISecureSocketContext* (*createSecureSocketContextSecret_t)(const char *mtlsSecretName, SecureSocketType); +typedef ISecureSocketContext* (*createSecureSocketContextSecretSrv_t)(const char *mtlsSecretName, const char *optTrustedPeers, bool requireMtlsConfig); extern "C" { //The following allow the creation of a secure socket context where the certificates will automatically be updated when they expire. SECURESOCKET_API ISecureSocketContext* createSecureSocketContextSynced(const ISyncedPropertyTree * config, SecureSocketType sockettype); // Will become the primary (only) factory method SECURESOCKET_API ISecureSocketContext* createSecureSocketContextSecret(const char *mtlsSecretName, SecureSocketType); -SECURESOCKET_API ISecureSocketContext* createSecureSocketContextSecretSrv(const char *mtlsSecretName, bool requireMtlsConfig); +SECURESOCKET_API ISecureSocketContext* createSecureSocketContextSecretSrv(const char *mtlsSecretName, const char *optTrustedPeers, bool requireMtlsConfig); SECURESOCKET_API ISecureSocketContext* createSecureSocketContextSSF(ISmartSocketFactory* ssf); //Helper function to aid migration to the functions above. This should eventually be removed. diff --git a/testing/regress/ecl/key/wasmembed.xml b/testing/regress/ecl/key/wasmembed.xml new file mode 100644 index 00000000000..4400c029084 --- /dev/null +++ b/testing/regress/ecl/key/wasmembed.xml @@ -0,0 +1,102 @@ + + true + + + true + + + true + + + true + + + true + + + true + + + true + + + true + + + true + + + true + + + true + + + true + + + true + + + true + + + true + + + true + + + true + + + true + + + true + + + true + + + true + + + true + + + true + + + true + + + true + + + true + + + true + + + true + + + true + + + true + + + true + + + true + + + true + + + true + diff --git a/testing/regress/ecl/wasmembed.ecl b/testing/regress/ecl/wasmembed.ecl new file mode 100644 index 00000000000..75d885e777e --- /dev/null +++ b/testing/regress/ecl/wasmembed.ecl @@ -0,0 +1,112 @@ +import wasm; + +boolean boolTest (boolean a, boolean b) := IMPORT(wasm, 'wasmembed.bool-test'); +real4 float32Test (real4 a, real4 b) := IMPORT(wasm, 'wasmembed.float32-test'); +real8 float64Test (real8 a, real8 b) := IMPORT(wasm, 'wasmembed.float64-test'); +unsigned1 u8Test (unsigned1 a, unsigned1 b) := IMPORT(wasm, 'wasmembed.u8-test'); +unsigned2 u16Test (unsigned2 a, unsigned2 b) := IMPORT(wasm, 'wasmembed.u16-test'); +unsigned4 u32Test (unsigned4 a, unsigned4 b) := IMPORT(wasm, 'wasmembed.u32-test'); +unsigned8 u64Test (unsigned8 a, unsigned8 b) := IMPORT(wasm, 'wasmembed.u64-test'); +integer1 s8Test (integer1 a, integer1 b) := IMPORT(wasm, 'wasmembed.s8-test'); +integer2 s16Test (integer2 a, integer2 b) := IMPORT(wasm, 'wasmembed.s16-test'); +integer4 s32Test (integer4 a, integer4 b) := IMPORT(wasm, 'wasmembed.s32-test'); +integer8 s64Test (integer8 a, integer8 b) := IMPORT(wasm, 'wasmembed.s64-test'); +string stringTest (string a, string b) := IMPORT(wasm, 'wasmembed.string-test'); +string12 string5Test (string5 a, string5 b) := IMPORT(wasm, 'wasmembed.string-test'); +varstring varstringTest (varstring a, varstring b) := IMPORT(wasm, 'wasmembed.string-test'); +unicode12 unicode5Test (unicode5 a, unicode5 b) := IMPORT(wasm, 'wasmembed.string-test'); +unicode unicodeTest (unicode a, unicode b) := IMPORT(wasm, 'wasmembed.string-test'); +utf8_12 utf8_5Test (utf8_5 a, utf8_5 b) := IMPORT(wasm, 'wasmembed.string-test'); +utf8 utf8Test (utf8 a, utf8 b) := IMPORT(wasm, 'wasmembed.string-test'); + +// '--- bool ---'; +boolTest(false, false) = (false AND false); +boolTest(false, true) = (false AND true); +boolTest(true, false) = (true AND false); +boolTest(true, true) = (true AND true); +// '--- float ---'; +ROUND(float32Test((real4)1234.1234, (real4)2345.2345), 3) = ROUND((real4)((real4)1234.1234 + (real4)2345.2345), 3); +float64Test(123456789.123456789, 23456789.23456789) = (real8)((real8)123456789.123456789 + (real8)23456789.23456789); +// '--- unsigned ---'; +u8Test(1, 2) = (unsigned1)(1 + 2); +u8Test(254, 1) = (unsigned1)(254 + 1); +u16Test(1, 2) = (unsigned2)(1 + 2); +u16Test(65534, 1) = (unsigned2)(65534 + 1); +u32Test(1, 2) = (unsigned4)(1 + 2); +u32Test(4294967294, 1) = (unsigned4)(4294967294 + 1); +u64Test(1, 2) = (unsigned8)(1 + 2); +u64Test(18446744073709551614, 1) = (unsigned8)(18446744073709551614 + 1); +// '--- signed ---'; +s8Test(1, 2) = (integer1)(1 + 2); +s8Test(126, 1) = (integer1)(126 + 1); +s8Test(-127, -1) = (integer1)(-127 - 1); + +s16Test(1, 2) = (integer2)(1 + 2); +s16Test(32766, 1) = (integer2)(32766 + 1); +s16Test(-32767, -1) = (integer2)(-32767 - 1); + +s32Test(1, 2) = (integer4)(1 + 2); +s32Test(2147483646, 1) = (integer4)(2147483646 + 1); +s32Test(-2147483647, -1) = (integer4)(-2147483647 - 1); + +s64Test(1, 2) = (integer8)(1 + 2); +s64Test(9223372036854775806, 1) = (integer8)(9223372036854775806 + 1); +s64Test(-9223372036854775807, -1) = (integer8)(-9223372036854775807 - 1); +// '--- string ---'; +varstringTest('1234567890', 'abcdefghij') = '1234567890' + 'abcdefghij'; +stringTest('1234567890', 'abcdefghij') = '1234567890' + 'abcdefghij'; +unicodeTest(U'1234567890您好1231231230', U'abcdefghij欢迎光临abcdefghij') = U'1234567890您好1231231230' + U'abcdefghij欢迎光临abcdefghij'; +utf8Test(U8'您好', U8'欢迎光临') = U8'您好' + U8'欢迎光临'; +// '--- string (fixed length) ---'; +string5Test('1234567890', 'abcdefghij') = (string12)((string5)'1234567890' + (string5)'abcdefghij'); +utf8_5Test(U8'您好1234567890', U8'欢迎光临abcdefghij') = (utf8_12)((utf8_5)U8'您好1234567890' + (utf8_5)U8'欢迎光临abcdefghij'); +unicode5Test(U'您好1234567890', U'欢迎光临abcdefghij') = (unicode12)((unicode5)U'您好1234567890' + (unicode5)U'欢迎光临abcdefghij'); +// '--- reentry ---'; +r := RECORD + unsigned1 kind; + string20 word; + unsigned8 doc; + unsigned1 segment; + unsigned8 wpos; + END; +d := dataset('~regress::multi::searchsource', r, THOR); + +r2 := RECORD(r) + unsigned8 newUnsigned; + string newWord; + boolean passed; +END; + +r2 t(r L) := TRANSFORM + SELF.newUnsigned := u64Test(L.doc, L.wpos); + boolean a := SELF.newUnsigned = (unsigned8)(L.doc + L.wpos); + SELF.newWord := stringTest(L.word, L.word); + boolean b := SELF.newWord = L.word + L.word; + SELF.passed := a and B; + SELF := L; +END; + +r2 t2(r L) := TRANSFORM + SELF.newUnsigned := u64Test(L.doc, L.wpos); + boolean a := SELF.newUnsigned = L.doc+ L.wpos; + SELF.newWord := L.word + L.word; + boolean b := SELF.newWord = L.word + L.word; + SELF.passed := a and B; + SELF := L; +END; + +r2 t3(r L) := TRANSFORM + SELF.newUnsigned := L.doc+ L.wpos; + boolean a := SELF.newUnsigned = L.doc+ L.wpos; + SELF.newWord := L.word + L.word; + boolean b := SELF.newWord = L.word + L.word; + SELF.passed := a and B; + SELF := L; +END; + +unsigned sampleSize := 10000; +d2 := project(choosen(d, sampleSize), t(LEFT)); +d3 := project(choosen(d, sampleSize, 5000), t(LEFT)); +d4 := project(choosen(d, sampleSize, 10001), t(LEFT)); +count(d2(passed=false)) = 0 AND count(d3(passed=false)) = 0 AND count(d4(passed=false)) = 0; +// '--- --- ---'; diff --git a/testing/regress/ecl/wasmembed.manifest b/testing/regress/ecl/wasmembed.manifest new file mode 100644 index 00000000000..8a2d68eb5c2 --- /dev/null +++ b/testing/regress/ecl/wasmembed.manifest @@ -0,0 +1,3 @@ + + + diff --git a/testing/regress/ecl/wasmembed.wasm b/testing/regress/ecl/wasmembed.wasm new file mode 100644 index 00000000000..f2f32c33c95 Binary files /dev/null and b/testing/regress/ecl/wasmembed.wasm differ diff --git a/testing/unittests/jlibtests.cpp b/testing/unittests/jlibtests.cpp index 7efdd4611c9..9918b77c5ef 100644 --- a/testing/unittests/jlibtests.cpp +++ b/testing/unittests/jlibtests.cpp @@ -32,11 +32,14 @@ #include "jlzw.hpp" #include "jqueue.hpp" #include "jregexp.hpp" +#include "jsecrets.hpp" #include "jutil.hpp" #include "junicode.hpp" #include "unittests.hpp" +#define CPPUNIT_ASSERT_EQUAL_STR(x, y) CPPUNIT_ASSERT_EQUAL(std::string(x),std::string(y)) + static const unsigned oneMinute = 60000; // msec class JlibTraceTest : public CppUnit::TestFixture @@ -3627,4 +3630,204 @@ CPPUNIT_TEST_SUITE_REGISTRATION( JLibUnicodeTest ); CPPUNIT_TEST_SUITE_NAMED_REGISTRATION( JLibUnicodeTest, "JLibUnicodeTest" ); +class JLibSecretsTest : public CppUnit::TestFixture +{ +public: + CPPUNIT_TEST_SUITE(JLibSecretsTest); + CPPUNIT_TEST(setup); + CPPUNIT_TEST(testUpdate1); + CPPUNIT_TEST(testUpdate2); + CPPUNIT_TEST_SUITE_END(); + + //Each test creates a different instance of the class(!) so member values cannot be used to pass items + //from one test to another + StringBuffer secretRoot; + +protected: + void checkSecret(const char * secret, const char * key, const char * expectedValue) + { + Owned match = getSecret("testing", secret); + CPPUNIT_ASSERT(match); + const char * secretValue = match->queryProp(key); + if (secretValue) + { + CPPUNIT_ASSERT_EQUAL_STR(secretValue, expectedValue); + } + else + { + //IPropertyTree doesn't allow blank values, so a missing value is the same as a blank value + //We should probably revisit some day, but it is likely to break existing code if we do. + CPPUNIT_ASSERT_EQUAL_STR("", expectedValue); + } + } + + bool hasSecret(const char * name) + { + Owned match = getSecret("testing", name); + return match != nullptr; + } + + void initPath() + { + char cwd[1024]; + CPPUNIT_ASSERT(GetCurrentDirectory(1024, cwd)); + secretRoot.set(cwd).append(PATHSEPCHAR).append("unittest-secrets"); + secretRoot.append(PATHSEPCHAR).append("testing"); // catgegory + } + + void setup() + { + char cwd[1024]; + CPPUNIT_ASSERT(GetCurrentDirectory(1024, cwd)); + secretRoot.append(cwd).append(PATHSEPCHAR).append("unittest-secrets"); + + recursiveRemoveDirectory(secretRoot); + CPPUNIT_ASSERT(recursiveCreateDirectory(secretRoot.str())); + setSecretMount(secretRoot); + setSecretTimeout(100); // Set the timeout so we can check it is working. + + secretRoot.append(PATHSEPCHAR).append("testing"); // catgegory + CPPUNIT_ASSERT(recursiveCreateDirectory(secretRoot.str())); + } + + void testUpdate1() + { + initPath(); // secretRoot needs to be called for each test + + CPPUNIT_ASSERT(!hasSecret("secret1")); + writeTestingSecret("secret1", "value", "secret1Value"); + //Secret should not appear yet - null should be cached. + CPPUNIT_ASSERT(!hasSecret("secret1")); + + Owned secret2 = resolveSecret("testing", "secret2", nullptr, nullptr); + CPPUNIT_ASSERT(!secret2->isValid()); + CPPUNIT_ASSERT(!secret2->isStale()); + + MilliSleep(50); + //Secret should not appear yet - null should be cached. + CPPUNIT_ASSERT(!hasSecret("secret1")); + CPPUNIT_ASSERT(!secret2->isValid()); + CPPUNIT_ASSERT(!secret2->isStale()); + + MilliSleep(100); + //Secret1 should now be updated - enough time has passed + checkSecret("secret1", "value", "secret1Value"); + CPPUNIT_ASSERT(!secret2->isValid()); + CPPUNIT_ASSERT(secret2->isStale()); + + //Cleanup + writeTestingSecret("secret1", "value", nullptr); + } + + void testUpdate2() + { + initPath(); // secretRoot needs to be called for each test + + Owned secret3 = resolveSecret("testing", "secret3", nullptr, nullptr); + unsigned version = secret3->getVersion(); + CPPUNIT_ASSERT(!secret3->isValid()); + CPPUNIT_ASSERT(!secret3->isStale()); + writeTestingSecret("secret3", "value", "secret3Value"); + CPPUNIT_ASSERT(!secret3->isValid()); + CPPUNIT_ASSERT_EQUAL(version, secret3->getVersion()); + + //After sleep new value should not have been picked up + MilliSleep(50); + CPPUNIT_ASSERT(!secret3->isValid()); + CPPUNIT_ASSERT_EQUAL(version, secret3->getVersion()); + + //After sleep new value should now have been picked up + MilliSleep(100); + checkSecret("secret3", "value", "secret3Value"); + unsigned version2 = secret3->getVersion(); + CPPUNIT_ASSERT(!secret3->isStale()); + CPPUNIT_ASSERT(secret3->isValid()); + CPPUNIT_ASSERT(version != version2); + + //Sleep and check that the hash value has not changed + MilliSleep(200); + checkSecret("secret3", "value", "secret3Value"); + CPPUNIT_ASSERT(!secret3->isStale()); + CPPUNIT_ASSERT(secret3->isValid()); + CPPUNIT_ASSERT_EQUAL(version2, secret3->getVersion()); + + //Remove the secret - should have no immediate effect + writeTestingSecret("secret3", "value", nullptr); + CPPUNIT_ASSERT(!secret3->isStale()); + CPPUNIT_ASSERT(secret3->isValid()); + CPPUNIT_ASSERT_EQUAL(version2, secret3->getVersion()); + + MilliSleep(50); + CPPUNIT_ASSERT(!secret3->isStale()); + CPPUNIT_ASSERT(secret3->isValid()); + CPPUNIT_ASSERT_EQUAL(version2, secret3->getVersion()); + checkSecret("secret3", "value", "secret3Value"); + + MilliSleep(100); + CPPUNIT_ASSERT(secret3->isStale()); // Value has gone, but the old value is still returned + CPPUNIT_ASSERT(secret3->isValid()); + CPPUNIT_ASSERT_EQUAL(version2, secret3->getVersion()); + checkSecret("secret3", "value", "secret3Value"); + + //Update the value = the change should not be seen until the cache entry expires + writeTestingSecret("secret3", "value", "secret3NewValue"); + CPPUNIT_ASSERT(secret3->isStale()); + CPPUNIT_ASSERT(secret3->isValid()); + CPPUNIT_ASSERT_EQUAL(version2, secret3->getVersion()); + checkSecret("secret3", "value", "secret3Value"); + + MilliSleep(50); + CPPUNIT_ASSERT(secret3->isStale()); + CPPUNIT_ASSERT(secret3->isValid()); + CPPUNIT_ASSERT_EQUAL(version2, secret3->getVersion()); + checkSecret("secret3", "value", "secret3Value"); + + MilliSleep(100); + //These functions do not check for up to date values, so they return the same as before + CPPUNIT_ASSERT(secret3->isStale()); // Value still appears to be out of date + CPPUNIT_ASSERT(secret3->isValid()); + + //The getVersion() should force the value to be updated + unsigned version3 = secret3->getVersion(); + CPPUNIT_ASSERT(version2 != version3); + CPPUNIT_ASSERT(!secret3->isStale()); // New value has now been picked up + CPPUNIT_ASSERT(secret3->isValid()); + checkSecret("secret3", "value", "secret3NewValue"); + + //Finally check that writing a blank value is spotted as a change. + writeTestingSecret("secret3", "value", ""); + MilliSleep(150); + //Check the version to ensure that the value has been updated + CPPUNIT_ASSERT(version3 != secret3->getVersion()); + CPPUNIT_ASSERT(secret3->isValid()); + checkSecret("secret3", "value", ""); + + //Cleanup + writeTestingSecret("secret3", "value", nullptr); + } + + void writeTestingSecret(const char * secret, const char * key, const char * value) + { + StringBuffer filename; + filename.append(secretRoot).append(PATHSEPCHAR).append(secret); + CPPUNIT_ASSERT(recursiveCreateDirectory(filename.str())); + + filename.append(PATHSEPCHAR).append(key); + + Owned file = createIFile(filename.str()); + if (value) + { + Owned io = file->open(IFOcreate); + io->write(0, strlen(value), value); + } + else + file->remove(); + } +}; + +CPPUNIT_TEST_SUITE_REGISTRATION( JLibSecretsTest ); +CPPUNIT_TEST_SUITE_NAMED_REGISTRATION( JLibSecretsTest, "JLibSecretsTest" ); + + + #endif // _USE_CPPUNIT diff --git a/thorlcr/msort/tsorts1.cpp b/thorlcr/msort/tsorts1.cpp index e1f0c10cfb0..abe76aeeef9 100644 --- a/thorlcr/msort/tsorts1.cpp +++ b/thorlcr/msort/tsorts1.cpp @@ -315,7 +315,7 @@ protected: friend class CSortMerge; #if defined(_USE_OPENSSL) if (slave.queryTLS()) { - secureContextServer.setown(createSecureSocketContextSecretSrv("local", true)); + secureContextServer.setown(createSecureSocketContextSecretSrv("local", nullptr, true)); secureContextClients.setown(createSecureSocketContextSecret("local", ClientSocket)); } #endif diff --git a/vcpkg.json.in b/vcpkg.json.in index 7396715d0af..199c2b42bde 100644 --- a/vcpkg.json.in +++ b/vcpkg.json.in @@ -181,6 +181,10 @@ "name": "tbb", "platform": "@VCPKG_TBB@" }, + { + "name": "wasmtime-cpp-api", + "platform": "@VCPKG_WASMEMBED@" + }, { "name": "winflexbison", "platform": "windows"