From 4c7085d5b895def78c98c1d38d91dc1ec5b67ae2 Mon Sep 17 00:00:00 2001 From: Jeremy Clements <79224539+jeclrsg@users.noreply.github.com> Date: Thu, 16 Nov 2023 13:13:04 -0500 Subject: [PATCH 01/16] HPCC-30777 ECL Watch v9 add missing TotalClusterTime to WU Summary Signed-off-by: Jeremy Clements <79224539+jeclrsg@users.noreply.github.com> --- esp/src/package-lock.json | 8 ++++---- esp/src/package.json | 2 +- esp/src/src-react/hooks/workunit.ts | 2 +- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/esp/src/package-lock.json b/esp/src/package-lock.json index 77fb9cfbc06..2d17e26a187 100644 --- a/esp/src/package-lock.json +++ b/esp/src/package-lock.json @@ -17,7 +17,7 @@ "@hpcc-js/chart": "2.81.7", "@hpcc-js/codemirror": "2.60.12", "@hpcc-js/common": "2.71.12", - "@hpcc-js/comms": "2.84.4", + "@hpcc-js/comms": "2.85.0", "@hpcc-js/dataflow": "8.1.6", "@hpcc-js/eclwatch": "2.73.27", "@hpcc-js/graph": "2.85.8", @@ -1577,9 +1577,9 @@ } }, "node_modules/@hpcc-js/comms": { - "version": "2.84.4", - "resolved": "https://registry.npmjs.org/@hpcc-js/comms/-/comms-2.84.4.tgz", - "integrity": "sha512-GkKUyttPRYPb/jluoNtxY6jEqJhF2Dr0srzXrF+Vs0biyr4GvwQzMcoQfU97PQl7PJx8M+FUquMchfuQ7sqSWA==", + "version": "2.85.0", + "resolved": "https://registry.npmjs.org/@hpcc-js/comms/-/comms-2.85.0.tgz", + "integrity": "sha512-ZnvI7T35qyj6Dm1uT7f+j1Xgq0QGPyMAypGq8jacq2VAo2Q24eg/AL57E8PBLOM1Q0UAV/WYJqz4eTk9n+UwgQ==", "dependencies": { "@hpcc-js/ddl-shim": "^2.20.6", "@hpcc-js/util": "^2.50.6", diff --git a/esp/src/package.json b/esp/src/package.json index ef77a32ff76..6df836b8b9d 100644 --- a/esp/src/package.json +++ b/esp/src/package.json @@ -42,7 +42,7 @@ "@hpcc-js/chart": "2.81.7", "@hpcc-js/codemirror": "2.60.12", "@hpcc-js/common": "2.71.12", - "@hpcc-js/comms": "2.84.4", + "@hpcc-js/comms": "2.85.0", "@hpcc-js/dataflow": "8.1.6", "@hpcc-js/eclwatch": "2.73.27", "@hpcc-js/graph": "2.85.8", diff --git a/esp/src/src-react/hooks/workunit.ts b/esp/src/src-react/hooks/workunit.ts index 783061de755..9b331369fbc 100644 --- a/esp/src/src-react/hooks/workunit.ts +++ b/esp/src/src-react/hooks/workunit.ts @@ -23,7 +23,7 @@ export function useWorkunit(wuid: string, full: boolean = false): [Workunit, WUS let active = true; let handle; const refresh = singletonDebounce(wu, "refresh"); - refresh(full) + refresh(full, { IncludeTotalClusterTime: true }) .then(() => { if (active) { setRetVal({ workunit: wu, state: wu.StateID, lastUpdate: Date.now(), isComplete: wu.isComplete(), refresh }); From 1be8e2d6cb8de8e95e83c3bc5e7cac7b5a88133a Mon Sep 17 00:00:00 2001 From: wangkx Date: Thu, 16 Nov 2023 14:24:07 -0500 Subject: [PATCH 02/16] HPCC-30685 HPCC-30685 Report State always for Scheduled ECL WUs Without this fix, the WU state is displayed inside the ECLWatch Event Scheduler tab only when a state filter is set inside the tab. Signed-off-by: wangkx --- .../ws_workunits/ws_workunitsService.cpp | 32 +++++++------------ 1 file changed, 12 insertions(+), 20 deletions(-) diff --git a/esp/services/ws_workunits/ws_workunitsService.cpp b/esp/services/ws_workunits/ws_workunitsService.cpp index 0f5efa8f24b..3450b185109 100644 --- a/esp/services/ws_workunits/ws_workunitsService.cpp +++ b/esp/services/ws_workunits/ws_workunitsService.cpp @@ -3626,6 +3626,16 @@ void getScheduledWUs(IEspContext &context, WUShowScheduledFilters *filters, cons { jobName.set(cw->queryJobName()); owner.set(cw->queryUser()); + if ((cw->getState() == WUStateScheduled) && cw->aborting()) + { + stateID = WUStateAborting; + state.set("aborting"); + } + else + { + stateID = cw->getState(); + state.set(cw->queryStateDesc()); + } } if (!filters->jobName.isEmpty() && (jobName.isEmpty() || !WildMatch(jobName.str(), filters->jobName, true))) @@ -3634,26 +3644,8 @@ void getScheduledWUs(IEspContext &context, WUShowScheduledFilters *filters, cons match = false; else if (!filters->eventText.isEmpty() && (ieventText.isEmpty() || !WildMatch(ieventText, filters->eventText, true))) match = false; - else if (!filters->state.isEmpty()) - { - if (!cw) - match = false; - else - { - if ((cw->getState() == WUStateScheduled) && cw->aborting()) - { - stateID = WUStateAborting; - state.set("aborting"); - } - else - { - stateID = cw->getState(); - state.set(cw->queryStateDesc()); - } - if (!strieq(filters->state, state.str())) - match = false; - } - } + else if (!filters->state.isEmpty() && !strisame(filters->state, state.str())) + match = false; } catch (IException *e) { From d93e2fb2c0bf9dca708b8d39bddba0d5392b1640 Mon Sep 17 00:00:00 2001 From: Anthony Fishbeck Date: Thu, 16 Nov 2023 16:02:58 -0500 Subject: [PATCH 03/16] HPCC-30755 Allow ESP server TLS config to be based on an issuer name Signed-off-by: Anthony Fishbeck --- esp/bindings/http/platform/httpprot.cpp | 48 +++++++++++-------- helm/hpcc/templates/esp.yaml | 1 + system/mp/mpcomm.cpp | 2 +- system/security/securesocket/securesocket.cpp | 11 +++-- system/security/securesocket/securesocket.hpp | 3 +- thorlcr/msort/tsorts1.cpp | 2 +- 6 files changed, 38 insertions(+), 29 deletions(-) diff --git a/esp/bindings/http/platform/httpprot.cpp b/esp/bindings/http/platform/httpprot.cpp index d4b2b69285e..e4e6af64ec1 100644 --- a/esp/bindings/http/platform/httpprot.cpp +++ b/esp/bindings/http/platform/httpprot.cpp @@ -216,32 +216,38 @@ CSecureHttpProtocol::CSecureHttpProtocol(IPropertyTree* cfg) { m_config.setown(cfg); - //ensure keys are specified. Passphrase is optional - StringBuffer sb; - cfg->getProp("certificate", sb); - if(sb.length() == 0) - { - throw MakeStringException(-1, "certificate file not specified in config file"); - } + IEspPlugin *pplg = loadPlugin(SSLIB); + if (!pplg) + throw MakeStringException(-1, "dll/shared-object %s can't be loaded", SSLIB); - cfg->getProp("privatekey", sb.clear()); - if(sb.length() == 0) + const char *issuer = cfg->queryProp("issuer"); + if (!isEmptyString(issuer)) { - throw MakeStringException(-1, "private key file not specified in config file"); + const char *trustedPeers = nullptr; + if (cfg->hasProp("verify")) + trustedPeers = cfg->queryProp("verify/trusted_peers"); + createSecureSocketContextSecretSrv_t xproc = (createSecureSocketContextSecretSrv_t) pplg->getProcAddress("createSecureSocketContextSecretSrv"); + if (!xproc) + throw MakeStringException(-1, "procedure createSecureSocketContextSecretSrv can't be loaded"); + m_ssctx.setown(xproc(issuer, trustedPeers, false)); } - - createSecureSocketContextEx2_t xproc = NULL; - IEspPlugin *pplg = loadPlugin(SSLIB); - if (pplg) - xproc = (createSecureSocketContextEx2_t) pplg->getProcAddress("createSecureSocketContextEx2"); else - throw MakeStringException(-1, "dll/shared-object %s can't be loaded", SSLIB); - - - if (xproc) + { + //ensure keys are specified. Passphrase is optional + StringBuffer sb; + cfg->getProp("certificate", sb); + if(sb.isEmpty()) + throw MakeStringException(-1, "certificate file not specified in config file"); + + cfg->getProp("privatekey", sb.clear()); + if(sb.isEmpty()) + throw MakeStringException(-1, "private key file not specified in config file"); + + createSecureSocketContextEx2_t xproc = (createSecureSocketContextEx2_t) pplg->getProcAddress("createSecureSocketContextEx2"); + if (!xproc) + throw MakeStringException(-1, "procedure createSecureSocketContextEx2 can't be loaded"); m_ssctx.setown(xproc(cfg, ServerSocket)); - else - throw MakeStringException(-1, "procedure createSecureSocketContextEx2 can't be loaded"); + } } } diff --git a/helm/hpcc/templates/esp.yaml b/helm/hpcc/templates/esp.yaml index 032e6dc943b..30b47df9773 100644 --- a/helm/hpcc/templates/esp.yaml +++ b/helm/hpcc/templates/esp.yaml @@ -44,6 +44,7 @@ data: tls: {{ include "hpcc.isIssuerEnabled" (dict "root" .root "issuerKeyName" $issuerKeyName) }} {{- end }} tls_config: + issuer: {{ $issuerKeyName }} {{- if $externalCert }} certificate: /opt/HPCCSystems/secrets/certificates/{{ $issuerKeyName }}/tls.crt privatekey: /opt/HPCCSystems/secrets/certificates/{{ $issuerKeyName }}/tls.key diff --git a/system/mp/mpcomm.cpp b/system/mp/mpcomm.cpp index 483f79765e5..299f674e1fb 100644 --- a/system/mp/mpcomm.cpp +++ b/system/mp/mpcomm.cpp @@ -2132,7 +2132,7 @@ CMPConnectThread::CMPConnectThread(CMPServer *_parent, unsigned port, bool _list #if defined(_USE_OPENSSL) if (parent->useTLS) - secureContextServer.setown(createSecureSocketContextSecretSrv("local", true)); + secureContextServer.setown(createSecureSocketContextSecretSrv("local", nullptr, true)); #endif } diff --git a/system/security/securesocket/securesocket.cpp b/system/security/securesocket/securesocket.cpp index 7c61a6c9a5f..cf35bd3ef40 100644 --- a/system/security/securesocket/securesocket.cpp +++ b/system/security/securesocket/securesocket.cpp @@ -2011,14 +2011,15 @@ SECURESOCKET_API ISecureSocketContext* createSecureSocketContextSecret(const cha } -SECURESOCKET_API ISecureSocketContext* createSecureSocketContextSecretSrv(const char *issuer, bool requireMtlsFlag) +SECURESOCKET_API ISecureSocketContext* createSecureSocketContextSecretSrv(const char *issuer, const char *optTrustedPeers, bool requireMtlsFlag) { if (requireMtlsFlag && !queryMtls()) - throw makeStringException(-100, "TLS secure communication requested but not configured"); + throw makeStringException(-100, "MTLS secure context required but not configured"); + + Owned info = getIssuerTlsSyncedConfig(issuer, optTrustedPeers, false); - Owned info = getIssuerTlsSyncedConfig(issuer); if (!info->isValid()) - throw makeStringException(-101, "TLS secure communication requested but not configured (2)"); + throw makeStringExceptionV(-101, "TLS issuer %s secure context requested but not configured (2)", issuer); return createSecureSocketContextSynced(info, ServerSocket); } @@ -2216,7 +2217,7 @@ class CSingletonSecureSocketConnection: public CSingletonSocketConnection state = Snone; cancelling = false; secureContextClient.setown(createSecureSocketContextSecret("local", ClientSocket)); - secureContextServer.setown(createSecureSocketContextSecretSrv("local", true)); + secureContextServer.setown(createSecureSocketContextSecretSrv("local", nullptr, true)); #ifdef _CONTAINERIZED tlsLogLevel = getComponentConfigSP()->getPropInt("logging/@detail", SSLogMin); if (tlsLogLevel >= ExtraneousMsgThreshold) // or InfoMsgThreshold ? diff --git a/system/security/securesocket/securesocket.hpp b/system/security/securesocket/securesocket.hpp index cc790676f7d..4d75bb56e75 100644 --- a/system/security/securesocket/securesocket.hpp +++ b/system/security/securesocket/securesocket.hpp @@ -84,13 +84,14 @@ typedef ISecureSocketContext* (*createSecureSocketContext_t)(SecureSocketType); typedef ISecureSocketContext* (*createSecureSocketContextEx_t)(const char* certFileOrBuf, const char* privKeyFileOrBuf, const char* passphrase, SecureSocketType); typedef ISecureSocketContext* (*createSecureSocketContextEx2_t)(IPropertyTree* config, SecureSocketType); typedef ISecureSocketContext* (*createSecureSocketContextSecret_t)(const char *mtlsSecretName, SecureSocketType); +typedef ISecureSocketContext* (*createSecureSocketContextSecretSrv_t)(const char *mtlsSecretName, const char *optTrustedPeers, bool requireMtlsConfig); extern "C" { //The following allow the creation of a secure socket context where the certificates will automatically be updated when they expire. SECURESOCKET_API ISecureSocketContext* createSecureSocketContextSynced(const ISyncedPropertyTree * config, SecureSocketType sockettype); // Will become the primary (only) factory method SECURESOCKET_API ISecureSocketContext* createSecureSocketContextSecret(const char *mtlsSecretName, SecureSocketType); -SECURESOCKET_API ISecureSocketContext* createSecureSocketContextSecretSrv(const char *mtlsSecretName, bool requireMtlsConfig); +SECURESOCKET_API ISecureSocketContext* createSecureSocketContextSecretSrv(const char *mtlsSecretName, const char *optTrustedPeers, bool requireMtlsConfig); SECURESOCKET_API ISecureSocketContext* createSecureSocketContextSSF(ISmartSocketFactory* ssf); //Helper function to aid migration to the functions above. This should eventually be removed. diff --git a/thorlcr/msort/tsorts1.cpp b/thorlcr/msort/tsorts1.cpp index e1f0c10cfb0..abe76aeeef9 100644 --- a/thorlcr/msort/tsorts1.cpp +++ b/thorlcr/msort/tsorts1.cpp @@ -315,7 +315,7 @@ protected: friend class CSortMerge; #if defined(_USE_OPENSSL) if (slave.queryTLS()) { - secureContextServer.setown(createSecureSocketContextSecretSrv("local", true)); + secureContextServer.setown(createSecureSocketContextSecretSrv("local", nullptr, true)); secureContextClients.setown(createSecureSocketContextSecret("local", ClientSocket)); } #endif From 6eea4ca1ab74b380ee59ebf0decab67e4bef9473 Mon Sep 17 00:00:00 2001 From: Jake Smith Date: Fri, 17 Nov 2023 11:23:44 +0000 Subject: [PATCH 04/16] Split off 8.10.66 Signed-off-by: Jake Smith --- helm/hpcc/Chart.yaml | 4 ++-- helm/hpcc/templates/_helpers.tpl | 2 +- helm/hpcc/templates/dafilesrv.yaml | 2 +- helm/hpcc/templates/dali.yaml | 2 +- helm/hpcc/templates/dfuserver.yaml | 2 +- helm/hpcc/templates/eclagent.yaml | 4 ++-- helm/hpcc/templates/eclccserver.yaml | 4 ++-- helm/hpcc/templates/eclscheduler.yaml | 2 +- helm/hpcc/templates/esp.yaml | 2 +- helm/hpcc/templates/localroxie.yaml | 2 +- helm/hpcc/templates/roxie.yaml | 8 ++++---- helm/hpcc/templates/sasha.yaml | 2 +- helm/hpcc/templates/thor.yaml | 10 +++++----- version.cmake | 2 +- 14 files changed, 24 insertions(+), 24 deletions(-) diff --git a/helm/hpcc/Chart.yaml b/helm/hpcc/Chart.yaml index 0598d9d6e8f..6778d9b6194 100644 --- a/helm/hpcc/Chart.yaml +++ b/helm/hpcc/Chart.yaml @@ -6,9 +6,9 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. -version: 8.10.65-closedown0 +version: 8.10.67-closedown0 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. -appVersion: 8.10.65-closedown0 +appVersion: 8.10.67-closedown0 diff --git a/helm/hpcc/templates/_helpers.tpl b/helm/hpcc/templates/_helpers.tpl index 1cb56ed2bad..df2b93e61d5 100644 --- a/helm/hpcc/templates/_helpers.tpl +++ b/helm/hpcc/templates/_helpers.tpl @@ -1208,7 +1208,7 @@ kind: Service metadata: name: {{ $lvars.serviceName | quote }} labels: - helmVersion: 8.10.65-closedown0 + helmVersion: 8.10.67-closedown0 {{- include "hpcc.addStandardLabels" (dict "root" $.root "instance" $lvars.serviceName ) | indent 4 }} {{- if $lvars.labels }} {{ toYaml $lvars.labels | indent 4 }} diff --git a/helm/hpcc/templates/dafilesrv.yaml b/helm/hpcc/templates/dafilesrv.yaml index d3d7793c01d..1a716392fb3 100644 --- a/helm/hpcc/templates/dafilesrv.yaml +++ b/helm/hpcc/templates/dafilesrv.yaml @@ -50,7 +50,7 @@ spec: labels: {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "dafilesrv" "name" "dafilesrv" "instance" .name) | indent 8 }} server: {{ .name | quote }} - helmVersion: 8.10.65-closedown0 + helmVersion: 8.10.67-closedown0 annotations: checksum/config: {{ $configSHA }} spec: diff --git a/helm/hpcc/templates/dali.yaml b/helm/hpcc/templates/dali.yaml index 4d11f5b9b4c..d351bb6c9c3 100644 --- a/helm/hpcc/templates/dali.yaml +++ b/helm/hpcc/templates/dali.yaml @@ -81,7 +81,7 @@ spec: run: {{ $dali.name | quote }} server: {{ $dali.name | quote }} app: dali - helmVersion: 8.10.65-closedown0 + helmVersion: 8.10.67-closedown0 {{- if hasKey $.Values.global "metrics" }} {{- include "hpcc.generateMetricsReporterLabel" $.Values.global.metrics | nindent 8 }} {{- end }} diff --git a/helm/hpcc/templates/dfuserver.yaml b/helm/hpcc/templates/dfuserver.yaml index 228b7243926..e1fe83eee0f 100644 --- a/helm/hpcc/templates/dfuserver.yaml +++ b/helm/hpcc/templates/dfuserver.yaml @@ -56,7 +56,7 @@ spec: {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "dfuserver" "name" "dfuserver" "instance" .name) | indent 8 }} run: {{ .name | quote }} accessDali: "yes" - helmVersion: 8.10.65-closedown0 + helmVersion: 8.10.67-closedown0 {{- if hasKey . "labels" }} {{ toYaml .labels | indent 8 }} {{- end }} diff --git a/helm/hpcc/templates/eclagent.yaml b/helm/hpcc/templates/eclagent.yaml index a82404c56b2..97d528ff667 100644 --- a/helm/hpcc/templates/eclagent.yaml +++ b/helm/hpcc/templates/eclagent.yaml @@ -58,7 +58,7 @@ data: {{- include "hpcc.addStandardLabels" (dict "root" $ "component" $apptype "name" "eclagent" "instance" $appJobName "instanceOf" (printf "%s-job" .me.name)) | indent 12 }} accessDali: "yes" accessEsp: "yes" - helmVersion: 8.10.65-closedown0 + helmVersion: 8.10.67-closedown0 {{- if hasKey .me "labels" }} {{ toYaml .me.labels | indent 12 }} {{- end }} @@ -137,7 +137,7 @@ spec: run: {{ .name | quote }} accessDali: "yes" accessEsp: {{ .useChildProcesses | default false | ternary "yes" "no" | quote }} - helmVersion: 8.10.65-closedown0 + helmVersion: 8.10.67-closedown0 {{- if hasKey . "labels" }} {{ toYaml .labels | indent 8 }} {{- end }} diff --git a/helm/hpcc/templates/eclccserver.yaml b/helm/hpcc/templates/eclccserver.yaml index c2730b3d562..3640b3dd055 100644 --- a/helm/hpcc/templates/eclccserver.yaml +++ b/helm/hpcc/templates/eclccserver.yaml @@ -57,7 +57,7 @@ data: {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "eclccserver" "name" "eclccserver" "instance" $compileJobName "instanceOf" (printf "%s-job" .me.name)) | indent 12 }} accessDali: "yes" accessEsp: "yes" - helmVersion: 8.10.65-closedown0 + helmVersion: 8.10.67-closedown0 {{- if hasKey .me "labels" }} {{ toYaml .me.labels | indent 12 }} {{- end }} @@ -142,7 +142,7 @@ spec: run: {{ .name | quote }} accessDali: "yes" accessEsp: {{ .useChildProcesses | default false | ternary "yes" "no" | quote }} - helmVersion: 8.10.65-closedown0 + helmVersion: 8.10.67-closedown0 {{- if hasKey . "labels" }} {{ toYaml .labels | indent 8 }} {{- end }} diff --git a/helm/hpcc/templates/eclscheduler.yaml b/helm/hpcc/templates/eclscheduler.yaml index 753e23a02ea..fef0cd9a0f8 100644 --- a/helm/hpcc/templates/eclscheduler.yaml +++ b/helm/hpcc/templates/eclscheduler.yaml @@ -64,7 +64,7 @@ spec: run: {{ .name | quote }} accessDali: "yes" accessEsp: "no" - helmVersion: 8.10.65-closedown0 + helmVersion: 8.10.67-closedown0 {{- if hasKey . "labels" }} {{ toYaml .labels | indent 8 }} {{- end }} diff --git a/helm/hpcc/templates/esp.yaml b/helm/hpcc/templates/esp.yaml index 102b7bd6739..52c7d16ae88 100644 --- a/helm/hpcc/templates/esp.yaml +++ b/helm/hpcc/templates/esp.yaml @@ -116,7 +116,7 @@ spec: server: {{ .name | quote }} accessDali: "yes" app: {{ $application }} - helmVersion: 8.10.65-closedown0 + helmVersion: 8.10.67-closedown0 {{- include "hpcc.addStandardLabels" (dict "root" $ "name" $application "component" "esp" "instance" .name) | indent 8 }} {{- if hasKey $.Values.global "metrics" }} {{- include "hpcc.generateMetricsReporterLabel" $.Values.global.metrics | nindent 8 }} diff --git a/helm/hpcc/templates/localroxie.yaml b/helm/hpcc/templates/localroxie.yaml index 055451c74b4..4f9723720a1 100644 --- a/helm/hpcc/templates/localroxie.yaml +++ b/helm/hpcc/templates/localroxie.yaml @@ -70,7 +70,7 @@ spec: server: {{ $servername | quote }} accessDali: "yes" accessEsp: "yes" - helmVersion: 8.10.65-closedown0 + helmVersion: 8.10.67-closedown0 {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "roxie-server" "name" "roxie" "instance" $roxie.name) | indent 8 }} {{- if hasKey . "labels" }} {{ toYaml .labels | indent 8 }} diff --git a/helm/hpcc/templates/roxie.yaml b/helm/hpcc/templates/roxie.yaml index f005bcbcf38..a274643e164 100644 --- a/helm/hpcc/templates/roxie.yaml +++ b/helm/hpcc/templates/roxie.yaml @@ -118,7 +118,7 @@ spec: {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "topology-server" "name" "roxie" "instance" $commonCtx.toponame) | indent 8 }} run: {{ $commonCtx.toponame | quote }} roxie-cluster: {{ $roxie.name | quote }} - helmVersion: 8.10.65-closedown0 + helmVersion: 8.10.67-closedown0 {{- if hasKey $.Values.global "metrics" }} {{- include "hpcc.generateMetricsReporterLabel" $.Values.global.metrics | nindent 8}} {{- end }} @@ -178,7 +178,7 @@ kind: Service metadata: name: {{ $commonCtx.toponame | quote }} labels: - helmVersion: 8.10.65-closedown0 + helmVersion: 8.10.67-closedown0 {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "topology-server" "name" "roxie" "instance" $commonCtx.toponame) | indent 4 }} spec: ports: @@ -240,7 +240,7 @@ spec: roxie-cluster: {{ $roxie.name | quote }} accessDali: "yes" accessEsp: "yes" - helmVersion: 8.10.65-closedown0 + helmVersion: 8.10.67-closedown0 {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "roxie-server" "name" "roxie" "instance" $servername) | indent 8 }} {{- if hasKey $.Values.global "metrics" }} {{- include "hpcc.generateMetricsReporterLabel" $.Values.global.metrics | nindent 8}} @@ -343,7 +343,7 @@ spec: roxie-cluster: {{ $roxie.name | quote }} accessDali: "yes" accessEsp: "yes" - helmVersion: 8.10.65-closedown0 + helmVersion: 8.10.67-closedown0 {{- if hasKey $.Values.global "metrics" }} {{- include "hpcc.generateMetricsReporterLabel" $.Values.global.metrics | nindent 8}} {{- end }} diff --git a/helm/hpcc/templates/sasha.yaml b/helm/hpcc/templates/sasha.yaml index eec4ca49f7e..342ec5a5e08 100644 --- a/helm/hpcc/templates/sasha.yaml +++ b/helm/hpcc/templates/sasha.yaml @@ -52,7 +52,7 @@ spec: run: {{ $serviceName | quote }} server: {{ $serviceName | quote }} accessDali: {{ (has "dali" $sasha.access) | ternary "yes" "no" | quote }} - helmVersion: 8.10.65-closedown0 + helmVersion: 8.10.67-closedown0 {{- if hasKey $sasha "labels" }} {{ toYaml $sasha.labels | indent 8 }} {{- end }} diff --git a/helm/hpcc/templates/thor.yaml b/helm/hpcc/templates/thor.yaml index 2994349984a..c886547b315 100644 --- a/helm/hpcc/templates/thor.yaml +++ b/helm/hpcc/templates/thor.yaml @@ -82,7 +82,7 @@ data: labels: accessDali: "yes" accessEsp: "yes" - helmVersion: 8.10.65-closedown0 + helmVersion: 8.10.67-closedown0 {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "eclagent" "name" "thor" "instance" $eclAgentJobName "instanceOf" (printf "%s-job" .eclAgentName)) | indent 8 }} {{- if hasKey .me "labels" }} {{ toYaml .me.labels | indent 12 }} @@ -149,7 +149,7 @@ data: accessEsp: "yes" app: "thor" component: "thormanager" - helmVersion: 8.10.65-closedown0 + helmVersion: 8.10.67-closedown0 instance: "_HPCC_JOBNAME_" job: "_HPCC_JOBNAME_" {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "thormanager" "name" "thor" "instance" $thorManagerJobName "instanceOf" (printf "%s-thormanager-job" .me.name)) | indent 12 }} @@ -218,7 +218,7 @@ data: accessEsp: "yes" app: "thor" component: "thorworker" - helmVersion: 8.10.65-closedown0 + helmVersion: 8.10.67-closedown0 instance: "_HPCC_JOBNAME_" job: "_HPCC_JOBNAME_" {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "thorworker" "name" "thor" "instance" $thorWorkerJobName "instanceOf" (printf "%s-thorworker-job" .me.name)) | indent 12 }} @@ -353,7 +353,7 @@ spec: accessEsp: {{ $commonCtx.eclAgentUseChildProcesses | ternary "yes" "no" | quote }} app: "thor" component: "thor-eclagent" - helmVersion: 8.10.65-closedown0 + helmVersion: 8.10.67-closedown0 instance: {{ $commonCtx.eclAgentName | quote }} {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "eclagent" "name" "thor" "instance" $commonCtx.eclAgentName ) | indent 8 }} {{- if hasKey $commonCtx.me "labels" }} @@ -418,7 +418,7 @@ spec: accessEsp: "no" app: "thor" component: "thor-thoragent" - helmVersion: 8.10.65-closedown0 + helmVersion: 8.10.67-closedown0 instance: {{ $commonCtx.thorAgentName | quote }} {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "eclagent" "name" "thor" "instance" $commonCtx.thorAgentName ) | indent 8 }} {{- if hasKey $commonCtx.me "labels" }} diff --git a/version.cmake b/version.cmake index 82b6e10ff62..f8e2039a0e5 100644 --- a/version.cmake +++ b/version.cmake @@ -5,7 +5,7 @@ set ( HPCC_NAME "Community Edition" ) set ( HPCC_PROJECT "community" ) set ( HPCC_MAJOR 8 ) set ( HPCC_MINOR 10 ) -set ( HPCC_POINT 65 ) +set ( HPCC_POINT 67 ) set ( HPCC_MATURITY "closedown" ) set ( HPCC_SEQUENCE 0 ) ### From 607df4efa138209f1d2785b298f1b66be2e96c55 Mon Sep 17 00:00:00 2001 From: Jake Smith Date: Fri, 17 Nov 2023 11:27:20 +0000 Subject: [PATCH 05/16] Split off 8.12.70 Signed-off-by: Jake Smith --- helm/hpcc/Chart.yaml | 4 ++-- helm/hpcc/templates/_helpers.tpl | 2 +- helm/hpcc/templates/dafilesrv.yaml | 2 +- helm/hpcc/templates/dali.yaml | 2 +- helm/hpcc/templates/dfuserver.yaml | 2 +- helm/hpcc/templates/eclagent.yaml | 4 ++-- helm/hpcc/templates/eclccserver.yaml | 4 ++-- helm/hpcc/templates/eclscheduler.yaml | 2 +- helm/hpcc/templates/esp.yaml | 2 +- helm/hpcc/templates/localroxie.yaml | 2 +- helm/hpcc/templates/roxie.yaml | 8 ++++---- helm/hpcc/templates/sasha.yaml | 2 +- helm/hpcc/templates/thor.yaml | 10 +++++----- version.cmake | 4 ++-- 14 files changed, 25 insertions(+), 25 deletions(-) diff --git a/helm/hpcc/Chart.yaml b/helm/hpcc/Chart.yaml index c06cbcfbdcf..eb812c72a62 100644 --- a/helm/hpcc/Chart.yaml +++ b/helm/hpcc/Chart.yaml @@ -6,9 +6,9 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. -version: 8.12.69-closedown0 +version: 8.12.71-closedown0 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. -appVersion: 8.12.69-closedown0 +appVersion: 8.12.71-closedown0 diff --git a/helm/hpcc/templates/_helpers.tpl b/helm/hpcc/templates/_helpers.tpl index 08e3ae178de..e82a832f9b4 100644 --- a/helm/hpcc/templates/_helpers.tpl +++ b/helm/hpcc/templates/_helpers.tpl @@ -1240,7 +1240,7 @@ kind: Service metadata: name: {{ $lvars.serviceName | quote }} labels: - helmVersion: 8.12.69-closedown0 + helmVersion: 8.12.71-closedown0 {{- include "hpcc.addStandardLabels" (dict "root" $.root "instance" $lvars.serviceName ) | indent 4 }} {{- if $lvars.labels }} {{ toYaml $lvars.labels | indent 4 }} diff --git a/helm/hpcc/templates/dafilesrv.yaml b/helm/hpcc/templates/dafilesrv.yaml index 9a29a5c2e8f..3d3d7073547 100644 --- a/helm/hpcc/templates/dafilesrv.yaml +++ b/helm/hpcc/templates/dafilesrv.yaml @@ -50,7 +50,7 @@ spec: labels: {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "dafilesrv" "name" "dafilesrv" "instance" .name) | indent 8 }} server: {{ .name | quote }} - helmVersion: 8.12.69-closedown0 + helmVersion: 8.12.71-closedown0 annotations: checksum/config: {{ $configSHA }} spec: diff --git a/helm/hpcc/templates/dali.yaml b/helm/hpcc/templates/dali.yaml index 85de10b6726..6655615858d 100644 --- a/helm/hpcc/templates/dali.yaml +++ b/helm/hpcc/templates/dali.yaml @@ -82,7 +82,7 @@ spec: run: {{ $dali.name | quote }} server: {{ $dali.name | quote }} app: dali - helmVersion: 8.12.69-closedown0 + helmVersion: 8.12.71-closedown0 {{- if hasKey $.Values.global "metrics" }} {{- include "hpcc.generateMetricsReporterLabel" $.Values.global.metrics | nindent 8 }} {{- end }} diff --git a/helm/hpcc/templates/dfuserver.yaml b/helm/hpcc/templates/dfuserver.yaml index eaa6e638b6a..dfa31da5fc5 100644 --- a/helm/hpcc/templates/dfuserver.yaml +++ b/helm/hpcc/templates/dfuserver.yaml @@ -56,7 +56,7 @@ spec: {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "dfuserver" "name" "dfuserver" "instance" .name) | indent 8 }} run: {{ .name | quote }} accessDali: "yes" - helmVersion: 8.12.69-closedown0 + helmVersion: 8.12.71-closedown0 {{- if hasKey . "labels" }} {{ toYaml .labels | indent 8 }} {{- end }} diff --git a/helm/hpcc/templates/eclagent.yaml b/helm/hpcc/templates/eclagent.yaml index 38654d09e98..63a9f338a46 100644 --- a/helm/hpcc/templates/eclagent.yaml +++ b/helm/hpcc/templates/eclagent.yaml @@ -58,7 +58,7 @@ data: {{- include "hpcc.addStandardLabels" (dict "root" $ "component" $apptype "name" "eclagent" "instance" $appJobName "instanceOf" (printf "%s-job" .me.name)) | indent 12 }} accessDali: "yes" accessEsp: "yes" - helmVersion: 8.12.69-closedown0 + helmVersion: 8.12.71-closedown0 {{- if hasKey .me "labels" }} {{ toYaml .me.labels | indent 12 }} {{- end }} @@ -137,7 +137,7 @@ spec: run: {{ .name | quote }} accessDali: "yes" accessEsp: {{ .useChildProcesses | default false | ternary "yes" "no" | quote }} - helmVersion: 8.12.69-closedown0 + helmVersion: 8.12.71-closedown0 {{- if hasKey . "labels" }} {{ toYaml .labels | indent 8 }} {{- end }} diff --git a/helm/hpcc/templates/eclccserver.yaml b/helm/hpcc/templates/eclccserver.yaml index a53f00ff780..dc37b864583 100644 --- a/helm/hpcc/templates/eclccserver.yaml +++ b/helm/hpcc/templates/eclccserver.yaml @@ -57,7 +57,7 @@ data: {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "eclccserver" "name" "eclccserver" "instance" $compileJobName "instanceOf" (printf "%s-job" .me.name)) | indent 12 }} accessDali: "yes" accessEsp: "yes" - helmVersion: 8.12.69-closedown0 + helmVersion: 8.12.71-closedown0 {{- if hasKey .me "labels" }} {{ toYaml .me.labels | indent 12 }} {{- end }} @@ -142,7 +142,7 @@ spec: run: {{ .name | quote }} accessDali: "yes" accessEsp: {{ .useChildProcesses | default false | ternary "yes" "no" | quote }} - helmVersion: 8.12.69-closedown0 + helmVersion: 8.12.71-closedown0 {{- if hasKey . "labels" }} {{ toYaml .labels | indent 8 }} {{- end }} diff --git a/helm/hpcc/templates/eclscheduler.yaml b/helm/hpcc/templates/eclscheduler.yaml index 318f8906312..63d27d164f1 100644 --- a/helm/hpcc/templates/eclscheduler.yaml +++ b/helm/hpcc/templates/eclscheduler.yaml @@ -64,7 +64,7 @@ spec: run: {{ .name | quote }} accessDali: "yes" accessEsp: "no" - helmVersion: 8.12.69-closedown0 + helmVersion: 8.12.71-closedown0 {{- if hasKey . "labels" }} {{ toYaml .labels | indent 8 }} {{- end }} diff --git a/helm/hpcc/templates/esp.yaml b/helm/hpcc/templates/esp.yaml index 0d3dd8db618..9094bf0a2a4 100644 --- a/helm/hpcc/templates/esp.yaml +++ b/helm/hpcc/templates/esp.yaml @@ -117,7 +117,7 @@ spec: server: {{ .name | quote }} accessDali: "yes" app: {{ $application }} - helmVersion: 8.12.69-closedown0 + helmVersion: 8.12.71-closedown0 {{- include "hpcc.addStandardLabels" (dict "root" $ "name" $application "component" "esp" "instance" .name) | indent 8 }} {{- if hasKey $.Values.global "metrics" }} {{- include "hpcc.generateMetricsReporterLabel" $.Values.global.metrics | nindent 8 }} diff --git a/helm/hpcc/templates/localroxie.yaml b/helm/hpcc/templates/localroxie.yaml index 75f175adc94..a672ee1cbaa 100644 --- a/helm/hpcc/templates/localroxie.yaml +++ b/helm/hpcc/templates/localroxie.yaml @@ -70,7 +70,7 @@ spec: server: {{ $servername | quote }} accessDali: "yes" accessEsp: "yes" - helmVersion: 8.12.69-closedown0 + helmVersion: 8.12.71-closedown0 {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "roxie-server" "name" "roxie" "instance" $roxie.name) | indent 8 }} {{- if hasKey . "labels" }} {{ toYaml .labels | indent 8 }} diff --git a/helm/hpcc/templates/roxie.yaml b/helm/hpcc/templates/roxie.yaml index 0e7830ba79e..b2e8a83cc94 100644 --- a/helm/hpcc/templates/roxie.yaml +++ b/helm/hpcc/templates/roxie.yaml @@ -120,7 +120,7 @@ spec: {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "topology-server" "name" "roxie" "instance" $commonCtx.toponame) | indent 8 }} run: {{ $commonCtx.toponame | quote }} roxie-cluster: {{ $roxie.name | quote }} - helmVersion: 8.12.69-closedown0 + helmVersion: 8.12.71-closedown0 {{- if hasKey $.Values.global "metrics" }} {{- include "hpcc.generateMetricsReporterLabel" $.Values.global.metrics | nindent 8}} {{- end }} @@ -180,7 +180,7 @@ kind: Service metadata: name: {{ $commonCtx.toponame | quote }} labels: - helmVersion: 8.12.69-closedown0 + helmVersion: 8.12.71-closedown0 {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "topology-server" "name" "roxie" "instance" $commonCtx.toponame) | indent 4 }} spec: ports: @@ -242,7 +242,7 @@ spec: roxie-cluster: {{ $roxie.name | quote }} accessDali: "yes" accessEsp: "yes" - helmVersion: 8.12.69-closedown0 + helmVersion: 8.12.71-closedown0 {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "roxie-server" "name" "roxie" "instance" $servername) | indent 8 }} {{- if hasKey $.Values.global "metrics" }} {{- include "hpcc.generateMetricsReporterLabel" $.Values.global.metrics | nindent 8}} @@ -347,7 +347,7 @@ spec: roxie-cluster: {{ $roxie.name | quote }} accessDali: "yes" accessEsp: "yes" - helmVersion: 8.12.69-closedown0 + helmVersion: 8.12.71-closedown0 {{- if hasKey $.Values.global "metrics" }} {{- include "hpcc.generateMetricsReporterLabel" $.Values.global.metrics | nindent 8}} {{- end }} diff --git a/helm/hpcc/templates/sasha.yaml b/helm/hpcc/templates/sasha.yaml index a49b06454c4..3a911713542 100644 --- a/helm/hpcc/templates/sasha.yaml +++ b/helm/hpcc/templates/sasha.yaml @@ -52,7 +52,7 @@ spec: run: {{ $serviceName | quote }} server: {{ $serviceName | quote }} accessDali: {{ (has "dali" $sasha.access) | ternary "yes" "no" | quote }} - helmVersion: 8.12.69-closedown0 + helmVersion: 8.12.71-closedown0 {{- if hasKey $sasha "labels" }} {{ toYaml $sasha.labels | indent 8 }} {{- end }} diff --git a/helm/hpcc/templates/thor.yaml b/helm/hpcc/templates/thor.yaml index 80a849d897a..14d04fb4fbd 100644 --- a/helm/hpcc/templates/thor.yaml +++ b/helm/hpcc/templates/thor.yaml @@ -82,7 +82,7 @@ data: labels: accessDali: "yes" accessEsp: "yes" - helmVersion: 8.12.69-closedown0 + helmVersion: 8.12.71-closedown0 {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "eclagent" "name" "thor" "instance" $eclAgentJobName "instanceOf" (printf "%s-job" .eclAgentName)) | indent 8 }} {{- if hasKey .me "labels" }} {{ toYaml .me.labels | indent 12 }} @@ -149,7 +149,7 @@ data: accessEsp: "yes" app: "thor" component: "thormanager" - helmVersion: 8.12.69-closedown0 + helmVersion: 8.12.71-closedown0 instance: "_HPCC_JOBNAME_" job: "_HPCC_JOBNAME_" {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "thormanager" "name" "thor" "instance" $thorManagerJobName "instanceOf" (printf "%s-thormanager-job" .me.name)) | indent 12 }} @@ -218,7 +218,7 @@ data: accessEsp: "yes" app: "thor" component: "thorworker" - helmVersion: 8.12.69-closedown0 + helmVersion: 8.12.71-closedown0 instance: "_HPCC_JOBNAME_" job: "_HPCC_JOBNAME_" {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "thorworker" "name" "thor" "instance" $thorWorkerJobName "instanceOf" (printf "%s-thorworker-job" .me.name)) | indent 12 }} @@ -353,7 +353,7 @@ spec: accessEsp: {{ $commonCtx.eclAgentUseChildProcesses | ternary "yes" "no" | quote }} app: "thor" component: "thor-eclagent" - helmVersion: 8.12.69-closedown0 + helmVersion: 8.12.71-closedown0 instance: {{ $commonCtx.eclAgentName | quote }} {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "eclagent" "name" "thor" "instance" $commonCtx.eclAgentName ) | indent 8 }} {{- if hasKey $commonCtx.me "labels" }} @@ -418,7 +418,7 @@ spec: accessEsp: "no" app: "thor" component: "thor-thoragent" - helmVersion: 8.12.69-closedown0 + helmVersion: 8.12.71-closedown0 instance: {{ $commonCtx.thorAgentName | quote }} {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "eclagent" "name" "thor" "instance" $commonCtx.thorAgentName ) | indent 8 }} {{- if hasKey $commonCtx.me "labels" }} diff --git a/version.cmake b/version.cmake index 6cce0f14488..fb2b5d16ea4 100644 --- a/version.cmake +++ b/version.cmake @@ -5,8 +5,8 @@ set ( HPCC_NAME "Community Edition" ) set ( HPCC_PROJECT "community" ) set ( HPCC_MAJOR 8 ) set ( HPCC_MINOR 12 ) -set ( HPCC_POINT 69 ) +set ( HPCC_POINT 71 ) set ( HPCC_MATURITY "closedown" ) set ( HPCC_SEQUENCE 0 ) -set ( HPCC_TAG_TIMESTAMP "2023-11-09T17:02:40Z" ) +set ( HPCC_TAG_TIMESTAMP "2023-11-17T11:27:20Z" ) ### From bb718e0291114a463a5cc76e776f254844562192 Mon Sep 17 00:00:00 2001 From: Gordon Smith Date: Fri, 17 Nov 2023 11:04:15 +0100 Subject: [PATCH 06/16] HPCC-30872 Test UI failing Signed-off-by: Gordon Smith --- .github/workflows/test-smoke-gh_runner.yml | 10 +++++++ .github/workflows/test-ui-gh_runner.yml | 34 ++++++++++++---------- 2 files changed, 28 insertions(+), 16 deletions(-) diff --git a/.github/workflows/test-smoke-gh_runner.yml b/.github/workflows/test-smoke-gh_runner.yml index b01fdd3ebbc..f0fdfbaeab5 100644 --- a/.github/workflows/test-smoke-gh_runner.yml +++ b/.github/workflows/test-smoke-gh_runner.yml @@ -139,3 +139,13 @@ jobs: /var/log/HPCCSystems /home/runner/HPCCSystems-regression if-no-files-found: error + + succeeded: + runs-on: ${{ inputs.os }} + needs: main + steps: + - shell: "bash" + run: | + echo "...all tests passed..." + + diff --git a/.github/workflows/test-ui-gh_runner.yml b/.github/workflows/test-ui-gh_runner.yml index 0126f2c6e39..16530443396 100644 --- a/.github/workflows/test-ui-gh_runner.yml +++ b/.github/workflows/test-ui-gh_runner.yml @@ -70,20 +70,6 @@ jobs: gdb \ ${{ inputs.dependencies }} - - name: Install UI Dependencies - if: steps.check.outputs.runtests - run: | - wget https://dl.google.com/linux/direct/google-chrome-stable_current_amd64.deb - sudo apt-get install -y ./google-chrome-stable_current_amd64.deb - wget https://chromedriver.storage.googleapis.com/2.41/chromedriver_linux64.zip - unzip chromedriver_linux64.zip - sudo mv chromedriver /usr/bin/chromedriver - sudo chown root:root /usr/bin/chromedriver - sudo chmod +x /usr/bin/chromedriver - wget https://selenium-release.storage.googleapis.com/3.141/selenium-server-standalone-3.141.59.jar - wget http://www.java2s.com/Code/JarDownload/testng/testng-6.8.7.jar.zip - unzip testng-6.8.7.jar.zip - - name: Download Package if: steps.check.outputs.runtests uses: actions/download-artifact@v3 @@ -109,13 +95,29 @@ jobs: sudo update-locale sudo /etc/init.d/hpcc-init start + - name: Install UI Dependencies + if: steps.check.outputs.runtests + shell: "bash" + run: | + wget https://dl.google.com/linux/direct/google-chrome-stable_current_amd64.deb + sudo apt-get install -y ./google-chrome-stable_current_amd64.deb + wget https://chromedriver.storage.googleapis.com/2.41/chromedriver_linux64.zip + unzip chromedriver_linux64.zip + sudo mv chromedriver /usr/bin/chromedriver + sudo chown root:root /usr/bin/chromedriver + sudo chmod +x /usr/bin/chromedriver + wget https://selenium-release.storage.googleapis.com/3.141/selenium-server-standalone-3.141.59.jar + wget http://www.java2s.com/Code/JarDownload/testng/testng-6.8.7.jar.zip + unzip testng-6.8.7.jar.zip + - name: Run Tests timeout-minutes: 10 # generous, current runtime is ~1min, this should be increased if new tests are added if: steps.check.outputs.runtests + shell: "bash" run: | - export CLASSPATH=".:$(realpath selenium-server-standalone-3.141.59.jar):$(realpath testng-6.8.7.jar)" + export CLASSPATH=".:${{ github.workspace }}/selenium-server-standalone-3.141.59.jar:${{ github.workspace }}/testng-6.8.7.jar" pushd ${{ inputs.asset-name }}-ui_test-files - sudo ./run.sh tests http://localhost:8010 > eclWatchUiTest.log 2>&1 + ./run.sh tests http://localhost:8010 > eclWatchUiTest.log 2>&1 retCode=$? echo "UI test done" [[ $retCode -ne 0 ]] && exit 1 From 3770a07191ad7a163b5f90cdfe4b8cc452897ed8 Mon Sep 17 00:00:00 2001 From: Jeremy Clements <79224539+jeclrsg@users.noreply.github.com> Date: Fri, 17 Nov 2023 11:24:44 -0500 Subject: [PATCH 07/16] HPCC-30402 ECL Watch v9 fix permissions tabs not viewable Signed-off-by: Jeremy Clements <79224539+jeclrsg@users.noreply.github.com> --- esp/src/eclwatch/PermissionsWidget.js | 7 ++++++- esp/src/eclwatch/ShowAccountPermissionsWidget.js | 7 ++++++- esp/src/src-react/layouts/DojoAdapter.tsx | 8 ++++++-- 3 files changed, 18 insertions(+), 4 deletions(-) diff --git a/esp/src/eclwatch/PermissionsWidget.js b/esp/src/eclwatch/PermissionsWidget.js index 5a0bf49fe7f..77d7a166570 100644 --- a/esp/src/eclwatch/PermissionsWidget.js +++ b/esp/src/eclwatch/PermissionsWidget.js @@ -11,7 +11,12 @@ define([ "hpcc/GridDetailsWidget", "src/ws_access", "src/ESPUtil", - "src/Utility" + "src/Utility", + + "dijit/Dialog", + "dijit/form/TextBox", + + "hpcc/TableContainer" ], function (declare, nlsHPCCMod, registry, CheckBox, diff --git a/esp/src/eclwatch/ShowAccountPermissionsWidget.js b/esp/src/eclwatch/ShowAccountPermissionsWidget.js index cd87980db35..05fe65e894c 100644 --- a/esp/src/eclwatch/ShowAccountPermissionsWidget.js +++ b/esp/src/eclwatch/ShowAccountPermissionsWidget.js @@ -13,7 +13,12 @@ define([ "src/ws_access", "src/ESPUtil", "src/Utility", - "hpcc/ShowInheritedPermissionsWidget" + "hpcc/ShowInheritedPermissionsWidget", + + "dijit/Dialog", + "dijit/form/TextBox", + + "hpcc/TableContainer" ], function (declare, lang, nlsHPCCMod, arrayUtil, registry, CheckBox, diff --git a/esp/src/src-react/layouts/DojoAdapter.tsx b/esp/src/src-react/layouts/DojoAdapter.tsx index f48e3a5455c..1861875a3a6 100644 --- a/esp/src/src-react/layouts/DojoAdapter.tsx +++ b/esp/src/src-react/layouts/DojoAdapter.tsx @@ -60,8 +60,12 @@ export const DojoAdapter: React.FunctionComponent = ({ }, ...delayProps }, elem); - widget.startup(); - widget.resize(); + if (widget.startup) { + widget.startup(); + } + if (widget.resize) { + widget.resize(); + } if (widget.init) { widget.init(params || {}); } From 0a97ab6b55747005f1af9e26b1082865f2a9f64b Mon Sep 17 00:00:00 2001 From: Jake Smith Date: Fri, 17 Nov 2023 13:57:16 +0000 Subject: [PATCH 08/16] HPCC-30883 Fix k8s foreign lookup meta change issues A foreign lookup into a k8s system (when enabled), was causing spurious file and group meta data to be published. Fix by ensuring the live Dali IPT is not altered, and avoid "anon" groups being created during IFileDescriptor setup. Signed-off-by: Jake Smith --- dali/base/dadfs.cpp | 26 +++++++++++++++++--------- 1 file changed, 17 insertions(+), 9 deletions(-) diff --git a/dali/base/dadfs.cpp b/dali/base/dadfs.cpp index c6c1fdb2442..00fea1d1ddb 100644 --- a/dali/base/dadfs.cpp +++ b/dali/base/dadfs.cpp @@ -11168,6 +11168,7 @@ class CDaliDFSServer: public Thread, public CTransactionLogTracker, implements I CScopeConnectLock sconnlock("getFileTree", *logicalname, false, false, false, defaultTimeout); IPropertyTree* sroot = sconnlock.conn()?sconnlock.conn()->queryRoot():NULL; logicalname->getTail(tail); + INamedGroupStore *groupResolver = &queryNamedGroupStore(); if (version >= 2) { Owned tree = getNamedPropTree(sroot,queryDfsXmlBranchName(DXB_File),"@name",tail.str(),false); @@ -11179,7 +11180,9 @@ class CDaliDFSServer: public Thread, public CTransactionLogTracker, implements I // asking for the returned meta data to be remapped to point to the dafilesrv service. if (hasMask(opts, GetFileTreeOpts::remapToService)) { + tree.setown(createPTreeFromIPT(tree)); // copy live Dali tree, because it is about to be altered by remapGroupsToDafilesrv remapGroupsToDafilesrv(tree, &queryNamedGroupStore()); + groupResolver = nullptr; // do not attempt to resolve remapped group (it will not exist and cause addUnique to create a new anon one) const char *remotePlaneName = tree->queryProp("@group"); Owned filePlane = getStoragePlane(remotePlaneName); @@ -11189,7 +11192,7 @@ class CDaliDFSServer: public Thread, public CTransactionLogTracker, implements I } } - Owned fdesc = deserializeFileDescriptorTree(tree,&queryNamedGroupStore(),IFDSF_EXCLUDE_CLUSTERNAMES); + Owned fdesc = deserializeFileDescriptorTree(tree,groupResolver,IFDSF_EXCLUDE_CLUSTERNAMES); mb.append((int)1); // 1 == standard file fdesc->serialize(mb); @@ -11220,15 +11223,20 @@ class CDaliDFSServer: public Thread, public CTransactionLogTracker, implements I { if (version == MDFS_GET_FILE_TREE_V2) { -#ifdef _CONTAINERIZED - // NB: to be here, the client is by definition legacy, and should be foreign. - // foreignAccess must also have been configured in this environment - - if (getComponentConfigSP()->getPropBool("@foreignAccess")) - remapGroupsToDafilesrv(tree, &queryNamedGroupStore()); -#endif + if (isContainerized()) + { + // NB: to be here, the client is by definition legacy, and should only be via ~foreign. + // NB: foreignAccess is a auto-generated template setting, that is set to true if Dali and directio, + // have been exposed in the helm chart for foreign access. + if (getComponentConfigSP()->getPropBool("@foreignAccess")) + { + tree.setown(createPTreeFromIPT(tree)); // copy live Dali tree, because it is about to be altered by remapGroupsToDafilesrv + remapGroupsToDafilesrv(tree, &queryNamedGroupStore()); + groupResolver = nullptr; // do not attempt to resolve remapped group (it will not exist and cause addUnique to create a new anon one) + } + } - Owned fdesc = deserializeFileDescriptorTree(tree,&queryNamedGroupStore(),IFDSF_EXCLUDE_CLUSTERNAMES); + Owned fdesc = deserializeFileDescriptorTree(tree,groupResolver,IFDSF_EXCLUDE_CLUSTERNAMES); mb.append((int)-2).append(version); From f3fe0ea6913297b17443fac0713d277a3b36f06c Mon Sep 17 00:00:00 2001 From: Gordon Smith Date: Mon, 20 Nov 2023 15:05:43 +0000 Subject: [PATCH 09/16] HPCC-30896 Windows + opentelemetry build issue Signed-off-by: Gordon Smith --- system/jlib/CMakeLists.txt | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/system/jlib/CMakeLists.txt b/system/jlib/CMakeLists.txt index 0b5a9a0490f..711d8958b90 100644 --- a/system/jlib/CMakeLists.txt +++ b/system/jlib/CMakeLists.txt @@ -42,10 +42,10 @@ endif(NOT TARGET libbase58) find_package(yaml CONFIG REQUIRED) -#For OpenTel exporter find_package(Protobuf REQUIRED) -#For http exporter -#find_package(CURL REQUIRED) +find_package(CURL CONFIG REQUIRED) +find_package(gRPC CONFIG REQUIRED) +find_package(nlohmann_json CONFIG REQUIRED) find_package(opentelemetry-cpp CONFIG REQUIRED) SET (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${STRICT_CXX_FLAGS}") @@ -239,8 +239,6 @@ if ( ${HAVE_LIBCRYPT} ) target_link_libraries ( jlib crypt) endif ( ${HAVE_LIBCRYPT} ) -find_package(gRPC CONFIG REQUIRED) - target_link_libraries ( jlib opentelemetry-cpp::ostream_span_exporter From 0820f74b2c2bb73d7af9c6ae51dad93d8b1932a3 Mon Sep 17 00:00:00 2001 From: Gordon Smith Date: Tue, 11 Jul 2023 09:36:25 +0200 Subject: [PATCH 10/16] HPCC-29914 Add embedded wasm support Signed-off-by: Gordon Smith --- .github/workflows/build-assets.yml | 2 +- .github/workflows/build-docker.yml | 2 +- .github/workflows/build-gh_runner.yml | 2 +- CMakeLists.txt | 1 + cmake_modules/plugins.cmake | 1 + plugins/CMakeLists.txt | 1 + plugins/wasmembed/CMakeLists.txt | 66 ++ plugins/wasmembed/abi.cpp | 269 ++++++++ plugins/wasmembed/abi.hpp | 3 + plugins/wasmembed/hpcc-platform.wit | 25 + plugins/wasmembed/secure-enclave.cpp | 626 ++++++++++++++++++ plugins/wasmembed/secure-enclave.hpp | 4 + plugins/wasmembed/test/CMakeLists.txt | 33 + plugins/wasmembed/test/build.sh | 23 + plugins/wasmembed/test/hpcc-scalar-test.wit | 28 + plugins/wasmembed/test/main.cpp | 70 ++ plugins/wasmembed/test/wasm32-wasi/Dockerfile | 41 ++ plugins/wasmembed/util.cpp | 46 ++ plugins/wasmembed/util.hpp | 7 + plugins/wasmembed/wasm.ecllib | 10 + plugins/wasmembed/wasmembed.cpp | 65 ++ system/jlib/jarray.hpp | 2 +- system/jlib/jbuff.hpp | 8 +- system/jlib/jhash.hpp | 6 +- system/jlib/jiface.hpp | 4 +- system/jlib/jsuperhash.hpp | 26 +- testing/regress/ecl/key/wasmembed.xml | 102 +++ testing/regress/ecl/wasmembed.ecl | 112 ++++ testing/regress/ecl/wasmembed.manifest | 3 + testing/regress/ecl/wasmembed.wasm | Bin 0 -> 53388 bytes vcpkg.json.in | 4 + 31 files changed, 1566 insertions(+), 26 deletions(-) create mode 100644 plugins/wasmembed/CMakeLists.txt create mode 100644 plugins/wasmembed/abi.cpp create mode 100644 plugins/wasmembed/abi.hpp create mode 100644 plugins/wasmembed/hpcc-platform.wit create mode 100644 plugins/wasmembed/secure-enclave.cpp create mode 100644 plugins/wasmembed/secure-enclave.hpp create mode 100644 plugins/wasmembed/test/CMakeLists.txt create mode 100755 plugins/wasmembed/test/build.sh create mode 100644 plugins/wasmembed/test/hpcc-scalar-test.wit create mode 100644 plugins/wasmembed/test/main.cpp create mode 100644 plugins/wasmembed/test/wasm32-wasi/Dockerfile create mode 100644 plugins/wasmembed/util.cpp create mode 100644 plugins/wasmembed/util.hpp create mode 100644 plugins/wasmembed/wasm.ecllib create mode 100644 plugins/wasmembed/wasmembed.cpp create mode 100644 testing/regress/ecl/key/wasmembed.xml create mode 100644 testing/regress/ecl/wasmembed.ecl create mode 100644 testing/regress/ecl/wasmembed.manifest create mode 100644 testing/regress/ecl/wasmembed.wasm diff --git a/.github/workflows/build-assets.yml b/.github/workflows/build-assets.yml index 9aeb67f532c..7d3ec45658d 100644 --- a/.github/workflows/build-assets.yml +++ b/.github/workflows/build-assets.yml @@ -189,7 +189,7 @@ jobs: run: | mkdir -p ${{ needs.preamble.outputs.folder_build }} echo "${{ secrets.SIGNING_SECRET }}" > ${{ needs.preamble.outputs.folder_build }}/private.key - plugins=("CASSANDRAEMBED" "COUCHBASEEMBED" "ECLBLAS" "H3" "JAVAEMBED" "KAFKA" "MEMCACHED" "MONGODBEMBED" "MYSQLEMBED" "NLP" "PARQUETEMBED" "REDIS" "REMBED" "SQLITE3EMBED" "SQS" "PLATFORM" "CLIENTTOOLS_ONLY") + plugins=("CASSANDRAEMBED" "COUCHBASEEMBED" "ECLBLAS" "H3" "JAVAEMBED" "KAFKA" "MEMCACHED" "MONGODBEMBED" "MYSQLEMBED" "NLP" "PARQUETEMBED" "REDIS" "REMBED" "SQLITE3EMBED" "SQS" "WASMEMBED" "PLATFORM" "CLIENTTOOLS_ONLY") for plugin in "${plugins[@]}"; do sudo rm -f ${{ needs.preamble.outputs.folder_build }}/CMakeCache.txt sudo rm -rf ${{ needs.preamble.outputs.folder_build }}/CMakeFiles diff --git a/.github/workflows/build-docker.yml b/.github/workflows/build-docker.yml index e7bfab4a02f..24560aa6331 100644 --- a/.github/workflows/build-docker.yml +++ b/.github/workflows/build-docker.yml @@ -128,7 +128,7 @@ jobs: mkdir -p ${{ github.workspace }}/build mkdir -p ${{ github.workspace }}/.ccache declare -a plugins - plugins=(${{ inputs.single-package == true && '"PLATFORM"' || '"PLATFORM" "CASSANDRAEMBED" "COUCHBASEEMBED" "ECLBLAS" "H3" "JAVAEMBED" "KAFKA" "MEMCACHED" "MONGODBEMBED" "MYSQLEMBED" "NLP" "PARQUETEMBED" "REDIS" "REMBED" "SQLITE3EMBED" "SQS"' }}) + plugins=(${{ inputs.single-package == true && '"PLATFORM"' || '"PLATFORM" "CASSANDRAEMBED" "COUCHBASEEMBED" "ECLBLAS" "H3" "JAVAEMBED" "KAFKA" "MEMCACHED" "MONGODBEMBED" "MYSQLEMBED" "NLP" "PARQUETEMBED" "REDIS" "REMBED" "SQLITE3EMBED" "SQS" "WASMEMBED"' }}) for plugin in "${plugins[@]}"; do sudo rm -f ${{ github.workspace }}/build/CMakeCache.txt sudo rm -rf ${{ github.workspace }}/build/CMakeFiles diff --git a/.github/workflows/build-gh_runner.yml b/.github/workflows/build-gh_runner.yml index 48823c004c5..2b352e2cb99 100644 --- a/.github/workflows/build-gh_runner.yml +++ b/.github/workflows/build-gh_runner.yml @@ -205,7 +205,7 @@ jobs: mkdir -p ${{ github.workspace }}/LN mkdir -p ${{ github.workspace }}/build declare -a plugins - plugins=(${{ inputs.single-package == true && '"PLATFORM"' || '"PLATFORM" "CASSANDRAEMBED" "COUCHBASEEMBED" "ECLBLAS" "H3" "JAVAEMBED" "KAFKA" "MEMCACHED" "MONGODBEMBED" "MYSQLEMBED" "NLP" "PARQUETEMBED" "REDIS" "REMBED" "SQLITE3EMBED" "SQS"' }}) + plugins=(${{ inputs.single-package == true && '"PLATFORM"' || '"PLATFORM" "CASSANDRAEMBED" "COUCHBASEEMBED" "ECLBLAS" "H3" "JAVAEMBED" "KAFKA" "MEMCACHED" "MONGODBEMBED" "MYSQLEMBED" "NLP" "PARQUETEMBED" "REDIS" "REMBED" "SQLITE3EMBED" "SQS" "WASMEMBED"' }}) for plugin in "${plugins[@]}"; do rm -f ./build/CMakeCache.txt rm -rf ./build/CMakeFiles diff --git a/CMakeLists.txt b/CMakeLists.txt index b26f33ec7e0..621f77deaaa 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -169,6 +169,7 @@ if ( PLUGIN ) HPCC_ADD_SUBDIRECTORY (dali/base) HPCC_ADD_SUBDIRECTORY (plugins/Rembed "REMBED") HPCC_ADD_SUBDIRECTORY (plugins/v8embed "V8EMBED") + HPCC_ADD_SUBDIRECTORY (plugins/wasmembed "WASMEMBED") HPCC_ADD_SUBDIRECTORY (plugins/memcached "MEMCACHED") HPCC_ADD_SUBDIRECTORY (plugins/redis "REDIS") HPCC_ADD_SUBDIRECTORY (plugins/javaembed "JAVAEMBED") diff --git a/cmake_modules/plugins.cmake b/cmake_modules/plugins.cmake index b96a7e7cd09..f87df647507 100644 --- a/cmake_modules/plugins.cmake +++ b/cmake_modules/plugins.cmake @@ -42,6 +42,7 @@ set(PLUGINS_LIST SQLITE3EMBED SQS V8EMBED + WASMEMBED EXAMPLEPLUGIN ) diff --git a/plugins/CMakeLists.txt b/plugins/CMakeLists.txt index 109e74b7abb..63de3d1cc68 100644 --- a/plugins/CMakeLists.txt +++ b/plugins/CMakeLists.txt @@ -30,6 +30,7 @@ add_subdirectory (proxies) add_subdirectory (sqlite3) add_subdirectory (mysql) add_subdirectory (v8embed) +add_subdirectory (wasmembed) HPCC_ADD_SUBDIRECTORY (py3embed "USE_PYTHON3") HPCC_ADD_SUBDIRECTORY (pyembed "USE_PYTHON2") add_subdirectory (javaembed) diff --git a/plugins/wasmembed/CMakeLists.txt b/plugins/wasmembed/CMakeLists.txt new file mode 100644 index 00000000000..7dff7328253 --- /dev/null +++ b/plugins/wasmembed/CMakeLists.txt @@ -0,0 +1,66 @@ +project(wasmembed) + +if(WASMEMBED) + ADD_PLUGIN(wasmembed) + if(MAKE_WASMEMBED) + + set(CMAKE_CXX_STANDARD 20) + find_path(WASMTIME_CPP_API_INCLUDE_DIRS "wasmtime-cpp-api/wasmtime.hh" + PATHS ${VCPKG_INSTALLED_DIR}/${VCPKG_TARGET_TRIPLET} + ) + if (WIN32) + find_library(WASMTIME_LIB NAMES wasmtime.dll + PATHS ${VCPKG_INSTALLED_DIR}/${VCPKG_TARGET_TRIPLET} + ) + else() + find_library(WASMTIME_LIB NAMES wasmtime + PATHS ${VCPKG_INSTALLED_DIR}/${VCPKG_TARGET_TRIPLET} + ) + endif() + + include_directories( + ${WASMTIME_CPP_API_INCLUDE_DIRS}/wasmtime-c-api + ${WASMTIME_CPP_API_INCLUDE_DIRS}/wasmtime-cpp-api + ./../../system/include + ./../../system/jlib + ./../../rtl/eclrtl + ./../../rtl/include + + # Following includes are needed for "enginecontext.hpp" + ./../../common/thorhelper + ./../../dali/base + ./../../system/mp + ) + + add_definitions(-D_USRDLL -DWASMEMBED_EXPORTS) + + add_library(wasmembed SHARED + wasmembed.cpp + secure-enclave.cpp + abi.cpp + util.cpp + ) + + target_link_libraries(wasmembed + ${WASMTIME_LIB} + eclrtl + jlib + ) + + install( + TARGETS wasmembed + DESTINATION plugins + CALC_DEPS + ) + + else() + message(WARNING "Cannot build wasmembed plugin") + endif() +endif() + +if(PLATFORM OR CLIENTTOOLS_ONLY) + install( + FILES ${CMAKE_CURRENT_SOURCE_DIR}/wasm.ecllib + DESTINATION plugins + COMPONENT Runtime) +endif() diff --git a/plugins/wasmembed/abi.cpp b/plugins/wasmembed/abi.cpp new file mode 100644 index 00000000000..43b91339e4b --- /dev/null +++ b/plugins/wasmembed/abi.cpp @@ -0,0 +1,269 @@ +/* + See: https://github.com/WebAssembly/component-model/blob/main/design/mvp/CanonicalABI.md + https://github.com/WebAssembly/component-model/blob/main/design/mvp/canonical-abi/definitions.py +*/ + +#include "abi.hpp" + +#include "jexcept.hpp" + +auto UTF16_TAG = 1U << 31; + +// +/* canonical despecialize (python) ------------------------------------------------------------- + +def despecialize(t): + match t: + case Tuple(ts) : return Record([ Field(str(i), t) for i,t in enumerate(ts) ]) + case Union(ts) : return Variant([ Case(str(i), t) for i,t in enumerate(ts) ]) + case Enum(labels) : return Variant([ Case(l, None) for l in labels ]) + case Option(t) : return Variant([ Case("none", None), Case("some", t) ]) + case Result(ok, error) : return Variant([ Case("ok", ok), Case("error", error) ]) + case _ : return t + +*/ + +// template +// wasmtime::ValType despecialize(const T &t) +// { +// switch (t.kind()) +// { +// case wasmtime::ValKind::I32: +// case wasmtime::ValKind::I64: +// case wasmtime::ValKind::F32: +// case wasmtime::ValKind::F64: +// case wasmtime::ValKind::V128: +// return t.kind(); +// default: +// return wasmtime::ValType::i32(); +// } +// } + +/* canonical alignment (python) ------------------------------------------------------------- + +def alignment(t): + match despecialize(t): + case Bool() : return 1 + case S8() | U8() : return 1 + case S16() | U16() : return 2 + case S32() | U32() : return 4 + case S64() | U64() : return 8 + case Float32() : return 4 + case Float64() : return 8 + case Char() : return 4 + case String() | List(_) : return 4 + case Record(fields) : return alignment_record(fields) + case Variant(cases) : return alignment_variant(cases) + case Flags(labels) : return alignment_flags(labels) + case Own(_) | Borrow(_) : return 4 + +*/ + +// int alignment(const wasmtime::ValType &t) +// { +// switch (t.kind()) +// { +// case wasmtime::ValKind::I32: +// case wasmtime::ValKind::F32: +// return 4; +// case wasmtime::ValKind::I64: +// case wasmtime::ValKind::F64: +// return 8; +// case wasmtime::ValKind::V128: +// return 16; +// default: +// return 1; +// } +// } + +/* canonical align_to (python) ------------------------------------------------------------- + +def align_to(ptr, alignment): + return math.ceil(ptr / alignment) * alignment + +*/ + +uint32_t align_to(uint32_t ptr, uint32_t alignment) +{ + return (ptr + alignment - 1) & ~(alignment - 1); +} + +bool isAligned(uint32_t ptr, uint32_t alignment) +{ + return (ptr & (alignment - 1)) == 0; +} + +// loading --- + +/* canonical load_int (python) ------------------------------------------------------------- + +def load_int(cx, ptr, nbytes, signed = False): + return int.from_bytes(cx.opts.memory[ptr : ptr+nbytes], 'little', signed=signed) + +*/ + +template +T load_int(const wasmtime::Span &data, uint32_t ptr) +{ + T retVal = 0; + if constexpr (sizeof(T) == 1) + { + retVal = static_cast(data[ptr]); + } + else if constexpr (sizeof(T) == 2) + { + retVal = static_cast((static_cast(data[ptr + 1]) << 8) | + static_cast(data[ptr])); + } + else if constexpr (sizeof(T) == 4) + { + retVal = static_cast((static_cast(data[ptr + 3]) << 24) | + (static_cast(data[ptr + 2]) << 16) | + (static_cast(data[ptr + 1]) << 8) | + static_cast(data[ptr])); + } + else if constexpr (sizeof(T) == 8) + { + retVal = static_cast((static_cast(data[ptr + 7]) << 56) | + (static_cast(data[ptr + 6]) << 48) | + (static_cast(data[ptr + 5]) << 40) | + (static_cast(data[ptr + 4]) << 32) | + (static_cast(data[ptr + 3]) << 24) | + (static_cast(data[ptr + 2]) << 16) | + (static_cast(data[ptr + 1]) << 8) | + static_cast(data[ptr])); + } + return retVal; +} +/* canonical load_string_from_range (python) ------------------------------------------------------------- + +def load_string_from_range(cx, ptr, tagged_code_units): + match cx.opts.string_encoding: + case 'utf8': + alignment = 1 + byte_length = tagged_code_units + encoding = 'utf-8' + case 'utf16': + alignment = 2 + byte_length = 2 * tagged_code_units + encoding = 'utf-16-le' + case 'latin1+utf16': + alignment = 2 + if bool(tagged_code_units & UTF16_TAG): + byte_length = 2 * (tagged_code_units ^ UTF16_TAG) + encoding = 'utf-16-le' + else: + byte_length = tagged_code_units + encoding = 'latin-1' + + trap_if(ptr != align_to(ptr, alignment)) + trap_if(ptr + byte_length > len(cx.opts.memory)) + try: + s = cx.opts.memory[ptr : ptr+byte_length].decode(encoding) + except UnicodeError: + trap() + + return (s, cx.opts.string_encoding, tagged_code_units) + +*/ + +// More: Not currently available from the wasmtime::context object, see https://github.com/bytecodealliance/wasmtime/issues/6719 +static const std::string global_encoding = "utf8"; + +std::tuple load_string_from_range(const wasmtime::Span &data, uint32_t ptr, uint32_t tagged_code_units) +{ + std::string encoding = "utf-8"; + uint32_t byte_length = tagged_code_units; + uint32_t alignment = 1; + if (global_encoding.compare("utf8") == 0) + { + alignment = 1; + byte_length = tagged_code_units; + encoding = "utf-8"; + } + else if (global_encoding.compare("utf16") == 0) + { + alignment = 2; + byte_length = 2 * tagged_code_units; + encoding = "utf-16-le"; + } + else if (global_encoding.compare("latin1+utf16") == 0) + { + alignment = 2; + if (tagged_code_units & UTF16_TAG) + { + byte_length = 2 * (tagged_code_units ^ UTF16_TAG); + encoding = "utf-16-le"; + } + else + { + byte_length = tagged_code_units; + encoding = "latin-1"; + } + } + + if (!isAligned(ptr, alignment)) + { + throw makeStringException(3, "Invalid alignment"); + } + + if (ptr + byte_length > data.size()) + { + throw makeStringException(1, "Out of bounds"); + } + + return std::make_tuple(ptr, encoding, byte_length); +} + +/* canonical load_string (python) ------------------------------------------------------------- + +def load_string(cx, ptr): + begin = load_int(cx, ptr, 4) + tagged_code_units = load_int(cx, ptr + 4, 4) + return load_string_from_range(cx, begin, tagged_code_units) + +*/ +std::tuple load_string(const wasmtime::Span &data, uint32_t ptr) +{ + uint32_t begin = load_int(data, ptr); + uint32_t tagged_code_units = load_int(data, ptr + 4); + return load_string_from_range(data, begin, tagged_code_units); +} + +/* canonical load_list_from_range (python) ------------------------------------------------------------- + +def load_list_from_range(cx, ptr, length, elem_type): + trap_if(ptr != align_to(ptr, alignment(elem_type))) + trap_if(ptr + length * size(elem_type) > len(cx.opts.memory)) + a = [] + for i in range(length): + a.append(load(cx, ptr + i * size(elem_type), elem_type)) + return a + +*/ + +template +std::vector load_list_from_range(const wasmtime::Span &data, uint32_t ptr, uint32_t length) +{ + if (!isAligned(ptr, alignment(T{}))) + throw makeStringException(2, "Pointer is not aligned"); + if (ptr + length * sizeof(T) > data.size()) + throw makeStringException(1, "Out of bounds access"); + std::vector a; + for (uint32_t i = 0; i < length; i++) + { + a.push_back(load(data, ptr + i * sizeof(T))); + } + return a; +} + +/* canonical load_list (python) ------------------------------------------------------------- + +def load_list(cx, ptr, elem_type): + begin = load_int(cx, ptr, 4) + length = load_int(cx, ptr + 4, 4) + return load_list_from_range(cx, begin, length, elem_type) + +*/ + +// Storing --- diff --git a/plugins/wasmembed/abi.hpp b/plugins/wasmembed/abi.hpp new file mode 100644 index 00000000000..38f09fe1020 --- /dev/null +++ b/plugins/wasmembed/abi.hpp @@ -0,0 +1,3 @@ +#include + +std::tuple load_string(const wasmtime::Span &data, uint32_t ptr); diff --git a/plugins/wasmembed/hpcc-platform.wit b/plugins/wasmembed/hpcc-platform.wit new file mode 100644 index 00000000000..9ff9693c540 --- /dev/null +++ b/plugins/wasmembed/hpcc-platform.wit @@ -0,0 +1,25 @@ +/* WebAssembly Component Interface + + See: https://component-model.bytecodealliance.org/design/wit.html + + HPCC-Platform is the "host". + The WebAssembly module is the "guest". + + Memory management rules: + * Guests calling host functions: + - Guests must dispose all params as needed + - Guests must dispose "results" as needed + * Hosts calling guest functions: + - Guests must dispose all params as needed + - Hosts must dispose "results" as needed by + calling cabi_post_XXX for each function call + */ + +package hpcc-systems:hpcc-platform + +world wasmembed { + + import dbglog: func(msg: string) + + // export myfunc(params: string) -> string +} diff --git a/plugins/wasmembed/secure-enclave.cpp b/plugins/wasmembed/secure-enclave.cpp new file mode 100644 index 00000000000..07a76f7d656 --- /dev/null +++ b/plugins/wasmembed/secure-enclave.cpp @@ -0,0 +1,626 @@ +#include "secure-enclave.hpp" + +#include "eclrtl_imp.hpp" +#include "jexcept.hpp" +#include "jiface.hpp" +#include "eclhelper.hpp" +#include "enginecontext.hpp" + +#include "abi.hpp" +#include "util.hpp" + +#include +#include + +// From deftype.hpp in common +#define UNKNOWN_LENGTH 0xFFFFFFF1 + +// #define ENABLE_TRACE +#ifdef ENABLE_TRACE +#define TRACE(format, ...) DBGLOG(format __VA_OPT__(, ) __VA_ARGS__) +#else +#define TRACE(format, ...) \ + do \ + { \ + } while (0) +#endif +class WasmEngine +{ +private: + std::once_flag wasmLoadedFlag; + std::unordered_map wasmModules; + + wasmtime::Module createModule(const std::string &wasmName, const wasmtime::Span &wasm) + { + TRACE("WasmEngine createModule %s", wasmName.c_str()); + try + { + wasmtime::Store store(engine); + wasmtime::WasiConfig wasi; + wasi.inherit_argv(); + wasi.inherit_env(); + wasi.inherit_stdin(); + wasi.inherit_stdout(); + wasi.inherit_stderr(); + store.context().set_wasi(std::move(wasi)).unwrap(); + return wasmtime::Module::compile(engine, wasm).unwrap(); + } + catch (const wasmtime::Error &e) + { + throw makeStringExceptionV(100, "WasmEngine createModule failed: %s", e.message().c_str()); + } + } + + void loadWasmFiles(ICodeContext *codeCtx) + { + TRACE("WasmEngine loadWasmFiles"); + IEngineContext *engine = codeCtx->queryEngineContext(); + if (!engine) + throw makeStringException(100, "Failed to get engine context"); + + StringArray manifestModules; + engine->getManifestFiles("wasm", manifestModules); + + ForEachItemIn(idx, manifestModules) + { + const char *path = manifestModules.item(idx); + TRACE("WasmEngine loadWasmFiles %s", path); + std::vector contents = readWasmBinaryToBuffer(path); + auto module = createModule(path, contents); + std::filesystem::path p(path); + wasmModules.insert(std::make_pair(p.stem(), module)); + } + } + +public: + wasmtime::Engine engine; + + WasmEngine() + { + TRACE("WASM SE WasmEngine"); + } + + ~WasmEngine() + { + TRACE("WASM SE ~WasmEngine"); + } + + void setCodeContext(ICodeContext *codeCtx) + { + TRACE("WASM SE setCodeContext"); + std::call_once(wasmLoadedFlag, &WasmEngine::loadWasmFiles, this, codeCtx); + } + + bool hasModule(const std::string &wasmName) const + { + TRACE("WASM SE hasModule"); + return wasmModules.find(wasmName) != wasmModules.end(); + } + + wasmtime::Module getModule(const std::string &wasmName) const + { + TRACE("WASM SE getModule"); + auto found = wasmModules.find(wasmName); + if (found == wasmModules.end()) + throw makeStringExceptionV(100, "Wasm module not found: %s", wasmName.c_str()); + return found->second; + } +}; +static std::unique_ptr wasmEngine = std::make_unique(); + +class WasmStore +{ +private: + wasmtime::Store store; + + std::unordered_map wasmInstances; + std::unordered_map wasmMems; + std::unordered_map wasmFuncs; + +public: + WasmStore() : store(wasmEngine->engine) + { + TRACE("WASM SE WasmStore"); + } + + ~WasmStore() + { + TRACE("WASM SE ~WasmStore"); + } + + bool hasInstance(const std::string &wasmName) const + { + TRACE("WASM SE hasInstance"); + return wasmInstances.find(wasmName) != wasmInstances.end(); + } + + wasmtime::Instance getInstance(const std::string &wasmName) const + { + TRACE("WASM SE getInstance"); + auto found = wasmInstances.find(wasmName); + if (found == wasmInstances.end()) + throw makeStringExceptionV(100, "Wasm instance not found: %s", wasmName.c_str()); + return found->second; + } + + void registerInstance(const std::string &wasmName) + { + TRACE("WASM SE registerInstance %s", wasmName.c_str()); + if (hasInstance(wasmName)) + { + throw makeStringExceptionV(100, "Wasm instance already registered: %s", wasmName.c_str()); + } + TRACE("WASM SE createInstance %s", wasmName.c_str()); + auto module = wasmEngine->getModule(wasmName); + try + { + wasmtime::Linker linker(wasmEngine->engine); + linker.define_wasi().unwrap(); + + auto callback = [this, wasmName](wasmtime::Caller caller, uint32_t msg, uint32_t msg_len) + { + auto data = this->getData(wasmName); + auto msg_ptr = (char *)&data[msg]; + std::string str(msg_ptr, msg_len); + DBGLOG("from wasm: %s", str.c_str()); + }; + auto host_func = linker.func_wrap("$root", "dbglog", callback).unwrap(); + + auto newInstance = linker.instantiate(store, module).unwrap(); + linker.define_instance(store, "linking2", newInstance).unwrap(); + + for (auto exportItem : module.exports()) + { + auto externType = wasmtime::ExternType::from_export(exportItem); + std::string name(exportItem.name()); + if (std::holds_alternative(externType)) + { + TRACE("WASM SE Exported function: %s", name.c_str()); + auto func = std::get(*newInstance.get(store, name)); + wasmFuncs.insert(std::make_pair(wasmName + "." + name, func)); + } + else if (std::holds_alternative(externType)) + { + TRACE("WASM SE Exported memory: %s", name.c_str()); + auto memory = std::get(*newInstance.get(store, name)); + wasmMems.insert(std::make_pair(wasmName + "." + name, memory)); + } + else if (std::holds_alternative(externType)) + { + TRACE("WASM SE Exported table: %s", name.c_str()); + } + else if (std::holds_alternative(externType)) + { + TRACE("WASM SE Exported global: %s", name.c_str()); + } + else + { + TRACE("WASM SE Unknown export type"); + } + } + wasmInstances.insert(std::make_pair(wasmName, newInstance)); + } + catch (const wasmtime::Error &e) + { + throw makeStringExceptionV(100, "WASM SE createInstance: %s", e.message().c_str()); + } + } + + bool hasFunc(const std::string &qualifiedID) const + { + TRACE("WASM SE hasFunc"); + return wasmFuncs.find(qualifiedID) != wasmFuncs.end(); + } + + wasmtime::Func getFunc(const std::string &qualifiedID) const + { + TRACE("WASM SE getFunc"); + auto found = wasmFuncs.find(qualifiedID); + if (found == wasmFuncs.end()) + throw makeStringExceptionV(100, "Wasm function not found: %s", qualifiedID.c_str()); + return found->second; + } + + wasmtime::ValType::ListRef getFuncParams(const std::string &qualifiedID) + { + TRACE("WASM SE getFuncParams"); + auto func = getFunc(qualifiedID); + wasmtime::FuncType funcType = func.type(store.context()); + return funcType->params(); + } + + wasmtime::ValType::ListRef getFuncResults(const std::string &qualifiedID) + { + TRACE("WASM SE getFuncResults"); + auto func = getFunc(qualifiedID); + wasmtime::FuncType funcType = func.type(store.context()); + return funcType->results(); + } + + std::vector call(const std::string &qualifiedID, const std::vector ¶ms) + { + TRACE("WASM SE call"); + auto func = getFunc(qualifiedID); + try + { + auto retVal = func.call(store, params).unwrap(); + return retVal; + } + catch (const wasmtime::Trap &e) + { + throw makeStringExceptionV(100, "WASM SE call: %s", e.message().c_str()); + } + } + + std::vector callRealloc(const std::string &wasmName, const std::vector ¶ms) + { + TRACE("WASM SE callRealloc"); + return call(createQualifiedID(wasmName, "cabi_realloc"), params); + } + + wasmtime::Span getData(const std::string &wasmName) + { + TRACE("WASM SE getData"); + auto found = wasmMems.find(createQualifiedID(wasmName, "memory")); + if (found == wasmMems.end()) + throw makeStringExceptionV(100, "Wasm memory not found: %s", wasmName.c_str()); + return found->second.data(store.context()); + } +}; +thread_local std::unique_ptr wasmStore = std::make_unique(); + +class SecureFunction : public CInterfaceOf +{ + std::string wasmName; + std::string funcName; + std::string qualifiedID; + + std::vector args; + std::vector wasmResults; + +public: + SecureFunction(ICodeContext *codeCtx) + { + TRACE("WASM SE se:constructor"); + wasmEngine->setCodeContext(codeCtx); + } + + virtual ~SecureFunction() + { + TRACE("WASM SE se:destructor"); + + // Garbage Collection --- + // Function results --- + auto gc_func_name = createQualifiedID(wasmName, "cabi_post_" + funcName); + if (wasmStore->hasFunc(gc_func_name)) + { + for (auto &result : wasmResults) + { + wasmStore->call(gc_func_name, {result}); + } + } + } + + // IEmbedFunctionContext --- + void setActivityContext(const IThorActivityContext *activityCtx) + { + } + + virtual IInterface *bindParamWriter(IInterface *esdl, const char *esdlservice, const char *esdltype, const char *name) + { + TRACE("WASM SE paramWriterCommit"); + return NULL; + } + virtual void paramWriterCommit(IInterface *writer) + { + TRACE("WASM SE paramWriterCommit"); + } + virtual void writeResult(IInterface *esdl, const char *esdlservice, const char *esdltype, IInterface *writer) + { + TRACE("WASM SE writeResult"); + } + virtual void bindBooleanParam(const char *name, bool val) + { + TRACE("WASM SE bindBooleanParam %s %i", name, val); + args.push_back(val); + } + virtual void bindDataParam(const char *name, size32_t len, const void *val) + { + TRACE("WASM SE bindDataParam %s %d", name, len); + } + virtual void bindFloatParam(const char *name, float val) + { + TRACE("WASM SE bindFloatParam %s %f", name, val); + args.push_back(val); + } + virtual void bindRealParam(const char *name, double val) + { + TRACE("WASM SE bindRealParam %s %f", name, val); + args.push_back(val); + } + virtual void bindSignedSizeParam(const char *name, int size, __int64 val) + { + TRACE("WASM SE bindSignedSizeParam %s %i %lld", name, size, val); + if (size <= 4) + args.push_back(static_cast(val)); + else + args.push_back(static_cast(val)); + } + virtual void bindSignedParam(const char *name, __int64 val) + { + TRACE("WASM SE bindSignedParam %s %lld", name, val); + args.push_back(static_cast(val)); + } + virtual void bindUnsignedSizeParam(const char *name, int size, unsigned __int64 val) + { + TRACE("WASM SE bindUnsignedSizeParam %s %i %llu", name, size, val); + if (size <= 4) + args.push_back(static_cast(val)); + else + args.push_back(static_cast(val)); + } + virtual void bindUnsignedParam(const char *name, unsigned __int64 val) + { + TRACE("WASM SE bindUnsignedParam %s %llu", name, val); + args.push_back(static_cast(val)); + } + virtual void bindStringParam(const char *name, size32_t bytes, const char *val) + { + TRACE("WASM SE bindStringParam %s %d %s", name, bytes, val); + size32_t utfCharCount; + rtlDataAttr utfText; + rtlStrToUtf8X(utfCharCount, utfText.refstr(), bytes, val); + bindUTF8Param(name, utfCharCount, utfText.getstr()); + } + virtual void bindVStringParam(const char *name, const char *val) + { + TRACE("WASM SE bindVStringParam %s %s", name, val); + bindStringParam(name, strlen(val), val); + } + virtual void bindUTF8Param(const char *name, size32_t chars, const char *val) + { + TRACE("WASM SE bindUTF8Param %s %d %s", name, chars, val); + auto bytes = rtlUtf8Size(chars, val); + auto memIdxVar = wasmStore->callRealloc(wasmName, {0, 0, 1, (int32_t)bytes}); + auto memIdx = memIdxVar[0].i32(); + auto mem = wasmStore->getData(wasmName); + memcpy(&mem[memIdx], val, bytes); + args.push_back(memIdx); + args.push_back((int32_t)bytes); + } + virtual void bindUnicodeParam(const char *name, size32_t chars, const UChar *val) + { + TRACE("WASM SE bindUnicodeParam %s %d", name, chars); + size32_t utfCharCount; + rtlDataAttr utfText; + rtlUnicodeToUtf8X(utfCharCount, utfText.refstr(), chars, val); + bindUTF8Param(name, utfCharCount, utfText.getstr()); + } + + virtual void bindSetParam(const char *name, int elemType, size32_t elemSize, bool isAll, size32_t totalBytes, const void *setData) + { + TRACE("WASM SE bindSetParam %s %d %d %d %d %p", name, elemType, elemSize, isAll, totalBytes, setData); + throw makeStringException(200, "bindSetParam not implemented"); + + type_vals typecode = (type_vals)elemType; + const byte *inData = (const byte *)setData; + const byte *endData = inData + totalBytes; + int numElems; + if (elemSize == UNKNOWN_LENGTH) + { + numElems = 0; + // Will need 2 passes to work out how many elements there are in the set :( + while (inData < endData) + { + int thisSize; + switch (elemType) + { + case type_varstring: + thisSize = strlen((const char *)inData) + 1; + break; + case type_string: + thisSize = *(size32_t *)inData + sizeof(size32_t); + break; + case type_unicode: + thisSize = (*(size32_t *)inData) * sizeof(UChar) + sizeof(size32_t); + break; + case type_utf8: + thisSize = rtlUtf8Size(*(size32_t *)inData, inData + sizeof(size32_t)) + sizeof(size32_t); + break; + default: + rtlFail(0, "wasmembed: Unsupported parameter type"); + break; + } + inData += thisSize; + numElems++; + } + inData = (const byte *)setData; + } + else + numElems = totalBytes / elemSize; + + std::vector memIdxVar; + int32_t memIdx; + + switch (typecode) + { + case type_boolean: + memIdxVar = wasmStore->callRealloc(wasmName, {0, 0, 1, (int32_t)numElems}); + memIdx = memIdxVar[0].i32(); + break; + default: + rtlFail(0, "wasmembed: Unsupported parameter type"); + break; + } + + auto mem = wasmStore->getData(wasmName); + size32_t thisSize = elemSize; + for (int idx = 0; idx < numElems; idx++) + { + switch (typecode) + { + case type_boolean: + mem[memIdx + idx] = *(bool *)inData; + break; + default: + rtlFail(0, "v8embed: Unsupported parameter type"); + break; + } + inData += thisSize; + } + args.push_back(memIdx); + args.push_back(numElems); + } + + virtual void bindRowParam(const char *name, IOutputMetaData &metaVal, const byte *val) override + { + TRACE("WASM SE bindRowParam %s %p", name, val); + throw makeStringException(200, "bindRowParam not implemented"); + } + virtual void bindDatasetParam(const char *name, IOutputMetaData &metaVal, IRowStream *val) + { + TRACE("WASM SE bindDatasetParam %s %p", name, val); + throw makeStringException(200, "bindDatasetParam not implemented"); + } + virtual bool getBooleanResult() + { + TRACE("WASM SE getBooleanResult"); + return wasmResults[0].i32(); + } + virtual void getDataResult(size32_t &len, void *&result) + { + TRACE("WASM SE getDataResult"); + throw makeStringException(200, "getDataResult not implemented"); + } + virtual double getRealResult() + { + TRACE("WASM SE getRealResult"); + if (wasmResults[0].kind() == wasmtime::ValKind::F64) + return wasmResults[0].f64(); + return wasmResults[0].f32(); + } + virtual __int64 getSignedResult() + { + TRACE("WASM SE getSignedResult1 %i", (uint8_t)wasmResults[0].kind()); + if (wasmResults[0].kind() == wasmtime::ValKind::I64) + { + return wasmResults[0].i64(); + } + return static_cast<__int64>(wasmResults[0].i32()); + } + virtual unsigned __int64 getUnsignedResult() + { + TRACE("WASM SE getUnsignedResult"); + if (wasmResults[0].kind() == wasmtime::ValKind::I64) + return wasmResults[0].i64(); + return static_cast(wasmResults[0].i32()); + } + virtual void getStringResult(size32_t &chars, char *&result) + { + TRACE("WASM SE getStringResult %zu", wasmResults.size()); + auto ptr = wasmResults[0].i32(); + auto data = wasmStore->getData(wasmName); + uint32_t strPtr; + std::string encoding; + uint32_t bytes; + std::tie(strPtr, encoding, bytes) = load_string(data, ptr); + size32_t codepoints = rtlUtf8Length(bytes, &data[strPtr]); + rtlUtf8ToStrX(chars, result, codepoints, reinterpret_cast(&data[strPtr])); + } + virtual void getUTF8Result(size32_t &chars, char *&result) + { + TRACE("WASM SE getUTF8Result"); + auto ptr = wasmResults[0].i32(); + auto data = wasmStore->getData(wasmName); + uint32_t strPtr; + std::string encoding; + uint32_t bytes; + std::tie(strPtr, encoding, bytes) = load_string(data, ptr); + chars = rtlUtf8Length(bytes, &data[strPtr]); + TRACE("WASM SE getUTF8Result %d %d", bytes, chars); + result = (char *)rtlMalloc(bytes); + memcpy(result, &data[strPtr], bytes); + } + virtual void getUnicodeResult(size32_t &chars, UChar *&result) + { + TRACE("WASM SE getUnicodeResult"); + auto ptr = wasmResults[0].i32(); + auto data = wasmStore->getData(wasmName); + uint32_t strPtr; + std::string encoding; + uint32_t bytes; + std::tie(strPtr, encoding, bytes) = load_string(data, ptr); + unsigned numchars = rtlUtf8Length(bytes, &data[strPtr]); + rtlUtf8ToUnicodeX(chars, result, numchars, reinterpret_cast(&data[strPtr])); + } + virtual void getSetResult(bool &__isAllResult, size32_t &resultBytes, void *&result, int elemType, size32_t elemSize) + { + TRACE("WASM SE getSetResult %d %d %zu", elemType, elemSize, wasmResults.size()); + auto ptr = wasmResults[0].i32(); + auto data = wasmStore->getData(wasmName); + + throw makeStringException(200, "getSetResult not implemented"); + } + virtual IRowStream *getDatasetResult(IEngineRowAllocator *_resultAllocator) + { + TRACE("WASM SE getDatasetResult"); + throw makeStringException(200, "getDatasetResult not implemented"); + return NULL; + } + virtual byte *getRowResult(IEngineRowAllocator *_resultAllocator) + { + TRACE("WASM SE getRowResult"); + throw makeStringException(200, "getRowResult not implemented"); + return NULL; + } + virtual size32_t getTransformResult(ARowBuilder &builder) + { + TRACE("WASM SE getTransformResult"); + throw makeStringException(200, "getTransformResult not implemented"); + return 0; + } + virtual void loadCompiledScript(size32_t chars, const void *_script) override + { + TRACE("WASM SE loadCompiledScript %p", _script); + throw makeStringException(200, "loadCompiledScript not implemented"); + } + virtual void enter() override + { + TRACE("WASM SE enter"); + } + virtual void reenter(ICodeContext *codeCtx) override + { + TRACE("WASM SE reenter"); + } + virtual void exit() override + { + TRACE("WASM SE exit"); + } + virtual void compileEmbeddedScript(size32_t lenChars, const char *_utf) override + { + TRACE("WASM SE compileEmbeddedScript"); + throw makeStringException(200, "compileEmbeddedScript not supported"); + } + virtual void importFunction(size32_t lenChars, const char *qualifiedName) override + { + TRACE("WASM SE importFunction: %s", qualifiedName); + + qualifiedID = std::string(qualifiedName, lenChars); + std::tie(wasmName, funcName) = splitQualifiedID(qualifiedID); + + if (!wasmStore->hasInstance(wasmName)) + { + wasmStore->registerInstance(wasmName); + } + } + virtual void callFunction() + { + TRACE("WASM SE callFunction %s", qualifiedID.c_str()); + wasmResults = wasmStore->call(qualifiedID, args); + } +}; + +IEmbedFunctionContext *createISecureEnclave(ICodeContext *codeCtx) +{ + return new SecureFunction(codeCtx); +} diff --git a/plugins/wasmembed/secure-enclave.hpp b/plugins/wasmembed/secure-enclave.hpp new file mode 100644 index 00000000000..fe81927e191 --- /dev/null +++ b/plugins/wasmembed/secure-enclave.hpp @@ -0,0 +1,4 @@ +#include "platform.h" +#include "eclrtl.hpp" + +IEmbedFunctionContext *createISecureEnclave(ICodeContext *codeCtx); diff --git a/plugins/wasmembed/test/CMakeLists.txt b/plugins/wasmembed/test/CMakeLists.txt new file mode 100644 index 00000000000..14079ec94ec --- /dev/null +++ b/plugins/wasmembed/test/CMakeLists.txt @@ -0,0 +1,33 @@ +cmake_minimum_required(VERSION 3.22) + +project(wasmembed) + +set(WASM_PATH "${CMAKE_CURRENT_BINARY_DIR}/bin/${PROJECT_NAME}.wasm") + +add_custom_command( + MAIN_DEPENDENCY ${CMAKE_CURRENT_SOURCE_DIR}/hpcc-scalar-test.wit + OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/hpcc_scalar_test.c ${CMAKE_CURRENT_BINARY_DIR}/hpcc_scalar_test.h ${CMAKE_CURRENT_BINARY_DIR}/hpcc_scalar_test_component_type.o + COMMAND wit-bindgen c --out-dir ${CMAKE_CURRENT_BINARY_DIR} ${CMAKE_CURRENT_SOURCE_DIR}/hpcc-scalar-test.wit +) +add_custom_target(wit-generate ALL DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/hpcc_scalar_test.c) + +set(CMAKE_EXECUTABLE_SUFFIX ".wasm") +set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -nostartfiles -fno-exceptions --sysroot=/${WASI_SDK_PREFIX}/share/wasi-sysroot -Wl,--no-entry") + +include_directories( + ${CMAKE_CURRENT_BINARY_DIR} +) + +add_executable(wasmembed + main.cpp + ${CMAKE_CURRENT_BINARY_DIR}/hpcc_scalar_test.c + ${CMAKE_CURRENT_BINARY_DIR}/hpcc_scalar_test.h +) + +target_link_libraries(wasmembed + ${CMAKE_CURRENT_BINARY_DIR}/hpcc_scalar_test_component_type.o +) + +install(TARGETS wasmembed + RUNTIME DESTINATION ${CMAKE_CURRENT_BINARY_DIR}/../install-target +) diff --git a/plugins/wasmembed/test/build.sh b/plugins/wasmembed/test/build.sh new file mode 100755 index 00000000000..7f91603130c --- /dev/null +++ b/plugins/wasmembed/test/build.sh @@ -0,0 +1,23 @@ +#!/bin/bash + +SCRIPT_DIR="$( cd -- "$( dirname -- "${BASH_SOURCE[0]:-$0}"; )" &> /dev/null && pwd 2> /dev/null; )" +ROOT_DIR="${SCRIPT_DIR}/../../.." + +echo "SCRIPT_DIR: ${SCRIPT_DIR}" +echo "ROOT_DIR: $ROOT_DIR" + +docker build --progress plain -f "${SCRIPT_DIR}/wasm32-wasi/Dockerfile" \ + -t wasm32-wasi:latest \ + "${SCRIPT_DIR}/." + +CMAKE_OPTIONS="-G Ninja -DCMAKE_BUILD_TYPE=MinSizeRel -DCMAKE_TOOLCHAIN_FILE=/hpcc-dev/wasi-sdk/share/cmake/wasi-sdk.cmake -DWASI_SDK_PREFIX=/hpcc-dev/wasi-sdk" + +docker run --rm \ + --mount source="${SCRIPT_DIR}",target=/hpcc-dev/wasmembed,type=bind,consistency=cached \ + --mount source="${ROOT_DIR}/testing/regress/ecl",target=/hpcc-dev/install-target,type=bind,consistency=cached \ + wasm32-wasi:latest \ + "rm -rf ./build && \ + cmake -S . -B /hpcc-dev/build ${CMAKE_OPTIONS} && \ + cmake --build /hpcc-dev/build --target install" + +echo "docker run -it --mount source=\"${SCRIPT_DIR}\",target=/hpcc-dev/wasmembed,type=bind,consistency=cached wasm32-wasi:latest bash" diff --git a/plugins/wasmembed/test/hpcc-scalar-test.wit b/plugins/wasmembed/test/hpcc-scalar-test.wit new file mode 100644 index 00000000000..3b632d46ea1 --- /dev/null +++ b/plugins/wasmembed/test/hpcc-scalar-test.wit @@ -0,0 +1,28 @@ +package hpcc-systems:hpcc-platform + +world hpcc-scalar-test { +/* imports --- + + guests dispose all params as needed + guests should dispose "results" as needed +*/ + import dbglog: func(msg: string) + +/* exports --- + + guests dispose all params as needed + hosts call cabi_post_XXX to dispose "results" as needed +*/ + export bool-test: func(a: bool, b: bool) -> bool + export float32-test: func(a: float32, b: float32) -> float32 + export float64-test: func(a: float64, b: float64) -> float64 + export u8-test: func(a: u8, b: u8) -> u8 + export u16-test: func(a: u16, b: u16) -> u16 + export u32-test: func(a: u32, b: u32) -> u32 + export u64-test: func(a: u64, b: u64) -> u64 + export s8-test: func(a: s8, b: s8) -> s8 + export s16-test: func(a: s16, b: s16) -> s16 + export s32-test: func(a: s32, b: s32) -> s32 + export s64-test: func(a: s64, b: s64) -> s64 + export utf8-string-test: func(a: string, b: string) -> string +} diff --git a/plugins/wasmembed/test/main.cpp b/plugins/wasmembed/test/main.cpp new file mode 100644 index 00000000000..c0d7420e5f1 --- /dev/null +++ b/plugins/wasmembed/test/main.cpp @@ -0,0 +1,70 @@ +#include "hpcc_scalar_test.h" + +#include + +void dbglog(const std::string str) +{ + hpcc_scalar_test_string_t msg; + hpcc_scalar_test_string_set(&msg, str.c_str()); + hpcc_scalar_test_dbglog(&msg); +} + +bool hpcc_scalar_test_bool_test(bool a, bool b) +{ + return a && b; +} +float hpcc_scalar_test_float32_test(float a, float b) +{ + return a + b; +} +double hpcc_scalar_test_float64_test(double a, double b) +{ + return a + b; +} +uint8_t hpcc_scalar_test_u8_test(uint8_t a, uint8_t b) +{ + return a + b; +} +uint16_t hpcc_scalar_test_u16_test(uint16_t a, uint16_t b) +{ + return a + b; +} +uint32_t hpcc_scalar_test_u32_test(uint32_t a, uint32_t b) +{ + return a + b; +} +uint64_t hpcc_scalar_test_u64_test(uint64_t a, uint64_t b) +{ + return a + b; +} +int8_t hpcc_scalar_test_s8_test(int8_t a, int8_t b) +{ + return a + b; +} +int16_t hpcc_scalar_test_s16_test(int16_t a, int16_t b) +{ + return a + b; +} +int32_t hpcc_scalar_test_s32_test(int32_t a, int32_t b) +{ + return a + b; +} +int64_t hpcc_scalar_test_s64_test(int64_t a, int64_t b) +{ + return a + b; +} +uint32_t hpcc_scalar_test_char_test(uint32_t a, uint32_t b) +{ + return a + b; +} +static uint32_t tally = 0; +void hpcc_scalar_test_utf8_string_test(hpcc_scalar_test_string_t *a, hpcc_scalar_test_string_t *b, hpcc_scalar_test_string_t *ret) +{ + std::string s1(a->ptr, a->len); + hpcc_scalar_test_string_free(a); + std::string s2(b->ptr, b->len); + hpcc_scalar_test_string_free(b); + std::string r = s1 + s2; + dbglog(std::to_string(++tally) + ": " + r); + hpcc_scalar_test_string_dup(ret, r.c_str()); +} diff --git a/plugins/wasmembed/test/wasm32-wasi/Dockerfile b/plugins/wasmembed/test/wasm32-wasi/Dockerfile new file mode 100644 index 00000000000..5746ac1eb34 --- /dev/null +++ b/plugins/wasmembed/test/wasm32-wasi/Dockerfile @@ -0,0 +1,41 @@ +FROM ubuntu:22.04 + +RUN apt-get update && \ + apt-get install -y \ + autoconf \ + autogen \ + automake \ + clang \ + cmake \ + curl \ + libtool \ + lld \ + llvm \ + make \ + ninja-build \ + wget + +RUN curl --proto '=https' --tlsv1.3 https://sh.rustup.rs -sSf | sh -s -- -y + +SHELL ["/bin/bash", "--login", "-c"] + +WORKDIR /hpcc-dev + +ARG WIT_VERSION=0.9.0 +RUN cargo install wasm-tools && \ + cargo install --git https://github.com/bytecodealliance/wit-bindgen --tag wit-bindgen-cli-${WIT_VERSION} wit-bindgen-cli && \ + curl https://wasmtime.dev/install.sh -sSf | bash + +# List of current vertsion can be found in https://github.com/bytecodealliance/wit-bindgen/releases --- +ARG WASI_VERSION=20 +ARG WASI_MINOR_VERSION=0 +ARG WASI_VERSION_FULL=${WASI_VERSION}.${WASI_MINOR_VERSION} +RUN wget https://github.com/WebAssembly/wasi-sdk/releases/download/wasi-sdk-${WASI_VERSION}/wasi-sdk-${WASI_VERSION_FULL}-linux.tar.gz +RUN tar xvf wasi-sdk-${WASI_VERSION_FULL}-linux.tar.gz && rm wasi-sdk-${WASI_VERSION_FULL}-linux.tar.gz +RUN mv wasi-sdk-${WASI_VERSION_FULL} wasi-sdk + +WORKDIR /hpcc-dev/wasmembed + +ENTRYPOINT ["/bin/bash", "--login", "-c"] + +CMD ["bash"] diff --git a/plugins/wasmembed/util.cpp b/plugins/wasmembed/util.cpp new file mode 100644 index 00000000000..7f65691c7b4 --- /dev/null +++ b/plugins/wasmembed/util.cpp @@ -0,0 +1,46 @@ +#include "util.hpp" + +#include "platform.h" +#include "jexcept.hpp" +#include "jfile.hpp" + +std::vector readWasmBinaryToBuffer(const char *filename) +{ + Owned file = createIFile(filename); + Owned fileIO = file->open(IFOread); + if (!fileIO) + throw makeStringExceptionV(0, "Failed to open %s", filename); + + MemoryBuffer mb; + size32_t count = read(fileIO, 0, (size32_t)-1, mb); + uint8_t *ptr = (uint8_t *)mb.detach(); + return std::vector(ptr, ptr + count); +} + +std::string extractContentInDoubleQuotes(const std::string &input) +{ + + std::size_t firstQuote = input.find_first_of('"'); + if (firstQuote == std::string::npos) + return ""; + + std::size_t secondQuote = input.find('"', firstQuote + 1); + if (secondQuote == std::string::npos) + return ""; + + return input.substr(firstQuote + 1, secondQuote - firstQuote - 1); +} + +std::pair splitQualifiedID(const std::string &qualifiedName) +{ + std::size_t firstDot = qualifiedName.find_first_of('.'); + if (firstDot == std::string::npos || firstDot == 0 || firstDot == qualifiedName.size() - 1) + throw makeStringExceptionV(3, "Invalid import function '%s', expected format: .", qualifiedName.c_str()); + + return std::make_pair(qualifiedName.substr(0, firstDot), qualifiedName.substr(firstDot + 1)); +} + +std::string createQualifiedID(const std::string &wasmName, const std::string &funcName) +{ + return wasmName + "." + funcName; +} \ No newline at end of file diff --git a/plugins/wasmembed/util.hpp b/plugins/wasmembed/util.hpp new file mode 100644 index 00000000000..22282eb0579 --- /dev/null +++ b/plugins/wasmembed/util.hpp @@ -0,0 +1,7 @@ +#include +#include + +std::vector readWasmBinaryToBuffer(const char *filename); +std::string extractContentInDoubleQuotes(const std::string &input); +std::pair splitQualifiedID(const std::string &qualifiedName); +std::string createQualifiedID(const std::string &wasmName, const std::string &funcName); diff --git a/plugins/wasmembed/wasm.ecllib b/plugins/wasmembed/wasm.ecllib new file mode 100644 index 00000000000..eb5d8313c02 --- /dev/null +++ b/plugins/wasmembed/wasm.ecllib @@ -0,0 +1,10 @@ +EXPORT Language := SERVICE : plugin('wasmembed') + integer getEmbedContext() : cpp, pure, fold, namespace='wasmLanguageHelper', entrypoint='getEmbedContext', prototype='IEmbedContext* getEmbedContext()'; + string syntaxCheck(const varstring funcname, UTF8 body, const varstring argnames, const varstring compileOptions, const varstring persistOptions) : cpp, pure, fold, namespace='wasmLanguageHelper', entrypoint='syntaxCheck'; +END; +EXPORT getEmbedContext := Language.getEmbedContext; +EXPORT syntaxCheck := Language.syntaxCheck; +EXPORT boolean supportsImport := true; +EXPORT boolean supportsScript := true; +EXPORT boolean prebind := false; +EXPORT boolean singletonEmbedContext := false; \ No newline at end of file diff --git a/plugins/wasmembed/wasmembed.cpp b/plugins/wasmembed/wasmembed.cpp new file mode 100644 index 00000000000..a2ce1d35d43 --- /dev/null +++ b/plugins/wasmembed/wasmembed.cpp @@ -0,0 +1,65 @@ +#include "secure-enclave.hpp" + +#include "jexcept.hpp" +#include "hqlplugins.hpp" + +static const char *compatibleVersions[] = { + "WASM Embed Helper 1.0.0", + NULL}; + +static const char *version = "WASM Embed Helper 1.0.0"; + +extern "C" DECL_EXPORT bool getECLPluginDefinition(ECLPluginDefinitionBlock *pb) +{ + if (pb->size == sizeof(ECLPluginDefinitionBlockEx)) + { + ECLPluginDefinitionBlockEx *pbx = (ECLPluginDefinitionBlockEx *)pb; + pbx->compatibleVersions = compatibleVersions; + } + else if (pb->size != sizeof(ECLPluginDefinitionBlock)) + return false; + pb->magicVersion = PLUGIN_VERSION; + pb->version = version; + pb->moduleName = "wasm"; + pb->ECL = NULL; + pb->flags = PLUGIN_MULTIPLE_VERSIONS; + pb->description = "WASM Embed Helper"; + return true; +} + +namespace wasmLanguageHelper +{ + class WasmEmbedContext : public CInterfaceOf + { + public: + virtual IEmbedFunctionContext *createFunctionContext(unsigned flags, const char *options) override + { + return createFunctionContextEx(nullptr, nullptr, flags, options); + } + + virtual IEmbedFunctionContext *createFunctionContextEx(ICodeContext *codeCtx, const IThorActivityContext *activityContext, unsigned flags, const char *options) override + { + return createISecureEnclave(codeCtx); + } + + virtual IEmbedServiceContext *createServiceContext(const char *service, unsigned flags, const char *options) override + { + throwUnexpected(); + return nullptr; + } + } theEmbedContext; + + extern DECL_EXPORT IEmbedContext *getEmbedContext() + { + return LINK(&theEmbedContext); + } + + extern DECL_EXPORT void syntaxCheck(size32_t &__lenResult, char *&__result, const char *funcname, size32_t charsBody, const char *body, const char *argNames, const char *compilerOptions, const char *persistOptions) + { + StringBuffer result; + // MORE - ::syntaxCheck(__lenResult, __result, funcname, charsBody, body, argNames, compilerOptions, persistOptions); + __lenResult = result.length(); + __result = result.detach(); + } + +} // namespace diff --git a/system/jlib/jarray.hpp b/system/jlib/jarray.hpp index a2c1240bb6d..9870c8d9c31 100644 --- a/system/jlib/jarray.hpp +++ b/system/jlib/jarray.hpp @@ -153,7 +153,7 @@ class ArrayOf : public AllocatorOf typedef int (*CompareFunc)(const MEMBER *, const MEMBER *); // Should really be const, as should the original array functions public: - ~ArrayOf() { kill(); } + ~ArrayOf() { kill(); } MEMBER & operator[](size_t pos) { return element((aindex_t)pos); } const MEMBER & operator[](size_t pos) const { return element((aindex_t)pos); } diff --git a/system/jlib/jbuff.hpp b/system/jlib/jbuff.hpp index 5d6392dbec6..010e9dcc66d 100644 --- a/system/jlib/jbuff.hpp +++ b/system/jlib/jbuff.hpp @@ -66,10 +66,10 @@ template class OwnedPtrCustomFree CLASS *ptr = nullptr; public: - OwnedPtrCustomFree() { } - OwnedPtrCustomFree(CLASS *_ptr) : ptr(_ptr) { } - OwnedPtrCustomFree(SELF &&_ptr) { ptr = _ptr.getClear(); } - ~OwnedPtrCustomFree() { safeFree(ptr); } + OwnedPtrCustomFree() { } + OwnedPtrCustomFree(CLASS *_ptr) : ptr(_ptr) { } + OwnedPtrCustomFree(SELF &&_ptr) { ptr = _ptr.getClear(); } + ~OwnedPtrCustomFree() { safeFree(ptr); } void operator = (CLASS * _ptr) { diff --git a/system/jlib/jhash.hpp b/system/jlib/jhash.hpp index c8790fe2e4c..3653abd4c21 100644 --- a/system/jlib/jhash.hpp +++ b/system/jlib/jhash.hpp @@ -468,13 +468,13 @@ template class CMinHashTable } public: - CMinHashTable(unsigned _initialSize = 7) + CMinHashTable(unsigned _initialSize = 7) { htn = _initialSize; n = 0; table = (C **)calloc(sizeof(C *),htn); } - ~CMinHashTable() + ~CMinHashTable() { C **t = table+htn; while (t--!=table) @@ -625,7 +625,7 @@ template class CTimeLimitedCache { public: - CTimeLimitedCache(unsigned timeoutMs=defaultCacheTimeoutMs) + CTimeLimitedCache(unsigned timeoutMs=defaultCacheTimeoutMs) { timeoutPeriodCycles = ((cycle_t)timeoutMs) * queryOneSecCycles() / 1000; } diff --git a/system/jlib/jiface.hpp b/system/jlib/jiface.hpp index 4f3d6b2a744..ca502ab010d 100644 --- a/system/jlib/jiface.hpp +++ b/system/jlib/jiface.hpp @@ -88,8 +88,8 @@ class CSimpleInterfaceOf : public INTERFACE } private: - CSimpleInterfaceOf(const CSimpleInterfaceOf&) = delete; - CSimpleInterfaceOf(CSimpleInterfaceOf &&) = delete; + CSimpleInterfaceOf(const CSimpleInterfaceOf&) = delete; + CSimpleInterfaceOf(CSimpleInterfaceOf &&) = delete; CSimpleInterfaceOf & operator = (const CSimpleInterfaceOf &) = delete; mutable std::atomic xxcount; }; diff --git a/system/jlib/jsuperhash.hpp b/system/jlib/jsuperhash.hpp index 8806ad52993..2cd6216f412 100644 --- a/system/jlib/jsuperhash.hpp +++ b/system/jlib/jsuperhash.hpp @@ -178,9 +178,9 @@ class SimpleHashTableOf : public SuperHashTableOf { typedef SimpleHashTableOf SELF; public: - SimpleHashTableOf(void) : SuperHashTableOf() { } - SimpleHashTableOf(unsigned initsize) : SuperHashTableOf(initsize) { } - ~SimpleHashTableOf() { SELF::_releaseAll(); } + SimpleHashTableOf(void) : SuperHashTableOf() { } + SimpleHashTableOf(unsigned initsize) : SuperHashTableOf(initsize) { } + ~SimpleHashTableOf() { SELF::_releaseAll(); } IMPLEMENT_SUPERHASHTABLEOF_REF_FIND(ET, FP); @@ -209,9 +209,9 @@ class OwningSimpleHashTableOf : public SimpleHashTableOf { typedef OwningSimpleHashTableOf SELF; public: - OwningSimpleHashTableOf(void) : SimpleHashTableOf() { } - OwningSimpleHashTableOf(unsigned initsize) : SimpleHashTableOf(initsize) { } - ~OwningSimpleHashTableOf() { SELF::_releaseAll(); } + OwningSimpleHashTableOf(void) : SimpleHashTableOf() { } + OwningSimpleHashTableOf(unsigned initsize) : SimpleHashTableOf(initsize) { } + ~OwningSimpleHashTableOf() { SELF::_releaseAll(); } virtual void onRemove(void *et) { ((ET *)et)->Release(); } }; @@ -288,9 +288,9 @@ class StringSuperHashTableOf : public SuperHashTableOf { typedef StringSuperHashTableOf SELF; public: - StringSuperHashTableOf(void) : SuperHashTableOf() { } - StringSuperHashTableOf(unsigned initsize) : SuperHashTableOf(initsize) { } - ~StringSuperHashTableOf() { SELF::_releaseAll(); } + StringSuperHashTableOf(void) : SuperHashTableOf() { } + StringSuperHashTableOf(unsigned initsize) : SuperHashTableOf(initsize) { } + ~StringSuperHashTableOf() { SELF::_releaseAll(); } virtual void onAdd(void *et __attribute__((unused))) { } virtual void onRemove(void *et __attribute__((unused))) { } @@ -318,9 +318,9 @@ class OwningStringSuperHashTableOf : public StringSuperHashTableOf { typedef OwningStringSuperHashTableOf SELF; public: - OwningStringSuperHashTableOf(void) : StringSuperHashTableOf() { } - OwningStringSuperHashTableOf(unsigned initsize) : StringSuperHashTableOf(initsize) { } - ~OwningStringSuperHashTableOf() { SELF::_releaseAll(); } + OwningStringSuperHashTableOf(void) : StringSuperHashTableOf() { } + OwningStringSuperHashTableOf(unsigned initsize) : StringSuperHashTableOf(initsize) { } + ~OwningStringSuperHashTableOf() { SELF::_releaseAll(); } virtual void onRemove(void *et) { ((ET *)et)->Release(); } }; @@ -422,7 +422,7 @@ class ThreadSafeOwningSimpleHashTableOf : public ThreadSafeSimpleHashTableOf SELF; public: - ~ThreadSafeOwningSimpleHashTableOf() { SELF::_releaseAll(); } + ~ThreadSafeOwningSimpleHashTableOf() { SELF::_releaseAll(); } virtual void onRemove(void *et) { ((ET *)et)->Release(); } }; diff --git a/testing/regress/ecl/key/wasmembed.xml b/testing/regress/ecl/key/wasmembed.xml new file mode 100644 index 00000000000..4400c029084 --- /dev/null +++ b/testing/regress/ecl/key/wasmembed.xml @@ -0,0 +1,102 @@ + + true + + + true + + + true + + + true + + + true + + + true + + + true + + + true + + + true + + + true + + + true + + + true + + + true + + + true + + + true + + + true + + + true + + + true + + + true + + + true + + + true + + + true + + + true + + + true + + + true + + + true + + + true + + + true + + + true + + + true + + + true + + + true + + + true + + + true + diff --git a/testing/regress/ecl/wasmembed.ecl b/testing/regress/ecl/wasmembed.ecl new file mode 100644 index 00000000000..75d885e777e --- /dev/null +++ b/testing/regress/ecl/wasmembed.ecl @@ -0,0 +1,112 @@ +import wasm; + +boolean boolTest (boolean a, boolean b) := IMPORT(wasm, 'wasmembed.bool-test'); +real4 float32Test (real4 a, real4 b) := IMPORT(wasm, 'wasmembed.float32-test'); +real8 float64Test (real8 a, real8 b) := IMPORT(wasm, 'wasmembed.float64-test'); +unsigned1 u8Test (unsigned1 a, unsigned1 b) := IMPORT(wasm, 'wasmembed.u8-test'); +unsigned2 u16Test (unsigned2 a, unsigned2 b) := IMPORT(wasm, 'wasmembed.u16-test'); +unsigned4 u32Test (unsigned4 a, unsigned4 b) := IMPORT(wasm, 'wasmembed.u32-test'); +unsigned8 u64Test (unsigned8 a, unsigned8 b) := IMPORT(wasm, 'wasmembed.u64-test'); +integer1 s8Test (integer1 a, integer1 b) := IMPORT(wasm, 'wasmembed.s8-test'); +integer2 s16Test (integer2 a, integer2 b) := IMPORT(wasm, 'wasmembed.s16-test'); +integer4 s32Test (integer4 a, integer4 b) := IMPORT(wasm, 'wasmembed.s32-test'); +integer8 s64Test (integer8 a, integer8 b) := IMPORT(wasm, 'wasmembed.s64-test'); +string stringTest (string a, string b) := IMPORT(wasm, 'wasmembed.string-test'); +string12 string5Test (string5 a, string5 b) := IMPORT(wasm, 'wasmembed.string-test'); +varstring varstringTest (varstring a, varstring b) := IMPORT(wasm, 'wasmembed.string-test'); +unicode12 unicode5Test (unicode5 a, unicode5 b) := IMPORT(wasm, 'wasmembed.string-test'); +unicode unicodeTest (unicode a, unicode b) := IMPORT(wasm, 'wasmembed.string-test'); +utf8_12 utf8_5Test (utf8_5 a, utf8_5 b) := IMPORT(wasm, 'wasmembed.string-test'); +utf8 utf8Test (utf8 a, utf8 b) := IMPORT(wasm, 'wasmembed.string-test'); + +// '--- bool ---'; +boolTest(false, false) = (false AND false); +boolTest(false, true) = (false AND true); +boolTest(true, false) = (true AND false); +boolTest(true, true) = (true AND true); +// '--- float ---'; +ROUND(float32Test((real4)1234.1234, (real4)2345.2345), 3) = ROUND((real4)((real4)1234.1234 + (real4)2345.2345), 3); +float64Test(123456789.123456789, 23456789.23456789) = (real8)((real8)123456789.123456789 + (real8)23456789.23456789); +// '--- unsigned ---'; +u8Test(1, 2) = (unsigned1)(1 + 2); +u8Test(254, 1) = (unsigned1)(254 + 1); +u16Test(1, 2) = (unsigned2)(1 + 2); +u16Test(65534, 1) = (unsigned2)(65534 + 1); +u32Test(1, 2) = (unsigned4)(1 + 2); +u32Test(4294967294, 1) = (unsigned4)(4294967294 + 1); +u64Test(1, 2) = (unsigned8)(1 + 2); +u64Test(18446744073709551614, 1) = (unsigned8)(18446744073709551614 + 1); +// '--- signed ---'; +s8Test(1, 2) = (integer1)(1 + 2); +s8Test(126, 1) = (integer1)(126 + 1); +s8Test(-127, -1) = (integer1)(-127 - 1); + +s16Test(1, 2) = (integer2)(1 + 2); +s16Test(32766, 1) = (integer2)(32766 + 1); +s16Test(-32767, -1) = (integer2)(-32767 - 1); + +s32Test(1, 2) = (integer4)(1 + 2); +s32Test(2147483646, 1) = (integer4)(2147483646 + 1); +s32Test(-2147483647, -1) = (integer4)(-2147483647 - 1); + +s64Test(1, 2) = (integer8)(1 + 2); +s64Test(9223372036854775806, 1) = (integer8)(9223372036854775806 + 1); +s64Test(-9223372036854775807, -1) = (integer8)(-9223372036854775807 - 1); +// '--- string ---'; +varstringTest('1234567890', 'abcdefghij') = '1234567890' + 'abcdefghij'; +stringTest('1234567890', 'abcdefghij') = '1234567890' + 'abcdefghij'; +unicodeTest(U'1234567890您好1231231230', U'abcdefghij欢迎光临abcdefghij') = U'1234567890您好1231231230' + U'abcdefghij欢迎光临abcdefghij'; +utf8Test(U8'您好', U8'欢迎光临') = U8'您好' + U8'欢迎光临'; +// '--- string (fixed length) ---'; +string5Test('1234567890', 'abcdefghij') = (string12)((string5)'1234567890' + (string5)'abcdefghij'); +utf8_5Test(U8'您好1234567890', U8'欢迎光临abcdefghij') = (utf8_12)((utf8_5)U8'您好1234567890' + (utf8_5)U8'欢迎光临abcdefghij'); +unicode5Test(U'您好1234567890', U'欢迎光临abcdefghij') = (unicode12)((unicode5)U'您好1234567890' + (unicode5)U'欢迎光临abcdefghij'); +// '--- reentry ---'; +r := RECORD + unsigned1 kind; + string20 word; + unsigned8 doc; + unsigned1 segment; + unsigned8 wpos; + END; +d := dataset('~regress::multi::searchsource', r, THOR); + +r2 := RECORD(r) + unsigned8 newUnsigned; + string newWord; + boolean passed; +END; + +r2 t(r L) := TRANSFORM + SELF.newUnsigned := u64Test(L.doc, L.wpos); + boolean a := SELF.newUnsigned = (unsigned8)(L.doc + L.wpos); + SELF.newWord := stringTest(L.word, L.word); + boolean b := SELF.newWord = L.word + L.word; + SELF.passed := a and B; + SELF := L; +END; + +r2 t2(r L) := TRANSFORM + SELF.newUnsigned := u64Test(L.doc, L.wpos); + boolean a := SELF.newUnsigned = L.doc+ L.wpos; + SELF.newWord := L.word + L.word; + boolean b := SELF.newWord = L.word + L.word; + SELF.passed := a and B; + SELF := L; +END; + +r2 t3(r L) := TRANSFORM + SELF.newUnsigned := L.doc+ L.wpos; + boolean a := SELF.newUnsigned = L.doc+ L.wpos; + SELF.newWord := L.word + L.word; + boolean b := SELF.newWord = L.word + L.word; + SELF.passed := a and B; + SELF := L; +END; + +unsigned sampleSize := 10000; +d2 := project(choosen(d, sampleSize), t(LEFT)); +d3 := project(choosen(d, sampleSize, 5000), t(LEFT)); +d4 := project(choosen(d, sampleSize, 10001), t(LEFT)); +count(d2(passed=false)) = 0 AND count(d3(passed=false)) = 0 AND count(d4(passed=false)) = 0; +// '--- --- ---'; diff --git a/testing/regress/ecl/wasmembed.manifest b/testing/regress/ecl/wasmembed.manifest new file mode 100644 index 00000000000..8a2d68eb5c2 --- /dev/null +++ b/testing/regress/ecl/wasmembed.manifest @@ -0,0 +1,3 @@ + + + diff --git a/testing/regress/ecl/wasmembed.wasm b/testing/regress/ecl/wasmembed.wasm new file mode 100644 index 0000000000000000000000000000000000000000..f2f32c33c958c4eba0a09a7f0c7317aa0b165c2c GIT binary patch literal 53388 zcmd7531A%8c_vs#ccX!Bpo@nn@Dg9OC<;vxBtU=w2+huS8;ges{PDa zo98K<>!W_=$d&W-)ArM6ou{6%pE~P2^9&lZt92GnE%dX_W^pNhth1It!G(1;t#3K! z+#Q$3Yt!Z0-1tK6iTQeYVX1O)x^k&Mdv2;cIXhpgSgW;YLyKCaGGnFI0C8z)x?Zu; z?$_+E+L^X&CX;FB-;P{9pU&m-v0Leki$9(G$8~c#H;-#HvpaIQaPfav-gR-$#^a8R zop&>~oy!$)+2y)cMjjOkg$~!tWI8$uuARxCZ3hk=0D-~yW(J25{$tDpUWotI}zmGRlx`AO@uc6;OD z=j{Bs+4=GMz){`!^X7VJP+xz+Zd)GFr+;W?m-~nG`4@2>c6lD>Ve>E9Z8g*U%XYRF zHvc1>ht0o&^RW3>?Oc2me{6R((f&2NJ#SlqopV27|Azf(yW`!@I$yQhEM+NsF{h6| zX^zk3a=5FkYVZP{<-_wATxrM$Y>4AixvYNpe0cID^Q6(@oo#mJ>aExC#|pk@%@p1J za_)>NtH((dQ^EIMc;SV%hdV(@k7N01XAF;0LB|8Q^S5+o>Li>Rw?gy*i@yefi-FYjrUU|M%hj1 zaSeDY&_4?Fe**MY9DUlLCyD6@{Uf-thMdzH{ewAWfq-W8wn6XqI1eiKko9QMBAb;J zSf_UvF`gdhao(RS+H&8P`{`XEA77AHDuh;GyL zoxq`O$YF)oKc%4^;X-un);_5OknFve`PNIj78s8q?pkh)b0Bf|gzTJ8rurgyi{ zgArh?%I&r8EvB9ayNcFs(s@Fql(pOY?^cK0vjXyWFfMfM0mvSl9kK?;f(1U zSMR54qLKHrRIHr7e`^rl?@rv`7TwR9`^sVOkm^I$y@KL{4HWlAT}Be19*ypwPTZf2 z?mrfyaA9jzZE1y^_ba5+%DrIQZCAC~Io>bwp6@DKIYUkg2N%4>P_$Lvhasi{chOI4 zv&;mRZ{ukyNS*0~9DFBWgQpf}{5G%|!hfpjXO-()E76 z!)v>7?czFWoPxPQ4;=t4I;T3V9FRFPKI}hD!ay_|fan=wzFPHdmsmppw%mfcNV7Dq zifI7J<@U+|gH#n77TBi&in``KCt_UZsn5PH(Src*U$RTl>9j)LDkrytk8Elv4&Lvw z1vr}r zZS<7)Ho~E=CZ@N?FMdQWwzDz)jCd9AN986d4qkv?=Up7UAB%T{T{b|zQ!cvD;dy00 z@4YL!{F1W2;Ty z9{;V-v%}8&>0aEtc^|?asUAA4gj%0j1 zK}L}CV+7bBAgA&(ekW+@0xextzn}_+md--%Ix30lCCPi~AQJ$9-!N0I_5KXrg|55RF17ojzXvkg ztM;h9GyW}Vr@v3#;w!aJsWZNhMn&bT;-Y`6XymQLAYk_M@N)k)wO`#<^$!5Wfgwlf zXSbtUkGfs;)cq13lwhoZ@(<$lVAa1v-Qgb!_V~TH?5+BT)nWgLI;4(xza_->sXllt zzhCuxzb!Y8s-yk@^&a>R`GWv`Obx1IGyaeo@&|Hiz#j&r;UQ-~2K+nH1uXAz_WQ@t zc2pf#qgDS-43Y9UQT1P@UgqDW?ouP_PVaZwoQ*yIZalr4Mv1GuKgPp$KTu;T@P1EP z<-FeyTc7an0k`ilyvY0as(Za(5zNp4%OOPhByXJXPpOxylj@ZB8MfLbV;S^M z1IXzJ$QgCU^toU6c|hH-9;o_Iu^y+-e~5=c{}t*%^^kf+9MG8mFrSV1kEn;$BQyRh zb@x}PSE^Ud_>W>-kJ5pmN!EWgP`)}s`5N^agYvby&+F7{)$6MMW4s#lU#}ihudn)V z;8lu zr{F#+YRW&SD*kzOPMtsHKY^#y>IpTy=vP5yl~jfTozW!B@^IR}fR=OWf|{%P^J+%T zs(J4Z$X8)&-d}(xTJSy_3TwN+h#QOE=fWE&{3XH}^lNHK)vA77lf10zYPsrP1ivnd z4fi->{v~w1b3x zpJGRPkE?isaO0Y-uG!ju6msvi?X97U+0ZQUbwWd6mF~52eg^p3kS4TU^s}I%9dk6C z6U4lT`2gp6)vof3a40#|dCKns%fy>3!kc*TCa&62g*Vx1oS9e1z1q&cIBF1RfEo|f zATMArmo-2$h1{&2e{n$d5MQwL7#A@{j7fa(qTc~zIUvhbG0MC_)=|jKy;x+_*bD+0 zAyt31V!5kR>6j zLhha@wfB?6=xMk?L{_Ui^K~Jq4bgoD$#^06amT(j9MJ04gUo^iF~)j6WWV<%%_(RQ z=4C{WRNyx6%W^l1k&2pK#3;VIJS*i2>g02*WV5ZH>vIFrbT~;K-dvMxEXBfRyV8e8Gk3H;9X>;fZmNhd(>{V z=c2zCZEjI})h#oaqWAfjMRxfL2o%ss>pbRb5A(IFidFwsb*quAU&sxw7WSAXIvx$K zF{2h{{SldA<=)iL#-yA~zm`fPY9Vo~_h-V>Hc$nXQrRk4+itqH6>`ry_P&ja5Hkj9 zG7ONlDxQH)Xt)I7X9xHhbZFn8WuV2MN5f8)SDjTzu*>@sp{jtJo+>DBM!UmE6}AGC ziLtb}PWn60_GYz1-Mr{`V{Yn>fT1nOMq`c3ib%T!1HbT?K!08eSoHz;;7~Nd<`;av`^`(JX1J z(yWmCmTm70;jBWqBur`TorM0BCQstO( z@k_fZh~Apcp7Pjvi|M>oZBbhn{cWbRq)1DUqNi>`FAQe~e9(?64Ebiwj_b-V+Q&+M zZ8+aNX?KKtZDApFDX0{z?E)1-pkKD_Eg_su$fwsr02B!Y8o=-iAs1tyjB01dbqxk( zqytadu>|3?sm@Ugr8OfCh>6{h_Opivp?P==wrm04(T$mHOIDL1^Sg(Hn9oG z(|{;>9TO$I=mXk|!Uk!Q@Sb^0d*1&1ubzb1qBP*2O*AC*+TA}b|Yojlxr5IF?^Kygwv1A zCo-p+#DR`k+%{K6z(eobGLk&_(y7uW7BPIuZ=Nsz?V5bqtP)Dm83oAJVZ~;l{o6HZ zTWq+x>Ghb!Z*tdX=ps?Pb;r7lnM|`^1V7z#}!K8_#XXD+0jf{>{KH`OBieQ$q2b43VYf zE)cYfR~b`|+^u$-LK4S=7+I_)ixQ;ZkranfGOoIPrGfaYM=Iaclk~l!ZjVHV`Obd8 z+HYvi`?o0ATt3K5~;~&*c2J~T2w>-u}w?71I!yu?v9aRIUT@I^bYRLOl z5|@LlM}QqmuE*8!{kR2!Q8!sHU|w@tFexqcnHfP8S8RG#5EqYN!245@1?__7Bm9(g zQCJ@Hft2cq?E{%dvliL+?ziHds!y!9B+q!3j+9hp3ij>pk z4XzwP+}#+acq>GD9PD2SJ?$7>N_kayf-RiW=@++w-mPkz_m{#c5X1$z?NugCnh%BC zOk-}wikqH-XFyCQWE4r=1Gd7Kb>kqbcp17hgdwB_`3^bH1QgC5Ri?L-_W(tYAJ@A= zuCm%ZuH%4~c_V-`BY|{N^qjnEM+Vu!6*W5bGsr_s-9U1XLT*i~Yb@t&elq_Sxwz`| zn?pPrT=gXu-7pCoZrC0zMSs?|y@rmhRwHDR1Hr+&ka4UL{~jpa`swGb+3N=OgD z9vUSi=1(uR;PrFVP|U?Eb3uiI*Jk4=y;W(osBFU$ep@2S-F$G-t*vA`xMk|S#(Q9G zL02UH7mA9Ol!whINN;6)oA3rA2bEso4d{QRG$q~uK9CT_8)UUNpe}{p;O~f3W}#+p zaQ$)pudCwvuw&m68cizbTgA;C(gW+k0d8(T2X;sgtOs1}#;7pyG`Pwt&v%5UXxPnZ ziEgozhy4uTy#K(5#uze@Yle8X0noN8yzzFp;;mR3M%7aly#Fl(b_fw0pm@5R>HWRH zq4&iCvG9983eae5(nO%f9=!jJ!5VAG`1i5l|e4eEO`n$-+ysWD;cCWv>q zO(`s}w$bn9F&Bx7V!4YfLvldR)<&;|q=Y^Zvj`^>Y!KkAh0NstFauyF7+Hg+F z$$(b#H!szj{pU5IKmL*h`+pMtE^(I4lKqL;7MhK#O~(5#nq+7SHd~kyF(!9&Un-_r zj~lgYm{%IMWzkf;{~D6i1rYH~EQQ=(#cH3M3TWs*#egMgozccF`4;UA!3>QXu9Rk( zN3I}C6?ND-f)Loc>_SCM%*F-XO4cnHk7d*rs3aW7R-Ecm3znJku>Fj>sR~=)0YkU2 z=5(|7e{*DT&SLCUE;H@NoolFXZFXAHTOC%jqF4<&DY#Wau~?N`%!XC&aP@(b z2|ivt)JwOS|A(Vqb&g(mG3%rfvow|{O?oRMp}4U^3f~SrZ(a#y$gx@O|6fA>|8Lyi zEV*A;liZi?L;ivATlre<_ z7X5aq1htz=PzpwjZ2_DtB~X;wN|w%-2F`sd^uIqKOHAP<4;P4sS$I9{gY*88kGP&7 zI8^!&6ceFc4-N?JP)THTjx3Se-lRaV zpP?P1h!0A7XQBwj22zYLTR9Vj)&~-MuADLD=^ALbIaB(gpyK@@b;RF_L=>A2)V4)` zyO_;(V>U|9<$5;_1g|%`Vli6nK=p){f)^$hLQU%SiRnD z_CJCYkM&lyPfE;;Mr3T;5yrFVs`o#l)gEjv$*7{)TZbFl{abP4RNX!GSYI}Wp7Ki=^g&fp2N!+p3cat>W~)6u#6iyYsza(*mtyse z!)V?pzN$Wv7(4Xi!BN$(j#hmXVATK_gKhg7+8`n-6k^q}QL7LXuxm7}hN`H?4x{io zs7A~ly*|`m)9QE?8%9U>gPn|vKIqtcL*-dL;Mta={+S1MEUQ+g6CwL z(w4wM)a$9Ob#P>hNqRYz+=RszY>L(y?is%uB?jg+ppWm#&hpo(&hl#VE1`G?@F7Le|zwqHaZ!d>O!3#7df z(!&~{coy&70G@!L={{)e-+56<{0sJ^stpz3Sy>{QH3Rq`FU?T=Y)??Wss~r-N&s{M(d&MxFNW z#|D4dV}8be5X~P_*jc{lzXHu)5mEWDMj|`Qy&o3}UI_|b8K?49y3M2NRq9di{nF;u zX!Ghgm9Gh|eemzU<@&Exuko?T96QWkrye`wzaDtspkA-uu;{-Lc;6V|eO$woE#}@& z2+TJF=9?QZv)FMi3^=RG>a6#Z(qFmqk55)yz8a#gnn`@rto=! zE81JE#+B^QLV`yz^R&kp>CQ zXdk*7XRb3fO|020H9tfGFJw$5(}t`c6mmn2 zh;L=@6>nWIibWo-w}Av7;!CT(1($CtOWEE}3!RQ~3c3Fj+c%wzB#n31TOUobj~O!!pCM!;LI%qtJ0NuRi$qm(=$TYm45mMxRPci|J zuzLUFDaqULB#VHOuSB@2 zJ65Y!Hy3i7t#8On+JMzulqlpr>)6Hh1M4Q~fQe~gqSSQ4SOc5Hur`L2ropc>34Wm) zIK~Auv!I>T%$Fdepl5Zd>U;k}cq)5l!Ba33Obl~Awf$t`olm~;8A^dKLSqq6q7Vyv z(JiKaVdSm$oIlQPbJ%!whECK!kfkORO zmH@AxB6nbGi1%S4%DrbGTE7VJF1xMicg6yIAmDfCNcimmN}o6W3*L^miu^VKul|8o zJ}Ux&b{5HXm z^A6pOr{5&R;M%p6k6Hph{csI$7+LtF&fBj+P5A)^087m*>ifI(gPCGF__vn+SOnCc zj_R)kj_Ek$d@FhR2@VSlLB{j?G`KqCe4eK+JYMiko>g)3bvYrqXpHNx=xf=4m1L17Ur)Azv8 z!s6~?3g4~3XC_oSI9)A*Mtt)iuw;GJV zSR~D-w447^R`4Hfd@08nBiNrOkRoP2+}=jX;+Q+L8-j2z7V$qn=m8}m5DXsQf^e4t zyH?BuZoTM+Y=rwa9LDrjqzf;m@d*x)*%mybGM5&IG#;@&N|iYUM!8P4#0Cko3Xq!QS(c+g8_mc1swV20}7(pdc%|tT7@MO9#9>V zSsVwvpOpn=!bXf8^a=u;f+*n@ct^a?3Fq*hoUO1mSar-SJ`p(e;Op-NIaM+PZBihX zdvg#w@UuoPrH+cCS48lDfo+#|;JT@M=+^uI5i6i)ikaQ$iKJHH1Y*GFmspAg+-_E# zc#T$$|&+NkZpb;!(VMHKVIsju4_#r6W1%%w!3F7e84L_4}K^#7? zkt(()+F^St*D|npEOh7a{SI)&%6H^2v!v1ai&vrkqVJ$!0pF`hum@`EXF)B#(d5t- z?PCVa_l(-uD;=QX6l5=+FQ#TL(jK6o^_k$Vs-Fdyg?FLBL9z+{W+-92$%DOy;%K4v zz#QyEJNx1c{@r(Scim^3_K@*L9z_IZ{MuC;*qcF^(I$}ec?NEqwXaA#Qmeq|_FfE9 zi&gM~{L7r&jV5ioffG?6uqaR?JO;m)1YA1|C>t3NDutTY+Mt?r1x}sDhcAP{2qU#2 znP=piH@N2&fdSkRoq@1WL4n}3frwi$B-xP(_14}k~Tf6%vZ8VbD~e81jQseK1jfNGGTMuk?4)6lk!(}Y56 zx@01N6{a5oa)-#lSJLB#;K(68N61_$9oW(k;E4AScp9S2=_WS#fy4%`sO);NLB86E z4c3ngT#V08L0sg^*Q{!hAM(OD-I-$BA{<2;1_uK{HTFH6{G##xB`KnInmO$2at-NL zi}(l&nSkmXW(WbF1Q1xgmJUr)i=99%UfAJCC9?2SVtG(sOk>KS!9z??aD4ygflggw zhW62P0?YeVhFzfe>d8)Q^WcYjs3j69!Li^AQc!qKo^Wl}3;^+?Ym9ji0Mi^GE^Ji_ z{z#)8)1bbYl9o)$dIb3)B~NgCW9UJe5%vV10YmoMuBe;MA&;sBF6yK9LP!f zPuMTm4jO4uY56i+8~BEMT$&+fkvir#5yv7lNysE`Q~0c18@sz2x&|WOe}GBJLv(7J z(NA|t@!m(b;OeQ)G+Y&A-VOjbSol(;0{c#3K7p8k)rbaDpqydS3)y3154iA~mSN85 z%P<0455xusQV;VA+=eSk;{$J62_jD7Z32E>PPi)L2g+|yc`z=}j7M^X!Ri0lQAf@a zC`T$nOyX6ArNwqI71NM7rcMh(6aefjbDL+Ba?$QPq}v_C*nfC}T*%`xm@mOC&KrR(w1ZJClsAlOF>o?y^{5t6j<6n#!}r=4 z(aLH|uayt7h?QVY8SQH=P8BW?y9TivB1Q};NVL0y>>Zd0oHV0qmTfSRR&myn61d$k zz6P^3CGZG_5}W!ju(gESK|@TkFfasckSt$2G+Gntls-~$G|7ypZ>TvAqNS7`At%?c zYr`kO3D75jk2Gcagvcu)3}CPY<(EQ$w9w;gDHSp!bpIJJFJqXOG0el~GWAq~Xb^sq zv$qbD5xtw4g!0p@<@9Cv#lT)O7%Tq#{J zUBvfZXalTpf+CY?Qi#L{`EdT(N7%$^Li$v209Op@T^-W9VMq_sCEddI8Gb$y3~2D1 z`BB7P$j@f>$t(_(0WV3!#19gM<$wc?;@g{sCQ=AufS4HoFd_qRbp`>=4i-6RlCRwz zIGG9X7GvZh&Pc_g8U$5DX~;{V*(e#b2e+=~O+)Vhri9fBiNyn}Xe5OddqV8&+ z1-;-GFX>IdsYn;$0G+XIdYVML4unDuBE0+oak(mQ|?)J(b;L`Kj> zI#+&BQ>T{}bP*G0i}VtDExqw^8Lf+{)pQYiJyfcW5L$GR#z|(Xwn!HRJ?ra+CLCms z1=>Lu;l81ZDXoj(Aa$`FjBSr}5eYn#7w97Ng}TUeqdn9`7`;_&r!Kgq0aK|tix?>o*)IXq`B$zK|YM0AwmrQ3y1nq%G#$$uD8ziEy z8x*M2uvJM9P2LCb(MT9fm^mAo3lGBV6;1{U!yqYS1MX>GFYvULa2gIhb_*-vT#PDN zXeCT{b!`jNLS!XS1wCu(0kPZ$`tvx5rXUW|R)TrnScwZOflyjoiS))Ex0Y4{=8JKp zwQ$s$@YihKvOAj^$n^60spOgvE8B^TWB?1SW<^Z9gjHs|d7Z@Of${15c zY_o3{_Dc$5j#1jjJLCvr-i? zk4cbgRFSEwu9(29Myj}0l~t=`RAWVYEU|-jhO1OrMfY^HfMJPulF>WMH? zM@sIt!hd9?gQXn$Gm}f6L5Ay~l;A_oz=#Jz5Ta3mpv@8;=~@BlA}je(D4dZ{7*F#E z|5GbM;dCPuhEXD0LMV)OGz9s?vTS%mZ@kqcq7*=OZDnOwNjbK6!4F-{3yeUd*Rn&N445lV0Qv@d$Xrv~BsQ18`H$q_|qT3Y; zU@gs|aEjy9z8B*}U=D?6u#1sE;XyU}NRb~{uLtL8&`@|h*v<#G)L$9h z7_{BAVJ=``RR>86vnOc0qoWTP9t>J%JkSLbe&bAQdI4~C#n%9^qcs>OaG)dLs7uky zOeJFECiWG18A@tEqjR(fcuZ+dbpi@~icq3YMVktgnP8C12!qjmqSnM~4ADT0p#qm4 z5(YvAH|iQwP=@NOD=40>tDp#GTPdh!dZ?fR2+o2EDkz=El3MGjn@~Omw%0*VAuh6$ z6uf9X#c*SK>c;hyb4949K+#%H@j&zx3QVsY(^Kl&3Jpvs3FnDWNg&Zki(7T1;Ov3#Skgll!0NA|OlC_`L&VI&718i9twIlcR@P}T z#&Z*x@!YUvGOLJHO3H}q)Wp}WnT`X#crO(5`^!;%%|=566}ebYk!{r1K&;Fak)b2i z$u)usSRnQiBd9PRWq}^(2!WK~TL*#;r}-k5A?UMlo{dF6@Y%36ZE$!`*u)lX^w}7= z#b4~8muy^t0W=bKOxZqgW32I}_%LME$AIWF(G`2SX*f9YYj>I!Qm_XhROBcG5$AIB zqXK4|V+)C)qPz}!vk_b58Cyu1P4|&tI?K;;}7zQ~!hlIKXE8i^aA`Nn$aBBy(YG%koPYGhlS4%!>(5mwC~Q$sF|$D_@AS zsDF57&~~$wNM(W8bZl!wAxw=I{w8uVGG~L0aM0q?L=%}+=sFE1JOJfhRA(%NXTn=Z zD-qtpFjs`P_^l{l(!MxC0lh>|@8l41nMAkwl!C*+ZnN#%qfE}tTPV<;99jdCJTu)UU`ajkIiY~iQ zRl}=>s`ATDu1>?khgwyo7zkBqQdLWfj}#=s+fAq{h8t5=XSMJaC|av39+(x0W>tk6 zhvMQ{IAO<>gyC(dvEtdn#flak8AtHfkT@UO5Pk3!22_&1J7bY3CA>vUg7B7J4e${| zB2W-HE7!MW28WLLkSClSV>egAG>}M(Af<3K65g^hz@J*u1!6lmfU8v&E2MK&`id`B zq%?Jn;1-J&GVSZiBq)R$fkGYJN@c031_sW6Q7hLfs3Ll=SS+?hMF8QHj&0%Dxmpp6 zZ6PcZ+hP`>Q;Fl_*l`nM*P*RpkI;#FoDOZJL&&9E?vt*aM)V3sVdac}AK1;=3~{bR zpOQ`l%6Hu{D7K~Pf%+r3Ij*ZAd?}SKs1>F zsnS8O&SG3f#$rdDA1P+D`PIVaS46umcu%*Ac)0MuZdP6q?Yj6p7O-P7;{YHW2Df74 zXo^J<$&V;AIFCj+%qCd##wL-*JPr>_K4Q>O%nVQ6)ChOEOx;KfhJ?y0* zB@}B2iumAz8O7R+&~+>UGMi!v%;N?;WRtC$A1Q;BXhdPML0H~LrCye@EGzd4%Yj@v zt-B%4x{(EcZ;Vs~S09ub2p(M=!@?K0kR*h6722cwg}=2{KsQg`Hh?MNM=5X8gW<@5 z^BVKV*AZ433EgwmIClKr+zKRH zw1yC%_bJhmtQCBS+uCw(bMcXJXUzLe-ll70omhB{D^L%iNi51(fmdn^8eX&TTT;## zuTw=CUbZN400-;Ld?Jv0`?rW1^Jk~fa8QG5#!^TaKvG?X+U9h!R|1Kq`6(9D)Uy8)Ejsr{V$XZ~%{6|{aqZY&v zJfmAWx+QIEMN4PYq8>f1Te`X>9Q4YTE`Lpizi%nnWtWAGRc=2m3;PQAe$Zh-Lm0!b z390kb5I2UzLQMGj2en`JO!=(2;Gr8Vz+(pi3xh~0B7}r^XYyeAhS%phzvbrKI zQnH;AO<*Q;30PIrO~?%m#0i@mv8+2Szzc-?Na4|vgtL~8Vj6!+%i^2cGSQPs!YDUT zph8Y5X#N7Vjm?!BrN9GHr`lr!KMQv0AX6>bU>Mev1AK*zyQ*-ogD!fjpo=ZB!MqFW zonV^=BkzmjpzoFiMd&640%2dImhvo% zGfl9osb^%D4%#&J*Z_*cpn$}|2EAlL!BF2x%|RT9cU&Mm$U@m1GjISdd-3HW5F(?~ zqQ*V)NSZ*#Y=VPnqD75+M%1Y1>LxhIaAetpXtb!|1ar@~$L-vw9l&6@xyHr`EEKy4 zNe^nq59rNMwzeyV>}XPhA?e8yRH7#U;6#=}xqM}bocRdt06VA+b=l zC@qWS$_S9YI(R*!3expPHJif%`7%2AEchR+P2(TqNo3fJzc@GvBoKwrAK7q@6f6&j zfS4zt1;R7|flMzSf?$jQHe?&u+`tIt8WcJZr`vM0!{uI1L@eMdW&pDS40=VG!k@dWBp>Pn0$s z{9~LW7;MfYj{#@k9$2U{yiNlXc%XTW&F2`41tvk8%%F`{xmw5ATwFi6j7lOPrGh=+ za-xfL#?l2-7l1)6l(8$$_?c*Z=IJsO?rmp)Cr|MLEcZ0w9s+Bu@o=(Xwt@q88gR0~ zl&U$-UxXGX3daS0L&|Bbwm4`~%%H_csTD9wfaU?tk2<3HjVUUkKIBtmv*^%ki9O1h z*qWT-x7KBBy;gb@o>}hUm+c^4_GrTo696MTAuh!sLLf*KHNPx=4&FiiG%So7LXaKR57lT+k+S?0l}gy3ooWWDf@-@jDF zJDhKQ_U)GmLn`fp9au%apk^iqXWwOczsLC2`z;B6+byJHp7;B9PR_RQj47Ie!XYKw z%Iln5dplz1H^Y-Ua2?{{3w@@}`R=YG_-#_lOt*M9m#mi0IPHEsRr z&mq3hfBO3R`umRd4fGB69qSwF8}1wFJKo>d-`{_*S4fhWp z9Ud4S96mNYG(0>!GJJfbZ=`?Z=*Ymx;K;F&p^@Q{k&)xaLE>?sKMv5x(d{^HA;sy) z)zjgyaI|CR;i-cYFva>%TX;twd7pn!`+7m2ZS3D}BK$KqcX+BYv3$NfJ$G*Y4}mu& zy;JtzR=ba`IO`1Iv1xRf!$ANO#skovA=fB3by;Fp1>qtnWI6 z$@5eC+DhY^=cn`5KO=xo#f3p_x#QTx` z_uy|zsHBR#KDy!=Vd5EhgA@JdaHnD4()ir@O09sq9Lpp4`#Aod!(UPv{rQ0%b{LAsvrXYf zI*o>Wc4iwmI~coYyBUsN_sSv;=sL5F7bED?zAbtJg=NRwVbHKOGuHhx_v~!Izk4Ty zk+z(h8yGqV6PI0k?fz}Ax%si&>vOLI-~t+V_XD2CW<1$4Xzl6b$yS^ccmiEd?VFsx zurNPYnXC8KuPjtXpIDfj?5$0X&yFwk)+@ETYkAGLtnu1~PAF~L2V6u}cCgnTv)v1| z^EDd|E;BW8es=!6ZP{avJ$}`mxN2QZS?v?^^Roiaw&lS^d*Y&XF=ge?&CZY42af7z zjyzkoCzh?{@Y&Fye&))vr|pTSt*28~+wzFMos!#cu_xYQy(ML3m-~nG{j}V_VozML zuJC?{A|v-N*%OznOS~U~ZmJNL6J?#ri^Ha-{m8F`U$LVCHR;x_)PF!(1FHP5b zBY9`~4)-5D+$T3Drst;4SLS*r(|w1JAMVTTx9wZjzzR6@q~CVJ*=KfouJUTQv6S^? z+=q<6-vtf9A70`AO9&lmQ!}GSj$Az2cewA!#Pam))RF1AT77(W_Q?D~{Rmt1qRElk z6XQ!2b9tq3Yp-231>{JTg0b@j@?#TCGghk8ps~vz4XZrTUTC z=?TGA2P(?~F6wyg@Dn6zdahm$FX7cMEYH@b2M$k)hfldy+M$PbZ@1HjI_^z(XX@#E z#$9?WoqO$k`almZPNZ+mxXbTPdznw9@rljp6X_ipxBLF|zG6C^dCWbBACAemd+tvc zUHoQ+`$~N2GUM)jHhes;(pxgFe?MLp#Ydo7cYfqVdSAv>=h8X%%KL9myQnrhDKOD` zcQhwJZLVNWpPh&Dw&ZAR?AFvWf`YnLpO(XvJ}l$b&|n$2FU>Da)vP5lg#ar{ zx$%YC6Z7@*0^Ip@Zri;Y%37tqebt9k%L_NHI`VT%mCBBubqI{K z>gGGvX&KkC?hQ3Ive|iH9mq!icHO>SJEJ1I*KKGNX3x3}4T*c#Bhl#XE$aacwRqpU z4GoEE-G+ulAKqi5Dq8Gar)^R#Z-qinjgFSfM@L8F`S^)ROq+*P;~}rh^`&w6RC#t+ z{GsUaczu3JKe}slv^M=@<&9AC(Tn{<$jkfQ)KgNE^O&Fa*BUUdC&J0`h4IPh`jwVQ zZ(F)CNXzB(6-*5Cn1d=yO_&c1k!1&CEG(Dn^X2iWsU-wZiN>01J<`5(`(M~MjM%05 zOXZ0xGI3L7(la-#t)O<)?$4d@J%lu%er+%ZxG+iq*s7UZL)V0!+Hwr7oPPFA! z_59#lZX|95B$%pJMScfj`x{}^THj^hu(y^xKXmDaqi$_)y_at!V$JZ@m=8a7qc9@? zXd&c;jh2oKH?Xg8(xao}HHO~eTMisj4Vyer>bpUn_`>+*7DOBO-+#41?e%p3@U2hK zyUkYS&exwPSC*FMn;o+DMMtX%_UW4kVqO~LClvccqWxV-hiSSu`Au`TH1PC|8thU9 znc0QP+*E?pByFX^*&Bsk=KCb-(y>2_&0FQN2?>yCVXCPi7e3J-CJGnAfTG2tD3UNX zcUMeA5%fxV9DcYmUaKdFX@$zb>sb{QtRo@m($G6&1G>JtI#D@~ymWT@T%|sJp(#$1 zD4>~^;lH&n&QDJzc%bX_QxiyWFOJVnPmR~Xj@jAOIC1?(a950p*AME!(#Xpj!fD~j zCg&Hf=ol&P#|}2L>iD&o*fynUQ#v059}_%`jWz~v$Y+jtbNUH4t0XZrkL>8=|#XPq+@ zs#lzGb7#YE%`IQ3EMZl9cKQM{ zT2U^uRLjf~b<2hE>804{uCD7++XG-WJ!HCW1?kd~3~Mzg9u%XELFI zH^kY;XTIy_bqfWnpn^WZ+gSKtE>Bg? zjU$&u$~BF}-tqILS79ybMpW8XYDwwQPdN=-D+s@_ENdWc@(`_fm|$Owu9fUd8hkTz z;|jR4?*U|A4YFE2FTMI*F-^OEDx6#?NsiX|MaGXhi9m8F_e!tv8`NSQK(8t`DmGX` zD82SS-w=9ZZ0Z@Go1eRKVSc%$xaM8Mnn|TJIy%jj2~AN0nxh$eQhMyuZ;AGM%14 zC|X`C2Ro$dOUv~9D>Jdj&&FXjrBlr^YNGOx>W?7HB1uaIzv&m&VsJBx>t}uumlHGJ zN;PH0$gdlPLG=`^B8JEBVGX_nVg%+L?=YZQp#~RMH*K&C`~-_QnAKd zq3~hy4RJnPUmcu!txQBQpUQMA!PR84CCwA1u%^PRG$>U*9MftkJ@CG^&v5ojR_|Bh zjg9)=SuR;k2D9fMnV+4WywU<^6YOLxy%Krp+&^uW!l`LYjrGYVkoX}Zh5lWDLMBEE@d zDwDDgG?6J7BrJn)L*cBL7R&d9Jbm8}#Ip0ElOObT3EZnX?$UgG4Qs=5O6W0#ngpg87xi!wV@xQE z8E$FeRExo`XU++vGQ5MO#hC`6)rNxdH){5Y-Z7HHS@e+Vk$|IQ5*@?dxQLcsiiou0 zC#YCmM738mh821GrWGvq;-i}!08@o^EVXp$GcVSG9e^ib$^V9Vu+Uj0%sadZ zeq4U(wirTN%@`Z*^2$%y>r7~^r|Pcv-|`WpRgjeVpD()NfTLJYkWd* ztX=8phhHqpyR>mOkKvhfD*!g9s8c9@H|;(-VaRQ!FGAA%pmg<~7mvq4EnTyJEN0{D zR;<>nhP29JlXFa5S(=N$rMKDxjpt2>qNU#zoBf`(A8+(3mr?a>1PA9%&_d!C)dW`J zm1LJB!KJs^pO5jU<#H&h7MI5H-Vu#7S_oxaW|lxHB;yWadVPJ1K3t}dIO3po2hn8v zcKd@d`or4l%BrhuS*K-lL3e&U!G%WsL~6Qh6~d3Hz7%E~qQGV4JSsa#5Baj*T>S*! zLJ1dSNce(+YF3{ zXMU^kta5p}{@dV1<-+8`mEW=3aKf7W@7fu?BkT9<^wjc&3s-*M-eO*W#5X5q%TstM zvGy7JK67uA@Aw`@?0SK3%jtf9V4q$GPV(i-_&dK%pgwEwj=uxk^z?J~@H&X1cXyk5 ze%@v*wASm)@ppNfI)7nfoy}XRO`ZSHKDG{K^R91Gzc1SBy?~8xaLB9NO&yqF2eBL;iisXS-)fJA%1oJj;)9I$M$-!wd-vntpxHl z8{0wG)V-v6G!3>5UwMM-!DKo)6TVa1zQwWH4&(D9n+1N$8UBG-E#WaJ*0-U*e>5t}>4I_bWB`j@-oZ>`X5_$^6olI|>(;>)h4a Vi_gtWV0+jdS-v>eTe)2S{{Xq^%Vhun literal 0 HcmV?d00001 diff --git a/vcpkg.json.in b/vcpkg.json.in index 7396715d0af..199c2b42bde 100644 --- a/vcpkg.json.in +++ b/vcpkg.json.in @@ -181,6 +181,10 @@ "name": "tbb", "platform": "@VCPKG_TBB@" }, + { + "name": "wasmtime-cpp-api", + "platform": "@VCPKG_WASMEMBED@" + }, { "name": "winflexbison", "platform": "windows" From 6fb36668c97ba1259302f9bcace64a4aeb16d1c1 Mon Sep 17 00:00:00 2001 From: Gordon Smith Date: Mon, 20 Nov 2023 11:18:54 +0000 Subject: [PATCH 11/16] HPCC-30895 image.sh failing due to missing file File removed as part of smoketest cleanup. Signed-off-by: Gordon Smith --- dockerfiles/vcpkg/amazonlinux.dockerfile | 14 ++------------ dockerfiles/vcpkg/centos-7.dockerfile | 6 ++++++ dockerfiles/vcpkg/centos-8.dockerfile | 6 ++++++ dockerfiles/vcpkg/ubuntu-20.04.dockerfile | 6 ++++++ dockerfiles/vcpkg/ubuntu-22.04.dockerfile | 6 ++++++ dockerfiles/vcpkg/ubuntu-23.10.dockerfile | 6 ++++++ 6 files changed, 32 insertions(+), 12 deletions(-) create mode 100644 dockerfiles/vcpkg/centos-7.dockerfile create mode 100644 dockerfiles/vcpkg/centos-8.dockerfile create mode 100644 dockerfiles/vcpkg/ubuntu-20.04.dockerfile create mode 100644 dockerfiles/vcpkg/ubuntu-22.04.dockerfile create mode 100644 dockerfiles/vcpkg/ubuntu-23.10.dockerfile diff --git a/dockerfiles/vcpkg/amazonlinux.dockerfile b/dockerfiles/vcpkg/amazonlinux.dockerfile index 190badf60f4..c7ea3c4e138 100644 --- a/dockerfiles/vcpkg/amazonlinux.dockerfile +++ b/dockerfiles/vcpkg/amazonlinux.dockerfile @@ -1,16 +1,6 @@ ARG VCPKG_REF=latest FROM hpccsystems/platform-build-base-amazonlinux:$VCPKG_REF -RUN amazon-linux-extras install java-openjdk11 && yum install -y \ - java-11-openjdk-devel \ - python3-devel \ - epel-release -RUN yum install -y \ - ccache \ - R-core-devel \ - R-Rcpp-devel \ - R-RInside-devel - -WORKDIR /hpcc-dev - ENTRYPOINT ["/bin/bash", "--login", "-c"] + +CMD ["/bin/bash"] diff --git a/dockerfiles/vcpkg/centos-7.dockerfile b/dockerfiles/vcpkg/centos-7.dockerfile new file mode 100644 index 00000000000..04d1ba29af5 --- /dev/null +++ b/dockerfiles/vcpkg/centos-7.dockerfile @@ -0,0 +1,6 @@ +ARG VCPKG_REF=latest +FROM hpccsystems/platform-build-base-centos-7:$VCPKG_REF + +ENTRYPOINT ["/bin/bash", "--login", "-c"] + +CMD ["/bin/bash"] diff --git a/dockerfiles/vcpkg/centos-8.dockerfile b/dockerfiles/vcpkg/centos-8.dockerfile new file mode 100644 index 00000000000..108398a2711 --- /dev/null +++ b/dockerfiles/vcpkg/centos-8.dockerfile @@ -0,0 +1,6 @@ +ARG VCPKG_REF=latest +FROM hpccsystems/platform-build-base-centos-8:$VCPKG_REF + +ENTRYPOINT ["/bin/bash", "--login", "-c"] + +CMD ["/bin/bash"] \ No newline at end of file diff --git a/dockerfiles/vcpkg/ubuntu-20.04.dockerfile b/dockerfiles/vcpkg/ubuntu-20.04.dockerfile new file mode 100644 index 00000000000..af0e1dc41b2 --- /dev/null +++ b/dockerfiles/vcpkg/ubuntu-20.04.dockerfile @@ -0,0 +1,6 @@ +ARG VCPKG_REF=latest +FROM hpccsystems/platform-build-base-ubuntu-20.04:$VCPKG_REF + +ENTRYPOINT ["/bin/bash", "--login", "-c"] + +CMD ["/bin/bash"] \ No newline at end of file diff --git a/dockerfiles/vcpkg/ubuntu-22.04.dockerfile b/dockerfiles/vcpkg/ubuntu-22.04.dockerfile new file mode 100644 index 00000000000..4c7fe928b5c --- /dev/null +++ b/dockerfiles/vcpkg/ubuntu-22.04.dockerfile @@ -0,0 +1,6 @@ +ARG VCPKG_REF=latest +FROM hpccsystems/platform-build-base-ubuntu-22.04:$VCPKG_REF + +ENTRYPOINT ["/bin/bash", "--login", "-c"] + +CMD ["/bin/bash"] diff --git a/dockerfiles/vcpkg/ubuntu-23.10.dockerfile b/dockerfiles/vcpkg/ubuntu-23.10.dockerfile new file mode 100644 index 00000000000..652e1e1a42e --- /dev/null +++ b/dockerfiles/vcpkg/ubuntu-23.10.dockerfile @@ -0,0 +1,6 @@ +ARG VCPKG_REF=latest +FROM hpccsystems/platform-build-base-ubuntu-23.10:$VCPKG_REF + +ENTRYPOINT ["/bin/bash", "--login", "-c"] + +CMD ["/bin/bash"] \ No newline at end of file From d3ab1c33455480cbb15da47e0c86be792843b0bc Mon Sep 17 00:00:00 2001 From: Gavin Halliday Date: Mon, 13 Nov 2023 14:14:08 +0000 Subject: [PATCH 12/16] HPCC-30767 Move created date for secrets out of the IPropertyTree Signed-off-by: Gavin Halliday --- common/remote/hooks/git/gitfile.cpp | 2 +- common/thorhelper/thorsoapcall.cpp | 2 +- ecl/hql/hqlrepository.cpp | 2 +- esp/clients/ws_dfsclient/ws_dfsclient.cpp | 2 +- esp/esdlscriptlib/esdl_script.cpp | 6 +- rtl/eclrtl/eclrtl.cpp | 2 +- system/codesigner/gpgcodesigner.cpp | 2 +- system/jlib/jptree.hpp | 7 +- system/jlib/jsecrets.cpp | 478 ++++++++---------- system/jlib/jsecrets.hpp | 4 +- .../AzureLogAnalyticsCurlClient.cpp | 2 +- .../security/LdapSecurity/ldapconnection.cpp | 4 +- .../testauthSecurity/testauthSecurity.cpp | 2 +- testing/unittests/jlibtests.cpp | 203 ++++++++ 14 files changed, 443 insertions(+), 275 deletions(-) diff --git a/common/remote/hooks/git/gitfile.cpp b/common/remote/hooks/git/gitfile.cpp index ccebaddb28b..1a3e7d14ebe 100644 --- a/common/remote/hooks/git/gitfile.cpp +++ b/common/remote/hooks/git/gitfile.cpp @@ -244,7 +244,7 @@ class GitRepositoryFileIO : implements CSimpleInterfaceOf addPathSepChar(scriptPath).append("bin/hpccaskpass.sh"); env.emplace_back("GIT_ASKPASS", scriptPath); - Owned secret = getSecret("git", gitUser); + Owned secret = getSecret("git", gitUser); if (secret) { MemoryBuffer gitKey; diff --git a/common/thorhelper/thorsoapcall.cpp b/common/thorhelper/thorsoapcall.cpp index 98df33c7a32..b4a0d56976e 100644 --- a/common/thorhelper/thorsoapcall.cpp +++ b/common/thorhelper/thorsoapcall.cpp @@ -878,7 +878,7 @@ class CWSCHelperThread : public Thread bool loadConnectSecret(const char *vaultId, const char *secretName, UrlArray &urlArray, StringBuffer &issuer, StringBuffer &proxyAddress, bool required, WSCType wscType) { - Owned secret; + Owned secret; if (!isEmptyString(secretName)) secret.setown(getSecret("ecl", secretName, vaultId, nullptr)); if (!secret) diff --git a/ecl/hql/hqlrepository.cpp b/ecl/hql/hqlrepository.cpp index 4a6ad7e5b75..75ca659e302 100644 --- a/ecl/hql/hqlrepository.cpp +++ b/ecl/hql/hqlrepository.cpp @@ -959,7 +959,7 @@ unsigned EclRepositoryManager::runGitCommand(StringBuffer * output, const char * } else { - Owned secret = getSecret("git", options.gitUser.str()); + Owned secret = getSecret("git", options.gitUser.str()); if (secret) { MemoryBuffer gitKey; diff --git a/esp/clients/ws_dfsclient/ws_dfsclient.cpp b/esp/clients/ws_dfsclient/ws_dfsclient.cpp index 6a9045952f0..8270b912a8b 100644 --- a/esp/clients/ws_dfsclient/ws_dfsclient.cpp +++ b/esp/clients/ws_dfsclient/ws_dfsclient.cpp @@ -599,7 +599,7 @@ IClientWsDfs *getDfsClient(const char *serviceUrl, IUserDescriptor *userDesc) static void configureClientSSL(IEspClientRpcSettings &rpc, const char *secretName) { - Owned secretPTree = getSecret("storage", secretName); + Owned secretPTree = getSecret("storage", secretName); if (!secretPTree) throw makeStringExceptionV(-1, "secret %s.%s not found", "storage", secretName); diff --git a/esp/esdlscriptlib/esdl_script.cpp b/esp/esdlscriptlib/esdl_script.cpp index 4c4c24b7f5c..6d6427387f8 100644 --- a/esp/esdlscriptlib/esdl_script.cpp +++ b/esp/esdlscriptlib/esdl_script.cpp @@ -911,7 +911,7 @@ class CEsdlTransformOperationMySqlCall : public CEsdlTransformOperationBase recordException(ESDL_SCRIPT_MissingOperationAttr, msg.append(name)); } } - IPropertyTree *getSecretInfo(IXpathContext * sourceContext) + const IPropertyTree *getSecretInfo(IXpathContext * sourceContext) { //leaving flexibility for the secret to be configured multiple ways // the most secure option in my opinion is to at least have the server, name, and password all in the secret @@ -943,7 +943,7 @@ class CEsdlTransformOperationMySqlCall : public CEsdlTransformOperationBase options.append(name).append('=').append(value); } - void appendOption(StringBuffer &options, const char *name, IXpathContext * sourceContext, ICompiledXpath *cx, IPropertyTree *secret, bool required) + void appendOption(StringBuffer &options, const char *name, IXpathContext * sourceContext, ICompiledXpath *cx, const IPropertyTree *secret, bool required) { if (secret && secret->hasProp(name)) { @@ -971,7 +971,7 @@ class CEsdlTransformOperationMySqlCall : public CEsdlTransformOperationBase } IEmbedFunctionContext *createFunctionContext(IXpathContext * sourceContext) { - Owned secret = getSecretInfo(sourceContext); + Owned secret = getSecretInfo(sourceContext); StringBuffer options; appendOption(options, "server", sourceContext, m_server, secret, true); appendOption(options, "user", sourceContext, m_user, secret, true); diff --git a/rtl/eclrtl/eclrtl.cpp b/rtl/eclrtl/eclrtl.cpp index 7fb01efd1cf..980ebf2198e 100644 --- a/rtl/eclrtl/eclrtl.cpp +++ b/rtl/eclrtl/eclrtl.cpp @@ -6279,7 +6279,7 @@ void rtlBase64Decode(size32_t & tlen, void * & tgt, size32_t slen, const char * void rtlGetEclUserSecret(size32_t & outlen, void * & out, const char *name, const char *key) { - Owned secret = getSecret("eclUser", name); + Owned secret = getSecret("eclUser", name); if (secret) { MemoryBuffer data; diff --git a/system/codesigner/gpgcodesigner.cpp b/system/codesigner/gpgcodesigner.cpp index 8ed528fd21b..3e3c124a470 100644 --- a/system/codesigner/gpgcodesigner.cpp +++ b/system/codesigner/gpgcodesigner.cpp @@ -138,7 +138,7 @@ void GpgCodeSigner::importKeysFromSecret(const char * cat, const char *keytype) for (int keyentry = 1; ; keyentry++) { VStringBuffer keysecretname("gpg-%s-key-%d", keytype, keyentry); - Owned secretKey = getSecret(cat, keysecretname.str()); + Owned secretKey = getSecret(cat, keysecretname.str()); if (secretKey) { StringBuffer gpgKey; diff --git a/system/jlib/jptree.hpp b/system/jlib/jptree.hpp index 80888c8218d..e52db5eec1a 100644 --- a/system/jlib/jptree.hpp +++ b/system/jlib/jptree.hpp @@ -438,11 +438,14 @@ extern jlib_decl unsigned getPropertyTreeHash(const IPropertyTree & source, unsi //to not be modified and to remain valid and consistent until it is released. interface ISyncedPropertyTree : extends IInterface { +//The following functions check whether something is up to date before returning their values. + //Return a version-hash which changes whenever the property tree changes - so that a caller can determine whether it needs to update + virtual unsigned getVersion() const = 0; virtual const IPropertyTree * getTree() const = 0; virtual bool getProp(MemoryBuffer & result, const char * xpath) const = 0; virtual bool getProp(StringBuffer & result, const char * xpath) const = 0; - //Return a version-hash which changes whenever the property tree changes - so that a caller can determine whether it needs to update - virtual unsigned getVersion() const = 0; + +// The following functions return the current cached state - they do not force a check to see if the value is up to date virtual bool isStale() const = 0; // An indication that the property tree may be out of date because it couldn't be resynchronized. virtual bool isValid() const = 0; // Is the property tree non-null? Typically called at startup to check configuration is provided. }; diff --git a/system/jlib/jsecrets.cpp b/system/jlib/jsecrets.cpp index 5951ad6fbb7..322c6c12f73 100644 --- a/system/jlib/jsecrets.cpp +++ b/system/jlib/jsecrets.cpp @@ -64,36 +64,19 @@ CVaultKind getSecretType(const char *s) } interface IVaultManager : extends IInterface { - virtual bool getCachedSecretFromVault(const char *category, const char *vaultId, CVaultKind &kind, StringBuffer &content, const char *secret, const char *version) = 0; virtual bool requestSecretFromVault(const char *category, const char *vaultId, CVaultKind &kind, StringBuffer &content, const char *secret, const char *version) = 0; - virtual bool getCachedSecretByCategory(const char *category, CVaultKind &kind, StringBuffer &content, const char *secret, const char *version) = 0; virtual bool requestSecretByCategory(const char *category, CVaultKind &kind, StringBuffer &content, const char *secret, const char *version) = 0; }; -static CriticalSection secretCacheCS; -static Owned secretCache; -static CriticalSection mtlsInfoCacheCS; -static std::unordered_map> mtlsInfoCache; static Owned vaultManager; static MemoryAttr udpKey; static bool udpKeyInitialized = false; -MODULE_INIT(INIT_PRIORITY_SYSTEM) +static const IPropertyTree *getLocalSecret(const char *category, const char * name) { - secretCache.setown(createPTree()); - return true; + return getSecret(category, name, "k8s", nullptr); } -MODULE_EXIT() -{ - vaultManager.clear(); - secretCache.clear(); - mtlsInfoCache.clear(); - udpKey.clear(); -} - -static IPropertyTree *getLocalSecret(const char *category, const char * name); - //based on kubernetes secret / key names. Even if some vault backends support additional characters we'll restrict to this subset for now static const char *validSecretNameChrs = ".-"; @@ -319,8 +302,8 @@ extern jlib_decl StringBuffer &generateDynamicUrlSecretName(StringBuffer &secret unsigned portNum = port.length() ? atoi(port) : 0; return generateDynamicUrlSecretName(secretName, scheme, username, host, portNum, path); } -//--------------------------------------------------------------------------------------------------------------------- +//--------------------------------------------------------------------------------------------------------------------- static StringBuffer secretDirectory; static CriticalSection secretCS; @@ -364,18 +347,6 @@ static StringBuffer &buildSecretPath(StringBuffer &path, const char *category, c return addPathSepChar(path.append(ensureSecretDirectory())).append(category).append(PATHSEPCHAR).append(name).append(PATHSEPCHAR); } -static bool checkSecretExpired(unsigned created) -{ - if (!created) - return false; - unsigned age = msTick() - created; - return age > getSecretTimeout(); -} - -static bool hasCacheExpired(const IPropertyTree * secret) -{ - return checkSecretExpired((unsigned)secret->getPropInt("@created")); -} enum class VaultAuthType {unknown, k8s, appRole, token, clientcert}; @@ -395,6 +366,134 @@ static bool isEmptyTimeval(const timeval &tv) return (tv.tv_sec==0 && tv.tv_usec==0); } +//--------------------------------------------------------------------------------------------------------------------- + +//Represents an entry in the secret cache. Once created it is always used for the secret. +using cache_timestamp = unsigned; +class SecretCacheEntry : public CInterface +{ + friend class SecretCache; + +public: + //A cache entry is initally created that has a create and access time of now, but the checkTimestamp + //is set so that needsRefresh() will return true. + SecretCacheEntry(cache_timestamp _now) + : contentTimestamp(_now), accessedTimestamp(_now), checkedTimestamp(_now - 2 * secretTimeoutMs) + { + } + + unsigned getHash() const + { + return contentHash; + } + + // We should never replace known contents for unknown contents + // so once this returns true it should always return true + bool hasContents() const + { + return contents != nullptr; + } + + //Is the secret potentially out of date? + bool isStale() const + { + cache_timestamp now = msTick(); + cache_timestamp elapsed = (now - contentTimestamp); + return (elapsed > secretTimeoutMs); + } + + // Is it time to check if there is a new value for this secret? + bool needsRefresh(cache_timestamp now) const + { + cache_timestamp elapsed = (now - checkedTimestamp); + return (elapsed > secretTimeoutMs); + } + + bool needsRefresh() const + { + return needsRefresh(msTick()); + } + + void noteFailedUpdate(cache_timestamp now) + { + //Update the checked timestamp - so that we do not continually check for updates to secrets which + //are stale because the vault or other source of values in inaccessible. + //Keep using the last good value + checkedTimestamp = now; + } + + //The following functions can only be called from member functions of SecretCache +private: + void updateContents(IPropertyTree * _contents, cache_timestamp now) + { + contents.set(_contents); + updateHash(); + contentTimestamp = now; + accessedTimestamp = now; + checkedTimestamp = now; + } + + void updateHash() + { + if (contents) + contentHash = getPropertyTreeHash(*contents, 0x811C9DC5); + else + contentHash = 0; + } +private: + Linked contents;// Can only be accessed when SecretCache::cs is held + cache_timestamp contentTimestamp = 0; // When was this secret read from disk/vault + cache_timestamp accessedTimestamp = 0; // When was this secret last accessed? + cache_timestamp checkedTimestamp = 0; // When was this last checked for updates? + unsigned contentHash = 0; +}; + +// A cache of (secret[:version] to a secret cache entry) +// Once a hash table entry has been created for a secret it is never removed and the associated +// value is never replaced. This means it is safe to keep a pointer to the entry in another class. +class SecretCache +{ +public: + const IPropertyTree * getContents(SecretCacheEntry * match) + { + //Return contents within the critical section so no other thread can modify it + CriticalBlock block(cs); + return LINK(match->contents); + } + + //Check to see if a secret exists, and if not add a null entry that has expired. + SecretCacheEntry * resolveSecret(const std::string & secretKey, cache_timestamp now) + { + SecretCacheEntry * result; + CriticalBlock block(cs); + auto match = secrets.find(secretKey); + if (match != secrets.cend()) + { + result = match->second.get(); + result->accessedTimestamp = now; + } + else + { + //Insert an entry with a null value that is marked as out of date + result = new SecretCacheEntry(now); + secrets.emplace(secretKey, result); + } + return result; + } + + void updateSecret(SecretCacheEntry * match, IPropertyTree * value, cache_timestamp now) + { + CriticalBlock block(cs); + match->updateContents(value, now); + } + +private: + CriticalSection cs; + std::unordered_map> secrets; +}; + +//--------------------------------------------------------------------------------------------------------------------- + class CVault { private: @@ -402,7 +501,6 @@ class CVault CVaultKind kind; CriticalSection vaultCS; - Owned cache; std::string clientCertPath; std::string clientKeyPath; @@ -445,7 +543,6 @@ class CVault if (!checkFileExists(clientKeyPath.c_str())) WARNLOG("vault: client key not found, %s", clientKeyPath.c_str()); - cache.setown(createPTree()); StringBuffer url; replaceEnvVariables(url, vault->queryProp("@url"), false); PROGLOG("vault url %s", url.str()); @@ -486,7 +583,7 @@ class CVault } else if (vault->hasProp("@client-secret")) { - Owned clientSecret = getLocalSecret("system", vault->queryProp("@client-secret")); + Owned clientSecret = getLocalSecret("system", vault->queryProp("@client-secret")); if (clientSecret) { StringBuffer tokenText; @@ -672,7 +769,7 @@ class CVault return; DBGLOG("appRoleLogin%s", permissionDenied ? " because existing token permission denied" : ""); StringBuffer appRoleSecretId; - Owned appRoleSecret = getLocalSecret("system", appRoleSecretName); + Owned appRoleSecret = getLocalSecret("system", appRoleSecretName); if (!appRoleSecret) vaultAuthErrorV("appRole secret %s not found", appRoleSecretName.str()); else if (!getSecretKeyValue(appRoleSecretId, appRoleSecret, "secret-id")) @@ -712,42 +809,6 @@ class CVault if (clientToken.isEmpty()) vaultAuthError("no vault access token"); } - bool getCachedSecret(CVaultKind &rkind, StringBuffer &content, const char *secret, const char *version) - { - CriticalBlock block(vaultCS); - IPropertyTree *tree = cache->queryPropTree(secret); - if (tree) - { - VStringBuffer vername("v.%s", isEmptyString(version) ? "latest" : version); - IPropertyTree *envelope = tree->queryPropTree(vername); - if (!envelope) - return false; - if (hasCacheExpired(envelope)) - { - tree->removeTree(envelope); - return false; - } - const char *s = envelope->queryProp(""); - rkind = kind; - if (!isEmptyString(s)) - content.append(s); - return true; - } - return false; - } - void addCachedSecret(const char *content, const char *secret, const char *version) - { - VStringBuffer vername("v.%s", isEmptyString(version) ? "latest" : version); - Owned envelope = createPTree(vername); - envelope->setPropInt("@created", (int) msTick()); - if (!isEmptyString(content)) - envelope->setProp("", content); - { - CriticalBlock block(vaultCS); - IPropertyTree *parent = ensurePTree(cache, secret); - parent->setPropTree(vername, envelope.getClear()); - } - } bool requestSecretAtLocation(CVaultKind &rkind, StringBuffer &content, const char *location, const char *secretCacheKey, const char *version, bool permissionDenied) { checkAuthentication(permissionDenied); @@ -779,7 +840,6 @@ class CVault { rkind = kind; content.append(res->body.c_str()); - addCachedSecret(content.str(), secretCacheKey, version); return true; } else if (res->status == 403) @@ -801,7 +861,6 @@ class CVault else OERRLOG("Error: Vault %s http error (%d) accessing secret %s.%s location %s", name.str(), res.error(), secretCacheKey, version ? version : "", location); - addCachedSecret("", secretCacheKey, version); //cache misses so we don't keep calling the vault return false; } bool requestSecret(CVaultKind &rkind, StringBuffer &content, const char *secret, const char *version) @@ -831,16 +890,6 @@ class CVaultSet if (!isEmptyString(name)) vaults.emplace(name, std::unique_ptr(new CVault(vault))); } - bool getCachedSecret(CVaultKind &kind, StringBuffer &content, const char *secret, const char *version) - { - auto it = vaults.begin(); - for (; it != vaults.end(); it++) - { - if (it->second->getCachedSecret(kind, content, secret, version)) - return true; - } - return false; - } bool requestSecret(CVaultKind &kind, StringBuffer &content, const char *secret, const char *version) { auto it = vaults.begin(); @@ -851,15 +900,6 @@ class CVaultSet } return false; } - bool getCachedSecretFromVault(const char *vaultId, CVaultKind &kind, StringBuffer &content, const char *secret, const char *version) - { - if (isEmptyString(vaultId)) - return false; - auto it = vaults.find(vaultId); - if (it == vaults.end()) - return false; - return it->second->getCachedSecret(kind, content, secret, version); - } bool requestSecretFromVault(const char *vaultId, CVaultKind &kind, StringBuffer &content, const char *secret, const char *version) { if (isEmptyString(vaultId)) @@ -906,15 +946,6 @@ class CVaultManager : public CInterfaceOf it->second->addVault(&vault); } } - bool getCachedSecretFromVault(const char *category, const char *vaultId, CVaultKind &kind, StringBuffer &content, const char *secret, const char *version) override - { - if (isEmptyString(category)) - return false; - auto it = categories.find(category); - if (it == categories.end()) - return false; - return it->second->getCachedSecretFromVault(vaultId, kind, content, secret, version); - } bool requestSecretFromVault(const char *category, const char *vaultId, CVaultKind &kind, StringBuffer &content, const char *secret, const char *version) override { if (isEmptyString(category)) @@ -925,15 +956,6 @@ class CVaultManager : public CInterfaceOf return it->second->requestSecretFromVault(vaultId, kind, content, secret, version); } - bool getCachedSecretByCategory(const char *category, CVaultKind &kind, StringBuffer &content, const char *secret, const char *version) override - { - if (isEmptyString(category)) - return false; - auto it = categories.find(category); - if (it == categories.end()) - return false; - return it->second->getCachedSecret(kind, content, secret, version); - } bool requestSecretByCategory(const char *category, CVaultKind &kind, StringBuffer &content, const char *secret, const char *version) override { if (isEmptyString(category)) @@ -953,56 +975,34 @@ IVaultManager *ensureVaultManager() return vaultManager; } -static IPropertyTree *getCachedLocalSecret(const char *category, const char *name, bool &cachedMiss) +//--------------------------------------------------------------------------------------------------------------------- + +static SecretCache globalSecretCache; +static CriticalSection mtlsInfoCacheCS; +static std::unordered_map> mtlsInfoCache; + +MODULE_INIT(INIT_PRIORITY_SYSTEM) { - if (isEmptyString(name)) - return nullptr; - Owned secret; - { - CriticalBlock block(secretCacheCS); - IPropertyTree *tree = secretCache->queryPropTree(category); - if (!tree) - return nullptr; - secret.setown(tree->getPropTree(name)); - if (secret) - { - if (hasCacheExpired(secret)) - { - secretCache->removeProp(name); - return nullptr; - } - if (secret->hasProp("@miss")) - { - cachedMiss = true; - return nullptr; - } - return secret.getClear(); - } - } - return nullptr; + return true; } -static void addCachedLocalSecret(const char *category, const char *name, IPropertyTree *secret) +MODULE_EXIT() { - if (!secret || isEmptyString(name) || isEmptyString(category)) - return; - secret->setPropInt("@created", (int)msTick()); - { - CriticalBlock block(secretCacheCS); - IPropertyTree *tree = ensurePTree(secretCache, category); - tree->setPropTree(name, LINK(secret)); - } + vaultManager.clear(); + udpKey.clear(); } -static IPropertyTree *loadLocalSecret(const char *category, const char * name) + +static IPropertyTree * resolveLocalSecret(const char *category, const char * name) { StringBuffer path; buildSecretPath(path, category, name); + Owned entries = createDirectoryIterator(path); if (!entries || !entries->first()) return nullptr; - Owned tree = createPTree(name); - tree->setPropInt("@created", (int) msTick()); + + Owned tree(createPTree(name)); ForEach(*entries) { if (entries->isDir()) @@ -1018,22 +1018,8 @@ static IPropertyTree *loadLocalSecret(const char *category, const char * name) continue; tree->setPropBin(name, content.length(), content.bufferBase()); } - addCachedLocalSecret(category, name, tree); - return tree.getClear(); -} - -static IPropertyTree *getLocalSecret(const char *category, const char * name) -{ - validateCategoryName(category); - validateSecretName(name); - bool skipLocalFetch = false; - Owned tree = getCachedLocalSecret(category, name, skipLocalFetch); - if (skipLocalFetch) - return nullptr; - if (tree) - return tree.getClear(); - return loadLocalSecret(category, name); + return tree.getClear(); } static IPropertyTree *createPTreeFromVaultSecret(const char *content, CVaultKind kind) @@ -1056,30 +1042,8 @@ static IPropertyTree *createPTreeFromVaultSecret(const char *content, CVaultKind } return tree.getClear(); } -static IPropertyTree *getCachedVaultSecret(const char *category, const char *vaultId, const char * name, const char *version, bool &cachedMiss) -{ - CVaultKind kind; - StringBuffer json; - IVaultManager *vaultmgr = ensureVaultManager(); - if (isEmptyString(vaultId)) - { - if (!vaultmgr->getCachedSecretByCategory(category, kind, json, name, version)) - return nullptr; - } - else - { - if (!vaultmgr->getCachedSecretFromVault(category, vaultId, kind, json, name, version)) - return nullptr; - } - if (json.isEmpty()) - { - cachedMiss = true; - return nullptr; - } - return createPTreeFromVaultSecret(json.str(), kind); -} -static IPropertyTree *requestVaultSecret(const char *category, const char *vaultId, const char * name, const char *version) +static IPropertyTree *resolveVaultSecret(const char *category, const char * name, const char *vaultId, const char *version) { CVaultKind kind; StringBuffer json; @@ -1097,55 +1061,58 @@ static IPropertyTree *requestVaultSecret(const char *category, const char *vault return createPTreeFromVaultSecret(json.str(), kind); } -static IPropertyTree *getVaultSecret(const char *category, const char * name, const char *vaultId, const char *version) + +static SecretCacheEntry * getSecretEntry(const char *category, const char * name, const char * optVaultId, const char * optVersion) { - CVaultKind kind; - StringBuffer json; - IVaultManager *vaultmgr = ensureVaultManager(); + cache_timestamp now = msTick(); - bool cachedMiss = false; + std::string key; + key.append(category).append("/").append(name); + if (optVaultId) + key.append("@").append(optVaultId); + if (optVersion) + key.append("#").append(optVersion); - if (isEmptyString(vaultId)) + SecretCacheEntry * match = globalSecretCache.resolveSecret(key, now); + if (!match->needsRefresh(now)) + return match; + + Owned resolved; + if (!isEmptyString(optVaultId)) { - if (vaultmgr->getCachedSecretByCategory(category, kind, json, name, version)) - cachedMiss = json.isEmpty(); + if (strieq(optVaultId, "k8s")) + resolved.setown(resolveLocalSecret(category, name)); else - vaultmgr->requestSecretByCategory(category, kind, json, name, version); + resolved.setown(resolveVaultSecret(category, name, optVaultId, optVersion)); } else { - if (!vaultmgr->getCachedSecretFromVault(category, vaultId, kind, json, name, version)) - cachedMiss = json.isEmpty(); - else - vaultmgr->requestSecretFromVault(category, vaultId, kind, json, name, version); + resolved.setown(resolveLocalSecret(category, name)); + if (!resolved) + resolved.setown(resolveVaultSecret(category, name, nullptr, optVersion)); } - if (cachedMiss) - return nullptr; - return createPTreeFromVaultSecret(json.str(), kind); + + //If the secret could no longer be resolved (e.g. a vault has gone down) then keep the old one + if (resolved) + globalSecretCache.updateSecret(match, resolved, now); + else + match->noteFailedUpdate(now); + + return match; } -IPropertyTree *getSecretTree(const char *category, const char * name, const char * optVaultId, const char * optVersion) +static const IPropertyTree *getSecretTree(const char *category, const char * name, const char * optVaultId, const char * optVersion) { - if (!isEmptyString(optVaultId)) - return getVaultSecret(category, name, optVaultId, optVersion); + SecretCacheEntry * secret = getSecretEntry(category, name, optVaultId, optVersion); + if (secret) + return globalSecretCache.getContents(secret); + return nullptr; +} - //if we get back a null secret, it might be a cached miss, so don't go to the source if flag gets set - bool skipVaultFetch = false; - bool skipLocalFetch = false; - //check for any chached first - Owned secret = getCachedLocalSecret(category, name, skipLocalFetch); - if (!secret) - secret.setown(getCachedVaultSecret(category, nullptr, name, nullptr, skipVaultFetch)); - //now check local, then vaults - if (!secret && !skipLocalFetch) - secret.setown(loadLocalSecret(category, name)); - if (!secret && !skipVaultFetch) - secret.setown(requestVaultSecret(category, nullptr, name, nullptr)); - return secret.getClear(); -} +//Public interface to the secrets -IPropertyTree *getSecret(const char *category, const char * name, const char * optVaultId, const char * optVersion) +const IPropertyTree *getSecret(const char *category, const char * name, const char * optVaultId, const char * optVersion) { validateCategoryName(category); validateSecretName(name); @@ -1172,7 +1139,7 @@ bool getSecretKeyValue(StringBuffer & result, const IPropertyTree *secret, const extern jlib_decl bool getSecretValue(StringBuffer & result, const char *category, const char * name, const char * key, bool required) { - Owned secret = getSecret(category, name); + Owned secret = getSecret(category, name); if (required && !secret) throw MakeStringException(-1, "secret %s.%s not found", category, name); bool found = getSecretKeyValue(result, secret, key); @@ -1186,10 +1153,9 @@ extern jlib_decl bool getSecretValue(StringBuffer & result, const char *category class CSecret final : public CInterfaceOf { public: - CSecret(const char *_category, const char * _name, const char * _vaultId, const char * _version, const IPropertyTree * _secret) + CSecret(const char *_category, const char * _name, const char * _vaultId, const char * _version, SecretCacheEntry * _secret) : category(_category), name(_name), vaultId(_vaultId), version(_version), secret(_secret) { - updateHash(); } virtual const IPropertyTree * getTree() const override; @@ -1197,34 +1163,34 @@ class CSecret final : public CInterfaceOf virtual bool getProp(MemoryBuffer & result, const char * key) const override { CriticalBlock block(secretCs); - checkStale(); - return getSecretKeyValue(result, secret, key); + checkUptoDate(); + Owned contents = globalSecretCache.getContents(secret); + return getSecretKeyValue(result, contents, key); } virtual bool getProp(StringBuffer & result, const char * key) const override { CriticalBlock block(secretCs); - checkStale(); - return getSecretKeyValue(result, secret, key); + checkUptoDate(); + Owned contents = globalSecretCache.getContents(secret); + return getSecretKeyValue(result, contents, key); } virtual bool isStale() const override { - return secret && hasCacheExpired(secret); + return secret->isStale(); } virtual unsigned getVersion() const override { CriticalBlock block(secretCs); - checkStale(); - return secretHash; + checkUptoDate(); + return secret->getHash(); } virtual bool isValid() const override { - CriticalBlock block(secretCs); - return secret != nullptr; + return secret->hasContents(); } protected: - void checkStale() const; - void updateHash() const; + void checkUptoDate() const; protected: StringAttr category; @@ -1232,21 +1198,20 @@ class CSecret final : public CInterfaceOf StringAttr vaultId; StringAttr version; mutable CriticalSection secretCs; - mutable Linked secret; - mutable unsigned secretHash = 0; + mutable SecretCacheEntry * secret; }; const IPropertyTree * CSecret::getTree() const { CriticalBlock block(secretCs); - checkStale(); - return LINK(secret); + checkUptoDate(); + return globalSecretCache.getContents(secret); } -void CSecret::checkStale() const +void CSecret::checkUptoDate() const { - if (isStale()) + if (secret->needsRefresh()) { #ifdef TRACE_SECRETS DBGLOG("Secret %s/%s is stale updating from %u...", category.str(), name.str(), secretHash); @@ -1254,8 +1219,10 @@ void CSecret::checkStale() const //MORE: This could block or fail - in roxie especially it would be better to return the old value try { - secret.setown(getSecretTree(category, name, vaultId, version)); - updateHash(); + SecretCacheEntry * newSecret = getSecretEntry(category, name, vaultId, version); + //Check the secret is always returned consistently. It would be possible to call a slightly + //more optimal function to refresh a secret, but this is simplest. + assertex(secret == newSecret); } catch (IException * e) { @@ -1266,17 +1233,12 @@ void CSecret::checkStale() const } } -void CSecret::updateHash() const -{ - if (secret) - secretHash = getPropertyTreeHash(*secret.get(), 0x811C9DC5); - else - secretHash = 0; -} - ISyncedPropertyTree * resolveSecret(const char *category, const char * name, const char * optVaultId, const char * optVersion) { - Owned resolved = getSecret(category, name, optVaultId, optVersion); + validateCategoryName(category); + validateSecretName(name); + + SecretCacheEntry * resolved = getSecretEntry(category, name, optVaultId, optVersion); return new CSecret(category, name, optVaultId, optVersion, resolved); } @@ -1354,13 +1316,13 @@ class CSyncedCertificateBase : public CInterfaceOf virtual bool getProp(MemoryBuffer & result, const char * key) const override final { CriticalBlock block(secretCs); - checkStale(); + checkUptoDate(); return getSecretKeyValue(result, config, key); } virtual bool getProp(StringBuffer & result, const char * key) const override final { CriticalBlock block(secretCs); - checkStale(); + checkUptoDate(); return getSecretKeyValue(result, config, key); } virtual bool isStale() const override final @@ -1375,7 +1337,7 @@ class CSyncedCertificateBase : public CInterfaceOf virtual unsigned getVersion() const override final { CriticalBlock block(secretCs); - checkStale(); + checkUptoDate(); //If information that is combined with the secret (e.g. trusted peers) can also change dynamically this would //need to be a separate hash calculated from the config tree return secretHash; @@ -1385,7 +1347,7 @@ class CSyncedCertificateBase : public CInterfaceOf virtual void updateConfigFromSecret(const IPropertyTree * secretInfo) const = 0; protected: - void checkStale() const; + void checkUptoDate() const; void createConfig() const; void createDefaultConfigFromSecret(const IPropertyTree * secretInfo, bool addCertificates, bool addCertificateAuthority) const; void updateCertificateFromSecret(const IPropertyTree * secretInfo) const; @@ -1403,11 +1365,11 @@ class CSyncedCertificateBase : public CInterfaceOf const IPropertyTree * CSyncedCertificateBase::getTree() const { CriticalBlock block(secretCs); - checkStale(); + checkUptoDate(); return LINK(config); } -void CSyncedCertificateBase::checkStale() const +void CSyncedCertificateBase::checkUptoDate() const { if (secretHash != secret->getVersion()) createConfig(); diff --git a/system/jlib/jsecrets.hpp b/system/jlib/jsecrets.hpp index 9601deb4d7d..2d7f1c286be 100644 --- a/system/jlib/jsecrets.hpp +++ b/system/jlib/jsecrets.hpp @@ -28,9 +28,9 @@ extern jlib_decl void setSecretMount(const char * path); extern jlib_decl void setSecretTimeout(unsigned timeoutMs); //Return the current (cached) value of a secret. If the secret is not defined, return nullptr. -extern jlib_decl IPropertyTree *getSecret(const char *category, const char * name, const char * optVaultId = nullptr, const char * optVersion = nullptr); +extern jlib_decl const IPropertyTree *getSecret(const char *category, const char * name, const char * optVaultId = nullptr, const char * optVersion = nullptr); // resolveSecret() always returns an object, which will potentially be updated behind the scenes. If no secret is originally -// defined, but it then configured in a vault or Kubernetes secret, it will be bicked up when the cache entry is +// defined, but it then added to a vault or Kubernetes secret, it will then be picked up when the cache entry is // refreshed - allowing missing configuration to be updated for a live system. extern jlib_decl ISyncedPropertyTree * resolveSecret(const char *category, const char * name, const char * optRequiredVault, const char* optVersion); diff --git a/system/logaccess/Azure/LogAnalytics/CurlClient/AzureLogAnalyticsCurlClient.cpp b/system/logaccess/Azure/LogAnalytics/CurlClient/AzureLogAnalyticsCurlClient.cpp index fad1077b23d..cd02bcceb9a 100644 --- a/system/logaccess/Azure/LogAnalytics/CurlClient/AzureLogAnalyticsCurlClient.cpp +++ b/system/logaccess/Azure/LogAnalytics/CurlClient/AzureLogAnalyticsCurlClient.cpp @@ -286,7 +286,7 @@ AzureLogAnalyticsCurlClient::AzureLogAnalyticsCurlClient(IPropertyTree & logAcce { PROGLOG("%s: Resolving all required configuration values...", COMPONENT_NAME); - Owned secretTree = getSecret(azureLogAccessSecretCategory, azureLogAccessSecretName); + Owned secretTree = getSecret(azureLogAccessSecretCategory, azureLogAccessSecretName); if (!secretTree) throw makeStringExceptionV(-1, "%s: Could not fetch %s information!", COMPONENT_NAME, azureLogAccessSecretName); diff --git a/system/security/LdapSecurity/ldapconnection.cpp b/system/security/LdapSecurity/ldapconnection.cpp index 3b52cf7ea4e..3c55315f9a6 100644 --- a/system/security/LdapSecurity/ldapconnection.cpp +++ b/system/security/LdapSecurity/ldapconnection.cpp @@ -409,7 +409,7 @@ class CLdapConfig : implements ILdapConfig, public CInterface DBGLOG("Retrieving LDAP Admin username/password from secrets repo: %s %s", !vaultId.isEmpty() ? vaultId.str() : "", adminUserSecretKey.str()); - Owned secretTree(getSecret("authn", adminUserSecretKey.str(), vaultId, nullptr)); + Owned secretTree(getSecret("authn", adminUserSecretKey.str(), vaultId, nullptr)); if (!secretTree) throw MakeStringException(-1, "Error retrieving LDAP Admin username/password"); @@ -494,7 +494,7 @@ class CLdapConfig : implements ILdapConfig, public CInterface cfg->getProp(".//@hpccAdminVaultId", vaultId);//optional HashiCorp vault ID DBGLOG("Retrieving optional HPCC Admin username/password from secrets repo: %s %s", !vaultId.isEmpty() ? vaultId.str() : "", adminUserSecretKey.str()); - Owned secretTree(getSecret("authn", adminUserSecretKey.str(), vaultId, nullptr)); + Owned secretTree(getSecret("authn", adminUserSecretKey.str(), vaultId, nullptr)); if (secretTree) { getSecretKeyValue(m_HPCCAdminUser_username, secretTree, "username"); diff --git a/system/security/plugins/testauthSecurity/testauthSecurity.cpp b/system/security/plugins/testauthSecurity/testauthSecurity.cpp index 7da09257991..feb800cbf8f 100644 --- a/system/security/plugins/testauthSecurity/testauthSecurity.cpp +++ b/system/security/plugins/testauthSecurity/testauthSecurity.cpp @@ -230,7 +230,7 @@ class CTestAuthSecurityManager : public CBaseSecurityManager const char* secretKey = userSettings.queryProp("@secretKey"); if (!isEmptyString(secretKey)) { - Owned secretTree = getSecret("authn", secretKey); + Owned secretTree = getSecret("authn", secretKey); if (!secretTree) throw makeStringExceptionV(-1, "Error retrieving the secret for %s.", secretKey); diff --git a/testing/unittests/jlibtests.cpp b/testing/unittests/jlibtests.cpp index 6a074989d43..b925505cba7 100644 --- a/testing/unittests/jlibtests.cpp +++ b/testing/unittests/jlibtests.cpp @@ -32,11 +32,14 @@ #include "jlzw.hpp" #include "jqueue.hpp" #include "jregexp.hpp" +#include "jsecrets.hpp" #include "jutil.hpp" #include "junicode.hpp" #include "unittests.hpp" +#define CPPUNIT_ASSERT_EQUAL_STR(x, y) CPPUNIT_ASSERT_EQUAL(std::string(x),std::string(y)) + static const unsigned oneMinute = 60000; // msec class JlibTraceTest : public CppUnit::TestFixture @@ -3611,4 +3614,204 @@ CPPUNIT_TEST_SUITE_REGISTRATION( JLibUnicodeTest ); CPPUNIT_TEST_SUITE_NAMED_REGISTRATION( JLibUnicodeTest, "JLibUnicodeTest" ); +class JLibSecretsTest : public CppUnit::TestFixture +{ +public: + CPPUNIT_TEST_SUITE(JLibSecretsTest); + CPPUNIT_TEST(setup); + CPPUNIT_TEST(testUpdate1); + CPPUNIT_TEST(testUpdate2); + CPPUNIT_TEST_SUITE_END(); + + //Each test creates a different instance of the class(!) so member values cannot be used to pass items + //from one test to another + StringBuffer secretRoot; + +protected: + void checkSecret(const char * secret, const char * key, const char * expectedValue) + { + Owned match = getSecret("testing", secret); + CPPUNIT_ASSERT(match); + const char * secretValue = match->queryProp(key); + if (secretValue) + { + CPPUNIT_ASSERT_EQUAL_STR(secretValue, expectedValue); + } + else + { + //IPropertyTree doesn't allow blank values, so a missing value is the same as a blank value + //We should probably revisit some day, but it is likely to break existing code if we do. + CPPUNIT_ASSERT_EQUAL_STR("", expectedValue); + } + } + + bool hasSecret(const char * name) + { + Owned match = getSecret("testing", name); + return match != nullptr; + } + + void initPath() + { + char cwd[1024]; + CPPUNIT_ASSERT(GetCurrentDirectory(1024, cwd)); + secretRoot.set(cwd).append(PATHSEPCHAR).append("unittest-secrets"); + secretRoot.append(PATHSEPCHAR).append("testing"); // catgegory + } + + void setup() + { + char cwd[1024]; + CPPUNIT_ASSERT(GetCurrentDirectory(1024, cwd)); + secretRoot.append(cwd).append(PATHSEPCHAR).append("unittest-secrets"); + + recursiveRemoveDirectory(secretRoot); + CPPUNIT_ASSERT(recursiveCreateDirectory(secretRoot.str())); + setSecretMount(secretRoot); + setSecretTimeout(100); // Set the timeout so we can check it is working. + + secretRoot.append(PATHSEPCHAR).append("testing"); // catgegory + CPPUNIT_ASSERT(recursiveCreateDirectory(secretRoot.str())); + } + + void testUpdate1() + { + initPath(); // secretRoot needs to be called for each test + + CPPUNIT_ASSERT(!hasSecret("secret1")); + writeTestingSecret("secret1", "value", "secret1Value"); + //Secret should not appear yet - null should be cached. + CPPUNIT_ASSERT(!hasSecret("secret1")); + + Owned secret2 = resolveSecret("testing", "secret2", nullptr, nullptr); + CPPUNIT_ASSERT(!secret2->isValid()); + CPPUNIT_ASSERT(!secret2->isStale()); + + MilliSleep(50); + //Secret should not appear yet - null should be cached. + CPPUNIT_ASSERT(!hasSecret("secret1")); + CPPUNIT_ASSERT(!secret2->isValid()); + CPPUNIT_ASSERT(!secret2->isStale()); + + MilliSleep(100); + //Secret1 should now be updated - enough time has passed + checkSecret("secret1", "value", "secret1Value"); + CPPUNIT_ASSERT(!secret2->isValid()); + CPPUNIT_ASSERT(secret2->isStale()); + + //Cleanup + writeTestingSecret("secret1", "value", nullptr); + } + + void testUpdate2() + { + initPath(); // secretRoot needs to be called for each test + + Owned secret3 = resolveSecret("testing", "secret3", nullptr, nullptr); + unsigned version = secret3->getVersion(); + CPPUNIT_ASSERT(!secret3->isValid()); + CPPUNIT_ASSERT(!secret3->isStale()); + writeTestingSecret("secret3", "value", "secret3Value"); + CPPUNIT_ASSERT(!secret3->isValid()); + CPPUNIT_ASSERT_EQUAL(version, secret3->getVersion()); + + //After sleep new value should not have been picked up + MilliSleep(50); + CPPUNIT_ASSERT(!secret3->isValid()); + CPPUNIT_ASSERT_EQUAL(version, secret3->getVersion()); + + //After sleep new value should now have been picked up + MilliSleep(100); + checkSecret("secret3", "value", "secret3Value"); + unsigned version2 = secret3->getVersion(); + CPPUNIT_ASSERT(!secret3->isStale()); + CPPUNIT_ASSERT(secret3->isValid()); + CPPUNIT_ASSERT(version != version2); + + //Sleep and check that the hash value has not changed + MilliSleep(200); + checkSecret("secret3", "value", "secret3Value"); + CPPUNIT_ASSERT(!secret3->isStale()); + CPPUNIT_ASSERT(secret3->isValid()); + CPPUNIT_ASSERT_EQUAL(version2, secret3->getVersion()); + + //Remove the secret - should have no immediate effect + writeTestingSecret("secret3", "value", nullptr); + CPPUNIT_ASSERT(!secret3->isStale()); + CPPUNIT_ASSERT(secret3->isValid()); + CPPUNIT_ASSERT_EQUAL(version2, secret3->getVersion()); + + MilliSleep(50); + CPPUNIT_ASSERT(!secret3->isStale()); + CPPUNIT_ASSERT(secret3->isValid()); + CPPUNIT_ASSERT_EQUAL(version2, secret3->getVersion()); + checkSecret("secret3", "value", "secret3Value"); + + MilliSleep(100); + CPPUNIT_ASSERT(secret3->isStale()); // Value has gone, but the old value is still returned + CPPUNIT_ASSERT(secret3->isValid()); + CPPUNIT_ASSERT_EQUAL(version2, secret3->getVersion()); + checkSecret("secret3", "value", "secret3Value"); + + //Update the value = the change should not be seen until the cache entry expires + writeTestingSecret("secret3", "value", "secret3NewValue"); + CPPUNIT_ASSERT(secret3->isStale()); + CPPUNIT_ASSERT(secret3->isValid()); + CPPUNIT_ASSERT_EQUAL(version2, secret3->getVersion()); + checkSecret("secret3", "value", "secret3Value"); + + MilliSleep(50); + CPPUNIT_ASSERT(secret3->isStale()); + CPPUNIT_ASSERT(secret3->isValid()); + CPPUNIT_ASSERT_EQUAL(version2, secret3->getVersion()); + checkSecret("secret3", "value", "secret3Value"); + + MilliSleep(100); + //These functions do not check for up to date values, so they return the same as before + CPPUNIT_ASSERT(secret3->isStale()); // Value still appears to be out of date + CPPUNIT_ASSERT(secret3->isValid()); + + //The getVersion() should force the value to be updated + unsigned version3 = secret3->getVersion(); + CPPUNIT_ASSERT(version2 != version3); + CPPUNIT_ASSERT(!secret3->isStale()); // New value has now been picked up + CPPUNIT_ASSERT(secret3->isValid()); + checkSecret("secret3", "value", "secret3NewValue"); + + //Finally check that writing a blank value is spotted as a change. + writeTestingSecret("secret3", "value", ""); + MilliSleep(150); + //Check the version to ensure that the value has been updated + CPPUNIT_ASSERT(version3 != secret3->getVersion()); + CPPUNIT_ASSERT(secret3->isValid()); + checkSecret("secret3", "value", ""); + + //Cleanup + writeTestingSecret("secret3", "value", nullptr); + } + + void writeTestingSecret(const char * secret, const char * key, const char * value) + { + StringBuffer filename; + filename.append(secretRoot).append(PATHSEPCHAR).append(secret); + CPPUNIT_ASSERT(recursiveCreateDirectory(filename.str())); + + filename.append(PATHSEPCHAR).append(key); + + Owned file = createIFile(filename.str()); + if (value) + { + Owned io = file->open(IFOcreate); + io->write(0, strlen(value), value); + } + else + file->remove(); + } +}; + +CPPUNIT_TEST_SUITE_REGISTRATION( JLibSecretsTest ); +CPPUNIT_TEST_SUITE_NAMED_REGISTRATION( JLibSecretsTest, "JLibSecretsTest" ); + + + #endif // _USE_CPPUNIT From 84f16727e82d4da9721a2dbf267fe39d28c9b6e9 Mon Sep 17 00:00:00 2001 From: Gavin Halliday Date: Tue, 14 Nov 2023 16:59:46 +0000 Subject: [PATCH 13/16] HPCC-30432 Add options to control if span start/finish are logged Signed-off-by: Gavin Halliday --- helm/examples/tracing/README.md | 3 +++ helm/hpcc/values.schema.json | 12 ++++++++- system/jlib/jtrace.cpp | 45 +++++++++++++++++++++++++-------- system/jlib/jtrace.hpp | 8 +++--- 4 files changed, 53 insertions(+), 15 deletions(-) diff --git a/helm/examples/tracing/README.md b/helm/examples/tracing/README.md index bd1778a4aa5..5caba096c10 100644 --- a/helm/examples/tracing/README.md +++ b/helm/examples/tracing/README.md @@ -10,6 +10,9 @@ All configuration options detailed here are part of the HPCC Systems Helm chart, ### Tracing cofiguration options - disabled - (default: false) disables tracking and reporting of internal traces and spans - alwaysCreateGlobalIds - If true, assign newly created global ID to any requests that do not supply one. +- optAlwaysCreateTraceIds - If true components generate trace/span ids if none are provided by the remote caller. +- logSpanStart - If true, generate a log entry whenever a span is started (default: false) +- logSpanFinish - If true, generate a log entry whenever a span is finished (default: true) - exporter - Defines The type of exporter in charge of forwarding span data to target back-end - type - (defalt: NONE) "OTLP-HTTP" | "OTLP-GRCP" | "OS" | "NONE" - OTLP-HTTP diff --git a/helm/hpcc/values.schema.json b/helm/hpcc/values.schema.json index c57bd13ae45..d73687e2039 100644 --- a/helm/hpcc/values.schema.json +++ b/helm/hpcc/values.schema.json @@ -1114,7 +1114,7 @@ }, "alwaysCreateTraceIds": { "type": "boolean", - "description": "If true, components generate trace/span ids to aid in unit of worktracing" + "description": "If true, components generate trace/span ids if none are provided by the remote caller" }, "exporter": { "type": "object", @@ -1135,6 +1135,16 @@ "description": "Defines the manner in which trace data is processed - in batches, or simple as available" } } + }, + "logSpanStart": { + "type": "boolean", + "description": "If true, generate a log entry whenever a span is started", + "default": false + }, + "logSpanFinish": { + "type": "boolean", + "description": "If true, generate a log entry whenever a span is finished", + "default": true } }, "additionalProperties": { "type": ["integer", "string", "boolean"] } diff --git a/system/jlib/jtrace.cpp b/system/jlib/jtrace.cpp index 3914417ce8e..a776a5236dd 100644 --- a/system/jlib/jtrace.cpp +++ b/system/jlib/jtrace.cpp @@ -161,9 +161,12 @@ class CSpan : public CInterfaceOf virtual void beforeDispose() override { - StringBuffer out; - toLog(out); - LOG(MCmonitorEvent, "Span end: {%s}", out.str()); + if (queryTraceManager().logSpanFinish()) + { + StringBuffer out; + toLog(out); + LOG(MCmonitorEvent, "SpanFinish: {%s}", out.str()); + } } const char * getSpanID() const @@ -453,9 +456,12 @@ class CSpan : public CInterfaceOf { storeSpanContext(); - StringBuffer out; - toLog(out); - LOG(MCmonitorEvent, "Span start: {%s}", out.str()); + if (queryTraceManager().logSpanStart()) + { + StringBuffer out; + toLog(out); + LOG(MCmonitorEvent, "SpanStart: {%s}", out.str()); + } } } } @@ -755,6 +761,8 @@ class CTraceManager : implements ITraceManager, public CInterface bool enabled = true; bool optAlwaysCreateGlobalIds = false; bool optAlwaysCreateTraceIds = true; + bool optLogSpanStart = false; + bool optLogSpanFinish = true; StringAttr moduleName; //Initializes the global trace provider which is required for all Otel based tracing operations. @@ -942,6 +950,8 @@ class CTraceManager : implements ITraceManager, public CInterface //Non open-telemetry tracing configuration if (traceConfig) { + optLogSpanStart = traceConfig->getPropBool("@logSpanStart", optLogSpanStart); + optLogSpanFinish = traceConfig->getPropBool("@logSpanFinish", optLogSpanFinish); optAlwaysCreateGlobalIds = traceConfig->getPropBool("@alwaysCreateGlobalIds", optAlwaysCreateGlobalIds); optAlwaysCreateTraceIds = traceConfig->getPropBool("@alwaysCreateTraceIds", optAlwaysCreateTraceIds); } @@ -991,28 +1001,31 @@ class CTraceManager : implements ITraceManager, public CInterface throw makeStringExceptionV(-1, "TraceManager must be intialized!"); } - ISpan * createServerSpan(const char * name, StringArray & httpHeaders, SpanFlags flags) const override + virtual ISpan * createServerSpan(const char * name, StringArray & httpHeaders, SpanFlags flags) const override { Owned headerProperties = getHeadersAsProperties(httpHeaders); return new CServerSpan(name, moduleName.get(), headerProperties, flags); } - ISpan * createServerSpan(const char * name, const IProperties * httpHeaders, SpanFlags flags) const override + virtual ISpan * createServerSpan(const char * name, const IProperties * httpHeaders, SpanFlags flags) const override { return new CServerSpan(name, moduleName.get(), httpHeaders, flags); } - const char * getTracedComponentName() const override + virtual const char * getTracedComponentName() const override { return moduleName.get(); } - bool isTracingEnabled() const override + virtual bool isTracingEnabled() const override { return enabled; } - virtual bool alwaysCreateGlobalIds() const + //These functions are only accessed within this module, and should not really be in the public interface + //Something to possibly revisit later - either by passing as flags to the spans, or having a static + //function to provide the result of queryTraceManager() as a CTraceManager. + virtual bool alwaysCreateGlobalIds() const override { return optAlwaysCreateGlobalIds; } @@ -1021,6 +1034,16 @@ class CTraceManager : implements ITraceManager, public CInterface { return optAlwaysCreateTraceIds; } + + virtual bool logSpanStart() const override + { + return optLogSpanStart; + } + + virtual bool logSpanFinish() const override + { + return optLogSpanFinish; + } }; static Singleton theTraceManager; diff --git a/system/jlib/jtrace.hpp b/system/jlib/jtrace.hpp index f305fe49829..2c1234b0622 100644 --- a/system/jlib/jtrace.hpp +++ b/system/jlib/jtrace.hpp @@ -32,9 +32,9 @@ static constexpr const char *kCallerIdOtelAttributeName = "hpcc.callerid"; enum class SpanFlags : unsigned { - None = 0x000000000, - EnsureGlobalId = 0x000000001, - EnsureTraceId = 0x000000010, + None = 0x00000000, + EnsureGlobalId = 0x00000001, + EnsureTraceId = 0x00000002, }; BITMASK_ENUM(SpanFlags); @@ -66,6 +66,8 @@ interface ITraceManager : extends IInterface virtual bool isTracingEnabled() const = 0; virtual bool alwaysCreateGlobalIds() const = 0; virtual bool alwaysCreateTraceIds() const = 0; + virtual bool logSpanStart() const = 0; + virtual bool logSpanFinish() const = 0; virtual const char * getTracedComponentName() const = 0; }; From 4369b67c7ee24f94a8af03257d5b6196d62d4d76 Mon Sep 17 00:00:00 2001 From: Gavin Halliday Date: Tue, 21 Nov 2023 12:50:06 +0000 Subject: [PATCH 14/16] HPCC-30898 Resolve build errors with ubuntu 23.10 Signed-off-by: Gavin Halliday --- ecl/hql/hqlgram2.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ecl/hql/hqlgram2.cpp b/ecl/hql/hqlgram2.cpp index 8d906ad57f0..26bf61e3ff0 100644 --- a/ecl/hql/hqlgram2.cpp +++ b/ecl/hql/hqlgram2.cpp @@ -9689,11 +9689,11 @@ void HqlGram::checkDerivedCompatible(IIdAtom * name, IHqlExpression * scope, IHq if (match) { if (!canBeVirtual(match)) - reportError(ERR_MISMATCH_PROTO, errpos, "Definition %s, cannot override this kind of definition", str(name)); + reportError(ERR_MISMATCH_PROTO, errpos, "Definition %s, cannot override this kind of definition", nullText(str(name))); else { if (!areSymbolsCompatible(expr, isParametered, parameters, match)) - reportError(ERR_MISMATCH_PROTO, errpos, "Prototypes for %s in base and derived modules must match", str(name)); + reportError(ERR_MISMATCH_PROTO, errpos, "Prototypes for %s in base and derived modules must match", nullText(str(name))); } } } From 5caf5c97eb6a8820c4584e91e7419f5150115f02 Mon Sep 17 00:00:00 2001 From: Gavin Halliday Date: Tue, 21 Nov 2023 10:41:00 +0000 Subject: [PATCH 15/16] HPCC-30898 Resolve build errors with ubuntu 23.10 Signed-off-by: Gavin Halliday --- system/jlib/jsecrets.cpp | 3 +++ 1 file changed, 3 insertions(+) diff --git a/system/jlib/jsecrets.cpp b/system/jlib/jsecrets.cpp index 13d697120fe..71965327b10 100644 --- a/system/jlib/jsecrets.cpp +++ b/system/jlib/jsecrets.cpp @@ -31,6 +31,9 @@ #if defined(__clang__) || defined(__GNUC__) #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Wformat-nonliteral" + +//httplib also generates warning about access outside of array bounds in gcc +#pragma GCC diagnostic ignored "-Warray-bounds" #endif #ifdef _USE_OPENSSL From db10f484449a1014472a54339a2ee9471fb9f7af Mon Sep 17 00:00:00 2001 From: wangkx Date: Thu, 16 Nov 2023 14:24:07 -0500 Subject: [PATCH 16/16] HPCC-30685 HPCC-30685 Report State always for Scheduled ECL WUs Without this fix, the WU state is displayed inside the ECLWatch Event Scheduler tab only when a state filter is set inside the tab. Signed-off-by: wangkx --- .../ws_workunits/ws_workunitsService.cpp | 32 +++++++------------ 1 file changed, 12 insertions(+), 20 deletions(-) diff --git a/esp/services/ws_workunits/ws_workunitsService.cpp b/esp/services/ws_workunits/ws_workunitsService.cpp index 0f5efa8f24b..3450b185109 100644 --- a/esp/services/ws_workunits/ws_workunitsService.cpp +++ b/esp/services/ws_workunits/ws_workunitsService.cpp @@ -3626,6 +3626,16 @@ void getScheduledWUs(IEspContext &context, WUShowScheduledFilters *filters, cons { jobName.set(cw->queryJobName()); owner.set(cw->queryUser()); + if ((cw->getState() == WUStateScheduled) && cw->aborting()) + { + stateID = WUStateAborting; + state.set("aborting"); + } + else + { + stateID = cw->getState(); + state.set(cw->queryStateDesc()); + } } if (!filters->jobName.isEmpty() && (jobName.isEmpty() || !WildMatch(jobName.str(), filters->jobName, true))) @@ -3634,26 +3644,8 @@ void getScheduledWUs(IEspContext &context, WUShowScheduledFilters *filters, cons match = false; else if (!filters->eventText.isEmpty() && (ieventText.isEmpty() || !WildMatch(ieventText, filters->eventText, true))) match = false; - else if (!filters->state.isEmpty()) - { - if (!cw) - match = false; - else - { - if ((cw->getState() == WUStateScheduled) && cw->aborting()) - { - stateID = WUStateAborting; - state.set("aborting"); - } - else - { - stateID = cw->getState(); - state.set(cw->queryStateDesc()); - } - if (!strieq(filters->state, state.str())) - match = false; - } - } + else if (!filters->state.isEmpty() && !strisame(filters->state, state.str())) + match = false; } catch (IException *e) {