From 1b2cf13003ccf260117878e57567d6bce29f03fc Mon Sep 17 00:00:00 2001 From: wangkx Date: Mon, 11 Dec 2023 09:17:03 -0500 Subject: [PATCH 01/32] HPCC-31038 Add option for removing 'lost' file from super file In the existing code, a 'lost' file found by XRef can't be deleted if that file is a sub of a super file without manually being removed from the super file. With this option is set inside delete request, the file will be automatically removed from its owning super files before the 'lost' file is deleted. Signed-off-by: wangkx --- esp/scm/ws_dfuXref.ecm | 3 +- esp/services/ws_dfu/ws_dfuXRefService.cpp | 46 ++++++++++++++++++++++- esp/services/ws_dfu/ws_dfuXRefService.hpp | 2 + 3 files changed, 49 insertions(+), 2 deletions(-) diff --git a/esp/scm/ws_dfuXref.ecm b/esp/scm/ws_dfuXref.ecm index 6a5205aa13e..36d04d41c1e 100644 --- a/esp/scm/ws_dfuXref.ecm +++ b/esp/scm/ws_dfuXref.ecm @@ -170,6 +170,7 @@ DFUXRefArrayActionRequest string Cluster; string Action; ESParray XRefFiles; + [min_ver("1.04")] bool RemoveFromSuperfiles(false); }; ESPresponse @@ -226,7 +227,7 @@ ESPresponse [exceptions_inline] DFUXRefUnusedFilesResponse }; // =========================================================================== -ESPservice [version("1.03"), default_client_version("1.03"), auth_feature("DEFERRED"), exceptions_inline("./smc_xslt/exceptions.xslt")] WsDFUXRef +ESPservice [version("1.04"), default_client_version("1.04"), auth_feature("DEFERRED"), exceptions_inline("./smc_xslt/exceptions.xslt")] WsDFUXRef { ///ESPmethod [resp_xsl_default("./smc_xslt/xref_main.xslt")] DFUXRefList(DFUXRefListRequest, DFUXRefListResponse); ESPmethod [resp_xsl_default("/esp/xslt/xref_main.xslt")] DFUXRefList(DFUXRefListRequest, DFUXRefListResponse); diff --git a/esp/services/ws_dfu/ws_dfuXRefService.cpp b/esp/services/ws_dfu/ws_dfuXRefService.cpp index b8f169c0687..e732cbd488d 100644 --- a/esp/services/ws_dfu/ws_dfuXRefService.cpp +++ b/esp/services/ws_dfu/ws_dfuXRefService.cpp @@ -177,7 +177,10 @@ bool CWsDfuXRefEx::onDFUXRefArrayAction(IEspContext &context, IEspDFUXRefArrayAc else { // DeleteLogical: // Note we don't want to physically delete 'lost' files - this will end up with orphans on next time round but that is safer - if (fileNode->RemoveLogical(file, userDesc, cluster, err)) + if (!canRemoveLogicalFile(file, userDesc, req.getRemoveFromSuperfiles(), err)) + appendReplyMessage(fmt==ESPSerializationJSON, returnStr, nullptr, + "Error(s) removing File %s\n%s", file, err.str()); + else if (fileNode->RemoveLogical(file, userDesc, cluster, err)) appendReplyMessage(fmt==ESPSerializationJSON, returnStr, nullptr, "Removed Logical File %s", file); else @@ -202,6 +205,47 @@ bool CWsDfuXRefEx::onDFUXRefArrayAction(IEspContext &context, IEspDFUXRefArrayAc return true; } +bool CWsDfuXRefEx::canRemoveLogicalFile(const char *logicalFile, IUserDescriptor *userDesc, bool removeFromSuperfiles, StringBuffer &errStr) +{ + Owned df = queryDistributedFileDirectory().lookup(logicalFile, userDesc, AccessMode::tbdWrite, false, false, nullptr, defaultPrivilegedUser); + if (!df) + { + errStr.appendf("Logical file %s not found", logicalFile); + return false; + } + Owned superOwners = df->getOwningSuperFiles(); + if (!superOwners->first()) + return true; + + if (!removeFromSuperfiles) + { + errStr.appendf("Cannot remove logical file %s as owned by SuperFile(s)", logicalFile); + return false; + } + return removeLogicalFileFromSuperfiles(logicalFile, superOwners, errStr); +} + +bool CWsDfuXRefEx::removeLogicalFileFromSuperfiles(const char *logicalFile, IDistributedSuperFileIterator *superOwners, StringBuffer &errStr) +{ + ForEach(*superOwners) + { + IDistributedSuperFile &superOwner = superOwners->query(); + try + { + superOwner.removeSubFile(logicalFile, false, false, nullptr); + PROGLOG("File %s is removed from superfile %s", logicalFile, superOwner.queryLogicalName()); + } + catch(IException *e) + { + errStr.appendf("Could not remove file %s from superfile %s: ", logicalFile, superOwner.queryLogicalName()); + e->errorMessage(errStr); + e->Release(); + return false; + } + } + return true; +} + IXRefFilesNode *CWsDfuXRefEx::getFileNodeInterface(IXRefNode &XRefNode, const char *nodeType) { if (strieq("Found", nodeType)) diff --git a/esp/services/ws_dfu/ws_dfuXRefService.hpp b/esp/services/ws_dfu/ws_dfuXRefService.hpp index cc9d0b0a4a5..9a17c4664a5 100644 --- a/esp/services/ws_dfu/ws_dfuXRefService.hpp +++ b/esp/services/ws_dfu/ws_dfuXRefService.hpp @@ -200,6 +200,8 @@ class CWsDfuXRefEx : public CWsDFUXRef const char* formatResult(IEspContext& context, IPropertyTree* result, StringBuffer& formatedResult); const char* formatResult(IEspContext& context, StringBuffer& result, StringBuffer& encodedResult); void getRoxieFiles(const char *process, bool checkPackageMaps, MapStringTo &usedFileMap); + bool canRemoveLogicalFile(const char *logicalFile, IUserDescriptor *userDesc, bool removeFromSuperfiles, StringBuffer &errStr); + bool removeLogicalFileFromSuperfiles(const char *logicalFile, IDistributedSuperFileIterator *superOwners, StringBuffer &errStr); public: IMPLEMENT_IINTERFACE; From 7cc0c2785e7724e0a34acb03735d415e97e7ce08 Mon Sep 17 00:00:00 2001 From: Shamser Ahmed Date: Thu, 21 Dec 2023 12:11:57 +0000 Subject: [PATCH 02/32] HPCC-31035 Move ctx stats merging to caller from CKeyLevelManager mergeStats Prior to this change, it wasn't possible to share a IContextLogger between different CKeyLevelManager. The reason was that a mergeStats call needed to made of each of the CKeyLevelManagers but each call to the CKeyLevelManager::mergeStat would have the same jhtree stats being merged. By removing the IContextLogger merge from CKeyLevelManager::mergeStats it is possible for the same IContextLogger to be shared between different CKeyLevelManagers. This change required, the caller to merge the stats from the IContextLogger. This change make the IndexRead activity jhtree stats change simpler and more efficient as it can use the same IContextLogger from the different file parts.) Signed-off-by: Shamser Ahmed --- system/jhtree/jhtree.cpp | 3 --- thorlcr/activities/keyedjoin/thkeyedjoinslave.cpp | 8 ++++++-- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/system/jhtree/jhtree.cpp b/system/jhtree/jhtree.cpp index cfb9ec45b20..26264459d8f 100644 --- a/system/jhtree/jhtree.cpp +++ b/system/jhtree/jhtree.cpp @@ -574,11 +574,8 @@ class jhtree_decl CKeyLevelManager : implements IKeyManager, public CInterface virtual void mergeStats(CRuntimeStatisticCollection & targetStats) const { - // IO Stats coming from the keyCursor and jhtree cache stats coming from this class's stats if (keyCursor) keyCursor->mergeStats(targetStats); // merge IO stats - if (ctx) - targetStats.merge(ctx->queryStats()); // merge jhtree cache stats } }; diff --git a/thorlcr/activities/keyedjoin/thkeyedjoinslave.cpp b/thorlcr/activities/keyedjoin/thkeyedjoinslave.cpp index 3edc40700b1..59997c87a34 100644 --- a/thorlcr/activities/keyedjoin/thkeyedjoinslave.cpp +++ b/thorlcr/activities/keyedjoin/thkeyedjoinslave.cpp @@ -1409,10 +1409,14 @@ class CKeyedJoinSlave : public CSlaveActivity, implements IJoinProcessor, implem if (isSuper) { unsigned subfile = subFileNum[i]; - keyManager->mergeStats(*fileStats[startOffset+subfile]); + keyManager->mergeStats(*fileStats[startOffset+subfile]); // merge IO stats + fileStats[startOffset+subfile]->merge(contextLoggers[i]->queryStats()); // merge jhtreeStats } else - keyManager->mergeStats(*fileStats[startOffset]); + { + keyManager->mergeStats(*fileStats[startOffset]); // merge IO stats + fileStats[startOffset]->merge(contextLoggers[i]->queryStats()); // merge jhtreeStats + } } } }; From 62ba672da04df0218c392714df8a8ecc24e33b34 Mon Sep 17 00:00:00 2001 From: Jim DeFabia Date: Wed, 3 Jan 2024 10:35:16 -0500 Subject: [PATCH 03/32] HPCC-31064 Document SashaAccess LDAP settings Signed-off-by: Jim DeFabia --- .../Inst-Mods/UserSecurityMaint.xml | 24 +++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/docs/EN_US/Installing_and_RunningTheHPCCPlatform/Inst-Mods/UserSecurityMaint.xml b/docs/EN_US/Installing_and_RunningTheHPCCPlatform/Inst-Mods/UserSecurityMaint.xml index 005389012ec..9aa2297ea5b 100644 --- a/docs/EN_US/Installing_and_RunningTheHPCCPlatform/Inst-Mods/UserSecurityMaint.xml +++ b/docs/EN_US/Installing_and_RunningTheHPCCPlatform/Inst-Mods/UserSecurityMaint.xml @@ -1632,6 +1632,30 @@ Read + + + SashaAccess + + Access to WsSasha service + + Access + + + + + + List Workunits + + Read + + + + + + Archive Workunits, restore archived Workunits + + Full + From 950d526fa4be3ffd3335b31717634755d34c427f Mon Sep 17 00:00:00 2001 From: Gordon Smith Date: Thu, 4 Jan 2024 16:55:21 +0000 Subject: [PATCH 04/32] Split off 8.12.82 Signed-off-by: Gordon Smith --- helm/hpcc/Chart.yaml | 4 ++-- helm/hpcc/templates/_helpers.tpl | 2 +- helm/hpcc/templates/dafilesrv.yaml | 2 +- helm/hpcc/templates/dali.yaml | 2 +- helm/hpcc/templates/dfuserver.yaml | 2 +- helm/hpcc/templates/eclagent.yaml | 4 ++-- helm/hpcc/templates/eclccserver.yaml | 4 ++-- helm/hpcc/templates/eclscheduler.yaml | 2 +- helm/hpcc/templates/esp.yaml | 2 +- helm/hpcc/templates/localroxie.yaml | 2 +- helm/hpcc/templates/roxie.yaml | 8 ++++---- helm/hpcc/templates/sasha.yaml | 2 +- helm/hpcc/templates/thor.yaml | 10 +++++----- version.cmake | 4 ++-- 14 files changed, 25 insertions(+), 25 deletions(-) diff --git a/helm/hpcc/Chart.yaml b/helm/hpcc/Chart.yaml index 36f8bc752b2..5c66e0b4187 100644 --- a/helm/hpcc/Chart.yaml +++ b/helm/hpcc/Chart.yaml @@ -6,9 +6,9 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. -version: 8.12.81-closedown0 +version: 8.12.83-closedown0 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. -appVersion: 8.12.81-closedown0 +appVersion: 8.12.83-closedown0 diff --git a/helm/hpcc/templates/_helpers.tpl b/helm/hpcc/templates/_helpers.tpl index 01f6c6ce1f1..82044e42d77 100644 --- a/helm/hpcc/templates/_helpers.tpl +++ b/helm/hpcc/templates/_helpers.tpl @@ -1240,7 +1240,7 @@ kind: Service metadata: name: {{ $lvars.serviceName | quote }} labels: - helmVersion: 8.12.81-closedown0 + helmVersion: 8.12.83-closedown0 {{- include "hpcc.addStandardLabels" (dict "root" $.root "instance" $lvars.serviceName ) | indent 4 }} {{- if $lvars.labels }} {{ toYaml $lvars.labels | indent 4 }} diff --git a/helm/hpcc/templates/dafilesrv.yaml b/helm/hpcc/templates/dafilesrv.yaml index 4c2529a9b13..24915904a3b 100644 --- a/helm/hpcc/templates/dafilesrv.yaml +++ b/helm/hpcc/templates/dafilesrv.yaml @@ -50,7 +50,7 @@ spec: labels: {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "dafilesrv" "name" "dafilesrv" "instance" .name) | indent 8 }} server: {{ .name | quote }} - helmVersion: 8.12.81-closedown0 + helmVersion: 8.12.83-closedown0 annotations: checksum/config: {{ $configSHA }} spec: diff --git a/helm/hpcc/templates/dali.yaml b/helm/hpcc/templates/dali.yaml index 480c311e00c..277d1837157 100644 --- a/helm/hpcc/templates/dali.yaml +++ b/helm/hpcc/templates/dali.yaml @@ -82,7 +82,7 @@ spec: run: {{ $dali.name | quote }} server: {{ $dali.name | quote }} app: dali - helmVersion: 8.12.81-closedown0 + helmVersion: 8.12.83-closedown0 {{- if hasKey $.Values.global "metrics" }} {{- include "hpcc.generateMetricsReporterLabel" $.Values.global.metrics | nindent 8 }} {{- end }} diff --git a/helm/hpcc/templates/dfuserver.yaml b/helm/hpcc/templates/dfuserver.yaml index 125007b79ec..196f3d112d4 100644 --- a/helm/hpcc/templates/dfuserver.yaml +++ b/helm/hpcc/templates/dfuserver.yaml @@ -56,7 +56,7 @@ spec: {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "dfuserver" "name" "dfuserver" "instance" .name) | indent 8 }} run: {{ .name | quote }} accessDali: "yes" - helmVersion: 8.12.81-closedown0 + helmVersion: 8.12.83-closedown0 {{- if hasKey . "labels" }} {{ toYaml .labels | indent 8 }} {{- end }} diff --git a/helm/hpcc/templates/eclagent.yaml b/helm/hpcc/templates/eclagent.yaml index e86dda57048..b42445d29db 100644 --- a/helm/hpcc/templates/eclagent.yaml +++ b/helm/hpcc/templates/eclagent.yaml @@ -58,7 +58,7 @@ data: {{- include "hpcc.addStandardLabels" (dict "root" $ "component" $apptype "name" "eclagent" "instance" $appJobName "instanceOf" (printf "%s-job" .me.name)) | indent 12 }} accessDali: "yes" accessEsp: "yes" - helmVersion: 8.12.81-closedown0 + helmVersion: 8.12.83-closedown0 {{- if hasKey .me "labels" }} {{ toYaml .me.labels | indent 12 }} {{- end }} @@ -137,7 +137,7 @@ spec: run: {{ .name | quote }} accessDali: "yes" accessEsp: {{ .useChildProcesses | default false | ternary "yes" "no" | quote }} - helmVersion: 8.12.81-closedown0 + helmVersion: 8.12.83-closedown0 {{- if hasKey . "labels" }} {{ toYaml .labels | indent 8 }} {{- end }} diff --git a/helm/hpcc/templates/eclccserver.yaml b/helm/hpcc/templates/eclccserver.yaml index 580bb2ad2d6..c875cac745b 100644 --- a/helm/hpcc/templates/eclccserver.yaml +++ b/helm/hpcc/templates/eclccserver.yaml @@ -57,7 +57,7 @@ data: {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "eclccserver" "name" "eclccserver" "instance" $compileJobName "instanceOf" (printf "%s-job" .me.name)) | indent 12 }} accessDali: "yes" accessEsp: "yes" - helmVersion: 8.12.81-closedown0 + helmVersion: 8.12.83-closedown0 {{- if hasKey .me "labels" }} {{ toYaml .me.labels | indent 12 }} {{- end }} @@ -142,7 +142,7 @@ spec: run: {{ .name | quote }} accessDali: "yes" accessEsp: {{ .useChildProcesses | default false | ternary "yes" "no" | quote }} - helmVersion: 8.12.81-closedown0 + helmVersion: 8.12.83-closedown0 {{- if hasKey . "labels" }} {{ toYaml .labels | indent 8 }} {{- end }} diff --git a/helm/hpcc/templates/eclscheduler.yaml b/helm/hpcc/templates/eclscheduler.yaml index b69d24fc76a..f4abef54d15 100644 --- a/helm/hpcc/templates/eclscheduler.yaml +++ b/helm/hpcc/templates/eclscheduler.yaml @@ -64,7 +64,7 @@ spec: run: {{ .name | quote }} accessDali: "yes" accessEsp: "no" - helmVersion: 8.12.81-closedown0 + helmVersion: 8.12.83-closedown0 {{- if hasKey . "labels" }} {{ toYaml .labels | indent 8 }} {{- end }} diff --git a/helm/hpcc/templates/esp.yaml b/helm/hpcc/templates/esp.yaml index bc5753b7df9..dcd26219b10 100644 --- a/helm/hpcc/templates/esp.yaml +++ b/helm/hpcc/templates/esp.yaml @@ -117,7 +117,7 @@ spec: server: {{ .name | quote }} accessDali: "yes" app: {{ $application }} - helmVersion: 8.12.81-closedown0 + helmVersion: 8.12.83-closedown0 {{- include "hpcc.addStandardLabels" (dict "root" $ "name" $application "component" "esp" "instance" .name) | indent 8 }} {{- if hasKey $.Values.global "metrics" }} {{- include "hpcc.generateMetricsReporterLabel" $.Values.global.metrics | nindent 8 }} diff --git a/helm/hpcc/templates/localroxie.yaml b/helm/hpcc/templates/localroxie.yaml index 2a4a97b601f..e0a475866e4 100644 --- a/helm/hpcc/templates/localroxie.yaml +++ b/helm/hpcc/templates/localroxie.yaml @@ -70,7 +70,7 @@ spec: server: {{ $servername | quote }} accessDali: "yes" accessEsp: "yes" - helmVersion: 8.12.81-closedown0 + helmVersion: 8.12.83-closedown0 {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "roxie-server" "name" "roxie" "instance" $roxie.name) | indent 8 }} {{- if hasKey . "labels" }} {{ toYaml .labels | indent 8 }} diff --git a/helm/hpcc/templates/roxie.yaml b/helm/hpcc/templates/roxie.yaml index 70a083edb5e..aebc380a231 100644 --- a/helm/hpcc/templates/roxie.yaml +++ b/helm/hpcc/templates/roxie.yaml @@ -120,7 +120,7 @@ spec: {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "topology-server" "name" "roxie" "instance" $commonCtx.toponame) | indent 8 }} run: {{ $commonCtx.toponame | quote }} roxie-cluster: {{ $roxie.name | quote }} - helmVersion: 8.12.81-closedown0 + helmVersion: 8.12.83-closedown0 {{- if hasKey $.Values.global "metrics" }} {{- include "hpcc.generateMetricsReporterLabel" $.Values.global.metrics | nindent 8}} {{- end }} @@ -180,7 +180,7 @@ kind: Service metadata: name: {{ $commonCtx.toponame | quote }} labels: - helmVersion: 8.12.81-closedown0 + helmVersion: 8.12.83-closedown0 {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "topology-server" "name" "roxie" "instance" $commonCtx.toponame) | indent 4 }} spec: ports: @@ -242,7 +242,7 @@ spec: roxie-cluster: {{ $roxie.name | quote }} accessDali: "yes" accessEsp: "yes" - helmVersion: 8.12.81-closedown0 + helmVersion: 8.12.83-closedown0 {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "roxie-server" "name" "roxie" "instance" $servername) | indent 8 }} {{- if hasKey $.Values.global "metrics" }} {{- include "hpcc.generateMetricsReporterLabel" $.Values.global.metrics | nindent 8}} @@ -347,7 +347,7 @@ spec: roxie-cluster: {{ $roxie.name | quote }} accessDali: "yes" accessEsp: "yes" - helmVersion: 8.12.81-closedown0 + helmVersion: 8.12.83-closedown0 {{- if hasKey $.Values.global "metrics" }} {{- include "hpcc.generateMetricsReporterLabel" $.Values.global.metrics | nindent 8}} {{- end }} diff --git a/helm/hpcc/templates/sasha.yaml b/helm/hpcc/templates/sasha.yaml index 11c54c245c8..652c69f38ff 100644 --- a/helm/hpcc/templates/sasha.yaml +++ b/helm/hpcc/templates/sasha.yaml @@ -52,7 +52,7 @@ spec: run: {{ $serviceName | quote }} server: {{ $serviceName | quote }} accessDali: {{ (has "dali" $sasha.access) | ternary "yes" "no" | quote }} - helmVersion: 8.12.81-closedown0 + helmVersion: 8.12.83-closedown0 {{- if hasKey $sasha "labels" }} {{ toYaml $sasha.labels | indent 8 }} {{- end }} diff --git a/helm/hpcc/templates/thor.yaml b/helm/hpcc/templates/thor.yaml index 98d4ffa5902..2fa220712a3 100644 --- a/helm/hpcc/templates/thor.yaml +++ b/helm/hpcc/templates/thor.yaml @@ -82,7 +82,7 @@ data: labels: accessDali: "yes" accessEsp: "yes" - helmVersion: 8.12.81-closedown0 + helmVersion: 8.12.83-closedown0 {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "eclagent" "name" "thor" "instance" $eclAgentJobName "instanceOf" (printf "%s-job" .eclAgentName)) | indent 8 }} {{- if hasKey .me "labels" }} {{ toYaml .me.labels | indent 12 }} @@ -149,7 +149,7 @@ data: accessEsp: "yes" app: "thor" component: "thormanager" - helmVersion: 8.12.81-closedown0 + helmVersion: 8.12.83-closedown0 instance: "_HPCC_JOBNAME_" job: "_HPCC_JOBNAME_" {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "thormanager" "name" "thor" "instance" $thorManagerJobName "instanceOf" (printf "%s-thormanager-job" .me.name)) | indent 12 }} @@ -218,7 +218,7 @@ data: accessEsp: "yes" app: "thor" component: "thorworker" - helmVersion: 8.12.81-closedown0 + helmVersion: 8.12.83-closedown0 instance: "_HPCC_JOBNAME_" job: "_HPCC_JOBNAME_" {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "thorworker" "name" "thor" "instance" $thorWorkerJobName "instanceOf" (printf "%s-thorworker-job" .me.name)) | indent 12 }} @@ -353,7 +353,7 @@ spec: accessEsp: {{ $commonCtx.eclAgentUseChildProcesses | ternary "yes" "no" | quote }} app: "thor" component: "thor-eclagent" - helmVersion: 8.12.81-closedown0 + helmVersion: 8.12.83-closedown0 instance: {{ $commonCtx.eclAgentName | quote }} {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "eclagent" "name" "thor" "instance" $commonCtx.eclAgentName ) | indent 8 }} {{- if hasKey $commonCtx.me "labels" }} @@ -418,7 +418,7 @@ spec: accessEsp: "no" app: "thor" component: "thor-thoragent" - helmVersion: 8.12.81-closedown0 + helmVersion: 8.12.83-closedown0 instance: {{ $commonCtx.thorAgentName | quote }} {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "eclagent" "name" "thor" "instance" $commonCtx.thorAgentName ) | indent 8 }} {{- if hasKey $commonCtx.me "labels" }} diff --git a/version.cmake b/version.cmake index a2efe40a237..2dc80a3f7bc 100644 --- a/version.cmake +++ b/version.cmake @@ -5,8 +5,8 @@ set ( HPCC_NAME "Community Edition" ) set ( HPCC_PROJECT "community" ) set ( HPCC_MAJOR 8 ) set ( HPCC_MINOR 12 ) -set ( HPCC_POINT 81 ) +set ( HPCC_POINT 83 ) set ( HPCC_MATURITY "closedown" ) set ( HPCC_SEQUENCE 0 ) -set ( HPCC_TAG_TIMESTAMP "2023-12-21T11:25:54Z" ) +set ( HPCC_TAG_TIMESTAMP "2024-01-04T16:55:21Z" ) ### From 77938c2f49d737f123d4b9a6a46f93a0e38f982a Mon Sep 17 00:00:00 2001 From: Gordon Smith Date: Thu, 4 Jan 2024 16:56:55 +0000 Subject: [PATCH 05/32] Split off 9.0.72 Signed-off-by: Gordon Smith --- helm/hpcc/Chart.yaml | 4 ++-- helm/hpcc/templates/_helpers.tpl | 2 +- helm/hpcc/templates/dafilesrv.yaml | 2 +- helm/hpcc/templates/dali.yaml | 2 +- helm/hpcc/templates/dfuserver.yaml | 2 +- helm/hpcc/templates/eclagent.yaml | 4 ++-- helm/hpcc/templates/eclccserver.yaml | 4 ++-- helm/hpcc/templates/eclscheduler.yaml | 2 +- helm/hpcc/templates/esp.yaml | 2 +- helm/hpcc/templates/localroxie.yaml | 2 +- helm/hpcc/templates/roxie.yaml | 8 ++++---- helm/hpcc/templates/sasha.yaml | 2 +- helm/hpcc/templates/thor.yaml | 10 +++++----- version.cmake | 4 ++-- 14 files changed, 25 insertions(+), 25 deletions(-) diff --git a/helm/hpcc/Chart.yaml b/helm/hpcc/Chart.yaml index 22b949fa6ad..5a0743d3d2b 100644 --- a/helm/hpcc/Chart.yaml +++ b/helm/hpcc/Chart.yaml @@ -6,9 +6,9 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. -version: 9.0.71-closedown0 +version: 9.0.73-closedown0 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. -appVersion: 9.0.71-closedown0 +appVersion: 9.0.73-closedown0 diff --git a/helm/hpcc/templates/_helpers.tpl b/helm/hpcc/templates/_helpers.tpl index eae6a7a05e1..727ff242efb 100644 --- a/helm/hpcc/templates/_helpers.tpl +++ b/helm/hpcc/templates/_helpers.tpl @@ -1314,7 +1314,7 @@ kind: Service metadata: name: {{ $lvars.serviceName | quote }} labels: - helmVersion: 9.0.71-closedown0 + helmVersion: 9.0.73-closedown0 {{- include "hpcc.addStandardLabels" (dict "root" $.root "instance" $lvars.serviceName ) | indent 4 }} {{- if $lvars.labels }} {{ toYaml $lvars.labels | indent 4 }} diff --git a/helm/hpcc/templates/dafilesrv.yaml b/helm/hpcc/templates/dafilesrv.yaml index bc5c2d2533c..dd5c31e3b9a 100644 --- a/helm/hpcc/templates/dafilesrv.yaml +++ b/helm/hpcc/templates/dafilesrv.yaml @@ -50,7 +50,7 @@ spec: labels: {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "dafilesrv" "name" "dafilesrv" "instance" .name) | indent 8 }} server: {{ .name | quote }} - helmVersion: 9.0.71-closedown0 + helmVersion: 9.0.73-closedown0 annotations: checksum/config: {{ $configSHA }} spec: diff --git a/helm/hpcc/templates/dali.yaml b/helm/hpcc/templates/dali.yaml index 19230e0df78..c4492590de1 100644 --- a/helm/hpcc/templates/dali.yaml +++ b/helm/hpcc/templates/dali.yaml @@ -82,7 +82,7 @@ spec: run: {{ $dali.name | quote }} server: {{ $dali.name | quote }} app: dali - helmVersion: 9.0.71-closedown0 + helmVersion: 9.0.73-closedown0 {{- if hasKey $.Values.global "metrics" }} {{- include "hpcc.generateMetricsReporterLabel" $.Values.global.metrics | nindent 8 }} {{- end }} diff --git a/helm/hpcc/templates/dfuserver.yaml b/helm/hpcc/templates/dfuserver.yaml index 4ade8a2c238..5e058453283 100644 --- a/helm/hpcc/templates/dfuserver.yaml +++ b/helm/hpcc/templates/dfuserver.yaml @@ -56,7 +56,7 @@ spec: {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "dfuserver" "name" "dfuserver" "instance" .name) | indent 8 }} run: {{ .name | quote }} accessDali: "yes" - helmVersion: 9.0.71-closedown0 + helmVersion: 9.0.73-closedown0 {{- if hasKey . "labels" }} {{ toYaml .labels | indent 8 }} {{- end }} diff --git a/helm/hpcc/templates/eclagent.yaml b/helm/hpcc/templates/eclagent.yaml index 75779a9d9cf..2b503d6250d 100644 --- a/helm/hpcc/templates/eclagent.yaml +++ b/helm/hpcc/templates/eclagent.yaml @@ -58,7 +58,7 @@ data: {{- include "hpcc.addStandardLabels" (dict "root" $ "component" $apptype "name" "eclagent" "instance" $appJobName "instanceOf" (printf "%s-job" .me.name)) | indent 12 }} accessDali: "yes" accessEsp: "yes" - helmVersion: 9.0.71-closedown0 + helmVersion: 9.0.73-closedown0 {{- if hasKey .me "labels" }} {{ toYaml .me.labels | indent 12 }} {{- end }} @@ -135,7 +135,7 @@ spec: run: {{ .name | quote }} accessDali: "yes" accessEsp: {{ .useChildProcesses | default false | ternary "yes" "no" | quote }} - helmVersion: 9.0.71-closedown0 + helmVersion: 9.0.73-closedown0 {{- if hasKey . "labels" }} {{ toYaml .labels | indent 8 }} {{- end }} diff --git a/helm/hpcc/templates/eclccserver.yaml b/helm/hpcc/templates/eclccserver.yaml index 36da057a770..e24ce42d821 100644 --- a/helm/hpcc/templates/eclccserver.yaml +++ b/helm/hpcc/templates/eclccserver.yaml @@ -57,7 +57,7 @@ data: {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "eclccserver" "name" "eclccserver" "instance" $compileJobName "instanceOf" (printf "%s-job" .me.name)) | indent 12 }} accessDali: "yes" accessEsp: "yes" - helmVersion: 9.0.71-closedown0 + helmVersion: 9.0.73-closedown0 {{- if hasKey .me "labels" }} {{ toYaml .me.labels | indent 12 }} {{- end }} @@ -142,7 +142,7 @@ spec: run: {{ .name | quote }} accessDali: "yes" accessEsp: {{ .useChildProcesses | default false | ternary "yes" "no" | quote }} - helmVersion: 9.0.71-closedown0 + helmVersion: 9.0.73-closedown0 {{- if hasKey . "labels" }} {{ toYaml .labels | indent 8 }} {{- end }} diff --git a/helm/hpcc/templates/eclscheduler.yaml b/helm/hpcc/templates/eclscheduler.yaml index af2358c4182..158421cbad7 100644 --- a/helm/hpcc/templates/eclscheduler.yaml +++ b/helm/hpcc/templates/eclscheduler.yaml @@ -64,7 +64,7 @@ spec: run: {{ .name | quote }} accessDali: "yes" accessEsp: "no" - helmVersion: 9.0.71-closedown0 + helmVersion: 9.0.73-closedown0 {{- if hasKey . "labels" }} {{ toYaml .labels | indent 8 }} {{- end }} diff --git a/helm/hpcc/templates/esp.yaml b/helm/hpcc/templates/esp.yaml index 1c6a6314bc0..1b2a2e57d25 100644 --- a/helm/hpcc/templates/esp.yaml +++ b/helm/hpcc/templates/esp.yaml @@ -117,7 +117,7 @@ spec: server: {{ .name | quote }} accessDali: "yes" app: {{ $application }} - helmVersion: 9.0.71-closedown0 + helmVersion: 9.0.73-closedown0 {{- include "hpcc.addStandardLabels" (dict "root" $ "name" $application "component" "esp" "instance" .name) | indent 8 }} {{- if hasKey $.Values.global "metrics" }} {{- include "hpcc.generateMetricsReporterLabel" $.Values.global.metrics | nindent 8 }} diff --git a/helm/hpcc/templates/localroxie.yaml b/helm/hpcc/templates/localroxie.yaml index e79c71a9046..71796c561d1 100644 --- a/helm/hpcc/templates/localroxie.yaml +++ b/helm/hpcc/templates/localroxie.yaml @@ -70,7 +70,7 @@ spec: server: {{ $servername | quote }} accessDali: "yes" accessEsp: "yes" - helmVersion: 9.0.71-closedown0 + helmVersion: 9.0.73-closedown0 {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "roxie-server" "name" "roxie" "instance" $roxie.name) | indent 8 }} {{- if hasKey . "labels" }} {{ toYaml .labels | indent 8 }} diff --git a/helm/hpcc/templates/roxie.yaml b/helm/hpcc/templates/roxie.yaml index 950adb8943e..3d1a14a8af7 100644 --- a/helm/hpcc/templates/roxie.yaml +++ b/helm/hpcc/templates/roxie.yaml @@ -120,7 +120,7 @@ spec: {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "topology-server" "name" "roxie" "instance" $commonCtx.toponame) | indent 8 }} run: {{ $commonCtx.toponame | quote }} roxie-cluster: {{ $roxie.name | quote }} - helmVersion: 9.0.71-closedown0 + helmVersion: 9.0.73-closedown0 {{- if hasKey $.Values.global "metrics" }} {{- include "hpcc.generateMetricsReporterLabel" $.Values.global.metrics | nindent 8}} {{- end }} @@ -180,7 +180,7 @@ kind: Service metadata: name: {{ $commonCtx.toponame | quote }} labels: - helmVersion: 9.0.71-closedown0 + helmVersion: 9.0.73-closedown0 {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "topology-server" "name" "roxie" "instance" $commonCtx.toponame) | indent 4 }} spec: ports: @@ -242,7 +242,7 @@ spec: roxie-cluster: {{ $roxie.name | quote }} accessDali: "yes" accessEsp: "yes" - helmVersion: 9.0.71-closedown0 + helmVersion: 9.0.73-closedown0 {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "roxie-server" "name" "roxie" "instance" $servername) | indent 8 }} {{- if hasKey $.Values.global "metrics" }} {{- include "hpcc.generateMetricsReporterLabel" $.Values.global.metrics | nindent 8}} @@ -347,7 +347,7 @@ spec: roxie-cluster: {{ $roxie.name | quote }} accessDali: "yes" accessEsp: "yes" - helmVersion: 9.0.71-closedown0 + helmVersion: 9.0.73-closedown0 {{- if hasKey $.Values.global "metrics" }} {{- include "hpcc.generateMetricsReporterLabel" $.Values.global.metrics | nindent 8}} {{- end }} diff --git a/helm/hpcc/templates/sasha.yaml b/helm/hpcc/templates/sasha.yaml index 0af938f0de1..46e68dbdba0 100644 --- a/helm/hpcc/templates/sasha.yaml +++ b/helm/hpcc/templates/sasha.yaml @@ -52,7 +52,7 @@ spec: run: {{ $serviceName | quote }} server: {{ $serviceName | quote }} accessDali: {{ (has "dali" $sasha.access) | ternary "yes" "no" | quote }} - helmVersion: 9.0.71-closedown0 + helmVersion: 9.0.73-closedown0 {{- if hasKey $sasha "labels" }} {{ toYaml $sasha.labels | indent 8 }} {{- end }} diff --git a/helm/hpcc/templates/thor.yaml b/helm/hpcc/templates/thor.yaml index 413b1d13726..8a1eabdf9a6 100644 --- a/helm/hpcc/templates/thor.yaml +++ b/helm/hpcc/templates/thor.yaml @@ -82,7 +82,7 @@ data: labels: accessDali: "yes" accessEsp: "yes" - helmVersion: 9.0.71-closedown0 + helmVersion: 9.0.73-closedown0 {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "eclagent" "name" "thor" "instance" $eclAgentJobName "instanceOf" (printf "%s-job" .eclAgentName)) | indent 8 }} {{- if hasKey .me "labels" }} {{ toYaml .me.labels | indent 12 }} @@ -147,7 +147,7 @@ data: accessEsp: "yes" app: "thor" component: "thormanager" - helmVersion: 9.0.71-closedown0 + helmVersion: 9.0.73-closedown0 instance: "_HPCC_JOBNAME_" job: "_HPCC_JOBNAME_" {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "thormanager" "name" "thor" "instance" $thorManagerJobName "instanceOf" (printf "%s-thormanager-job" .me.name)) | indent 12 }} @@ -214,7 +214,7 @@ data: accessEsp: "yes" app: "thor" component: "thorworker" - helmVersion: 9.0.71-closedown0 + helmVersion: 9.0.73-closedown0 instance: "_HPCC_JOBNAME_" job: "_HPCC_JOBNAME_" {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "thorworker" "name" "thor" "instance" $thorWorkerJobName "instanceOf" (printf "%s-thorworker-job" .me.name)) | indent 12 }} @@ -347,7 +347,7 @@ spec: accessEsp: {{ $commonCtx.eclAgentUseChildProcesses | ternary "yes" "no" | quote }} app: "thor" component: "thor-eclagent" - helmVersion: 9.0.71-closedown0 + helmVersion: 9.0.73-closedown0 instance: {{ $commonCtx.eclAgentName | quote }} {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "eclagent" "name" "thor" "instance" $commonCtx.eclAgentName ) | indent 8 }} {{- if hasKey $commonCtx.me "labels" }} @@ -412,7 +412,7 @@ spec: accessEsp: "no" app: "thor" component: "thor-thoragent" - helmVersion: 9.0.71-closedown0 + helmVersion: 9.0.73-closedown0 instance: {{ $commonCtx.thorAgentName | quote }} {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "eclagent" "name" "thor" "instance" $commonCtx.thorAgentName ) | indent 8 }} {{- if hasKey $commonCtx.me "labels" }} diff --git a/version.cmake b/version.cmake index 60d9de1f8eb..2c8c95119db 100644 --- a/version.cmake +++ b/version.cmake @@ -5,8 +5,8 @@ set ( HPCC_NAME "Community Edition" ) set ( HPCC_PROJECT "community" ) set ( HPCC_MAJOR 9 ) set ( HPCC_MINOR 0 ) -set ( HPCC_POINT 71 ) +set ( HPCC_POINT 73 ) set ( HPCC_MATURITY "closedown" ) set ( HPCC_SEQUENCE 0 ) -set ( HPCC_TAG_TIMESTAMP "2023-12-21T11:27:33Z" ) +set ( HPCC_TAG_TIMESTAMP "2024-01-04T16:56:55Z" ) ### From c8ac479ba027162c76a26767c3bc92c5eab9b11e Mon Sep 17 00:00:00 2001 From: Jake Smith Date: Mon, 8 Jan 2024 17:53:44 +0000 Subject: [PATCH 06/32] HPCC-31073 Fix IStoragePlane hosts when plane based on 'hostGroup' A plane based on a hostGroup was not being expanded correctly when a IStoragePlane was constucted. As a consequence, use of a IStoragePlane based on a hostGroup had no hosts, and was being treated as if it was a non-host based plane that was locally mounted. Signed-off-by: Jake Smith --- dali/base/dafdesc.cpp | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/dali/base/dafdesc.cpp b/dali/base/dafdesc.cpp index a1e9d24005f..0765eb3055d 100644 --- a/dali/base/dafdesc.cpp +++ b/dali/base/dafdesc.cpp @@ -3812,9 +3812,10 @@ class CStoragePlaneInfo : public CInterfaceOf Owned srcAliases = xml->getElements("aliases"); ForEach(*srcAliases) aliases.push_back(new CStoragePlaneAlias(&srcAliases->query())); - Owned srcHosts = xml->getElements("hosts"); - ForEach(*srcHosts) - hosts.emplace_back(srcHosts->query().queryProp(nullptr)); + StringArray planeHosts; + getPlaneHosts(planeHosts, xml); + ForEachItemIn(h, planeHosts) + hosts.emplace_back(planeHosts.item(h)); } virtual const char * queryPrefix() const override { return xml->queryProp("@prefix"); } From 9d3a82213c3e05f49e79c10c3fad2b734a39ffb2 Mon Sep 17 00:00:00 2001 From: wangkx Date: Wed, 13 Dec 2023 13:06:01 -0500 Subject: [PATCH 07/32] HPCC-30641 Fetch pod logs for ZAP report Before this PR, WU ZAP report has only one log file which contains the WU log lines from all pods. In this PR, each pod where the WU works on has a log file in the WU ZAP report. The createWULogFile() is replaced by new readWULogToFiles() which does the following: 1. check whether Remote Log Accessor is available or not. 2. find out the pods where the WU works on. 3. read the log lines into the log files for the pods. Signed-off-by: wangkx --- .../ws_workunits/ws_workunitsHelpers.cpp | 36 ++++++++++++++----- .../ws_workunits/ws_workunitsHelpers.hpp | 3 +- 2 files changed, 29 insertions(+), 10 deletions(-) diff --git a/esp/services/ws_workunits/ws_workunitsHelpers.cpp b/esp/services/ws_workunits/ws_workunitsHelpers.cpp index 4deb23ac4c7..9398231c187 100644 --- a/esp/services/ws_workunits/ws_workunitsHelpers.cpp +++ b/esp/services/ws_workunits/ws_workunitsHelpers.cpp @@ -276,9 +276,6 @@ void streamFilteredLogsToFile(const char* outFile, LogAccessConditions & logFetc void WsWuInfo::readWorkunitComponentLogs(const char* outFile, CWsWuZAPInfoReq& zapLogFilterOptions) { - if (!queryRemoteLogAccessor()) - throw makeStringException(ECLWATCH_LOGACCESS_UNAVAILABLE, "WsWuInfo: Remote Log Access plug-in not available!"); - if (isEmptyString(outFile)) throw makeStringException(ECLWATCH_INVALID_FILE_NAME, "WsWuInfo: Target filename not provided!"); @@ -4201,13 +4198,15 @@ void CWsWuFileHelper::createZAPECLQueryArchiveFiles(IConstWorkUnit* cwu, const c } } -void CWsWuFileHelper::createWULogFile(IConstWorkUnit *cwu, WsWuInfo &winfo, const char *path, CWsWuZAPInfoReq & zapLogFilterOptions) +void CWsWuFileHelper::readWULogToFiles(IConstWorkUnit *cwu, WsWuInfo &winfo, const char *path, CWsWuZAPInfoReq &zapLogFilterOptions) { if (cwu->getWuidVersion() == 0) return; - LogAccessLogFormat logFormat = zapLogFilterOptions.logFilter.logDataFormat; + if (!queryRemoteLogAccessor()) + throw makeStringException(ECLWATCH_LOGACCESS_UNAVAILABLE, "CWsWuFileHelper: Remote Log Access plug-in not available!"); + LogAccessLogFormat logFormat = zapLogFilterOptions.logFilter.logDataFormat; StringBuffer logfileextension; if (logFormat == LOGACCESS_LOGFORMAT_csv) logfileextension.set("csv"); @@ -4218,18 +4217,37 @@ void CWsWuFileHelper::createWULogFile(IConstWorkUnit *cwu, WsWuInfo &winfo, cons else logfileextension.set("log"); - VStringBuffer fileName("%s%c%s-log.%s", path, PATHSEPCHAR, cwu->queryWuid(), logfileextension.str()); + const char *wuid = cwu->queryWuid(); + ILogAccessFilter *logFetchFilter = getJobIDLogAccessFilter(wuid); + zapLogFilterOptions.logFilter.logFetchOptions.setFilter(logFetchFilter); + + VStringBuffer componentLog("%s%c%s-log.%s", path, PATHSEPCHAR, wuid, logfileextension.str()); + readWULogToFile(componentLog, winfo, zapLogFilterOptions); + Owned iter = cwu->getProcesses("*", nullptr); + ForEach(*iter) + { + const char *processName = iter->query().queryProp("@podName"); + ILogAccessFilter *processLogFetchFilter = getBinaryLogAccessFilter(logFetchFilter, getPodLogAccessFilter(processName), LOGACCESS_FILTER_and); + zapLogFilterOptions.logFilter.logFetchOptions.setFilter(processLogFetchFilter); + + VStringBuffer processLog("%s%c%s-%s-log.%s", path, PATHSEPCHAR, wuid, processName, logfileextension.str()); + readWULogToFile(processLog, winfo, zapLogFilterOptions); + } +} + +void CWsWuFileHelper::readWULogToFile(const char *logFileName, WsWuInfo &winfo, CWsWuZAPInfoReq &zapLogFilterOptions) +{ try { - winfo.readWorkunitComponentLogs(fileName.str(), zapLogFilterOptions); + winfo.readWorkunitComponentLogs(logFileName, zapLogFilterOptions); } catch(IException* e) { StringBuffer s; e->errorMessage(s); IERRLOG(e, "Error accessing WU logs"); - writeToFile(fileName.str(), s.length(), s.str()); + writeToFile(logFileName, s.length(), s.str()); e->Release(); } } @@ -4480,7 +4498,7 @@ void CWsWuFileHelper::createWUZAPFile(IEspContext& context, IConstWorkUnit* cwu, if (request.includeThorSlaveLog.isEmpty() || strieq(request.includeThorSlaveLog.str(), "on")) createThorSlaveLogfile(cwu, winfo, tempDirName); #else - createWULogFile(cwu, winfo, tempDirName, request); + readWULogToFiles(cwu, winfo, tempDirName, request); #endif //Write out to ZIP file diff --git a/esp/services/ws_workunits/ws_workunitsHelpers.hpp b/esp/services/ws_workunits/ws_workunitsHelpers.hpp index 16eca77c5b7..78b0347795f 100644 --- a/esp/services/ws_workunits/ws_workunitsHelpers.hpp +++ b/esp/services/ws_workunits/ws_workunitsHelpers.hpp @@ -897,7 +897,8 @@ class CWsWuFileHelper void createThorSlaveLogfile(IConstWorkUnit *cwu, WsWuInfo &winfo, const char *path); #endif LogAccessLogFormat getComponentLogFormatFromLogName(const char *log); - void createWULogFile(IConstWorkUnit *cwu, WsWuInfo &winfo, const char *path, CWsWuZAPInfoReq & zapLogFilterOptions); + void readWULogToFiles(IConstWorkUnit *cwu, WsWuInfo &winfo, const char *path, CWsWuZAPInfoReq &zapLogFilterOptions); + void readWULogToFile(const char *logFileName, WsWuInfo &winfo, CWsWuZAPInfoReq &zapLogFilterOptions); void writeZAPWUInfoToIOStream(IFileIOStream *outFile, const char *name, SCMStringBuffer &value); void writeZAPWUInfoToIOStream(IFileIOStream *outFile, const char *name, const char *value); void readWUIDRequest(CHttpRequest *request, StringBuffer &wuid); From 616a491a49ce526ca73266c35e2706cbe652e98e Mon Sep 17 00:00:00 2001 From: Jim DeFabia Date: Wed, 10 Jan 2024 14:29:26 -0500 Subject: [PATCH 08/32] HPCC-20267 Document TimestampToString Signed-off-by: Jim DeFabia --- .../SLR-Mods/TimestampToString.xml | 88 +++++++++++++++++++ .../SLR-includer.xml | 3 + 2 files changed, 91 insertions(+) create mode 100644 docs/EN_US/ECLStandardLibraryReference/SLR-Mods/TimestampToString.xml diff --git a/docs/EN_US/ECLStandardLibraryReference/SLR-Mods/TimestampToString.xml b/docs/EN_US/ECLStandardLibraryReference/SLR-Mods/TimestampToString.xml new file mode 100644 index 00000000000..085ccf79aaa --- /dev/null +++ b/docs/EN_US/ECLStandardLibraryReference/SLR-Mods/TimestampToString.xml @@ -0,0 +1,88 @@ + + + + TimestampToString + + STD.Date.TimestampToString + (timestamp, format) + + + + + + + + + + timestamp + + An INTEGER8 holding the number of microseconds since epoch + (January 1, 1970 UTC) + + + + format + + OPTIONAL. The format of the string to return. See strftime + documentation for details (http://strftime.org/). If + omitted, it defaults to '%Y-%m-%dT%H:%M:%S.%@' which is + YYYY-MM-DDTHH:MM:SS.ssssss. + + + + Return: + + The converted timestamp as a string in + the specified format. + + + + + + The TimestampToString + TimestampToString + function converts a Timestamp_t value containing + the number of microseconds since epoch (January 1, 1970 UTC) into a + human-readable string using a format template of strftime standards. Two + additional format specifiers are available to show fractional + seconds: + + + + + + + + + + %@ + + Fraction of seconds in microseconds (6 digits) + + + + %# + + Fraction of seconds in milliseconds (3 digits) + + + + + + Millisecond fractions are truncated from microseconds when + necessary. + + The maximum length of the resulting string is 255 characters. + + Example: + + IMPORT STD; +STD.Date.TimestampToString(1048998120000000, '%A %B %d, %Y T%H:%M:%S.%#'); + // returns Sunday March 30, 2003 T04:22:00.000 + + + + diff --git a/docs/EN_US/ECLStandardLibraryReference/SLR-includer.xml b/docs/EN_US/ECLStandardLibraryReference/SLR-includer.xml index da6de3c274c..917e3cac08a 100644 --- a/docs/EN_US/ECLStandardLibraryReference/SLR-includer.xml +++ b/docs/EN_US/ECLStandardLibraryReference/SLR-includer.xml @@ -486,6 +486,9 @@ + + Date: Wed, 20 Dec 2023 13:09:14 -0500 Subject: [PATCH 09/32] HPCC-30389 Split JTrace getSpanContext function - Adjusts ISpan getSpanContex signature - Adds new ISpan getClientHeaders function - Ensures tracestate is produced Signed-off-by: Rodrigo Pastrana --- system/jlib/jtrace.cpp | 126 +++++++++++++++----------------- system/jlib/jtrace.hpp | 3 +- testing/unittests/jlibtests.cpp | 91 ++++++++++------------- 3 files changed, 102 insertions(+), 118 deletions(-) diff --git a/system/jlib/jtrace.cpp b/system/jlib/jtrace.cpp index 714ec9e5f85..d7e72a3069f 100644 --- a/system/jlib/jtrace.cpp +++ b/system/jlib/jtrace.cpp @@ -573,8 +573,7 @@ class CSpan : public CInterfaceOf bool getSpanContext(CHPCCHttpTextMapCarrier * carrier) const { - if (!carrier) - return false; + assertex(carrier); auto propagator = opentelemetry::context::propagation::GlobalTextMapPropagator::GetGlobalPropagator(); @@ -594,66 +593,68 @@ class CSpan : public CInterfaceOf } /** - * Retrieves the Span's context as key/value pairs into the provided IProperties. - * Optionally, output follows OpenTelemetry Span context format for propogation + * Retrieves the Span's client headers traceparent and tracestate + * Output follows OpenTelemetry Span context format for propogation * accross process boundaries. * - * @param ctxProps IProperties container for span context key/value pairs. - * @param otelFormatted If true, output follows OpenTelemetry Span context format. - * @return True if the span context was successfully retrieved, false otherwise. + * @param clientHeaders IProperties container for client headers. */ - bool getSpanContext(IProperties * ctxProps, bool otelFormatted) const override + void getClientHeaders(IProperties * clientHeaders) const { - if (ctxProps == nullptr) - return false; + assertex(clientHeaders); - ctxProps->setNonEmptyProp(kGlobalIdHttpHeaderName, queryGlobalId()); + clientHeaders->setNonEmptyProp(kGlobalIdHttpHeaderName, queryGlobalId()); - if (otelFormatted) - { - //The localid is passed as the callerid for the client request.... - ctxProps->setNonEmptyProp(kCallerIdHttpHeaderName, queryLocalId()); - } - else - { - ctxProps->setNonEmptyProp(kCallerIdHttpHeaderName, queryCallerId()); - } + //The localid is passed as the callerid for the client request.... + clientHeaders->setNonEmptyProp(kCallerIdHttpHeaderName, queryLocalId()); if (span == nullptr) - return false; + return; - if (otelFormatted) - { - if (isEmptyString(traceID.get()) || isEmptyString(spanID.get()) || isEmptyString(traceFlags.get())) - return false; + if (isEmptyString(traceID.get()) || isEmptyString(spanID.get()) || isEmptyString(traceFlags.get())) + return; - //The traceparent header uses the version-trace_id-parent_id-trace_flags format where: - //version is always 00. trace_id is a hex-encoded trace id. span_id is a hex-encoded span id. trace_flags is a hex-encoded 8-bit field that contains tracing flags such as sampling, trace level, etc. - //Example: "traceparent", "00-beca49ca8f3138a2842e5cf21402bfff-4b960b3e4647da3f-01" + //The traceparent header uses the version-trace_id-parent_id-trace_flags format where: + //version is always 00. trace_id is a hex-encoded trace id. span_id is a hex-encoded span id. trace_flags is a hex-encoded 8-bit field that contains tracing flags such as sampling, trace level, etc. + //Example: "traceparent", "00-beca49ca8f3138a2842e5cf21402bfff-4b960b3e4647da3f-01" - StringBuffer contextHTTPHeader; - //https://www.w3.org/TR/trace-context/#header-name - contextHTTPHeader.append("00-").append(traceID.get()).append("-").append(spanID.get()).append("-").append(traceFlags.get()); - ctxProps->setProp(opentelemetry::trace::propagation::kTraceParent.data(), contextHTTPHeader.str()); + StringBuffer contextHTTPHeader; + //https://www.w3.org/TR/trace-context/#header-name + contextHTTPHeader.append("00-").append(traceID.get()).append("-").append(spanID.get()).append("-").append(traceFlags.get()); + clientHeaders->setProp(opentelemetry::trace::propagation::kTraceParent.data(), contextHTTPHeader.str()); - //The main purpose of the tracestate HTTP header is to provide additional vendor-specific trace identification - // information across different distributed tracing systems and is a companion header for the traceparent field. - // It also conveys information about the request’s position in multiple distributed tracing graphs. + //The main purpose of the tracestate HTTP header is to provide additional vendor-specific trace identification + // information across different distributed tracing systems and is a companion header for the traceparent field. + // It also conveys information about the request’s position in multiple distributed tracing graphs. - //https://www.w3.org/TR/trace-context/#trace-context-http-headers-format - //StringBuffer traceStateHTTPHeader; - //traceStateHTTPHeader.append("hpcc=").append(spanID.get()); + //https://www.w3.org/TR/trace-context/#trace-context-http-headers-format + //StringBuffer traceStateHTTPHeader; + //traceStateHTTPHeader.append("hpcc=").append(spanID.get()); - ctxProps->setNonEmptyProp(opentelemetry::trace::propagation::kTraceState.data(), span->GetContext().trace_state()->ToHeader().c_str()); - } - else - { - ctxProps->setNonEmptyProp("traceID", traceID.get()); - ctxProps->setNonEmptyProp("spanID", spanID.get()); - ctxProps->setNonEmptyProp("traceFlags", traceFlags.get()); - } + clientHeaders->setNonEmptyProp(opentelemetry::trace::propagation::kTraceState.data(), span->GetContext().trace_state()->ToHeader().c_str()); + } - return true; + /** + * Retrieves the Span's context as key/value pairs into the provided IProperties. + * Optionally, output follows OpenTelemetry Span context format for propogation + * accross process boundaries. + * + * @param ctxProps IProperties container for span context key/value pairs. + */ + void getSpanContext(IProperties * ctxProps) const override + { + if (ctxProps == nullptr) + return; + + ctxProps->setNonEmptyProp(kGlobalIdHttpHeaderName, queryGlobalId()); + ctxProps->setNonEmptyProp(kCallerIdHttpHeaderName, queryCallerId()); + + if (span == nullptr) + return; + + ctxProps->setNonEmptyProp("traceID", traceID.get()); + ctxProps->setNonEmptyProp("spanID", spanID.get()); + ctxProps->setNonEmptyProp("traceFlags", traceFlags.get()); } opentelemetry::v1::trace::SpanContext querySpanContext() const @@ -783,7 +784,8 @@ class CNullSpan : public CInterfaceOf virtual void setSpanAttributes(const IProperties * attributes) override {} virtual void addSpanEvent(const char * eventName) override {} virtual void addSpanEvent(const char * eventName, IProperties * attributes) override {}; - virtual bool getSpanContext(IProperties * ctxProps, bool otelFormatted) const override { return false; } + virtual void getSpanContext(IProperties * ctxProps) const override {} + virtual void getClientHeaders(IProperties * clientHeaders) const override {} virtual void toString(StringBuffer & out) const override {} virtual void getLogPrefix(StringBuffer & out) const override {} @@ -818,19 +820,13 @@ class CChildSpan : public CSpan } public: - virtual bool getSpanContext(IProperties * ctxProps, bool otelFormatted) const override + virtual void getSpanContext(IProperties * ctxProps) const override { - //MORE: It is not clear what this return value represents, and whether it would ever be checked. - bool ok = CSpan::getSpanContext(ctxProps, otelFormatted); + CSpan::getSpanContext(ctxProps); - if (ok && !otelFormatted) - { - Owned localParentSpanCtxProps = createProperties(); - localParentSpan->getSpanContext(localParentSpanCtxProps, false); - ctxProps->setNonEmptyProp("localParentSpanID", localParentSpanCtxProps->queryProp("spanID")); - } - - return ok; + Owned localParentSpanCtxProps = createProperties(); + localParentSpan->getSpanContext(localParentSpanCtxProps); + ctxProps->setNonEmptyProp("localParentSpanID", localParentSpanCtxProps->queryProp("spanID")); } virtual const char* queryGlobalId() const override @@ -951,11 +947,11 @@ class CServerSpan : public CSpan } } - bool getSpanContext(IProperties * ctxProps, bool otelFormatted) const override + void getSpanContext(IProperties * ctxProps) const override { - bool success = CSpan::getSpanContext(ctxProps, otelFormatted); + CSpan::getSpanContext(ctxProps); - if (!otelFormatted && remoteParentSpanCtx.IsValid()) + if (remoteParentSpanCtx.IsValid()) { StringBuffer remoteParentSpanID; char remoteParentSpanId[16] = {0}; @@ -963,8 +959,6 @@ class CServerSpan : public CSpan remoteParentSpanID.append(16, remoteParentSpanId); ctxProps->setProp("remoteParentSpanID", remoteParentSpanID.str()); } - - return success; } void init(SpanFlags flags) @@ -1046,14 +1040,14 @@ class CServerSpan : public CSpan IProperties * getClientHeaders(const ISpan * span) { Owned headers = createProperties(true); - span->getSpanContext(headers, true); // Return value is not helpful + span->getClientHeaders(headers); return headers.getClear(); } IProperties * getSpanContext(const ISpan * span) { Owned headers = createProperties(true); - span->getSpanContext(headers, false); + span->getSpanContext(headers); return headers.getClear(); } diff --git a/system/jlib/jtrace.hpp b/system/jlib/jtrace.hpp index 6e0b34137e0..e0053684b55 100644 --- a/system/jlib/jtrace.hpp +++ b/system/jlib/jtrace.hpp @@ -61,7 +61,8 @@ interface ISpan : extends IInterface virtual void setSpanAttributes(const IProperties * attributes) = 0; virtual void addSpanEvent(const char * eventName) = 0; virtual void addSpanEvent(const char * eventName, IProperties * attributes) = 0; - virtual bool getSpanContext(IProperties * ctxProps, bool otelFormatted) const = 0; + virtual void getSpanContext(IProperties * ctxProps) const = 0; + virtual void getClientHeaders(IProperties * clientHeaders) const = 0; virtual void toString(StringBuffer & out) const = 0; virtual void getLogPrefix(StringBuffer & out) const = 0; diff --git a/testing/unittests/jlibtests.cpp b/testing/unittests/jlibtests.cpp index 862eb0d75e5..3827f30fdf9 100644 --- a/testing/unittests/jlibtests.cpp +++ b/testing/unittests/jlibtests.cpp @@ -259,9 +259,7 @@ class JlibTraceTest : public CppUnit::TestFixture Owned serverSpan = queryTraceManager().createServerSpan("noRemoteParentEnsureTraceID", emptyMockHTTPHeaders, flags); Owned retrievedSpanCtxAttributes = createProperties(); - bool getSpanCtxSuccess = serverSpan->getSpanContext(retrievedSpanCtxAttributes.get(), false); - - CPPUNIT_ASSERT_EQUAL_MESSAGE("Unexpected getSpanContext failure detected - EnsureTraceID flag was set", true, getSpanCtxSuccess); + serverSpan->getSpanContext(retrievedSpanCtxAttributes.get()); CPPUNIT_ASSERT_EQUAL_MESSAGE("Unexpected empty TraceID detected", false, isEmptyString(retrievedSpanCtxAttributes->queryProp("traceID"))); CPPUNIT_ASSERT_EQUAL_MESSAGE("Unexpected empty SpanID detected", false, isEmptyString(retrievedSpanCtxAttributes->queryProp("spanID"))); @@ -279,9 +277,7 @@ class JlibTraceTest : public CppUnit::TestFixture //retrieve serverSpan context with the intent to interrogate attributes { Owned retrievedSpanCtxAttributes = createProperties(); - bool getSpanCtxSuccess = serverSpan->getSpanContext(retrievedSpanCtxAttributes.get(), false); - - CPPUNIT_ASSERT_EQUAL_MESSAGE("Unexpected getSpanContext failure detected", true, getSpanCtxSuccess); + serverSpan->getSpanContext(retrievedSpanCtxAttributes.get()); CPPUNIT_ASSERT_MESSAGE("Unexpected GlobalID detected", strsame("IncomingUGID", retrievedSpanCtxAttributes->queryProp(kGlobalIdHttpHeaderName))); @@ -296,12 +292,17 @@ class JlibTraceTest : public CppUnit::TestFixture //retrieve serverSpan context with the intent to propagate it to a remote child span { - Owned retrievedSpanCtxAttributes = createProperties(); - bool getSpanCtxSuccess = serverSpan->getSpanContext(retrievedSpanCtxAttributes.get(), true); + Owned retrievedClientHeaders = createProperties(); + serverSpan->getClientHeaders(retrievedClientHeaders.get()); - CPPUNIT_ASSERT_EQUAL_MESSAGE("Unexpected getSpanContext failure detected", true, getSpanCtxSuccess); + CPPUNIT_ASSERT_EQUAL_MESSAGE("getClientHeaders failed to produce traceParent!", true, retrievedClientHeaders->hasProp("traceparent")); CPPUNIT_ASSERT_EQUAL_MESSAGE("Unexpected Otel traceparent header len detected", (size_t)55, - strlen(retrievedSpanCtxAttributes->queryProp("traceparent"))); + strlen(retrievedClientHeaders->queryProp("traceparent"))); + + CPPUNIT_ASSERT_EQUAL_MESSAGE("getClientHeaders failed to produce traceState!", true, retrievedClientHeaders->hasProp("tracestate")); + const char * tracestate = retrievedClientHeaders->queryProp("tracestate"); + CPPUNIT_ASSERT_MESSAGE("Unexpected traceState detected", + strsame("hpcc=4b960b3e4647da3f", retrievedClientHeaders->queryProp("tracestate"))); } } @@ -326,7 +327,7 @@ class JlibTraceTest : public CppUnit::TestFixture { Owned headers = createProperties(true); - nullSpan->getSpanContext(headers, true); + nullSpan->getClientHeaders(headers); } Owned nullSpanChild = nullSpan->createClientSpan("nullSpanChild"); @@ -339,8 +340,7 @@ class JlibTraceTest : public CppUnit::TestFixture Owned serverSpan = queryTraceManager().createServerSpan("propegatedServerSpan", emptyMockHTTPHeaders); Owned retrievedSpanCtxAttributes = createProperties(); - bool getSpanCtxSuccess = serverSpan->getSpanContext(retrievedSpanCtxAttributes, false); - CPPUNIT_ASSERT_EQUAL_MESSAGE("Unexpected getSpanContext failure detected", true, getSpanCtxSuccess); + serverSpan->getSpanContext(retrievedSpanCtxAttributes); CPPUNIT_ASSERT_EQUAL_MESSAGE("Unexpected empty spanID detected", true, retrievedSpanCtxAttributes->hasProp("spanID")); const char * serverSpanID = retrievedSpanCtxAttributes->queryProp("spanID"); @@ -353,8 +353,7 @@ class JlibTraceTest : public CppUnit::TestFixture //retrieve clientSpan context with the intent to propogate otel and HPCC context { Owned retrievedSpanCtxAttributes = createProperties(); - bool getSpanCtxSuccess = internalSpan->getSpanContext(retrievedSpanCtxAttributes, false); - CPPUNIT_ASSERT_EQUAL_MESSAGE("Unexpected getSpanContext failure detected", true, getSpanCtxSuccess); + internalSpan->getSpanContext(retrievedSpanCtxAttributes); CPPUNIT_ASSERT_EQUAL_MESSAGE("Unexpected missing localParentSpanID detected", true, retrievedSpanCtxAttributes->hasProp("localParentSpanID")); @@ -388,8 +387,7 @@ class JlibTraceTest : public CppUnit::TestFixture Owned serverSpan = queryTraceManager().createServerSpan("propegatedServerSpan", emptyMockHTTPHeaders); Owned retrievedSpanCtxAttributes = createProperties(); - bool getSpanCtxSuccess = serverSpan->getSpanContext(retrievedSpanCtxAttributes, false); - CPPUNIT_ASSERT_EQUAL_MESSAGE("Unexpected getSpanContext failure detected", true, getSpanCtxSuccess); + serverSpan->getSpanContext(retrievedSpanCtxAttributes); CPPUNIT_ASSERT_EQUAL_MESSAGE("Unexpected empty spanID detected", true, retrievedSpanCtxAttributes->hasProp("spanID")); const char * serverSpanID = retrievedSpanCtxAttributes->queryProp("spanID"); @@ -402,8 +400,7 @@ class JlibTraceTest : public CppUnit::TestFixture //retrieve internalSpan context with the intent to interrogate attributes { Owned retrievedSpanCtxAttributes = createProperties(); - bool getSpanCtxSuccess = internalSpan->getSpanContext(retrievedSpanCtxAttributes, false); - CPPUNIT_ASSERT_EQUAL_MESSAGE("Unexpected getSpanContext failure detected", true, getSpanCtxSuccess); + internalSpan->getSpanContext(retrievedSpanCtxAttributes); CPPUNIT_ASSERT_EQUAL_MESSAGE("Unexpected missing localParentSpanID detected", true, retrievedSpanCtxAttributes->hasProp("localParentSpanID")); @@ -434,25 +431,24 @@ class JlibTraceTest : public CppUnit::TestFixture Owned emptyMockHTTPHeaders = createProperties(); Owned serverSpan = queryTraceManager().createServerSpan("propegatedServerSpan", emptyMockHTTPHeaders); - Owned retrievedSpanCtxAttributes = createProperties(); - //at this point the serverSpan should have the following context attributes - //traceID, spanID,traceFlags, traceState - //but no remoteParentSpanID, globalID, callerID - //retrieve serverSpan context with the intent to propagate it to a remote child span { - bool getSpanCtxSuccess = serverSpan->getSpanContext(retrievedSpanCtxAttributes, true); + Owned retrievedClientHeaders = createProperties(); + serverSpan->getClientHeaders(retrievedClientHeaders); - CPPUNIT_ASSERT_EQUAL_MESSAGE("Unexpected getSpanContext failure detected", true, getSpanCtxSuccess); + CPPUNIT_ASSERT_EQUAL_MESSAGE("getClientHeaders failed to produce traceParent!", true, retrievedClientHeaders->hasProp("traceparent")); CPPUNIT_ASSERT_EQUAL_MESSAGE("Unexpected Otel traceparent header len detected", (size_t)55, - strlen(retrievedSpanCtxAttributes->queryProp("traceparent"))); + strlen(retrievedClientHeaders->queryProp("traceparent"))); + + CPPUNIT_ASSERT_EQUAL_MESSAGE("detected unexpected tracestate from getClientHeaders", false, retrievedClientHeaders->hasProp("tracestate")); } //retrieve serverSpan context with the intent to interrogate attributes { - bool getSpanCtxSuccess = serverSpan->getSpanContext(retrievedSpanCtxAttributes, false); - - CPPUNIT_ASSERT_EQUAL_MESSAGE("Unexpected getSpanContext failure detected", true, getSpanCtxSuccess); + Owned retrievedSpanCtxAttributes = createProperties(); + serverSpan->getSpanContext(retrievedSpanCtxAttributes); + //at this point the serverSpan should have the following context attributes + //traceID, spanID //but no remoteParentSpanID, globalID, callerID CPPUNIT_ASSERT_EQUAL_MESSAGE("Unexpected GlobalID detected", false, retrievedSpanCtxAttributes->hasProp(kGlobalIdHttpHeaderName)); @@ -474,9 +470,8 @@ class JlibTraceTest : public CppUnit::TestFixture Owned serverSpan = queryTraceManager().createServerSpan("invalidPropegatedServerSpan", mockHTTPHeaders); Owned retrievedSpanCtxAttributes = createProperties(); - bool getSpanCtxSuccess = serverSpan->getSpanContext(retrievedSpanCtxAttributes.get(), true); + serverSpan->getClientHeaders(retrievedSpanCtxAttributes.get()); - CPPUNIT_ASSERT_EQUAL_MESSAGE("Unexpected getSpanContext failure detected", true, getSpanCtxSuccess); const char * traceParent = retrievedSpanCtxAttributes->queryProp("remoteParentSpanID"); DBGLOG("testInvalidPropegatedServerSpan: traceparent: %s", traceParent); } @@ -498,9 +493,7 @@ class JlibTraceTest : public CppUnit::TestFixture //remoteParentSpanID, globalID, callerID Owned retrievedSpanCtxAttributes = createProperties(); - bool getSpanCtxSuccess = serverSpan->getSpanContext(retrievedSpanCtxAttributes.get(), false); - - CPPUNIT_ASSERT_EQUAL_MESSAGE("Unexpected getSpanContext failure detected", true, getSpanCtxSuccess); + serverSpan->getSpanContext(retrievedSpanCtxAttributes.get()); CPPUNIT_ASSERT_MESSAGE("Unexpected GlobalID detected", strsame("IncomingUGID", retrievedSpanCtxAttributes->queryProp(kGlobalIdHttpHeaderName))); @@ -560,9 +553,7 @@ class JlibTraceTest : public CppUnit::TestFixture //retrieve serverSpan context with the intent to interrogate attributes { Owned retrievedClientSpanCtxAttributes = createProperties(); - bool getClientSpanCtxSuccess = clientSpan->getSpanContext(retrievedClientSpanCtxAttributes.get(), false); - - CPPUNIT_ASSERT_EQUAL_MESSAGE("Unexpected getSpanContext failure detected on client span", true, getClientSpanCtxSuccess); + clientSpan->getSpanContext(retrievedClientSpanCtxAttributes.get()); CPPUNIT_ASSERT_MESSAGE("Unexpected GlobalID detected", strsame("IncomingUGID", retrievedClientSpanCtxAttributes->queryProp(kGlobalIdHttpHeaderName))); @@ -586,9 +577,7 @@ class JlibTraceTest : public CppUnit::TestFixture //retrieve serverSpan context with the intent to interrogate attributes { Owned retrievedSpanCtxAttributes = createProperties(); - bool getSpanCtxSuccess = serverSpan->getSpanContext(retrievedSpanCtxAttributes.get(), false); - - CPPUNIT_ASSERT_EQUAL_MESSAGE("Unexpected getSpanContext failure detected", true, getSpanCtxSuccess); + serverSpan->getSpanContext(retrievedSpanCtxAttributes.get()); CPPUNIT_ASSERT_MESSAGE("Unexpected GlobalID detected", strsame("IncomingUGID", retrievedSpanCtxAttributes->queryProp(kGlobalIdHttpHeaderName))); @@ -602,14 +591,18 @@ class JlibTraceTest : public CppUnit::TestFixture CPPUNIT_ASSERT_EQUAL_MESSAGE("Unexpected empty SpanID detected", false, isEmptyString(retrievedSpanCtxAttributes->queryProp("spanID"))); } - //retrieve serverSpan context with the intent to propagate it to a remote child span + //retrieve serverSpan client headers with the intent to propagate them onto the next hop { - Owned retrievedSpanCtxAttributes = createProperties(); - bool getSpanCtxSuccess = serverSpan->getSpanContext(retrievedSpanCtxAttributes.get(), true); + Owned retrievedClientHeaders = createProperties(); + serverSpan->getClientHeaders(retrievedClientHeaders.get()); - CPPUNIT_ASSERT_EQUAL_MESSAGE("Unexpected getSpanContext failure detected", true, getSpanCtxSuccess); CPPUNIT_ASSERT_EQUAL_MESSAGE("Unexpected Otel traceparent header len detected", (size_t)55, - strlen(retrievedSpanCtxAttributes->queryProp("traceparent"))); + strlen(retrievedClientHeaders->queryProp("traceparent"))); + + CPPUNIT_ASSERT_EQUAL_MESSAGE("getClientHeaders failed to produce traceState!", true, retrievedClientHeaders->hasProp("tracestate")); + const char * tracestate = retrievedClientHeaders->queryProp("tracestate"); + CPPUNIT_ASSERT_MESSAGE("Unexpected traceState detected", + strsame("hpcc=4b960b3e4647da3f", retrievedClientHeaders->queryProp("tracestate"))); } } @@ -630,11 +623,7 @@ class JlibTraceTest : public CppUnit::TestFixture //retrieve serverSpan context with the intent to interrogate attributes { Owned retrievedSpanCtxAttributes = createProperties(); - bool getSpanCtxSuccess = serverSpan->getSpanContext(retrievedSpanCtxAttributes.get(), false); - - CPPUNIT_ASSERT_EQUAL_MESSAGE("Unexpected getSpanContext failure detected", true, getSpanCtxSuccess); - - CPPUNIT_ASSERT_EQUAL_MESSAGE("Unexpected getSpanContext failure detected", true, getSpanCtxSuccess); + serverSpan->getSpanContext(retrievedSpanCtxAttributes.get()); CPPUNIT_ASSERT_EQUAL_MESSAGE("Unexpected GlobalID detected", true, strsame("someGlobalID", retrievedSpanCtxAttributes->queryProp(kGlobalIdHttpHeaderName))); From 831ba96beaff0a4b6d5ef2ff508e6cf682f2528f Mon Sep 17 00:00:00 2001 From: Shamser Ahmed Date: Tue, 19 Dec 2023 11:04:37 +0000 Subject: [PATCH 10/32] HPCC-30974 Copying super files via fileservices/dfu does not track read/cost stats. Signed-off-by: Shamser Ahmed --- dali/ft/filecopy.cpp | 94 ++++++++++++++++++++++++++++++++------------ 1 file changed, 69 insertions(+), 25 deletions(-) diff --git a/dali/ft/filecopy.cpp b/dali/ft/filecopy.cpp index b9feea834ef..5802a8bf1c5 100644 --- a/dali/ft/filecopy.cpp +++ b/dali/ft/filecopy.cpp @@ -3384,25 +3384,9 @@ void FileSprayer::spray() afterTransfer(); //If got here then we have succeeded + //Note: On failure, costs will not be updated. Future: would be useful to have a way to update costs on failure. updateTargetProperties(); - //Calculate and store file access cost - cost_type fileAccessCost = 0; - if (distributedTarget) - { - StringBuffer cluster; - distributedTarget->getClusterName(0, cluster); - if (!cluster.isEmpty()) - fileAccessCost += calcFileAccessCost(cluster, totalNumWrites, 0); - } - if (distributedSource && distributedSource->querySuperFile()==nullptr) - { - StringBuffer cluster; - distributedSource->getClusterName(0, cluster); - if (!cluster.isEmpty()) - fileAccessCost += calcFileAccessCost(cluster, 0, totalNumReads); - } - progressReport->setFileAccessCost(fileAccessCost); StringBuffer copyEventText; // [logical-source] > [logical-target] if (distributedSource) copyEventText.append(distributedSource->queryLogicalName()); @@ -3455,6 +3439,7 @@ void FileSprayer::updateTargetProperties() { TimeSection timer("FileSprayer::updateTargetProperties() time"); Owned error; + cost_type totalWriteCost = 0; if (distributedTarget) { StringBuffer failedParts; @@ -3467,6 +3452,7 @@ void FileSprayer::updateTargetProperties() bool sameSizeHeaderFooter = isSameSizeHeaderFooter(); bool sameSizeSourceTarget = (sources.ordinality() == distributedTarget->numParts()); offset_t partCompressedLength = 0; + IDistributedSuperFile * superTgt = distributedTarget->querySuperFile(); ForEachItemIn(idx, partition) { @@ -3585,6 +3571,26 @@ void FileSprayer::updateTargetProperties() partLength = 0; partCompressedLength = 0; } + + // Update @writeCost and @numWrites in subfile properties and update totalWriteCost + if (superTgt) + { + if (cur.whichOutput != (unsigned)-1) + { + unsigned targetPartNum = targets.item(cur.whichOutput).partNum; + IDistributedFile &subfile = superTgt->querySubFile(targetPartNum, true); + DistributedFilePropertyLock lock(&subfile); + IPropertyTree &subFileProps = lock.queryAttributes(); + cost_type prevNumWrites = subFileProps.getPropInt64(getDFUQResultFieldName(DFUQRFnumDiskWrites)); + cost_type prevWriteCost = subFileProps.getPropInt64(getDFUQResultFieldName(DFUQRFwriteCost)); + cost_type curWriteCost = calcFileAccessCost(&subfile, curProgress.numWrites, 0); + subFileProps.setPropInt64(getDFUQResultFieldName(DFUQRFwriteCost), prevWriteCost + curWriteCost); + subFileProps.setPropInt64(getDFUQResultFieldName(DFUQRFnumDiskWrites), prevNumWrites + curProgress.numWrites); + totalWriteCost += curWriteCost; + } + else // not sure if tgt superfile can have whichOutput==-1 (but if so, the following cost calc works) + totalWriteCost += calcFileAccessCost(distributedTarget, curProgress.numWrites, 0); + } } if (failedParts.length()) @@ -3592,8 +3598,10 @@ void FileSprayer::updateTargetProperties() DistributedFilePropertyLock lock(distributedTarget); IPropertyTree &curProps = lock.queryAttributes(); - cost_type writeCost = calcFileAccessCost(distributedTarget, totalNumWrites, 0); - curProps.setPropInt64(getDFUQResultFieldName(DFUQRFwriteCost), writeCost); + + if (!superTgt) + totalWriteCost = calcFileAccessCost(distributedTarget, totalNumWrites, 0); + curProps.setPropInt64(getDFUQResultFieldName(DFUQRFwriteCost), totalWriteCost); curProps.setPropInt64(getDFUQResultFieldName(DFUQRFnumDiskWrites), totalNumWrites); if (calcCRC()) @@ -3772,17 +3780,53 @@ void FileSprayer::updateTargetProperties() if (expireDays != -1) curProps.setPropInt("@expireDays", expireDays); } + // Update file readCost and numReads in file properties and do the same for subfiles + // Update totalReadCost + cost_type totalReadCost = 0; if (distributedSource) { - if (distributedSource->querySuperFile()==nullptr) + IDistributedSuperFile * superSrc = distributedSource->querySuperFile(); + if (superSrc) { - IPropertyTree & fileAttr = distributedSource->queryAttributes(); - cost_type legacyReadCost = getLegacyReadCost(fileAttr, distributedSource); - cost_type curReadCost = calcFileAccessCost(distributedSource, 0, totalNumReads); - distributedSource->addAttrValue(getDFUQResultFieldName(DFUQRFreadCost), legacyReadCost+curReadCost); - distributedSource->addAttrValue(getDFUQResultFieldName(DFUQRFnumDiskReads), totalNumReads); + ForEachItemIn(idx, partition) + { + PartitionPoint & cur = partition.item(idx); + OutputProgress & curProgress = progress.item(idx); + + if (cur.whichInput != (unsigned)-1) + { + unsigned sourcePartNum = sources.item(cur.whichInput).partNum; + IDistributedFile &subfile = superSrc->querySubFile(sourcePartNum, true); + DistributedFilePropertyLock lock(&subfile); + IPropertyTree &subFileProps = lock.queryAttributes(); + stat_type prevNumReads = subFileProps.getPropInt64(getDFUQResultFieldName(DFUQRFnumDiskReads), 0); + cost_type legacyReadCost = getLegacyReadCost(subfile.queryAttributes(), &subfile); + cost_type prevReadCost = subFileProps.getPropInt64(getDFUQResultFieldName(DFUQRFreadCost), 0); + cost_type curReadCost = calcFileAccessCost(&subfile, 0, curProgress.numReads); + subFileProps.setPropInt64(getDFUQResultFieldName(DFUQRFnumDiskReads), prevNumReads + curProgress.numReads); + subFileProps.setPropInt64(getDFUQResultFieldName(DFUQRFreadCost), legacyReadCost + prevReadCost + curReadCost); + totalReadCost += curReadCost; + } + else + { + // not sure if src superfile can have whichInput==-1 (but if so, this is best effort to calc cost) + totalReadCost += calcFileAccessCost(distributedSource, 0, curProgress.numReads); + } + } } + else + { + totalReadCost = calcFileAccessCost(distributedSource, 0, totalNumReads); + } + DistributedFilePropertyLock lock(distributedSource); + IPropertyTree &curProps = lock.queryAttributes(); + stat_type prevNumReads = curProps.getPropInt64(getDFUQResultFieldName(DFUQRFnumDiskReads), 0); + cost_type legacyReadCost = getLegacyReadCost(curProps, distributedSource); + cost_type prevReadCost = curProps.getPropInt64(getDFUQResultFieldName(DFUQRFreadCost), 0); + curProps.setPropInt64(getDFUQResultFieldName(DFUQRFnumDiskReads), prevNumReads + totalNumReads); + curProps.setPropInt64(getDFUQResultFieldName(DFUQRFreadCost), legacyReadCost + prevReadCost + totalReadCost); } + progressReport->setFileAccessCost(cost_type2money(totalReadCost+totalWriteCost)); if (error) throw error.getClear(); } From 6e62b80ada53c8c91ebe1c903c58426773fa849c Mon Sep 17 00:00:00 2001 From: Jake Smith Date: Thu, 11 Jan 2024 17:47:59 +0000 Subject: [PATCH 11/32] Split off 9.0.74 Signed-off-by: Jake Smith --- helm/hpcc/Chart.yaml | 4 ++-- helm/hpcc/templates/_helpers.tpl | 2 +- helm/hpcc/templates/dafilesrv.yaml | 2 +- helm/hpcc/templates/dali.yaml | 2 +- helm/hpcc/templates/dfuserver.yaml | 2 +- helm/hpcc/templates/eclagent.yaml | 4 ++-- helm/hpcc/templates/eclccserver.yaml | 4 ++-- helm/hpcc/templates/eclscheduler.yaml | 2 +- helm/hpcc/templates/esp.yaml | 2 +- helm/hpcc/templates/localroxie.yaml | 2 +- helm/hpcc/templates/roxie.yaml | 8 ++++---- helm/hpcc/templates/sasha.yaml | 2 +- helm/hpcc/templates/thor.yaml | 10 +++++----- version.cmake | 4 ++-- 14 files changed, 25 insertions(+), 25 deletions(-) diff --git a/helm/hpcc/Chart.yaml b/helm/hpcc/Chart.yaml index 5a0743d3d2b..57509b04b89 100644 --- a/helm/hpcc/Chart.yaml +++ b/helm/hpcc/Chart.yaml @@ -6,9 +6,9 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. -version: 9.0.73-closedown0 +version: 9.0.75-closedown0 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. -appVersion: 9.0.73-closedown0 +appVersion: 9.0.75-closedown0 diff --git a/helm/hpcc/templates/_helpers.tpl b/helm/hpcc/templates/_helpers.tpl index 727ff242efb..412cc7a1549 100644 --- a/helm/hpcc/templates/_helpers.tpl +++ b/helm/hpcc/templates/_helpers.tpl @@ -1314,7 +1314,7 @@ kind: Service metadata: name: {{ $lvars.serviceName | quote }} labels: - helmVersion: 9.0.73-closedown0 + helmVersion: 9.0.75-closedown0 {{- include "hpcc.addStandardLabels" (dict "root" $.root "instance" $lvars.serviceName ) | indent 4 }} {{- if $lvars.labels }} {{ toYaml $lvars.labels | indent 4 }} diff --git a/helm/hpcc/templates/dafilesrv.yaml b/helm/hpcc/templates/dafilesrv.yaml index dd5c31e3b9a..0d341434f2d 100644 --- a/helm/hpcc/templates/dafilesrv.yaml +++ b/helm/hpcc/templates/dafilesrv.yaml @@ -50,7 +50,7 @@ spec: labels: {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "dafilesrv" "name" "dafilesrv" "instance" .name) | indent 8 }} server: {{ .name | quote }} - helmVersion: 9.0.73-closedown0 + helmVersion: 9.0.75-closedown0 annotations: checksum/config: {{ $configSHA }} spec: diff --git a/helm/hpcc/templates/dali.yaml b/helm/hpcc/templates/dali.yaml index c4492590de1..b2a75725678 100644 --- a/helm/hpcc/templates/dali.yaml +++ b/helm/hpcc/templates/dali.yaml @@ -82,7 +82,7 @@ spec: run: {{ $dali.name | quote }} server: {{ $dali.name | quote }} app: dali - helmVersion: 9.0.73-closedown0 + helmVersion: 9.0.75-closedown0 {{- if hasKey $.Values.global "metrics" }} {{- include "hpcc.generateMetricsReporterLabel" $.Values.global.metrics | nindent 8 }} {{- end }} diff --git a/helm/hpcc/templates/dfuserver.yaml b/helm/hpcc/templates/dfuserver.yaml index 5e058453283..b3f85adf708 100644 --- a/helm/hpcc/templates/dfuserver.yaml +++ b/helm/hpcc/templates/dfuserver.yaml @@ -56,7 +56,7 @@ spec: {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "dfuserver" "name" "dfuserver" "instance" .name) | indent 8 }} run: {{ .name | quote }} accessDali: "yes" - helmVersion: 9.0.73-closedown0 + helmVersion: 9.0.75-closedown0 {{- if hasKey . "labels" }} {{ toYaml .labels | indent 8 }} {{- end }} diff --git a/helm/hpcc/templates/eclagent.yaml b/helm/hpcc/templates/eclagent.yaml index 2b503d6250d..64d98be7596 100644 --- a/helm/hpcc/templates/eclagent.yaml +++ b/helm/hpcc/templates/eclagent.yaml @@ -58,7 +58,7 @@ data: {{- include "hpcc.addStandardLabels" (dict "root" $ "component" $apptype "name" "eclagent" "instance" $appJobName "instanceOf" (printf "%s-job" .me.name)) | indent 12 }} accessDali: "yes" accessEsp: "yes" - helmVersion: 9.0.73-closedown0 + helmVersion: 9.0.75-closedown0 {{- if hasKey .me "labels" }} {{ toYaml .me.labels | indent 12 }} {{- end }} @@ -135,7 +135,7 @@ spec: run: {{ .name | quote }} accessDali: "yes" accessEsp: {{ .useChildProcesses | default false | ternary "yes" "no" | quote }} - helmVersion: 9.0.73-closedown0 + helmVersion: 9.0.75-closedown0 {{- if hasKey . "labels" }} {{ toYaml .labels | indent 8 }} {{- end }} diff --git a/helm/hpcc/templates/eclccserver.yaml b/helm/hpcc/templates/eclccserver.yaml index e24ce42d821..9cdca8d7d34 100644 --- a/helm/hpcc/templates/eclccserver.yaml +++ b/helm/hpcc/templates/eclccserver.yaml @@ -57,7 +57,7 @@ data: {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "eclccserver" "name" "eclccserver" "instance" $compileJobName "instanceOf" (printf "%s-job" .me.name)) | indent 12 }} accessDali: "yes" accessEsp: "yes" - helmVersion: 9.0.73-closedown0 + helmVersion: 9.0.75-closedown0 {{- if hasKey .me "labels" }} {{ toYaml .me.labels | indent 12 }} {{- end }} @@ -142,7 +142,7 @@ spec: run: {{ .name | quote }} accessDali: "yes" accessEsp: {{ .useChildProcesses | default false | ternary "yes" "no" | quote }} - helmVersion: 9.0.73-closedown0 + helmVersion: 9.0.75-closedown0 {{- if hasKey . "labels" }} {{ toYaml .labels | indent 8 }} {{- end }} diff --git a/helm/hpcc/templates/eclscheduler.yaml b/helm/hpcc/templates/eclscheduler.yaml index 158421cbad7..cc4f6dbce4c 100644 --- a/helm/hpcc/templates/eclscheduler.yaml +++ b/helm/hpcc/templates/eclscheduler.yaml @@ -64,7 +64,7 @@ spec: run: {{ .name | quote }} accessDali: "yes" accessEsp: "no" - helmVersion: 9.0.73-closedown0 + helmVersion: 9.0.75-closedown0 {{- if hasKey . "labels" }} {{ toYaml .labels | indent 8 }} {{- end }} diff --git a/helm/hpcc/templates/esp.yaml b/helm/hpcc/templates/esp.yaml index 1b2a2e57d25..e820963d68a 100644 --- a/helm/hpcc/templates/esp.yaml +++ b/helm/hpcc/templates/esp.yaml @@ -117,7 +117,7 @@ spec: server: {{ .name | quote }} accessDali: "yes" app: {{ $application }} - helmVersion: 9.0.73-closedown0 + helmVersion: 9.0.75-closedown0 {{- include "hpcc.addStandardLabels" (dict "root" $ "name" $application "component" "esp" "instance" .name) | indent 8 }} {{- if hasKey $.Values.global "metrics" }} {{- include "hpcc.generateMetricsReporterLabel" $.Values.global.metrics | nindent 8 }} diff --git a/helm/hpcc/templates/localroxie.yaml b/helm/hpcc/templates/localroxie.yaml index 71796c561d1..e3efec9a936 100644 --- a/helm/hpcc/templates/localroxie.yaml +++ b/helm/hpcc/templates/localroxie.yaml @@ -70,7 +70,7 @@ spec: server: {{ $servername | quote }} accessDali: "yes" accessEsp: "yes" - helmVersion: 9.0.73-closedown0 + helmVersion: 9.0.75-closedown0 {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "roxie-server" "name" "roxie" "instance" $roxie.name) | indent 8 }} {{- if hasKey . "labels" }} {{ toYaml .labels | indent 8 }} diff --git a/helm/hpcc/templates/roxie.yaml b/helm/hpcc/templates/roxie.yaml index 3d1a14a8af7..d0ec5a8021b 100644 --- a/helm/hpcc/templates/roxie.yaml +++ b/helm/hpcc/templates/roxie.yaml @@ -120,7 +120,7 @@ spec: {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "topology-server" "name" "roxie" "instance" $commonCtx.toponame) | indent 8 }} run: {{ $commonCtx.toponame | quote }} roxie-cluster: {{ $roxie.name | quote }} - helmVersion: 9.0.73-closedown0 + helmVersion: 9.0.75-closedown0 {{- if hasKey $.Values.global "metrics" }} {{- include "hpcc.generateMetricsReporterLabel" $.Values.global.metrics | nindent 8}} {{- end }} @@ -180,7 +180,7 @@ kind: Service metadata: name: {{ $commonCtx.toponame | quote }} labels: - helmVersion: 9.0.73-closedown0 + helmVersion: 9.0.75-closedown0 {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "topology-server" "name" "roxie" "instance" $commonCtx.toponame) | indent 4 }} spec: ports: @@ -242,7 +242,7 @@ spec: roxie-cluster: {{ $roxie.name | quote }} accessDali: "yes" accessEsp: "yes" - helmVersion: 9.0.73-closedown0 + helmVersion: 9.0.75-closedown0 {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "roxie-server" "name" "roxie" "instance" $servername) | indent 8 }} {{- if hasKey $.Values.global "metrics" }} {{- include "hpcc.generateMetricsReporterLabel" $.Values.global.metrics | nindent 8}} @@ -347,7 +347,7 @@ spec: roxie-cluster: {{ $roxie.name | quote }} accessDali: "yes" accessEsp: "yes" - helmVersion: 9.0.73-closedown0 + helmVersion: 9.0.75-closedown0 {{- if hasKey $.Values.global "metrics" }} {{- include "hpcc.generateMetricsReporterLabel" $.Values.global.metrics | nindent 8}} {{- end }} diff --git a/helm/hpcc/templates/sasha.yaml b/helm/hpcc/templates/sasha.yaml index 46e68dbdba0..9a8782cab99 100644 --- a/helm/hpcc/templates/sasha.yaml +++ b/helm/hpcc/templates/sasha.yaml @@ -52,7 +52,7 @@ spec: run: {{ $serviceName | quote }} server: {{ $serviceName | quote }} accessDali: {{ (has "dali" $sasha.access) | ternary "yes" "no" | quote }} - helmVersion: 9.0.73-closedown0 + helmVersion: 9.0.75-closedown0 {{- if hasKey $sasha "labels" }} {{ toYaml $sasha.labels | indent 8 }} {{- end }} diff --git a/helm/hpcc/templates/thor.yaml b/helm/hpcc/templates/thor.yaml index 8a1eabdf9a6..cd71b8e21e7 100644 --- a/helm/hpcc/templates/thor.yaml +++ b/helm/hpcc/templates/thor.yaml @@ -82,7 +82,7 @@ data: labels: accessDali: "yes" accessEsp: "yes" - helmVersion: 9.0.73-closedown0 + helmVersion: 9.0.75-closedown0 {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "eclagent" "name" "thor" "instance" $eclAgentJobName "instanceOf" (printf "%s-job" .eclAgentName)) | indent 8 }} {{- if hasKey .me "labels" }} {{ toYaml .me.labels | indent 12 }} @@ -147,7 +147,7 @@ data: accessEsp: "yes" app: "thor" component: "thormanager" - helmVersion: 9.0.73-closedown0 + helmVersion: 9.0.75-closedown0 instance: "_HPCC_JOBNAME_" job: "_HPCC_JOBNAME_" {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "thormanager" "name" "thor" "instance" $thorManagerJobName "instanceOf" (printf "%s-thormanager-job" .me.name)) | indent 12 }} @@ -214,7 +214,7 @@ data: accessEsp: "yes" app: "thor" component: "thorworker" - helmVersion: 9.0.73-closedown0 + helmVersion: 9.0.75-closedown0 instance: "_HPCC_JOBNAME_" job: "_HPCC_JOBNAME_" {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "thorworker" "name" "thor" "instance" $thorWorkerJobName "instanceOf" (printf "%s-thorworker-job" .me.name)) | indent 12 }} @@ -347,7 +347,7 @@ spec: accessEsp: {{ $commonCtx.eclAgentUseChildProcesses | ternary "yes" "no" | quote }} app: "thor" component: "thor-eclagent" - helmVersion: 9.0.73-closedown0 + helmVersion: 9.0.75-closedown0 instance: {{ $commonCtx.eclAgentName | quote }} {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "eclagent" "name" "thor" "instance" $commonCtx.eclAgentName ) | indent 8 }} {{- if hasKey $commonCtx.me "labels" }} @@ -412,7 +412,7 @@ spec: accessEsp: "no" app: "thor" component: "thor-thoragent" - helmVersion: 9.0.73-closedown0 + helmVersion: 9.0.75-closedown0 instance: {{ $commonCtx.thorAgentName | quote }} {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "eclagent" "name" "thor" "instance" $commonCtx.thorAgentName ) | indent 8 }} {{- if hasKey $commonCtx.me "labels" }} diff --git a/version.cmake b/version.cmake index 2c8c95119db..09bd5710d99 100644 --- a/version.cmake +++ b/version.cmake @@ -5,8 +5,8 @@ set ( HPCC_NAME "Community Edition" ) set ( HPCC_PROJECT "community" ) set ( HPCC_MAJOR 9 ) set ( HPCC_MINOR 0 ) -set ( HPCC_POINT 73 ) +set ( HPCC_POINT 75 ) set ( HPCC_MATURITY "closedown" ) set ( HPCC_SEQUENCE 0 ) -set ( HPCC_TAG_TIMESTAMP "2024-01-04T16:56:55Z" ) +set ( HPCC_TAG_TIMESTAMP "2024-01-11T17:47:59Z" ) ### From 583eb563219422c73d1cb6f161bcd71505dee49d Mon Sep 17 00:00:00 2001 From: Jake Smith Date: Thu, 11 Jan 2024 17:49:06 +0000 Subject: [PATCH 12/32] Split off 9.2.52 Signed-off-by: Jake Smith --- helm/hpcc/Chart.yaml | 4 ++-- helm/hpcc/templates/_helpers.tpl | 2 +- helm/hpcc/templates/dafilesrv.yaml | 2 +- helm/hpcc/templates/dali.yaml | 2 +- helm/hpcc/templates/dfuserver.yaml | 2 +- helm/hpcc/templates/eclagent.yaml | 4 ++-- helm/hpcc/templates/eclccserver.yaml | 4 ++-- helm/hpcc/templates/eclscheduler.yaml | 2 +- helm/hpcc/templates/esp.yaml | 2 +- helm/hpcc/templates/localroxie.yaml | 2 +- helm/hpcc/templates/roxie.yaml | 8 ++++---- helm/hpcc/templates/sasha.yaml | 2 +- helm/hpcc/templates/thor.yaml | 10 +++++----- version.cmake | 4 ++-- 14 files changed, 25 insertions(+), 25 deletions(-) diff --git a/helm/hpcc/Chart.yaml b/helm/hpcc/Chart.yaml index 46c633c9d0f..b13994cde37 100644 --- a/helm/hpcc/Chart.yaml +++ b/helm/hpcc/Chart.yaml @@ -6,9 +6,9 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. -version: 9.2.51-closedown0 +version: 9.2.53-closedown0 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. -appVersion: 9.2.51-closedown0 +appVersion: 9.2.53-closedown0 diff --git a/helm/hpcc/templates/_helpers.tpl b/helm/hpcc/templates/_helpers.tpl index faf28d21272..17259c62c4f 100644 --- a/helm/hpcc/templates/_helpers.tpl +++ b/helm/hpcc/templates/_helpers.tpl @@ -1405,7 +1405,7 @@ kind: Service metadata: name: {{ $lvars.serviceName | quote }} labels: - helmVersion: 9.2.51-closedown0 + helmVersion: 9.2.53-closedown0 {{- include "hpcc.addStandardLabels" (dict "root" $.root "instance" $lvars.serviceName ) | indent 4 }} {{- if $lvars.labels }} {{ toYaml $lvars.labels | indent 4 }} diff --git a/helm/hpcc/templates/dafilesrv.yaml b/helm/hpcc/templates/dafilesrv.yaml index 931a3464288..1b810c42e74 100644 --- a/helm/hpcc/templates/dafilesrv.yaml +++ b/helm/hpcc/templates/dafilesrv.yaml @@ -50,7 +50,7 @@ spec: labels: {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "dafilesrv" "name" "dafilesrv" "instance" .name) | indent 8 }} server: {{ .name | quote }} - helmVersion: 9.2.51-closedown0 + helmVersion: 9.2.53-closedown0 annotations: checksum/config: {{ $configSHA }} {{- include "hpcc.generateAnnotations" $commonCtx | indent 8 }} diff --git a/helm/hpcc/templates/dali.yaml b/helm/hpcc/templates/dali.yaml index f0a86e93fac..03333bb5403 100644 --- a/helm/hpcc/templates/dali.yaml +++ b/helm/hpcc/templates/dali.yaml @@ -82,7 +82,7 @@ spec: run: {{ $dali.name | quote }} server: {{ $dali.name | quote }} app: dali - helmVersion: 9.2.51-closedown0 + helmVersion: 9.2.53-closedown0 {{- if hasKey $.Values.global "metrics" }} {{- include "hpcc.generateMetricsReporterLabel" $.Values.global.metrics | nindent 8 }} {{- end }} diff --git a/helm/hpcc/templates/dfuserver.yaml b/helm/hpcc/templates/dfuserver.yaml index 2d5765da373..b90565ea2df 100644 --- a/helm/hpcc/templates/dfuserver.yaml +++ b/helm/hpcc/templates/dfuserver.yaml @@ -56,7 +56,7 @@ spec: {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "dfuserver" "name" "dfuserver" "instance" .name) | indent 8 }} run: {{ .name | quote }} accessDali: "yes" - helmVersion: 9.2.51-closedown0 + helmVersion: 9.2.53-closedown0 {{- if hasKey . "labels" }} {{ toYaml .labels | indent 8 }} {{- end }} diff --git a/helm/hpcc/templates/eclagent.yaml b/helm/hpcc/templates/eclagent.yaml index 998be80d96c..891fdcd51ee 100644 --- a/helm/hpcc/templates/eclagent.yaml +++ b/helm/hpcc/templates/eclagent.yaml @@ -58,7 +58,7 @@ data: {{- include "hpcc.addStandardLabels" (dict "root" $ "component" $apptype "name" "eclagent" "instance" $appJobName "instanceOf" (printf "%s-job" .me.name)) | indent 12 }} accessDali: "yes" accessEsp: "yes" - helmVersion: 9.2.51-closedown0 + helmVersion: 9.2.53-closedown0 {{- if hasKey .me "labels" }} {{ toYaml .me.labels | indent 12 }} {{- end }} @@ -133,7 +133,7 @@ spec: run: {{ .name | quote }} accessDali: "yes" accessEsp: {{ .useChildProcesses | default false | ternary "yes" "no" | quote }} - helmVersion: 9.2.51-closedown0 + helmVersion: 9.2.53-closedown0 {{- if hasKey . "labels" }} {{ toYaml .labels | indent 8 }} {{- end }} diff --git a/helm/hpcc/templates/eclccserver.yaml b/helm/hpcc/templates/eclccserver.yaml index 96634b821ee..248ccd0f7b0 100644 --- a/helm/hpcc/templates/eclccserver.yaml +++ b/helm/hpcc/templates/eclccserver.yaml @@ -57,7 +57,7 @@ data: {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "eclccserver" "name" "eclccserver" "instance" $compileJobName "instanceOf" (printf "%s-job" .me.name)) | indent 12 }} accessDali: "yes" accessEsp: "yes" - helmVersion: 9.2.51-closedown0 + helmVersion: 9.2.53-closedown0 {{- if hasKey .me "labels" }} {{ toYaml .me.labels | indent 12 }} {{- end }} @@ -140,7 +140,7 @@ spec: run: {{ .name | quote }} accessDali: "yes" accessEsp: {{ .useChildProcesses | default false | ternary "yes" "no" | quote }} - helmVersion: 9.2.51-closedown0 + helmVersion: 9.2.53-closedown0 {{- if hasKey . "labels" }} {{ toYaml .labels | indent 8 }} {{- end }} diff --git a/helm/hpcc/templates/eclscheduler.yaml b/helm/hpcc/templates/eclscheduler.yaml index a1a327d5760..923c3f9bb96 100644 --- a/helm/hpcc/templates/eclscheduler.yaml +++ b/helm/hpcc/templates/eclscheduler.yaml @@ -64,7 +64,7 @@ spec: run: {{ .name | quote }} accessDali: "yes" accessEsp: "no" - helmVersion: 9.2.51-closedown0 + helmVersion: 9.2.53-closedown0 {{- if hasKey . "labels" }} {{ toYaml .labels | indent 8 }} {{- end }} diff --git a/helm/hpcc/templates/esp.yaml b/helm/hpcc/templates/esp.yaml index 2312ed4aebc..ab736c5e1a5 100644 --- a/helm/hpcc/templates/esp.yaml +++ b/helm/hpcc/templates/esp.yaml @@ -120,7 +120,7 @@ spec: accessSasha: "yes" {{- end }} app: {{ $application }} - helmVersion: 9.2.51-closedown0 + helmVersion: 9.2.53-closedown0 {{- include "hpcc.addStandardLabels" (dict "root" $ "name" $application "component" "esp" "instance" .name) | indent 8 }} {{- if hasKey $.Values.global "metrics" }} {{- include "hpcc.generateMetricsReporterLabel" $.Values.global.metrics | nindent 8 }} diff --git a/helm/hpcc/templates/localroxie.yaml b/helm/hpcc/templates/localroxie.yaml index 2df89dc5ba5..04fb38bfd9e 100644 --- a/helm/hpcc/templates/localroxie.yaml +++ b/helm/hpcc/templates/localroxie.yaml @@ -70,7 +70,7 @@ spec: server: {{ $servername | quote }} accessDali: "yes" accessEsp: "yes" - helmVersion: 9.2.51-closedown0 + helmVersion: 9.2.53-closedown0 {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "roxie-server" "name" "roxie" "instance" $roxie.name) | indent 8 }} {{- if hasKey . "labels" }} {{ toYaml .labels | indent 8 }} diff --git a/helm/hpcc/templates/roxie.yaml b/helm/hpcc/templates/roxie.yaml index eee4d56a741..e32f7f4fe81 100644 --- a/helm/hpcc/templates/roxie.yaml +++ b/helm/hpcc/templates/roxie.yaml @@ -120,7 +120,7 @@ spec: {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "topology-server" "name" "roxie" "instance" $commonCtx.toponame) | indent 8 }} run: {{ $commonCtx.toponame | quote }} roxie-cluster: {{ $roxie.name | quote }} - helmVersion: 9.2.51-closedown0 + helmVersion: 9.2.53-closedown0 {{- if hasKey $.Values.global "metrics" }} {{- include "hpcc.generateMetricsReporterLabel" $.Values.global.metrics | nindent 8}} {{- end }} @@ -178,7 +178,7 @@ kind: Service metadata: name: {{ $commonCtx.toponame | quote }} labels: - helmVersion: 9.2.51-closedown0 + helmVersion: 9.2.53-closedown0 {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "topology-server" "name" "roxie" "instance" $commonCtx.toponame) | indent 4 }} spec: ports: @@ -240,7 +240,7 @@ spec: roxie-cluster: {{ $roxie.name | quote }} accessDali: "yes" accessEsp: "yes" - helmVersion: 9.2.51-closedown0 + helmVersion: 9.2.53-closedown0 {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "roxie-server" "name" "roxie" "instance" $servername) | indent 8 }} {{- if hasKey $.Values.global "metrics" }} {{- include "hpcc.generateMetricsReporterLabel" $.Values.global.metrics | nindent 8}} @@ -346,7 +346,7 @@ spec: roxie-cluster: {{ $roxie.name | quote }} accessDali: "yes" accessEsp: "yes" - helmVersion: 9.2.51-closedown0 + helmVersion: 9.2.53-closedown0 {{- if hasKey $.Values.global "metrics" }} {{- include "hpcc.generateMetricsReporterLabel" $.Values.global.metrics | nindent 8}} {{- end }} diff --git a/helm/hpcc/templates/sasha.yaml b/helm/hpcc/templates/sasha.yaml index 85d0e2e8ee0..252a5a7d7ab 100644 --- a/helm/hpcc/templates/sasha.yaml +++ b/helm/hpcc/templates/sasha.yaml @@ -53,7 +53,7 @@ spec: server: {{ $serviceName | quote }} app: sasha accessDali: {{ (has "dali" $sasha.access) | ternary "yes" "no" | quote }} - helmVersion: 9.2.51-closedown0 + helmVersion: 9.2.53-closedown0 {{- if hasKey $sasha "labels" }} {{ toYaml $sasha.labels | indent 8 }} {{- end }} diff --git a/helm/hpcc/templates/thor.yaml b/helm/hpcc/templates/thor.yaml index 2ede2631ded..732ef1bea96 100644 --- a/helm/hpcc/templates/thor.yaml +++ b/helm/hpcc/templates/thor.yaml @@ -82,7 +82,7 @@ data: labels: accessDali: "yes" accessEsp: "yes" - helmVersion: 9.2.51-closedown0 + helmVersion: 9.2.53-closedown0 {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "eclagent" "name" "thor" "instance" $eclAgentJobName "instanceOf" (printf "%s-job" .eclAgentName)) | indent 8 }} {{- if hasKey .me "labels" }} {{ toYaml .me.labels | indent 12 }} @@ -145,7 +145,7 @@ data: accessEsp: "yes" app: "thor" component: "thormanager" - helmVersion: 9.2.51-closedown0 + helmVersion: 9.2.53-closedown0 instance: "_HPCC_JOBNAME_" job: "_HPCC_JOBNAME_" {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "thormanager" "name" "thor" "instance" $thorManagerJobName "instanceOf" (printf "%s-thormanager-job" .me.name)) | indent 12 }} @@ -210,7 +210,7 @@ data: accessEsp: "yes" app: "thor" component: "thorworker" - helmVersion: 9.2.51-closedown0 + helmVersion: 9.2.53-closedown0 instance: "_HPCC_JOBNAME_" job: "_HPCC_JOBNAME_" {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "thorworker" "name" "thor" "instance" $thorWorkerJobName "instanceOf" (printf "%s-thorworker-job" .me.name)) | indent 12 }} @@ -341,7 +341,7 @@ spec: accessEsp: {{ $commonCtx.eclAgentUseChildProcesses | ternary "yes" "no" | quote }} app: "thor" component: "thor-eclagent" - helmVersion: 9.2.51-closedown0 + helmVersion: 9.2.53-closedown0 instance: {{ $commonCtx.eclAgentName | quote }} {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "eclagent" "name" "thor" "instance" $commonCtx.eclAgentName ) | indent 8 }} {{- if hasKey $commonCtx.me "labels" }} @@ -404,7 +404,7 @@ spec: accessEsp: "no" app: "thor" component: "thor-thoragent" - helmVersion: 9.2.51-closedown0 + helmVersion: 9.2.53-closedown0 instance: {{ $commonCtx.thorAgentName | quote }} {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "eclagent" "name" "thor" "instance" $commonCtx.thorAgentName ) | indent 8 }} {{- if hasKey $commonCtx.me "labels" }} diff --git a/version.cmake b/version.cmake index a59cd11b5c4..046a2d562cf 100644 --- a/version.cmake +++ b/version.cmake @@ -5,8 +5,8 @@ set ( HPCC_NAME "Community Edition" ) set ( HPCC_PROJECT "community" ) set ( HPCC_MAJOR 9 ) set ( HPCC_MINOR 2 ) -set ( HPCC_POINT 51 ) +set ( HPCC_POINT 53 ) set ( HPCC_MATURITY "closedown" ) set ( HPCC_SEQUENCE 0 ) -set ( HPCC_TAG_TIMESTAMP "2024-01-04T16:58:20Z" ) +set ( HPCC_TAG_TIMESTAMP "2024-01-11T17:49:06Z" ) ### From 1d8e10081f9c2b51520f57b625b822052b3f2a28 Mon Sep 17 00:00:00 2001 From: Jake Smith Date: Thu, 11 Jan 2024 17:53:31 +0000 Subject: [PATCH 13/32] Split off 9.4.26 Signed-off-by: Jake Smith --- helm/hpcc/Chart.yaml | 4 ++-- helm/hpcc/templates/_helpers.tpl | 2 +- helm/hpcc/templates/dafilesrv.yaml | 2 +- helm/hpcc/templates/dali.yaml | 2 +- helm/hpcc/templates/dfuserver.yaml | 2 +- helm/hpcc/templates/eclagent.yaml | 4 ++-- helm/hpcc/templates/eclccserver.yaml | 4 ++-- helm/hpcc/templates/eclscheduler.yaml | 2 +- helm/hpcc/templates/esp.yaml | 2 +- helm/hpcc/templates/localroxie.yaml | 2 +- helm/hpcc/templates/roxie.yaml | 8 ++++---- helm/hpcc/templates/sasha.yaml | 2 +- helm/hpcc/templates/thor.yaml | 10 +++++----- version.cmake | 4 ++-- 14 files changed, 25 insertions(+), 25 deletions(-) diff --git a/helm/hpcc/Chart.yaml b/helm/hpcc/Chart.yaml index 5d49e863f40..2bcfaf6b174 100644 --- a/helm/hpcc/Chart.yaml +++ b/helm/hpcc/Chart.yaml @@ -6,9 +6,9 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. -version: 9.4.25-closedown0 +version: 9.4.27-closedown0 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. -appVersion: 9.4.25-closedown0 +appVersion: 9.4.27-closedown0 diff --git a/helm/hpcc/templates/_helpers.tpl b/helm/hpcc/templates/_helpers.tpl index a7781ad7bd0..cfd602f3881 100644 --- a/helm/hpcc/templates/_helpers.tpl +++ b/helm/hpcc/templates/_helpers.tpl @@ -1452,7 +1452,7 @@ kind: Service metadata: name: {{ $lvars.serviceName | quote }} labels: - helmVersion: 9.4.25-closedown0 + helmVersion: 9.4.27-closedown0 {{- include "hpcc.addStandardLabels" (dict "root" $.root "instance" $lvars.serviceName ) | indent 4 }} {{- if $lvars.labels }} {{ toYaml $lvars.labels | indent 4 }} diff --git a/helm/hpcc/templates/dafilesrv.yaml b/helm/hpcc/templates/dafilesrv.yaml index d3d43cfccd6..04e5e6b2d88 100644 --- a/helm/hpcc/templates/dafilesrv.yaml +++ b/helm/hpcc/templates/dafilesrv.yaml @@ -51,7 +51,7 @@ spec: labels: {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "dafilesrv" "name" "dafilesrv" "instance" .name) | indent 8 }} server: {{ .name | quote }} - helmVersion: 9.4.25-closedown0 + helmVersion: 9.4.27-closedown0 annotations: checksum/config: {{ $configSHA }} {{- include "hpcc.generateAnnotations" $commonCtx | indent 8 }} diff --git a/helm/hpcc/templates/dali.yaml b/helm/hpcc/templates/dali.yaml index f83c7861ab3..38e678a8afa 100644 --- a/helm/hpcc/templates/dali.yaml +++ b/helm/hpcc/templates/dali.yaml @@ -84,7 +84,7 @@ spec: run: {{ $dali.name | quote }} server: {{ $dali.name | quote }} app: dali - helmVersion: 9.4.25-closedown0 + helmVersion: 9.4.27-closedown0 {{- if hasKey $.Values.global "metrics" }} {{- include "hpcc.generateMetricsReporterLabel" $.Values.global.metrics | nindent 8 }} {{- end }} diff --git a/helm/hpcc/templates/dfuserver.yaml b/helm/hpcc/templates/dfuserver.yaml index 27e80e24e09..99a8a21da93 100644 --- a/helm/hpcc/templates/dfuserver.yaml +++ b/helm/hpcc/templates/dfuserver.yaml @@ -57,7 +57,7 @@ spec: {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "dfuserver" "name" "dfuserver" "instance" .name) | indent 8 }} run: {{ .name | quote }} accessDali: "yes" - helmVersion: 9.4.25-closedown0 + helmVersion: 9.4.27-closedown0 {{- if hasKey . "labels" }} {{ toYaml .labels | indent 8 }} {{- end }} diff --git a/helm/hpcc/templates/eclagent.yaml b/helm/hpcc/templates/eclagent.yaml index c8c8bd3fdb4..9c5627c1ea0 100644 --- a/helm/hpcc/templates/eclagent.yaml +++ b/helm/hpcc/templates/eclagent.yaml @@ -60,7 +60,7 @@ data: {{- include "hpcc.addStandardLabels" (dict "root" $ "component" $apptype "name" "eclagent" "instance" $appJobName "instanceOf" (printf "%s-job" .me.name)) | indent 12 }} accessDali: "yes" accessEsp: "yes" - helmVersion: 9.4.25-closedown0 + helmVersion: 9.4.27-closedown0 {{- if hasKey .me "labels" }} {{ toYaml .me.labels | indent 12 }} {{- end }} @@ -137,7 +137,7 @@ spec: run: {{ .name | quote }} accessDali: "yes" accessEsp: {{ .useChildProcesses | default false | ternary "yes" "no" | quote }} - helmVersion: 9.4.25-closedown0 + helmVersion: 9.4.27-closedown0 {{- if hasKey . "labels" }} {{ toYaml .labels | indent 8 }} {{- end }} diff --git a/helm/hpcc/templates/eclccserver.yaml b/helm/hpcc/templates/eclccserver.yaml index e02aaab02bd..c3ba3c9d661 100644 --- a/helm/hpcc/templates/eclccserver.yaml +++ b/helm/hpcc/templates/eclccserver.yaml @@ -58,7 +58,7 @@ data: {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "eclccserver" "name" "eclccserver" "instance" $compileJobName "instanceOf" (printf "%s-job" .me.name)) | indent 12 }} accessDali: "yes" accessEsp: "yes" - helmVersion: 9.4.25-closedown0 + helmVersion: 9.4.27-closedown0 {{- if hasKey .me "labels" }} {{ toYaml .me.labels | indent 12 }} {{- end }} @@ -143,7 +143,7 @@ spec: run: {{ .name | quote }} accessDali: "yes" accessEsp: {{ .useChildProcesses | default false | ternary "yes" "no" | quote }} - helmVersion: 9.4.25-closedown0 + helmVersion: 9.4.27-closedown0 {{- if hasKey . "labels" }} {{ toYaml .labels | indent 8 }} {{- end }} diff --git a/helm/hpcc/templates/eclscheduler.yaml b/helm/hpcc/templates/eclscheduler.yaml index dc7c070af10..e1211405d9e 100644 --- a/helm/hpcc/templates/eclscheduler.yaml +++ b/helm/hpcc/templates/eclscheduler.yaml @@ -65,7 +65,7 @@ spec: run: {{ .name | quote }} accessDali: "yes" accessEsp: "no" - helmVersion: 9.4.25-closedown0 + helmVersion: 9.4.27-closedown0 {{- if hasKey . "labels" }} {{ toYaml .labels | indent 8 }} {{- end }} diff --git a/helm/hpcc/templates/esp.yaml b/helm/hpcc/templates/esp.yaml index 18f4b4c9be6..3ca318da32e 100644 --- a/helm/hpcc/templates/esp.yaml +++ b/helm/hpcc/templates/esp.yaml @@ -122,7 +122,7 @@ spec: accessSasha: "yes" {{- end }} app: {{ $application }} - helmVersion: 9.4.25-closedown0 + helmVersion: 9.4.27-closedown0 {{- include "hpcc.addStandardLabels" (dict "root" $ "name" $application "component" "esp" "instance" .name) | indent 8 }} {{- if hasKey $.Values.global "metrics" }} {{- include "hpcc.generateMetricsReporterLabel" $.Values.global.metrics | nindent 8 }} diff --git a/helm/hpcc/templates/localroxie.yaml b/helm/hpcc/templates/localroxie.yaml index 0e64aaa029d..ab03f0e15d6 100644 --- a/helm/hpcc/templates/localroxie.yaml +++ b/helm/hpcc/templates/localroxie.yaml @@ -73,7 +73,7 @@ spec: server: {{ $servername | quote }} accessDali: "yes" accessEsp: "yes" - helmVersion: 9.4.25-closedown0 + helmVersion: 9.4.27-closedown0 {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "roxie-server" "name" "roxie" "instance" $roxie.name) | indent 8 }} {{- if hasKey . "labels" }} {{ toYaml .labels | indent 8 }} diff --git a/helm/hpcc/templates/roxie.yaml b/helm/hpcc/templates/roxie.yaml index 5b015bec770..03a66d34c9c 100644 --- a/helm/hpcc/templates/roxie.yaml +++ b/helm/hpcc/templates/roxie.yaml @@ -125,7 +125,7 @@ spec: {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "topology-server" "name" "roxie" "instance" $commonCtx.toponame) | indent 8 }} run: {{ $commonCtx.toponame | quote }} roxie-cluster: {{ $roxie.name | quote }} - helmVersion: 9.4.25-closedown0 + helmVersion: 9.4.27-closedown0 {{- if hasKey $.Values.global "metrics" }} {{- include "hpcc.generateMetricsReporterLabel" $.Values.global.metrics | nindent 8}} {{- end }} @@ -183,7 +183,7 @@ kind: Service metadata: name: {{ $commonCtx.toponame | quote }} labels: - helmVersion: 9.4.25-closedown0 + helmVersion: 9.4.27-closedown0 {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "topology-server" "name" "roxie" "instance" $commonCtx.toponame) | indent 4 }} spec: ports: @@ -245,7 +245,7 @@ spec: roxie-cluster: {{ $roxie.name | quote }} accessDali: "yes" accessEsp: "yes" - helmVersion: 9.4.25-closedown0 + helmVersion: 9.4.27-closedown0 {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "roxie-server" "name" "roxie" "instance" $servername) | indent 8 }} {{- if hasKey $.Values.global "metrics" }} {{- include "hpcc.generateMetricsReporterLabel" $.Values.global.metrics | nindent 8}} @@ -353,7 +353,7 @@ spec: roxie-cluster: {{ $roxie.name | quote }} accessDali: "yes" accessEsp: "yes" - helmVersion: 9.4.25-closedown0 + helmVersion: 9.4.27-closedown0 {{- if hasKey $.Values.global "metrics" }} {{- include "hpcc.generateMetricsReporterLabel" $.Values.global.metrics | nindent 8}} {{- end }} diff --git a/helm/hpcc/templates/sasha.yaml b/helm/hpcc/templates/sasha.yaml index d0422146526..baa6c0b5344 100644 --- a/helm/hpcc/templates/sasha.yaml +++ b/helm/hpcc/templates/sasha.yaml @@ -53,7 +53,7 @@ spec: server: {{ $serviceName | quote }} app: sasha accessDali: {{ (has "dali" $sasha.access) | ternary "yes" "no" | quote }} - helmVersion: 9.4.25-closedown0 + helmVersion: 9.4.27-closedown0 {{- if hasKey $sasha "labels" }} {{ toYaml $sasha.labels | indent 8 }} {{- end }} diff --git a/helm/hpcc/templates/thor.yaml b/helm/hpcc/templates/thor.yaml index e80d73c44cb..2b8d6a367da 100644 --- a/helm/hpcc/templates/thor.yaml +++ b/helm/hpcc/templates/thor.yaml @@ -86,7 +86,7 @@ data: labels: accessDali: "yes" accessEsp: "yes" - helmVersion: 9.4.25-closedown0 + helmVersion: 9.4.27-closedown0 {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "eclagent" "name" "thor" "instance" $eclAgentJobName "instanceOf" (printf "%s-job" .eclAgentName)) | indent 8 }} {{- if hasKey .me "labels" }} {{ toYaml .me.labels | indent 12 }} @@ -151,7 +151,7 @@ data: accessEsp: "yes" app: "thor" component: "thormanager" - helmVersion: 9.4.25-closedown0 + helmVersion: 9.4.27-closedown0 instance: "_HPCC_JOBNAME_" job: "_HPCC_JOBNAME_" {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "thormanager" "name" "thor" "instance" $thorManagerJobName "instanceOf" (printf "%s-thormanager-job" .me.name)) | indent 12 }} @@ -218,7 +218,7 @@ data: accessEsp: "yes" app: "thor" component: "thorworker" - helmVersion: 9.4.25-closedown0 + helmVersion: 9.4.27-closedown0 instance: "_HPCC_JOBNAME_" job: "_HPCC_JOBNAME_" {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "thorworker" "name" "thor" "instance" $thorWorkerJobName "instanceOf" (printf "%s-thorworker-job" .me.name)) | indent 12 }} @@ -351,7 +351,7 @@ spec: accessEsp: {{ $commonCtx.eclAgentUseChildProcesses | ternary "yes" "no" | quote }} app: "thor" component: "thor-eclagent" - helmVersion: 9.4.25-closedown0 + helmVersion: 9.4.27-closedown0 instance: {{ $commonCtx.eclAgentName | quote }} {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "eclagent" "name" "thor" "instance" $commonCtx.eclAgentName ) | indent 8 }} {{- if hasKey $commonCtx.me "labels" }} @@ -416,7 +416,7 @@ spec: accessEsp: "no" app: "thor" component: "thor-thoragent" - helmVersion: 9.4.25-closedown0 + helmVersion: 9.4.27-closedown0 instance: {{ $commonCtx.thorAgentName | quote }} {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "eclagent" "name" "thor" "instance" $commonCtx.thorAgentName ) | indent 8 }} {{- if hasKey $commonCtx.me "labels" }} diff --git a/version.cmake b/version.cmake index 45cfc723054..4558d41dba5 100644 --- a/version.cmake +++ b/version.cmake @@ -5,8 +5,8 @@ set ( HPCC_NAME "Community Edition" ) set ( HPCC_PROJECT "community" ) set ( HPCC_MAJOR 9 ) set ( HPCC_MINOR 4 ) -set ( HPCC_POINT 25 ) +set ( HPCC_POINT 27 ) set ( HPCC_MATURITY "closedown" ) set ( HPCC_SEQUENCE 0 ) -set ( HPCC_TAG_TIMESTAMP "2024-01-04T16:59:34Z" ) +set ( HPCC_TAG_TIMESTAMP "2024-01-11T17:53:30Z" ) ### From 36db927bdbbdc594282bf1f6bbd9294c834c6766 Mon Sep 17 00:00:00 2001 From: Jake Smith Date: Thu, 11 Jan 2024 17:55:48 +0000 Subject: [PATCH 14/32] Split off 8.12.84 Signed-off-by: Jake Smith --- helm/hpcc/Chart.yaml | 4 ++-- helm/hpcc/templates/_helpers.tpl | 2 +- helm/hpcc/templates/dafilesrv.yaml | 2 +- helm/hpcc/templates/dali.yaml | 2 +- helm/hpcc/templates/dfuserver.yaml | 2 +- helm/hpcc/templates/eclagent.yaml | 4 ++-- helm/hpcc/templates/eclccserver.yaml | 4 ++-- helm/hpcc/templates/eclscheduler.yaml | 2 +- helm/hpcc/templates/esp.yaml | 2 +- helm/hpcc/templates/localroxie.yaml | 2 +- helm/hpcc/templates/roxie.yaml | 8 ++++---- helm/hpcc/templates/sasha.yaml | 2 +- helm/hpcc/templates/thor.yaml | 10 +++++----- version.cmake | 4 ++-- 14 files changed, 25 insertions(+), 25 deletions(-) diff --git a/helm/hpcc/Chart.yaml b/helm/hpcc/Chart.yaml index 5c66e0b4187..ed82a20d9a5 100644 --- a/helm/hpcc/Chart.yaml +++ b/helm/hpcc/Chart.yaml @@ -6,9 +6,9 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. -version: 8.12.83-closedown0 +version: 8.12.85-closedown0 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. -appVersion: 8.12.83-closedown0 +appVersion: 8.12.85-closedown0 diff --git a/helm/hpcc/templates/_helpers.tpl b/helm/hpcc/templates/_helpers.tpl index 82044e42d77..e762231af0f 100644 --- a/helm/hpcc/templates/_helpers.tpl +++ b/helm/hpcc/templates/_helpers.tpl @@ -1240,7 +1240,7 @@ kind: Service metadata: name: {{ $lvars.serviceName | quote }} labels: - helmVersion: 8.12.83-closedown0 + helmVersion: 8.12.85-closedown0 {{- include "hpcc.addStandardLabels" (dict "root" $.root "instance" $lvars.serviceName ) | indent 4 }} {{- if $lvars.labels }} {{ toYaml $lvars.labels | indent 4 }} diff --git a/helm/hpcc/templates/dafilesrv.yaml b/helm/hpcc/templates/dafilesrv.yaml index 24915904a3b..8e41bf25e4b 100644 --- a/helm/hpcc/templates/dafilesrv.yaml +++ b/helm/hpcc/templates/dafilesrv.yaml @@ -50,7 +50,7 @@ spec: labels: {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "dafilesrv" "name" "dafilesrv" "instance" .name) | indent 8 }} server: {{ .name | quote }} - helmVersion: 8.12.83-closedown0 + helmVersion: 8.12.85-closedown0 annotations: checksum/config: {{ $configSHA }} spec: diff --git a/helm/hpcc/templates/dali.yaml b/helm/hpcc/templates/dali.yaml index 277d1837157..e17a44795f5 100644 --- a/helm/hpcc/templates/dali.yaml +++ b/helm/hpcc/templates/dali.yaml @@ -82,7 +82,7 @@ spec: run: {{ $dali.name | quote }} server: {{ $dali.name | quote }} app: dali - helmVersion: 8.12.83-closedown0 + helmVersion: 8.12.85-closedown0 {{- if hasKey $.Values.global "metrics" }} {{- include "hpcc.generateMetricsReporterLabel" $.Values.global.metrics | nindent 8 }} {{- end }} diff --git a/helm/hpcc/templates/dfuserver.yaml b/helm/hpcc/templates/dfuserver.yaml index 196f3d112d4..a45c78a84ce 100644 --- a/helm/hpcc/templates/dfuserver.yaml +++ b/helm/hpcc/templates/dfuserver.yaml @@ -56,7 +56,7 @@ spec: {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "dfuserver" "name" "dfuserver" "instance" .name) | indent 8 }} run: {{ .name | quote }} accessDali: "yes" - helmVersion: 8.12.83-closedown0 + helmVersion: 8.12.85-closedown0 {{- if hasKey . "labels" }} {{ toYaml .labels | indent 8 }} {{- end }} diff --git a/helm/hpcc/templates/eclagent.yaml b/helm/hpcc/templates/eclagent.yaml index b42445d29db..985f2325921 100644 --- a/helm/hpcc/templates/eclagent.yaml +++ b/helm/hpcc/templates/eclagent.yaml @@ -58,7 +58,7 @@ data: {{- include "hpcc.addStandardLabels" (dict "root" $ "component" $apptype "name" "eclagent" "instance" $appJobName "instanceOf" (printf "%s-job" .me.name)) | indent 12 }} accessDali: "yes" accessEsp: "yes" - helmVersion: 8.12.83-closedown0 + helmVersion: 8.12.85-closedown0 {{- if hasKey .me "labels" }} {{ toYaml .me.labels | indent 12 }} {{- end }} @@ -137,7 +137,7 @@ spec: run: {{ .name | quote }} accessDali: "yes" accessEsp: {{ .useChildProcesses | default false | ternary "yes" "no" | quote }} - helmVersion: 8.12.83-closedown0 + helmVersion: 8.12.85-closedown0 {{- if hasKey . "labels" }} {{ toYaml .labels | indent 8 }} {{- end }} diff --git a/helm/hpcc/templates/eclccserver.yaml b/helm/hpcc/templates/eclccserver.yaml index c875cac745b..b5e7e3c0683 100644 --- a/helm/hpcc/templates/eclccserver.yaml +++ b/helm/hpcc/templates/eclccserver.yaml @@ -57,7 +57,7 @@ data: {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "eclccserver" "name" "eclccserver" "instance" $compileJobName "instanceOf" (printf "%s-job" .me.name)) | indent 12 }} accessDali: "yes" accessEsp: "yes" - helmVersion: 8.12.83-closedown0 + helmVersion: 8.12.85-closedown0 {{- if hasKey .me "labels" }} {{ toYaml .me.labels | indent 12 }} {{- end }} @@ -142,7 +142,7 @@ spec: run: {{ .name | quote }} accessDali: "yes" accessEsp: {{ .useChildProcesses | default false | ternary "yes" "no" | quote }} - helmVersion: 8.12.83-closedown0 + helmVersion: 8.12.85-closedown0 {{- if hasKey . "labels" }} {{ toYaml .labels | indent 8 }} {{- end }} diff --git a/helm/hpcc/templates/eclscheduler.yaml b/helm/hpcc/templates/eclscheduler.yaml index f4abef54d15..58b6fcdb7cb 100644 --- a/helm/hpcc/templates/eclscheduler.yaml +++ b/helm/hpcc/templates/eclscheduler.yaml @@ -64,7 +64,7 @@ spec: run: {{ .name | quote }} accessDali: "yes" accessEsp: "no" - helmVersion: 8.12.83-closedown0 + helmVersion: 8.12.85-closedown0 {{- if hasKey . "labels" }} {{ toYaml .labels | indent 8 }} {{- end }} diff --git a/helm/hpcc/templates/esp.yaml b/helm/hpcc/templates/esp.yaml index dcd26219b10..0fad67ccb88 100644 --- a/helm/hpcc/templates/esp.yaml +++ b/helm/hpcc/templates/esp.yaml @@ -117,7 +117,7 @@ spec: server: {{ .name | quote }} accessDali: "yes" app: {{ $application }} - helmVersion: 8.12.83-closedown0 + helmVersion: 8.12.85-closedown0 {{- include "hpcc.addStandardLabels" (dict "root" $ "name" $application "component" "esp" "instance" .name) | indent 8 }} {{- if hasKey $.Values.global "metrics" }} {{- include "hpcc.generateMetricsReporterLabel" $.Values.global.metrics | nindent 8 }} diff --git a/helm/hpcc/templates/localroxie.yaml b/helm/hpcc/templates/localroxie.yaml index e0a475866e4..b46b2bdf471 100644 --- a/helm/hpcc/templates/localroxie.yaml +++ b/helm/hpcc/templates/localroxie.yaml @@ -70,7 +70,7 @@ spec: server: {{ $servername | quote }} accessDali: "yes" accessEsp: "yes" - helmVersion: 8.12.83-closedown0 + helmVersion: 8.12.85-closedown0 {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "roxie-server" "name" "roxie" "instance" $roxie.name) | indent 8 }} {{- if hasKey . "labels" }} {{ toYaml .labels | indent 8 }} diff --git a/helm/hpcc/templates/roxie.yaml b/helm/hpcc/templates/roxie.yaml index aebc380a231..3ade79ea3a1 100644 --- a/helm/hpcc/templates/roxie.yaml +++ b/helm/hpcc/templates/roxie.yaml @@ -120,7 +120,7 @@ spec: {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "topology-server" "name" "roxie" "instance" $commonCtx.toponame) | indent 8 }} run: {{ $commonCtx.toponame | quote }} roxie-cluster: {{ $roxie.name | quote }} - helmVersion: 8.12.83-closedown0 + helmVersion: 8.12.85-closedown0 {{- if hasKey $.Values.global "metrics" }} {{- include "hpcc.generateMetricsReporterLabel" $.Values.global.metrics | nindent 8}} {{- end }} @@ -180,7 +180,7 @@ kind: Service metadata: name: {{ $commonCtx.toponame | quote }} labels: - helmVersion: 8.12.83-closedown0 + helmVersion: 8.12.85-closedown0 {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "topology-server" "name" "roxie" "instance" $commonCtx.toponame) | indent 4 }} spec: ports: @@ -242,7 +242,7 @@ spec: roxie-cluster: {{ $roxie.name | quote }} accessDali: "yes" accessEsp: "yes" - helmVersion: 8.12.83-closedown0 + helmVersion: 8.12.85-closedown0 {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "roxie-server" "name" "roxie" "instance" $servername) | indent 8 }} {{- if hasKey $.Values.global "metrics" }} {{- include "hpcc.generateMetricsReporterLabel" $.Values.global.metrics | nindent 8}} @@ -347,7 +347,7 @@ spec: roxie-cluster: {{ $roxie.name | quote }} accessDali: "yes" accessEsp: "yes" - helmVersion: 8.12.83-closedown0 + helmVersion: 8.12.85-closedown0 {{- if hasKey $.Values.global "metrics" }} {{- include "hpcc.generateMetricsReporterLabel" $.Values.global.metrics | nindent 8}} {{- end }} diff --git a/helm/hpcc/templates/sasha.yaml b/helm/hpcc/templates/sasha.yaml index 652c69f38ff..7d6a937eb6a 100644 --- a/helm/hpcc/templates/sasha.yaml +++ b/helm/hpcc/templates/sasha.yaml @@ -52,7 +52,7 @@ spec: run: {{ $serviceName | quote }} server: {{ $serviceName | quote }} accessDali: {{ (has "dali" $sasha.access) | ternary "yes" "no" | quote }} - helmVersion: 8.12.83-closedown0 + helmVersion: 8.12.85-closedown0 {{- if hasKey $sasha "labels" }} {{ toYaml $sasha.labels | indent 8 }} {{- end }} diff --git a/helm/hpcc/templates/thor.yaml b/helm/hpcc/templates/thor.yaml index 2fa220712a3..3b8ed70d34e 100644 --- a/helm/hpcc/templates/thor.yaml +++ b/helm/hpcc/templates/thor.yaml @@ -82,7 +82,7 @@ data: labels: accessDali: "yes" accessEsp: "yes" - helmVersion: 8.12.83-closedown0 + helmVersion: 8.12.85-closedown0 {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "eclagent" "name" "thor" "instance" $eclAgentJobName "instanceOf" (printf "%s-job" .eclAgentName)) | indent 8 }} {{- if hasKey .me "labels" }} {{ toYaml .me.labels | indent 12 }} @@ -149,7 +149,7 @@ data: accessEsp: "yes" app: "thor" component: "thormanager" - helmVersion: 8.12.83-closedown0 + helmVersion: 8.12.85-closedown0 instance: "_HPCC_JOBNAME_" job: "_HPCC_JOBNAME_" {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "thormanager" "name" "thor" "instance" $thorManagerJobName "instanceOf" (printf "%s-thormanager-job" .me.name)) | indent 12 }} @@ -218,7 +218,7 @@ data: accessEsp: "yes" app: "thor" component: "thorworker" - helmVersion: 8.12.83-closedown0 + helmVersion: 8.12.85-closedown0 instance: "_HPCC_JOBNAME_" job: "_HPCC_JOBNAME_" {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "thorworker" "name" "thor" "instance" $thorWorkerJobName "instanceOf" (printf "%s-thorworker-job" .me.name)) | indent 12 }} @@ -353,7 +353,7 @@ spec: accessEsp: {{ $commonCtx.eclAgentUseChildProcesses | ternary "yes" "no" | quote }} app: "thor" component: "thor-eclagent" - helmVersion: 8.12.83-closedown0 + helmVersion: 8.12.85-closedown0 instance: {{ $commonCtx.eclAgentName | quote }} {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "eclagent" "name" "thor" "instance" $commonCtx.eclAgentName ) | indent 8 }} {{- if hasKey $commonCtx.me "labels" }} @@ -418,7 +418,7 @@ spec: accessEsp: "no" app: "thor" component: "thor-thoragent" - helmVersion: 8.12.83-closedown0 + helmVersion: 8.12.85-closedown0 instance: {{ $commonCtx.thorAgentName | quote }} {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "eclagent" "name" "thor" "instance" $commonCtx.thorAgentName ) | indent 8 }} {{- if hasKey $commonCtx.me "labels" }} diff --git a/version.cmake b/version.cmake index 2dc80a3f7bc..29cfdff36ee 100644 --- a/version.cmake +++ b/version.cmake @@ -5,8 +5,8 @@ set ( HPCC_NAME "Community Edition" ) set ( HPCC_PROJECT "community" ) set ( HPCC_MAJOR 8 ) set ( HPCC_MINOR 12 ) -set ( HPCC_POINT 83 ) +set ( HPCC_POINT 85 ) set ( HPCC_MATURITY "closedown" ) set ( HPCC_SEQUENCE 0 ) -set ( HPCC_TAG_TIMESTAMP "2024-01-04T16:55:21Z" ) +set ( HPCC_TAG_TIMESTAMP "2024-01-11T17:55:48Z" ) ### From ebcab16801a4e7b2d64434a82e82c866fac58250 Mon Sep 17 00:00:00 2001 From: Michael Gardner Date: Thu, 11 Jan 2024 12:50:35 -0500 Subject: [PATCH 15/32] HPCC-30887 Add Enterprise builds to build-assets in 9.4 Signed-off-by: Michael Gardner --- .github/workflows/build-assets.yml | 44 +++++++++++++++++++++++++----- 1 file changed, 37 insertions(+), 7 deletions(-) diff --git a/.github/workflows/build-assets.yml b/.github/workflows/build-assets.yml index 07cc9c97392..3a417614141 100644 --- a/.github/workflows/build-assets.yml +++ b/.github/workflows/build-assets.yml @@ -112,6 +112,9 @@ jobs: - os: ubuntu-22.04 name: LN ln: true + - os: ubuntu-22.04 + name: Enterprise + ee: true - os: ubuntu-20.04 name: LN ln: true @@ -138,7 +141,7 @@ jobs: path: ${{ needs.preamble.outputs.folder_platform }} - name: Checkout LN - if: ${{ matrix.ln }} + if: ${{ matrix.ln || matrix.ee }} uses: actions/checkout@v3 with: repository: ${{ github.repository_owner }}/LN @@ -192,7 +195,7 @@ jobs: # Communtiy Build - name: CMake Packages (community) - if: ${{ !matrix.ln && !matrix.container && !matrix.documentation }} + if: ${{ !matrix.ln && !matrix.ee && !matrix.container && !matrix.documentation }} run: | mkdir -p ${{ needs.preamble.outputs.folder_build }} echo "${{ secrets.SIGNING_SECRET }}" > ${{ needs.preamble.outputs.folder_build }}/private.key @@ -209,7 +212,7 @@ jobs: done - name: CMake Containerized Packages (community) - if: ${{ !matrix.ln && matrix.container && !matrix.documentation }} + if: ${{ !matrix.ln && !matrix.ee && matrix.container && !matrix.documentation }} run: | mkdir -p ${{ needs.preamble.outputs.folder_build }} echo "${{ secrets.SIGNING_SECRET }}" > ${{ needs.preamble.outputs.folder_build }}/private.key @@ -223,7 +226,7 @@ jobs: cmake --build /hpcc-dev/build --parallel $(nproc) --target package" - name: CMake documentation (community) - if: ${{ !matrix.ln && !matrix.container && matrix.documentation }} + if: ${{ !matrix.ln && !matrix.ee && !matrix.container && matrix.documentation }} run: | mkdir -p {${{needs.preamble.outputs.folder_build }},EN_US,PT_BR} sudo rm -f ${{ needs.preamble.outputs.folder_build }}/CMakeCache.txt @@ -235,7 +238,7 @@ jobs: docker run --rm --mount ${{ needs.preamble.outputs.mount_platform }} --mount ${{ needs.preamble.outputs.mount_build }} ${{ steps.vars.outputs.docker_tag_candidate_base }} "cd /hpcc-dev/build/Release/docs/PT_BR && zip ALL_HPCC_DOCS_PT_BR-${{ needs.preamble.outputs.community_tag }}.zip *.pdf" - name: Upload Assets (community) - if: ${{ !matrix.ln }} + if: ${{ !matrix.ln && !matrix.ee }} uses: ncipollo/release-action@v1.12.0 with: allowUpdates: true @@ -244,7 +247,7 @@ jobs: artifacts: "${{ needs.preamble.outputs.folder_build }}/*.deb,${{ needs.preamble.outputs.folder_build }}/*.rpm,${{ needs.preamble.outputs.folder_build }}/Release/docs/*.zip,${{ needs.preamble.outputs.folder_build }}/Release/docs/EN_US/*.zip,${{ needs.preamble.outputs.folder_build }}/Release/docs/PT_BR/*.zip,${{ needs.preamble.outputs.folder_build }}/docs/EN_US/EclipseHelp/*.zip,${{ needs.preamble.outputs.folder_build }}/docs/EN_US/HTMLHelp/*.zip,${{ needs.preamble.outputs.folder_build }}/docs/PT_BR/HTMLHelp/*.zip" - name: Locate k8s deb file (community) - if: ${{ !matrix.ln && matrix.container && !matrix.documentation }} + if: ${{ !matrix.ln && !matrix.ee && matrix.container && !matrix.documentation }} id: container run: | k8s_pkg_path=$(ls -t ${{ needs.preamble.outputs.folder_build }}/*64_k8s.deb 2>/dev/null | head -1) @@ -254,7 +257,7 @@ jobs: - name: Create Docker Image (community) uses: docker/build-push-action@v4 - if: ${{ !matrix.ln && matrix.container && !matrix.documentation }} + if: ${{ !matrix.ln && !matrix.ee && matrix.container && !matrix.documentation }} with: builder: ${{ steps.buildx.outputs.name }} file: ${{ needs.preamble.outputs.folder_platform }}/dockerfiles/vcpkg/platform-core-${{ matrix.os }}/Dockerfile @@ -360,6 +363,33 @@ jobs: jf docker push ${{ secrets.JFROG_REGISTRY || 'dummy.io' }}/hpccpl-docker-local/platform-core-ln:${{ needs.preamble.outputs.hpcc_version }} --build-name=platform-core-ln --build-number=${{ needs.preamble.outputs.hpcc_version }} --project=hpccpl jf rt bp platform-core-ln ${{ needs.preamble.outputs.hpcc_version }} --project=hpccpl + + # Enterprise Build --- + - name: CMake Packages (enterprise) + if: ${{ matrix.ee }} + run: | + mkdir -p ${{ needs.preamble.outputs.folder_build }} + echo "${{ secrets.SIGNING_SECRET }}" > ${{ needs.preamble.outputs.folder_build }}/private.key + sudo rm -f ${{ needs.preamble.outputs.folder_build }}/CMakeCache.txt + sudo rm -rf ${{ needs.preamble.outputs.folder_build }}/CMakeFiles + docker run --rm --mount ${{ needs.preamble.outputs.mount_platform }} --mount ${{ needs.preamble.outputs.mount_ln }} --mount ${{ needs.preamble.outputs.mount_build }} ${{ steps.vars.outputs.docker_tag_candidate_base }} "${{ needs.preamble.outputs.gpg_import }} && \ + cmake -S /hpcc-dev/LN -B /hpcc-dev/build -DHPCC_SOURCE_DIR=/hpcc-dev/HPCC-Platform ${{ needs.preamble.outputs.cmake_docker_config }} -DBUILD_LEVEL=ENTERPRISE -DSIGN_MODULES_PASSPHRASE=${{ secrets.SIGN_MODULES_PASSPHRASE }} -DSIGN_MODULES_KEYID=${{ secrets.SIGN_MODULES_KEYID }} -DPLATFORM=ON -DINCLUDE_PLUGINS=ON -DCONTAINERIZED=OFF -DSUPPRESS_REMBED=ON -DSUPPRESS_V8EMBED=ON -DSUPPRESS_SPARK=ON -DCPACK_STRIP_FILES=OFF && \ + cmake --build /hpcc-dev/build --parallel $(nproc) --target package" + + - name: Upload Assets (enterprise) + if: ${{ matrix.ee }} + uses: ncipollo/release-action@v1.12.0 + with: + allowUpdates: true + generateReleaseNotes: false + prerelease: ${{ contains(github.ref, '-rc') }} + owner: ${{ secrets.LNB_ACTOR }} + repo: LN + token: ${{ secrets.LNB_TOKEN }} + tag: ${{ needs.preamble.outputs.internal_tag }} + artifacts: "${{ needs.preamble.outputs.folder_build }}/hpccsystems-platform-enterprise*.deb,${{ needs.preamble.outputs.folder_build }}/hpccsystems-platform-enterprise*.rpm" + + # Common --- - name: Cleanup Environment if: always() From 32442049b70884de8e8a38b39c6bfb99fb537d3a Mon Sep 17 00:00:00 2001 From: Gavin Halliday Date: Fri, 12 Jan 2024 09:49:22 +0000 Subject: [PATCH 16/32] HPCC-31119 Fix problem loading roxie queries compiled with 9.4.20-24 Signed-off-by: Gavin Halliday --- rtl/eclrtl/eclhelper_base.cpp | 2 ++ rtl/include/eclhelper_base.hpp | 2 ++ 2 files changed, 4 insertions(+) diff --git a/rtl/eclrtl/eclhelper_base.cpp b/rtl/eclrtl/eclhelper_base.cpp index fac7194d86b..238840a21b9 100644 --- a/rtl/eclrtl/eclhelper_base.cpp +++ b/rtl/eclrtl/eclhelper_base.cpp @@ -620,6 +620,7 @@ const char * CThorSoapActionArg::getXpathHintsXml() { return nullptr;} const char * CThorSoapActionArg::getRequestHeader() { return nullptr; } const char * CThorSoapActionArg::getRequestFooter() { return nullptr; } unsigned CThorSoapActionArg::getPersistMaxRequests() { return 0; } +unsigned CThorSoapActionArg::getPersistPoolSize() { return 0; } //CThorSoapCallArg @@ -646,6 +647,7 @@ const char * CThorSoapCallArg::getXpathHintsXml() { return nullptr; } const char * CThorSoapCallArg::getRequestHeader() { return nullptr; } const char * CThorSoapCallArg::getRequestFooter() { return nullptr; } unsigned CThorSoapCallArg::getPersistMaxRequests() { return 0; } +unsigned CThorSoapCallArg::getPersistPoolSize() { return 0; } size32_t CThorSoapCallArg::onFailTransform(ARowBuilder & rowBuilder, const void * left, IException * e) { return 0; } void CThorSoapCallArg::getLogText(size32_t & lenText, char * & text, const void * left) { lenText =0; text = NULL; } diff --git a/rtl/include/eclhelper_base.hpp b/rtl/include/eclhelper_base.hpp index 0a0ef1c5e08..07dc23f8d4f 100644 --- a/rtl/include/eclhelper_base.hpp +++ b/rtl/include/eclhelper_base.hpp @@ -823,6 +823,7 @@ class ECLRTL_API CThorSoapActionArg : public CThorSinkArgOf virtual const char * getRequestHeader() override; virtual const char * getRequestFooter() override; virtual unsigned getPersistMaxRequests() override; + unsigned getPersistPoolSize(); // Included for backward compatibility to prevent linking problems }; class ECLRTL_API CThorSoapCallArg : public CThorArgOf @@ -854,6 +855,7 @@ class ECLRTL_API CThorSoapCallArg : public CThorArgOf virtual const char * getRequestHeader() override; virtual const char * getRequestFooter() override; virtual unsigned getPersistMaxRequests() override; + unsigned getPersistPoolSize(); // Included for backward compatibility to prevent linking problems }; typedef CThorSoapCallArg CThorHttpCallArg; From ece60e861bc9a38a1dfed01be42f18935e955d8e Mon Sep 17 00:00:00 2001 From: Jim DeFabia Date: Wed, 3 Jan 2024 20:52:39 -0500 Subject: [PATCH 17/32] HPCC-29378 Document Pipe Restrictions Signed-off-by: Jim DeFabia --- .../EN_US/ConfiguringHPCC/ConfiguringHPCC.xml | 56 ++++++++- .../ContainerizedMods/CustomConfig.xml | 119 ++++++++++++++++++ 2 files changed, 172 insertions(+), 3 deletions(-) diff --git a/docs/EN_US/ConfiguringHPCC/ConfiguringHPCC.xml b/docs/EN_US/ConfiguringHPCC/ConfiguringHPCC.xml index afff88f92d9..21fbc070187 100644 --- a/docs/EN_US/ConfiguringHPCC/ConfiguringHPCC.xml +++ b/docs/EN_US/ConfiguringHPCC/ConfiguringHPCC.xml @@ -1155,8 +1155,10 @@ sudo -u hpcc cp /etc/HPCCSystems/source/NewEnvironment.xml /etc/HPCCSystems/envi Directory Description - - + + + + log @@ -1449,6 +1451,30 @@ sudo -u hpcc cp /etc/HPCCSystems/source/NewEnvironment.xml /etc/HPCCSystems/envi + + Allowed Pipe Programs + + In version 9.2.0 and greater, commands used in a PIPE action + are restricted by default. However, for legacy reasons, the default + stock behavior is different in bare-metal and containerized + deployments. In both types of systems, if allowedPipePrograms is + unset, then all but "built-in" programs are restricted (The only + built-in program currently is 'roxiepipe'). + + In bare-metal, the default environment.xml includes a setting + value of "*" for allowedPipePrograms. This means (by default) + any PIPE program can be used. + + + In a secure system, this should be removed + or edited to avoid arbitrary programs, including system programs, + from being executed. + + + + + EclAgentProcessNotes @@ -1702,7 +1728,7 @@ sudo -u hpcc cp /etc/HPCCSystems/source/NewEnvironment.xml /etc/HPCCSystems/envi If you update the platform, but are using a preexisting configuration, you could encounter a situation where Feature level access flags are not automatically created. Missing flags can deny - access to users trying to access features in the system. + access to users trying to access features in the system. @@ -2653,6 +2679,30 @@ sudo -u hpcc cp /etc/HPCCSystems/source/NewEnvironment.xml /etc/HPCCSystems/envi + + Allowed Pipe Programs + + In version 9.2.0 and greater, commands used in a PIPE action + are restricted by default. However, for legacy reasons, the default + stock behavior is different in bare-metal and containerized + deployments. In both types of systems, if allowedPipePrograms is + unset, then all but "built-in" programs are restricted (The only + built-in program currently is 'roxiepipe'). + + In bare-metal, the default environment.xml includes a setting + value of "*" for allowedPipePrograms. This means (by default) + any PIPE program can be used. + + + In a secure system, this should be removed + or edited to avoid arbitrary programs, including system programs, + from being executed. + + + + + diff --git a/docs/EN_US/ContainerizedHPCC/ContainerizedMods/CustomConfig.xml b/docs/EN_US/ContainerizedHPCC/ContainerizedMods/CustomConfig.xml index e4ae7ac4ae6..c01e7bfb060 100644 --- a/docs/EN_US/ContainerizedHPCC/ContainerizedMods/CustomConfig.xml +++ b/docs/EN_US/ContainerizedHPCC/ContainerizedMods/CustomConfig.xml @@ -319,6 +319,125 @@ helm upgrade mycluster hpcc/hpcc -f tworoxies.yaml -f twothors.yaml + + Create a Custom Configuration Chart to AllowPipePrograms + + You can specify more than one custom configuration by repeating + the -f parameter. + + For example: + + helm install mycluster hpcc/hpcc -f tworoxies.yaml -f thorWithPipe.yaml + + In this section, we will modify the Thor to allow some Pipe + Programs. In version 9.2.0 and greater, commands used in PIPE are + restricted by default in containerized deployments unless explicitly + allowed in the Helm chart. + + + + If you have not already added the HPCC Systems repository to + the helm repository list, add it now. + + helm repo add hpcc https://hpcc-systems.github.io/helm-chart/ + + If you have added it, update to the latest charts: + + helm repo update + + + + Create a new text file and name it thorWithPipe.yaml and open it in a text + editor. + + You can use any text editor. + + + + Open the default values file that you saved earlier + (myvalues.yaml) in a text editor. + + + + Copy the entire thor: section + and paste it into the new thorWithPipe.yaml file. + + + + Add a block at the end: + + allowedPipePrograms: + - sort + - grep + - echoThis example enables three common programs. You can + use the ones you want instead. + + + + Save the file and close the text editor. + + The resulting thorWithPipe.yaml file should look like + this + + Note: The comments have been + removed to simplify the example: + + thor: +- name: thor + prefix: thor + numWorkers: 2 + maxJobs: 4 + maxGraphs: 2 + allowedPipePrograms: + - sort + - grep + - echo + + + + Deploy using the new custom configuration + chart. + + + + Open a terminal and navigate to the folder where you saved the + thorWithPipe.yaml file file. + + + + Deploy your HPCC Systems Platform, adding the new + configuration to your command: + + # If you have previously stopped your cluster + +helm install mycluster hpcc/hpcc -f thorWithPipe.yaml + +# To upgrade without stopping + +helm upgrade mycluster hpcc/hpcc -f thorWithPipe.yaml + + + + + After you confirm that your deployment is running, submit a + job that uses a PIPE action and specifies one of the programs you + specified. + + Note: If the job is too + simple, it will execute on hThor instead of Thor and this example + doesn't enable Pipe programs on hThor. + + You can create another yaml file to allow Pipe Programs on ECL + Agent or you can use: + + #OPTION('pickBestEngine',FALSE); + + to force the job to run on Thor. + + + + Create a Custom Configuration Chart for No Thor From 22e78da4bbe4736d273cb61cd4cedb1e449bc2f1 Mon Sep 17 00:00:00 2001 From: Jake Smith Date: Fri, 12 Jan 2024 16:14:00 +0000 Subject: [PATCH 18/32] HPCC-30333 Suppress unsupported checkpoint recovery Signed-off-by: Jake Smith --- thorlcr/graph/thgraph.cpp | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/thorlcr/graph/thgraph.cpp b/thorlcr/graph/thgraph.cpp index deb69708ab8..4994ba9607d 100644 --- a/thorlcr/graph/thgraph.cpp +++ b/thorlcr/graph/thgraph.cpp @@ -2907,7 +2907,13 @@ void CJobBase::addDependencies(IPropertyTree *xgmml, bool failIfMissing) bool CJobBase::queryUseCheckpoints() const { - return globals->getPropBool("@checkPointRecovery") || 0 != getWorkUnitValueInt("checkPointRecovery", 0); + bool configured = globals->getPropBool("@checkPointRecovery") || 0 != getWorkUnitValueInt("checkPointRecovery", 0); + if (configured && isContainerized()) + { + WARNLOG("Containerized Thor does not support checkpoint recovery"); + return false; + } + return configured; } void CJobBase::abort(IException *e) From c2444cebbbaa63b0d15cb47caccccf78208ec44d Mon Sep 17 00:00:00 2001 From: Gavin Halliday Date: Sat, 13 Jan 2024 13:18:07 +0000 Subject: [PATCH 19/32] HPCC-31117 Fix roxie crash listening to a secure socket Signed-off-by: Gavin Halliday --- system/security/securesocket/securesocket.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/system/security/securesocket/securesocket.cpp b/system/security/securesocket/securesocket.cpp index 0b2452cf564..b5896a88c3f 100644 --- a/system/security/securesocket/securesocket.cpp +++ b/system/security/securesocket/securesocket.cpp @@ -1285,7 +1285,7 @@ class CSecureSocketContext : implements ISecureSocketContext, implements ISecure Owned m_peers; StringAttr password; CriticalSection cs; - Owned syncedConfig; + Linked syncedConfig; unsigned configVersion = 0; void setSessionIdContext() From 0926a6475f580498deb168c3886d163b41d30fb1 Mon Sep 17 00:00:00 2001 From: Gavin Halliday Date: Mon, 15 Jan 2024 09:24:35 +0000 Subject: [PATCH 20/32] HPCC-31123 Fix regression in roxie AES encryption using openSSL Signed-off-by: Gavin Halliday --- system/jlib/jencrypt.cpp | 8 +++-- testing/unittests/jlibtests.cpp | 60 ++++++++++++++++++++++----------- 2 files changed, 46 insertions(+), 22 deletions(-) diff --git a/system/jlib/jencrypt.cpp b/system/jlib/jencrypt.cpp index c79d8fd7343..1ff66bb571d 100644 --- a/system/jlib/jencrypt.cpp +++ b/system/jlib/jencrypt.cpp @@ -1870,7 +1870,8 @@ MemoryBuffer &aesEncrypt(const void *key, size_t keylen, const void *plaintext, encryptError("Unsupported key length"); break; } - byte *ciphertext = (byte *) output.reserve(plaintext_len + 100); + unsigned originalLen = output.length(); + byte *ciphertext = (byte *) output.reserve(plaintext_len + 16); int ciphertext_len = 0; int thislen = 0; if(1 != EVP_EncryptUpdate(ctx, ciphertext, &thislen, (const unsigned char *) plaintext, plaintext_len)) @@ -1880,7 +1881,7 @@ MemoryBuffer &aesEncrypt(const void *key, size_t keylen, const void *plaintext, encryptError("Error in EVP_EncryptFinal_ex"); ciphertext_len += thislen; EVP_CIPHER_CTX_free(ctx); - output.setLength(ciphertext_len); + output.setLength(originalLen + ciphertext_len); return output; } catch (...) @@ -1928,6 +1929,7 @@ MemoryBuffer &aesDecrypt(const void *key, size_t keylen, const void *ciphertext, decryptError("Unsupported key length"); break; } + unsigned originalLen = output.length(); byte *plaintext = (byte *) output.reserve(ciphertext_len); if(1 != EVP_DecryptUpdate(ctx, plaintext, &thislen, (const unsigned char *) ciphertext, ciphertext_len)) decryptError("Error in EVP_DecryptUpdate"); @@ -1936,7 +1938,7 @@ MemoryBuffer &aesDecrypt(const void *key, size_t keylen, const void *ciphertext, if(1 != EVP_DecryptFinal_ex(ctx, plaintext + plaintext_len, &thislen)) decryptError("Error in EVP_DecryptFinal_ex"); plaintext_len += thislen; - output.setLength(plaintext_len); + output.setLength(originalLen + plaintext_len); EVP_CIPHER_CTX_free(ctx); return output; } diff --git a/testing/unittests/jlibtests.cpp b/testing/unittests/jlibtests.cpp index ca59f416148..e0ac1789d8d 100644 --- a/testing/unittests/jlibtests.cpp +++ b/testing/unittests/jlibtests.cpp @@ -3778,49 +3778,71 @@ class JLibOpensslAESTest : public CppUnit::TestFixture 0x36, 0x37, 0x38, 0x39, 0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, 0x38, 0x39, 0x30, 0x31 }; + constexpr const char * prefix = "ABCDEFGHIJKLMNOPQRSTUVWXYZ"; + constexpr unsigned lenPrefix = strlen(prefix); MemoryBuffer ciphertext1, ciphertext2, decrypted1, decrypted2; + ciphertext1.append(lenPrefix, prefix); + ciphertext2.append(lenPrefix, prefix); openssl::aesEncrypt(key, 32, intext, len, ciphertext1); jlib::aesEncrypt(key, 32, intext, len, ciphertext2); + CPPUNIT_ASSERT(memcmp(ciphertext1.bytes(), prefix, lenPrefix) == 0); + CPPUNIT_ASSERT(memcmp(ciphertext2.bytes(), prefix, lenPrefix) == 0); + if (len) + CPPUNIT_ASSERT(ciphertext1.length() > len + lenPrefix); + else + CPPUNIT_ASSERT(ciphertext1.length() == len + lenPrefix); + CPPUNIT_ASSERT(ciphertext1.length() <= len + lenPrefix + 16); CPPUNIT_ASSERT(ciphertext1.length()==ciphertext2.length()); CPPUNIT_ASSERT(memcmp(ciphertext1.bytes(), ciphertext2.bytes(), ciphertext1.length()) == 0); + unsigned cipherlen = ciphertext1.length() - lenPrefix; + /* Decrypt the ciphertext */ - openssl::aesDecrypt(key, 32, ciphertext1.bytes(), ciphertext1.length(), decrypted1); - assert(decrypted1.length() == len); - CPPUNIT_ASSERT(decrypted1.length() == len); - CPPUNIT_ASSERT(memcmp(decrypted1.bytes(), intext, len) == 0); + decrypted1.append(lenPrefix, prefix); + openssl::aesDecrypt(key, 32, ciphertext1.bytes()+lenPrefix, cipherlen, decrypted1); + CPPUNIT_ASSERT(decrypted1.length() == len+lenPrefix); + CPPUNIT_ASSERT(memcmp(decrypted1.bytes(), prefix, lenPrefix) == 0); + CPPUNIT_ASSERT(memcmp(decrypted1.bytes()+lenPrefix, intext, len) == 0); CPPUNIT_ASSERT(memcmp(ciphertext1.bytes(), ciphertext2.bytes(), ciphertext1.length()) == 0); // check input unchanged - jlib::aesDecrypt(key, 32, ciphertext2.bytes(), ciphertext2.length(), decrypted2); - CPPUNIT_ASSERT(decrypted2.length() == len); - CPPUNIT_ASSERT(memcmp(decrypted2.bytes(), intext, len) == 0); + decrypted2.append(lenPrefix, prefix); + jlib::aesDecrypt(key, 32, ciphertext2.bytes()+lenPrefix, cipherlen, decrypted2); + CPPUNIT_ASSERT(memcmp(decrypted2.bytes(), prefix, lenPrefix) == 0); + CPPUNIT_ASSERT(decrypted2.length() == len + lenPrefix); + CPPUNIT_ASSERT(memcmp(decrypted2.bytes() + lenPrefix, intext, len) == 0); CPPUNIT_ASSERT(memcmp(ciphertext1.bytes(), ciphertext2.bytes(), ciphertext1.length()) == 0); // check input unchanged // Now test in-place decrypt - unsigned cipherlen = ciphertext1.length(); ciphertext1.append(4, "XXXX"); // Marker - unsigned decryptedlen = openssl::aesDecryptInPlace(key, 32, (void *) ciphertext1.bytes(), cipherlen); + unsigned decryptedlen = openssl::aesDecryptInPlace(key, 32, (void *)(ciphertext1.bytes() + lenPrefix), cipherlen); CPPUNIT_ASSERT(decryptedlen == len); - CPPUNIT_ASSERT(memcmp(ciphertext1.bytes(), intext, len) == 0); - CPPUNIT_ASSERT(memcmp(ciphertext1.bytes()+cipherlen, "XXXX", 4) == 0); + CPPUNIT_ASSERT(memcmp(ciphertext1.bytes()+lenPrefix, intext, len) == 0); + CPPUNIT_ASSERT(memcmp(ciphertext1.bytes()+lenPrefix+cipherlen, "XXXX", 4) == 0); - cipherlen = ciphertext2.length(); ciphertext2.append(4, "XXXX"); // Marker - decryptedlen = jlib::aesDecryptInPlace(key, 32, (void *) ciphertext2.bytes(), cipherlen); + decryptedlen = jlib::aesDecryptInPlace(key, 32, (void *)(ciphertext2.bytes() + lenPrefix), cipherlen); CPPUNIT_ASSERT(decryptedlen == len); - CPPUNIT_ASSERT(memcmp(ciphertext2.bytes(), intext, len) == 0); - CPPUNIT_ASSERT(memcmp(ciphertext2.bytes()+cipherlen, "XXXX", 4) == 0); + CPPUNIT_ASSERT(memcmp(ciphertext2.bytes()+lenPrefix, intext, len) == 0); + CPPUNIT_ASSERT(memcmp(ciphertext2.bytes()+lenPrefix+cipherlen, "XXXX", 4) == 0); } void test() { - /* Message to be encrypted */ - const char *plaintext = "The quick brown fox jumps over the lazy dog"; - for (unsigned l = 0; l < strlen(plaintext); l++) - testOne(l, plaintext); + try + { + /* Message to be encrypted */ + const char *plaintext = "The quick brown fox jumps over the lazy dog"; + for (unsigned l = 0; l < strlen(plaintext); l++) + testOne(l, plaintext); + } + catch (IException * e) + { + EXCLOG(e, "Exception in AES unit test"); + throw; + } } }; From 25aa10ed9f6485ad6b8bb821b8a017cd5134f672 Mon Sep 17 00:00:00 2001 From: Gavin Halliday Date: Mon, 15 Jan 2024 10:06:18 +0000 Subject: [PATCH 21/32] HPCC-31118 Fix roxie crash restarting while cluster is processing queries Revert HPCC-30151 ONCE reading remote data cores roxie at startup This reverts commit 5db2478c338a164d232563dc252622b341d86ee7. --- roxie/ccd/ccdmain.cpp | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/roxie/ccd/ccdmain.cpp b/roxie/ccd/ccdmain.cpp index 0b0358b6c90..1f69ae3e696 100644 --- a/roxie/ccd/ccdmain.cpp +++ b/roxie/ccd/ccdmain.cpp @@ -1413,6 +1413,12 @@ int CCD_API roxie_main(int argc, const char *argv[], const char * defaultYaml) #endif configurePreferredPlanes(); createDelayedReleaser(); + CCycleTimer loadPackageTimer; + globalPackageSetManager = createRoxiePackageSetManager(standAloneDll.getClear()); + globalPackageSetManager->load(); + if (traceLevel) + DBGLOG("Loading all packages took %ums", loadPackageTimer.elapsedMs()); + ROQ = createOutputQueueManager(numAgentThreads, encryptInTransit); ROQ->setHeadRegionSize(headRegionSize); ROQ->start(); @@ -1428,13 +1434,6 @@ int CCD_API roxie_main(int argc, const char *argv[], const char * defaultYaml) EnableSEHtoExceptionMapping(); setSEHtoExceptionHandler(&abortHandler); - - CCycleTimer loadPackageTimer; - globalPackageSetManager = createRoxiePackageSetManager(standAloneDll.getClear()); - globalPackageSetManager->load(); - if (traceLevel) - DBGLOG("Loading all packages took %ums", loadPackageTimer.elapsedMs()); - Owned protocolCtx = new CHpccProtocolPluginCtx(); if (runOnce) { From f2fce5b503f13378b7403f974a690f201dd9fc79 Mon Sep 17 00:00:00 2001 From: Gavin Halliday Date: Mon, 15 Jan 2024 10:18:58 +0000 Subject: [PATCH 22/32] HPCC-31112 Update python test case to remove new stack trace Signed-off-by: Gavin Halliday --- testing/regress/ecl/embedpy3-catch.ecl | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/testing/regress/ecl/embedpy3-catch.ecl b/testing/regress/ecl/embedpy3-catch.ecl index 9074de26ad5..eb8847554bd 100644 --- a/testing/regress/ecl/embedpy3-catch.ecl +++ b/testing/regress/ecl/embedpy3-catch.ecl @@ -23,6 +23,7 @@ //skip type==thorlcr TBD IMPORT Python3; +IMPORT Std.Str; integer testThrow(integer val) := EMBED(Python3) raise Exception('Error from Python') @@ -32,8 +33,9 @@ ENDEMBED; d := dataset([{ 1, '' }], { integer a, string m} ) : stored('nofold'); d t := transform + eol := Str.Find(FAILMESSAGE, '\n'); self.a := FAILCODE; - self.m := FAILMESSAGE; + self.m := IF(eol != 0, FAILMESSAGE[1..eol-1], FAILMESSAGE); self := []; end; From acc5f542148214753426c85a9d2e148882c96f95 Mon Sep 17 00:00:00 2001 From: Gavin Halliday Date: Tue, 16 Jan 2024 17:16:58 +0000 Subject: [PATCH 23/32] HPCC-31131 Allow roxie to recover from partially read datagrams Signed-off-by: Gavin Halliday --- roxie/udplib/udptrr.cpp | 36 ++++++++++++++++++++++++++++++------ roxie/udplib/udptrs.cpp | 2 +- 2 files changed, 31 insertions(+), 7 deletions(-) diff --git a/roxie/udplib/udptrr.cpp b/roxie/udplib/udptrr.cpp index e4598a84974..8dec8182b4f 100644 --- a/roxie/udplib/udptrr.cpp +++ b/roxie/udplib/udptrr.cpp @@ -1303,7 +1303,7 @@ class CReceiveManager : implements IReceiveManager, public CInterface Thread::start(); started.wait(); } - + ~receive_data() { DBGLOG("Total data packets seen = %u OOO(%u) Requests(%u) Permits(%u)", dataPacketsReceived.load(), packetsOOO.load(), flowRequestsReceived.load(), flowRequestsSent.load()); @@ -1336,11 +1336,37 @@ class CReceiveManager : implements IReceiveManager, public CInterface try { unsigned int res; + UdpPacketHeader &hdr = *(UdpPacketHeader *) b->data; while (true) { - receive_socket->readtms(b->data, 1, DATA_PAYLOAD, res, timeout); - if (res!=sizeof(UdpRequestToSendMsg)) - break; + //Read at least the size of the smallest packet we can receive + //static assert to check we are reading the smaller of the two possible packet types + static_assert(sizeof(UdpRequestToSendMsg) <= sizeof(UdpPacketHeader)); + receive_socket->readtms(b->data, sizeof(UdpRequestToSendMsg), DATA_PAYLOAD, res, timeout); + + //Even if a UDP packet is not split, very occasionally only some of the data may be present for the read. + //Slightly horribly this packet could be one of two different formats(!) + // a UdpRequestToSendMsg, which has a 2 byte command at the start of the header, with a maximum value of max_flow_cmd + // a UdpPacketHeader which has a 2 byte length. This length must be > sizeof(UdpPacketHeader). + //Since max_flow_cmd < sizeof(UdpPacketHeader) this can be used to distinguish a true data packet(!) + static_assert(flowType::max_flow_cmd < sizeof(UdpPacketHeader)); // assert to check the above comment is correct + + if (hdr.length >= sizeof(UdpPacketHeader)) + { + if (res == hdr.length) + break; + + //Very rare situation - log it so that there is some evidence that it is occurring + OWARNLOG("Received partial network packet - %u bytes out of %u received", res, hdr.length); + + //Because we are reading UDP datgrams rather than tcp packets, if we failed to read the whole datagram + //the rest of the datgram is lost - you cannot call readtms to read the rest of the datagram. + //Therefore throw this incomplete datagram away and allow the resend mechanism to retransmit it. + continue; + } + + //Sanity check + assertex(res == sizeof(UdpRequestToSendMsg)); //Sending flow packets (eg send_completed) to the data thread ensures they do not overtake the data //Redirect them to the flow thread to process them. @@ -1348,8 +1374,6 @@ class CReceiveManager : implements IReceiveManager, public CInterface } dataPacketsReceived++; - UdpPacketHeader &hdr = *(UdpPacketHeader *) b->data; - assert(hdr.length == res && hdr.length > sizeof(hdr)); UdpSenderEntry *sender = &parent.sendersTable[hdr.node]; if (sender->noteSeen(hdr)) { diff --git a/roxie/udplib/udptrs.cpp b/roxie/udplib/udptrs.cpp index 5ed9996118f..96912bc604f 100644 --- a/roxie/udplib/udptrs.cpp +++ b/roxie/udplib/udptrs.cpp @@ -573,7 +573,7 @@ class UdpReceiverEntry : public IUdpReceiverEntry aesEncrypt(udpkey.get(), udpkey.length(), data, length, encryptBuffer); header->length = encryptBuffer.length(); encryptBuffer.writeDirect(0, sizeof(UdpPacketHeader), header); // Only really need length updating - assert(length <= DATA_PAYLOAD); + assertex(encryptBuffer.length() <= DATA_PAYLOAD); data_socket->write(encryptBuffer.toByteArray(), encryptBuffer.length()); } else From 7936c3661dbafa1cb0ce5a72b50062f069a514c9 Mon Sep 17 00:00:00 2001 From: Gavin Halliday Date: Mon, 15 Jan 2024 17:50:03 +0000 Subject: [PATCH 24/32] HPCC-31111 Fix unusual roxie crash on keyed join within library Only occurs if the library is used more than once within the same query and the keyed joins execute at the same time. Signed-off-by: Gavin Halliday --- common/thorhelper/roxierow.cpp | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/common/thorhelper/roxierow.cpp b/common/thorhelper/roxierow.cpp index 48d297361be..ea4d7706e42 100644 --- a/common/thorhelper/roxierow.cpp +++ b/common/thorhelper/roxierow.cpp @@ -488,9 +488,12 @@ class CAllocatorCache : public CSimpleInterfaceOf CAllocatorCacheItem *container = _lookup(meta, activityId, flags); if (container) { - if (0 == (roxiemem::RHFunique & flags)) + if (0 == ((roxiemem::RHFunique|roxiemem::RHFblocked) & flags)) return LINK(&container->queryElement()); - // if in cache but unique, reuse allocatorId + // If in cache but unique, reuse allocatorId, but create a unique allocator (and heap) + // If blocked the allocator must not be commoned up! (The underlying heap will be within roxiemem.) + // This is very unusual, but can happen if a library is used more than once within the same query + // since you will have multiple activity instances with the same activityId. SpinUnblock b(allAllocatorsLock); return callback->createAllocator(this, meta, activityId, container->queryAllocatorId(), flags); } From 1f2578382aa4d3d4e210cd3560636790234d1079 Mon Sep 17 00:00:00 2001 From: Richard Chapman Date: Wed, 10 Jan 2024 15:26:13 +0000 Subject: [PATCH 25/32] HPCC-31099 Incorrect use of msTick in hrpc.cpp Signed-off-by: Richard Chapman --- system/hrpc/hrpc.cpp | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/system/hrpc/hrpc.cpp b/system/hrpc/hrpc.cpp index f94cce181b7..40de0767580 100644 --- a/system/hrpc/hrpc.cpp +++ b/system/hrpc/hrpc.cpp @@ -305,9 +305,7 @@ struct SafeInc }; void HRPCcommon::doproxy(HRPCcallframe &frame,int fn,IHRPCtransport *tr,HRPCbuffer &buff,HRPCbuffer &rbuff,int cb,int locked) { - unsigned duetime = 0; - if (timelimit) - duetime = msTick()+timelimit; + unsigned start = msTick(); size32_t base=buff.markwrite(); HRPCtrace(">doproxy in fn=%d base=%d cb=%d locked=%d\n",fn,base,cb,locked); HRPCpacketheader *h=(HRPCpacketheader *)buff.buffptr(frame.mark); @@ -328,11 +326,14 @@ void HRPCcommon::doproxy(HRPCcallframe &frame,int fn,IHRPCtransport *tr,HRPCbuff buff.releasewrite(base); int timeleft=-1; HRPCtrace(" doproxy pre-receive\n"); - if (timelimit) { - timeleft = (duetime-msTick()); - if (timeleft<=0) { + if (timelimit) + { + unsigned now = msTick(); + if (now - start >= timelimit) + { THROWHRPCEXCEPTION(HRPCERR_call_timeout); } + timeleft = timelimit - (now - start); } tr->Receive(buff,timeleft); HRPCtrace(" doproxy post-receive\n"); From 267452f35b23918f73b9f91d745226f2e92150f8 Mon Sep 17 00:00:00 2001 From: Richard Chapman Date: Wed, 10 Jan 2024 14:31:16 +0000 Subject: [PATCH 26/32] HPCC-31097 Suspicious use of msTick() in CTimedCacheItem Was not handling msTick() wrapping properly Signed-off-by: Richard Chapman --- dali/base/dautils.cpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/dali/base/dautils.cpp b/dali/base/dautils.cpp index fa4b545773c..98fa5c7d448 100644 --- a/dali/base/dautils.cpp +++ b/dali/base/dautils.cpp @@ -2299,7 +2299,7 @@ void setMaxPageCacheItems(unsigned _maxPageCacheItems) class CTimedCacheItem: public CInterface { protected: friend class CTimedCache; - unsigned due = 0; + unsigned timestamp = 0; StringAttr owner; public: DALI_UID hint; @@ -2342,9 +2342,9 @@ class CTimedCache ForEachItemIn(i, items) { CTimedCacheItem &item = items.item(i); - if (item.due > now) + if (now - item.timestamp < pageCacheTimeoutMilliSeconds) { - res = item.due - now; + res = pageCacheTimeoutMilliSeconds - (now - item.timestamp); break; } expired++; @@ -2392,7 +2392,7 @@ class CTimedCache CriticalBlock block(sect); if ((maxPageCacheItems > 0) && (maxPageCacheItems == items.length())) items.remove(0); - item->due = msTick() + pageCacheTimeoutMilliSeconds; + item->timestamp = msTick(); items.append(*item); DALI_UID ret = item->hint; sem.signal(); From 1cecf573572d7d3fa77024c44bd3f15aa985f430 Mon Sep 17 00:00:00 2001 From: Richard Chapman Date: Wed, 17 Jan 2024 12:30:44 +0000 Subject: [PATCH 27/32] HPCC-31137 Add aesEncryptInPlace function Signed-off-by: Richard Chapman --- system/jlib/jencrypt.cpp | 79 +++++++++++++++++++++++++++++++++ system/jlib/jencrypt.hpp | 3 ++ testing/unittests/jlibtests.cpp | 58 +++++++++++++++++++++++- 3 files changed, 139 insertions(+), 1 deletion(-) diff --git a/system/jlib/jencrypt.cpp b/system/jlib/jencrypt.cpp index 1ff66bb571d..da7208c27aa 100644 --- a/system/jlib/jencrypt.cpp +++ b/system/jlib/jencrypt.cpp @@ -1818,6 +1818,25 @@ MemoryBuffer &aesDecrypt(const void *key, size_t keylen, const void *input, size return output; } +size_t aesEncryptInPlace(const void *key, size_t keylen, void *buffer, size_t inlen, size_t maxlen) +{ + if (!buffer || !inlen) + return 0; + // Make sure there is space for the padding. This is up to 16 bytes + if (maxlen - inlen < 16) + throw MakeStringException(-1,"AES Encryption error: Insufficient space in input buffer"); + Rijndael rin; + Rijndael::KeyLength keyType = getAesKeyType(keylen); + + rin.init(Rijndael::CBC, Rijndael::Encrypt, (const UINT8 *)key, keyType); + size32_t truncInLen = (size32_t)inlen; //MORE: Modify the padEncrypt function + int len = rin.padEncrypt((const UINT8 *)buffer, truncInLen, (UINT8 *) buffer); + if(len >= 0) + return len; + else + throw MakeStringException(-1,"AES Encryption error: %d, %s", len, getAesErrorText(len)); +} + size_t aesDecryptInPlace(const void *key, size_t keylen, void *data, size_t inlen) { Rijndael rin; @@ -1891,6 +1910,56 @@ MemoryBuffer &aesEncrypt(const void *key, size_t keylen, const void *plaintext, } } +extern jlib_decl size_t aesEncryptInPlace(const void *key, size_t keylen, void *buffer, size_t inlen, size_t buflen) +{ + if (!buffer || !inlen) + return 0; + byte *data = (byte *) buffer; + EVP_CIPHER_CTX *ctx = EVP_CIPHER_CTX_new(); + if (!ctx) + encryptError("Failed to create context"); + try + { + unsigned char iv[16] = { 0 }; + switch (keylen) + { + case 32: + if(1 != EVP_EncryptInit_ex(ctx, EVP_aes_256_cbc(), NULL, (const unsigned char *) key, iv)) + encryptError("Failed to initialize context"); + break; + case 24: + if(1 != EVP_EncryptInit_ex(ctx, EVP_aes_192_cbc(), NULL, (const unsigned char *) key, iv)) + encryptError("Failed to initialize context"); + break; + case 16: + if(1 != EVP_EncryptInit_ex(ctx, EVP_aes_128_cbc(), NULL, (const unsigned char *) key, iv)) + encryptError("Failed to initialize context"); + break; + default: + encryptError("Unsupported key length"); + break; + } + // Make sure there is space for the padding. This is up to 16 bytes + if (buflen - inlen < 16) + encryptError("Insufficient space in input buffer"); + int ciphertext_len = 0; + int thislen = 0; + if(1 != EVP_EncryptUpdate(ctx, data, &thislen, (const unsigned char *) data, inlen)) + encryptError("Error in EVP_EncryptUpdate"); + ciphertext_len += thislen; + if(1 != EVP_EncryptFinal_ex(ctx, data + ciphertext_len, &thislen)) + encryptError("Error in EVP_EncryptFinal_ex"); + ciphertext_len += thislen; + EVP_CIPHER_CTX_free(ctx); + return ciphertext_len; + } + catch (...) + { + EVP_CIPHER_CTX_free(ctx); + throw; + } +} + static void decryptError(const char *what) { VStringBuffer s("openssl::aesDecrypt: unexpected failure in %s", what); @@ -2022,6 +2091,16 @@ MemoryBuffer &aesDecrypt(const void *key, size_t keylen, const void *input, size #endif } +size_t aesEncryptInPlace(const void *key, size_t keylen, void *buffer, size_t inlen, size_t buflen) +{ +#ifdef _USE_OPENSSL + return openssl::aesEncryptInPlace(key, keylen, buffer, inlen, buflen); +#else + return jlib::aesEncryptInPlace(key, keylen, buffer, inlen, buflen); +#endif +} + + size_t aesDecryptInPlace(const void *key, size_t keylen, void *data, size_t inlen) { #ifdef _USE_OPENSSL diff --git a/system/jlib/jencrypt.hpp b/system/jlib/jencrypt.hpp index b9711ca394a..767688c9b15 100644 --- a/system/jlib/jencrypt.hpp +++ b/system/jlib/jencrypt.hpp @@ -30,6 +30,7 @@ namespace jlib { extern jlib_decl MemoryBuffer &aesEncrypt(const void *key, size_t keylen, const void *input, size_t inlen, MemoryBuffer &output); extern jlib_decl MemoryBuffer &aesDecrypt(const void *key, size_t keylen, const void *input, size_t inlen, MemoryBuffer &output); + extern jlib_decl size_t aesEncryptInPlace(const void *key, size_t keylen, void *data, size_t inlen, size_t buflen); extern jlib_decl size_t aesDecryptInPlace(const void *key, size_t keylen, void *data, size_t inlen); } // end of namespace jlib; @@ -38,6 +39,7 @@ namespace openssl { extern jlib_decl MemoryBuffer &aesEncrypt(const void *key, size_t keylen, const void *input, size_t inlen, MemoryBuffer &output); extern jlib_decl MemoryBuffer &aesDecrypt(const void *key, size_t keylen, const void *input, size_t inlen, MemoryBuffer &output); + extern jlib_decl size_t aesEncryptInPlace(const void *key, size_t keylen, void *data, size_t inlen, size_t buflen); extern jlib_decl size_t aesDecryptInPlace(const void *key, size_t keylen, void *data, size_t inlen); } // end of namespace openssl; #endif @@ -46,6 +48,7 @@ namespace openssl extern jlib_decl MemoryBuffer &aesEncrypt(const void *key, size_t keylen, const void *input, size_t inlen, MemoryBuffer &output); extern jlib_decl MemoryBuffer &aesDecrypt(const void *key, size_t keylen, const void *input, size_t inlen, MemoryBuffer &output); +extern jlib_decl size_t aesEncryptInPlace(const void *key, size_t keylen, void *data, size_t inlen, size_t buflen); extern jlib_decl size_t aesDecryptInPlace(const void *key, size_t keylen, void *data, size_t inlen); diff --git a/testing/unittests/jlibtests.cpp b/testing/unittests/jlibtests.cpp index e0ac1789d8d..08f20787e86 100644 --- a/testing/unittests/jlibtests.cpp +++ b/testing/unittests/jlibtests.cpp @@ -3779,7 +3779,7 @@ class JLibOpensslAESTest : public CppUnit::TestFixture 0x34, 0x35, 0x36, 0x37, 0x38, 0x39, 0x30, 0x31 }; constexpr const char * prefix = "ABCDEFGHIJKLMNOPQRSTUVWXYZ"; - constexpr unsigned lenPrefix = strlen(prefix); + unsigned lenPrefix = strlen(prefix); MemoryBuffer ciphertext1, ciphertext2, decrypted1, decrypted2; @@ -3827,6 +3827,62 @@ class JLibOpensslAESTest : public CppUnit::TestFixture CPPUNIT_ASSERT(decryptedlen == len); CPPUNIT_ASSERT(memcmp(ciphertext2.bytes()+lenPrefix, intext, len) == 0); CPPUNIT_ASSERT(memcmp(ciphertext2.bytes()+lenPrefix+cipherlen, "XXXX", 4) == 0); + + // Now in-place encrypt + ciphertext1.clear().append(lenPrefix, prefix).append(len, intext); + ciphertext1.append(16, "1234123412341234"); // Filler to be used by AES padding + ciphertext1.append(4, "WXYZ"); // Marker - check this is untouched + unsigned encryptedlen = openssl::aesEncryptInPlace(key, 32, (void *)(ciphertext1.bytes() + lenPrefix), len, len+16); + CPPUNIT_ASSERT(encryptedlen >= len); + CPPUNIT_ASSERT(encryptedlen <= len+16); + CPPUNIT_ASSERT(memcmp(ciphertext1.bytes()+lenPrefix+len+16, "WXYZ", 4) == 0); + CPPUNIT_ASSERT(len == 0 || memcmp(ciphertext1.bytes()+lenPrefix, intext, len) != 0); // Check it actually did encrypt! + decryptedlen = openssl::aesDecryptInPlace(key, 32, (void *)(ciphertext1.bytes() + lenPrefix), encryptedlen); + CPPUNIT_ASSERT(decryptedlen == len); + CPPUNIT_ASSERT(memcmp(ciphertext1.bytes()+lenPrefix, intext, len) == 0); + CPPUNIT_ASSERT(memcmp(ciphertext1.bytes()+lenPrefix+len+16, "WXYZ", 4) == 0); + + ciphertext2.clear().append(lenPrefix, prefix).append(len, intext); + ciphertext2.append(16, "1234123412341234"); // Filler to be used by AES padding + ciphertext2.append(4, "ABCD"); // Marker - check this is untouched + encryptedlen = jlib::aesEncryptInPlace(key, 32, (void *)(ciphertext2.bytes() + lenPrefix), len, len+16); + CPPUNIT_ASSERT(encryptedlen >= len); + CPPUNIT_ASSERT(encryptedlen <= len+16); + CPPUNIT_ASSERT(memcmp(ciphertext2.bytes()+lenPrefix+len+16, "ABCD", 4) == 0); + CPPUNIT_ASSERT(len == 0 || memcmp(ciphertext2.bytes()+lenPrefix, intext, len) != 0); // Check it actually did encrypt! + decryptedlen = jlib::aesDecryptInPlace(key, 32, (void *)(ciphertext2.bytes() + lenPrefix), encryptedlen); + CPPUNIT_ASSERT(decryptedlen == len); + CPPUNIT_ASSERT(memcmp(ciphertext2.bytes()+lenPrefix, intext, len) == 0); + CPPUNIT_ASSERT(memcmp(ciphertext2.bytes()+lenPrefix+len+16, "ABCD", 4) == 0); + + // Test some error cases + if (len) + { + ciphertext1.clear().append(lenPrefix, prefix).append(len, intext); + ciphertext1.append(4, "WXYZ"); // Marker - check this is untouched + try + { + encryptedlen = openssl::aesEncryptInPlace(key, 32, (void *)(ciphertext1.bytes() + lenPrefix), len, len); + CPPUNIT_ASSERT(!"Should have reported insufficient length"); + } + catch (IException *E) + { + CPPUNIT_ASSERT(memcmp(ciphertext1.bytes()+lenPrefix+len, "WXYZ", 4) == 0); + E->Release(); + } + ciphertext2.clear().append(lenPrefix, prefix).append(len, intext); + ciphertext2.append(4, "ABCD"); // Marker - check this is untouched + try + { + encryptedlen = jlib::aesEncryptInPlace(key, 32, (void *)(ciphertext2.bytes() + lenPrefix), len, len); + CPPUNIT_ASSERT(!"Should have reported insufficient length"); + } + catch (IException *E) + { + CPPUNIT_ASSERT(memcmp(ciphertext2.bytes()+lenPrefix+len, "ABCD", 4) == 0); + E->Release(); + } + } } void test() From e4a70a4310ffb85b1d6f3bb9e4ee809dbe657b8c Mon Sep 17 00:00:00 2001 From: Gavin Halliday Date: Tue, 16 Jan 2024 17:16:58 +0000 Subject: [PATCH 28/32] HPCC-31131 Allow roxie to recover from partially read datagrams Signed-off-by: Gavin Halliday --- roxie/udplib/udptrr.cpp | 36 ++++++++++++++++++++++++++++++------ roxie/udplib/udptrs.cpp | 2 +- 2 files changed, 31 insertions(+), 7 deletions(-) diff --git a/roxie/udplib/udptrr.cpp b/roxie/udplib/udptrr.cpp index a729839ef09..b6a571fbf74 100644 --- a/roxie/udplib/udptrr.cpp +++ b/roxie/udplib/udptrr.cpp @@ -1303,7 +1303,7 @@ class CReceiveManager : implements IReceiveManager, public CInterface Thread::start(); started.wait(); } - + ~receive_data() { DBGLOG("Total data packets seen = %u OOO(%u) Requests(%u) Permits(%u)", dataPacketsReceived.load(), packetsOOO.load(), flowRequestsReceived.load(), flowRequestsSent.load()); @@ -1338,11 +1338,37 @@ class CReceiveManager : implements IReceiveManager, public CInterface b = bufferManager->allocate(); unsigned int res; + UdpPacketHeader &hdr = *(UdpPacketHeader *) b->data; while (true) { - receive_socket->readtms(b->data, 1, DATA_PAYLOAD, res, timeout); - if (res!=sizeof(UdpRequestToSendMsg)) - break; + //Read at least the size of the smallest packet we can receive + //static assert to check we are reading the smaller of the two possible packet types + static_assert(sizeof(UdpRequestToSendMsg) <= sizeof(UdpPacketHeader)); + receive_socket->readtms(b->data, sizeof(UdpRequestToSendMsg), DATA_PAYLOAD, res, timeout); + + //Even if a UDP packet is not split, very occasionally only some of the data may be present for the read. + //Slightly horribly this packet could be one of two different formats(!) + // a UdpRequestToSendMsg, which has a 2 byte command at the start of the header, with a maximum value of max_flow_cmd + // a UdpPacketHeader which has a 2 byte length. This length must be > sizeof(UdpPacketHeader). + //Since max_flow_cmd < sizeof(UdpPacketHeader) this can be used to distinguish a true data packet(!) + static_assert(flowType::max_flow_cmd < sizeof(UdpPacketHeader)); // assert to check the above comment is correct + + if (hdr.length >= sizeof(UdpPacketHeader)) + { + if (res == hdr.length) + break; + + //Very rare situation - log it so that there is some evidence that it is occurring + OWARNLOG("Received partial network packet - %u bytes out of %u received", res, hdr.length); + + //Because we are reading UDP datgrams rather than tcp packets, if we failed to read the whole datagram + //the rest of the datgram is lost - you cannot call readtms to read the rest of the datagram. + //Therefore throw this incomplete datagram away and allow the resend mechanism to retransmit it. + continue; + } + + //Sanity check + assertex(res == sizeof(UdpRequestToSendMsg)); //Sending flow packets (eg send_completed) to the data thread ensures they do not overtake the data //Redirect them to the flow thread to process them. @@ -1350,8 +1376,6 @@ class CReceiveManager : implements IReceiveManager, public CInterface } dataPacketsReceived++; - UdpPacketHeader &hdr = *(UdpPacketHeader *) b->data; - assert(hdr.length == res && hdr.length > sizeof(hdr)); UdpSenderEntry *sender = &parent.sendersTable[hdr.node]; if (sender->noteSeen(hdr)) { diff --git a/roxie/udplib/udptrs.cpp b/roxie/udplib/udptrs.cpp index e488c011324..72add1a5215 100644 --- a/roxie/udplib/udptrs.cpp +++ b/roxie/udplib/udptrs.cpp @@ -574,7 +574,7 @@ class UdpReceiverEntry : public IUdpReceiverEntry aesEncrypt(udpkey.get(), udpkey.length(), data, length, encryptBuffer); header->length = encryptBuffer.length(); encryptBuffer.writeDirect(0, sizeof(UdpPacketHeader), header); // Only really need length updating - assert(length <= DATA_PAYLOAD); + assertex(encryptBuffer.length() <= DATA_PAYLOAD); if (udpTraceLevel > 5) DBGLOG("ENCRYPT: Writing %u bytes to data socket", encryptBuffer.length()); data_socket->write(encryptBuffer.toByteArray(), encryptBuffer.length()); From 695be14ddb3e243b5d7f321dbfe9cb17d935d2db Mon Sep 17 00:00:00 2001 From: Richard Chapman Date: Wed, 17 Jan 2024 14:45:30 +0000 Subject: [PATCH 29/32] HPCC-31139 Length field gets corrupted in resent encrypted packets Signed-off-by: Richard Chapman --- roxie/udplib/udptrs.cpp | 5 +++-- testing/unittests/jlibtests.cpp | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/roxie/udplib/udptrs.cpp b/roxie/udplib/udptrs.cpp index 96912bc604f..93004768aae 100644 --- a/roxie/udplib/udptrs.cpp +++ b/roxie/udplib/udptrs.cpp @@ -571,8 +571,9 @@ class UdpReceiverEntry : public IUdpReceiverEntry const char *data = buffer->data + sizeof(UdpPacketHeader); const MemoryAttr &udpkey = getSecretUdpKey(true); aesEncrypt(udpkey.get(), udpkey.length(), data, length, encryptBuffer); - header->length = encryptBuffer.length(); - encryptBuffer.writeDirect(0, sizeof(UdpPacketHeader), header); // Only really need length updating + UdpPacketHeader newHeader; + newHeader.length = encryptBuffer.length(); + encryptBuffer.writeDirect(offsetof(UdpPacketHeader, length), sizeof(newHeader.length), &newHeader.length); // Only need to update the length - rest is the same assertex(encryptBuffer.length() <= DATA_PAYLOAD); data_socket->write(encryptBuffer.toByteArray(), encryptBuffer.length()); } diff --git a/testing/unittests/jlibtests.cpp b/testing/unittests/jlibtests.cpp index e0ac1789d8d..3b0e7695569 100644 --- a/testing/unittests/jlibtests.cpp +++ b/testing/unittests/jlibtests.cpp @@ -3779,7 +3779,7 @@ class JLibOpensslAESTest : public CppUnit::TestFixture 0x34, 0x35, 0x36, 0x37, 0x38, 0x39, 0x30, 0x31 }; constexpr const char * prefix = "ABCDEFGHIJKLMNOPQRSTUVWXYZ"; - constexpr unsigned lenPrefix = strlen(prefix); + unsigned lenPrefix = strlen(prefix); MemoryBuffer ciphertext1, ciphertext2, decrypted1, decrypted2; From 8b62a8cb8bbe9ea063b305b1ebb22ce2b9aba1c2 Mon Sep 17 00:00:00 2001 From: wangkx Date: Fri, 12 Jan 2024 10:06:48 -0500 Subject: [PATCH 30/32] HPCC-31122 Remove ZAP 'Log Access plug-in' warning for bare-metal When creating a ZAP report, if Remote Log Access plug-in is not available, ECLWatch shows a warning: "Warning: may not be able to include log file because Remote Log Access plug-in is not available!". Before this PR, the warning shows for both cloud and bare-metal. In this PR, the warning is removed for bare-metal since the Remote Log Access interface is not used in bare-metal at the moment. Revise based on review: 1. Revise the warning message. 2. If Containerized, skip the code to report thor processes. 3. Add comments about reporting thor processes for bare-metal. Signed-off-by: wangkx --- .../ws_workunits/ws_workunitsService.cpp | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/esp/services/ws_workunits/ws_workunitsService.cpp b/esp/services/ws_workunits/ws_workunitsService.cpp index ffbdf933181..7e3a0d49605 100644 --- a/esp/services/ws_workunits/ws_workunitsService.cpp +++ b/esp/services/ws_workunits/ws_workunitsService.cpp @@ -4979,8 +4979,8 @@ bool CWsWorkunitsEx::onWUGetZAPInfo(IEspContext &context, IEspWUGetZAPInfoReques ipaddr.getHostText(EspIP); resp.setESPIPAddress(EspIP.str()); } - if ((version >= 1.96) && !queryRemoteLogAccessor()) - resp.setMessage("Warning: may not be able to include log file because Remote Log Access plug-in is not available!"); + if ((version >= 1.96) && isContainerized() & !queryRemoteLogAccessor()) + resp.setMessage("Warning: the log file cannot be included because Remote Log Access plug-in is not available!"); //Get Archive Owned query = cw->getQuery(); @@ -4992,8 +4992,18 @@ bool CWsWorkunitsEx::onWUGetZAPInfo(IEspContext &context, IEspWUGetZAPInfoReques resp.setArchive(queryText.str()); } + if (isContainerized()) + { + resp.setEmailTo(zapEmailTo.get()); + resp.setEmailFrom(zapEmailFrom.get()); + return true; + } + + //The code below is for bare-metal only. if (version >= 1.95) { + //Using the getProcesses(), find out possible Thor processes used by the WU. + //The processes may be displayed inside the dialog box of creating ZAP report. StringBuffer thorProcesses; Owned thorInstances = cw->getProcesses("Thor"); ForEach (*thorInstances) @@ -5014,7 +5024,7 @@ bool CWsWorkunitsEx::onWUGetZAPInfo(IEspContext &context, IEspWUGetZAPInfoReques return true; } - //Get Thor IP + //Get Thor IP which may be displayed inside the dialog box of creating ZAP report. BoolHash uniqueProcesses; Owned thorInstances = cw->getProcesses("Thor"); ForEach (*thorInstances) From 063a9ec9fe1188dfffb6c1ae3e0a9ba6d796e7d9 Mon Sep 17 00:00:00 2001 From: Richard Chapman Date: Wed, 17 Jan 2024 14:45:30 +0000 Subject: [PATCH 31/32] HPCC-31139 Length field gets corrupted in resent encrypted packets Signed-off-by: Richard Chapman --- roxie/udplib/udptrs.cpp | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/roxie/udplib/udptrs.cpp b/roxie/udplib/udptrs.cpp index 72add1a5215..57dd631f005 100644 --- a/roxie/udplib/udptrs.cpp +++ b/roxie/udplib/udptrs.cpp @@ -572,8 +572,9 @@ class UdpReceiverEntry : public IUdpReceiverEntry const char *data = buffer->data + sizeof(UdpPacketHeader); const MemoryAttr &udpkey = getSecretUdpKey(true); aesEncrypt(udpkey.get(), udpkey.length(), data, length, encryptBuffer); - header->length = encryptBuffer.length(); - encryptBuffer.writeDirect(0, sizeof(UdpPacketHeader), header); // Only really need length updating + UdpPacketHeader newHeader; + newHeader.length = encryptBuffer.length(); + encryptBuffer.writeDirect(offsetof(UdpPacketHeader, length), sizeof(newHeader.length), &newHeader.length); // Only need to update the length - rest is the same assertex(encryptBuffer.length() <= DATA_PAYLOAD); if (udpTraceLevel > 5) DBGLOG("ENCRYPT: Writing %u bytes to data socket", encryptBuffer.length()); From 7b4366e4027febd8ca813ebc1d09eaaa2ee5f601 Mon Sep 17 00:00:00 2001 From: Gavin Halliday Date: Wed, 17 Jan 2024 13:52:15 +0000 Subject: [PATCH 32/32] HPCC-31138 Allow roxie to config whether it uses legacy or openssl aes encryption Signed-off-by: Gavin Halliday --- roxie/ccd/ccdmain.cpp | 1 + system/jlib/jencrypt.cpp | 31 +++++++++++++++++++------------ system/jlib/jencrypt.hpp | 2 ++ 3 files changed, 22 insertions(+), 12 deletions(-) diff --git a/roxie/ccd/ccdmain.cpp b/roxie/ccd/ccdmain.cpp index 1f69ae3e696..40f01b73e6b 100644 --- a/roxie/ccd/ccdmain.cpp +++ b/roxie/ccd/ccdmain.cpp @@ -1254,6 +1254,7 @@ int CCD_API roxie_main(int argc, const char *argv[], const char * defaultYaml) defaultSinkMode = getSinkMode(sinkModeText); cacheReportPeriodSeconds = topology->getPropInt("@cacheReportPeriodSeconds", 5*60); + setLegacyAES(topology->getPropBool("expert/@useLegacyAES", false)); // NB: these directories will have been setup by topology earlier const char *primaryDirectory = queryBaseDirectory(grp_unknown, 0); diff --git a/system/jlib/jencrypt.cpp b/system/jlib/jencrypt.cpp index da7208c27aa..6412a476ff1 100644 --- a/system/jlib/jencrypt.cpp +++ b/system/jlib/jencrypt.cpp @@ -2073,41 +2073,48 @@ size_t aesDecryptInPlace(const void *key, size_t keylen, void *ciphertext, size_ } // namespace #endif +static bool useLegacyAES = false; + MemoryBuffer &aesEncrypt(const void *key, size_t keylen, const void *input, size_t inlen, MemoryBuffer &output) { #ifdef _USE_OPENSSL - return openssl::aesEncrypt(key, keylen, input, inlen, output); -#else - return jlib::aesEncrypt(key, keylen, input, inlen, output); + if (!useLegacyAES) + return openssl::aesEncrypt(key, keylen, input, inlen, output); #endif + return jlib::aesEncrypt(key, keylen, input, inlen, output); } MemoryBuffer &aesDecrypt(const void *key, size_t keylen, const void *input, size_t inlen, MemoryBuffer &output) { #ifdef _USE_OPENSSL - return openssl::aesDecrypt(key, keylen, input, inlen, output); -#else - return jlib::aesDecrypt(key, keylen, input, inlen, output); + if (!useLegacyAES) + return openssl::aesDecrypt(key, keylen, input, inlen, output); #endif + return jlib::aesDecrypt(key, keylen, input, inlen, output); } size_t aesEncryptInPlace(const void *key, size_t keylen, void *buffer, size_t inlen, size_t buflen) { #ifdef _USE_OPENSSL - return openssl::aesEncryptInPlace(key, keylen, buffer, inlen, buflen); -#else - return jlib::aesEncryptInPlace(key, keylen, buffer, inlen, buflen); + if (!useLegacyAES) + return openssl::aesEncryptInPlace(key, keylen, buffer, inlen, buflen); #endif + return jlib::aesEncryptInPlace(key, keylen, buffer, inlen, buflen); } size_t aesDecryptInPlace(const void *key, size_t keylen, void *data, size_t inlen) { #ifdef _USE_OPENSSL - return openssl::aesDecryptInPlace(key, keylen, data, inlen); -#else - return jlib::aesDecryptInPlace(key, keylen, data, inlen); + if (!useLegacyAES) + return openssl::aesDecryptInPlace(key, keylen, data, inlen); #endif + return jlib::aesDecryptInPlace(key, keylen, data, inlen); +} + +void setLegacyAES(bool value) +{ + useLegacyAES = value; } #define CRYPTSIZE 32 diff --git a/system/jlib/jencrypt.hpp b/system/jlib/jencrypt.hpp index 767688c9b15..6a04949ab94 100644 --- a/system/jlib/jencrypt.hpp +++ b/system/jlib/jencrypt.hpp @@ -51,6 +51,8 @@ extern jlib_decl MemoryBuffer &aesDecrypt(const void *key, size_t keylen, const extern jlib_decl size_t aesEncryptInPlace(const void *key, size_t keylen, void *data, size_t inlen, size_t buflen); extern jlib_decl size_t aesDecryptInPlace(const void *key, size_t keylen, void *data, size_t inlen); +//Control whether to use the legacy jlib implementation of the AES functions, rather than openSSL when available. +extern jlib_decl void setLegacyAES(bool value); #define encrypt _LogProcessError12 #define decrypt _LogProcessError15