From 3ca761134e9354a897a27f9a35927c1edcb28e7c Mon Sep 17 00:00:00 2001 From: Mark Kelly Date: Fri, 23 Feb 2024 10:47:53 -0500 Subject: [PATCH 01/17] HPCC-31339 HPCC-Platform updates to go with HPCC-14361 update Signed-off-by: Mark Kelly --- plugins/eeproxies/lib_addrclean.ecllib | 1 + 1 file changed, 1 insertion(+) diff --git a/plugins/eeproxies/lib_addrclean.ecllib b/plugins/eeproxies/lib_addrclean.ecllib index 3c4fc3172ac..95ae781c443 100644 --- a/plugins/eeproxies/lib_addrclean.ecllib +++ b/plugins/eeproxies/lib_addrclean.ecllib @@ -30,4 +30,5 @@ export AddrCleanLib := SERVICE : plugin('addrcleanlib'),time string34 Zip9ToGeo34(const string zip9, const string server = '', unsigned2 port = 0) : c, pure, context, entrypoint='aclZip9ToGeo34Ctx'; boolean SetTracing(boolean trace=true) : c, action, context, entrypoint='aclSetTracingCtx'; boolean SetTiming(boolean timing=true) : c, action, context, entrypoint='aclSetTimingCtx'; + CleanUtil(unsigned4 flags = 0) : c, pure, context, entrypoint='aclCleanUtilCtx'; END; From d3a5014892a425c088ad5fcaa7a3c9105221947b Mon Sep 17 00:00:00 2001 From: Jake Smith Date: Thu, 29 Feb 2024 16:47:01 +0000 Subject: [PATCH 02/17] Split off 9.2.66 Signed-off-by: Jake Smith --- helm/hpcc/Chart.yaml | 4 ++-- helm/hpcc/templates/_helpers.tpl | 2 +- helm/hpcc/templates/dafilesrv.yaml | 2 +- helm/hpcc/templates/dali.yaml | 2 +- helm/hpcc/templates/dfuserver.yaml | 2 +- helm/hpcc/templates/eclagent.yaml | 4 ++-- helm/hpcc/templates/eclccserver.yaml | 4 ++-- helm/hpcc/templates/eclscheduler.yaml | 2 +- helm/hpcc/templates/esp.yaml | 2 +- helm/hpcc/templates/localroxie.yaml | 2 +- helm/hpcc/templates/roxie.yaml | 8 ++++---- helm/hpcc/templates/sasha.yaml | 2 +- helm/hpcc/templates/thor.yaml | 10 +++++----- version.cmake | 4 ++-- 14 files changed, 25 insertions(+), 25 deletions(-) diff --git a/helm/hpcc/Chart.yaml b/helm/hpcc/Chart.yaml index 49f31f92590..b36187391e9 100644 --- a/helm/hpcc/Chart.yaml +++ b/helm/hpcc/Chart.yaml @@ -6,9 +6,9 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. -version: 9.2.65-closedown0 +version: 9.2.67-closedown0 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. -appVersion: 9.2.65-closedown0 +appVersion: 9.2.67-closedown0 diff --git a/helm/hpcc/templates/_helpers.tpl b/helm/hpcc/templates/_helpers.tpl index 160701f2df3..6f1375ad63b 100644 --- a/helm/hpcc/templates/_helpers.tpl +++ b/helm/hpcc/templates/_helpers.tpl @@ -1405,7 +1405,7 @@ kind: Service metadata: name: {{ $lvars.serviceName | quote }} labels: - helmVersion: 9.2.65-closedown0 + helmVersion: 9.2.67-closedown0 {{- include "hpcc.addStandardLabels" (dict "root" $.root "instance" $lvars.serviceName ) | indent 4 }} {{- if $lvars.labels }} {{ toYaml $lvars.labels | indent 4 }} diff --git a/helm/hpcc/templates/dafilesrv.yaml b/helm/hpcc/templates/dafilesrv.yaml index f0850fdb836..257c73db7be 100644 --- a/helm/hpcc/templates/dafilesrv.yaml +++ b/helm/hpcc/templates/dafilesrv.yaml @@ -50,7 +50,7 @@ spec: labels: {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "dafilesrv" "name" "dafilesrv" "instance" .name) | indent 8 }} server: {{ .name | quote }} - helmVersion: 9.2.65-closedown0 + helmVersion: 9.2.67-closedown0 annotations: checksum/config: {{ $configSHA }} {{- include "hpcc.generateAnnotations" $commonCtx | indent 8 }} diff --git a/helm/hpcc/templates/dali.yaml b/helm/hpcc/templates/dali.yaml index dbea2014a08..3b4fc294026 100644 --- a/helm/hpcc/templates/dali.yaml +++ b/helm/hpcc/templates/dali.yaml @@ -82,7 +82,7 @@ spec: run: {{ $dali.name | quote }} server: {{ $dali.name | quote }} app: dali - helmVersion: 9.2.65-closedown0 + helmVersion: 9.2.67-closedown0 {{- if hasKey $.Values.global "metrics" }} {{- include "hpcc.generateMetricsReporterLabel" $.Values.global.metrics | nindent 8 }} {{- end }} diff --git a/helm/hpcc/templates/dfuserver.yaml b/helm/hpcc/templates/dfuserver.yaml index d61acb8b72e..878cc0b50f7 100644 --- a/helm/hpcc/templates/dfuserver.yaml +++ b/helm/hpcc/templates/dfuserver.yaml @@ -56,7 +56,7 @@ spec: {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "dfuserver" "name" "dfuserver" "instance" .name) | indent 8 }} run: {{ .name | quote }} accessDali: "yes" - helmVersion: 9.2.65-closedown0 + helmVersion: 9.2.67-closedown0 {{- if hasKey . "labels" }} {{ toYaml .labels | indent 8 }} {{- end }} diff --git a/helm/hpcc/templates/eclagent.yaml b/helm/hpcc/templates/eclagent.yaml index bccbfd74dd2..36a490e5c8a 100644 --- a/helm/hpcc/templates/eclagent.yaml +++ b/helm/hpcc/templates/eclagent.yaml @@ -58,7 +58,7 @@ data: {{- include "hpcc.addStandardLabels" (dict "root" $ "component" $apptype "name" "eclagent" "instance" $appJobName "instanceOf" (printf "%s-job" .me.name)) | indent 12 }} accessDali: "yes" accessEsp: "yes" - helmVersion: 9.2.65-closedown0 + helmVersion: 9.2.67-closedown0 {{- if hasKey .me "labels" }} {{ toYaml .me.labels | indent 12 }} {{- end }} @@ -133,7 +133,7 @@ spec: run: {{ .name | quote }} accessDali: "yes" accessEsp: {{ .useChildProcesses | default false | ternary "yes" "no" | quote }} - helmVersion: 9.2.65-closedown0 + helmVersion: 9.2.67-closedown0 {{- if hasKey . "labels" }} {{ toYaml .labels | indent 8 }} {{- end }} diff --git a/helm/hpcc/templates/eclccserver.yaml b/helm/hpcc/templates/eclccserver.yaml index 25a0c0893b9..c6715e0ebc8 100644 --- a/helm/hpcc/templates/eclccserver.yaml +++ b/helm/hpcc/templates/eclccserver.yaml @@ -57,7 +57,7 @@ data: {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "eclccserver" "name" "eclccserver" "instance" $compileJobName "instanceOf" (printf "%s-job" .me.name)) | indent 12 }} accessDali: "yes" accessEsp: "yes" - helmVersion: 9.2.65-closedown0 + helmVersion: 9.2.67-closedown0 {{- if hasKey .me "labels" }} {{ toYaml .me.labels | indent 12 }} {{- end }} @@ -140,7 +140,7 @@ spec: run: {{ .name | quote }} accessDali: "yes" accessEsp: {{ .useChildProcesses | default false | ternary "yes" "no" | quote }} - helmVersion: 9.2.65-closedown0 + helmVersion: 9.2.67-closedown0 {{- if hasKey . "labels" }} {{ toYaml .labels | indent 8 }} {{- end }} diff --git a/helm/hpcc/templates/eclscheduler.yaml b/helm/hpcc/templates/eclscheduler.yaml index 33cc9b9f38d..4602c3bfc1c 100644 --- a/helm/hpcc/templates/eclscheduler.yaml +++ b/helm/hpcc/templates/eclscheduler.yaml @@ -64,7 +64,7 @@ spec: run: {{ .name | quote }} accessDali: "yes" accessEsp: "no" - helmVersion: 9.2.65-closedown0 + helmVersion: 9.2.67-closedown0 {{- if hasKey . "labels" }} {{ toYaml .labels | indent 8 }} {{- end }} diff --git a/helm/hpcc/templates/esp.yaml b/helm/hpcc/templates/esp.yaml index c39803e432c..67af35b335c 100644 --- a/helm/hpcc/templates/esp.yaml +++ b/helm/hpcc/templates/esp.yaml @@ -120,7 +120,7 @@ spec: accessSasha: "yes" {{- end }} app: {{ $application }} - helmVersion: 9.2.65-closedown0 + helmVersion: 9.2.67-closedown0 {{- include "hpcc.addStandardLabels" (dict "root" $ "name" $application "component" "esp" "instance" .name) | indent 8 }} {{- if hasKey $.Values.global "metrics" }} {{- include "hpcc.generateMetricsReporterLabel" $.Values.global.metrics | nindent 8 }} diff --git a/helm/hpcc/templates/localroxie.yaml b/helm/hpcc/templates/localroxie.yaml index 56210d11d8d..07e0e22895b 100644 --- a/helm/hpcc/templates/localroxie.yaml +++ b/helm/hpcc/templates/localroxie.yaml @@ -70,7 +70,7 @@ spec: server: {{ $servername | quote }} accessDali: "yes" accessEsp: "yes" - helmVersion: 9.2.65-closedown0 + helmVersion: 9.2.67-closedown0 {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "roxie-server" "name" "roxie" "instance" $roxie.name) | indent 8 }} {{- if hasKey . "labels" }} {{ toYaml .labels | indent 8 }} diff --git a/helm/hpcc/templates/roxie.yaml b/helm/hpcc/templates/roxie.yaml index 8b06d3408bc..062930aa659 100644 --- a/helm/hpcc/templates/roxie.yaml +++ b/helm/hpcc/templates/roxie.yaml @@ -120,7 +120,7 @@ spec: {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "topology-server" "name" "roxie" "instance" $commonCtx.toponame) | indent 8 }} run: {{ $commonCtx.toponame | quote }} roxie-cluster: {{ $roxie.name | quote }} - helmVersion: 9.2.65-closedown0 + helmVersion: 9.2.67-closedown0 {{- if hasKey $.Values.global "metrics" }} {{- include "hpcc.generateMetricsReporterLabel" $.Values.global.metrics | nindent 8}} {{- end }} @@ -178,7 +178,7 @@ kind: Service metadata: name: {{ $commonCtx.toponame | quote }} labels: - helmVersion: 9.2.65-closedown0 + helmVersion: 9.2.67-closedown0 {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "topology-server" "name" "roxie" "instance" $commonCtx.toponame) | indent 4 }} spec: ports: @@ -240,7 +240,7 @@ spec: roxie-cluster: {{ $roxie.name | quote }} accessDali: "yes" accessEsp: "yes" - helmVersion: 9.2.65-closedown0 + helmVersion: 9.2.67-closedown0 {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "roxie-server" "name" "roxie" "instance" $servername) | indent 8 }} {{- if hasKey $.Values.global "metrics" }} {{- include "hpcc.generateMetricsReporterLabel" $.Values.global.metrics | nindent 8}} @@ -346,7 +346,7 @@ spec: roxie-cluster: {{ $roxie.name | quote }} accessDali: "yes" accessEsp: "yes" - helmVersion: 9.2.65-closedown0 + helmVersion: 9.2.67-closedown0 {{- if hasKey $.Values.global "metrics" }} {{- include "hpcc.generateMetricsReporterLabel" $.Values.global.metrics | nindent 8}} {{- end }} diff --git a/helm/hpcc/templates/sasha.yaml b/helm/hpcc/templates/sasha.yaml index ae7c47d2baa..4adf65c8685 100644 --- a/helm/hpcc/templates/sasha.yaml +++ b/helm/hpcc/templates/sasha.yaml @@ -53,7 +53,7 @@ spec: server: {{ $serviceName | quote }} app: sasha accessDali: {{ (has "dali" $sasha.access) | ternary "yes" "no" | quote }} - helmVersion: 9.2.65-closedown0 + helmVersion: 9.2.67-closedown0 {{- if hasKey $sasha "labels" }} {{ toYaml $sasha.labels | indent 8 }} {{- end }} diff --git a/helm/hpcc/templates/thor.yaml b/helm/hpcc/templates/thor.yaml index b5547330d3e..78df1a6c1bc 100644 --- a/helm/hpcc/templates/thor.yaml +++ b/helm/hpcc/templates/thor.yaml @@ -82,7 +82,7 @@ data: labels: accessDali: "yes" accessEsp: "yes" - helmVersion: 9.2.65-closedown0 + helmVersion: 9.2.67-closedown0 {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "eclagent" "name" "thor" "instance" $eclAgentJobName "instanceOf" (printf "%s-job" .eclAgentName)) | indent 8 }} {{- if hasKey .me "labels" }} {{ toYaml .me.labels | indent 12 }} @@ -145,7 +145,7 @@ data: accessEsp: "yes" app: "thor" component: "thormanager" - helmVersion: 9.2.65-closedown0 + helmVersion: 9.2.67-closedown0 instance: "_HPCC_JOBNAME_" job: "_HPCC_JOBNAME_" {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "thormanager" "name" "thor" "instance" $thorManagerJobName "instanceOf" (printf "%s-thormanager-job" .me.name)) | indent 12 }} @@ -210,7 +210,7 @@ data: accessEsp: "yes" app: "thor" component: "thorworker" - helmVersion: 9.2.65-closedown0 + helmVersion: 9.2.67-closedown0 instance: "_HPCC_JOBNAME_" job: "_HPCC_JOBNAME_" {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "thorworker" "name" "thor" "instance" $thorWorkerJobName "instanceOf" (printf "%s-thorworker-job" .me.name)) | indent 12 }} @@ -341,7 +341,7 @@ spec: accessEsp: {{ $commonCtx.eclAgentUseChildProcesses | ternary "yes" "no" | quote }} app: "thor" component: "thor-eclagent" - helmVersion: 9.2.65-closedown0 + helmVersion: 9.2.67-closedown0 instance: {{ $commonCtx.eclAgentName | quote }} {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "eclagent" "name" "thor" "instance" $commonCtx.eclAgentName ) | indent 8 }} {{- if hasKey $commonCtx.me "labels" }} @@ -404,7 +404,7 @@ spec: accessEsp: "no" app: "thor" component: "thor-thoragent" - helmVersion: 9.2.65-closedown0 + helmVersion: 9.2.67-closedown0 instance: {{ $commonCtx.thorAgentName | quote }} {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "eclagent" "name" "thor" "instance" $commonCtx.thorAgentName ) | indent 8 }} {{- if hasKey $commonCtx.me "labels" }} diff --git a/version.cmake b/version.cmake index 7833a82e358..e633902a6f2 100644 --- a/version.cmake +++ b/version.cmake @@ -5,8 +5,8 @@ set ( HPCC_NAME "Community Edition" ) set ( HPCC_PROJECT "community" ) set ( HPCC_MAJOR 9 ) set ( HPCC_MINOR 2 ) -set ( HPCC_POINT 65 ) +set ( HPCC_POINT 67 ) set ( HPCC_MATURITY "closedown" ) set ( HPCC_SEQUENCE 0 ) -set ( HPCC_TAG_TIMESTAMP "2024-02-22T17:02:01Z" ) +set ( HPCC_TAG_TIMESTAMP "2024-02-29T16:47:01Z" ) ### From e054901cdad906ded69de4b1af948acd1be21465 Mon Sep 17 00:00:00 2001 From: Jake Smith Date: Thu, 29 Feb 2024 16:48:00 +0000 Subject: [PATCH 03/17] Split off 9.4.40 Signed-off-by: Jake Smith --- helm/hpcc/Chart.yaml | 4 ++-- helm/hpcc/templates/_helpers.tpl | 2 +- helm/hpcc/templates/dafilesrv.yaml | 2 +- helm/hpcc/templates/dali.yaml | 2 +- helm/hpcc/templates/dfuserver.yaml | 2 +- helm/hpcc/templates/eclagent.yaml | 4 ++-- helm/hpcc/templates/eclccserver.yaml | 4 ++-- helm/hpcc/templates/eclscheduler.yaml | 2 +- helm/hpcc/templates/esp.yaml | 2 +- helm/hpcc/templates/localroxie.yaml | 2 +- helm/hpcc/templates/roxie.yaml | 8 ++++---- helm/hpcc/templates/sasha.yaml | 2 +- helm/hpcc/templates/thor.yaml | 10 +++++----- version.cmake | 4 ++-- 14 files changed, 25 insertions(+), 25 deletions(-) diff --git a/helm/hpcc/Chart.yaml b/helm/hpcc/Chart.yaml index 5f582c320ce..f0c4e4a4054 100644 --- a/helm/hpcc/Chart.yaml +++ b/helm/hpcc/Chart.yaml @@ -6,9 +6,9 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. -version: 9.4.39-closedown0 +version: 9.4.41-closedown0 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. -appVersion: 9.4.39-closedown0 +appVersion: 9.4.41-closedown0 diff --git a/helm/hpcc/templates/_helpers.tpl b/helm/hpcc/templates/_helpers.tpl index 0d9651df49d..2dac1e6384d 100644 --- a/helm/hpcc/templates/_helpers.tpl +++ b/helm/hpcc/templates/_helpers.tpl @@ -1499,7 +1499,7 @@ kind: Service metadata: name: {{ $lvars.serviceName | quote }} labels: - helmVersion: 9.4.39-closedown0 + helmVersion: 9.4.41-closedown0 {{- include "hpcc.addStandardLabels" (dict "root" $.root "instance" $lvars.serviceName ) | indent 4 }} {{- if $lvars.labels }} {{ toYaml $lvars.labels | indent 4 }} diff --git a/helm/hpcc/templates/dafilesrv.yaml b/helm/hpcc/templates/dafilesrv.yaml index 4be7c69957f..4553c3497fe 100644 --- a/helm/hpcc/templates/dafilesrv.yaml +++ b/helm/hpcc/templates/dafilesrv.yaml @@ -51,7 +51,7 @@ spec: labels: {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "dafilesrv" "name" "dafilesrv" "instance" .name) | indent 8 }} server: {{ .name | quote }} - helmVersion: 9.4.39-closedown0 + helmVersion: 9.4.41-closedown0 annotations: checksum/config: {{ $configSHA }} {{- include "hpcc.generateAnnotations" $commonCtx | indent 8 }} diff --git a/helm/hpcc/templates/dali.yaml b/helm/hpcc/templates/dali.yaml index ed9eb832b22..8aadbf80c77 100644 --- a/helm/hpcc/templates/dali.yaml +++ b/helm/hpcc/templates/dali.yaml @@ -84,7 +84,7 @@ spec: run: {{ $dali.name | quote }} server: {{ $dali.name | quote }} app: dali - helmVersion: 9.4.39-closedown0 + helmVersion: 9.4.41-closedown0 {{- if hasKey $.Values.global "metrics" }} {{- include "hpcc.generateMetricsReporterLabel" $.Values.global.metrics | nindent 8 }} {{- end }} diff --git a/helm/hpcc/templates/dfuserver.yaml b/helm/hpcc/templates/dfuserver.yaml index b023ed1eb3a..73f24e7c2ef 100644 --- a/helm/hpcc/templates/dfuserver.yaml +++ b/helm/hpcc/templates/dfuserver.yaml @@ -57,7 +57,7 @@ spec: {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "dfuserver" "name" "dfuserver" "instance" .name) | indent 8 }} run: {{ .name | quote }} accessDali: "yes" - helmVersion: 9.4.39-closedown0 + helmVersion: 9.4.41-closedown0 {{- if hasKey . "labels" }} {{ toYaml .labels | indent 8 }} {{- end }} diff --git a/helm/hpcc/templates/eclagent.yaml b/helm/hpcc/templates/eclagent.yaml index 2e1ba74eb1e..88d2cb86feb 100644 --- a/helm/hpcc/templates/eclagent.yaml +++ b/helm/hpcc/templates/eclagent.yaml @@ -60,7 +60,7 @@ data: {{- include "hpcc.addStandardLabels" (dict "root" $ "component" $apptype "name" "eclagent" "instance" $appJobName "instanceOf" (printf "%s-job" .me.name)) | indent 12 }} accessDali: "yes" accessEsp: "yes" - helmVersion: 9.4.39-closedown0 + helmVersion: 9.4.41-closedown0 {{- if hasKey .me "labels" }} {{ toYaml .me.labels | indent 12 }} {{- end }} @@ -137,7 +137,7 @@ spec: run: {{ .name | quote }} accessDali: "yes" accessEsp: {{ .useChildProcesses | default false | ternary "yes" "no" | quote }} - helmVersion: 9.4.39-closedown0 + helmVersion: 9.4.41-closedown0 {{- if hasKey . "labels" }} {{ toYaml .labels | indent 8 }} {{- end }} diff --git a/helm/hpcc/templates/eclccserver.yaml b/helm/hpcc/templates/eclccserver.yaml index a1b24e0b458..50f1aeeffc5 100644 --- a/helm/hpcc/templates/eclccserver.yaml +++ b/helm/hpcc/templates/eclccserver.yaml @@ -58,7 +58,7 @@ data: {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "eclccserver" "name" "eclccserver" "instance" $compileJobName "instanceOf" (printf "%s-job" .me.name)) | indent 12 }} accessDali: "yes" accessEsp: "yes" - helmVersion: 9.4.39-closedown0 + helmVersion: 9.4.41-closedown0 {{- if hasKey .me "labels" }} {{ toYaml .me.labels | indent 12 }} {{- end }} @@ -143,7 +143,7 @@ spec: run: {{ .name | quote }} accessDali: "yes" accessEsp: {{ .useChildProcesses | default false | ternary "yes" "no" | quote }} - helmVersion: 9.4.39-closedown0 + helmVersion: 9.4.41-closedown0 {{- if hasKey . "labels" }} {{ toYaml .labels | indent 8 }} {{- end }} diff --git a/helm/hpcc/templates/eclscheduler.yaml b/helm/hpcc/templates/eclscheduler.yaml index 1392ba16475..c9a85e4b523 100644 --- a/helm/hpcc/templates/eclscheduler.yaml +++ b/helm/hpcc/templates/eclscheduler.yaml @@ -65,7 +65,7 @@ spec: run: {{ .name | quote }} accessDali: "yes" accessEsp: "no" - helmVersion: 9.4.39-closedown0 + helmVersion: 9.4.41-closedown0 {{- if hasKey . "labels" }} {{ toYaml .labels | indent 8 }} {{- end }} diff --git a/helm/hpcc/templates/esp.yaml b/helm/hpcc/templates/esp.yaml index 1335cf07309..0f6d42fe71e 100644 --- a/helm/hpcc/templates/esp.yaml +++ b/helm/hpcc/templates/esp.yaml @@ -122,7 +122,7 @@ spec: accessSasha: "yes" {{- end }} app: {{ $application }} - helmVersion: 9.4.39-closedown0 + helmVersion: 9.4.41-closedown0 {{- include "hpcc.addStandardLabels" (dict "root" $ "name" $application "component" "esp" "instance" .name) | indent 8 }} {{- if hasKey $.Values.global "metrics" }} {{- include "hpcc.generateMetricsReporterLabel" $.Values.global.metrics | nindent 8 }} diff --git a/helm/hpcc/templates/localroxie.yaml b/helm/hpcc/templates/localroxie.yaml index 5b4b0d4e8f3..6013ddec3bb 100644 --- a/helm/hpcc/templates/localroxie.yaml +++ b/helm/hpcc/templates/localroxie.yaml @@ -73,7 +73,7 @@ spec: server: {{ $servername | quote }} accessDali: "yes" accessEsp: "yes" - helmVersion: 9.4.39-closedown0 + helmVersion: 9.4.41-closedown0 {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "roxie-server" "name" "roxie" "instance" $roxie.name) | indent 8 }} {{- if hasKey . "labels" }} {{ toYaml .labels | indent 8 }} diff --git a/helm/hpcc/templates/roxie.yaml b/helm/hpcc/templates/roxie.yaml index debfe165fc0..57f8d523ded 100644 --- a/helm/hpcc/templates/roxie.yaml +++ b/helm/hpcc/templates/roxie.yaml @@ -125,7 +125,7 @@ spec: {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "topology-server" "name" "roxie" "instance" $commonCtx.toponame) | indent 8 }} run: {{ $commonCtx.toponame | quote }} roxie-cluster: {{ $roxie.name | quote }} - helmVersion: 9.4.39-closedown0 + helmVersion: 9.4.41-closedown0 {{- if hasKey $.Values.global "metrics" }} {{- include "hpcc.generateMetricsReporterLabel" $.Values.global.metrics | nindent 8}} {{- end }} @@ -183,7 +183,7 @@ kind: Service metadata: name: {{ $commonCtx.toponame | quote }} labels: - helmVersion: 9.4.39-closedown0 + helmVersion: 9.4.41-closedown0 {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "topology-server" "name" "roxie" "instance" $commonCtx.toponame) | indent 4 }} spec: ports: @@ -245,7 +245,7 @@ spec: roxie-cluster: {{ $roxie.name | quote }} accessDali: "yes" accessEsp: "yes" - helmVersion: 9.4.39-closedown0 + helmVersion: 9.4.41-closedown0 {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "roxie-server" "name" "roxie" "instance" $servername) | indent 8 }} {{- if hasKey $.Values.global "metrics" }} {{- include "hpcc.generateMetricsReporterLabel" $.Values.global.metrics | nindent 8}} @@ -353,7 +353,7 @@ spec: roxie-cluster: {{ $roxie.name | quote }} accessDali: "yes" accessEsp: "yes" - helmVersion: 9.4.39-closedown0 + helmVersion: 9.4.41-closedown0 {{- if hasKey $.Values.global "metrics" }} {{- include "hpcc.generateMetricsReporterLabel" $.Values.global.metrics | nindent 8}} {{- end }} diff --git a/helm/hpcc/templates/sasha.yaml b/helm/hpcc/templates/sasha.yaml index 0700424fc40..7a3dd69bdb7 100644 --- a/helm/hpcc/templates/sasha.yaml +++ b/helm/hpcc/templates/sasha.yaml @@ -53,7 +53,7 @@ spec: server: {{ $serviceName | quote }} app: sasha accessDali: {{ (has "dali" $sasha.access) | ternary "yes" "no" | quote }} - helmVersion: 9.4.39-closedown0 + helmVersion: 9.4.41-closedown0 {{- if hasKey $sasha "labels" }} {{ toYaml $sasha.labels | indent 8 }} {{- end }} diff --git a/helm/hpcc/templates/thor.yaml b/helm/hpcc/templates/thor.yaml index 30fb477ff95..13c617f0459 100644 --- a/helm/hpcc/templates/thor.yaml +++ b/helm/hpcc/templates/thor.yaml @@ -86,7 +86,7 @@ data: labels: accessDali: "yes" accessEsp: "yes" - helmVersion: 9.4.39-closedown0 + helmVersion: 9.4.41-closedown0 {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "eclagent" "name" "thor" "instance" $eclAgentJobName "instanceOf" (printf "%s-job" .eclAgentName)) | indent 8 }} {{- if hasKey .me "labels" }} {{ toYaml .me.labels | indent 12 }} @@ -151,7 +151,7 @@ data: accessEsp: "yes" app: "thor" component: "thormanager" - helmVersion: 9.4.39-closedown0 + helmVersion: 9.4.41-closedown0 instance: "_HPCC_JOBNAME_" job: "_HPCC_JOBNAME_" {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "thormanager" "name" "thor" "instance" $thorManagerJobName "instanceOf" (printf "%s-thormanager-job" .me.name)) | indent 12 }} @@ -218,7 +218,7 @@ data: accessEsp: "yes" app: "thor" component: "thorworker" - helmVersion: 9.4.39-closedown0 + helmVersion: 9.4.41-closedown0 instance: "_HPCC_JOBNAME_" job: "_HPCC_JOBNAME_" {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "thorworker" "name" "thor" "instance" $thorWorkerJobName "instanceOf" (printf "%s-thorworker-job" .me.name)) | indent 12 }} @@ -351,7 +351,7 @@ spec: accessEsp: {{ $commonCtx.eclAgentUseChildProcesses | ternary "yes" "no" | quote }} app: "thor" component: "thor-eclagent" - helmVersion: 9.4.39-closedown0 + helmVersion: 9.4.41-closedown0 instance: {{ $commonCtx.eclAgentName | quote }} {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "eclagent" "name" "thor" "instance" $commonCtx.eclAgentName ) | indent 8 }} {{- if hasKey $commonCtx.me "labels" }} @@ -416,7 +416,7 @@ spec: accessEsp: "no" app: "thor" component: "thor-thoragent" - helmVersion: 9.4.39-closedown0 + helmVersion: 9.4.41-closedown0 instance: {{ $commonCtx.thorAgentName | quote }} {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "eclagent" "name" "thor" "instance" $commonCtx.thorAgentName ) | indent 8 }} {{- if hasKey $commonCtx.me "labels" }} diff --git a/version.cmake b/version.cmake index 08681876b08..64412a8589d 100644 --- a/version.cmake +++ b/version.cmake @@ -5,8 +5,8 @@ set ( HPCC_NAME "Community Edition" ) set ( HPCC_PROJECT "community" ) set ( HPCC_MAJOR 9 ) set ( HPCC_MINOR 4 ) -set ( HPCC_POINT 39 ) +set ( HPCC_POINT 41 ) set ( HPCC_MATURITY "closedown" ) set ( HPCC_SEQUENCE 0 ) -set ( HPCC_TAG_TIMESTAMP "2024-02-22T17:02:48Z" ) +set ( HPCC_TAG_TIMESTAMP "2024-02-29T16:47:59Z" ) ### From f3b071e9c58584576b0c0b3bc5b2860066016d9f Mon Sep 17 00:00:00 2001 From: Jake Smith Date: Thu, 29 Feb 2024 17:19:51 +0000 Subject: [PATCH 04/17] HPCC-31374 Fix auxQueues helm error if 0 thors Fix helm template generation error if 0 thors are defined. Validate combinedList size before trying to generate the yaml list. Signed-off-by: Jake Smith --- helm/hpcc/templates/_helpers.tpl | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/helm/hpcc/templates/_helpers.tpl b/helm/hpcc/templates/_helpers.tpl index 2dac1e6384d..b52b5929b48 100644 --- a/helm/hpcc/templates/_helpers.tpl +++ b/helm/hpcc/templates/_helpers.tpl @@ -1175,7 +1175,9 @@ Generate instance queue names {{- range $auxQueue := $auxThorQueues }} {{- $combinedList = append $combinedList $auxQueue -}} {{- end -}} -{{- toYaml $combinedList -}} +{{- if $combinedList -}} + {{- toYaml $combinedList -}} +{{- end -}} {{- end -}} {{- define "hpcc.usesRemoteIssuer" -}} From c0cbf94d3919303989bf1935cc25cb595c63772d Mon Sep 17 00:00:00 2001 From: Jim DeFabia Date: Thu, 29 Feb 2024 12:35:35 -0500 Subject: [PATCH 05/17] HPCC-31359 Remove mentions of 32-bit Client Tools Signed-off-by: Jim DeFabia --- .../HPCCClientTools/CT_Mods/CT_Overview.xml | 16 ---------------- .../CT_Mods/CT_Overview_withoutIDE.xml | 14 -------------- 2 files changed, 30 deletions(-) diff --git a/docs/EN_US/HPCCClientTools/CT_Mods/CT_Overview.xml b/docs/EN_US/HPCCClientTools/CT_Mods/CT_Overview.xml index 6cace2d5391..d9ef060c461 100644 --- a/docs/EN_US/HPCCClientTools/CT_Mods/CT_Overview.xml +++ b/docs/EN_US/HPCCClientTools/CT_Mods/CT_Overview.xml @@ -129,22 +129,6 @@ Install the client tools software to your machine. - - - - Note: - - - For large ECL workunits, the 32-bit compiler (eclcc) - could run out of memory quicker than the 64-bit version. - Therefore in most cases, you should install the 64-bit - version. However, for machines with 4GB memory or less, you - should use the 32-bit client tools. If you have both versions - on your local machine, the IDE will give preference to the - 64-bit version. - - - diff --git a/docs/EN_US/HPCCClientTools/CT_Mods/CT_Overview_withoutIDE.xml b/docs/EN_US/HPCCClientTools/CT_Mods/CT_Overview_withoutIDE.xml index d6cabd58c77..07fba16fa8e 100644 --- a/docs/EN_US/HPCCClientTools/CT_Mods/CT_Overview_withoutIDE.xml +++ b/docs/EN_US/HPCCClientTools/CT_Mods/CT_Overview_withoutIDE.xml @@ -112,20 +112,6 @@ Install the client tools software to your machine. - - - - Note: - - - For large ECL workunits, the 32-bit compiler (eclcc) - could run out of memory quicker than the 64-bit version. - Therefore in most cases, you should install the 64-bit - version. However, for machines with 4GB memory or less, you - should use the 32-bit client tools. - - - From a42c45ddb410bc81c294e16b9fc1f13edb79f7b2 Mon Sep 17 00:00:00 2001 From: Rodrigo Pastrana Date: Mon, 26 Feb 2024 16:12:52 -0500 Subject: [PATCH 06/17] HPCC-31320 JTrace Reports name of traced service - Creates OTEL resource with service.name attr - Reports current component as service.name Signed-off-by: Rodrigo Pastrana --- system/jlib/jtrace.cpp | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/system/jlib/jtrace.cpp b/system/jlib/jtrace.cpp index 4aa800d848e..e33a9f7681d 100644 --- a/system/jlib/jtrace.cpp +++ b/system/jlib/jtrace.cpp @@ -1220,9 +1220,12 @@ void CTraceManager::initTracerProviderAndGlobalInternals(const IPropertyTree * t processors.push_back(opentelemetry::sdk::trace::SimpleSpanProcessorFactory::Create(std::move(exporter))); } + opentelemetry::sdk::resource::ResourceAttributes resourceAtts = {{"service.name", moduleName.get()}}; + auto jtraceResource = opentelemetry::sdk::resource::Resource::Create(resourceAtts); + // Default is an always-on sampler. std::shared_ptr context = - opentelemetry::sdk::trace::TracerContextFactory::Create(std::move(processors)); + opentelemetry::sdk::trace::TracerContextFactory::Create(std::move(processors), jtraceResource); std::shared_ptr provider = opentelemetry::sdk::trace::TracerProviderFactory::Create(context); From 778e42da31b5ed1d9f8f605515a6c7287793fa72 Mon Sep 17 00:00:00 2001 From: Gordon Smith Date: Thu, 29 Feb 2024 16:52:11 +0000 Subject: [PATCH 07/17] HPCC-31378 Buffer overrun in rtlPadTruncString Signed-off-by: Gordon Smith --- rtl/eclrtl/rtltype.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rtl/eclrtl/rtltype.cpp b/rtl/eclrtl/rtltype.cpp index a0f0409b452..ecee7e5a8bc 100644 --- a/rtl/eclrtl/rtltype.cpp +++ b/rtl/eclrtl/rtltype.cpp @@ -62,7 +62,7 @@ void rtlPadTruncString(unsigned & tgtLen, char * & tgt, unsigned newLen, unsigne { if (len > newLen) len = newLen; - char * buff = (char *)malloc(len); + char * buff = (char *)malloc(newLen); memcpy(buff, src, len); if (len < newLen) memset(buff+len, ' ', newLen - len); From 07820be87ad7308e850a3a8e6b60968e4033c753 Mon Sep 17 00:00:00 2001 From: Mark Kelly Date: Fri, 1 Mar 2024 11:24:36 -0500 Subject: [PATCH 08/17] HPCC-31385 Roxie complete line elapsed time not correct for longer WU queries Signed-off-by: Mark Kelly --- roxie/ccd/ccdlistener.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roxie/ccd/ccdlistener.cpp b/roxie/ccd/ccdlistener.cpp index 7badb16cc46..c26eb8b9534 100644 --- a/roxie/ccd/ccdlistener.cpp +++ b/roxie/ccd/ccdlistener.cpp @@ -1316,7 +1316,7 @@ class RoxieWorkUnitWorker : public RoxieQueryWorker reportUnknownException(wu, logctx); } #endif - unsigned elapsedNs = nsTick() - startNs; + stat_type elapsedNs = nsTick() - startNs; unsigned elapsedMs = nanoToMilli(elapsedNs); noteQuery(failed, elapsedMs, priority); queryFactory->noteQuery(startTime, failed, elapsedMs, memused, agentsReplyLen, 0); From 6491527f784af0bfaa105997a39db4ef6ed33cfe Mon Sep 17 00:00:00 2001 From: Rodrigo Pastrana Date: Mon, 26 Feb 2024 11:15:08 -0500 Subject: [PATCH 09/17] HPCC-31347 JLog trace export catch all Iexceptions - Catches all IExceptions thrown in noexcept function - Logs issue - Releases exception - Reports export failure if exception caught Signed-off-by: Rodrigo Pastrana --- system/jlib/jtrace.cpp | 113 ++++++++++++++++++++++------------------- 1 file changed, 61 insertions(+), 52 deletions(-) diff --git a/system/jlib/jtrace.cpp b/system/jlib/jtrace.cpp index bc213c6c197..c69094d784c 100644 --- a/system/jlib/jtrace.cpp +++ b/system/jlib/jtrace.cpp @@ -179,76 +179,85 @@ class JLogSpanExporter final : public opentelemetry::sdk::trace::SpanExporter opentelemetry::sdk::common::ExportResult Export( const nostd::span> &recordables) noexcept override { - if (isShutDown()) - return opentelemetry::sdk::common::ExportResult::kFailure; - - for (auto &recordable : recordables) + try { - //Casting the recordable object to the type of the object that was previously created by - //JLogSpanExporter::MakeRecordable() - - auto span = std::unique_ptr( - static_cast(recordable.release())); + if (isShutDown()) + return opentelemetry::sdk::common::ExportResult::kFailure; - if (span != nullptr) + for (auto &recordable : recordables) { - char traceID[32] = {0}; - char spanID[16] = {0}; + //Casting the recordable object to the type of the object that was previously created by + //JLogSpanExporter::MakeRecordable() - + auto span = std::unique_ptr( + static_cast(recordable.release())); - span->GetTraceId().ToLowerBase16(traceID); - span->GetSpanId().ToLowerBase16(spanID); + if (span != nullptr) + { + char traceID[32] = {0}; + char spanID[16] = {0}; - StringBuffer out("{ \"type\": \"span\""); //for simple identification in log scraping - out.appendf(", \"name\": \"%s\"", span->GetName().data()); - out.append(", \"trace_id\": \"").append(32, traceID).append("\""); - out.append(", \"span_id\": \"").append(16, spanID).append("\""); - out.appendf(", \"start\": %lld", (long long)(span->GetStartTime().time_since_epoch()).count()); - out.appendf(", \"duration\": %lld", (long long)span->GetDuration().count()); + span->GetTraceId().ToLowerBase16(traceID); + span->GetSpanId().ToLowerBase16(spanID); - if (hasMask(logFlags, SpanLogFlags::LogParentInfo)) - { - if (span->GetParentSpanId().IsValid()) + StringBuffer out("{ \"type\": \"span\""); //for simple identification in log scraping + out.appendf(", \"name\": \"%s\"", span->GetName().data()); + out.append(", \"trace_id\": \"").append(32, traceID).append("\""); + out.append(", \"span_id\": \"").append(16, spanID).append("\""); + out.appendf(", \"start\": %lld", (long long)(span->GetStartTime().time_since_epoch()).count()); + out.appendf(", \"duration\": %lld", (long long)span->GetDuration().count()); + + if (hasMask(logFlags, SpanLogFlags::LogParentInfo)) { - char parentSpanID[16] = {0}; - span->GetParentSpanId().ToLowerBase16(parentSpanID); - out.append(", \"parent_span_id\": \"").append(16, parentSpanID).append("\""); - } + if (span->GetParentSpanId().IsValid()) + { + char parentSpanID[16] = {0}; + span->GetParentSpanId().ToLowerBase16(parentSpanID); + out.append(", \"parent_span_id\": \"").append(16, parentSpanID).append("\""); + } - std::string traceStatestr = span->GetSpanContext().trace_state()->ToHeader(); - if (!traceStatestr.empty()) - out.appendf(", \"trace_state\": \"%s\"", traceStatestr.c_str()); - } + std::string traceStatestr = span->GetSpanContext().trace_state()->ToHeader(); + if (!traceStatestr.empty()) + out.appendf(", \"trace_state\": \"%s\"", traceStatestr.c_str()); + } - if (hasMask(logFlags, SpanLogFlags::LogSpanDetails)) - { - out.appendf(", \"status\": \"%s\"", spanStatusToString(span->GetStatus())); - out.appendf(", \"kind\": \"%s\"", spanKindToString(span->GetSpanKind())); - const char * description = span->GetDescription().data(); - if (!isEmptyString(description)) + if (hasMask(logFlags, SpanLogFlags::LogSpanDetails)) { - StringBuffer encoded; - encodeJSON(encoded, description); - out.appendf(", \"description\": \"%s\"", encoded.str()); + out.appendf(", \"status\": \"%s\"", spanStatusToString(span->GetStatus())); + out.appendf(", \"kind\": \"%s\"", spanKindToString(span->GetSpanKind())); + const char * description = span->GetDescription().data(); + if (!isEmptyString(description)) + { + StringBuffer encoded; + encodeJSON(encoded, description); + out.appendf(", \"description\": \"%s\"", encoded.str()); + } + printInstrumentationScope(out, span->GetInstrumentationScope()); } - printInstrumentationScope(out, span->GetInstrumentationScope()); - } - if (hasMask(logFlags, SpanLogFlags::LogAttributes)) - printAttributes(out, span->GetAttributes()); + if (hasMask(logFlags, SpanLogFlags::LogAttributes)) + printAttributes(out, span->GetAttributes()); - if (hasMask(logFlags, SpanLogFlags::LogEvents)) - printEvents(out, span->GetEvents()); + if (hasMask(logFlags, SpanLogFlags::LogEvents)) + printEvents(out, span->GetEvents()); - if (hasMask(logFlags, SpanLogFlags::LogLinks)) - printLinks(out, span->GetLinks()); + if (hasMask(logFlags, SpanLogFlags::LogLinks)) + printLinks(out, span->GetLinks()); - if (hasMask(logFlags, SpanLogFlags::LogResources)) - printResources(out, span->GetResource()); + if (hasMask(logFlags, SpanLogFlags::LogResources)) + printResources(out, span->GetResource()); - out.append(" }"); - LOG(MCmonitorEvent, "%s",out.str()); + out.append(" }"); + LOG(MCmonitorEvent, "%s",out.str()); + } } + return opentelemetry::sdk::common::ExportResult::kSuccess; + } + catch (IException * e) + { + EXCLOG(e, "JLogSpanExporter::Export"); + e->Release(); + return opentelemetry::sdk::common::ExportResult::kFailure; } - return opentelemetry::sdk::common::ExportResult::kSuccess; } /** From f9b7be0c27bb912bc04d34cf60db1d5579bc757a Mon Sep 17 00:00:00 2001 From: Mark Kelly Date: Mon, 4 Mar 2024 18:13:55 -0500 Subject: [PATCH 10/17] HPCC-31391 Roxie passphrase incorrectly decrypted Signed-off-by: Mark Kelly --- roxie/ccd/ccdmain.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roxie/ccd/ccdmain.cpp b/roxie/ccd/ccdmain.cpp index 3545d613b85..ecaa7722ca7 100644 --- a/roxie/ccd/ccdmain.cpp +++ b/roxie/ccd/ccdmain.cpp @@ -1537,7 +1537,7 @@ int CCD_API roxie_main(int argc, const char *argv[], const char * defaultYaml) const char *passPhrase = roxieFarm.queryProp("@passphrase"); if (!isEmptyString(passPhrase)) - decrypt(passPhraseStr, passPhrase); + passPhraseStr.append(passPhrase); // NB passphrase is decrypted in CSecureSocketContext::createNewContext() const char *certFile = roxieFarm.queryProp("@certificateFileName"); if (!certFile) From 20ff3254afb796d210bd21b1fe75b79704e00cca Mon Sep 17 00:00:00 2001 From: Gordon Smith Date: Mon, 4 Mar 2024 17:29:21 +0000 Subject: [PATCH 11/17] HPCC-31387 ECL Watch failing to display Signed-off-by: Gordon Smith --- esp/src/src-react/components/Frame.tsx | 2 +- esp/src/src-react/components/Title.tsx | 7 +------ 2 files changed, 2 insertions(+), 7 deletions(-) diff --git a/esp/src/src-react/components/Frame.tsx b/esp/src/src-react/components/Frame.tsx index 58967d6ff4d..60355ee8def 100644 --- a/esp/src/src-react/components/Frame.tsx +++ b/esp/src/src-react/components/Frame.tsx @@ -91,7 +91,7 @@ export const Frame: React.FunctionComponent = () => { }, []); React.useEffect(() => { - document.title = `${showEnvironmentTitle && environmentTitle.length ? environmentTitle : "ECL Watch "}${locationPathname.split("/").join(" | ")}`; + document.title = `${(showEnvironmentTitle && environmentTitle) ? environmentTitle : "ECL Watch "}${locationPathname.split("/").join(" | ")}`; }, [environmentTitle, locationPathname, showEnvironmentTitle]); React.useEffect(() => { diff --git a/esp/src/src-react/components/Title.tsx b/esp/src/src-react/components/Title.tsx index 9af34dfa076..ff235f3b50d 100644 --- a/esp/src/src-react/components/Title.tsx +++ b/esp/src/src-react/components/Title.tsx @@ -247,11 +247,6 @@ export const DevTitle: React.FunctionComponent = ({ } }, [currentUser]); - React.useEffect(() => { - if (!environmentTitle) return; - document.title = environmentTitle; - }, [environmentTitle]); - return
@@ -264,7 +259,7 @@ export const DevTitle: React.FunctionComponent = ({ - {showEnvironmentTitle && environmentTitle.length ? environmentTitle : "ECL Watch"} + {(showEnvironmentTitle && environmentTitle) ? environmentTitle : "ECL Watch"} From b66c8333a215fccca4e5740459312dfefe5e130b Mon Sep 17 00:00:00 2001 From: Gavin Halliday Date: Tue, 5 Mar 2024 13:57:39 +0000 Subject: [PATCH 12/17] HPCC-31392 Default failure to fetch from git to a warning Signed-off-by: Gavin Halliday --- ecl/hql/hqlrepository.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ecl/hql/hqlrepository.cpp b/ecl/hql/hqlrepository.cpp index 1c335ebe3cb..8b7c5a47649 100644 --- a/ecl/hql/hqlrepository.cpp +++ b/ecl/hql/hqlrepository.cpp @@ -827,11 +827,11 @@ IEclSourceCollection * EclRepositoryManager::resolveGitCollection(const char * r { if (options.updateRepos) { - unsigned retCode = runGitCommand(nullptr, "fetch origin", repoPath, true); + unsigned retCode = runGitCommand(nullptr, "fetch origin --prune", repoPath, true); if (retCode != 0) { VStringBuffer msg("Failed to download the latest version of '%s' error code (%u)", defaultUrl, retCode); - error.setown(createError(CategoryError, SeverityError, ERR_FAIL_UPDATE_REPO, msg.str(), nullptr, 0, 0, 0)); + error.setown(createError(CategoryError, SeverityWarning, ERR_FAIL_UPDATE_REPO, msg.str(), nullptr, 0, 0, 0)); } } From 8bd59469491bcb7ddac1392c74241f33d9440f0c Mon Sep 17 00:00:00 2001 From: Rodrigo Pastrana Date: Tue, 5 Mar 2024 23:12:23 -0500 Subject: [PATCH 13/17] HPCC-31398 Ensures JLog Exporter reports valid JSON - Moves comma logic into correct position - Adds unittests Signed-off-by: Rodrigo Pastrana --- system/jlib/jtrace.cpp | 11 +++++------ testing/unittests/jlibtests.cpp | 26 ++++++++++++++++++++++++++ 2 files changed, 31 insertions(+), 6 deletions(-) diff --git a/system/jlib/jtrace.cpp b/system/jlib/jtrace.cpp index d40abcbd2d6..45424274e4f 100644 --- a/system/jlib/jtrace.cpp +++ b/system/jlib/jtrace.cpp @@ -291,17 +291,17 @@ class JLogSpanExporter final : public opentelemetry::sdk::trace::SpanExporter bool first = true; for (const auto &kv : map) { - if (!first) - out.append(","); - else - first = false; - const auto & value = kv.second; std::ostringstream attsOS; //used to exploit OTel convenience functions for printing attribute values opentelemetry::exporter::ostream_common::print_value(value, attsOS); std::string val = attsOS.str(); if (val.size() > 0) { + if (!first) + out.append(","); + else + first = false; + switch (value.index()) { case opentelemetry::sdk::common::kTypeBool: @@ -335,7 +335,6 @@ class JLogSpanExporter final : public opentelemetry::sdk::trace::SpanExporter } } out.append(" }"); - } catch(const std::bad_variant_access & e) { diff --git a/testing/unittests/jlibtests.cpp b/testing/unittests/jlibtests.cpp index a7ab0975b2b..26b5674a8d2 100644 --- a/testing/unittests/jlibtests.cpp +++ b/testing/unittests/jlibtests.cpp @@ -63,6 +63,7 @@ class JlibTraceTest : public CppUnit::TestFixture CPPUNIT_TEST(testNullSpan); CPPUNIT_TEST(testClientSpanGlobalID); CPPUNIT_TEST(testEnsureTraceID); + CPPUNIT_TEST(manualTestsEventsOutput); //CPPUNIT_TEST(testJTraceJLOGExporterprintResources); //CPPUNIT_TEST(testJTraceJLOGExporterprintAttributes); @@ -189,6 +190,31 @@ class JlibTraceTest : public CppUnit::TestFixture CPPUNIT_ASSERT_EQUAL_MESSAGE("Missing resource attribute detected", true, jtraceAsTree->hasProp("resources/telemetry.sdk.name")); }*/ + //not able to programmatically test yet, but can visually inspect trace output + void manualTestsEventsOutput() + { + Owned emptyMockHTTPHeaders = createProperties(); + { + Owned serverSpan = queryTraceManager().createServerSpan("spanWithEventsNoAtts", emptyMockHTTPHeaders); + Owned emptyEventAtts = createProperties(); + serverSpan->addSpanEvent("event1", emptyEventAtts); + } + + Owned twoEventAtt = createProperties(); + twoEventAtt->setProp("key", "value"); + twoEventAtt->setProp("key2", ""); + + { + Owned serverSpan = queryTraceManager().createServerSpan("spanWithEvent1Att", emptyMockHTTPHeaders); + serverSpan->addSpanEvent("event2", twoEventAtt); + }//{ "type": "span", "name": "spanWithEvents1Att", "trace_id": "3b9f55aaf8fab51fb0d73a32db7d704f", "span_id": "2a25a44ae0b3abe0", "start": 1709696036335278770, "duration": 3363911469, "events":[ { "name": "event2", "time_stamp": 1709696038413023245, "attributes": {"key": "value" } } ] } + + { + Owned serverSpan = queryTraceManager().createServerSpan("spanWith2Events", emptyMockHTTPHeaders); + serverSpan->addSpanEvent("event1", twoEventAtt); + serverSpan->addSpanEvent("event2", twoEventAtt); + }//{ "type": "span", "name": "spanWith2Events", "trace_id": "ff5c5919b9c5f85913652b77f289bf0b", "span_id": "82f91ca1f9d469c1", "start": 1709698012480805016, "duration": 2811601377, "events":[ { "name": "event1", "time_stamp": 1709698013294323139, "attributes": {"key": "value" } },{ "name": "event2", "time_stamp": 1709698014500350802, "attributes": {"key": "value" } } ] } + } //not able to programmatically test yet, but can visually inspect trace output void manualTestsDeclaredSpanStartTime() { From 8c4c2995c1e44fabd88de1a1acd81e88f84c0cc2 Mon Sep 17 00:00:00 2001 From: Gavin Halliday Date: Wed, 6 Mar 2024 09:51:26 +0000 Subject: [PATCH 14/17] HPCC-31400 Fix build error with newer versions of clang Signed-off-by: Gavin Halliday --- system/jhtree/jhtree.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/system/jhtree/jhtree.cpp b/system/jhtree/jhtree.cpp index 66ee27999d2..e76b915a383 100644 --- a/system/jhtree/jhtree.cpp +++ b/system/jhtree/jhtree.cpp @@ -647,7 +647,7 @@ class CNodeCacheEntry : public CInterface public: CriticalSection cs; private: - std::atomic node = nullptr; + std::atomic node{nullptr}; public: ~CNodeCacheEntry() { From 32af3fb000b6a6607ab7ab1a5446942440c577fc Mon Sep 17 00:00:00 2001 From: Rodrigo Pastrana Date: Fri, 23 Feb 2024 21:55:17 -0500 Subject: [PATCH 15/17] HPCC-31343 JTrace status and record support - Exposes error reporting methods - Exposes span status support - Reports Status and description in JLog exporter by default - Updated test cases Signed-off-by: Rodrigo Pastrana --- roxie/ccd/ccdlistener.cpp | 4 +-- system/jlib/jtrace.cpp | 42 +++++++++++++++++++++++ system/jlib/jtrace.hpp | 56 ++++++++++++++++++++++++++++++ testing/unittests/jlibtests.cpp | 60 +++++++++++++++++++++++++++++++++ 4 files changed, 160 insertions(+), 2 deletions(-) diff --git a/roxie/ccd/ccdlistener.cpp b/roxie/ccd/ccdlistener.cpp index c26eb8b9534..2615434ab2e 100644 --- a/roxie/ccd/ccdlistener.cpp +++ b/roxie/ccd/ccdlistener.cpp @@ -949,9 +949,9 @@ void ContextLogger::exportStatsToSpan(bool failed, stat_type elapsedNs, unsigned { if (activeSpan->isRecording()) { + activeSpan->setSpanStatus(failed); setSpanAttribute("time_elapsed", elapsedNs); - if (failed) - setSpanAttribute("num_failures", 1); + if (memused) setSpanAttribute("size_peak_row_memory", memused * 0x100000); diff --git a/system/jlib/jtrace.cpp b/system/jlib/jtrace.cpp index 45424274e4f..1c1ba40dfb6 100644 --- a/system/jlib/jtrace.cpp +++ b/system/jlib/jtrace.cpp @@ -742,6 +742,44 @@ class CSpan : public CInterfaceOf return span ? span->IsRecording() : false; } + virtual void setSpanStatus(bool spanFailed, const char * statusMessage) + { + if (span != nullptr) + { + span->SetStatus(spanFailed ? opentelemetry::trace::StatusCode::kError : opentelemetry::trace::StatusCode::kOk, statusMessage); + } + } + + virtual void recordError(const SpanError & error) + { + if (span != nullptr) + { + if (error.spanFailed) + span->SetStatus(opentelemetry::trace::StatusCode::kError, error.errorMessage); + + //https://opentelemetry.io/docs/specs/semconv/exceptions/exceptions-spans/ + //The event name MUST be "exception". + //The table below indicates which attributes should be added to the Event and their types. + //exception.escaped boolean SHOULD be set to true if the exception event is recorded at a point where it is known that the exception is escaping the scope of the span. [1] Recommended + //exception.message string The exception message. Division by zero; Can't convert 'int' object to str implicitly See below + //exception.stacktrace string A stacktrace as a string in the natural representation for the language runtime. The representation is to be determined and documented by each language SIG. + // Exception in thread "main" java.lang.RuntimeException: Test exception\n at com.example.GenerateTrace.methodB(GenerateTrace.java:13)\n at com.example.GenerateTrace.methodA(GenerateTrace.java:9) + //exception.type string The type of the exception (its fully-qualified class name, if applicable). The dynamic type of the exception should be preferred over the static type in languages that support it. java.net.ConnectException; OSError See below + + if (error.errorCode != 0 && error.errorCode != -1) + span->AddEvent("Exception", {{"message", error.errorMessage}, {"escaped", error.escapeScope}, {"code", error.errorCode}}); + else + span->AddEvent("Exception", {{"message", error.errorMessage}, {"escaped", error.escapeScope}}); + } + } + + virtual void recordException(IException * e, bool spanFailed, bool escapedScope) + { + StringBuffer msg; + e->errorMessage(msg); + recordError(SpanError(msg.str(), e->errorCode(), spanFailed, escapedScope)); + }; + protected: CSpan(const char * spanName) { @@ -847,6 +885,10 @@ class CNullSpan : public CInterfaceOf virtual void getLogPrefix(StringBuffer & out) const override {} virtual bool isRecording() const { return false; } + virtual void recordException(IException * e, bool spanFailed, bool escapedScope) override {} + virtual void recordError(const SpanError & error) override {}; + virtual void setSpanStatus(bool spanFailed, const char * statusMessage) override {} + virtual const char* queryGlobalId() const override { return nullptr; } virtual const char* queryCallerId() const override { return nullptr; } virtual const char* queryLocalId() const override { return nullptr; } diff --git a/system/jlib/jtrace.hpp b/system/jlib/jtrace.hpp index c2cdeb2bae1..6689dff6597 100644 --- a/system/jlib/jtrace.hpp +++ b/system/jlib/jtrace.hpp @@ -55,6 +55,59 @@ enum class SpanFlags : unsigned }; BITMASK_ENUM(SpanFlags); +static constexpr const char * NO_STATUS_MESSAGE = ""; +static constexpr int UNKNOWN_ERROR_CODE = -1; + +struct SpanError +/** + * @brief Represents an error that occurred during a span. + * + * Used to store information about an error that occurred during the execution of a span. + * It includes the error message, error code, + * and flags indicating whether the span failed and whether the exception escaped the span's scope. + */ +{ + const char * errorMessage = NO_STATUS_MESSAGE; /**< The error message associated with the error. */ + int errorCode = UNKNOWN_ERROR_CODE; /**< The error code associated with the error. */ + bool spanFailed = true; /**< Flag indicating whether the span failed. */ + bool escapeScope = false; /**< Flag indicating whether the exception escaped the scope of the span. */ + + /** + * @brief Default constructor. + */ + SpanError() = default; + + /** + * @brief Constructor with error message. + * @param _errorMessage The error message. + */ + SpanError(const char * _errorMessage) : errorMessage(_errorMessage) {} + + /** + * @brief Constructor with error message, error code, span failure flag, and scope escape flag. + * @param _errorMessage The error message. + * @param _errorCode The error code. + * @param _spanFailed Flag indicating whether the span failed. + * @param _escapeScope Flag indicating whether the exception escaped the scope of the span. + */ + SpanError(const char * _errorMessage, int _errorCode, bool _spanFailed, bool _escapeScope) + : errorMessage(_errorMessage), errorCode(_errorCode), spanFailed(_spanFailed), escapeScope(_escapeScope) {} + + /** + * @brief Sets the span status. + * @param _spanFailed Flag indicating whether the span failed. + * @param _spanScopeEscape Flag indicating whether the exception escaped the scope of the span. + */ + void setSpanStatus(bool _spanFailed, bool _spanScopeEscape) { spanFailed = _spanFailed; escapeScope = _spanScopeEscape;} + + /** + * @brief Sets the error message and error code. + * @param _errorMessage The error message. + * @param _errorCode The error code. + */ + void setError(const char * _errorMessage, int _errorCode) { errorMessage = _errorMessage; errorCode = _errorCode; } +}; + interface ISpan : extends IInterface { virtual void setSpanAttribute(const char * key, const char * val) = 0; @@ -67,6 +120,9 @@ interface ISpan : extends IInterface virtual void toString(StringBuffer & out) const = 0; virtual void getLogPrefix(StringBuffer & out) const = 0; virtual bool isRecording() const = 0; // Is it worth adding any events/attributes to this span? + virtual void recordException(IException * e, bool spanFailed = true, bool escapedScope = true) = 0; + virtual void recordError(const SpanError & error = SpanError()) = 0; + virtual void setSpanStatus(bool spanFailed, const char * statusMessage = NO_STATUS_MESSAGE) = 0; virtual ISpan * createClientSpan(const char * name) = 0; virtual ISpan * createInternalSpan(const char * name) = 0; diff --git a/testing/unittests/jlibtests.cpp b/testing/unittests/jlibtests.cpp index 26b5674a8d2..6b96f023b38 100644 --- a/testing/unittests/jlibtests.cpp +++ b/testing/unittests/jlibtests.cpp @@ -64,6 +64,7 @@ class JlibTraceTest : public CppUnit::TestFixture CPPUNIT_TEST(testClientSpanGlobalID); CPPUNIT_TEST(testEnsureTraceID); CPPUNIT_TEST(manualTestsEventsOutput); + CPPUNIT_TEST(manualTestsDeclaredFailures); //CPPUNIT_TEST(testJTraceJLOGExporterprintResources); //CPPUNIT_TEST(testJTraceJLOGExporterprintAttributes); @@ -269,6 +270,65 @@ class JlibTraceTest : public CppUnit::TestFixture } } + //not able to programmatically test yet, but can visually inspect trace output + void manualTestsDeclaredFailures() + { + Owned emptyMockHTTPHeaders = createProperties(); + { + Owned serverSpan = queryTraceManager().createServerSpan("defaultErrorSpan", emptyMockHTTPHeaders); + serverSpan->recordError(); + }//{ "type": "span", "name": "defaultErrorSpan", "trace_id": "209b5d8cea0aec9785d2dfa3117e37ad", "span_id": "ab72e76c2f2466c2", "start": 1709675278129335702, "duration": 188292867932, "status": "Error", "kind": "Server", "instrumented_library": "unittests", "events":[ { "name": "Exception", "time_stamp": 1709675465508149013, "attributes": {"escaped": 0 } } ] } + + { + Owned serverSpan = queryTraceManager().createServerSpan("defaultErrorSpanStruct", emptyMockHTTPHeaders); + SpanError error; + serverSpan->recordError(error); + }//{ "type": "span", "name": "defaultErrorSpanStruct", "trace_id": "19803a446b971f2e0bdddc9c00db50fe", "span_id": "04c93a91ab8785a2", "start": 1709675487767044352, "duration": 2287497219, "status": "Error", "kind": "Server", "instrumented_library": "unittests", "events":[ { "name": "Exception", "time_stamp": 1709675489216412154, "attributes": {"escaped": 0 } } ] } + + { + Owned serverSpan = queryTraceManager().createServerSpan("failedErrorSpanEscaped", emptyMockHTTPHeaders); + SpanError * error = new SpanError("hello"); + error->setSpanStatus(true, true); + serverSpan->recordError(*error); + }//{ "type": "span", "name": "failedErrorSpanEscaped", "trace_id": "634f386c18a6140544c980e0d5a15905", "span_id": "e2f59c48f63a8f82", "start": 1709675508231168974, "duration": 7731717678, "status": "Error", "kind": "Server", "description": "hello", "instrumented_library": "unittests", "events":[ { "name": "Exception", "time_stamp": 1709675512164430668, "attributes": {"escaped": 1,"message": "hello" } } ] } + + { + Owned serverSpan = queryTraceManager().createServerSpan("failedErrEscapedMsgErrCode", emptyMockHTTPHeaders); + SpanError * error = new SpanError(); + error->setSpanStatus(true, true); + error->setError("hello", 34); + serverSpan->recordError(*error); + }//failedErrEscapedMsgErrCode + + { + Owned serverSpan = queryTraceManager().createServerSpan("containsErrorAndMessageSpan", emptyMockHTTPHeaders); + serverSpan->recordError(SpanError("Error Message!!")); + }//{ "type": "span", "name": "containsErrorAndMessageSpan", "trace_id": "9a6e00ea309bc0427733f9b2d452f9e2", "span_id": "de63e9c69b64e411", "start": 1709675552302360510, "duration": 5233037523, "status": "Error", "kind": "Server", "description": "Error Message!!", "instrumented_library": "unittests", "events":[ { "name": "Exception", "time_stamp": 1709675555149852711, "attributes": {"escaped": 0,"message": "Error Message!!" } } + + { + Owned serverSpan = queryTraceManager().createServerSpan("containsErrorAndMessageFailedNotEscapedSpan", emptyMockHTTPHeaders); + serverSpan->recordError(SpanError("Error Message!!", 23, true, false)); + }//{ "type": "span", "name": "containsErrorAndMessageFailedNotEscapedSpan", "trace_id": "02f4b2d215f8230b15063862f8a91e41", "span_id": "c665ec371d6db147", "start": 1709675573581678954, "duration": 3467489486, "status": "Error", "kind": "Server", "description": "Error Message!!", "instrumented_library": "unittests", "events":[ { "name": "Exception", "time_stamp": 1709675576145074240, "attributes": {"code": 23,"escaped": 0,"message": "Error Message!!" } } ] } + + { + Owned serverSpan = queryTraceManager().createServerSpan("mockExceptionSpanNotFailedNotEscaped", emptyMockHTTPHeaders); + serverSpan->recordException( makeStringExceptionV(76,"Mock exception"), false, false); + }//{ "type": "span", "name": "mockExceptionSpanNotFailedNotEscaped", "trace_id": "e01766474db05ce9085943fa3955cd73", "span_id": "7da620e96e10e42c", "start": 1709675595987480704, "duration": 2609091267, "status": "Unset", "kind": "Server", "instrumented_library": "unittests", "events":[ { "name": "Exception", "time_stamp": 1709675597728975355, "attributes": {"code": 76,"escaped": 0,"message": "Mock exception" } } ] + + { + Owned serverSpan = queryTraceManager().createServerSpan("thrownExceptionSpan", emptyMockHTTPHeaders); + try + { + throw makeStringExceptionV( 356, "Mock thrown exception"); + } + catch (IException *e) + { + serverSpan->recordException(e, false, true); + e->Release(); + } + }//{ "type": "span", "name": "thrownExceptionSpan", "trace_id": "4d6225e1cefdc6823d1134c71c522426", "span_id": "07f7bd070e008f53", "start": 1709675614823881961, "duration": 4665288686, "status": "Unset", "kind": "Server", "instrumented_library": "unittests", "events":[ { "name": "Exception", "time_stamp": 1709675616485586471, "attributes": {"code": 356,"escaped": 1,"message": "Mock thrown exception" } } ] } + } + void testTraceDisableConfig() { Owned testTree = createPTreeFromYAMLString(disableTracingYaml, ipt_none, ptr_ignoreWhiteSpace, nullptr); From 04b5ba8bfc4252af929268cad983437aa348429b Mon Sep 17 00:00:00 2001 From: Gavin Halliday Date: Wed, 6 Mar 2024 20:11:01 +0000 Subject: [PATCH 16/17] HPCC-31396 Introduce a class for tracking the lifetime of a span Signed-off-by: Gavin Halliday --- ecl/eclagent/eclagent.cpp | 4 +- esp/bindings/http/platform/httpservice.cpp | 9 ++-- esp/platform/esp.hpp | 4 +- esp/platform/espcontext.cpp | 18 ++++---- esp/services/esdl_svc_engine/esdl_binding.cpp | 3 +- esp/services/ws_ecl/ws_ecl_service.cpp | 3 +- .../ws_workunits/ws_workunitsHelpers.cpp | 2 +- roxie/ccd/ccdlistener.cpp | 6 +-- system/jlib/jtrace.cpp | 13 ++++-- system/jlib/jtrace.hpp | 13 ++++++ testing/unittests/jlibtests.cpp | 44 +++++++++---------- thorlcr/graph/thgraphmaster.cpp | 4 +- thorlcr/graph/thgraphslave.cpp | 4 +- 13 files changed, 70 insertions(+), 57 deletions(-) diff --git a/ecl/eclagent/eclagent.cpp b/ecl/eclagent/eclagent.cpp index a3a56824808..a49bfcf9ced 100644 --- a/ecl/eclagent/eclagent.cpp +++ b/ecl/eclagent/eclagent.cpp @@ -2131,9 +2131,9 @@ void EclAgent::runProcess(IEclProcess *process) allocatorMetaCache.setown(createRowAllocatorCache(this)); Owned traceHeaders = extractTraceDebugOptions(queryWorkUnit()); - Owned requestSpan = queryTraceManager().createServerSpan("run_workunit", traceHeaders); - requestSpan->setSpanAttribute("hpcc.wuid", queryWorkUnit()->queryWuid()); + OwnedSpanScope requestSpan = queryTraceManager().createServerSpan("run_workunit", traceHeaders); ContextSpanScope spanScope(updateDummyContextLogger(), requestSpan); + requestSpan->setSpanAttribute("hpcc.wuid", queryWorkUnit()->queryWuid()); // a component may specify an alternate name for the agent/workflow memory area, // e.g. Thor specifies in "eclAgentMemory" diff --git a/esp/bindings/http/platform/httpservice.cpp b/esp/bindings/http/platform/httpservice.cpp index 748155df69e..c5d82667d21 100644 --- a/esp/bindings/http/platform/httpservice.cpp +++ b/esp/bindings/http/platform/httpservice.cpp @@ -412,13 +412,10 @@ int CEspHttpServer::processRequest() return 0; } - //The context will be destroyed when this request is destroyed. So rather than using - //EspContextSpanScope spanContext(*ctx, serverSpan); - //which would remove the activeSpan when this function exits, use - //setActiveSpan() - //It is possible that using EspContextSpanScope would be better + //The context will be destroyed when this request is destroyed. So initialise a SpanScope in the context to + //ensure the span is also terminated at the same time. Owned serverSpan = m_request->createServerSpan(serviceName, methodName); - ctx->setActiveSpan(serverSpan); + ctx->setRequestSpan(serverSpan); if (thebinding!=NULL) { diff --git a/esp/platform/esp.hpp b/esp/platform/esp.hpp index 75dd142b14b..65c3008f412 100644 --- a/esp/platform/esp.hpp +++ b/esp/platform/esp.hpp @@ -235,15 +235,13 @@ interface IEspContext : extends IInterface virtual void setRequest(IHttpMessage* req) = 0; virtual IHttpMessage* queryRequest() = 0; - virtual void setActiveSpan(ISpan * span)=0; // Only call this function directly if this object's lifetime matches the lifetime of the span. If there is any doubt use EspContextSpanScope(ctx, span) + virtual void setRequestSpan(ISpan * span)=0; // Call this function to set the server span for the query. The spans's lifetime will match the lifetime of the context object. virtual ISpan * queryActiveSpan() const = 0; virtual IProperties * getClientSpanHeaders() const = 0; virtual const char* getGlobalId() const = 0; virtual const char* getCallerId() const = 0; virtual const char* getLocalId() const = 0; }; -//Class for managing the active span in the context -using EspContextSpanScope = ContextSpanScopeImp; typedef unsigned LogLevel; diff --git a/esp/platform/espcontext.cpp b/esp/platform/espcontext.cpp index a2bca7b701f..3270273fe5b 100755 --- a/esp/platform/espcontext.cpp +++ b/esp/platform/espcontext.cpp @@ -89,7 +89,7 @@ class CEspContext : public CInterface, implements IEspContext Owned m_secureContext; StringAttr m_transactionID; - Owned m_activeSpan; + OwnedSpanScope m_requestSpan; // When the context is destroy the span will end. IHttpMessage* m_request; public: @@ -114,7 +114,7 @@ class CEspContext : public CInterface, implements IEspContext updateTraceSummaryHeader(); m_secureContext.setown(secureContext); m_SecurityHandler.setSecureContext(secureContext); - m_activeSpan.set(getNullSpan()); + m_requestSpan.setown(getNullSpan()); } ~CEspContext() @@ -626,30 +626,30 @@ class CEspContext : public CInterface, implements IEspContext { return m_request; } - virtual void setActiveSpan(ISpan * span) override + virtual void setRequestSpan(ISpan * span) override { - m_activeSpan.set(span); + m_requestSpan.set(span); } virtual ISpan * queryActiveSpan() const override { - return m_activeSpan; + return m_requestSpan; } virtual const char* getGlobalId() const override { - return m_activeSpan->queryGlobalId(); + return m_requestSpan->queryGlobalId(); } virtual const char* getCallerId() const override { - return m_activeSpan->queryCallerId(); + return m_requestSpan->queryCallerId(); } virtual const char* getLocalId() const override { - return m_activeSpan->queryLocalId(); + return m_requestSpan->queryLocalId(); } virtual IProperties * getClientSpanHeaders() const override { - return ::getClientHeaders(m_activeSpan); + return ::getClientHeaders(m_requestSpan); } }; diff --git a/esp/services/esdl_svc_engine/esdl_binding.cpp b/esp/services/esdl_svc_engine/esdl_binding.cpp index 5c63e363737..54783bfafb4 100755 --- a/esp/services/esdl_svc_engine/esdl_binding.cpp +++ b/esp/services/esdl_svc_engine/esdl_binding.cpp @@ -1571,9 +1571,8 @@ void EsdlServiceImpl::sendTargetSOAP(IEspContext & context, httpclient->setPassword(password.str()); } - Owned clientSpan; ISpan * activeSpan = context.queryActiveSpan(); - clientSpan.setown(activeSpan->createClientSpan("soapcall")); + OwnedSpanScope clientSpan(activeSpan->createClientSpan("soapcall")); Owned headers = ::getClientHeaders(clientSpan); StringBuffer status; diff --git a/esp/services/ws_ecl/ws_ecl_service.cpp b/esp/services/ws_ecl/ws_ecl_service.cpp index c049e458a34..77f0bd8363e 100644 --- a/esp/services/ws_ecl/ws_ecl_service.cpp +++ b/esp/services/ws_ecl/ws_ecl_service.cpp @@ -1999,9 +1999,8 @@ int CWsEclBinding::submitWsEclWorkunit(IEspContext & context, WsEclWuInfo &wsinf bool noTimeout = false; - Owned clientSpan; ISpan * activeSpan = context.queryActiveSpan(); - clientSpan.setown(activeSpan->createClientSpan("run_workunit")); + OwnedSpanScope clientSpan(activeSpan->createClientSpan("run_workunit")); Owned httpHeaders = ::getClientHeaders(clientSpan); recordTraceDebugOptions(workunit, httpHeaders); diff --git a/esp/services/ws_workunits/ws_workunitsHelpers.cpp b/esp/services/ws_workunits/ws_workunitsHelpers.cpp index bb6fd445832..8598752f888 100644 --- a/esp/services/ws_workunits/ws_workunitsHelpers.cpp +++ b/esp/services/ws_workunits/ws_workunitsHelpers.cpp @@ -3779,7 +3779,7 @@ void WsWuHelpers::submitWsWorkunit(IEspContext& context, IConstWorkUnit* cw, con } ISpan * activeSpan = context.queryActiveSpan(); - Owned clientSpan(activeSpan->createClientSpan("run_workunit")); + OwnedSpanScope clientSpan(activeSpan->createClientSpan("run_workunit")); Owned httpHeaders = ::getClientHeaders(clientSpan); recordTraceDebugOptions(wu, httpHeaders); diff --git a/roxie/ccd/ccdlistener.cpp b/roxie/ccd/ccdlistener.cpp index 2615434ab2e..370a83bcdc7 100644 --- a/roxie/ccd/ccdlistener.cpp +++ b/roxie/ccd/ccdlistener.cpp @@ -1200,7 +1200,7 @@ class RoxieWorkUnitWorker : public RoxieQueryWorker Owned logctx = new StringContextLogger(wuid.get()); Owned traceHeaders = extractTraceDebugOptions(wu); - Owned requestSpan = queryTraceManager().createServerSpan("run_workunit", traceHeaders); + OwnedSpanScope requestSpan = queryTraceManager().createServerSpan("run_workunit", traceHeaders); requestSpan->setSpanAttribute("hpcc.wuid", wuid); ContextSpanScope spanScope(*logctx, requestSpan); @@ -1381,6 +1381,7 @@ class RoxieProtocolMsgContext : implements IHpccProtocolMsgContext, public CInte Owned debugCmdHandler; Owned logctx; Owned queryFactory; + OwnedSpanScope requestSpan; SocketEndpoint ep; time_t startTime; @@ -1484,8 +1485,7 @@ class RoxieProtocolMsgContext : implements IHpccProtocolMsgContext, public CInte if (spanName.length()) spanName.append('/'); spanName.append(spanQueryName); - Owned requestSpan = queryTraceManager().createServerSpan(spanName, allHeaders, flags); - //The span has a lifetime the same length as the logctx, so no need to restore it at the end of the query + requestSpan.setown(queryTraceManager().createServerSpan(spanName, allHeaders, flags)); logctx->setActiveSpan(requestSpan); const char * globalId = requestSpan->queryGlobalId(); diff --git a/system/jlib/jtrace.cpp b/system/jlib/jtrace.cpp index 1c1ba40dfb6..1d619407d4f 100644 --- a/system/jlib/jtrace.cpp +++ b/system/jlib/jtrace.cpp @@ -544,8 +544,7 @@ class CSpan : public CInterfaceOf virtual void beforeDispose() override { //Record the span as complete before we output the logging for the end of the span - if (span != nullptr) - span->End(); + endSpan(); } const char * getSpanID() const @@ -556,6 +555,13 @@ class CSpan : public CInterfaceOf ISpan * createClientSpan(const char * name) override; ISpan * createInternalSpan(const char * name) override; + virtual void endSpan() final override + { + //It is legal to call endSpan multiple times, but only the first call will have any effect + if (span != nullptr) + span->End(); + } + virtual void toString(StringBuffer & out) const { toString(out, true); @@ -868,7 +874,7 @@ class CSpan : public CInterfaceOf }; -class CNullSpan : public CInterfaceOf +class CNullSpan final : public CInterfaceOf { public: CNullSpan() = default; @@ -878,6 +884,7 @@ class CNullSpan : public CInterfaceOf virtual void setSpanAttributes(const IProperties * attributes) override {} virtual void addSpanEvent(const char * eventName) override {} virtual void addSpanEvent(const char * eventName, IProperties * attributes) override {}; + virtual void endSpan() override {} virtual void getSpanContext(IProperties * ctxProps) const override {} virtual void getClientHeaders(IProperties * clientHeaders) const override {} diff --git a/system/jlib/jtrace.hpp b/system/jlib/jtrace.hpp index 6689dff6597..31a957e289c 100644 --- a/system/jlib/jtrace.hpp +++ b/system/jlib/jtrace.hpp @@ -115,6 +115,7 @@ interface ISpan : extends IInterface virtual void setSpanAttributes(const IProperties * attributes) = 0; virtual void addSpanEvent(const char * eventName) = 0; virtual void addSpanEvent(const char * eventName, IProperties * attributes) = 0; + virtual void endSpan() = 0; // Indicate that the span has ended even if it has not yet been destroyed virtual void getSpanContext(IProperties * ctxProps) const = 0; virtual void getClientHeaders(IProperties * clientHeaders) const = 0; virtual void toString(StringBuffer & out) const = 0; @@ -133,6 +134,18 @@ interface ISpan : extends IInterface virtual const char* queryLocalId() const = 0; }; +class OwnedSpanScope : public Owned +{ +public: + inline OwnedSpanScope() { } + inline OwnedSpanScope(ISpan * _ptr) : Owned(_ptr) { } + ~OwnedSpanScope() + { + if (get()) + get()->endSpan(); + } +}; + extern jlib_decl IProperties * getClientHeaders(const ISpan * span); extern jlib_decl IProperties * getSpanContext(const ISpan * span); diff --git a/testing/unittests/jlibtests.cpp b/testing/unittests/jlibtests.cpp index 6b96f023b38..238e646a51e 100644 --- a/testing/unittests/jlibtests.cpp +++ b/testing/unittests/jlibtests.cpp @@ -226,7 +226,7 @@ class JlibTraceTest : public CppUnit::TestFixture { //duration should be at least 125 milliseconds - Owned serverSpan = queryTraceManager().createServerSpan("declaredSpanStartTime", emptyMockHTTPHeaders, &declaredSpanStartTime); + OwnedSpanScope serverSpan = queryTraceManager().createServerSpan("declaredSpanStartTime", emptyMockHTTPHeaders, &declaredSpanStartTime); //{ "type": "span", "name": "declaredSpanStartTime", "trace_id": "0a2eff24e1996540056745aaeb2f5824", "span_id": "46d0faf8b4da893e", //"start": 1702672311203213259, "duration": 125311051 } } @@ -244,7 +244,7 @@ class JlibTraceTest : public CppUnit::TestFixture SpanTimeStamp nowTimeStamp; //not used, printed out as "start" time for manual comparison nowTimeStamp.now(); { - Owned serverSpan = queryTraceManager().createServerSpan("msTickOffsetStartTime", emptyMockHTTPHeaders, &msTickOffsetTimeStamp); + OwnedSpanScope serverSpan = queryTraceManager().createServerSpan("msTickOffsetStartTime", emptyMockHTTPHeaders, &msTickOffsetTimeStamp); } DBGLOG("MsTickOffset span actual start-time timestamp: %lld", (long long)(nowTimeStamp.systemClockTime).count()); @@ -260,7 +260,7 @@ class JlibTraceTest : public CppUnit::TestFixture CPPUNIT_ASSERT_EQUAL_MESSAGE("Unexpected initialized spanTimeStamp", true, uninitializedTS.systemClockTime == std::chrono::nanoseconds::zero()); CPPUNIT_ASSERT_EQUAL_MESSAGE("Unexpected initialized spanTimeStamp", true, uninitializedTS.steadyClockTime == std::chrono::nanoseconds::zero()); { - Owned serverSpan = queryTraceManager().createServerSpan("uninitializeddeclaredSpanStartTime", emptyMockHTTPHeaders, &uninitializedTS); + OwnedSpanScope serverSpan = queryTraceManager().createServerSpan("uninitializeddeclaredSpanStartTime", emptyMockHTTPHeaders, &uninitializedTS); //sleep for 75 milliseconds after span creation, expect at least 75 milliseconds duration output MilliSleep(75); @@ -342,7 +342,7 @@ class JlibTraceTest : public CppUnit::TestFixture { SpanFlags flags = SpanFlags::EnsureTraceId; Owned emptyMockHTTPHeaders = createProperties(); - Owned serverSpan = queryTraceManager().createServerSpan("noRemoteParentEnsureTraceID", emptyMockHTTPHeaders, flags); + OwnedSpanScope serverSpan = queryTraceManager().createServerSpan("noRemoteParentEnsureTraceID", emptyMockHTTPHeaders, flags); Owned retrievedSpanCtxAttributes = createProperties(); serverSpan->getSpanContext(retrievedSpanCtxAttributes.get()); @@ -356,7 +356,7 @@ class JlibTraceTest : public CppUnit::TestFixture Owned mockHTTPHeaders = createProperties(); createMockHTTPHeaders(mockHTTPHeaders, true); - Owned serverSpan = queryTraceManager().createServerSpan("propegatedServerSpan", mockHTTPHeaders); + OwnedSpanScope serverSpan = queryTraceManager().createServerSpan("propegatedServerSpan", mockHTTPHeaders); //at this point the serverSpan should have the following context attributes //traceID, spanID, remoteParentSpanID, traceFlags, traceState, globalID, callerID @@ -408,7 +408,7 @@ class JlibTraceTest : public CppUnit::TestFixture return; } - Owned nullSpan = getNullSpan(); + OwnedSpanScope nullSpan = getNullSpan(); CPPUNIT_ASSERT_EQUAL_MESSAGE("Unexpected nullptr nullspan detected", true, nullSpan != nullptr); { @@ -416,14 +416,14 @@ class JlibTraceTest : public CppUnit::TestFixture nullSpan->getClientHeaders(headers); } - Owned nullSpanChild = nullSpan->createClientSpan("nullSpanChild"); + OwnedSpanScope nullSpanChild = nullSpan->createClientSpan("nullSpanChild"); CPPUNIT_ASSERT_EQUAL_MESSAGE("Unexpected nullptr nullSpanChild detected", true, nullSpanChild != nullptr); } void testClientSpan() { Owned emptyMockHTTPHeaders = createProperties(); - Owned serverSpan = queryTraceManager().createServerSpan("propegatedServerSpan", emptyMockHTTPHeaders); + OwnedSpanScope serverSpan = queryTraceManager().createServerSpan("propegatedServerSpan", emptyMockHTTPHeaders); Owned retrievedSpanCtxAttributes = createProperties(); serverSpan->getSpanContext(retrievedSpanCtxAttributes); @@ -434,7 +434,7 @@ class JlibTraceTest : public CppUnit::TestFixture const char * serverTraceID = retrievedSpanCtxAttributes->queryProp("traceID"); { - Owned internalSpan = serverSpan->createClientSpan("clientSpan"); + OwnedSpanScope internalSpan = serverSpan->createClientSpan("clientSpan"); //retrieve clientSpan context with the intent to propogate otel and HPCC context { @@ -470,7 +470,7 @@ class JlibTraceTest : public CppUnit::TestFixture void testInternalSpan() { Owned emptyMockHTTPHeaders = createProperties(); - Owned serverSpan = queryTraceManager().createServerSpan("propegatedServerSpan", emptyMockHTTPHeaders); + OwnedSpanScope serverSpan = queryTraceManager().createServerSpan("propegatedServerSpan", emptyMockHTTPHeaders); Owned retrievedSpanCtxAttributes = createProperties(); serverSpan->getSpanContext(retrievedSpanCtxAttributes); @@ -481,7 +481,7 @@ class JlibTraceTest : public CppUnit::TestFixture const char * serverTraceID = retrievedSpanCtxAttributes->queryProp("traceID"); { - Owned internalSpan = serverSpan->createInternalSpan("internalSpan"); + OwnedSpanScope internalSpan = serverSpan->createInternalSpan("internalSpan"); //retrieve internalSpan context with the intent to interrogate attributes { @@ -515,7 +515,7 @@ class JlibTraceTest : public CppUnit::TestFixture void testRootServerSpan() { Owned emptyMockHTTPHeaders = createProperties(); - Owned serverSpan = queryTraceManager().createServerSpan("propegatedServerSpan", emptyMockHTTPHeaders); + OwnedSpanScope serverSpan = queryTraceManager().createServerSpan("propegatedServerSpan", emptyMockHTTPHeaders); //retrieve serverSpan context with the intent to propagate it to a remote child span { @@ -553,7 +553,7 @@ class JlibTraceTest : public CppUnit::TestFixture { Owned mockHTTPHeaders = createProperties(); createMockHTTPHeaders(mockHTTPHeaders, false); - Owned serverSpan = queryTraceManager().createServerSpan("invalidPropegatedServerSpan", mockHTTPHeaders); + OwnedSpanScope serverSpan = queryTraceManager().createServerSpan("invalidPropegatedServerSpan", mockHTTPHeaders); Owned retrievedSpanCtxAttributes = createProperties(); serverSpan->getClientHeaders(retrievedSpanCtxAttributes.get()); @@ -574,7 +574,7 @@ class JlibTraceTest : public CppUnit::TestFixture Owned mockHTTPHeaders = createProperties(); createMockHTTPHeaders(mockHTTPHeaders, true); - Owned serverSpan = queryTraceManager().createServerSpan("propegatedServerSpan", mockHTTPHeaders); + OwnedSpanScope serverSpan = queryTraceManager().createServerSpan("propegatedServerSpan", mockHTTPHeaders); //at this point the serverSpan should have the following context attributes //remoteParentSpanID, globalID, callerID @@ -595,10 +595,10 @@ class JlibTraceTest : public CppUnit::TestFixture Owned mockHTTPHeaders = createProperties(); createMockHTTPHeaders(mockHTTPHeaders, true); - Owned serverSpan = queryTraceManager().createServerSpan("propegatedServerSpan", mockHTTPHeaders); - Owned clientSpan = serverSpan->createClientSpan("clientSpan"); - Owned internalSpan = clientSpan->createInternalSpan("internalSpan"); - Owned internalSpan2 = internalSpan->createInternalSpan("internalSpan2"); + OwnedSpanScope serverSpan = queryTraceManager().createServerSpan("propegatedServerSpan", mockHTTPHeaders); + OwnedSpanScope clientSpan = serverSpan->createClientSpan("clientSpan"); + OwnedSpanScope internalSpan = clientSpan->createInternalSpan("internalSpan"); + OwnedSpanScope internalSpan2 = internalSpan->createInternalSpan("internalSpan2"); StringBuffer out; out.set("{"); @@ -633,8 +633,8 @@ class JlibTraceTest : public CppUnit::TestFixture Owned mockHTTPHeaders = createProperties(); createMockHTTPHeaders(mockHTTPHeaders, true); //includes global ID - Owned serverSpan = queryTraceManager().createServerSpan("propegatedServerSpan", mockHTTPHeaders); - Owned clientSpan = serverSpan->createClientSpan("clientSpanWithGlobalID"); + OwnedSpanScope serverSpan = queryTraceManager().createServerSpan("propegatedServerSpan", mockHTTPHeaders); + OwnedSpanScope clientSpan = serverSpan->createClientSpan("clientSpanWithGlobalID"); //retrieve serverSpan context with the intent to interrogate attributes { @@ -656,7 +656,7 @@ class JlibTraceTest : public CppUnit::TestFixture Owned mockHTTPHeaders = createProperties(); createMockHTTPHeaders(mockHTTPHeaders, true); - Owned serverSpan = queryTraceManager().createServerSpan("propegatedServerSpan", mockHTTPHeaders); + OwnedSpanScope serverSpan = queryTraceManager().createServerSpan("propegatedServerSpan", mockHTTPHeaders); //at this point the serverSpan should have the following context attributes //traceID, spanID, remoteParentSpanID, traceFlags, traceState, globalID, callerID @@ -702,7 +702,7 @@ class JlibTraceTest : public CppUnit::TestFixture mockHTTPHeadersSA.append("HPCC-Global-Id:someGlobalID"); mockHTTPHeadersSA.append("HPCC-Caller-Id:IncomingCID"); - Owned serverSpan = queryTraceManager().createServerSpan("StringArrayPropegatedServerSpan", mockHTTPHeadersSA); + OwnedSpanScope serverSpan = queryTraceManager().createServerSpan("StringArrayPropegatedServerSpan", mockHTTPHeadersSA); //at this point the serverSpan should have the following context attributes //traceID, spanID, remoteParentSpanID, traceFlags, traceState, globalID, callerID diff --git a/thorlcr/graph/thgraphmaster.cpp b/thorlcr/graph/thgraphmaster.cpp index 7496b4e516b..7571ff59acf 100644 --- a/thorlcr/graph/thgraphmaster.cpp +++ b/thorlcr/graph/thgraphmaster.cpp @@ -1502,10 +1502,10 @@ CJobMaster::CJobMaster(IConstWorkUnit &_workunit, const char *graphName, ILoaded init(); Owned traceHeaders = extractTraceDebugOptions(workunit); - Owned requestSpan = queryTraceManager().createServerSpan("run_graph", traceHeaders); + OwnedSpanScope requestSpan = queryTraceManager().createServerSpan("run_graph", traceHeaders); + ContextSpanScope spanScope(*logctx, requestSpan); requestSpan->setSpanAttribute("hpcc.wuid", workunit->queryWuid()); requestSpan->setSpanAttribute("hpcc.graph", graphName); - ContextSpanScope spanScope(*logctx, requestSpan); resumed = WUActionResume == workunit->getAction(); fatalHandler.setown(new CFatalHandler(globals->getPropInt("@fatal_timeout", FATAL_TIMEOUT))); diff --git a/thorlcr/graph/thgraphslave.cpp b/thorlcr/graph/thgraphslave.cpp index c510d59960b..4c339769d5f 100644 --- a/thorlcr/graph/thgraphslave.cpp +++ b/thorlcr/graph/thgraphslave.cpp @@ -1680,10 +1680,10 @@ CJobSlave::CJobSlave(ISlaveWatchdog *_watchdog, IPropertyTree *_workUnitInfo, co init(); Owned traceHeaders = deserializeTraceDebugOptions(workUnitInfo->queryPropTree("Debug")); - Owned requestSpan = queryTraceManager().createServerSpan("run_graph", traceHeaders); + OwnedSpanScope requestSpan = queryTraceManager().createServerSpan("run_graph", traceHeaders); + ContextSpanScope spanScope(*logctx, requestSpan); requestSpan->setSpanAttribute("hpcc.wuid", wuid); requestSpan->setSpanAttribute("hpcc.graph", graphName); - ContextSpanScope spanScope(*logctx, requestSpan); oldNodeCacheMem = 0; slavemptag = _slavemptag; From 71ae37b2583e8803b1f0eaa92eb702f5448caee9 Mon Sep 17 00:00:00 2001 From: Shamser Ahmed Date: Tue, 5 Mar 2024 17:30:09 +0000 Subject: [PATCH 17/17] HPCC-31393 Subfile queried incorrectly For superfiles, use mapSubPart to obtain the subfile number. Signed-off-by: Shamser Ahmed --- dali/ft/filecopy.cpp | 49 +++++++++++++++++++++++++++++++++----------- 1 file changed, 37 insertions(+), 12 deletions(-) diff --git a/dali/ft/filecopy.cpp b/dali/ft/filecopy.cpp index d93ade88777..c92f42fbd86 100644 --- a/dali/ft/filecopy.cpp +++ b/dali/ft/filecopy.cpp @@ -3573,17 +3573,30 @@ void FileSprayer::updateTargetProperties() } // Update @writeCost and @numWrites in subfile properties and update totalWriteCost - if (superTgt) + if (superTgt && superTgt->numSubFiles() > 0) { if (cur.whichOutput != (unsigned)-1) { - unsigned targetPartNum = targets.item(cur.whichOutput).partNum; - IDistributedFile &subfile = superTgt->querySubFile(targetPartNum, true); - DistributedFilePropertyLock lock(&subfile); + IDistributedFile *subfile; + if (superTgt->numSubFiles() > 1) + { + Owned fDesc = superTgt->getFileDescriptor(); + ISuperFileDescriptor *superFDesc = fDesc->querySuperFileDescriptor(); + unsigned subfileNum, subFilePartNum; + superFDesc->mapSubPart(targets.item(cur.whichOutput).partNum, subfileNum, subFilePartNum); + subfile = superTgt->querySubPart(subfileNum, subFilePartNum); + } + else + { + // If there is a single subfile, it is not necessary to map part to subfile + // (also, querySuperFileDescriptor return nullptr if num subfile == 1) + subfile = &superTgt->querySubFile(0); + } + DistributedFilePropertyLock lock(subfile); IPropertyTree &subFileProps = lock.queryAttributes(); cost_type prevNumWrites = subFileProps.getPropInt64(getDFUQResultFieldName(DFUQRFnumDiskWrites)); cost_type prevWriteCost = subFileProps.getPropInt64(getDFUQResultFieldName(DFUQRFwriteCost)); - cost_type curWriteCost = calcFileAccessCost(&subfile, curProgress.numWrites, 0); + cost_type curWriteCost = calcFileAccessCost(subfile, curProgress.numWrites, 0); subFileProps.setPropInt64(getDFUQResultFieldName(DFUQRFwriteCost), prevWriteCost + curWriteCost); subFileProps.setPropInt64(getDFUQResultFieldName(DFUQRFnumDiskWrites), prevNumWrites + curProgress.numWrites); totalWriteCost += curWriteCost; @@ -3786,23 +3799,35 @@ void FileSprayer::updateTargetProperties() if (distributedSource) { IDistributedSuperFile * superSrc = distributedSource->querySuperFile(); - if (superSrc) + if (superSrc && superSrc->numSubFiles() > 0) { + Owned fDesc = superSrc->getFileDescriptor(); + ISuperFileDescriptor *superFDesc = fDesc->querySuperFileDescriptor(); ForEachItemIn(idx, partition) { PartitionPoint & cur = partition.item(idx); OutputProgress & curProgress = progress.item(idx); - if (cur.whichInput != (unsigned)-1) { - unsigned sourcePartNum = sources.item(cur.whichInput).partNum; - IDistributedFile &subfile = superSrc->querySubFile(sourcePartNum, true); - DistributedFilePropertyLock lock(&subfile); + IDistributedFile *subfile; + if (superFDesc) + { + unsigned subfileNum, subFilePartNum; + superFDesc->mapSubPart(sources.item(cur.whichInput).partNum, subfileNum, subFilePartNum); + subfile = superSrc->querySubPart(subfileNum, subFilePartNum); + } + else + { + // superFDesc==nullptr if there is a single file + // so query the first (and only) subfile + subfile = &superSrc->querySubFile(0); + } + DistributedFilePropertyLock lock(subfile); IPropertyTree &subFileProps = lock.queryAttributes(); stat_type prevNumReads = subFileProps.getPropInt64(getDFUQResultFieldName(DFUQRFnumDiskReads), 0); - cost_type legacyReadCost = getLegacyReadCost(subfile.queryAttributes(), &subfile); + cost_type legacyReadCost = getLegacyReadCost(subfile->queryAttributes(), subfile); cost_type prevReadCost = subFileProps.getPropInt64(getDFUQResultFieldName(DFUQRFreadCost), 0); - cost_type curReadCost = calcFileAccessCost(&subfile, 0, curProgress.numReads); + cost_type curReadCost = calcFileAccessCost(subfile, 0, curProgress.numReads); subFileProps.setPropInt64(getDFUQResultFieldName(DFUQRFnumDiskReads), prevNumReads + curProgress.numReads); subFileProps.setPropInt64(getDFUQResultFieldName(DFUQRFreadCost), legacyReadCost + prevReadCost + curReadCost); totalReadCost += curReadCost;