From c12219f62fa9845a34ab204af152329a79434b36 Mon Sep 17 00:00:00 2001 From: Anthony Fishbeck Date: Thu, 29 Jun 2023 13:35:42 -0400 Subject: [PATCH 1/8] HPCC-29855 Add the storage secret category to eclccserver Normally eclccserver doesn't need storage secrets, but sometimes the user 'instructs the compiler to fetch the source file's metadata for its record layout' in which case eclccserver may need to access remote (dfs) storage. Signed-off-by: Anthony Fishbeck --- helm/hpcc/templates/eclccserver.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/helm/hpcc/templates/eclccserver.yaml b/helm/hpcc/templates/eclccserver.yaml index cd4627f233b..9ea242b96fd 100644 --- a/helm/hpcc/templates/eclccserver.yaml +++ b/helm/hpcc/templates/eclccserver.yaml @@ -123,7 +123,7 @@ data: {{- if not .disabled -}} {{- $gitenv := get (fromYaml (include "hpcc.gatherGitEnvironment" (dict "root" $ "me" .))) "value" -}} {{- $env := concat ($.Values.global.env | default list) (.env | default list) $gitenv -}} -{{- $secretsCategories := list "system" "codeVerify" "git" }} +{{- $secretsCategories := list "system" "codeVerify" "git" "storage" }} {{- $commonCtx := dict "root" $ "me" . "includeCategories" (list "dll" "git" "debug") "secretsCategories" $secretsCategories "env" $env }} {{- $configSHA := include "hpcc.getConfigSHA" ($commonCtx | merge (dict "configMapHelper" "hpcc.eclccServerConfigMap" "component" "eclccserver" "excludeKeys" "global,eclccserver.queues")) }} apiVersion: apps/v1 From 75cc125edc80f17521ac2c75c9a603dff8d4884b Mon Sep 17 00:00:00 2001 From: Jake Smith Date: Fri, 11 Aug 2023 10:28:50 +0100 Subject: [PATCH 2/8] Split off 8.12.44 Signed-off-by: Jake Smith --- helm/hpcc/Chart.yaml | 4 ++-- helm/hpcc/templates/_helpers.tpl | 2 +- helm/hpcc/templates/dafilesrv.yaml | 2 +- helm/hpcc/templates/dali.yaml | 2 +- helm/hpcc/templates/dfuserver.yaml | 2 +- helm/hpcc/templates/eclagent.yaml | 4 ++-- helm/hpcc/templates/eclccserver.yaml | 4 ++-- helm/hpcc/templates/eclscheduler.yaml | 2 +- helm/hpcc/templates/esp.yaml | 2 +- helm/hpcc/templates/localroxie.yaml | 2 +- helm/hpcc/templates/roxie.yaml | 8 ++++---- helm/hpcc/templates/sasha.yaml | 2 +- helm/hpcc/templates/thor.yaml | 10 +++++----- version.cmake | 2 +- 14 files changed, 24 insertions(+), 24 deletions(-) diff --git a/helm/hpcc/Chart.yaml b/helm/hpcc/Chart.yaml index 57a338751da..2f6e2d50836 100644 --- a/helm/hpcc/Chart.yaml +++ b/helm/hpcc/Chart.yaml @@ -6,9 +6,9 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. -version: 8.12.43-closedown0 +version: 8.12.45-closedown0 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. -appVersion: 8.12.43-closedown0 +appVersion: 8.12.45-closedown0 diff --git a/helm/hpcc/templates/_helpers.tpl b/helm/hpcc/templates/_helpers.tpl index d56d09489b6..47c5b8d8502 100644 --- a/helm/hpcc/templates/_helpers.tpl +++ b/helm/hpcc/templates/_helpers.tpl @@ -1225,7 +1225,7 @@ kind: Service metadata: name: {{ $lvars.serviceName | quote }} labels: - helmVersion: 8.12.43-closedown0 + helmVersion: 8.12.45-closedown0 {{- include "hpcc.addStandardLabels" (dict "root" $.root "instance" $lvars.serviceName ) | indent 4 }} {{- if $lvars.labels }} {{ toYaml $lvars.labels | indent 4 }} diff --git a/helm/hpcc/templates/dafilesrv.yaml b/helm/hpcc/templates/dafilesrv.yaml index 86bde51460a..8328ad2afb8 100644 --- a/helm/hpcc/templates/dafilesrv.yaml +++ b/helm/hpcc/templates/dafilesrv.yaml @@ -50,7 +50,7 @@ spec: labels: {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "dafilesrv" "name" "dafilesrv" "instance" .name) | indent 8 }} server: {{ .name | quote }} - helmVersion: 8.12.43-closedown0 + helmVersion: 8.12.45-closedown0 annotations: checksum/config: {{ $configSHA }} spec: diff --git a/helm/hpcc/templates/dali.yaml b/helm/hpcc/templates/dali.yaml index 19fd535738d..aede118b26f 100644 --- a/helm/hpcc/templates/dali.yaml +++ b/helm/hpcc/templates/dali.yaml @@ -82,7 +82,7 @@ spec: run: {{ $dali.name | quote }} server: {{ $dali.name | quote }} app: dali - helmVersion: 8.12.43-closedown0 + helmVersion: 8.12.45-closedown0 {{- if hasKey $.Values.global "metrics" }} {{- include "hpcc.generateMetricsReporterLabel" $.Values.global.metrics | nindent 8 }} {{- end }} diff --git a/helm/hpcc/templates/dfuserver.yaml b/helm/hpcc/templates/dfuserver.yaml index be04a299741..2c2fdb3433c 100644 --- a/helm/hpcc/templates/dfuserver.yaml +++ b/helm/hpcc/templates/dfuserver.yaml @@ -56,7 +56,7 @@ spec: {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "dfuserver" "name" "dfuserver" "instance" .name) | indent 8 }} run: {{ .name | quote }} accessDali: "yes" - helmVersion: 8.12.43-closedown0 + helmVersion: 8.12.45-closedown0 {{- if hasKey . "labels" }} {{ toYaml .labels | indent 8 }} {{- end }} diff --git a/helm/hpcc/templates/eclagent.yaml b/helm/hpcc/templates/eclagent.yaml index dde133ec16a..0ddd21d4030 100644 --- a/helm/hpcc/templates/eclagent.yaml +++ b/helm/hpcc/templates/eclagent.yaml @@ -58,7 +58,7 @@ data: {{- include "hpcc.addStandardLabels" (dict "root" $ "component" $apptype "name" "eclagent" "instance" $appJobName "instanceOf" (printf "%s-job" .me.name)) | indent 12 }} accessDali: "yes" accessEsp: "yes" - helmVersion: 8.12.43-closedown0 + helmVersion: 8.12.45-closedown0 {{- if hasKey .me "labels" }} {{ toYaml .me.labels | indent 12 }} {{- end }} @@ -137,7 +137,7 @@ spec: run: {{ .name | quote }} accessDali: "yes" accessEsp: {{ .useChildProcesses | default false | ternary "yes" "no" | quote }} - helmVersion: 8.12.43-closedown0 + helmVersion: 8.12.45-closedown0 {{- if hasKey . "labels" }} {{ toYaml .labels | indent 8 }} {{- end }} diff --git a/helm/hpcc/templates/eclccserver.yaml b/helm/hpcc/templates/eclccserver.yaml index 83dccabf8c9..d6b53ceb3d1 100644 --- a/helm/hpcc/templates/eclccserver.yaml +++ b/helm/hpcc/templates/eclccserver.yaml @@ -57,7 +57,7 @@ data: {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "eclccserver" "name" "eclccserver" "instance" $compileJobName "instanceOf" (printf "%s-job" .me.name)) | indent 12 }} accessDali: "yes" accessEsp: "yes" - helmVersion: 8.12.43-closedown0 + helmVersion: 8.12.45-closedown0 {{- if hasKey .me "labels" }} {{ toYaml .me.labels | indent 12 }} {{- end }} @@ -142,7 +142,7 @@ spec: run: {{ .name | quote }} accessDali: "yes" accessEsp: {{ .useChildProcesses | default false | ternary "yes" "no" | quote }} - helmVersion: 8.12.43-closedown0 + helmVersion: 8.12.45-closedown0 {{- if hasKey . "labels" }} {{ toYaml .labels | indent 8 }} {{- end }} diff --git a/helm/hpcc/templates/eclscheduler.yaml b/helm/hpcc/templates/eclscheduler.yaml index 97f1bde3637..2d0137cd13d 100644 --- a/helm/hpcc/templates/eclscheduler.yaml +++ b/helm/hpcc/templates/eclscheduler.yaml @@ -64,7 +64,7 @@ spec: run: {{ .name | quote }} accessDali: "yes" accessEsp: "no" - helmVersion: 8.12.43-closedown0 + helmVersion: 8.12.45-closedown0 {{- if hasKey . "labels" }} {{ toYaml .labels | indent 8 }} {{- end }} diff --git a/helm/hpcc/templates/esp.yaml b/helm/hpcc/templates/esp.yaml index c0ed3e75611..d2db9482e9f 100644 --- a/helm/hpcc/templates/esp.yaml +++ b/helm/hpcc/templates/esp.yaml @@ -117,7 +117,7 @@ spec: server: {{ .name | quote }} accessDali: "yes" app: {{ $application }} - helmVersion: 8.12.43-closedown0 + helmVersion: 8.12.45-closedown0 {{- include "hpcc.addStandardLabels" (dict "root" $ "name" $application "component" "esp" "instance" .name) | indent 8 }} {{- if hasKey $.Values.global "metrics" }} {{- include "hpcc.generateMetricsReporterLabel" $.Values.global.metrics | nindent 8 }} diff --git a/helm/hpcc/templates/localroxie.yaml b/helm/hpcc/templates/localroxie.yaml index 57a6f0ace1d..5921c3e6acc 100644 --- a/helm/hpcc/templates/localroxie.yaml +++ b/helm/hpcc/templates/localroxie.yaml @@ -70,7 +70,7 @@ spec: server: {{ $servername | quote }} accessDali: "yes" accessEsp: "yes" - helmVersion: 8.12.43-closedown0 + helmVersion: 8.12.45-closedown0 {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "roxie-server" "name" "roxie" "instance" $roxie.name) | indent 8 }} {{- if hasKey . "labels" }} {{ toYaml .labels | indent 8 }} diff --git a/helm/hpcc/templates/roxie.yaml b/helm/hpcc/templates/roxie.yaml index 8b371e05ecb..834617bc552 100644 --- a/helm/hpcc/templates/roxie.yaml +++ b/helm/hpcc/templates/roxie.yaml @@ -120,7 +120,7 @@ spec: {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "topology-server" "name" "roxie" "instance" $commonCtx.toponame) | indent 8 }} run: {{ $commonCtx.toponame | quote }} roxie-cluster: {{ $roxie.name | quote }} - helmVersion: 8.12.43-closedown0 + helmVersion: 8.12.45-closedown0 {{- if hasKey $.Values.global "metrics" }} {{- include "hpcc.generateMetricsReporterLabel" $.Values.global.metrics | nindent 8}} {{- end }} @@ -180,7 +180,7 @@ kind: Service metadata: name: {{ $commonCtx.toponame | quote }} labels: - helmVersion: 8.12.43-closedown0 + helmVersion: 8.12.45-closedown0 {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "topology-server" "name" "roxie" "instance" $commonCtx.toponame) | indent 4 }} spec: ports: @@ -242,7 +242,7 @@ spec: roxie-cluster: {{ $roxie.name | quote }} accessDali: "yes" accessEsp: "yes" - helmVersion: 8.12.43-closedown0 + helmVersion: 8.12.45-closedown0 {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "roxie-server" "name" "roxie" "instance" $servername) | indent 8 }} {{- if hasKey $.Values.global "metrics" }} {{- include "hpcc.generateMetricsReporterLabel" $.Values.global.metrics | nindent 8}} @@ -347,7 +347,7 @@ spec: roxie-cluster: {{ $roxie.name | quote }} accessDali: "yes" accessEsp: "yes" - helmVersion: 8.12.43-closedown0 + helmVersion: 8.12.45-closedown0 {{- if hasKey $.Values.global "metrics" }} {{- include "hpcc.generateMetricsReporterLabel" $.Values.global.metrics | nindent 8}} {{- end }} diff --git a/helm/hpcc/templates/sasha.yaml b/helm/hpcc/templates/sasha.yaml index 4113a88f429..fe3a13f9865 100644 --- a/helm/hpcc/templates/sasha.yaml +++ b/helm/hpcc/templates/sasha.yaml @@ -52,7 +52,7 @@ spec: run: {{ $serviceName | quote }} server: {{ $serviceName | quote }} accessDali: {{ (has "dali" $sasha.access) | ternary "yes" "no" | quote }} - helmVersion: 8.12.43-closedown0 + helmVersion: 8.12.45-closedown0 {{- if hasKey $sasha "labels" }} {{ toYaml $sasha.labels | indent 8 }} {{- end }} diff --git a/helm/hpcc/templates/thor.yaml b/helm/hpcc/templates/thor.yaml index 8240b8d6a4f..887c483a326 100644 --- a/helm/hpcc/templates/thor.yaml +++ b/helm/hpcc/templates/thor.yaml @@ -82,7 +82,7 @@ data: labels: accessDali: "yes" accessEsp: "yes" - helmVersion: 8.12.43-closedown0 + helmVersion: 8.12.45-closedown0 {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "eclagent" "name" "thor" "instance" $eclAgentJobName "instanceOf" (printf "%s-job" .eclAgentName)) | indent 8 }} {{- if hasKey .me "labels" }} {{ toYaml .me.labels | indent 12 }} @@ -149,7 +149,7 @@ data: accessEsp: "yes" app: "thor" component: "thormanager" - helmVersion: 8.12.43-closedown0 + helmVersion: 8.12.45-closedown0 instance: "_HPCC_JOBNAME_" job: "_HPCC_JOBNAME_" {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "thormanager" "name" "thor" "instance" $thorManagerJobName "instanceOf" (printf "%s-thormanager-job" .me.name)) | indent 12 }} @@ -218,7 +218,7 @@ data: accessEsp: "yes" app: "thor" component: "thorworker" - helmVersion: 8.12.43-closedown0 + helmVersion: 8.12.45-closedown0 instance: "_HPCC_JOBNAME_" job: "_HPCC_JOBNAME_" {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "thorworker" "name" "thor" "instance" $thorWorkerJobName "instanceOf" (printf "%s-thorworker-job" .me.name)) | indent 12 }} @@ -353,7 +353,7 @@ spec: accessEsp: {{ $commonCtx.eclAgentUseChildProcesses | ternary "yes" "no" | quote }} app: "thor" component: "thor-eclagent" - helmVersion: 8.12.43-closedown0 + helmVersion: 8.12.45-closedown0 instance: {{ $commonCtx.eclAgentName | quote }} {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "eclagent" "name" "thor" "instance" $commonCtx.eclAgentName ) | indent 8 }} {{- if hasKey $commonCtx.me "labels" }} @@ -418,7 +418,7 @@ spec: accessEsp: "no" app: "thor" component: "thor-thoragent" - helmVersion: 8.12.43-closedown0 + helmVersion: 8.12.45-closedown0 instance: {{ $commonCtx.thorAgentName | quote }} {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "eclagent" "name" "thor" "instance" $commonCtx.thorAgentName ) | indent 8 }} {{- if hasKey $commonCtx.me "labels" }} diff --git a/version.cmake b/version.cmake index e6da983e169..534e8311f07 100644 --- a/version.cmake +++ b/version.cmake @@ -5,7 +5,7 @@ set ( HPCC_NAME "Community Edition" ) set ( HPCC_PROJECT "community" ) set ( HPCC_MAJOR 8 ) set ( HPCC_MINOR 12 ) -set ( HPCC_POINT 43 ) +set ( HPCC_POINT 45 ) set ( HPCC_MATURITY "closedown" ) set ( HPCC_SEQUENCE 0 ) ### From a5278bdd668e78c92b8330c82d5b0c54e91a6af3 Mon Sep 17 00:00:00 2001 From: Jake Smith Date: Fri, 11 Aug 2023 10:32:59 +0100 Subject: [PATCH 3/8] Split off 9.0.34 Signed-off-by: Jake Smith --- helm/hpcc/Chart.yaml | 4 ++-- helm/hpcc/templates/_helpers.tpl | 2 +- helm/hpcc/templates/dafilesrv.yaml | 2 +- helm/hpcc/templates/dali.yaml | 2 +- helm/hpcc/templates/dfuserver.yaml | 2 +- helm/hpcc/templates/eclagent.yaml | 4 ++-- helm/hpcc/templates/eclccserver.yaml | 4 ++-- helm/hpcc/templates/eclscheduler.yaml | 2 +- helm/hpcc/templates/esp.yaml | 2 +- helm/hpcc/templates/localroxie.yaml | 2 +- helm/hpcc/templates/roxie.yaml | 8 ++++---- helm/hpcc/templates/sasha.yaml | 2 +- helm/hpcc/templates/thor.yaml | 10 +++++----- version.cmake | 2 +- 14 files changed, 24 insertions(+), 24 deletions(-) diff --git a/helm/hpcc/Chart.yaml b/helm/hpcc/Chart.yaml index 7946b81023c..ad82957d86f 100644 --- a/helm/hpcc/Chart.yaml +++ b/helm/hpcc/Chart.yaml @@ -6,9 +6,9 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. -version: 9.0.33-closedown0 +version: 9.0.35-closedown0 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. -appVersion: 9.0.33-closedown0 +appVersion: 9.0.35-closedown0 diff --git a/helm/hpcc/templates/_helpers.tpl b/helm/hpcc/templates/_helpers.tpl index 4298299ce50..d3682d7268f 100644 --- a/helm/hpcc/templates/_helpers.tpl +++ b/helm/hpcc/templates/_helpers.tpl @@ -1299,7 +1299,7 @@ kind: Service metadata: name: {{ $lvars.serviceName | quote }} labels: - helmVersion: 9.0.33-closedown0 + helmVersion: 9.0.35-closedown0 {{- include "hpcc.addStandardLabels" (dict "root" $.root "instance" $lvars.serviceName ) | indent 4 }} {{- if $lvars.labels }} {{ toYaml $lvars.labels | indent 4 }} diff --git a/helm/hpcc/templates/dafilesrv.yaml b/helm/hpcc/templates/dafilesrv.yaml index 28c6f487789..723705598b8 100644 --- a/helm/hpcc/templates/dafilesrv.yaml +++ b/helm/hpcc/templates/dafilesrv.yaml @@ -50,7 +50,7 @@ spec: labels: {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "dafilesrv" "name" "dafilesrv" "instance" .name) | indent 8 }} server: {{ .name | quote }} - helmVersion: 9.0.33-closedown0 + helmVersion: 9.0.35-closedown0 annotations: checksum/config: {{ $configSHA }} spec: diff --git a/helm/hpcc/templates/dali.yaml b/helm/hpcc/templates/dali.yaml index de39c898b72..9f5a56d434c 100644 --- a/helm/hpcc/templates/dali.yaml +++ b/helm/hpcc/templates/dali.yaml @@ -82,7 +82,7 @@ spec: run: {{ $dali.name | quote }} server: {{ $dali.name | quote }} app: dali - helmVersion: 9.0.33-closedown0 + helmVersion: 9.0.35-closedown0 {{- if hasKey $.Values.global "metrics" }} {{- include "hpcc.generateMetricsReporterLabel" $.Values.global.metrics | nindent 8 }} {{- end }} diff --git a/helm/hpcc/templates/dfuserver.yaml b/helm/hpcc/templates/dfuserver.yaml index 44d64b55f6e..380383f21a3 100644 --- a/helm/hpcc/templates/dfuserver.yaml +++ b/helm/hpcc/templates/dfuserver.yaml @@ -56,7 +56,7 @@ spec: {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "dfuserver" "name" "dfuserver" "instance" .name) | indent 8 }} run: {{ .name | quote }} accessDali: "yes" - helmVersion: 9.0.33-closedown0 + helmVersion: 9.0.35-closedown0 {{- if hasKey . "labels" }} {{ toYaml .labels | indent 8 }} {{- end }} diff --git a/helm/hpcc/templates/eclagent.yaml b/helm/hpcc/templates/eclagent.yaml index d93ef473fe7..28506e9131e 100644 --- a/helm/hpcc/templates/eclagent.yaml +++ b/helm/hpcc/templates/eclagent.yaml @@ -58,7 +58,7 @@ data: {{- include "hpcc.addStandardLabels" (dict "root" $ "component" $apptype "name" "eclagent" "instance" $appJobName "instanceOf" (printf "%s-job" .me.name)) | indent 12 }} accessDali: "yes" accessEsp: "yes" - helmVersion: 9.0.33-closedown0 + helmVersion: 9.0.35-closedown0 {{- if hasKey .me "labels" }} {{ toYaml .me.labels | indent 12 }} {{- end }} @@ -135,7 +135,7 @@ spec: run: {{ .name | quote }} accessDali: "yes" accessEsp: {{ .useChildProcesses | default false | ternary "yes" "no" | quote }} - helmVersion: 9.0.33-closedown0 + helmVersion: 9.0.35-closedown0 {{- if hasKey . "labels" }} {{ toYaml .labels | indent 8 }} {{- end }} diff --git a/helm/hpcc/templates/eclccserver.yaml b/helm/hpcc/templates/eclccserver.yaml index 8bf72c44e94..d3d4abb0489 100644 --- a/helm/hpcc/templates/eclccserver.yaml +++ b/helm/hpcc/templates/eclccserver.yaml @@ -57,7 +57,7 @@ data: {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "eclccserver" "name" "eclccserver" "instance" $compileJobName "instanceOf" (printf "%s-job" .me.name)) | indent 12 }} accessDali: "yes" accessEsp: "yes" - helmVersion: 9.0.33-closedown0 + helmVersion: 9.0.35-closedown0 {{- if hasKey .me "labels" }} {{ toYaml .me.labels | indent 12 }} {{- end }} @@ -142,7 +142,7 @@ spec: run: {{ .name | quote }} accessDali: "yes" accessEsp: {{ .useChildProcesses | default false | ternary "yes" "no" | quote }} - helmVersion: 9.0.33-closedown0 + helmVersion: 9.0.35-closedown0 {{- if hasKey . "labels" }} {{ toYaml .labels | indent 8 }} {{- end }} diff --git a/helm/hpcc/templates/eclscheduler.yaml b/helm/hpcc/templates/eclscheduler.yaml index 0d5280792c8..b0733008f66 100644 --- a/helm/hpcc/templates/eclscheduler.yaml +++ b/helm/hpcc/templates/eclscheduler.yaml @@ -64,7 +64,7 @@ spec: run: {{ .name | quote }} accessDali: "yes" accessEsp: "no" - helmVersion: 9.0.33-closedown0 + helmVersion: 9.0.35-closedown0 {{- if hasKey . "labels" }} {{ toYaml .labels | indent 8 }} {{- end }} diff --git a/helm/hpcc/templates/esp.yaml b/helm/hpcc/templates/esp.yaml index 8adc1ff95e5..66f3e934a80 100644 --- a/helm/hpcc/templates/esp.yaml +++ b/helm/hpcc/templates/esp.yaml @@ -117,7 +117,7 @@ spec: server: {{ .name | quote }} accessDali: "yes" app: {{ $application }} - helmVersion: 9.0.33-closedown0 + helmVersion: 9.0.35-closedown0 {{- include "hpcc.addStandardLabels" (dict "root" $ "name" $application "component" "esp" "instance" .name) | indent 8 }} {{- if hasKey $.Values.global "metrics" }} {{- include "hpcc.generateMetricsReporterLabel" $.Values.global.metrics | nindent 8 }} diff --git a/helm/hpcc/templates/localroxie.yaml b/helm/hpcc/templates/localroxie.yaml index 70b33827755..0b602322931 100644 --- a/helm/hpcc/templates/localroxie.yaml +++ b/helm/hpcc/templates/localroxie.yaml @@ -70,7 +70,7 @@ spec: server: {{ $servername | quote }} accessDali: "yes" accessEsp: "yes" - helmVersion: 9.0.33-closedown0 + helmVersion: 9.0.35-closedown0 {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "roxie-server" "name" "roxie" "instance" $roxie.name) | indent 8 }} {{- if hasKey . "labels" }} {{ toYaml .labels | indent 8 }} diff --git a/helm/hpcc/templates/roxie.yaml b/helm/hpcc/templates/roxie.yaml index a79a4252282..fdb4da51bbe 100644 --- a/helm/hpcc/templates/roxie.yaml +++ b/helm/hpcc/templates/roxie.yaml @@ -120,7 +120,7 @@ spec: {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "topology-server" "name" "roxie" "instance" $commonCtx.toponame) | indent 8 }} run: {{ $commonCtx.toponame | quote }} roxie-cluster: {{ $roxie.name | quote }} - helmVersion: 9.0.33-closedown0 + helmVersion: 9.0.35-closedown0 {{- if hasKey $.Values.global "metrics" }} {{- include "hpcc.generateMetricsReporterLabel" $.Values.global.metrics | nindent 8}} {{- end }} @@ -180,7 +180,7 @@ kind: Service metadata: name: {{ $commonCtx.toponame | quote }} labels: - helmVersion: 9.0.33-closedown0 + helmVersion: 9.0.35-closedown0 {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "topology-server" "name" "roxie" "instance" $commonCtx.toponame) | indent 4 }} spec: ports: @@ -242,7 +242,7 @@ spec: roxie-cluster: {{ $roxie.name | quote }} accessDali: "yes" accessEsp: "yes" - helmVersion: 9.0.33-closedown0 + helmVersion: 9.0.35-closedown0 {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "roxie-server" "name" "roxie" "instance" $servername) | indent 8 }} {{- if hasKey $.Values.global "metrics" }} {{- include "hpcc.generateMetricsReporterLabel" $.Values.global.metrics | nindent 8}} @@ -347,7 +347,7 @@ spec: roxie-cluster: {{ $roxie.name | quote }} accessDali: "yes" accessEsp: "yes" - helmVersion: 9.0.33-closedown0 + helmVersion: 9.0.35-closedown0 {{- if hasKey $.Values.global "metrics" }} {{- include "hpcc.generateMetricsReporterLabel" $.Values.global.metrics | nindent 8}} {{- end }} diff --git a/helm/hpcc/templates/sasha.yaml b/helm/hpcc/templates/sasha.yaml index 914430575c8..ebf4ae541b7 100644 --- a/helm/hpcc/templates/sasha.yaml +++ b/helm/hpcc/templates/sasha.yaml @@ -52,7 +52,7 @@ spec: run: {{ $serviceName | quote }} server: {{ $serviceName | quote }} accessDali: {{ (has "dali" $sasha.access) | ternary "yes" "no" | quote }} - helmVersion: 9.0.33-closedown0 + helmVersion: 9.0.35-closedown0 {{- if hasKey $sasha "labels" }} {{ toYaml $sasha.labels | indent 8 }} {{- end }} diff --git a/helm/hpcc/templates/thor.yaml b/helm/hpcc/templates/thor.yaml index 6ba8cdbd30e..7d56d4c34fb 100644 --- a/helm/hpcc/templates/thor.yaml +++ b/helm/hpcc/templates/thor.yaml @@ -82,7 +82,7 @@ data: labels: accessDali: "yes" accessEsp: "yes" - helmVersion: 9.0.33-closedown0 + helmVersion: 9.0.35-closedown0 {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "eclagent" "name" "thor" "instance" $eclAgentJobName "instanceOf" (printf "%s-job" .eclAgentName)) | indent 8 }} {{- if hasKey .me "labels" }} {{ toYaml .me.labels | indent 12 }} @@ -147,7 +147,7 @@ data: accessEsp: "yes" app: "thor" component: "thormanager" - helmVersion: 9.0.33-closedown0 + helmVersion: 9.0.35-closedown0 instance: "_HPCC_JOBNAME_" job: "_HPCC_JOBNAME_" {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "thormanager" "name" "thor" "instance" $thorManagerJobName "instanceOf" (printf "%s-thormanager-job" .me.name)) | indent 12 }} @@ -214,7 +214,7 @@ data: accessEsp: "yes" app: "thor" component: "thorworker" - helmVersion: 9.0.33-closedown0 + helmVersion: 9.0.35-closedown0 instance: "_HPCC_JOBNAME_" job: "_HPCC_JOBNAME_" {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "thorworker" "name" "thor" "instance" $thorWorkerJobName "instanceOf" (printf "%s-thorworker-job" .me.name)) | indent 12 }} @@ -347,7 +347,7 @@ spec: accessEsp: {{ $commonCtx.eclAgentUseChildProcesses | ternary "yes" "no" | quote }} app: "thor" component: "thor-eclagent" - helmVersion: 9.0.33-closedown0 + helmVersion: 9.0.35-closedown0 instance: {{ $commonCtx.eclAgentName | quote }} {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "eclagent" "name" "thor" "instance" $commonCtx.eclAgentName ) | indent 8 }} {{- if hasKey $commonCtx.me "labels" }} @@ -412,7 +412,7 @@ spec: accessEsp: "no" app: "thor" component: "thor-thoragent" - helmVersion: 9.0.33-closedown0 + helmVersion: 9.0.35-closedown0 instance: {{ $commonCtx.thorAgentName | quote }} {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "eclagent" "name" "thor" "instance" $commonCtx.thorAgentName ) | indent 8 }} {{- if hasKey $commonCtx.me "labels" }} diff --git a/version.cmake b/version.cmake index 1555862be95..d06c6532d49 100644 --- a/version.cmake +++ b/version.cmake @@ -5,7 +5,7 @@ set ( HPCC_NAME "Community Edition" ) set ( HPCC_PROJECT "community" ) set ( HPCC_MAJOR 9 ) set ( HPCC_MINOR 0 ) -set ( HPCC_POINT 33 ) +set ( HPCC_POINT 35 ) set ( HPCC_MATURITY "closedown" ) set ( HPCC_SEQUENCE 0 ) ### From 2b372f765af905f375eaec3ca4044d8ef39c7e55 Mon Sep 17 00:00:00 2001 From: Russ Whitehead Date: Fri, 4 Aug 2023 16:51:40 -0400 Subject: [PATCH 4/8] HPCC-30028 LDAP connection pool improvement when host down Make connection requests fail over to next AD faster when current one goes down. Immediately blacklist current AD when LDAP_SERVER_DOWN returned. Check for current AD being blacklisted before attempting retry Signed-off-by: Russ Whitehead --- .../security/LdapSecurity/ldapconnection.cpp | 70 +++++++------------ 1 file changed, 26 insertions(+), 44 deletions(-) diff --git a/system/security/LdapSecurity/ldapconnection.cpp b/system/security/LdapSecurity/ldapconnection.cpp index 4a69b732090..36f5c428fbf 100644 --- a/system/security/LdapSecurity/ldapconnection.cpp +++ b/system/security/LdapSecurity/ldapconnection.cpp @@ -453,7 +453,6 @@ class CLdapConfig : implements ILdapConfig, public CInterface const char * ldapDomain = cfg->queryProp(".//@ldapDomain"); for (int numHosts=0; numHosts < getHostCount(); numHosts++) { - getLdapHost(hostbuf); unsigned port = strieq("ldaps",m_protocol) ? m_ldap_secure_port : m_ldapport; //Guesstimate system user baseDN based on config settings. It will be used if anonymous bind fails @@ -465,16 +464,14 @@ class CLdapConfig : implements ILdapConfig, public CInterface for(int retries = 0; retries <= LDAPSEC_MAX_RETRIES; retries++) { + getLdapHost(hostbuf);//get next available AD, as it may have changed rc = LdapUtils::getServerInfo(hostbuf.str(), sysUserDN.str(), m_sysuser_password.str(), m_protocol, port, m_cipherSuite, dcbuf, m_serverType, ldapDomain, m_timeout); - if(!LdapServerDown(rc) || retries >= LDAPSEC_MAX_RETRIES) + if(rc != LDAP_TIMEOUT || retries >= LDAPSEC_MAX_RETRIES) break; sleep(LDAPSEC_RETRY_WAIT); - if(retries < LDAPSEC_MAX_RETRIES) - { - DBGLOG("LDAP AD Server %s temporarily unreachable for user %s on port %d, retrying...", hostbuf.str(), sysUserDN.str(), port); - } + DBGLOG("LDAP AD Server %s temporarily unreachable for user %s on port %d, retrying...", hostbuf.str(), sysUserDN.str(), port); } - if (rc != LDAP_SUCCESS) + if(LdapServerDown(rc)) { rejectHost(hostbuf); } @@ -986,41 +983,21 @@ class CLdapConnection : implements ILdapConnection, public CInterface StringBuffer hostbuf; for (int numHosts=0; numHosts < m_ldapconfig->getHostCount(); numHosts++) { - m_ldapconfig->getLdapHost(hostbuf); - for(int retries = 0; retries <= LDAPSEC_MAX_RETRIES; retries++) { + m_ldapconfig->getLdapHost(hostbuf);//get next available AD, as it may have changed rc = connect(hostbuf.str(), proto); - if(!LdapServerDown(rc) || retries > LDAPSEC_MAX_RETRIES) + if(rc == LDAP_SUCCESS) + return true; + if(rc != LDAP_TIMEOUT || retries >= LDAPSEC_MAX_RETRIES) break; sleep(LDAPSEC_RETRY_WAIT); - if(retries < LDAPSEC_MAX_RETRIES) - DBGLOG("Server temporarily unreachable, retrying ..."); - } - - if(rc == LDAP_SERVER_DOWN) - { - StringBuffer dc; - LdapUtils::getDcName(m_ldapconfig->getDomain(), dc); - if(dc.length() > 0) - { - WARNLOG("Using automatically obtained LDAP Server %s", dc.str()); - rc = connect(dc.str(), proto); - } - } - - if (rc != LDAP_SUCCESS) - { - m_ldapconfig->rejectHost(hostbuf); + DBGLOG("Server %s temporarily unreachable, retrying ...", hostbuf.str()); } - else - break; + m_ldapconfig->rejectHost(hostbuf); } - if(rc == LDAP_SUCCESS) - return true; - else - return false; + return false; } virtual LDAP* getLd() @@ -1953,13 +1930,12 @@ class CLdapClient : implements ILdapClient, public CInterface ldap_memfree(userdn); StringBuffer hostbuf; - m_ldapconfig->getLdapHost(hostbuf); int rc = LDAP_SERVER_DOWN; char *ldap_errstring=NULL; - for(int retries = 0; retries <= LDAPSEC_MAX_RETRIES; retries++) { - DBGLOG("LdapBind for user %s (retries=%d).", username, retries); + m_ldapconfig->getLdapHost(hostbuf);//get next available AD, as it may have changed + DBGLOG("LdapBind for user %s (retries=%d) on host %s.", username, retries, hostbuf.str()); { LDAP* user_ld = LdapUtils::LdapInit(m_ldapconfig->getProtocol(), hostbuf.str(), m_ldapconfig->getLdapPort(), m_ldapconfig->getLdapSecurePort(), m_ldapconfig->getCipherSuite()); rc = LdapUtils::LdapBind(user_ld, m_ldapconfig->getLdapTimeout(), m_ldapconfig->getDomain(), username, password, userdnbuf.str(), m_ldapconfig->getServerType(), m_ldapconfig->getAuthMethod()); @@ -1968,16 +1944,22 @@ class CLdapClient : implements ILdapClient, public CInterface LDAP_UNBIND(user_ld); } DBGLOG("finished LdapBind for user %s, rc=%d", username, rc); - if(!LdapServerDown(rc) || retries > LDAPSEC_MAX_RETRIES) + + if(rc==LDAP_SERVER_DOWN || rc==LDAP_UNAVAILABLE) + { + m_ldapconfig->rejectHost(hostbuf); + continue;//try again with next configured LDAP host + } + else if(rc==LDAP_TIMEOUT && retries < LDAPSEC_MAX_RETRIES) + { + sleep(LDAPSEC_RETRY_WAIT); + DBGLOG("Server %s temporarily unreachable, retrying ...", hostbuf.str()); + } + else break; - sleep(LDAPSEC_RETRY_WAIT); - if(retries < LDAPSEC_MAX_RETRIES) - DBGLOG("Server temporarily unreachable, retrying ..."); - // Retrying next ldap sever, might be the same server - m_ldapconfig->getLdapHost(hostbuf); } - if(rc == LDAP_SERVER_DOWN) + if(LdapServerDown(rc)) { StringBuffer dc; LdapUtils::getDcName(NULL, dc); From e56d0673959cd5f697f7c5ad228902e87807f606 Mon Sep 17 00:00:00 2001 From: Anthony Fishbeck Date: Fri, 28 Jul 2023 17:11:11 -0400 Subject: [PATCH 5/8] HPCC-29969 Add connection retry logic to vault access code Signed-off-by: Anthony Fishbeck --- helm/hpcc/templates/_helpers.tpl | 15 +++++ helm/hpcc/values.schema.json | 20 +++++++ system/jlib/jsecrets.cpp | 99 +++++++++++++++++++++++++------- 3 files changed, 114 insertions(+), 20 deletions(-) diff --git a/helm/hpcc/templates/_helpers.tpl b/helm/hpcc/templates/_helpers.tpl index d56d09489b6..e0a39aeabb2 100644 --- a/helm/hpcc/templates/_helpers.tpl +++ b/helm/hpcc/templates/_helpers.tpl @@ -604,6 +604,21 @@ vaults: {{- if index $vault "appRoleSecret" }} appRoleSecret: {{ index $vault "appRoleSecret" }} {{- end -}} + {{- if (hasKey $vault "retries") }} + retries: {{ $vault.retries }} + {{- end }} + {{- if (hasKey $vault "retryWait") }} + retryWait: {{ $vault.retryWait }} + {{- end }} + {{- if (hasKey $vault "connectTimeout") }} + connectTimeout: {{ $vault.connectTimeout }} + {{- end }} + {{- if (hasKey $vault "readTimeout") }} + readTimeout: {{ $vault.readTimeout }} + {{- end }} + {{- if (hasKey $vault "writeTimeout") }} + writeTimeout: {{ $vault.writeTimeout }} + {{- end }} {{- end -}} {{- end -}} {{- end -}} diff --git a/helm/hpcc/values.schema.json b/helm/hpcc/values.schema.json index a4d46f0e572..5495a88127a 100644 --- a/helm/hpcc/values.schema.json +++ b/helm/hpcc/values.schema.json @@ -834,6 +834,26 @@ "verify_server": { "description": "optional relax server verification for trouble shooting", "type": "boolean" + }, + "retries": { + "description": "optional number of times to retry vault request in case of connect / socker error", + "type": "number" + }, + "retryWait": { + "description": "optional wait time (in ms) between socket retries", + "type": "number" + }, + "connectTimeout": { + "description": "optional timeout (in ms) for socket connect to vault", + "type": "number" + }, + "readTimeout": { + "description": "optional timeout (in ms) for socket reading from vault", + "type": "number" + }, + "writeTimeout": { + "description": "optional timeout (in ms) for socket writing to vault", + "type": "number" } }, "required": [ "name", "url" ], diff --git a/system/jlib/jsecrets.cpp b/system/jlib/jsecrets.cpp index 766ef4d8b76..13d697120fe 100644 --- a/system/jlib/jsecrets.cpp +++ b/system/jlib/jsecrets.cpp @@ -282,6 +282,22 @@ static inline bool checkSecretExpired(unsigned created) enum class VaultAuthType {unknown, k8s, appRole, token}; +static void setTimevalMS(timeval &tv, time_t ms) +{ + if (!ms) + tv = {0, 0}; + else + { + tv.tv_sec = ms / 1000; + tv.tv_usec = (ms % 1000)*1000; + } +} + +static bool isEmptyTimeval(const timeval &tv) +{ + return (tv.tv_sec==0 && tv.tv_usec==0); +} + class CVault { private: @@ -306,6 +322,11 @@ class CVault time_t clientTokenExpiration = 0; bool clientTokenRenewable = false; bool verify_server = true; + unsigned retries = 3; + time_t retryWait = 1000; + timeval connectTimeout = {0, 0}; + timeval readTimeout = {0, 0}; + timeval writeTimeout = {0, 0}; public: CVault(IPropertyTree *vault) @@ -330,6 +351,13 @@ class CVault PROGLOG("vault: namespace %s", vaultNamespace.str()); } verify_server = vault->getPropBool("@verify_server", true); + retries = (unsigned) vault->getPropInt("@retries", 3); + retryWait = (time_t) vault->getPropInt("@retryWait", 1000); + + setTimevalMS(connectTimeout, (time_t) vault->getPropInt("@connectTimeout")); + setTimevalMS(readTimeout, (time_t) vault->getPropInt("@readTimeout")); + setTimevalMS(writeTimeout, (time_t) vault->getPropInt("@writeTimeout")); + PROGLOG("Vault: httplib verify_server=%s", boolToStr(verify_server)); //set up vault client auth [appRole, clientToken (aka "token from the sky"), or kubernetes auth] @@ -396,7 +424,7 @@ class CVault void processClientTokenResponse(httplib::Result &res) { if (!res) - vaultAuthErrorV("missing login response, error %d", res.error()); + vaultAuthErrorV("login communication error %d", res.error()); if (res.error()!=0) OERRLOG("JSECRETS login calling HTTPLIB POST returned error %d", res.error()); if (res->status != 200) @@ -438,6 +466,22 @@ class CVault CVaultKind getVaultKind() const { return kind; } + void initClient(httplib::Client &cli, httplib::Headers &headers, unsigned &numRetries) + { + numRetries = retries; + cli.enable_server_certificate_verification(verify_server); + if (!isEmptyTimeval(connectTimeout)) + cli.set_connection_timeout(connectTimeout.tv_sec, connectTimeout.tv_usec); + if (!isEmptyTimeval(readTimeout)) + cli.set_read_timeout(readTimeout.tv_sec, readTimeout.tv_usec); + if (!isEmptyTimeval(writeTimeout)) + cli.set_write_timeout(writeTimeout.tv_sec, writeTimeout.tv_usec); + if (username.length() && password.length()) + cli.set_basic_auth(username, password); + if (vaultNamespace.length()) + headers.emplace("X-Vault-Namespace", vaultNamespace.str()); + } + //if we tried to use our token and it returned access denied it could be that we need to login again, or // perhaps it could be specific permissions about the secret that was being accessed, I don't think we can tell the difference void kubernetesLogin(bool permissionDenied) @@ -454,14 +498,19 @@ class CVault std::string json; json.append("{\"jwt\": \"").append(login_token.str()).append("\", \"role\": \"").append(k8sAuthRole.str()).append("\"}"); httplib::Client cli(schemeHostPort.str()); - cli.enable_server_certificate_verification(verify_server); - - if (username.length() && password.length()) - cli.set_basic_auth(username, password); httplib::Headers headers; - if (vaultNamespace.length()) - headers.emplace("X-Vault-Namespace", vaultNamespace.str()); + + unsigned numRetries = 0; + initClient(cli, headers, numRetries); httplib::Result res = cli.Post("/v1/auth/kubernetes/login", headers, json, "application/json"); + while (!res && numRetries--) + { + OERRLOG("Retrying vault %s kubernetes auth, communication error %d", name.str(), res.error()); + if (retryWait) + Sleep(retryWait); + res = cli.Post("/v1/auth/kubernetes/login", headers, json, "application/json"); + } + processClientTokenResponse(res); } //if we tried to use our token and it returned access denied it could be that we need to login again, or @@ -485,15 +534,19 @@ class CVault json.append("{\"role_id\": \"").append(appRoleId).append("\", \"secret_id\": \"").append(appRoleSecretId).append("\"}"); httplib::Client cli(schemeHostPort.str()); - cli.enable_server_certificate_verification(verify_server); - - if (username.length() && password.length()) - cli.set_basic_auth(username, password); httplib::Headers headers; - if (vaultNamespace.length()) - headers.emplace("X-Vault-Namespace", vaultNamespace.str()); + unsigned numRetries = 0; + initClient(cli, headers, numRetries); httplib::Result res = cli.Post("/v1/auth/approle/login", headers, json, "application/json"); + while (!res && numRetries--) + { + OERRLOG("Retrying vault %s appRole auth, communication error %d", name.str(), res.error()); + if (retryWait) + Sleep(retryWait); + res = cli.Post("/v1/auth/approle/login", headers, json, "application/json"); + } + processClientTokenResponse(res); } void checkAuthentication(bool permissionDenied) @@ -549,18 +602,20 @@ class CVault checkAuthentication(permissionDenied); httplib::Client cli(schemeHostPort.str()); - cli.enable_server_certificate_verification(verify_server); - - if (username.length() && password.length()) - cli.set_basic_auth(username.str(), password.str()); - httplib::Headers headers = { { "X-Vault-Token", clientToken.str() } }; - if (vaultNamespace.length()) - headers.emplace("X-Vault-Namespace", vaultNamespace.str()); + unsigned numRetries = 0; + initClient(cli, headers, numRetries); httplib::Result res = cli.Get(location, headers); + while (!res && numRetries--) + { + OERRLOG("Retrying vault %s get secret, communication error %d location %s", name.str(), res.error(), location ? location : "null"); + if (retryWait) + Sleep(retryWait); + res = cli.Get(location, headers); + } if (res) { @@ -578,6 +633,10 @@ class CVault return requestSecretAtLocation(rkind, content, location, secret, version, true); OERRLOG("Vault %s permission denied accessing secret (check namespace=%s?) %s.%s location %s [%d](%d) - response: %s", name.str(), vaultNamespace.str(), secret, version ? version : "", location ? location : "null", res->status, res.error(), res->body.c_str()); } + else if (res->status == 404) + { + OERRLOG("Vault %s secret not found %s.%s location %s", name.str(), secret, version ? version : "", location ? location : "null"); + } else { OERRLOG("Vault %s error accessing secret %s.%s location %s [%d](%d) - response: %s", name.str(), secret, version ? version : "", location ? location : "null", res->status, res.error(), res->body.c_str()); From cf2e1594917d4d744ef71604309abceb6c69daf2 Mon Sep 17 00:00:00 2001 From: Mark Kelly Date: Tue, 15 Aug 2023 11:45:02 -0400 Subject: [PATCH 6/8] HPCC-30099 Roxie dont log multicast in log if disabled Signed-off-by: Mark Kelly --- roxie/ccd/ccdqueue.cpp | 25 ++++++++++++++++--------- 1 file changed, 16 insertions(+), 9 deletions(-) diff --git a/roxie/ccd/ccdqueue.cpp b/roxie/ccd/ccdqueue.cpp index 9019d43c2ce..be64ea37b09 100644 --- a/roxie/ccd/ccdqueue.cpp +++ b/roxie/ccd/ccdqueue.cpp @@ -304,24 +304,31 @@ void openMulticastSocket() if (!multicastSocket) { multicastSocket.setown(ISocket::udp_create(ccdMulticastPort)); - if (multicastTTL) - { - multicastSocket->set_ttl(multicastTTL); - DBGLOG("Roxie: multicastTTL: %u", multicastTTL); - } - else - DBGLOG("Roxie: multicastTTL not set"); multicastSocket->set_receive_buffer_size(udpMulticastBufferSize); size32_t actualSize = multicastSocket->get_receive_buffer_size(); + + StringBuffer socketName; + if (roxieMulticastEnabled) + socketName.append("multicast"); + else + socketName.append("udp-agent"); + if (actualSize < udpMulticastBufferSize) { - DBGLOG("Roxie: multicast socket buffer size could not be set (requested=%d actual %d", udpMulticastBufferSize, actualSize); + DBGLOG("Roxie: %s socket buffer size could not be set (requested=%d actual %d", socketName.str(), udpMulticastBufferSize, actualSize); throwUnexpected(); } if (doTrace(TraceFlags::Always)) - DBGLOG("Roxie: multicast socket created port=%d sockbuffsize=%d actual %d", ccdMulticastPort, udpMulticastBufferSize, actualSize); + DBGLOG("Roxie: %s socket created port=%d sockbuffsize=%d actual %d", socketName.str(), ccdMulticastPort, udpMulticastBufferSize, actualSize); if (roxieMulticastEnabled && !localAgent) { + if (multicastTTL) + { + multicastSocket->set_ttl(multicastTTL); + DBGLOG("Roxie: multicastTTL: %u", multicastTTL); + } + else + DBGLOG("Roxie: multicastTTL not set"); Owned topology = getTopology(); for (unsigned channel : topology->queryChannels()) { From 7aa11cceae747c2388103917e6770de2d13c6e04 Mon Sep 17 00:00:00 2001 From: wangkx Date: Wed, 16 Aug 2023 17:35:43 -0400 Subject: [PATCH 7/8] HPCC-30114 Do not check dropzone access if dropzone not found Also pass correct path when dropzone not found. Signed-off-by: wangkx --- esp/services/ws_fs/ws_fsService.cpp | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/esp/services/ws_fs/ws_fsService.cpp b/esp/services/ws_fs/ws_fsService.cpp index 4f76cb19969..42e01348a07 100644 --- a/esp/services/ws_fs/ws_fsService.cpp +++ b/esp/services/ws_fs/ws_fsService.cpp @@ -2503,12 +2503,15 @@ bool CFileSprayEx::onDespray(IEspContext &context, IEspDespray &req, IEspDespray StringBuffer destfileWithPath, umask; if (!isEmptyString(destPlane)) // must be true, unless bare-metal and isDropZoneRestrictionEnabled()==false + { getDropZoneInfoByDestPlane(version, destPlane, destfile, destfileWithPath, umask, destip); - - SecAccessFlags permission = getDZFileScopePermissions(context, destPlane, destfileWithPath, destip); - if (permission < SecAccess_Write) - throw makeStringExceptionV(ECLWATCH_INVALID_INPUT, "Access DropZone Scope %s %s not allowed for user %s (permission:%s). Write Access Required.", - isEmptyString(destPlane) ? destip : destPlane, destfileWithPath.str(), context.queryUserId(), getSecAccessFlagName(permission)); + SecAccessFlags permission = getDZFileScopePermissions(context, destPlane, destfileWithPath, destip); + if (permission < SecAccess_Write) + throw makeStringExceptionV(ECLWATCH_INVALID_INPUT, "Access DropZone Scope %s %s not allowed for user %s (permission:%s). Write Access Required.", + destPlane, destfileWithPath.str(), context.queryUserId(), getSecAccessFlagName(permission)); + } + else + destfileWithPath.append(destfile).trim(); RemoteFilename rfn; SocketEndpoint ep(destip.str()); From 177504ed91295db144d573213917cfd041d5d02e Mon Sep 17 00:00:00 2001 From: Russ Whitehead Date: Tue, 15 Aug 2023 16:46:05 -0400 Subject: [PATCH 8/8] HPCC-30058 Handling of missing HPCCInternal:: scope Remove LDAP check for HpccInternal scopes. Instead, check with code that the requested username in scope (HpccInternal::) matches the username provided in the request. Users to be granted access to their scope and subscopes, anything else to be denied. Remove code to create HpccInteral root scope, and HpccInternal:: scopes, since these are no longer needed Also, remove the addScopes tool since, this is no longer needed Signed-off-by: Russ Whitehead --- dali/server/daldap.cpp | 18 --- .../security/LdapSecurity/ldapconnection.cpp | 27 ----- .../security/LdapSecurity/ldapconnection.hpp | 1 - system/security/LdapSecurity/ldapsecurity.cpp | 34 +++--- system/security/LdapSecurity/ldapsecurity.ipp | 2 +- system/security/shared/basesecurity.hpp | 5 - system/security/shared/seclib.hpp | 3 +- tools/CMakeLists.txt | 1 - tools/addScopes/CMakeLists.txt | 55 --------- tools/addScopes/addScopes.cpp | 108 ------------------ tools/addScopes/sourcedoc.xml | 26 ----- 11 files changed, 17 insertions(+), 263 deletions(-) delete mode 100644 tools/addScopes/CMakeLists.txt delete mode 100644 tools/addScopes/addScopes.cpp delete mode 100644 tools/addScopes/sourcedoc.xml diff --git a/dali/server/daldap.cpp b/dali/server/daldap.cpp index c5c6a8f9584..90e97132297 100644 --- a/dali/server/daldap.cpp +++ b/dali/server/daldap.cpp @@ -57,23 +57,6 @@ class CDaliLdapConnection: implements IDaliLdapConnection, public CInterface unsigned ldapflags; IDigitalSignatureManager * pDSM = nullptr; - void createDefaultScopes() - { - try { - Owned user = ldapsecurity->createUser(nullptr); - StringBuffer userTempFileScope(queryDfsXmlBranchName(DXB_Internal)); - if (ldapsecurity->addResourceEx(RT_FILE_SCOPE, *user, userTempFileScope.str(),PT_ADMINISTRATORS_ONLY, NULL)) - PROGLOG("LDAP: Created default '%s' scope", userTempFileScope.str()); - else - throw MakeStringException(-1, "Error adding LDAP resource '%s'",userTempFileScope.str()); - } - catch (IException *e) { - EXCLOG(e,"LDAP createDefaultScopes"); - throw; - } - } - - public: IMPLEMENT_IINTERFACE; @@ -113,7 +96,6 @@ class CDaliLdapConnection: implements IDaliLdapConnection, public CInterface EXCLOG(e,"LDAP server"); throw; } - createDefaultScopes(); } } } diff --git a/system/security/LdapSecurity/ldapconnection.cpp b/system/security/LdapSecurity/ldapconnection.cpp index 2f12fa5f48b..9b44281f289 100644 --- a/system/security/LdapSecurity/ldapconnection.cpp +++ b/system/security/LdapSecurity/ldapconnection.cpp @@ -4276,12 +4276,6 @@ class CLdapClient : implements ILdapClient, public CInterface continue; changeUserGroup("delete", username, grp); } - - //Remove tempfile scope for this user - StringBuffer resName(queryDfsXmlBranchName(DXB_Internal)); - resName.append("::").append(username); - deleteResource(RT_FILE_SCOPE, resName.str(), m_ldapconfig->getResourceBasedn(RT_FILE_SCOPE)); - return true; } @@ -6299,30 +6293,9 @@ class CLdapClient : implements ILdapClient, public CInterface throw; } } - - //Add tempfile scope for this user (spill, paused and checkpoint - //will be created under this user specific scope) - StringBuffer resName(queryDfsXmlBranchName(DXB_Internal)); - resName.append("::").append(username); - Owned resource = new CLdapSecResource(resName.str()); - if (!addResource(RT_FILE_SCOPE, user, resource, PT_ADMINISTRATORS_AND_USER, m_ldapconfig->getResourceBasedn(RT_FILE_SCOPE))) - { - throw MakeStringException(-1, "Error adding temp file scope %s",resName.str()); - } - return true; } - bool createUserScope(ISecUser& user) - { - //Add tempfile scope for given user (spill, paused and checkpoint - //files will be created under this user specific scope) - StringBuffer resName(queryDfsXmlBranchName(DXB_Internal)); - resName.append("::").append(user.getName()); - Owned resource = new CLdapSecResource(resName.str()); - return addResource(RT_FILE_SCOPE, user, resource, PT_ADMINISTRATORS_AND_USER, m_ldapconfig->getResourceBasedn(RT_FILE_SCOPE)); - } - virtual aindex_t getManagedScopeTree(LDAP* ld, SecResourceType rtype, const char * basedn, IArrayOf& scopes) { Owned lconn; diff --git a/system/security/LdapSecurity/ldapconnection.hpp b/system/security/LdapSecurity/ldapconnection.hpp index ceb179a53ca..51d148e763d 100644 --- a/system/security/LdapSecurity/ldapconnection.hpp +++ b/system/security/LdapSecurity/ldapconnection.hpp @@ -318,7 +318,6 @@ interface ILdapClient : extends IInterface virtual int countResources(const char* basedn, const char* searchstr, int limit) = 0; virtual ILdapConfig* queryConfig() = 0; virtual const char* getPasswordStorageScheme() = 0; - virtual bool createUserScope(ISecUser& user) = 0; virtual aindex_t getManagedScopeTree(LDAP* ld, SecResourceType rtype, const char * basedn, IArrayOf& scopes) = 0; virtual SecAccessFlags queryDefaultPermission(ISecUser& user) = 0; diff --git a/system/security/LdapSecurity/ldapsecurity.cpp b/system/security/LdapSecurity/ldapsecurity.cpp index 96130a9b623..1827c6adc9d 100644 --- a/system/security/LdapSecurity/ldapsecurity.cpp +++ b/system/security/LdapSecurity/ldapsecurity.cpp @@ -22,6 +22,7 @@ #include "authmap.ipp" #include "digisign.hpp" #include "caching.hpp" +#include "dautils.hpp" using namespace cryptohelper; @@ -631,6 +632,7 @@ void CLdapSecManager::init(const char *serviceName, IPropertyTree* cfg) m_permissionsCache->setSecManager(this); m_passwordExpirationWarningDays = cfg->getPropInt(".//@passwordExpirationWarningDays", 10); //Default to 10 days m_checkViewPermissions = cfg->getPropBool(".//@checkViewPermissions", false); + m_hpccInternalScope.set(queryDfsXmlBranchName(DXB_Internal)).append("::");//HpccInternal:: }; @@ -1014,6 +1016,19 @@ SecAccessFlags CLdapSecManager::authorizeFileScope(ISecUser & user, const char * if(filescope == 0 || filescope[0] == '\0') return SecAccess_Full; + //Preprocess "HpccInternal::" scopes, since they are not managed by LDAP + //Grant user access to their own hpccinternal:: scope, deny if anything else + if(startsWithIgnoreCase(filescope, m_hpccInternalScope.str())) + { + StringBuffer userName; + for (const char * p = &filescope[m_hpccInternalScope.length()]; *p && *p != ':'; p++)//extract scope username + userName.append(*p); + if(strieq(userName.str(), user.getName())) + return SecAccess_Full; + PROGLOG("Access denied to scope %s for user %s", filescope, user.getName()); + return SecAccess_None; + } + StringBuffer managedFilescope; if(m_permissionsCache->isCacheEnabled() && !m_usercache_off) { @@ -1510,25 +1525,6 @@ bool CLdapSecManager::getUserInfo(ISecUser& user, const char* infotype) return m_ldap_client->getUserInfo(user, infotype); } -bool CLdapSecManager::createUserScopes(IEspSecureContext* secureContext) -{ - Owned it = getAllUsers(secureContext); - it->first(); - bool rc = true; - while(it->isValid()) - { - ISecUser &user = it->get(); - if (!m_ldap_client->createUserScope(user)) - { - PROGLOG("CLdapSecManager::createUserScopes Error creating user scope for user '%s'", user.getName()); - rc = false; - } - it->next(); - } - return rc; -} - - aindex_t CLdapSecManager::getManagedScopeTree(SecResourceType rtype, const char * basedn, IArrayOf& scopes, IEspSecureContext* secureContext) { return m_ldap_client->getManagedScopeTree(nullptr, rtype, basedn, scopes); diff --git a/system/security/LdapSecurity/ldapsecurity.ipp b/system/security/LdapSecurity/ldapsecurity.ipp index 6ee0bc1e7cd..fb84a7c5488 100644 --- a/system/security/LdapSecurity/ldapsecurity.ipp +++ b/system/security/LdapSecurity/ldapsecurity.ipp @@ -321,6 +321,7 @@ private: bool m_checkViewPermissions; static const SecFeatureSet s_safeFeatures = SMF_ALL_FEATURES; static const SecFeatureSet s_implementedFeatures = s_safeFeatures & ~(SMF_RetrieveUserData | SMF_RemoveResources); + StringBuffer m_hpccInternalScope; public: IMPLEMENT_IINTERFACE @@ -434,7 +435,6 @@ public: return m_checkViewPermissions; } - bool createUserScopes(IEspSecureContext* secureContext = nullptr) override; aindex_t getManagedScopeTree(SecResourceType rtype, const char * basedn, IArrayOf& scopes, IEspSecureContext* secureContext = nullptr) override; SecAccessFlags queryDefaultPermission(ISecUser& user, IEspSecureContext* secureContext = nullptr) override; bool clearPermissionsCache(ISecUser &user, IEspSecureContext* secureContext = nullptr) override; diff --git a/system/security/shared/basesecurity.hpp b/system/security/shared/basesecurity.hpp index 960718fb6e5..caa70397f6d 100644 --- a/system/security/shared/basesecurity.hpp +++ b/system/security/shared/basesecurity.hpp @@ -317,11 +317,6 @@ class CBaseSecurityManager : implements ISecManager, public CInterface throwUnexpected(); } - bool createUserScopes(IEspSecureContext* secureContext = nullptr) override - { - throwUnexpected(); - } - aindex_t getManagedScopeTree(SecResourceType rtype, const char * basedn, IArrayOf& scopes, IEspSecureContext* secureContext = nullptr) override { throwUnexpected(); diff --git a/system/security/shared/seclib.hpp b/system/security/shared/seclib.hpp index ed190dd6bf4..d410f874f3a 100644 --- a/system/security/shared/seclib.hpp +++ b/system/security/shared/seclib.hpp @@ -461,7 +461,7 @@ static const SecFeatureBit SMF_AuthorizeWorkUnitScope_List = 0x0100000000; static const SecFeatureBit SMF_AuthorizeWorkUnitScope_Named = 0x0200000000; static const SecFeatureBit SMF_GetDescription = 0x0400000000; static const SecFeatureBit SMF_GetPasswordExpirationDays = 0x0800000000; -static const SecFeatureBit SMF_CreateUserScopes = 0x1000000000; +//static const SecFeatureBit SMF_CreateUserScopes = 0x1000000000;//feature removed in 9.x static const SecFeatureBit SMF_GetManagedScopeTree = 0x2000000000; static const SecFeatureBit SMF_QueryDefaultPermission = 0x4000000000; static const SecFeatureBit SMF_ClearPermissionsCache = 0x8000000000; @@ -511,7 +511,6 @@ interface ISecManager : extends ISecObject virtual bool authorizeWorkunitScope(ISecUser & user, ISecResourceList * resources, IEspSecureContext* secureContext = nullptr) = 0; virtual const char * getDescription() = 0; virtual unsigned getPasswordExpirationWarningDays(IEspSecureContext* secureContext = nullptr) = 0; - virtual bool createUserScopes(IEspSecureContext* secureContext = nullptr) = 0; virtual aindex_t getManagedScopeTree(SecResourceType rtype, const char * basedn, IArrayOf& scopes, IEspSecureContext* secureContext = nullptr) = 0; virtual SecAccessFlags queryDefaultPermission(ISecUser& user, IEspSecureContext* secureContext = nullptr) = 0; virtual bool clearPermissionsCache(ISecUser & user, IEspSecureContext* secureContext = nullptr) = 0; diff --git a/tools/CMakeLists.txt b/tools/CMakeLists.txt index 71faad2a68e..22cefa9f137 100644 --- a/tools/CMakeLists.txt +++ b/tools/CMakeLists.txt @@ -19,7 +19,6 @@ HPCC_ADD_SUBDIRECTORY (esdlcmd-xml) HPCC_ADD_SUBDIRECTORY (esdlcmd) HPCC_ADD_SUBDIRECTORY (backupnode "PLATFORM") IF (USE_OPENLDAP) -HPCC_ADD_SUBDIRECTORY (addScopes "PLATFORM") HPCC_ADD_SUBDIRECTORY (initldap "PLATFORM") ENDIF(USE_OPENLDAP) HPCC_ADD_SUBDIRECTORY (combine "PLATFORM") diff --git a/tools/addScopes/CMakeLists.txt b/tools/addScopes/CMakeLists.txt deleted file mode 100644 index 6fb5c8b3d23..00000000000 --- a/tools/addScopes/CMakeLists.txt +++ /dev/null @@ -1,55 +0,0 @@ -################################################################################ -# HPCC SYSTEMS software Copyright (C) 2012 HPCC Systems®. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -################################################################################ - -# Component: addScopes - -##################################################### -# Description: -# ------------ -# Cmake Input File for addScopes -##################################################### - - -project( addScopes ) - -set ( SRCS - addScopes.cpp - ./../../system/security/LdapSecurity/ldaputils.cpp - ) - -include_directories ( - ./../../system/security/LdapSecurity - ./../../system/security/shared - ./../../system/jlib - ./../../system/include - ./../../dali/base - ./../../system/mp - ) - -ADD_DEFINITIONS( -D_CONSOLE ) - - -HPCC_ADD_EXECUTABLE ( addScopes ${SRCS} ) -install ( TARGETS addScopes RUNTIME DESTINATION ${EXEC_DIR} ) - - - -target_link_libraries ( addScopes - jlib - LdapSecurity - ) - - diff --git a/tools/addScopes/addScopes.cpp b/tools/addScopes/addScopes.cpp deleted file mode 100644 index bddc099ebbc..00000000000 --- a/tools/addScopes/addScopes.cpp +++ /dev/null @@ -1,108 +0,0 @@ -/*############################################################################## - - HPCC SYSTEMS software Copyright (C) 2012 HPCC Systems®. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -############################################################################## */ - -#include "seclib.hpp" -#include "ldapsecurity.hpp" -#include "jliball.hpp" -#include "dasess.hpp" - -#ifndef _WIN32 -#include -#endif - -int main(int argc, char* argv[]) -{ - if(argc < 2 || argc > 3) - { - printf("usage: addScopes daliconf.xml [-c]\n"); - printf("\n\tCreates all user-specific LDAP private file scopes 'hpccinternal::'\n\tand grants users access to their scope. The configuration file\n\tdaliconf.xml is the dali configuration file, typically\n\tfound in /var/lib/HPCCSystems/mydali\n\tSpecify -c to make changes immediately visible by clearing permission caches\n\n"); - return -1; - } - - InitModuleObjects(); - - try - { - Owned cfg = createPTreeFromXMLFile(argv[1]); - Owned seccfg = cfg->getPropTree(".//ldapSecurity"); - if(seccfg == NULL) - { - printf("ldapSecurity not found\n"); - return -1; - } -#ifdef _NO_LDAP - printf("System was built with _NO_LDAP\n"); - return -1; -#else - Owned secmgr = newLdapSecManager("addScopes", *LINK(seccfg)); - if(secmgr == NULL) - { - printf("Security manager can't be created\n"); - releaseAtoms(); - return -1; - } - bool ok = secmgr->createUserScopes(); - printf(ok ? "User scopes added\n" : "Some scopes not added\n"); - - //Clear permission caches? - if (argc == 3 && 0==stricmp(argv[2], "-c")) - { - //Clear ESP Cache - StringBuffer sysuser; - StringBuffer passbuf; - seccfg->getProp(".//@systemUser", sysuser); - seccfg->getProp(".//@systemPassword", passbuf); - - if (0 == sysuser.length()) - { - printf("Error in configuration file %s - systemUser not specified", argv[1]); - releaseAtoms(); - return -1; - } - - if (0 == passbuf.length()) - { - printf("Error in configuration file %s - systemPassword not specified", argv[1]); - releaseAtoms(); - return -1; - } - - StringBuffer decPwd; - decrypt(decPwd, passbuf.str()); - - //Clear Dali cache - Owned userdesc(createUserDescriptor()); - userdesc->set(sysuser, decPwd); - ok = querySessionManager().clearPermissionsCache(userdesc); - printf(ok ? "Dali Cache cleared\n" : "Error clearing Dali Cache\n"); - } -#endif - } - catch(IException* e) - { - StringBuffer errmsg; - e->errorMessage(errmsg); - printf("%s\n", errmsg.str()); - } - catch(...) - { - printf("Unknown exception\n"); - } - - releaseAtoms(); - return 0; -} diff --git a/tools/addScopes/sourcedoc.xml b/tools/addScopes/sourcedoc.xml deleted file mode 100644 index 59994d81c14..00000000000 --- a/tools/addScopes/sourcedoc.xml +++ /dev/null @@ -1,26 +0,0 @@ - - - -
- tools/addScopes - - - The tools/addScopes directory contains the sources for the tools/addScopes tool. - -