diff --git a/.github/workflows/build-assets.yml b/.github/workflows/build-assets.yml index 12ebea45e5a..749aae5a0e6 100644 --- a/.github/workflows/build-assets.yml +++ b/.github/workflows/build-assets.yml @@ -368,6 +368,29 @@ jobs: tag: ${{ needs.preamble.outputs.internal_tag }} artifacts: "${{ needs.preamble.outputs.folder_build }}/hpccsystems-*-internal*.deb,${{ needs.preamble.outputs.folder_build }}/hpccsystems-*-internal*.rpm" + - name: Upload Assets to Jfrog (debian internal) + if: ${{ matrix.ln && !matrix.container && contains(matrix.os, 'ubuntu') && github.repository_owner == 'hpcc-systems'}} + shell: bash + run: | + cd ${{ needs.preamble.outputs.folder_build }} + version=$(echo "${{ needs.preamble.outputs.internal_tag }}" | sed 's/internal_//') + packages=($(ls -1 hpccsystems-*.deb )) + for _package in ${packages[@]}; do + distribution=$( echo "${_package}" | sed "s/^.*${version}//" | awk -F '_' '{print $1;}' ) + curl -u${{ secrets.JFROG_USERNAME }}:${{ secrets.JFROG_PASSWORD }} -XPUT "https://${{ secrets.JFROG_REGISTRY }}/hpccpl-debian-local/pool/LN/${_package};deb.distribution=${distribution};deb.component=LN;deb.architecture=amd64" -T ${{ needs.preamble.outputs.folder_build }}/${_package} + done + + - name: Upload Assets to Jfrog (centos internal) + if: ${{ matrix.ln && !matrix.container && !contains(matrix.os, 'ubuntu') && github.repository_owner == 'hpcc-systems'}} + shell: bash + run: | + cd ${{ needs.preamble.outputs.folder_build }} + packages=($(ls -1 hpccsystems-*.rpm )) + for _package in ${packages[@]}; do + distribution=$( echo "${_package}" | awk -F '.' '{print $4;}' ) + curl -u${{ secrets.JFROG_USERNAME }}:${{ secrets.JFROG_PASSWORD }} -XPUT "https://${{ secrets.JFROG_REGISTRY }}/hpccpl-rpm-local/LN/${distribution}/x86_64/${_package}" -T ${{ needs.preamble.outputs.folder_build }}/${_package} + done + - name: Locate k8s deb file (internal) if: ${{ matrix.ln && matrix.container && !matrix.documentation }} id: ln-container @@ -579,6 +602,26 @@ jobs: tag: ${{ needs.preamble.outputs.internal_tag }} artifacts: "./build/hpccsystems-clienttools-internal*.exe,./build/hpccsystems-clienttools-internal*.msi,./build/hpccsystems-clienttools-internal*.dmg,./build/hpccsystems-clienttools-internal*.pkg,./build/hpccsystems-clienttools-internal*.tar.gz" + - name: Upload Assets to Jfrog (windows) + if: ${{ contains(matrix.os, 'windows') && github.repository_owner == 'hpcc-systems' }} + shell: bash + run: | + cd ./build + packages=($(ls -1 hpccsystems-*.exe )) + for _package in ${packages[@]}; do + curl -u${{ secrets.JFROG_USERNAME }}:${{ secrets.JFROG_PASSWORD }} "https://${{ secrets.JFROG_REGISTRY }}/hpccpl-windows-local/LN/windows/x86_64/${_package}" -T ${_package} + done + + - name: Upload Assets to Jfrog (macos) + if: ${{ contains(matrix.os, 'macos') && github.repository_owner == 'hpcc-systems' }} + shell: bash + run: | + cd ./build + packages=($(ls -1 hpccsystems-*.pkg )) + for _package in ${packages[@]}; do + curl -u${{ secrets.JFROG_USERNAME }}:${{ secrets.JFROG_PASSWORD }} "https://${{ secrets.JFROG_REGISTRY }}/hpccpl-macos-local/LN/macos/x86_64/${_package}" -T ${_package} + done + - name: Upload error logs if: ${{ failure() || cancelled() }} uses: actions/upload-artifact@v4 diff --git a/helm/hpcc/templates/localroxie.yaml.fixed b/helm/hpcc/templates/localroxie.yaml.fixed new file mode 100644 index 00000000000..099b501840b --- /dev/null +++ b/helm/hpcc/templates/localroxie.yaml.fixed @@ -0,0 +1,161 @@ +{{/* + +--- DO NOT EDIT THIS FILE - all configuration of HPCC platform should be done via values.yaml ---- + +############################################################################## + + HPCC SYSTEMS software Copyright (C) 2021 HPCC Systems®. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +############################################################################## + +*/}} +{{/* +localroxie configmap +Pass in dict with root and me +*/}} +{{- define "hpcc.localroxieConfigMap" }} +apiVersion: v1 +metadata: + name: {{ .me.name }}-configmap +data: + {{ .me.name }}.yaml: + version: 1.0 + roxie: +{{ toYaml (omit .me "logging" "tracing" "env") | indent 6 }} +{{- include "hpcc.generateLoggingConfig" . | indent 6 }} +{{- include "hpcc.generateTracingConfig" . | indent 6 }} +{{ include "hpcc.generateVaultConfig" . | indent 6 }} + global: +{{ include "hpcc.generateGlobalConfigMap" .root | indent 6 }} +{{- end -}}{{/* define "hpcc.localroxieConfigMap */}} + +{{ range $roxie := $.Values.roxie -}} +{{- if not $roxie.disabled -}} +{{- $env := concat ($.Values.global.env | default list) (.env | default list) -}} +{{- $secretsCategories := list "system" "eclUser" "ecl" "storage" }} +{{- $enginePlaneDetails := dict -}} +{{- $_ := include "hpcc.getEnginePlanes" (dict "root" $ "me" . "result" $enginePlaneDetails) -}} +{{- $commonCtx := dict "root" $ "me" $roxie "includeCategories" $enginePlaneDetails.planeCategories "includeNames" $enginePlaneDetails.namedPlanes "secretsCategories" $secretsCategories "env" $env }} +{{- $configSHA := include "hpcc.getConfigSHA" ($commonCtx | merge (dict "configMapHelper" "hpcc.localroxieConfigMap" "component" "roxie" "excludeKeys" "global")) }} +{{- include "hpcc.checkDefaultStoragePlane" $commonCtx }} +{{- $singleNode := (hasKey $roxie "singleNode") | ternary $roxie.singleNode ((hasKey $roxie "localAgent") | ternary $roxie.localAgent false) }} +{{- if $singleNode -}} +{{- $localAgent := ((hasKey $roxie "localAgent") | ternary $roxie.localAgent true) -}} +{{- $name := $roxie.name -}} +{{- $servername := printf "%s-server" $roxie.name -}} + +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ $roxie.name | quote }} +spec: + replicas: {{ $roxie.replicas | default 1 }} + selector: + matchLabels: + run: {{ $roxie.name | quote }} + server: {{ $servername | quote }} + template: + metadata: + labels: + run: {{ $roxie.name | quote }} + server: {{ $servername | quote }} + accessDali: "yes" + accessEsp: "yes" +<<<<<<< HEAD + helmVersion: 9.0.117-closedown0 +{{- include "hpcc.addStandardLabels" (dict "root" $ "component" "roxie-server" "name" "roxie" "instance" $roxie.name) | indent 8 }} +{{- if hasKey . "labels" }} +{{ toYaml .labels | indent 8 }} +{{- end }} +======= + helmVersion: 9.0.117-closedown0 + {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "roxie-server" "name" "roxie" "instance" $roxie.name) | indent 8 }} +{{- if hasKey . "labels" }} +{{ toYaml .labels | indent 8 }} +{{- end }} +>>>>>>> origin/candidate-9.6.x + annotations: + checksum/config: {{ $configSHA }} +{{- include "hpcc.generateAnnotations" $commonCtx | indent 8 }} + spec: +{{- include "hpcc.placementsByPodTargetType" (dict "root" $ "pod" $roxie.name "target" $roxie.name "type" "roxie") | indent 6 }} + serviceAccountName: "hpcc-default" + initContainers: +{{- include "hpcc.createConfigInitContainers" $commonCtx | indent 6 }} +{{- include "hpcc.addImagePullSecrets" $commonCtx | nindent 6 -}} + containers: + - name: {{ $roxie.name | quote }} + workingDir: /var/lib/HPCCSystems + command: [ {{ include "hpcc.componentCommand" (dict "me" $roxie "root" $ "process" "roxie") }} ] + args: [ {{- include "hpcc.componentStartArgs" (dict "me" $roxie "root" $ "process" "roxie") | nindent 16 }} + {{ include "hpcc.configArg" $roxie }}, + {{ include "hpcc.daliArg" (dict "root" $ "component" "Local Roxie" "optional" false) }}, + "--server=true", + "--localAgent={{ $localAgent }}", + "--resolveLocally=false" + ] + env: +{{ include "hpcc.mergeEnvironments" $env | indent 8 -}} + - name: "SENTINEL" + value: "/tmp/{{ $roxie.name }}.sentinel" +{{- $local := dict "first" true }} +{{- range $service := $roxie.services }} +{{- if ne (int $service.servicePort) 0 }} +{{- if $local.first }} +{{- $_ := set $local "first" false }} + ports: +{{- end }} + - name: {{ $service.name }} + containerPort: {{ $service.servicePort }} +{{- end }} +{{- end }} +{{- include "hpcc.addSecurityContext" $commonCtx | indent 8 }} +{{- include "hpcc.addResources" (dict "me" $roxie.resources "root" $) | indent 8 }} +{{ include "hpcc.addImageAttrs" $commonCtx | indent 8 }} + volumeMounts: +{{ include "hpcc.addConfigMapVolumeMount" . | indent 8 }} +{{ include "hpcc.addVolumeMounts" $commonCtx | indent 8 }} +{{- include "hpcc.addSecretVolumeMounts" $commonCtx | indent 8 }} +{{ include "hpcc.addVaultClientCertificateVolumeMounts" $commonCtx | indent 8 }} +{{ include "hpcc.addCertificateVolumeMount" (dict "root" $ "name" $roxie.name "component" "localroxie" "external" false) | indent 8 }} +{{ include "hpcc.addCertificateVolumeMount" (dict "root" $ "name" $roxie.name "component" "localroxie" "external" true "includeRemote" true) | indent 8 }} +{{ include "hpcc.addUDPCertificateVolumeMount" (dict "root" $ "name" $roxie.name "component" "localudpkey" ) | indent 8 }} + volumes: +{{ include "hpcc.addConfigMapVolume" . | indent 6 }} +{{ include "hpcc.addVolumes" $commonCtx | indent 6 }} +{{ include "hpcc.addSecretVolumes" $commonCtx | indent 6 }} +{{ include "hpcc.addVaultClientCertificateVolumes" $commonCtx | indent 6 }} +{{ include "hpcc.addCertificateVolume" (dict "root" $ "name" $roxie.name "component" "localroxie" "external" false) | indent 6 }} +{{ include "hpcc.addCertificateVolume" (dict "root" $ "name" $roxie.name "component" "localroxie" "external" true "includeRemote" true) | indent 6 }} +{{ include "hpcc.addUDPCertificateVolume" (dict "root" $ "name" $roxie.name "component" "localudpkey" ) | indent 6 }} +--- +{{- range $service := $roxie.services }} +{{- if ne (int $service.servicePort) 0 }} +{{ include "hpcc.addService" ( dict "root" $ "name" $service.name "service" $service "selector" $servername "defaultPort" $service.servicePort ) }} +--- +{{- end }} +{{- end }} +kind: ConfigMap +{{ include "hpcc.generateConfig" ($commonCtx | merge (dict "configMapHelper" "hpcc.localroxieConfigMap")) }} +--- +{{ include "hpcc.addCertificate" (dict "root" $ "name" $roxie.name "services" $roxie.services "component" "localroxie" "external" false) }} +{{ include "hpcc.addCertificate" (dict "root" $ "name" $roxie.name "services" $roxie.services "component" "localroxie" "external" true "includeRemote" true) }} +{{ include "hpcc.addUDPCertificate" (dict "root" $ "name" $roxie.name "component" "localudpkey") }} +--- +{{ include "hpcc.addEgress" $commonCtx }} + +{{- end }}{{/* if singleNode */}} +{{- end }}{{/* if not disabled */}} +{{- end }}{{/* range */}} diff --git a/helm/hpcc/templates/roxie.yaml.fixed b/helm/hpcc/templates/roxie.yaml.fixed new file mode 100644 index 00000000000..3f919f7c048 --- /dev/null +++ b/helm/hpcc/templates/roxie.yaml.fixed @@ -0,0 +1,479 @@ +{{/* + +--- DO NOT EDIT THIS FILE - all configuration of HPCC platform should be done via values.yaml ---- + +############################################################################## + + HPCC SYSTEMS software Copyright (C) 2021 HPCC Systems®. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +############################################################################## + +*/}} + +{{/* +roxie configmap +Pass in dict with root and me +*/}} +{{- define "hpcc.roxieConfigMap" -}} +apiVersion: v1 +metadata: + name: {{ .me.name }}-configmap +data: + {{ .me.name }}.yaml: + version: 1.0 + roxie: +{{- $root := .root -}} +{{- $component := .me }} + services: +{{- range $service := .me.services }} + - name: {{ $service.name }} +{{ toYaml (omit $service "tls" "name") | indent 8 }} +{{- if ne (int $service.servicePort) 0 }} +{{- include "hpcc.addTLSServiceEntries" (dict "root" $root "service" $service "component" $component "visibility" $service.visibility "remoteClients" $service.remoteClients "trustClients" $service.trustClients "includeTrustedPeers" true "incluedRoxieAndEspServices" true) | indent 6 }} +{{- end }} +{{- end }} +{{ toYaml ( omit .me "logging" "tracing" "topoServer" "encryptInTransit" "env" "services") | indent 6 }} + numChannels: {{ .numChannels }} + topologyServers: "{{ .toponame }}:{{ .topoport }}" + heartbeatInterval: {{ .heartbeatInterval }} + resolveLocally: false +{{- $mtlsEnabled := (eq (include "hpcc.isMtlsEnabled" (dict "root" .root)) "true") -}} +{{/* By default use encryption if local certificates are enabled, but allow it to be turned off via roxie .encryptInTransit value */}} +{{- if (hasKey .me "encryptInTransit") -}} +{{- if and (.me.encryptInTransit) (not $mtlsEnabled) -}} +{{- $_ := fail (printf "Roxie %s encryptInTransit requires local cert-manager configuration." .me.name ) }} +{{- end }} + encryptInTransit: {{ .me.encryptInTransit }} +{{ else }} + encryptInTransit: {{ $mtlsEnabled }} +{{ end -}} +{{- include "hpcc.generateLoggingConfig" (dict "root" .root "me" .me) | indent 6 }} +{{- include "hpcc.generateTracingConfig" (dict "root" .root "me" .me) | indent 6 }} +{{ include "hpcc.generateVaultConfig" . | indent 6 }} + global: +{{ include "hpcc.generateGlobalConfigMap" .root | indent 6 }} +{{- end -}}{{/*define "hpcc.roxieConfigMap"*/}} + +{{- define "hpcc.roxieTopoConfigMap" -}} +apiVersion: v1 +metadata: + name: {{ .toponame }}-configmap +data: + {{ .toponame }}.yaml: + version: 1.0 + toposerver: +{{ toYaml ( omit .toposerver "logging" "tracing" "env") | indent 6 }} +{{- include "hpcc.generateLoggingConfig" (dict "root" .root "me" .toposerver) | indent 6 }} +{{- include "hpcc.generateTracingConfig" (dict "root" .root "me" .toposerver) | indent 6 }} + global: +{{ include "hpcc.generateGlobalConfigMap" .root | indent 6 }} +{{- end -}}{{/*define "hpcc.roxieConfigMap"*/}} + +{{ range $roxie := $.Values.roxie -}} +{{- if not $roxie.disabled -}} +{{- $env := concat ($.Values.global.env | default list) (.env | default list) -}} +{{- $secretsCategories := list "system" "eclUser" "ecl" "storage" }} +{{- $toposerver := ($roxie.topoServer | default dict) -}} +{{- $enginePlaneDetails := dict -}} +{{- $_ := include "hpcc.getEnginePlanes" (dict "root" $ "me" . "result" $enginePlaneDetails) -}} +{{- $commonCtx := dict "root" $ "me" $roxie "includeCategories" $enginePlaneDetails.planeCategories "includeNames" $enginePlaneDetails.namedPlanes "secretsCategories" $secretsCategories "toposerver" $toposerver "env" $env }} +{{- $_ := set $commonCtx "toponame" (printf "%s-toposerver" $roxie.name) -}} +{{- $_ := set $commonCtx "numChannels" ($roxie.numChannels | int | default 1) -}} +{{- $_ := set $commonCtx "topoport" ($toposerver.port | int | default 9004) -}} +{{- $_ := set $commonCtx "heartbeatInterval" ($toposerver.heartbeatInterval | int | default 10000) -}} +{{- $_ := set $toposerver "name" $commonCtx.toponame -}} +{{- $configSHA := include "hpcc.getConfigSHA" ($commonCtx | merge (dict "configMapHelper" "hpcc.roxieConfigMap" "component" "roxie" "excludeKeys" "global")) }} +{{- $topoconfigSHA := include "hpcc.getConfigSHA" ($commonCtx | merge (dict "configMapHelper" "hpcc.roxieTopoConfigMap" "component" "toposerver" "excludeKeys" "global")) }} +{{- include "hpcc.checkDefaultStoragePlane" $commonCtx }} +{{- $singleNode := (hasKey $roxie "singleNode") | ternary $roxie.singleNode ((hasKey $roxie "localAgent") | ternary $roxie.localAgent false) }} +{{- if not $singleNode -}} +{{- $servername := printf "%s-server" $roxie.name -}} +{{- $udpkeyname := $roxie.name -}} +{{- range $service := $roxie.services }} +{{- range $remoteClient := $service.remoteClients }} + {{ include "hpcc.addExternalRemoteClientCertificate" (dict "root" $ "client" $remoteClient.name "organization" $remoteClient.organization "instance" $service.name "component" "roxie" "visibility" $service.visibility "secretTemplate" $remoteClient.secretTemplate) }} +{{- end }} +{{- if ne (int $service.servicePort) 0 }} +{{- $_ := set $service "port" $service.servicePort }} +{{- end }} +{{- end }} + +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ $commonCtx.toponame | quote }} +spec: + replicas: {{ $toposerver.replicas | default 1 }} + selector: + matchLabels: + run: {{ $commonCtx.toponame | quote }} + template: + metadata: + labels: +{{- include "hpcc.addStandardLabels" (dict "root" $ "component" "topology-server" "name" "roxie" "instance" $commonCtx.toponame) | indent 8 }} + run: {{ $commonCtx.toponame | quote }} + roxie-cluster: {{ $roxie.name | quote }} +<<<<<<< HEAD + helmVersion: 9.0.117-closedown0 +{{- if hasKey $.Values.global "metrics" }} +{{- include "hpcc.generateMetricsReporterLabel" $.Values.global.metrics | nindent 8}} +{{- end }} +{{- if hasKey $toposerver "labels" }} +{{ toYaml $toposerver.labels | indent 8 }} +{{- end }} +======= + helmVersion: 9.0.117-closedown0 +{{- if hasKey $.Values.global "metrics" }} + {{- include "hpcc.generateMetricsReporterLabel" $.Values.global.metrics | nindent 8}} +{{- end }} +{{- if hasKey $toposerver "labels" }} +{{ toYaml $toposerver.labels | indent 8 }} +{{- end }} +>>>>>>> origin/candidate-9.6.x + annotations: + checksum/config: {{ $topoconfigSHA }} +{{- include "hpcc.generateAnnotations" (dict "root" $commonCtx.root "me" $toposerver) | indent 8 }} +{{- if hasKey $.Values.global "metrics" }} +{{- include "hpcc.addPrometheusScrapeAnnotations" $.Values.global.metrics | nindent 8 }} +{{- end }} + spec: +{{- include "hpcc.placementsByPodTargetType" (dict "root" $ "pod" $commonCtx.toponame "target" $roxie.name "type" "roxie") | indent 6 }} + serviceAccountName: "hpcc-default" +{{- include "hpcc.addImagePullSecrets" $commonCtx | nindent 6 -}} + containers: + - name: {{ $commonCtx.toponame | quote }} +{{ include "hpcc.addSentinelProbes" $toposerver | indent 8 }} +{{- include "hpcc.addSecurityContext" $commonCtx | indent 8 }} +{{- $defaultResources := dict "cpu" "500m" "memory" "200Mi" }} +{{- include "hpcc.addResources" (dict "me" .topoResources "defaults" $defaultResources "root" $) | indent 8 }} +{{ include "hpcc.addImageAttrs" $commonCtx | indent 8 }} + workingDir: /var/lib/HPCCSystems + command: [ {{ include "hpcc.componentCommand" (dict "me" $toposerver "root" $ "process" "toposerver") }} ] + args: [ {{- include "hpcc.componentStartArgs" (dict "me" $toposerver "root" $ "process" "toposerver") | nindent 16 }} + {{ include "hpcc.configArg" $toposerver }} + ] + env: +{{ include "hpcc.mergeEnvironments" $env | indent 8 -}} + - name: "SENTINEL" + value: "/tmp/{{ $commonCtx.toponame }}.sentinel" + volumeMounts: +{{ include "hpcc.addConfigMapVolumeMount" $toposerver | indent 8 }} +{{ include "hpcc.addCertificateVolumeMount" (dict "root" $ "component" "topo" "name" $commonCtx.toponame "external" false) | indent 8 }} + volumes: +{{ include "hpcc.addConfigMapVolume" $toposerver | indent 6 }} +{{ include "hpcc.addCertificateVolume" (dict "root" $ "component" "topo" "name" $commonCtx.toponame "external" false) | indent 6 }} + +--- +{{ include "hpcc.addCertificate" (dict "root" $ "name" $commonCtx.toponame "servicename" $commonCtx.toponame "component" "topo" "external" false) }} +{{ include "hpcc.addUDPCertificate" (dict "root" $ "name" $udpkeyname "component" "udpkey") }} + +--- +{{- range $service := $roxie.services }} +{{- if ne (int $service.servicePort) 0 }} +{{ include "hpcc.addService" ( dict "root" $ "name" $service.name "service" $service "selector" $servername "defaultPort" $service.servicePort) }} +--- +{{- end }} +{{- end }} + +apiVersion: v1 +kind: Service +metadata: + name: {{ $commonCtx.toponame | quote }} + labels: +<<<<<<< HEAD + helmVersion: 9.0.117-closedown0 +{{- include "hpcc.addStandardLabels" (dict "root" $ "component" "topology-server" "name" "roxie" "instance" $commonCtx.toponame) | indent 4 }} +======= + helmVersion: 9.0.117-closedown0 + {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "topology-server" "name" "roxie" "instance" $commonCtx.toponame) | indent 4 }} +>>>>>>> origin/candidate-9.6.x +spec: + ports: + - port: {{ $commonCtx.topoport }} + protocol: TCP + targetPort: {{ $commonCtx.topoport }} + selector: + run: {{ $commonCtx.toponame | quote }} + clusterIP: None # Headless service +--- + +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: {{ printf "%s-internal-traffic" $roxie.name }} +spec: + podSelector: + matchLabels: + roxie-cluster: {{ $roxie.name | quote }} + policyTypes: + - Ingress + - Egress + ingress: + - from: + - podSelector: + matchLabels: + roxie-cluster: {{ $roxie.name | quote }} + egress: + - to: + - podSelector: + matchLabels: + roxie-cluster: {{ $roxie.name | quote }} + +--- +kind: ConfigMap +{{ include "hpcc.generateConfig" ($commonCtx | merge (dict "configMapHelper" "hpcc.roxieConfigMap")) }} +--- +kind: ConfigMap +{{ include "hpcc.generateConfig" ($commonCtx | merge (dict "configMapHelper" "hpcc.roxieTopoConfigMap")) }} +--- + +{{- $_ := set $commonCtx "instanceNames" list -}} +{{ if $roxie.serverReplicas -}} +{{ $_ := set $commonCtx "instanceNames" (list $servername) -}} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ $servername | quote }} +spec: + replicas: {{ $roxie.serverReplicas }} + selector: + matchLabels: + run: {{ $servername | quote }} + template: + metadata: + labels: + run: {{ $servername | quote }} + server: {{ $roxie.name | quote }} + roxie-cluster: {{ $roxie.name | quote }} + accessDali: "yes" + accessEsp: "yes" +<<<<<<< HEAD + helmVersion: 9.0.117-closedown0 +{{- include "hpcc.addStandardLabels" (dict "root" $ "component" "roxie-server" "name" "roxie" "instance" $servername) | indent 8 }} +{{- if hasKey $.Values.global "metrics" }} +{{- include "hpcc.generateMetricsReporterLabel" $.Values.global.metrics | nindent 8}} +{{- end }} +{{- if hasKey $roxie "labels" }} +{{ toYaml $roxie.labels | indent 8 }} +{{- end }} +======= + helmVersion: 9.0.117-closedown0 + {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "roxie-server" "name" "roxie" "instance" $servername) | indent 8 }} +{{- if hasKey $.Values.global "metrics" }} + {{- include "hpcc.generateMetricsReporterLabel" $.Values.global.metrics | nindent 8}} +{{- end }} +{{- if hasKey $roxie "labels" }} +{{ toYaml $roxie.labels | indent 8 }} +{{- end }} +>>>>>>> origin/candidate-9.6.x + annotations: + checksum/config: {{ $configSHA }} +{{- include "hpcc.generateAnnotations" $commonCtx | indent 8 }} +{{- if hasKey $.Values.global "metrics" }} +{{- include "hpcc.addPrometheusScrapeAnnotations" $.Values.global.metrics | nindent 8 }} +{{- end }} +{{- if hasKey $roxie "annotations" }} +{{ toYaml $roxie.annotations | indent 8 }} +{{- end }} + spec: +{{- include "hpcc.placementsByPodTargetType" (dict "root" $ "pod" $servername "target" $roxie.name "type" "roxie") | indent 6 }} + serviceAccountName: "hpcc-default" + initContainers: +{{- include "hpcc.createConfigInitContainers" $commonCtx | indent 6 }} +{{- include "hpcc.addImagePullSecrets" $commonCtx | nindent 6 -}} + terminationGracePeriodSeconds: {{ add ($roxie.agentQueryReleaseDelaySeconds | default 60) 30 }} + containers: + - name: {{ $servername | quote }} + workingDir: /var/lib/HPCCSystems + command: [ {{ include "hpcc.componentCommand" (dict "me" $roxie "root" $ "process" "roxie") }} ] + args: [ {{- include "hpcc.componentStartArgs" (dict "me" $roxie "root" $ "process" "roxie") | nindent 16 }} + {{ include "hpcc.daliArg" (dict "root" $ "component" "Roxie" "optional" false) }}, + "--server=true" + ] + env: +{{ include "hpcc.mergeEnvironments" $env | indent 8 -}} + - name: "SENTINEL" + value: "/tmp/{{ $roxie.name }}.sentinel" +{{- $local := dict "first" true }} +{{- range $service := $roxie.services }} +{{- if ne (int $service.servicePort) 0 }} +{{- if $local.first }} +{{- $_ := set $local "first" false }} + ports: +{{- end }} + - name: {{ $service.name }} + containerPort: {{ $service.servicePort }} +{{- end }} +{{- end }} + lifecycle: + preStop: + exec: + command: ["testsocket", ".", "control:closedown"] +{{ include "hpcc.addSentinelProbes" ( $roxie | merge (dict "readyProbeName" ".ready" )) | indent 8 }} +{{ include "hpcc.addSecurityContext" (dict "root" $ "me" .) | indent 8 }} +{{- include "hpcc.addResources" (dict "me" ($roxie.serverResources | default $roxie.resources) "root" $) | indent 8 }} +{{ include "hpcc.addImageAttrs" $commonCtx | indent 8 }} + volumeMounts: +{{ include "hpcc.addConfigMapVolumeMount" $roxie | indent 8 }} +{{ include "hpcc.addVolumeMounts" $commonCtx | indent 8 }} +{{ include "hpcc.addSecretVolumeMounts" $commonCtx | indent 8 }} +{{ include "hpcc.addVaultClientCertificateVolumeMounts" $commonCtx | indent 8 }} +{{ include "hpcc.addCertificateVolumeMount" (dict "root" $ "component" "roxie-server" "name" $servername "external" false) | indent 8 }} +{{ include "hpcc.addCertificateVolumeMount" (dict "root" $ "component" "roxie-server" "name" $servername "certificate" $roxie.certificate "external" true "includeRemote" true) | indent 8 }} +{{ include "hpcc.addUDPCertificateVolumeMount" (dict "root" $ "component" "udpkey" "name" $udpkeyname ) | indent 8 }} + volumes: +{{ include "hpcc.addConfigMapVolume" $roxie | indent 6 }} +{{ include "hpcc.addVolumes" $commonCtx | indent 6 }} +{{ include "hpcc.addSecretVolumes" $commonCtx | indent 6 }} +{{ include "hpcc.addVaultClientCertificateVolumes" $commonCtx | indent 6 }} +{{ include "hpcc.addCertificateVolume" (dict "root" $ "component" "roxie-server" "name" $servername "external" false) | indent 6 }} +{{ include "hpcc.addCertificateVolume" (dict "root" $ "component" "roxie-server" "name" $servername "certificate" $roxie.certificate "external" true "includeRemote" true) | indent 6 }} +{{ include "hpcc.addUDPCertificateVolume" (dict "root" $ "component" "udpkey" "name" $udpkeyname) | indent 6 }} + +--- +{{ include "hpcc.addCertificate" (dict "root" $ "name" $servername "services" $roxie.services "component" "roxie-server" "external" false) }} +{{ include "hpcc.addCertificate" (dict "root" $ "name" $servername "services" $roxie.services "component" "roxie-server" "external" true "includeRemote" true) }} +--- +{{ end -}}{{/* if serverReplicas */}} + +{{- $agentPublicCertName := printf "%s-agent" $roxie.name }} +{{ include "hpcc.addCertificate" (dict "root" $ "name" $agentPublicCertName "services" $roxie.services "component" "roxie-agent" "external" true "includeRemote" true) }} + +{{ range $c, $e := until ($commonCtx.numChannels|int) -}} +{{- $channel := add $c 1 -}} +{{- $name := printf "%s-agent-%d" $roxie.name $channel }} +{{- $_ := set $commonCtx "instanceNames" (append $commonCtx.instanceNames $name) }} + +{{ include "hpcc.addCertificate" (dict "root" $ "name" $name "services" $roxie.services "component" "roxie-agent" "external" false) }} +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ $name | quote}} +spec: + replicas: {{ (hasKey $roxie "replicas") | ternary $roxie.replicas 1 }} + selector: + matchLabels: + run: {{ $name | quote}} + template: + metadata: + labels: +{{- include "hpcc.addStandardLabels" (dict "root" $ "component" "roxie-agent" "name" "roxie" "instance" $name) | indent 8 }} + run: {{ $name | quote}} +{{- if not $roxie.serverReplicas }} + server: {{ $servername | quote }} +{{- end }} + roxie-cluster: {{ $roxie.name | quote }} + accessDali: "yes" + accessEsp: "yes" +<<<<<<< HEAD + helmVersion: 9.0.117-closedown0 +{{- if hasKey $.Values.global "metrics" }} +{{- include "hpcc.generateMetricsReporterLabel" $.Values.global.metrics | nindent 8}} +{{- end }} +{{- if hasKey $roxie "labels" }} +{{ toYaml $roxie.labels | indent 8 }} +{{- end }} +======= + helmVersion: 9.0.117-closedown0 +{{- if hasKey $.Values.global "metrics" }} + {{- include "hpcc.generateMetricsReporterLabel" $.Values.global.metrics | nindent 8}} +{{- end }} +{{- if hasKey $roxie "labels" }} +{{ toYaml $roxie.labels | indent 8 }} +{{- end }} +>>>>>>> origin/candidate-9.6.x + annotations: + checksum/config: {{ $configSHA }} +{{- include "hpcc.generateAnnotations" $commonCtx | indent 8 }} +{{- if hasKey $.Values.global "metrics" }} +{{- include "hpcc.addPrometheusScrapeAnnotations" $.Values.global.metrics | nindent 8 }} +{{- end }} + spec: +{{- include "hpcc.placementsByPodTargetType" (dict "root" $ "pod" $name "target" $roxie.name "type" "roxie") | indent 6 }} + serviceAccountName: "hpcc-default" + initContainers: +{{- include "hpcc.createConfigInitContainers" $commonCtx | indent 6 }} +{{- include "hpcc.addImagePullSecrets" $commonCtx | nindent 6 -}} + terminationGracePeriodSeconds: {{ add ($roxie.agentQueryReleaseDelaySeconds | default 60) 30 }} + containers: + - name: {{ $name | quote}} + workingDir: /var/lib/HPCCSystems + command: [ {{ include "hpcc.componentCommand" (dict "me" $roxie "root" $ "process" "roxie") }} ] + args: [ {{- include "hpcc.componentStartArgs" (dict "me" $roxie "root" $ "process" "roxie") | nindent 16 }} + {{ include "hpcc.configArg" $roxie }}, + {{ include "hpcc.daliArg" (dict "root" $ "component" "Roxie" "optional" false) }}, + "--channels={{ $channel }}", + "--server={{ not $roxie.serverReplicas }}", + ] + env: +{{ include "hpcc.mergeEnvironments" $env | indent 8 -}} + - name: "SENTINEL" + value: "/tmp/{{ $roxie.name }}.sentinel" +{{- if not $roxie.serverReplicas }} +{{- $local := dict "first" true }} +{{- range $service := $roxie.services }} +{{- if ne (int $service.servicePort) 0 }} +{{- if $local.first }} +{{- $_ := set $local "first" false }} + ports: +{{- end }} + - name: {{ $service.name }} + containerPort: {{ $service.servicePort }} +{{- end }} +{{- end }} + lifecycle: + preStop: + exec: + command: ["testsocket", ".", "control:closedown"] +{{ include "hpcc.addSentinelProbes" ( $roxie | merge (dict "readyProbeName" ".ready" )) | indent 8 }} +{{- end }}{{/* not serverReplicas */}} +{{ include "hpcc.addSecurityContext" (dict "root" $ "me" .) | indent 8 }} +{{- include "hpcc.addResources" (dict "me" ($roxie.channelResources | default $roxie.resources) "root" $) | indent 8 }} +{{ include "hpcc.addImageAttrs" $commonCtx | indent 8 }} + volumeMounts: +{{ include "hpcc.addConfigMapVolumeMount" $roxie | indent 8 }} +{{ include "hpcc.addVolumeMounts" $commonCtx | indent 8 }} +{{ include "hpcc.addSecretVolumeMounts" $commonCtx | indent 8 }} +{{ include "hpcc.addVaultClientCertificateVolumeMounts" $commonCtx | indent 8 }} +{{- if not $roxie.serverReplicas }} + +{{ include "hpcc.addCertificateVolumeMount" (dict "root" $ "component" "roxie-agent" "name" $name "external" false) | indent 8 }} +{{ include "hpcc.addCertificateVolumeMount" (dict "root" $ "component" "roxie-agent" "name" $agentPublicCertName "certificate" $roxie.certificate "external" true "includeRemote" true) | indent 8 }} +{{ include "hpcc.addUDPCertificateVolumeMount" (dict "root" $ "component" "udpkey" "name" $udpkeyname ) | indent 8 }} +{{- end }}{{/* not serverReplicas */}} + + volumes: +{{ include "hpcc.addConfigMapVolume" $roxie | indent 6 }} +{{ include "hpcc.addVolumes" $commonCtx | indent 6 }} +{{ include "hpcc.addSecretVolumes" $commonCtx | indent 6 }} +{{ include "hpcc.addVaultClientCertificateVolumes" $commonCtx | indent 6 }} +{{- if not $roxie.serverReplicas }} +{{ include "hpcc.addCertificateVolume" (dict "root" $ "component" "roxie-agent" "name" $name "external" false) | indent 6 }} +{{ include "hpcc.addCertificateVolume" (dict "root" $ "component" "roxie-agent" "name" $agentPublicCertName "certificate" $roxie.certificate "external" true "includeRemote" true) | indent 6 }} +{{ include "hpcc.addUDPCertificateVolume" (dict "root" $ "component" "udpkey" "name" $udpkeyname) | indent 6 }} +{{- end }}{{/* not serverReplicas */}} +--- + +{{- end }} +{{- end }}{{/* if not singlenode */}} +--- +{{ include "hpcc.addEgress" (dict "root" $ "me" $roxie "labels" $commonCtx.instanceNames) }} +{{- if hasKey . "hpa" }} +{{- include "hpcc.addHorizontalPodAutoscaler" (dict "name" $roxie.name "kind" "Deployment" "hpa" $roxie.hpa) }} +{{- end }} +{{- end }}{{/* if not disabled */}} +{{- end }}{{/* range */}} + diff --git a/system/jlib/jptree.cpp b/system/jlib/jptree.cpp index bd8b9acd053..4aed0538292 100644 --- a/system/jlib/jptree.cpp +++ b/system/jlib/jptree.cpp @@ -1369,9 +1369,8 @@ bool PTree::hasProp(const char * xpath) const } else { - IPropertyTreeIterator *iter = getElements(xpath); + Owned iter = getElements(xpath); bool res = iter->first(); - iter->Release(); return res; } } diff --git a/thorlcr/activities/hashdistrib/thhashdistribslave.cpp b/thorlcr/activities/hashdistrib/thhashdistribslave.cpp index 5f543423889..6fcc98f6903 100644 --- a/thorlcr/activities/hashdistrib/thhashdistribslave.cpp +++ b/thorlcr/activities/hashdistrib/thhashdistribslave.cpp @@ -2727,7 +2727,7 @@ class CSpill : implements IRowWriter, public CSimpleInterface prefix.append(bucketN).append('_').append(desc); GetTempFilePath(tempname, prefix.str()); OwnedIFile iFile = createIFile(tempname.str()); - spillFile.setown(new CFileOwner(iFile.getLink(), tempFileSizeTracker)); + spillFile.setown(new CFileOwner(iFile, tempFileSizeTracker)); if (owner.getOptBool(THOROPT_COMPRESS_SPILLS, true)) { rwFlags |= rw_compress; diff --git a/thorlcr/activities/lookupjoin/thlookupjoinslave.cpp b/thorlcr/activities/lookupjoin/thlookupjoinslave.cpp index 47d4835213f..9465afc6a76 100644 --- a/thorlcr/activities/lookupjoin/thlookupjoinslave.cpp +++ b/thorlcr/activities/lookupjoin/thlookupjoinslave.cpp @@ -1878,7 +1878,8 @@ class CLookupJoinActivityBase : public CInMemJoinBase temp = createIFile(tempName.str()); + file.setown(new CFileOwner(temp)); VStringBuffer spillPrefixStr("clearAllNonLocalRows(%d)", SPILL_PRIORITY_SPILLABLE_STREAM); // 3rd param. is skipNulls = true, the row arrays may have had the non-local rows delete already. rows.save(file->queryIFile(), spillCompInfo, true, spillPrefixStr.str()); // saves committed rows @@ -2944,7 +2945,7 @@ class CLookupJoinActivityBase : public CInMemJoinBasegetProp("@query_so_dir", soPath); + StringBuffer soPattern("*."); +#ifdef _WIN32 + soPattern.append("dll"); +#else + soPattern.append("so"); +#endif + querySoCache.init(soPath.str(), DEFAULT_QUERYSO_LIMIT, soPattern); } CJobManager::~CJobManager() @@ -582,16 +592,6 @@ void CJobManager::run() setWuid(NULL); #ifndef _CONTAINERIZED - StringBuffer soPath; - globals->getProp("@query_so_dir", soPath); - StringBuffer soPattern("*."); -#ifdef _WIN32 - soPattern.append("dll"); -#else - soPattern.append("so"); -#endif - querySoCache.init(soPath.str(), DEFAULT_QUERYSO_LIMIT, soPattern); - SCMStringBuffer _queueNames; const char *thorName = globals->queryProp("@name"); if (!thorName) thorName = "thor"; diff --git a/thorlcr/thorutil/thmem.cpp b/thorlcr/thorutil/thmem.cpp index 3677e86647c..0e74d371462 100644 --- a/thorlcr/thorutil/thmem.cpp +++ b/thorlcr/thorutil/thmem.cpp @@ -1659,7 +1659,7 @@ class CThorRowCollectorBase : public CSpillable Owned iFile = createIFile(tempName.str()); VStringBuffer spillPrefixStr("%sRowCollector(%d)", tracingPrefix.str(), spillPriority); spillableRows.save(*iFile, spillCompInfo, false, spillPrefixStr.str()); // saves committed rows - spillFiles.append(new CFileOwner(iFile.getLink())); + spillFiles.append(new CFileOwner(iFile)); ++overflowCount; statOverflowCount.fastAdd(1); // NB: this is total over multiple uses of this class statSizeSpill.fastAdd(iFile->size()); diff --git a/thorlcr/thorutil/thormisc.hpp b/thorlcr/thorutil/thormisc.hpp index 73d57dbea31..2bc8f0c776d 100644 --- a/thorlcr/thorutil/thormisc.hpp +++ b/thorlcr/thorutil/thormisc.hpp @@ -340,7 +340,7 @@ class CFileSizeTracker: public CInterface // simple class which takes ownership of the underlying file and deletes it on destruction class graph_decl CFileOwner : public CSimpleInterface, implements IInterface { - OwnedIFile iFile; + Linked iFile; Linked fileSizeTracker; offset_t fileSize = 0; public: