From c53a237db332e52185cd57bf96ee0c15a7e75e21 Mon Sep 17 00:00:00 2001
From: Anthony Fishbeck <anthony.fishbeck@lexisnexis.com>
Date: Tue, 22 Aug 2023 16:50:38 -0400
Subject: [PATCH] HPCC-30409 Support HashiCorp vault auth using client
 certificates

Signed-off-by: Anthony Fishbeck <anthony.fishbeck@lexisnexis.com>
---
 ... using client cert vault authentication.md | 200 ++++++++++++++++++
 .../secrets/hpcc_vault_ecl_policies.hcl       |   3 +
 .../secrets/hpcc_vault_ecluser_policies.hcl   |   3 +
 .../values-secrets-client-cert-auth.yaml      |  57 +++++
 helm/hpcc/templates/_helpers.tpl              | 122 ++++++++++-
 helm/hpcc/templates/dafilesrv.yaml            |   2 +
 helm/hpcc/templates/dali.yaml                 |   7 +-
 helm/hpcc/templates/dfuserver.yaml            |   2 +
 helm/hpcc/templates/eclagent.yaml             |   5 +-
 helm/hpcc/templates/eclccserver.yaml          |   4 +
 helm/hpcc/templates/eclscheduler.yaml         |   2 +
 helm/hpcc/templates/esp.yaml                  |   2 +
 helm/hpcc/templates/issuers.yaml              |   8 +-
 helm/hpcc/templates/localroxie.yaml           |   2 +
 helm/hpcc/templates/roxie.yaml                |   4 +
 helm/hpcc/templates/sasha.yaml                |   2 +
 helm/hpcc/templates/thor.yaml                 |  10 +
 helm/hpcc/values.yaml                         |  36 +++-
 system/jlib/jsecrets.cpp                      |  91 ++++++--
 19 files changed, 528 insertions(+), 34 deletions(-)
 create mode 100644 helm/examples/secrets/README - vault secrets using client cert vault authentication.md
 create mode 100644 helm/examples/secrets/hpcc_vault_ecl_policies.hcl
 create mode 100644 helm/examples/secrets/hpcc_vault_ecluser_policies.hcl
 create mode 100644 helm/examples/secrets/values-secrets-client-cert-auth.yaml

diff --git a/helm/examples/secrets/README - vault secrets using client cert vault authentication.md b/helm/examples/secrets/README - vault secrets using client cert vault authentication.md
new file mode 100644
index 00000000000..0fb3c5c9592
--- /dev/null
+++ b/helm/examples/secrets/README - vault secrets using client cert vault authentication.md	
@@ -0,0 +1,200 @@
+# Containerized HPCC Systems Vault Secrets using appRole vault authentication
+
+This example demonstrates HPCC use use of Hashicorp Vault secrets using the "TLS certificates auth method".
+
+This example assumes you are starting from a linux command shell in the HPCC-Platform/helm directory.  From there you will find the example files and this README file in the examples/secrets directory.
+
+## Hashicorp Vault support:
+
+This example uses Hashicorp vault.  The following steps can be used to set up a development mode only instance of vault just for the purposes of this example.  This makes it easy to test out vault functionality without going through the much more extensive configuration process for a production ready vault installation.
+
+## Install hashicorp vault command line client on your local system:
+
+https://learn.hashicorp.com/tutorials/vault/getting-started-install
+
+--------------------------------------------------------------------------------------------------------
+
+## Install cert-manager
+
+Install cert-manager support following the example found at examples/certmanager.
+
+With the addition of the following step:
+
+Repeat the root certificate creation process from the cert-manager setup to create the CA secret for the vaultclient issuer.
+
+```bash
+openssl req -x509 -newkey rsa:2048 -nodes -keyout vaultclientca.key -sha256 -days 1825 -out vaultclientca.crt -config examples/certmanager/ca-req.cfg
+```
+
+## Create a Kubernetes TLS secret from the generated signing root certificate and privatekey
+
+```bash
+kubectl create secret tls hpcc-vaultclient-issuer-key-pair --cert=vaultclientca.crt --key=vaultclientca.key
+```
+
+
+## Install hashicorp vault service in standalone mode with tls:
+
+Install vault server with tls support.  This is currently outside the scope of this document.
+
+
+## Setting up vault
+
+Tell the vault command line application the server location (dev mode is http, default location is https)
+
+```bash
+export VAULT_ADDR=https://127.0.0.1:8200
+```
+
+In a separate terminal window start vault port forwarding.
+
+```bash
+kubectl port-forward vault-0 8200:8200
+```
+
+Login to the vault command line using the vault root token (development mode defaults to "root"):
+
+```bash
+vault login <token>
+```
+
+If you don't provide the token on the command line you will be prompted to input the value and it will be hidden from view.
+
+
+## Configure vault tls cert auth
+
+Enabling appRole auth will allow access the vault via the appRole authentication protocol.
+
+```bash
+vault auth enable cert
+```
+
+Setup vault auth policy granting access to the ecl secrets locations we plan to use:
+
+```bash
+vault policy write hpcc-ecl-ro examples/secrets/hpcc_vault_ecl_policies.hcl
+```
+
+Setup hpcc-vault-access auth role:
+
+```bash
+vault write auth/cert/certs/hpcc-ecl \
+    display_name=hpcc-ecl-cert \
+    policies=hpcc-ecl-ro \
+    certificate=@vaultclientca.crt \
+    allowed_common_names=ecl.vaultclient.hpcc.example.com \
+    ttl=3600
+```
+
+Setup vault auth policy granting access to the eclUser secrets locations we plan to use:
+
+```bash
+vault policy write hpcc-ecluser-ro examples/secrets/hpcc_vault_ecluser_policies.hcl
+```
+
+Setup hpcc-vault-access auth role:
+
+```bash
+vault write auth/cert/certs/hpcc-ecluser \
+    display_name=hpcc-ecluser-cert \
+    policies=hpcc-ecluser-ro \
+    certificate=@vaultclientca.crt \
+    allowed_common_names=ecluser.vaultclient.hpcc.example.com \
+    ttl=3600
+```
+
+
+## 'eclUser' category secrets
+
+Create example vault 'eclUser' secrets:
+
+```bash
+vault kv put secret/eclUser/vault-example crypt.key=@examples/secrets/crypt.key
+```
+
+## 'ecl' category secrets
+
+Secrets in the 'ecl' category are not accessible by ECL code directly and therefore not visible to ECL users.  They can be used by internal ECL feartures
+and commands.  For example:
+
+## HTTP-CONNECT Secrets:
+
+This example focuses on ECL secrets to provide HTTP connection strings and credentials for ECL SOAPCALL and HTTPCALL commands.
+
+These secrets are prefixed with the string "http-connect-" requiring this prefix ensures that HTTPCALL/SOAPCALL only accesses secrets which are intended for this use.
+
+HTTP-CONNECT secrets consist of a url string and optional additional secrets associated with that URL.  Requiring the url to be part of the secret prevents credentials from being easily hijacked via an HTTPCALL to an arbitrary location.  Instead the credentials are explicitly associated with the provided url.
+
+Besides the URL values can currently be set for proxy (trusted for keeping these secrets), username, and password.
+
+## Creating the HTTP-CONNECT Secrets
+
+## Create example vault secret:
+
+Create example vault 'ecl' secrets:
+
+```bash
+vault kv put secret/ecl/http-connect-vaultsecret url=@examples/secrets/url-basic username=@examples/secrets/username password=@examples/secrets/password
+```
+
+## Installing the HPCC with the secrets added to ECL components
+
+Install the HPCC helm chart with the secrets just defined added to all components that run ECL.
+
+```bash
+helm install myhpcc hpcc/ -f examples/secrets/values-secrets-client-cert-auth.yaml
+```
+
+Use kubectl to check the status of the deployed pods.  Wait until all pods are running before continuing.
+
+```bash
+kubectl get pods
+```
+--------------------------------------------------------------------------------------------------------
+
+If you don't already have the HPCC client tools installed please install them now:
+
+https://hpccsystems.com/download#HPCC-Platform
+
+
+## Using the created 'eclUser' category secrets directly in ECL code
+
+The following ecl command will run the example ECL file that demonstrates accessing a vault secret directly from ECL code.
+
+```bash
+ecl run hthor examples/secrets/crypto_vault_secret.ecl
+```
+
+The expected result would be:
+
+```xml
+<Result>
+<Dataset name='vault_message'>
+ <Row><vault_message>For your eyes only</vault_message></Row>
+</Dataset>
+</Result>
+```
+
+## Using the created 'ecl' category secrets via HTTPCALL from within ECL code
+
+If you don't already have the HPCC client tools installed please install them now:
+
+https://hpccsystems.com/download#HPCC-Platform
+
+--------------------------------------------------------------------------------------------------------
+
+The following ecl command will run the example ECL file that demonstrates an HTTPCALL that uses a vault secret for connection and  authentication.
+
+```bash
+ecl run hthor examples/secrets/httpcall_vault.ecl
+```
+
+For each job the expected result would be:
+
+```xml
+<Result>
+<Dataset name='Result 1'>
+ <Row><authenticated>true</authenticated></Row>
+</Dataset>
+</Result>
+```
diff --git a/helm/examples/secrets/hpcc_vault_ecl_policies.hcl b/helm/examples/secrets/hpcc_vault_ecl_policies.hcl
new file mode 100644
index 00000000000..1ff93b48113
--- /dev/null
+++ b/helm/examples/secrets/hpcc_vault_ecl_policies.hcl
@@ -0,0 +1,3 @@
+path "secret/data/ecl/*" {
+    capabilities = ["read"]
+}
diff --git a/helm/examples/secrets/hpcc_vault_ecluser_policies.hcl b/helm/examples/secrets/hpcc_vault_ecluser_policies.hcl
new file mode 100644
index 00000000000..c985e8754bb
--- /dev/null
+++ b/helm/examples/secrets/hpcc_vault_ecluser_policies.hcl
@@ -0,0 +1,3 @@
+path "secret/data/eclUser/*" {
+    capabilities = ["read"]
+}
diff --git a/helm/examples/secrets/values-secrets-client-cert-auth.yaml b/helm/examples/secrets/values-secrets-client-cert-auth.yaml
new file mode 100644
index 00000000000..4de36abdac3
--- /dev/null
+++ b/helm/examples/secrets/values-secrets-client-cert-auth.yaml
@@ -0,0 +1,57 @@
+# Overrides for http-connect secrets in hpcc.
+# NB: The "hpcc-connect-testsecret" should be created before installing the helm chart.
+
+certificates:
+  enabled: true
+  issuers:
+    local:
+      name: hpcc-local-issuer
+      ## kind can be changed to ClusterIssue to refer to a ClusterIssuer. https://cert-manager.io/docs/reference/api-docs/#cert-manager.io/v1.ClusterIssuer
+      kind: Issuer
+      ## do not define spec (set spec: null), to reference an Issuer resource that already exists in the cluster
+      ## change spec if you'd like to change how certificates get issued... see ## https://cert-manager.io/docs/configuration/#supported-issuer-types
+      ## for information on what spec should contain.
+      spec:
+        ca:
+          secretName: hpcc-local-issuer-key-pair
+    public:
+      name: hpcc-public-issuer
+      ## kind can be changed to ClusterIssue to refer to a ClusterIssuer. https://cert-manager.io/docs/reference/api-docs/#cert-manager.io/v1.ClusterIssuer
+      kind: Issuer
+      ## do not define spec (set spec: null), to reference an Issuer resource that already exists in the cluster
+      ## change spec if you'd like to change how certificates get issued... see ## https://cert-manager.io/docs/configuration/#supported-issuer-types
+      ## for information on what spec should contain.
+      spec:
+        selfSigned: {}
+    vaultclient:
+      name: hpcc-vault-issuer
+      enabled: true
+      domain: hpcc.example.com
+      rolePrefix: "hpcc-"
+      kind: Issuer
+      spec:
+        ca:
+          secretName: hpcc-vaultclient-issuer-key-pair
+
+    signing: # intended to be used for signing/verification purposes only, e.g. by dafilesrv
+      enabled: false
+      name: hpcc-signing-issuer
+      ## kind can be changed to ClusterIssue to refer to a ClusterIssuer. https://cert-manager.io/docs/reference/api-docs/#cert-manager.io/v1.ClusterIssuer
+      kind: Issuer
+      ## do not define spec (set spec: null), to reference an Issuer resource that already exists in the cluster
+      ## change spec if you'd like to change how certificates get issued... see ## https://cert-manager.io/docs/configuration/#supported-issuer-types
+      ## for information on what spec should contain.
+      spec:
+        ca:
+          secretName: hpcc-signing-issuer-key-pair
+
+vaults:
+  ecl:
+    - name: my-ecl-vault
+      url: https://vault.vaultns:8200/v1/secret/data/ecl/${secret}
+      kind: kv-v2
+  eclUser:
+    - name: my-eclUser-vault
+      url: https://vault.vaultns:8200/v1/secret/data/eclUser/${secret}
+      kind: kv-v2
+
diff --git a/helm/hpcc/templates/_helpers.tpl b/helm/hpcc/templates/_helpers.tpl
index 511ea27f3f4..5d9b8eeed88 100644
--- a/helm/hpcc/templates/_helpers.tpl
+++ b/helm/hpcc/templates/_helpers.tpl
@@ -599,7 +599,9 @@ readinessProbe:
 Generate vault info
 */}}
 {{- define "hpcc.generateVaultConfig" -}}
+{{- $root := .root -}}
 {{- $secretsCategories := .secretsCategories -}}
+{{- $vaultClientIssuerEnabled := eq (include "hpcc.isIssuerEnabled" (dict "root" .root "issuerKeyName" "vaultclient")) "true" -}}
 vaults:
 {{- range  $categoryname, $category := .root.Values.vaults -}}
  {{- if (has $categoryname $secretsCategories) }}
@@ -623,6 +625,21 @@ vaults:
     {{- if index $vault "appRoleSecret" }}
       appRoleSecret: {{ index $vault "appRoleSecret" }}
     {{- end -}}
+    {{- if $vaultClientIssuerEnabled }}
+     {{- if not (index $vault "client-secret") }}
+      {{- if not (index $vault "appRoleId") }}
+      useTLSCertificateAuth: true
+       {{- $issuer := $root.Values.certificates.issuers.vaultclient }}
+       {{- if index $vault "authRole" }}
+      role: {{ $vault.authRole }}
+       {{- else if index $issuer "rolePrefix" }}
+      role: {{ (printf "%s%s" $issuer.rolePrefix (lower $categoryname)) | quote }}
+       {{- else }}
+      role: {{ (printf "hpcc-%s" (lower $categoryname)) | quote }}
+       {{- end }}
+      {{- end }}
+     {{- end }}
+    {{- end }}
     {{- if (hasKey $vault "retries") }}
       retries: {{ $vault.retries }}
     {{- end }}
@@ -1959,7 +1976,110 @@ spec:
 {{- end }}
 
 {{/*
-Experimental: Use certmanager to generate a key for roxie udp encryption.
+*/}}
+
+{{/* When the cert-manager vaultclient issuer is enabled, a client certificate will be issued for each secret category.
+     These will be used to authenticate against a vault with granular category specific permissions.
+     Pass in root and category
+*/}}
+{{- define "hpcc.addVaultClientCertificate" }}
+ {{- if (.root.Values.certificates | default dict).enabled -}}
+  {{- $issuerKeyName := "vaultclient" -}}
+  {{- if eq (include "hpcc.isIssuerEnabled" (dict "root" .root "issuerKeyName" $issuerKeyName)) "true" -}}
+   {{- $issuer := get .root.Values.certificates.issuers $issuerKeyName -}}
+   {{- if not $issuer -}}
+    {{- $_ := fail (printf "Issuer %s for vault access client certificates not found." $issuerKeyName) -}}
+   {{- else -}}
+    {{- if not $issuer.enabled -}}
+     {{- $_ := fail (printf "Issuer %s for vault access client certificates not enabled." $issuerKeyName) -}}
+    {{- end }}
+    {{- if not $issuer.domain -}}
+     {{- $_ := fail (printf "Domain required for Issuer %s for vault access client certificates." $issuerKeyName) -}}
+    {{- end }}
+    {{- $namespace := .root.Release.Namespace -}}
+    {{- $category := .category -}}
+    {{- $secretTemplate := $issuer.secretTemplate -}}
+
+apiVersion: cert-manager.io/v1
+kind: Certificate
+metadata:
+  name: vaultclient-{{ lower $category }}-cert
+  namespace: {{ $namespace }}
+spec:
+  secretName: vaultclient-{{ lower $category }}-tls
+    {{- if $secretTemplate }}
+  secretTemplate:
+{{ toYaml $secretTemplate | indent 4 }}
+    {{- end }}
+  duration: 2160h # 90d
+  renewBefore: 360h # 15d
+  subject:
+    organizations:
+    - HPCC Vault Client
+  commonName: {{ $category }}.vaultclient.{{ $issuer.domain }}
+  isCA: false
+  privateKey:
+    algorithm: RSA
+    encoding: PKCS1
+    size: 2048
+  usages:
+    - client auth
+  issuerRef:
+    name: {{ $issuer.name }}
+    kind: {{ $issuer.kind }}
+    group: cert-manager.io
+---
+   {{- end }}
+  {{- end }}
+ {{- end }}
+{{- end }}
+
+
+{{/* When the cert-manager vaultclient issuer is enabled, generates the volume mounts for vault client certificate secrets.
+     Secrets are mounted for each category supported by the current component.
+     Pass in root and secretsCategories
+*/}}
+{{- define "hpcc.addVaultClientCertificateVolumeMounts" -}}
+ {{- if (.root.Values.certificates | default dict).enabled -}}
+  {{- $issuerKeyName := "vaultclient" -}}
+  {{- if eq (include "hpcc.isIssuerEnabled" (dict "root" .root "issuerKeyName" $issuerKeyName)) "true" -}}
+   {{- $issuer := get .root.Values.certificates.issuers $issuerKeyName -}}
+   {{- if $issuer -}}
+    {{- if $issuer.enabled -}}
+     {{- range $category := .secretsCategories }}
+- name: certificate-{{ $issuerKeyName }}-{{ lower $category }}
+  mountPath: /opt/HPCCSystems/secrets/certificates/{{ $issuerKeyName }}/{{ lower $category }}
+     {{- end -}}
+    {{- end -}}
+   {{ end -}}
+  {{- end -}}
+ {{- end -}}
+{{- end -}}
+
+{{/* When the cert-manager vaultclient issuer is enabled, generates the volumes for vault client certificate secrets.
+     Secrets are mounted for each category supported by the current component.
+     Pass in root and secretsCategories
+*/}}
+{{- define "hpcc.addVaultClientCertificateVolumes" -}}
+ {{- if (.root.Values.certificates | default dict).enabled -}}
+  {{- $issuerKeyName := "vaultclient" -}}
+  {{- if eq (include "hpcc.isIssuerEnabled" (dict "root" .root "issuerKeyName" $issuerKeyName)) "true" -}}
+   {{- $issuer := get .root.Values.certificates.issuers $issuerKeyName -}}
+   {{- if $issuer -}}
+    {{- if $issuer.enabled -}}
+     {{- range $category := .secretsCategories }}
+- name: certificate-{{ $issuerKeyName }}-{{ lower $category }}
+  secret:
+    secretName: {{ $issuerKeyName }}-{{ lower $category }}-tls
+     {{- end -}}
+    {{- end -}}
+   {{ end -}}
+  {{- end -}}
+ {{- end -}}
+{{- end -}}
+
+{{/*
+Use certmanager to generate a key for roxie udp encryption.
 A public certificate and private key are generated under /opt/HPCCSystems/secrets/certificates/udp.
 Current udp encryption design would only use the private key.
 Key is in pem format and the private key would need to be extracted.
diff --git a/helm/hpcc/templates/dafilesrv.yaml b/helm/hpcc/templates/dafilesrv.yaml
index 5d6f737b6d4..704ee7e581d 100644
--- a/helm/hpcc/templates/dafilesrv.yaml
+++ b/helm/hpcc/templates/dafilesrv.yaml
@@ -77,6 +77,7 @@ spec:
         volumeMounts:
 {{ include "hpcc.addConfigMapVolumeMount" . | indent 8 }}
 {{ include "hpcc.addVolumeMounts" $commonCtx | indent 8 }}
+{{ include "hpcc.addVaultClientCertificateVolumeMounts" $commonCtx | indent 8 }}
 {{- if $commonCtx.certificatesEnabled }}
 {{ include "hpcc.addCertificateVolumeMount" (dict "root" $ "name" .name "component" "dafilesrv" "certificate" .certificate "visibility" .service.visibility) | indent 8 }}
 {{- else }}
@@ -87,6 +88,7 @@ spec:
       volumes:
 {{ include "hpcc.addConfigMapVolume" . | indent 6 }}
 {{ include "hpcc.addVolumes" $commonCtx | indent 6 }}
+{{ include "hpcc.addVaultClientCertificateVolumes" $commonCtx | indent 6 }}
 {{- if $commonCtx.certificatesEnabled }}
 {{ include "hpcc.addCertificateVolume" (dict "root" $ "name" .name "component" "dafilesrv" "certificate" .certificate "visibility" .service.visibility) | indent 6 }}
 {{- end }}
diff --git a/helm/hpcc/templates/dali.yaml b/helm/hpcc/templates/dali.yaml
index c63af7b3e9d..b396c8687e6 100644
--- a/helm/hpcc/templates/dali.yaml
+++ b/helm/hpcc/templates/dali.yaml
@@ -123,7 +123,8 @@ spec:
 {{ include "hpcc.addConfigMapVolumeMount" $dali | indent 8 }}
 {{ include "hpcc.addVolumeMounts" $commonCtx | indent 8 }}
 {{- include "hpcc.addSecretVolumeMounts" (dict "root" $ "secretsCategories" $daliSecretsCategories) | nindent 8 }}
-{{- include "hpcc.addCertificateVolumeMount" (dict "root" $ "name" .name "component" "dali" "external" false) | nindent 8 }}
+{{ include "hpcc.addVaultClientCertificateVolumeMounts" (dict "root" $ "secretsCategories" $daliSecretsCategories) | indent 8 }}
+{{ include "hpcc.addCertificateVolumeMount" (dict "root" $ "name" .name "component" "dali" "external" false) | nindent 8 }}
 {{- range $sashaName, $_sasha := $daliSashaServicesCtx.services -}}
  {{- $sasha := deepCopy ($_sasha | default dict) -}}
  {{- $_ := set $sasha "name" $sashaName -}}
@@ -143,7 +144,8 @@ spec:
   {{- $sashaPlaneCategories := splitList " " (include "hpcc.getSashaPlanesFromAccess" .) -}}
   {{- include "hpcc.addVolumeMounts" (dict "root" $ "includeCategories" $sashaPlaneCategories "includeNames" (list $thisServiceCtx.sashaStoragePlane)) | indent 8 }}
   {{- include "hpcc.addSecretVolumeMounts" (dict "root" $ "secretsCategories" $sashaServiceSecretsCategories) | nindent 8 }}
-  {{- include "hpcc.addCertificateVolumeMount" (dict "root" $ "name" $dali.name "component" "dali" "external" false) | nindent 8 }}
+  {{ include "hpcc.addVaultClientCertificateVolumeMounts" (dict "root" $ "secretsCategories" $sashaServiceSecretsCategories) | indent 8 }}
+  {{ include "hpcc.addCertificateVolumeMount" (dict "root" $ "name" $dali.name "component" "dali" "external" false) | nindent 8 }}
  {{- end }}
 {{- end }}
       volumes:
@@ -157,6 +159,7 @@ spec:
 {{- end -}}
 {{- include "hpcc.addVolumes" (dict "root" $ "me" $dali "includeNames" $tmpDaliScope.aggregateSashaNamedPlanes "includeCategories" $tmpDaliScope.aggregatePlaneCategories) | indent 6 }}
 {{- include "hpcc.addSecretVolumes" (dict "root" $ "secretsCategories" $tmpDaliScope.aggregateSashaSecretsCategories) | indent 6 }}
+{{ include "hpcc.addVaultClientCertificateVolumes" (dict "root" $ "secretsCategories" $tmpDaliScope.aggregateSashaSecretsCategories) | indent 6 }}
 {{ include "hpcc.addCertificateVolume" (dict "root" $ "name" .name "component" "dali" "external" false) | indent 6 }}
 ---
 {{- $daliStoragePlane := $dali.daliPlane | default (include "hpcc.getFirstPlaneForCategory" (dict "root" $ "category" "dali")) }}
diff --git a/helm/hpcc/templates/dfuserver.yaml b/helm/hpcc/templates/dfuserver.yaml
index 6493f65a2ed..13d0601c081 100644
--- a/helm/hpcc/templates/dfuserver.yaml
+++ b/helm/hpcc/templates/dfuserver.yaml
@@ -92,11 +92,13 @@ spec:
 {{ include "hpcc.addConfigMapVolumeMount" . | indent 8 }}
 {{ include "hpcc.addCertificateVolumeMount" (dict "root" $ "name" .name "component" "dfuserver" "external" false) | indent 8 }}
 {{- include "hpcc.addSecretVolumeMounts" $commonCtx | nindent 8 }}
+{{ include "hpcc.addVaultClientCertificateVolumeMounts" $commonCtx | indent 8 }}
       volumes:
 {{ include "hpcc.addConfigMapVolume" . | indent 6 }}
 {{- include "hpcc.addVolumes" $commonCtx | indent 6 }}
 {{ include "hpcc.addCertificateVolume" (dict "root" $ "name" .name "component" "dfuserver" "external" false) | indent 6 }}
 {{ include "hpcc.addSecretVolumes" $commonCtx | indent 6 }}
+{{ include "hpcc.addVaultClientCertificateVolumes" $commonCtx | indent 6 }}
 ---
 {{- if hasKey . "hpa" }}
  {{- include "hpcc.addHorizontalPodAutoscaler" (dict "name" .name "kind" "Deployment" "hpa" .hpa) }}
diff --git a/helm/hpcc/templates/eclagent.yaml b/helm/hpcc/templates/eclagent.yaml
index 5f6b2016f1a..11bbe270ebe 100644
--- a/helm/hpcc/templates/eclagent.yaml
+++ b/helm/hpcc/templates/eclagent.yaml
@@ -92,6 +92,7 @@ data:
 {{ include "hpcc.addConfigMapVolumeMount" .me | indent 12 }}
 {{ include "hpcc.addVolumeMounts" . | indent 12 }}
 {{ include "hpcc.addSecretVolumeMounts" . | indent 12 }}
+{{ include "hpcc.addVaultClientCertificateVolumeMounts" . | indent 12 }}
 {{ include "hpcc.addCertificateVolumeMount" (dict "root" .root "name" .me.name "component" $apptype "includeRemote" true) | indent 12 }}
 {{- if $misc.postJobCommandViaSidecar }}
 {{ include "hpcc.addWaitAndRunVolumeMount" . | indent 12 }}
@@ -100,6 +101,7 @@ data:
 {{ include "hpcc.addConfigMapVolume" .me | indent 10 }}
 {{ include "hpcc.addVolumes" . | indent 10 }}
 {{ include "hpcc.addSecretVolumes" . | indent 10 }}
+{{ include "hpcc.addVaultClientCertificateVolumes" . | indent 10 }}
 {{ include "hpcc.addCertificateVolume" (dict "root" .root "name" .me.name "component" $apptype "includeRemote" true) | indent 10 }}
 {{- if $misc.postJobCommandViaSidecar }}
 {{ include "hpcc.addWaitAndRunVolume" . | indent 10 }}
@@ -174,13 +176,14 @@ spec:
 {{ include "hpcc.addConfigMapVolumeMount" . | indent 8 }}
 {{ include "hpcc.addVolumeMounts" $commonCtx | indent 8 }}
 {{ include "hpcc.addSecretVolumeMounts" $commonCtx | indent 8 }}
+{{ include "hpcc.addVaultClientCertificateVolumeMounts" $commonCtx | indent 8 }}
 {{ include "hpcc.addCertificateVolumeMount" (dict "root" $ "name" .name "component" $apptype "includeRemote" true) | indent 8 }}
       volumes:
 {{ include "hpcc.addConfigMapVolume" . | indent 6 }}
 {{ include "hpcc.addVolumes" $commonCtx | indent 6 }}
 {{ include "hpcc.addSecretVolumes" $commonCtx | indent 6 }}
+{{ include "hpcc.addVaultClientCertificateVolumes" $commonCtx | indent 6 }}
 {{ include "hpcc.addCertificateVolume" (dict "root" $ "name" .name "component" $apptype "includeRemote" true) | indent 6 }}
-
 ---
 {{- if and (hasKey . "hpa") .useChildProcesses }}
  {{- include "hpcc.addHorizontalPodAutoscaler" (dict "name" .name "kind" "Deployment" "hpa" .hpa) }}
diff --git a/helm/hpcc/templates/eclccserver.yaml b/helm/hpcc/templates/eclccserver.yaml
index 0ffece8ac94..0e4b4264021 100644
--- a/helm/hpcc/templates/eclccserver.yaml
+++ b/helm/hpcc/templates/eclccserver.yaml
@@ -91,6 +91,7 @@ data:
 {{ include "hpcc.addConfigMapVolumeMount" .me | indent 12 }}
 {{ include "hpcc.addVolumeMounts" . | indent 12 }}
 {{ include "hpcc.addSecretVolumeMounts" . | indent 12 }}
+{{ include "hpcc.addVaultClientCertificateVolumeMounts" . | indent 12 }}
 {{ include "hpcc.addCertificateVolumeMount" (dict "root" .root "name" .me.name "component" "compile" "includeRemote" true) | indent 12 }}
 {{- if $misc.postJobCommandViaSidecar }}
 {{ include "hpcc.addWaitAndRunVolumeMount" . | indent 12 }}
@@ -99,6 +100,7 @@ data:
 {{ include "hpcc.addConfigMapVolume" .me | indent 10 }}
 {{ include "hpcc.addVolumes" . | indent 10 }}
 {{ include "hpcc.addSecretVolumes" . | indent 10 }}
+{{ include "hpcc.addVaultClientCertificateVolumes" . | indent 10 }}
 {{ include "hpcc.addCertificateVolume" (dict "root" .root "name" .me.name "component" "compile" "includeRemote" true) | indent 10 }}
 {{- if $misc.postJobCommandViaSidecar }}
 {{ include "hpcc.addWaitAndRunVolume" . | indent 10 }}
@@ -183,6 +185,7 @@ spec:
 {{ include "hpcc.addConfigMapVolumeMount" . | indent 8 }}
 {{ include "hpcc.addVolumeMounts" $commonCtx | indent 8 -}}
 {{ include "hpcc.addSecretVolumeMounts" $commonCtx | indent 8 }}
+{{ include "hpcc.addVaultClientCertificateVolumeMounts" $commonCtx | indent 8 }}
 {{ include "hpcc.addCertificateVolumeMount" (dict "root" $ "name" .name "component" "eclccserver" "includeRemote" true) | indent 8 }}
         - name: "hpccbundles"
           mountPath: "/home/hpcc/.HPCCSystems"
@@ -190,6 +193,7 @@ spec:
 {{ include "hpcc.addConfigMapVolume" . | indent 6 }}
 {{ include "hpcc.addVolumes" $commonCtx | indent 6 }}
 {{ include "hpcc.addSecretVolumes" $commonCtx | indent 6 }}
+{{ include "hpcc.addVaultClientCertificateVolumes" $commonCtx | indent 6 }}
 {{ include "hpcc.addCertificateVolume" (dict "root" $ "name" .name "component" "eclccserver" "includeRemote" true) | indent 6 }}
       - name: hpccbundles
         emptyDir: {}
diff --git a/helm/hpcc/templates/eclscheduler.yaml b/helm/hpcc/templates/eclscheduler.yaml
index 714d25b1f88..7a75470931a 100644
--- a/helm/hpcc/templates/eclscheduler.yaml
+++ b/helm/hpcc/templates/eclscheduler.yaml
@@ -101,10 +101,12 @@ spec:
         volumeMounts:
 {{ include "hpcc.addConfigMapVolumeMount" . | indent 8 }}
 {{ include "hpcc.addSecretVolumeMounts" $commonCtx | indent 8 }}
+{{ include "hpcc.addVaultClientCertificateVolumeMounts" $commonCtx | indent 8 }}
 {{ include "hpcc.addCertificateVolumeMount" (dict "root" $ "name" .name "component" "eclscheduler") | indent 8 }}
       volumes:
 {{ include "hpcc.addConfigMapVolume" . | indent 6 }}
 {{ include "hpcc.addSecretVolumes" $commonCtx | indent 6 }}
+{{ include "hpcc.addVaultClientCertificateVolumes" $commonCtx | indent 6 }}
 {{ include "hpcc.addCertificateVolume" (dict "root" $ "name" .name "component" "eclscheduler") | indent 6 }}
 ---
 kind: ConfigMap
diff --git a/helm/hpcc/templates/esp.yaml b/helm/hpcc/templates/esp.yaml
index f2a0ff0cd7f..ce0f40a5063 100644
--- a/helm/hpcc/templates/esp.yaml
+++ b/helm/hpcc/templates/esp.yaml
@@ -158,6 +158,7 @@ spec:
 {{ include "hpcc.addConfigMapVolumeMount" . | indent 8 }}
 {{ include "hpcc.addVolumeMounts" $commonCtx | indent 8 }}
 {{ include "hpcc.addSecretVolumeMounts" $commonCtx | indent 8 }}
+{{ include "hpcc.addVaultClientCertificateVolumeMounts" $commonCtx | indent 8 }}
 {{ include "hpcc.addCertificateVolumeMount" (dict "root" $ "component" $application "name" .name "certificate" .certificate "visibility" .service.visibility "remoteClients" .remoteClients "trustClients" .trustClients "includeRemote" true) | indent 8 }}
 {{- if $commonCtx.externalCert }}
  {{- include "hpcc.addCertificateVolumeMount" (dict "root" $ "component" $application "name" .name "external" false) | nindent 8 }}
@@ -169,6 +170,7 @@ spec:
 {{ include "hpcc.addConfigMapVolume" . | indent 6 }}
 {{ include "hpcc.addVolumes" $commonCtx | indent 6 }}
 {{ include "hpcc.addSecretVolumes" $commonCtx | indent 6 }}
+{{ include "hpcc.addVaultClientCertificateVolumes" $commonCtx | indent 6 }}
 {{ include "hpcc.addCertificateVolume" (dict "root" $ "component" $application "name" .name "certificate" .certificate "visibility" .service.visibility "remoteClients" .remoteClients "trustClients" .trustClients "includeRemote" true) | indent 6 }}
 {{- if $commonCtx.externalCert }}
  {{- include "hpcc.addCertificateVolume" (dict "root" $ "component" $application "name" .name "external" false) | nindent 6 }}
diff --git a/helm/hpcc/templates/issuers.yaml b/helm/hpcc/templates/issuers.yaml
index c57ecad00b4..2cb9576fb25 100644
--- a/helm/hpcc/templates/issuers.yaml
+++ b/helm/hpcc/templates/issuers.yaml
@@ -132,8 +132,14 @@ spec:
     {{- $_ := fail (printf "\n\nEnabling certificate generation requires cert-manager resources. Please intall cert-manager. To disable mTLS security set \"certificates.enabled=false\". To bypass this validation check set \"global.noResourceValidation=true\". \n" ) -}}
    {{- end -}}
   {{- end -}}
-  {{- range $k, $v := .Values.certificates.issuers }}
+  {{- if hasKey $.Values.certificates "issuers" -}}
+   {{- range $k, $v := .Values.certificates.issuers }}
 {{- include "hpcc.addIssuer" (dict "root" $ "issuerKeyName" $k "me" $v ) }}
+   {{- end }}
+   {{- $categories := list "system" "storage" "esp" "codeSign" "codeVerify" "authn" "eclUser" "ecl" "git" -}}
+   {{- range $category := $categories }}
+    {{ include "hpcc.addVaultClientCertificate" (dict "root" $ "category" $category) }}
+   {{- end }}
   {{- end }}
  {{- end }}
 {{- end }}
diff --git a/helm/hpcc/templates/localroxie.yaml b/helm/hpcc/templates/localroxie.yaml
index 5c52a2f327a..e26e3102f9e 100644
--- a/helm/hpcc/templates/localroxie.yaml
+++ b/helm/hpcc/templates/localroxie.yaml
@@ -119,6 +119,7 @@ spec:
 {{ include "hpcc.addConfigMapVolumeMount" . | indent 8 }}
 {{ include "hpcc.addVolumeMounts" $commonCtx | indent 8 }}
 {{- include "hpcc.addSecretVolumeMounts" $commonCtx | indent 8 }}
+{{ include "hpcc.addVaultClientCertificateVolumeMounts" $commonCtx | indent 8 }}
 {{ include "hpcc.addCertificateVolumeMount" (dict "root" $ "name" $roxie.name "component" "localroxie" "external" false) | indent 8 }}
 {{ include "hpcc.addCertificateVolumeMount" (dict "root" $ "name" $roxie.name "component" "localroxie" "external" true "includeRemote" true) | indent 8 }}
 {{ include "hpcc.addUDPCertificateVolumeMount" (dict "root" $ "name" $roxie.name "component" "localudpkey" ) | indent 8 }}
@@ -126,6 +127,7 @@ spec:
 {{ include "hpcc.addConfigMapVolume" . | indent 6 }}
 {{ include "hpcc.addVolumes" $commonCtx | indent 6 }}
 {{ include "hpcc.addSecretVolumes" $commonCtx | indent 6 }}
+{{ include "hpcc.addVaultClientCertificateVolumes" $commonCtx | indent 6 }}
 {{ include "hpcc.addCertificateVolume" (dict "root" $ "name" $roxie.name "component" "localroxie" "external" false) | indent 6 }}
 {{ include "hpcc.addCertificateVolume" (dict "root" $ "name" $roxie.name "component" "localroxie" "external" true "includeRemote" true) | indent 6 }}
 {{ include "hpcc.addUDPCertificateVolume" (dict "root" $ "name" $roxie.name "component" "localudpkey" ) | indent 6 }}
diff --git a/helm/hpcc/templates/roxie.yaml b/helm/hpcc/templates/roxie.yaml
index b4abfa4a1df..67291a97188 100644
--- a/helm/hpcc/templates/roxie.yaml
+++ b/helm/hpcc/templates/roxie.yaml
@@ -303,6 +303,7 @@ spec:
 {{ include "hpcc.addConfigMapVolumeMount" $roxie | indent 8 }}
 {{ include "hpcc.addVolumeMounts" $commonCtx | indent 8 }}
 {{ include "hpcc.addSecretVolumeMounts" $commonCtx | indent 8 }}
+{{ include "hpcc.addVaultClientCertificateVolumeMounts" $commonCtx | indent 8 }}
 {{ include "hpcc.addCertificateVolumeMount" (dict "root" $ "component" "roxie-server" "name" $servername "external" false) | indent 8 }}
 {{ include "hpcc.addCertificateVolumeMount" (dict "root" $ "component" "roxie-server" "name" $servername "certificate" $roxie.certificate "external" true "includeRemote" true) | indent 8 }}
 {{ include "hpcc.addUDPCertificateVolumeMount" (dict "root" $ "component" "udpkey" "name" $udpkeyname ) | indent 8 }}
@@ -310,6 +311,7 @@ spec:
 {{ include "hpcc.addConfigMapVolume" $roxie | indent 6 }}
 {{ include "hpcc.addVolumes" $commonCtx | indent 6 }}
 {{ include "hpcc.addSecretVolumes" $commonCtx | indent 6 }}
+{{ include "hpcc.addVaultClientCertificateVolumes" $commonCtx | indent 6 }}
 {{ include "hpcc.addCertificateVolume" (dict "root" $ "component" "roxie-server" "name" $servername "external" false) | indent 6 }}
 {{ include "hpcc.addCertificateVolume" (dict "root" $ "component" "roxie-server" "name" $servername "certificate" $roxie.certificate "external" true "includeRemote" true) | indent 6 }}
 {{ include "hpcc.addUDPCertificateVolume" (dict "root" $ "component" "udpkey" "name" $udpkeyname) | indent 6 }}
@@ -411,6 +413,7 @@ spec:
 {{ include "hpcc.addConfigMapVolumeMount" $roxie | indent 8 }}
 {{ include "hpcc.addVolumeMounts" $commonCtx | indent 8 }}
 {{ include "hpcc.addSecretVolumeMounts" $commonCtx | indent 8 }}
+{{ include "hpcc.addVaultClientCertificateVolumeMounts" $commonCtx | indent 8 }}
 {{- if not $roxie.serverReplicas }}
 
 {{ include "hpcc.addCertificateVolumeMount" (dict "root" $ "component" "roxie-agent" "name" $name "external" false) | indent 8 }}
@@ -422,6 +425,7 @@ spec:
 {{ include "hpcc.addConfigMapVolume" $roxie | indent 6 }}
 {{ include "hpcc.addVolumes" $commonCtx | indent 6 }}
 {{ include "hpcc.addSecretVolumes" $commonCtx | indent 6 }}
+{{ include "hpcc.addVaultClientCertificateVolumes" $commonCtx | indent 6 }}
 {{- if not $roxie.serverReplicas }}
 {{ include "hpcc.addCertificateVolume" (dict "root" $ "component" "roxie-agent" "name" $name "external" false) | indent 6 }}
 {{ include "hpcc.addCertificateVolume" (dict "root" $ "component" "roxie-agent" "name" $agentPublicCertName "certificate" $roxie.certificate "external" true "includeRemote" true) | indent 6 }}
diff --git a/helm/hpcc/templates/sasha.yaml b/helm/hpcc/templates/sasha.yaml
index 174ae29b7d5..172c742947b 100644
--- a/helm/hpcc/templates/sasha.yaml
+++ b/helm/hpcc/templates/sasha.yaml
@@ -76,6 +76,7 @@ spec:
 {{ end -}}
 {{- include "hpcc.addVolumeMounts" $commonCtx | indent 8 }}
 {{ include "hpcc.addSecretVolumeMounts" $commonCtx | indent 8 }}
+{{ include "hpcc.addVaultClientCertificateVolumeMounts" $commonCtx | indent 8 }}
 {{ include "hpcc.addCertificateVolumeMount" (dict "root" $ "name" $sashaName "component" "sasha" "external" false) | indent 8 }}
       volumes:
 {{- with (dict "name" $serviceName) }}
@@ -83,6 +84,7 @@ spec:
 {{ end -}}
 {{- include "hpcc.addVolumes" $commonCtx | indent 6 }}
 {{- include "hpcc.addSecretVolumes"  $commonCtx | indent 6 }}
+{{ include "hpcc.addVaultClientCertificateVolumes" $commonCtx | indent 6 }}
 {{ include "hpcc.addCertificateVolume" (dict "root" $ "name" $sashaName "component" "sasha" "external" false) | indent 6 }}
 ---
 {{- if (hasKey $sasha "service") -}}
diff --git a/helm/hpcc/templates/thor.yaml b/helm/hpcc/templates/thor.yaml
index e66feb175a0..88775c13a96 100644
--- a/helm/hpcc/templates/thor.yaml
+++ b/helm/hpcc/templates/thor.yaml
@@ -117,6 +117,7 @@ data:
 {{ include "hpcc.addConfigMapVolumeMount" .me | indent 12 }}
 {{ include "hpcc.addVolumeMounts" . | indent 12 }}
 {{ include "hpcc.addSecretVolumeMounts" . | indent 12 }}
+{{ include "hpcc.addVaultClientCertificateVolumeMounts" . | indent 8 }}
 {{ include "hpcc.addCertificateVolumeMount" (dict "root" .root "name" .me.name "component" "eclagent" "includeRemote" true) | indent 12 }}
 {{- if $misc.postJobCommandViaSidecar }}
 {{ include "hpcc.addWaitAndRunVolumeMount" . | indent 12 }}
@@ -125,6 +126,7 @@ data:
 {{ include "hpcc.addConfigMapVolume" .me | indent 10 }}
 {{ include "hpcc.addVolumes" . | indent 10 }}
 {{ include "hpcc.addSecretVolumes" . | indent 10 }}
+{{ include "hpcc.addVaultClientCertificateVolumes" . | indent 10 }}
 {{ include "hpcc.addCertificateVolume" (dict "root" .root "name" .me.name "component" "eclagent" "includeRemote" true) | indent 10 }}
 {{- if $misc.postJobCommandViaSidecar }}
 {{ include "hpcc.addWaitAndRunVolume" . | indent 10 }}
@@ -184,6 +186,7 @@ data:
 {{ include "hpcc.addConfigMapVolumeMount" .me | indent 12 }}
 {{ include "hpcc.addVolumeMounts" . | indent 12 }}
 {{ include "hpcc.addSecretVolumeMounts" . | indent 12 }}
+{{ include "hpcc.addVaultClientCertificateVolumeMounts" . | indent 12 }}
 {{ include "hpcc.addCertificateVolumeMount" (dict "root" .root "name" .me.name "component" "thormanager" "includeRemote" true) | indent 12 }}
 {{- if $misc.postJobCommandViaSidecar }}
 {{ include "hpcc.addWaitAndRunVolumeMount" . | indent 12 }}
@@ -192,6 +195,7 @@ data:
 {{ include "hpcc.addConfigMapVolume" .me | indent 10 }}
 {{ include "hpcc.addVolumes" . | indent 10 }}
 {{ include "hpcc.addSecretVolumes" . | indent 10 }}
+{{ include "hpcc.addVaultClientCertificateVolumes" . | indent 10 }}
 {{ include "hpcc.addCertificateVolume" (dict "root" .root "name" .me.name "component" "thormanager" "includeRemote" true) | indent 10 }}
 {{- if $misc.postJobCommandViaSidecar }}
 {{ include "hpcc.addWaitAndRunVolume" . | indent 10 }}
@@ -253,6 +257,7 @@ data:
 {{ include "hpcc.addConfigMapVolumeMount" $configCtx.me | indent 12 }}
 {{ include "hpcc.addVolumeMounts" (deepCopy $configCtx | merge (dict "id" (toString $containerNum))) | indent 12 }}
 {{ include "hpcc.addSecretVolumeMounts" $configCtx | indent 12 }}
+{{ include "hpcc.addVaultClientCertificateVolumeMounts" $configCtx | indent 12 }}
 {{ include "hpcc.addCertificateVolumeMount" (dict "root" $configCtx.root "name" $configCtx.me.name "component" "thorworker" "includeRemote" true) | indent 12 }}
 {{- if and ($misc.postJobCommandViaSidecar) (eq $containerNum 1) }}
 {{ include "hpcc.addWaitAndRunVolumeMount" $configCtx | indent 12 }}
@@ -262,6 +267,7 @@ data:
 {{ include "hpcc.addConfigMapVolume" .me | indent 10 }}
 {{ include "hpcc.addVolumes" . | indent 10 }}
 {{ include "hpcc.addSecretVolumes" . | indent 10 }}
+{{ include "hpcc.addVaultClientCertificateVolumes" . | indent 10 }}
 {{ include "hpcc.addCertificateVolume" (dict "root" .root "name" .me.name "component" "thorworker" "includeRemote" true) | indent 10 }}
 {{- if $misc.postJobCommandViaSidecar }}
 {{ include "hpcc.addWaitAndRunVolume" . | indent 10 }}
@@ -389,11 +395,13 @@ spec:
 {{ include "hpcc.addConfigMapVolumeMount" . | indent 8 }}
 {{ include "hpcc.addVolumeMounts" $commonCtx | indent 8 }}
 {{ include "hpcc.addSecretVolumeMounts" $commonCtx | indent 8 }}
+{{ include "hpcc.addVaultClientCertificateVolumeMounts" $commonCtx | indent 8 }}
 {{ include "hpcc.addCertificateVolumeMount" (dict "root" $ "name" $commonCtx.eclAgentName "component" "eclagent" "includeRemote" true) | indent 8 }}
       volumes:
 {{ include "hpcc.addConfigMapVolume" . | indent 6 }}
 {{ include "hpcc.addVolumes" $commonCtx | indent 6 }}
 {{ include "hpcc.addSecretVolumes" $commonCtx | indent 6 }}
+{{ include "hpcc.addVaultClientCertificateVolumes" $commonCtx | indent 6 }}
 {{ include "hpcc.addCertificateVolume" (dict "root" $ "name" $commonCtx.eclAgentName "component" "eclagent" "includeRemote" true) | indent 6 }}
 ---
 apiVersion: apps/v1
@@ -450,11 +458,13 @@ spec:
 {{ include "hpcc.addConfigMapVolumeMount" . | indent 8 }}
 {{ include "hpcc.addVolumeMounts" $commonCtx | indent 8 }}
 {{ include "hpcc.addSecretVolumeMounts" $commonCtx | indent 8 }}
+{{ include "hpcc.addVaultClientCertificateVolumeMounts" $commonCtx | indent 8 }}
 {{ include "hpcc.addCertificateVolumeMount" (dict "root" $ "name" $commonCtx.thorAgentName "component" "thoragent" "includeRemote" true) | indent 8 }}
       volumes:
 {{ include "hpcc.addConfigMapVolume" . | indent 6 }}
 {{ include "hpcc.addVolumes" $commonCtx | indent 6 }}
 {{ include "hpcc.addSecretVolumes" $commonCtx | indent 6 }}
+{{ include "hpcc.addVaultClientCertificateVolumes" $commonCtx | indent 6 }}
 {{ include "hpcc.addCertificateVolume" (dict "root" $ "name" $commonCtx.thorAgentName "component" "thoragent" "includeRemote" true) | indent 6 }}
 ---
 kind: ConfigMap
diff --git a/helm/hpcc/values.yaml b/helm/hpcc/values.yaml
index ffb9c25f130..33143fa08b7 100644
--- a/helm/hpcc/values.yaml
+++ b/helm/hpcc/values.yaml
@@ -268,6 +268,18 @@ certificates:
       ## for information on what spec should contain.
       spec:
         selfSigned: {}
+    vaultclient:
+      name: hpcc-vaultclient-issuer
+      enabled: false
+      ## domain: hpcc.example.com
+      rolePrefix: "hpcc-"
+      kind: Issuer
+      ## do not define spec (set spec: null), to reference an Issuer resource that already exists in the cluster
+      ## change spec if you'd like to change how certificates get issued... see ## https://cert-manager.io/docs/configuration/#supported-issuer-types
+      ## for information on what spec should contain.
+      spec:
+        ca:
+          secretName: hpcc-vaultclient-issuer-key-pair
     remote:
       name: hpcc-remote-issuer
       ## set enabled to true if adding remoteClients for any components
@@ -349,21 +361,23 @@ secrets:
 
 vaults:
   storage:
-
   git:
-
   authn:
-
   ecl:
-
-  eclUser:
-    ## Category for vaults accessible via ecl code.  These are vaults that users can access directly.  Be cautious
-    ## about what vaults you add to this category as they are easily accessed by ECL code.
-    #- name: ecl-user-vault
-    #  url: http://${env.VAULT_SERVICE_HOST}:${env.VAULT_SERVICE_PORT}/v1/secret/data/ecluser/${secret}
+    # vault using vault client certs or kubernetes auth depending on whether certificates.issuers.vaultclient.enabled is true
+    #   to use approle authentication specify appRoleId and appRoleSecret
+    # - name: my-ecl-vault
+      #Note the data node in the URL is there for the REST APIs use. The path inside the vault starts after /data
+    #  url: http://${env.VAULT_SERVICE_HOST}:${env.VAULT_SERVICE_PORT}/v1/secret/data/ecl/${secret}
     #  kind: kv-v2
-    #  namespace: mynamespace # for use with enterprise vaults segmented by namespaces
-
+    # namespace: <vault-namespace>
+  eclUser:
+    # vault using vault client certs or kubernetes auth depending on whether certificates.issuers.vaultclient.enabled is true
+    #   to use approle authentication specify appRoleId and appRoleSecret
+    # - name: my-eclUser-vault
+      #Note the data node in the URL is there for the REST APIs use. The path inside the vault starts after /data
+    #   url: http://${env.VAULT_SERVICE_HOST}:${env.VAULT_SERVICE_PORT}/v1/secret/data/eclUser/${secret}
+    #   kind: kv-v2
   esp:
 
   ## The keys for code signing may be imported from the vault.  Multiple keys may be imported.
diff --git a/system/jlib/jsecrets.cpp b/system/jlib/jsecrets.cpp
index c5b30c42187..5828bc71166 100644
--- a/system/jlib/jsecrets.cpp
+++ b/system/jlib/jsecrets.cpp
@@ -329,6 +329,19 @@ extern jlib_decl void setSecretMount(const char * path)
         secretDirectory.set(path);
 }
 
+static const char *ensureSecretDirectory()
+{
+    CriticalBlock block(secretCS);
+    if (secretDirectory.isEmpty())
+        setSecretMount(nullptr);
+    return secretDirectory;
+}
+
+static StringBuffer &buildSecretPath(StringBuffer &path, const char *category, const char * name)
+{
+    return addPathSepChar(path.append(ensureSecretDirectory())).append(category).append(PATHSEPCHAR).append(name).append(PATHSEPCHAR);
+}
+
 static bool checkSecretExpired(unsigned created)
 {
     if (!created)
@@ -342,7 +355,7 @@ static bool hasCacheExpired(const IPropertyTree * secret)
     return checkSecretExpired((unsigned)secret->getPropInt("@created"));
 }
 
-enum class VaultAuthType {unknown, k8s, appRole, token};
+enum class VaultAuthType {unknown, k8s, appRole, token, clientcert};
 
 static void setTimevalMS(timeval &tv, time_t ms)
 {
@@ -369,6 +382,10 @@ class CVault
     CriticalSection vaultCS;
     Owned<IPropertyTree> cache;
 
+    std::string clientCertPath;
+    std::string clientKeyPath;
+
+    StringBuffer category;
     StringBuffer schemeHostPort;
     StringBuffer path;
     StringBuffer vaultNamespace;
@@ -376,7 +393,7 @@ class CVault
     StringBuffer password;
     StringAttr name;
 
-    StringAttr k8sAuthRole;
+    StringAttr authRole; //authRole is used by kubernetes and client cert auth, it's not part of appRole auth
     StringAttr appRoleId;
     StringBuffer appRoleSecretName;
 
@@ -393,6 +410,19 @@ class CVault
 public:
     CVault(IPropertyTree *vault)
     {
+        category.appendLower(vault->queryName());
+
+        StringBuffer clientTlsPath;
+        buildSecretPath(clientTlsPath, "certificates", "vaultclient");
+
+        clientCertPath.append(clientTlsPath.str()).append(category.str()).append("/tls.crt");
+        clientKeyPath.append(clientTlsPath.str()).append(category.str()).append("/tls.key");
+
+        if (!checkFileExists(clientCertPath.c_str()))
+            WARNLOG("vault: client cert not found, %s", clientCertPath.c_str());
+        if (!checkFileExists(clientKeyPath.c_str()))
+            WARNLOG("vault: client key not found, %s", clientKeyPath.c_str());
+
         cache.setown(createPTree());
         StringBuffer url;
         replaceEnvVariables(url, vault->queryProp("@url"), false);
@@ -445,13 +475,19 @@ class CVault
                 }
             }
         }
+        else if (vault->getPropBool("@useTLSCertificateAuth", false))
+        {
+            authType = VaultAuthType::clientcert;
+            if (vault->hasProp("@role"))
+                authRole.set(vault->queryProp("@role"));
+        }
         else if (isContainerized())
         {
             authType = VaultAuthType::k8s;
             if (vault->hasProp("@role"))
-                k8sAuthRole.set(vault->queryProp("@role"));
+                authRole.set(vault->queryProp("@role"));
             else
-                k8sAuthRole.set("hpcc-vault-access");
+                authRole.set("hpcc-vault-access");
             PROGLOG("using kubernetes vault auth");
         }
     }
@@ -465,6 +501,8 @@ class CVault
                 return "kubernetes";
             case VaultAuthType::token:
                 return "token";
+            case VaultAuthType::clientcert:
+                return "clientcert";
         }
         return "unknown";
     }
@@ -558,7 +596,7 @@ class CVault
             vaultAuthError("missing k8s auth token");
 
         std::string json;
-        json.append("{\"jwt\": \"").append(login_token.str()).append("\", \"role\": \"").append(k8sAuthRole.str()).append("\"}");
+        json.append("{\"jwt\": \"").append(login_token.str()).append("\", \"role\": \"").append(authRole.str()).append("\"}");
         httplib::Client cli(schemeHostPort.str());
         httplib::Headers headers;
 
@@ -575,6 +613,34 @@ class CVault
 
         processClientTokenResponse(res);
     }
+
+    void clientCertLogin(bool permissionDenied)
+    {
+        CriticalBlock block(vaultCS);
+        if (!permissionDenied && (clientToken.length() && !isClientTokenExpired()))
+            return;
+        DBGLOG("clientCertLogin%s", permissionDenied ? " because existing token permission denied" : "");
+
+        std::string json;
+        json.append("{\"name\": \"").append(authRole.str()).append("\"}"); //name can be empty but that is inefficient because vault would have to search for the cert being used
+
+        httplib::Client cli(schemeHostPort.str(), clientCertPath, clientKeyPath);
+        httplib::Headers headers;
+
+        unsigned numRetries = 0;
+        initClient(cli, headers, numRetries);
+        httplib::Result res = cli.Post("/v1/auth/cert/login", headers, json, "application/json");
+        while (!res && numRetries--)
+        {
+            OERRLOG("Retrying vault %s client cert auth, communication error %d", name.str(), res.error());
+            if (retryWait)
+                Sleep(retryWait);
+            res = cli.Post("/v1/auth/cert/login", headers, json, "application/json");
+        }
+
+        processClientTokenResponse(res);
+    }
+
     //if we tried to use our token and it returned access denied it could be that we need to login again, or
     //  perhaps it could be specific permissions about the secret that was being accessed, I don't think we can tell the difference
     void appRoleLogin(bool permissionDenied)
@@ -617,6 +683,8 @@ class CVault
             appRoleLogin(permissionDenied);
         else if (authType == VaultAuthType::k8s)
             kubernetesLogin(permissionDenied);
+        else if (authType == VaultAuthType::clientcert)
+            clientCertLogin(permissionDenied);
         else if (permissionDenied && authType == VaultAuthType::token)
             vaultAuthError("token permission denied"); //don't permenently invalidate token. Try again next time because it could be permissions for a particular secret rather than invalid token
         if (clientToken.isEmpty())
@@ -904,19 +972,6 @@ static void addCachedLocalSecret(const char *category, const char *name, IProper
     }
 }
 
-static const char *ensureSecretDirectory()
-{
-    CriticalBlock block(secretCS);
-    if (secretDirectory.isEmpty())
-        setSecretMount(nullptr);
-    return secretDirectory;
-}
-
-static StringBuffer &buildSecretPath(StringBuffer &path, const char *category, const char * name)
-{
-    return addPathSepChar(path.append(ensureSecretDirectory())).append(category).append(PATHSEPCHAR).append(name).append(PATHSEPCHAR);
-}
-
 static IPropertyTree *loadLocalSecret(const char *category, const char * name)
 {
     StringBuffer path;