Support external certificates for KES (#147)

Also update MinIO, KES and MCS images.
minio · Jun 23, 2020 · d6fcdca · d6fcdca
1 parent 72ec8e4
commit d6fcdca
Show file tree

Hide file tree

Showing 15 changed files with 216 additions and 106 deletions.
diff --git a/docs/tls.md b/docs/tls.md
@@ -3,22 +3,11 @@
 [![Slack](https://slack.min.io/slack?type=svg)](https://slack.min.io)
 [![Docker Pulls](https://img.shields.io/docker/pulls/minio/k8s-operator.svg?maxAge=604800)](https://hub.docker.com/r/minio/k8s-operator)
 
-This document explains how to enable TLS on MinIOInstance pods. There are a few approaches to enable TLS for MinIO:
+This document explains how to enable TLS on MinIOInstance pods. These are the approaches to enable TLS for MinIO:
 
-- Use the Kubernetes cluster's root Certificate Authority (CA) to establish trust.
+## Automatic TLS
 
-  The MinIO Operator creates a private key, and a certificate signing request (CSR) which is submitted to the `certificates.k8s.io` API for signing. Approving the CSR is either done by an automated approval process or on a one off basis by a Kubernetes cluster administrator.
-
-- Acquire a self-signed or CA signed certificate and use a Kubernetes Secret resource to store this information.
-
-  Once the Secret is manually created, pass the Secret name to MinIO Operator, which then uses the Secret to extract certificate data and mount it at relevant locations within MinIOInstance pods.
-
-- Configure MinIO to use a certificate issued by [cert-manager][1].
-
-  Similar to using a self-signed or CA signed certificate, but the operator is able to accept a Secret resource created by cert-manager.
-
-
-## Automatic CSR Generation
+This approach creates TLS certificates automatically using the Kubernetes cluster root Certificate Authority (CA) to establish trust. In this approach, MinIO Operator creates a private key, and a certificate signing request (CSR) which is submitted via the `certificates.k8s.io` API for signing. Approving the CSR is done on a one off basis by a Kubernetes cluster administrator. Automatic TLS approach creates other certificates required for KES as well as explained in [KES document](./kes.md).
 
 To enable automatic CSR generation on MinIOInstance, set `requestAutoCert` field in the config file to `true`. Optionally you can also pass additional configuration parameters to be used under `certConfig` section. The `certConfig` section currently supports below fields:
 
@@ -46,17 +35,17 @@ Once the CSR is approved and Certificate available, MinIO operator downloads the
 
 ## Pass Certificate Secret to MinIOInstance
 
-Follow this approach if you plan to use CA signed or self-signed certificates for MinIOInstance pods. Once you have the key and certificate file available, create a Kubernetes Secret using
+This approach involves acquiring a CA signed or self-signed certificate and use a Kubernetes Secret resource to store this information. Once you have the key and certificate file available, create a Kubernetes Secret using
 
 ```bash
 kubectl create secret generic tls-ssl-minio --from-file=path/to/private.key --from-file=path/to/public.crt
 ```
 
-Once created, set the name of Secret (here it is `tls-ssl-minio`) under `spec.externalCertSecret` field. Then create the MinIOInstance. MinIO Operator will use this Secret to fetch key and certificate and mount it to relevant locations inside the MinIOInstance pods.
+Once created, set the name of Secret (here it is `tls-ssl-minio`) under `spec.externalCertSecret` field. Then create the MinIOInstance. MinIO Operator will use this Secret to fetch key and certificate and mount it to relevant locations inside the MinIOInstance pods. 
 
 ### Using Kubernetes TLS
 
-Alternatively to the above, it's possible to use a TLS secret. First, create the Kubernetes secret:
+Alternatively, it's possible to use a TLS secret. First, create the Kubernetes secret:
 
 ```bash
 kubectl create secret tls tls-ssl-minio --key=private.key --cert=public.crt
@@ -70,10 +59,9 @@ Once created, set the name of the Secret (in this example `tls-ssl-minio`) under
     type: kubernetes.io/tls
 ```
 
-
 ## Using cert-manager
 
-[Certificate Manager][1] is a Kubernetes Operator capable of automatically issuing certificates from multiple Issuers. Integration with MinIO is simple. First, create a new certificate issuer; for this demonstration the issuer certificate will be self-signed:
+[Certificate Manager](https://cert-manager.io) is a Kubernetes Operator capable of automatically issuing certificates from multiple Issuers. Integration with MinIO is simple. First, create a new certificate issuer; for this demonstration the issuer certificate will be self-signed:
 
 ```yaml
 apiVersion: cert-manager.io/v1alpha2
@@ -110,5 +98,3 @@ Finally configure MinIO to use the newly created TLS certificate:
     name: tls-minio
     type: cert-manager.io/v1alpha2
 ```
-
-[1]: https://cert-manager.io
diff --git a/examples/kes-config-secret.yaml b/examples/kes-config-secret.yaml
@@ -8,8 +8,8 @@ stringData:
     address: 0.0.0.0:7373
     root: _ # Effectively disabled since no root identity necessary.
     tls:
-      key: /home/server.key   # Path to the TLS private key
-      cert: /home/server.crt # Path to the TLS certificate
+      key: /tmp/kes/server.key   # Path to the TLS private key
+      cert: /tmp/kes/server.crt # Path to the TLS certificate
       proxy:
         identities: []
         header:

diff --git a/examples/minioinstance-kes.yaml b/examples/minioinstance-kes.yaml
@@ -53,7 +53,7 @@ spec:
       prometheus.io/port: "9000"
       prometheus.io/scrape: "true"
   ## Registry location and Tag to download MinIO Server image
-  image: minio/minio:RELEASE.2020-06-03T22-13-49Z
+  image: minio/minio:RELEASE.2020-06-18T02-23-35Z
   ## A ClusterIP Service will be created with the given name
   serviceName: minio-internal-service
   ## Secret with credentials to be used by MinIO instance.
@@ -85,7 +85,7 @@ spec:
           storage: 1Ti
   ## Define configuration for MCS (Graphical user interface for MinIO)
   mcs:
-    image: minio/mcs:v0.0.6
+    image: minio/mcs:v0.1.0
     replicas: 2
     mcsSecret:
       name: mcs-secret
@@ -94,17 +94,21 @@ spec:
         app: mcs
   ## Define configuration for KES (stateless and distributed key-management system)
   kes:
-    image: minio/kes:v0.9.0
+    image: minio/kes:v0.10.1
     replicas: 2
     kesSecret:
       name: kes-config
     metadata:
       labels:
         app: kes
+    # externalCertSecret:
+    #   name: tls-ssl-kes
   ## Secret with certificates to configure TLS for MinIO certs. Create secrets as explained
   ## here: https://github.com/minio/minio/tree/master/docs/tls/kubernetes#2-create-kubernetes-secret
   # externalCertSecret:
-  # name: tls-ssl-minio
+  #   name: tls-ssl-minio
+  # externalClientCertSecret:
+  #   name: tls-ssl-minio-client
   ## Enable Kubernetes based certificate generation and signing as explained in
   ## https://kubernetes.io/docs/tasks/tls/managing-tls-in-a-cluster
   requestAutoCert: true

diff --git a/examples/minioinstance-mcs.yaml b/examples/minioinstance-mcs.yaml
@@ -53,7 +53,7 @@ spec:
       prometheus.io/port: "9000"
       prometheus.io/scrape: "true"
   ## Registry location and Tag to download MinIO Server image
-  image: minio/minio:RELEASE.2020-06-03T22-13-49Z
+  image: minio/minio:RELEASE.2020-06-18T02-23-35Z
   ## A ClusterIP Service will be created with the given name
   serviceName: minio-internal-service
   ## Secret with credentials to be used by MinIO instance.
@@ -85,7 +85,7 @@ spec:
           storage: 1Ti
   ## Define configuration for MCS (Graphical user interface for MinIO)
   mcs:
-    image: minio/mcs:v0.0.6
+    image: minio/mcs:v0.1.0
     replicas: 2
     mcsSecret:
       name: mcs-secret

diff --git a/examples/minioinstance-pod-security-policy.yaml b/examples/minioinstance-pod-security-policy.yaml
@@ -94,7 +94,7 @@ spec:
       prometheus.io/port: "9000"
       prometheus.io/scrape: "true"
   ## Registry location and Tag to download MinIO Server image
-  image: minio/minio:RELEASE.2020-06-03T22-13-49Z
+  image: minio/minio:RELEASE.2020-06-18T02-23-35Z
   ## A ClusterIP Service will be created with the given name
   serviceName: minio-internal-service
   ## Service account to be used for all the MinIO Pods

diff --git a/examples/minioinstance.yaml b/examples/minioinstance.yaml
@@ -42,7 +42,7 @@ spec:
       prometheus.io/port: "9000"
       prometheus.io/scrape: "true"
   ## Registry location and Tag to download MinIO Server image
-  image: minio/minio:RELEASE.2020-06-03T22-13-49Z
+  image: minio/minio:RELEASE.2020-06-18T02-23-35Z
   ## A ClusterIP Service will be created with the given name
   serviceName: minio-internal-service
   zones:

diff --git a/pkg/apis/operator.min.io/v1/constants.go b/pkg/apis/operator.min.io/v1/constants.go
@@ -48,6 +48,9 @@ const CSRNameSuffix = "-csr"
 
 // MinIO Related Constants
 
+// MinIOCertPath is the path where all MinIO certs are mounted
+const MinIOCertPath = "/tmp/certs"
+
 // InstanceLabel is applied to all components of a MinIOInstance cluster
 const InstanceLabel = "v1.min.io/instance"
 
@@ -67,7 +70,7 @@ const MinIOVolumeMountPath = "/export"
 const MinIOVolumeSubPath = ""
 
 // DefaultMinIOImage specifies the default MinIO Docker hub image
-const DefaultMinIOImage = "minio/minio:RELEASE.2020-06-03T22-13-49Z"
+const DefaultMinIOImage = "minio/minio:RELEASE.2020-06-18T02-23-35Z"
 
 // DefaultMinIOAccessKey specifies default access key for MinIOInstance
 const DefaultMinIOAccessKey = "AKIAIOSFODNN7EXAMPLE"
@@ -114,7 +117,7 @@ const LivenessTimeout = 1
 // MCS Related Constants
 
 // DefaultMCSImage specifies the latest MCS Docker hub image
-const DefaultMCSImage = "minio/mcs:v0.0.6"
+const DefaultMCSImage = "minio/mcs:v0.1.0"
 
 // MCSInstanceLabel is applied to the MCS pods of a MinIOInstance cluster
 const MCSInstanceLabel = "v1.min.io/mcs"
@@ -143,7 +146,7 @@ const DefaultMCSReplicas = 2
 // KES Related Constants
 
 // DefaultKESImage specifies the latest KES Docker hub image
-const DefaultKESImage = "minio/kes:v0.9.0"
+const DefaultKESImage = "minio/kes:v0.10.1"
 
 // KESInstanceLabel is applied to the KES pods of a MinIOInstance cluster
 const KESInstanceLabel = "v1.min.io/kes"
@@ -167,7 +170,8 @@ const KESHLSvcNameSuffix = "-kes-hl-svc"
 const KESName = "-kes"
 
 // KESConfigMountPath specifies the path where KES config file and all secrets are mounted
-const KESConfigMountPath = "/home"
+// We keep this to /tmp so it doesn't require any special permissions
+const KESConfigMountPath = "/tmp/kes"
 
 // DefaultKESReplicas specifies the default number of KES pods to be created if not specified
 const DefaultKESReplicas = 2

diff --git a/pkg/apis/operator.min.io/v1/globals.go b/pkg/apis/operator.min.io/v1/globals.go
@@ -41,7 +41,7 @@ func getEnv(key string) string {
 
 func identifyScheme(mi *MinIOInstance) string {
 	scheme := "http"
-	if mi.RequiresAutoCertSetup() || mi.RequiresExternalCertSetup() {
+	if mi.AutoCert() || mi.ExternalCert() {
 		scheme = "https"
 	}
 	return scheme

diff --git a/pkg/apis/operator.min.io/v1/helper.go b/pkg/apis/operator.min.io/v1/helper.go
@@ -72,17 +72,28 @@ func (mi *MinIOInstance) HasCertConfig() bool {
 	return mi.Spec.CertConfig != nil
 }
 
-// RequiresExternalCertSetup returns true is the user has provided a secret
-// that contains CA cert, server cert and server key for group replication
-// SSL support
-func (mi *MinIOInstance) RequiresExternalCertSetup() bool {
+// ExternalCert returns true is the user has provided a secret
+// that contains CA cert, server cert and server key
+func (mi *MinIOInstance) ExternalCert() bool {
 	return mi.Spec.ExternalCertSecret != nil
 }
 
-// RequiresAutoCertSetup returns true is the user has provided a secret
+// ExternalClientCert returns true is the user has provided a secret
+// that contains CA client cert, server cert and server key
+func (mi *MinIOInstance) ExternalClientCert() bool {
+	return mi.Spec.ExternalClientCertSecret != nil
+}
+
+// KESExternalCert returns true is the user has provided a secret
+// that contains CA cert, server cert and server key for KES pods
+func (mi *MinIOInstance) KESExternalCert() bool {
+	return mi.Spec.KES != nil && mi.Spec.KES.ExternalCertSecret != nil
+}
+
+// AutoCert returns true is the user has provided a secret
 // that contains CA cert, server cert and server key for group replication
 // SSL support
-func (mi *MinIOInstance) RequiresAutoCertSetup() bool {
+func (mi *MinIOInstance) AutoCert() bool {
 	return mi.Spec.RequestAutoCert
 }
 
@@ -152,7 +163,7 @@ func (mi *MinIOInstance) EnsureDefaults() *MinIOInstance {
 		mi.Spec.Subpath = MinIOVolumeSubPath
 	}
 
-	if mi.RequiresAutoCertSetup() {
+	if mi.AutoCert() {
 		if mi.Spec.CertConfig != nil {
 			if mi.Spec.CertConfig.CommonName == "" {
 				mi.Spec.CertConfig.CommonName = mi.MinIOWildCardName()

diff --git a/pkg/apis/operator.min.io/v1/helper_test.go b/pkg/apis/operator.min.io/v1/helper_test.go
@@ -16,12 +16,12 @@ func TestEnsureDefaults(t *testing.T) {
 		assert.Equal(t, mi.Spec.Image, DefaultMinIOImage)
 		assert.Equal(t, mi.Spec.Mountpath, MinIOVolumeMountPath)
 		assert.Equal(t, mi.Spec.Subpath, MinIOVolumeSubPath)
-		assert.False(t, mi.RequiresAutoCertSetup())
+		assert.False(t, mi.AutoCert())
 	})
 
 	t.Run("auto cert", func(t *testing.T) {
 		mi.Spec.RequestAutoCert = true
-		assert.True(t, mi.RequiresAutoCertSetup())
+		assert.True(t, mi.AutoCert())
 		assert.False(t, mi.HasCertConfig())
 
 		mi.EnsureDefaults()

diff --git a/pkg/apis/operator.min.io/v1/types.go b/pkg/apis/operator.min.io/v1/types.go
@@ -86,9 +86,14 @@ type MinIOInstanceSpec struct {
 	// If specified, affinity will define the pod's scheduling constraints
 	// +optional
 	Affinity *corev1.Affinity `json:"affinity,omitempty"`
-	// ExternalCertSecret allows a user to specify custom CA certificate, and private key for group replication SSL.
+	// ExternalCertSecret allows a user to specify custom CA certificate, and private key. This is
+	// used for enabling TLS support on MinIO Pods.
 	// +optional
 	ExternalCertSecret *LocalCertificateReference `json:"externalCertSecret,omitempty"`
+	// ExternalClientCertSecret allows a user to specify custom CA client certificate, and private key. This is
+	// used for adding client certificates on MinIO Pods --> used for KES authentication.
+	// +optional
+	ExternalClientCertSecret *LocalCertificateReference `json:"externalClientCertSecret,omitempty"`
 	// Mount path for MinIO volume (PV). Defaults to /export
 	// +optional
 	Mountpath string `json:"mountPath,omitempty"`
@@ -194,6 +199,9 @@ type KESConfig struct {
 	// This is a mandatory field
 	Configuration *corev1.LocalObjectReference `json:"kesSecret"`
 	Metadata      *metav1.ObjectMeta           `json:"metadata,omitempty"`
+	// ExternalCertSecret allows a user to specify custom CA certificate, and private key for group replication SSL.
+	// +optional
+	ExternalCertSecret *LocalCertificateReference `json:"externalCertSecret,omitempty"`
 }
 
 // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object