Openshift service certificates

config/manifests/bases/minio-operator-rhmp.clusterserviceversion.yaml

@@ -13,6 +13,7 @@ metadata:
     marketplace.openshift.io/remote-workflow: https://marketplace.redhat.com/en-us/operators/minio-operator-rhmp/pricing?utm_source=openshift_console
     marketplace.openshift.io/support-workflow: https://marketplace.redhat.com/en-us/operators/minio-operator-rhmp/support?utm_source=openshift_console
     capabilities: "Full Lifecycle"
+    operatorframework.io/suggested-namespace: minio-operator
   name: minio-operator-rhmp.v0.0.0
   namespace: placeholder
 spec:

config/manifests/bases/minio-operator.clusterserviceversion.yaml

@@ -11,6 +11,7 @@ metadata:
     k8sMinVersion: "1.18"
     repository: https://github.com/minio/operator
     capabilities: "Full Lifecycle"
+    operatorframework.io/suggested-namespace: minio-operator
   name: minio-operator.v0.0.0
   namespace: placeholder
 spec:

config/manifests/kustomization.yaml

@@ -8,6 +8,9 @@ resources:
 
 patchesStrategicMerge:
   - overlay/minio-operator-deployment.yaml
+  - overlay/console-deployment.yaml
+  - overlay/console_v1_service.yaml
+  - overlay/sts_v1_service.yaml
 
 patchesJson6902:
   - target:

config/manifests/overlay/console-deployment.yaml

@@ -0,0 +1,33 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: console
+  namespace: minio-operator
+spec:
+  template:
+    spec:
+      containers:
+        - name: console
+          volumeMounts:
+            - name: tls-certificates
+              mountPath: /tmp/certs
+      volumes:
+        - name: tls-certificates
+          projected:
+            defaultMode: 420
+            sources:
+              - secret:
+                  name: console-tls
+                  items:
+                    - key: tls.crt
+                      path: public.crt
+                    - key: tls.key
+                      path: private.key
+
+                  optional: true
+              - configMap:
+                  name: openshift-service-ca.crt
+                  items:
+                    - key: service-ca.crt
+                      path: CAs/ca.crt
+                  optional: true

config/manifests/overlay/console_v1_service.yaml

@@ -0,0 +1,7 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: console
+  namespace: minio-operator
+  annotations:
+    service.beta.openshift.io/serving-cert-secret-name: console-tls

config/manifests/overlay/minio-operator-deployment.yaml

@@ -12,3 +12,30 @@ spec:
           env:
             - name: MINIO_OPERATOR_RUNTIME
               value: "OpenShift"
+            - name: MINIO_CONSOLE_TLS_ENABLE
+              value: "on"
+          volumeMounts:
+            - name: openshift-service-ca
+              mountPath: /tmp/service-ca
+            - name: sts-tls
+              mountPath: /tmp/sts
+      volumes:
+        - name: sts-tls
+          projected:
+            defaultMode: 420
+            sources:
+              - secret:
+                  name: sts-tls
+                  items:
+                    - key: tls.crt
+                      path: public.crt
+                    - key: tls.key
+                      path: private.key
+                  optional: true
+        - name: openshift-service-ca
+          configMap:
+            name: openshift-service-ca.crt
+            items:
+              - key: service-ca.crt
+                path: service-ca.crt
+            optional: true

config/manifests/overlay/sts_v1_service.yaml

@@ -0,0 +1,7 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: sts
+  namespace: minio-operator
+  annotations:
+    service.beta.openshift.io/serving-cert-secret-name: sts-tls

olm.sh

@@ -7,15 +7,19 @@ OPERATOR_SDK_VERSION=v1.22.2
 TMP_BIN_DIR="$(mktemp -d)"
 
 function install_binaries() {
-
-  echo "Installing temporary Binaries into: $TMP_BIN_DIR";
-  echo "Installing temporary operator-sdk binary: $OPERATOR_SDK_VERSION"
-  ARCH=`{ case "$(uname -m)" in "x86_64") echo -n "amd64";; "aarch64") echo -n "arm64";; *) echo -n "$(uname -m)";; esac; }`
-  OS=$(uname | awk '{print tolower($0)}')
-  OPERATOR_SDK_DL_URL=https://github.com/operator-framework/operator-sdk/releases/download/$OPERATOR_SDK_VERSION
-  curl -L ${OPERATOR_SDK_DL_URL}/operator-sdk_${OS}_${ARCH} -o ${TMP_BIN_DIR}/operator-sdk
-  OPERATOR_SDK_BIN=${TMP_BIN_DIR}/operator-sdk
-  chmod +x $OPERATOR_SDK_BIN
+
+  if ! operator-sdk version; then
+    echo "Installing temporary Binaries into: $TMP_BIN_DIR";
+    echo "Installing temporary operator-sdk binary: $OPERATOR_SDK_VERSION"
+    ARCH=`{ case "$(uname -m)" in "x86_64") echo -n "amd64";; "aarch64") echo -n "arm64";; *) echo -n "$(uname -m)";; esac; }`
+    OS=$(uname | awk '{print tolower($0)}')
+    OPERATOR_SDK_DL_URL=https://github.com/operator-framework/operator-sdk/releases/download/$OPERATOR_SDK_VERSION
+    curl -L ${OPERATOR_SDK_DL_URL}/operator-sdk_${OS}_${ARCH} -o ${TMP_BIN_DIR}/operator-sdk
+    OPERATOR_SDK_BIN=${TMP_BIN_DIR}/operator-sdk
+    chmod +x $OPERATOR_SDK_BIN
+  else
+    OPERATOR_SDK_BIN="$(which operator-sdk)"
+  fi
 }
 
 install_binaries
@@ -92,6 +96,7 @@ for catalog in "${redhatCatalogs[@]}"; do
     echo "  com.redhat.openshift.versions: v4.8-v4.13"
     echo "  # Annotation to add default bundle channel as potential is declared"
     echo "  operators.operatorframework.io.bundle.channel.default.v1: stable"
+    echo "  operatorframework.io/suggested-namespace: minio-operator"
   } >> bundles/$catalog/$RELEASE/metadata/annotations.yaml
 
   echo "clean root level annotations.yaml"

pkg/apis/minio.min.io/v2/helper.go

@@ -110,6 +110,27 @@ func GetPodCAFromFile() []byte {
 	return namespace
 }
 
+// GetPodServiceCAFromFile extracts the service-ca.crt certificate in Openshift deployments coming from configmap openshift-service-ca.crt
+func GetPodServiceCAFromFile() []byte {
+	caPath, err := os.ReadFile("/tmp/service-ca/ca.crt")
+	if err != nil {
+		return nil
+	}
+	return caPath
+}
+
+// GetPublicCertFilePath return the path to the certificate file based for the serviceName
+func GetPublicCertFilePath(serviceName string) string {
+	publicCertPath := fmt.Sprintf("/tmp/%s/public.crt", serviceName)
+	return publicCertPath
+}
+
+// GetPrivateKeyFilePath return the path to the key file based for the serviceName
+func GetPrivateKeyFilePath(serviceName string) string {
+	privateKey := fmt.Sprintf("/tmp/%s/private.key", serviceName)
+	return privateKey
+}
+
 // GetNSFromFile assumes the operator is running inside a k8s pod and extract the
 // current namespace from the /var/run/secrets/kubernetes.io/serviceaccount/namespace file
 func GetNSFromFile() string {

pkg/controller/console.go

@@ -37,7 +37,7 @@ const (
 	// DefaultConsoleDeploymentName is the default name of the console deployment
 	DefaultConsoleDeploymentName = "console"
 	// OperatorConsoleTLSSecretName is the name of secret created with TLS certs for Operator console
-	OperatorConsoleTLSSecretName = "operator-console-tls"
+	OperatorConsoleTLSSecretName = "console-tls"
 )
 
 // checkConsoleSvc validates the existence of the MinIO service and validate it's status against what the specification

pkg/controller/main-controller.go

@@ -26,6 +26,8 @@ import (
 	"syscall"
 	"time"
 
+	"github.com/minio/operator/pkg/utils"
+
 	"github.com/minio/madmin-go/v2"
 	"github.com/minio/operator/pkg/common"
 	xcerts "github.com/minio/pkg/certs"
@@ -356,8 +358,13 @@ func (c *Controller) startUpgradeServer(ctx context.Context) <-chan error {
 // startUpgradeServer Starts the Upgrade tenant API server and notifies the start and stop via notificationChannel
 func (c *Controller) startSTSAPIServer(ctx context.Context, notificationChannel chan<- *EventNotification) {
 	klog.Infof("Starting STS API server")
-	publicCertPath, publicKeyPath := c.waitSTSTLSCert()
-	certsManager, err := xcerts.NewManager(ctx, *publicCertPath, *publicKeyPath, LoadX509KeyPair)
+
+	publicCertPath := miniov2.GetPublicCertFilePath("sts")
+	privateKeyPath := miniov2.GetPrivateKeyFilePath("sts")
+	if utils.GetOperatorRuntime() != common.OperatorRuntimeOpenshift {
+		publicCertPath, privateKeyPath = c.waitSTSTLSCert()
+	}
+	certsManager, err := xcerts.NewManager(ctx, publicCertPath, privateKeyPath, LoadX509KeyPair)
 	if err != nil {
 		klog.Errorf("HTTPS STS API server failed to load CA certificate: %v", err)
 		notificationChannel <- &EventNotification{
@@ -367,6 +374,7 @@ func (c *Controller) startSTSAPIServer(ctx context.Context, notificationChannel
 	}
 	serverCertsManager = certsManager
 	c.sts.TLSConfig = c.createTLSConfig(serverCertsManager)
+
 	if err := c.sts.ListenAndServeTLS("", ""); !errors.Is(err, http.ErrServerClosed) {
 		// only notify on server failure, on http.ErrServerClosed the channel should be already closed
 		notificationChannel <- &EventNotification{
@@ -410,15 +418,20 @@ func leaderRun(ctx context.Context, c *Controller, threadiness int, stopCh <-cha
 	if isOperatorConsoleTLS() {
 		klog.Info("Waiting for Console TLS")
 		go func() {
-			klog.Infof("Console TLS is enabled, starting console TLS certificate setup")
-			err := c.recreateOperatorConsoleCertsIfRequired(ctx)
-			if err != nil {
-				panic(err)
-			}
-			klog.Infof("Restarting Console pods")
-			err = c.rolloutRestartDeployment(getConsoleDeploymentName())
-			if err != nil {
-				klog.Errorf("Console deployment didn't restart: %s", err)
+			if utils.GetOperatorRuntime() == common.OperatorRuntimeOpenshift {
+				klog.Infof("Console TLS is enabled, skipping TLS certificate generation on Openshift deployment")
+			} else {
+				klog.Infof("Console TLS is enabled, starting console TLS certificate setup")
+
+				err := c.recreateOperatorConsoleCertsIfRequired(ctx)
+				if err != nil {
+					panic(err)
+				}
+				klog.Infof("Restarting Console pods")
+				err = c.rolloutRestartDeployment(getConsoleDeploymentName())
+				if err != nil {
+					klog.Errorf("Console deployment didn't restart: %s", err)
+				}
 			}
 		}()
 	} else {
@@ -428,8 +441,12 @@ func leaderRun(ctx context.Context, c *Controller, threadiness int, stopCh <-cha
 	// 2) we need to make sure we have STS API certificates (if enabled)
 	if IsSTSEnabled() {
 		go func() {
-			klog.Infof("STS is enabled, starting STS API certificate setup")
-			c.generateSTSTLSCert()
+			if utils.GetOperatorRuntime() == common.OperatorRuntimeOpenshift {
+				klog.Infof("STS is enabled, skipping TLS certificate generation on Openshift deployment")
+			} else {
+				klog.Infof("STS is enabled, starting API certificate setup")
+				c.generateSTSTLSCert()
+			}
 		}()
 	}
 
@@ -440,7 +457,7 @@ func leaderRun(ctx context.Context, c *Controller, threadiness int, stopCh <-cha
 				klog.Errorf("Upgrade Server stopped: %v, going to restart", err)
 				upgradeServerChannel = c.startUpgradeServer(ctx)
 			}
-			// webswerver was instructed to stop, do not attempt to restart
+			// webserver was instructed to stop, do not attempt to restart
 			continue
 		case <-stopCh:
 			return

pkg/controller/operator.go

@@ -137,6 +137,11 @@ func (c *Controller) fetchTransportCACertificates() (pool *x509.CertPool) {
 	// Default kubernetes CA certificate
 	rootCAs.AppendCertsFromPEM(miniov2.GetPodCAFromFile())
 
+	// Openshift Service CA certificate
+	if utils.GetOperatorRuntime() == common.OperatorRuntimeOpenshift {
+		rootCAs.AppendCertsFromPEM(miniov2.GetPodServiceCAFromFile())
+	}
+
 	// Custom ca certificate to be used by operator
 	operatorCATLSCert, err := c.kubeClientSet.CoreV1().Secrets(miniov2.GetNSFromFile()).Get(context.Background(), OperatorCATLSSecretName, metav1.GetOptions{})
 	if err == nil && operatorCATLSCert != nil {

pkg/controller/sts.go

@@ -400,6 +400,6 @@ func (c *Controller) generateSTSTLSCert() (*string, *string) {
 }
 
 // waitSTSTLSCert Waits for the Operator leader to issue the TLS Certificate for STS
-func (c *Controller) waitSTSTLSCert() (*string, *string) {
+func (c *Controller) waitSTSTLSCert() (string, string) {
 	return c.waitForCertSecretReady("sts", STSTLSSecretName)
 }

pkg/controller/tls.go

@@ -40,7 +40,7 @@ import (
 )
 
 // waitForCertSecretReady Function designed to run in a non-leader operator container to wait for the leader to issue a TLS certificate
-func (c *Controller) waitForCertSecretReady(serviceName string, secretName string) (*string, *string) {
+func (c *Controller) waitForCertSecretReady(serviceName string, secretName string) (string, string) {
 	ctx := context.Background()
 	namespace := miniov2.GetNSFromFile()
 	var publicCertPath, publicKeyPath string
@@ -65,7 +65,7 @@ func (c *Controller) waitForCertSecretReady(serviceName string, secretName strin
 		panic(err)
 	}
 
-	return &publicCertPath, &publicKeyPath
+	return publicCertPath, publicKeyPath
 }
 
 // getCertificateSecret gets a TLS Certificate secret
@@ -81,9 +81,9 @@ func (c *Controller) writeCertSecretToFile(tlsCertSecret *corev1.Secret, service
 		panic(mkdirerr)
 	}
 
-	publicCertPath := fmt.Sprintf("/tmp/%s/public.crt", serviceName)
-	publicKeyPath := fmt.Sprintf("/tmp/%s/private.key", serviceName)
-	publicCertKey, privateKeyKey := c.getKeyNames(tlsCertSecret)
+	publicCertPath := miniov2.GetPublicCertFilePath(serviceName)
+	privateKeyPath := miniov2.GetPrivateKeyFilePath(serviceName)
+	publicCertKey, privateKey := c.getKeyNames(tlsCertSecret)
 
 	if val, ok := tlsCertSecret.Data[publicCertKey]; ok {
 		err := os.WriteFile(publicCertPath, val, 0o644)
@@ -93,15 +93,15 @@ func (c *Controller) writeCertSecretToFile(tlsCertSecret *corev1.Secret, service
 	} else {
 		panic(fmt.Errorf("missing '%s' in %s/%s", publicCertKey, tlsCertSecret.Namespace, tlsCertSecret.Name))
 	}
-	if val, ok := tlsCertSecret.Data[privateKeyKey]; ok {
-		err := os.WriteFile(publicKeyPath, val, 0o644)
+	if val, ok := tlsCertSecret.Data[privateKey]; ok {
+		err := os.WriteFile(privateKeyPath, val, 0o644)
 		if err != nil {
 			panic(err)
 		}
 	} else {
-		panic(fmt.Errorf("missing '%s' in %s/%s", privateKeyKey, tlsCertSecret.Namespace, tlsCertSecret.Name))
+		panic(fmt.Errorf("missing '%s' in %s/%s", privateKey, tlsCertSecret.Namespace, tlsCertSecret.Name))
 	}
-	return publicCertPath, publicKeyPath
+	return publicCertPath, privateKeyPath
 }
 
 // generateTLSCert Generic method to generate TLS Certificartes for different services

resources/base/service.yaml

@@ -7,8 +7,6 @@ metadata:
     app.kubernetes.io/instance: minio-operator
     app.kubernetes.io/name: operator
   namespace: minio-operator
-  annotations:
-    service.beta.openshift.io/serving-cert-secret-name: operator-tls # To solve "remote error: tls: bad certificate" OpenShift v4
 spec:
   type: ClusterIP
   ports:
@@ -25,8 +23,6 @@ metadata:
   labels:
     name: minio-operator
   namespace: minio-operator
-  annotations:
-    service.beta.openshift.io/serving-cert-secret-name: operator-tls # To solve "remote error: tls: bad certificate"
 spec:
   type: ClusterIP
   ports:

testing/deploy-openshift-4.sh → testing/deploy-openshift-4.13.6.sh

@@ -24,17 +24,13 @@ function main() {
 
     install_binaries
 
-    setup_crc
+    setup_crc "4.13.6"
 
     create_marketplace_catalog "certified-operators"
 
-    install_operator "certified-operators" # "community-operators", "redhat-marketplace"
+    install_operator "certified-operators"
 
-    # install_operator
-    # install_tenant
-    # check_tenant_status tenant-lite myminio
-
-    destroy_crc
+    #destroy_crc
 }
 
 time main "$@"