From 437416bfa89a4cc6b9bc54adb0b0a12eb50e3f7d Mon Sep 17 00:00:00 2001 From: pjuarezd Date: Mon, 31 Jul 2023 19:39:59 -0600 Subject: [PATCH 1/2] - Support for TLS Certificates issued by service-ca in Openshift - Allow Operator to be installed on it's own namespace in OperatorHub Fix Openshift test: * Allow select Openshift version * Load Operator image from local build instead of remote registry * Only download operator-sdk if not existing in local Signed-off-by: pjuarezd --- ...o-operator-rhmp.clusterserviceversion.yaml | 1 + .../minio-operator.clusterserviceversion.yaml | 1 + config/manifests/kustomization.yaml | 3 + .../manifests/overlay/console-deployment.yaml | 33 ++ .../manifests/overlay/console_v1_service.yaml | 7 + .../overlay/minio-operator-deployment.yaml | 27 ++ config/manifests/overlay/sts_v1_service.yaml | 7 + olm.sh | 23 +- pkg/apis/minio.min.io/v2/helper.go | 21 + pkg/controller/console.go | 2 +- pkg/controller/main-controller.go | 45 +- pkg/controller/operator.go | 5 + pkg/controller/sts.go | 2 +- pkg/controller/tls.go | 8 +- resources/base/service.yaml | 4 - ...nshift-4.sh => deploy-openshift-4.13.6.sh} | 10 +- testing/openshift-common.sh | 411 +++++++++--------- 17 files changed, 370 insertions(+), 240 deletions(-) create mode 100644 config/manifests/overlay/console-deployment.yaml create mode 100644 config/manifests/overlay/console_v1_service.yaml create mode 100644 config/manifests/overlay/sts_v1_service.yaml rename testing/{deploy-openshift-4.sh => deploy-openshift-4.13.6.sh} (80%) diff --git a/config/manifests/bases/minio-operator-rhmp.clusterserviceversion.yaml b/config/manifests/bases/minio-operator-rhmp.clusterserviceversion.yaml index 0611737bd3f..5b1b4dcb66a 100644 --- a/config/manifests/bases/minio-operator-rhmp.clusterserviceversion.yaml +++ b/config/manifests/bases/minio-operator-rhmp.clusterserviceversion.yaml @@ -13,6 +13,7 @@ metadata: marketplace.openshift.io/remote-workflow: https://marketplace.redhat.com/en-us/operators/minio-operator-rhmp/pricing?utm_source=openshift_console marketplace.openshift.io/support-workflow: https://marketplace.redhat.com/en-us/operators/minio-operator-rhmp/support?utm_source=openshift_console capabilities: "Full Lifecycle" + operatorframework.io/suggested-namespace: minio-operator name: minio-operator-rhmp.v0.0.0 namespace: placeholder spec: diff --git a/config/manifests/bases/minio-operator.clusterserviceversion.yaml b/config/manifests/bases/minio-operator.clusterserviceversion.yaml index 7666f9302e2..c3c44e80a02 100644 --- a/config/manifests/bases/minio-operator.clusterserviceversion.yaml +++ b/config/manifests/bases/minio-operator.clusterserviceversion.yaml @@ -11,6 +11,7 @@ metadata: k8sMinVersion: "1.18" repository: https://github.com/minio/operator capabilities: "Full Lifecycle" + operatorframework.io/suggested-namespace: minio-operator name: minio-operator.v0.0.0 namespace: placeholder spec: diff --git a/config/manifests/kustomization.yaml b/config/manifests/kustomization.yaml index e79a09db6b8..e81650bc8e5 100644 --- a/config/manifests/kustomization.yaml +++ b/config/manifests/kustomization.yaml @@ -8,6 +8,9 @@ resources: patchesStrategicMerge: - overlay/minio-operator-deployment.yaml + - overlay/console-deployment.yaml + - overlay/console_v1_service.yaml + - overlay/sts_v1_service.yaml patchesJson6902: - target: diff --git a/config/manifests/overlay/console-deployment.yaml b/config/manifests/overlay/console-deployment.yaml new file mode 100644 index 00000000000..f362cb8651d --- /dev/null +++ b/config/manifests/overlay/console-deployment.yaml @@ -0,0 +1,33 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: console + namespace: minio-operator +spec: + template: + spec: + containers: + - name: console + volumeMounts: + - name: tls-certificates + mountPath: /tmp/certs + volumes: + - name: tls-certificates + projected: + defaultMode: 420 + sources: + - secret: + name: console-tls + items: + - key: tls.crt + path: public.crt + - key: tls.key + path: private.key + + optional: true + - configMap: + name: openshift-service-ca.crt + items: + - key: service-ca.crt + path: CAs/ca.crt + optional: true diff --git a/config/manifests/overlay/console_v1_service.yaml b/config/manifests/overlay/console_v1_service.yaml new file mode 100644 index 00000000000..cef8d0f155d --- /dev/null +++ b/config/manifests/overlay/console_v1_service.yaml @@ -0,0 +1,7 @@ +apiVersion: v1 +kind: Service +metadata: + name: console + namespace: minio-operator + annotations: + service.beta.openshift.io/serving-cert-secret-name: console-tls diff --git a/config/manifests/overlay/minio-operator-deployment.yaml b/config/manifests/overlay/minio-operator-deployment.yaml index a3612378d28..befb3c15356 100644 --- a/config/manifests/overlay/minio-operator-deployment.yaml +++ b/config/manifests/overlay/minio-operator-deployment.yaml @@ -12,3 +12,30 @@ spec: env: - name: MINIO_OPERATOR_RUNTIME value: "OpenShift" + - name: MINIO_CONSOLE_TLS_ENABLE + value: "on" + volumeMounts: + - name: openshift-service-ca + mountPath: /tmp/service-ca + - name: sts-tls + mountPath: /tmp/sts + volumes: + - name: sts-tls + projected: + defaultMode: 420 + sources: + - secret: + name: sts-tls + items: + - key: tls.crt + path: public.crt + - key: tls.key + path: private.key + optional: true + - name: openshift-service-ca + configMap: + name: openshift-service-ca.crt + items: + - key: service-ca.crt + path: service-ca.crt + optional: true diff --git a/config/manifests/overlay/sts_v1_service.yaml b/config/manifests/overlay/sts_v1_service.yaml new file mode 100644 index 00000000000..b805525360f --- /dev/null +++ b/config/manifests/overlay/sts_v1_service.yaml @@ -0,0 +1,7 @@ +apiVersion: v1 +kind: Service +metadata: + name: sts + namespace: minio-operator + annotations: + service.beta.openshift.io/serving-cert-secret-name: sts-tls \ No newline at end of file diff --git a/olm.sh b/olm.sh index 84d3745037f..2dd4a7ddcf1 100755 --- a/olm.sh +++ b/olm.sh @@ -7,15 +7,19 @@ OPERATOR_SDK_VERSION=v1.22.2 TMP_BIN_DIR="$(mktemp -d)" function install_binaries() { - - echo "Installing temporary Binaries into: $TMP_BIN_DIR"; - echo "Installing temporary operator-sdk binary: $OPERATOR_SDK_VERSION" - ARCH=`{ case "$(uname -m)" in "x86_64") echo -n "amd64";; "aarch64") echo -n "arm64";; *) echo -n "$(uname -m)";; esac; }` - OS=$(uname | awk '{print tolower($0)}') - OPERATOR_SDK_DL_URL=https://github.com/operator-framework/operator-sdk/releases/download/$OPERATOR_SDK_VERSION - curl -L ${OPERATOR_SDK_DL_URL}/operator-sdk_${OS}_${ARCH} -o ${TMP_BIN_DIR}/operator-sdk - OPERATOR_SDK_BIN=${TMP_BIN_DIR}/operator-sdk - chmod +x $OPERATOR_SDK_BIN + + if ! operator-sdk version; then + echo "Installing temporary Binaries into: $TMP_BIN_DIR"; + echo "Installing temporary operator-sdk binary: $OPERATOR_SDK_VERSION" + ARCH=`{ case "$(uname -m)" in "x86_64") echo -n "amd64";; "aarch64") echo -n "arm64";; *) echo -n "$(uname -m)";; esac; }` + OS=$(uname | awk '{print tolower($0)}') + OPERATOR_SDK_DL_URL=https://github.com/operator-framework/operator-sdk/releases/download/$OPERATOR_SDK_VERSION + curl -L ${OPERATOR_SDK_DL_URL}/operator-sdk_${OS}_${ARCH} -o ${TMP_BIN_DIR}/operator-sdk + OPERATOR_SDK_BIN=${TMP_BIN_DIR}/operator-sdk + chmod +x $OPERATOR_SDK_BIN + else + OPERATOR_SDK_BIN="$(which operator-sdk)" + fi } install_binaries @@ -92,6 +96,7 @@ for catalog in "${redhatCatalogs[@]}"; do echo " com.redhat.openshift.versions: v4.8-v4.13" echo " # Annotation to add default bundle channel as potential is declared" echo " operators.operatorframework.io.bundle.channel.default.v1: stable" + echo " operatorframework.io/suggested-namespace: minio-operator" } >> bundles/$catalog/$RELEASE/metadata/annotations.yaml echo "clean root level annotations.yaml" diff --git a/pkg/apis/minio.min.io/v2/helper.go b/pkg/apis/minio.min.io/v2/helper.go index 36e7c18af7b..01048ff8608 100644 --- a/pkg/apis/minio.min.io/v2/helper.go +++ b/pkg/apis/minio.min.io/v2/helper.go @@ -110,6 +110,27 @@ func GetPodCAFromFile() []byte { return namespace } +// GetPodServiceCAFromFile extracts the service-ca.crt certificate in Openshift deployments coming from configmap openshift-service-ca.crt +func GetPodServiceCAFromFile() []byte { + caPath, err := os.ReadFile("/tmp/service-ca/ca.crt") + if err != nil { + return nil + } + return caPath +} + +// GetPublicCertFilePath return the path to the certificate file based for the serviceName +func GetPublicCertFilePath(serviceName string) string { + publicCertPath := fmt.Sprintf("/tmp/%s/public.crt", serviceName) + return publicCertPath +} + +// GetPrivateKeyFilePath return the path to the key file based for the serviceName +func GetPrivateKeyFilePath(serviceName string) string { + publicKeyPath := fmt.Sprintf("/tmp/%s/private.key", serviceName) + return publicKeyPath +} + // GetNSFromFile assumes the operator is running inside a k8s pod and extract the // current namespace from the /var/run/secrets/kubernetes.io/serviceaccount/namespace file func GetNSFromFile() string { diff --git a/pkg/controller/console.go b/pkg/controller/console.go index 17f2be5fb28..fe65950ad55 100644 --- a/pkg/controller/console.go +++ b/pkg/controller/console.go @@ -37,7 +37,7 @@ const ( // DefaultConsoleDeploymentName is the default name of the console deployment DefaultConsoleDeploymentName = "console" // OperatorConsoleTLSSecretName is the name of secret created with TLS certs for Operator console - OperatorConsoleTLSSecretName = "operator-console-tls" + OperatorConsoleTLSSecretName = "console-tls" ) // checkConsoleSvc validates the existence of the MinIO service and validate it's status against what the specification diff --git a/pkg/controller/main-controller.go b/pkg/controller/main-controller.go index 76220966f0d..426c96d75b2 100644 --- a/pkg/controller/main-controller.go +++ b/pkg/controller/main-controller.go @@ -26,6 +26,8 @@ import ( "syscall" "time" + "github.com/minio/operator/pkg/utils" + "github.com/minio/madmin-go/v2" "github.com/minio/operator/pkg/common" xcerts "github.com/minio/pkg/certs" @@ -356,8 +358,13 @@ func (c *Controller) startUpgradeServer(ctx context.Context) <-chan error { // startUpgradeServer Starts the Upgrade tenant API server and notifies the start and stop via notificationChannel func (c *Controller) startSTSAPIServer(ctx context.Context, notificationChannel chan<- *EventNotification) { klog.Infof("Starting STS API server") - publicCertPath, publicKeyPath := c.waitSTSTLSCert() - certsManager, err := xcerts.NewManager(ctx, *publicCertPath, *publicKeyPath, LoadX509KeyPair) + + publicCertPath := miniov2.GetPublicCertFilePath("sts") + privateKeyPath := miniov2.GetPrivateKeyFilePath("sts") + if utils.GetOperatorRuntime() != common.OperatorRuntimeOpenshift { + publicCertPath, privateKeyPath = c.waitSTSTLSCert() + } + certsManager, err := xcerts.NewManager(ctx, publicCertPath, privateKeyPath, LoadX509KeyPair) if err != nil { klog.Errorf("HTTPS STS API server failed to load CA certificate: %v", err) notificationChannel <- &EventNotification{ @@ -367,6 +374,7 @@ func (c *Controller) startSTSAPIServer(ctx context.Context, notificationChannel } serverCertsManager = certsManager c.sts.TLSConfig = c.createTLSConfig(serverCertsManager) + if err := c.sts.ListenAndServeTLS("", ""); !errors.Is(err, http.ErrServerClosed) { // only notify on server failure, on http.ErrServerClosed the channel should be already closed notificationChannel <- &EventNotification{ @@ -410,15 +418,20 @@ func leaderRun(ctx context.Context, c *Controller, threadiness int, stopCh <-cha if isOperatorConsoleTLS() { klog.Info("Waiting for Console TLS") go func() { - klog.Infof("Console TLS is enabled, starting console TLS certificate setup") - err := c.recreateOperatorConsoleCertsIfRequired(ctx) - if err != nil { - panic(err) - } - klog.Infof("Restarting Console pods") - err = c.rolloutRestartDeployment(getConsoleDeploymentName()) - if err != nil { - klog.Errorf("Console deployment didn't restart: %s", err) + if utils.GetOperatorRuntime() == common.OperatorRuntimeOpenshift { + klog.Infof("Console TLS is enabled, skipping TLS certificate generation on Openshift deployment") + } else { + klog.Infof("Console TLS is enabled, starting console TLS certificate setup") + + err := c.recreateOperatorConsoleCertsIfRequired(ctx) + if err != nil { + panic(err) + } + klog.Infof("Restarting Console pods") + err = c.rolloutRestartDeployment(getConsoleDeploymentName()) + if err != nil { + klog.Errorf("Console deployment didn't restart: %s", err) + } } }() } else { @@ -428,8 +441,12 @@ func leaderRun(ctx context.Context, c *Controller, threadiness int, stopCh <-cha // 2) we need to make sure we have STS API certificates (if enabled) if IsSTSEnabled() { go func() { - klog.Infof("STS is enabled, starting STS API certificate setup") - c.generateSTSTLSCert() + if utils.GetOperatorRuntime() == common.OperatorRuntimeOpenshift { + klog.Infof("STS is enabled, skipping TLS certificate generation on Openshift deployment") + } else { + klog.Infof("STS is enabled, starting API certificate setup") + c.generateSTSTLSCert() + } }() } @@ -440,7 +457,7 @@ func leaderRun(ctx context.Context, c *Controller, threadiness int, stopCh <-cha klog.Errorf("Upgrade Server stopped: %v, going to restart", err) upgradeServerChannel = c.startUpgradeServer(ctx) } - // webswerver was instructed to stop, do not attempt to restart + // webserver was instructed to stop, do not attempt to restart continue case <-stopCh: return diff --git a/pkg/controller/operator.go b/pkg/controller/operator.go index c472efa41ad..42a8cb7c16c 100644 --- a/pkg/controller/operator.go +++ b/pkg/controller/operator.go @@ -137,6 +137,11 @@ func (c *Controller) fetchTransportCACertificates() (pool *x509.CertPool) { // Default kubernetes CA certificate rootCAs.AppendCertsFromPEM(miniov2.GetPodCAFromFile()) + // Openshift Service CA certificate + if utils.GetOperatorRuntime() == common.OperatorRuntimeOpenshift { + rootCAs.AppendCertsFromPEM(miniov2.GetPodServiceCAFromFile()) + } + // Custom ca certificate to be used by operator operatorCATLSCert, err := c.kubeClientSet.CoreV1().Secrets(miniov2.GetNSFromFile()).Get(context.Background(), OperatorCATLSSecretName, metav1.GetOptions{}) if err == nil && operatorCATLSCert != nil { diff --git a/pkg/controller/sts.go b/pkg/controller/sts.go index 920d482c7de..7dda413013a 100644 --- a/pkg/controller/sts.go +++ b/pkg/controller/sts.go @@ -400,6 +400,6 @@ func (c *Controller) generateSTSTLSCert() (*string, *string) { } // waitSTSTLSCert Waits for the Operator leader to issue the TLS Certificate for STS -func (c *Controller) waitSTSTLSCert() (*string, *string) { +func (c *Controller) waitSTSTLSCert() (string, string) { return c.waitForCertSecretReady("sts", STSTLSSecretName) } diff --git a/pkg/controller/tls.go b/pkg/controller/tls.go index d5f2c68c1cf..8375a2f4c2b 100644 --- a/pkg/controller/tls.go +++ b/pkg/controller/tls.go @@ -40,7 +40,7 @@ import ( ) // waitForCertSecretReady Function designed to run in a non-leader operator container to wait for the leader to issue a TLS certificate -func (c *Controller) waitForCertSecretReady(serviceName string, secretName string) (*string, *string) { +func (c *Controller) waitForCertSecretReady(serviceName string, secretName string) (string, string) { ctx := context.Background() namespace := miniov2.GetNSFromFile() var publicCertPath, publicKeyPath string @@ -65,7 +65,7 @@ func (c *Controller) waitForCertSecretReady(serviceName string, secretName strin panic(err) } - return &publicCertPath, &publicKeyPath + return publicCertPath, publicKeyPath } // getCertificateSecret gets a TLS Certificate secret @@ -81,8 +81,8 @@ func (c *Controller) writeCertSecretToFile(tlsCertSecret *corev1.Secret, service panic(mkdirerr) } - publicCertPath := fmt.Sprintf("/tmp/%s/public.crt", serviceName) - publicKeyPath := fmt.Sprintf("/tmp/%s/private.key", serviceName) + publicCertPath := miniov2.GetPublicCertFilePath(serviceName) + publicKeyPath := miniov2.GetPrivateKeyFilePath(serviceName) publicCertKey, privateKeyKey := c.getKeyNames(tlsCertSecret) if val, ok := tlsCertSecret.Data[publicCertKey]; ok { diff --git a/resources/base/service.yaml b/resources/base/service.yaml index 515b957888c..f5092c8cf4b 100644 --- a/resources/base/service.yaml +++ b/resources/base/service.yaml @@ -7,8 +7,6 @@ metadata: app.kubernetes.io/instance: minio-operator app.kubernetes.io/name: operator namespace: minio-operator - annotations: - service.beta.openshift.io/serving-cert-secret-name: operator-tls # To solve "remote error: tls: bad certificate" OpenShift v4 spec: type: ClusterIP ports: @@ -25,8 +23,6 @@ metadata: labels: name: minio-operator namespace: minio-operator - annotations: - service.beta.openshift.io/serving-cert-secret-name: operator-tls # To solve "remote error: tls: bad certificate" spec: type: ClusterIP ports: diff --git a/testing/deploy-openshift-4.sh b/testing/deploy-openshift-4.13.6.sh similarity index 80% rename from testing/deploy-openshift-4.sh rename to testing/deploy-openshift-4.13.6.sh index baeb5b66a0e..5659d79eefe 100755 --- a/testing/deploy-openshift-4.sh +++ b/testing/deploy-openshift-4.13.6.sh @@ -24,17 +24,13 @@ function main() { install_binaries - setup_crc + setup_crc "4.13.6" create_marketplace_catalog "certified-operators" - install_operator "certified-operators" # "community-operators", "redhat-marketplace" + install_operator "certified-operators" - # install_operator - # install_tenant - # check_tenant_status tenant-lite myminio - - destroy_crc + #destroy_crc } time main "$@" diff --git a/testing/openshift-common.sh b/testing/openshift-common.sh index 32f19191d3b..656462df318 100644 --- a/testing/openshift-common.sh +++ b/testing/openshift-common.sh @@ -14,7 +14,7 @@ # along with this program. If not, see #OPERATOR_SDK_VERSION=v1.22.2 -ARCH=`{ case "$(uname -m)" in "x86_64") echo -n "amd64";; "aarch64") echo -n "arm64";; *) echo -n "$(uname -m)";; esac; }` +ARCH=$({ case "$(uname -m)" in "x86_64") echo -n "amd64" ;; "aarch64") echo -n "arm64" ;; *) echo -n "$(uname -m)" ;; esac }) MACHINE="$(uname -m)" OS=$(uname | awk '{print tolower($0)}') # shellcheck disable=SC2155 @@ -22,224 +22,235 @@ export TMP_BIN_DIR="$(mktemp -d)" function install_binaries() { - echo -e "\e[34mInstalling temporal binaries in $TMP_BIN_DIR\e[0m" - - #echo "kubectl" - #curl -#L "https://dl.k8s.io/release/v1.23.1/bin/$OS/$ARCH/kubectl" -o $TMP_BIN_DIR/kubectl - #chmod +x $TMP_BIN_DIR/kubectl - - #echo "mc" - #curl -#L "https://dl.min.io/client/mc/release/${OS}-${ARCH}/mc" -o $TMP_BIN_DIR/mc - #chmod +x $TMP_BIN_DIR/mc - - echo "yq" - curl -#L "https://github.com/mikefarah/yq/releases/latest/download/yq_${OS}_${ARCH}" -o $TMP_BIN_DIR/yq - chmod +x $TMP_BIN_DIR/yq - - # latest kubectl and oc - # curl -#L "https://mirror.openshift.com/pub/openshift-v4/$MACHINE/clients/ocp/stable/openshift-client-$OS.tar.gz" -o $TMP_BIN_DIR/openshift-client-$OS.tar.gz - # tar -zxvf openshift-client-$OS.tar.gz - - echo "opm" - curl -#L "https://mirror.openshift.com/pub/openshift-v4/$MACHINE/clients/ocp/stable/opm-$OS.tar.gz" -o $TMP_BIN_DIR/opm-$OS.tar.gz - tar -zxf $TMP_BIN_DIR/opm-$OS.tar.gz -C $TMP_BIN_DIR/ - chmod +x $TMP_BIN_DIR/opm - - echo "crc" - curl -#L "https://developers.redhat.com/content-gateway/rest/mirror/pub/openshift-v4/clients/crc/latest/crc-$OS-$ARCH.tar.xz" -o $TMP_BIN_DIR/crc-$OS-$ARCH.tar.xz - tar -xJf $TMP_BIN_DIR/crc-$OS-$ARCH.tar.xz -C $TMP_BIN_DIR/ --strip-components=1 - chmod +x $TMP_BIN_DIR/crc - - #echo "operator-sdk" - #curl -#L "https://github.com/operator-framework/operator-sdk/releases/download/$OPERATOR_SDK_VERSION/operator-sdk_${OS}_${ARCH}" -o ${TMP_BIN_DIR}/operator-sdk - #chmod +x $TMP_BIN_DIR/operator-sdk + echo -e "\e[34mInstalling temporal binaries in $TMP_BIN_DIR\e[0m" + + #echo "kubectl" + #curl -#L "https://dl.k8s.io/release/v1.23.1/bin/$OS/$ARCH/kubectl" -o $TMP_BIN_DIR/kubectl + #chmod +x $TMP_BIN_DIR/kubectl + + #echo "mc" + #curl -#L "https://dl.min.io/client/mc/release/${OS}-${ARCH}/mc" -o $TMP_BIN_DIR/mc + #chmod +x $TMP_BIN_DIR/mc + + echo "yq" + curl -#L "https://github.com/mikefarah/yq/releases/latest/download/yq_${OS}_${ARCH}" -o $TMP_BIN_DIR/yq + chmod +x $TMP_BIN_DIR/yq + + # latest kubectl and oc + # curl -#L "https://mirror.openshift.com/pub/openshift-v4/$MACHINE/clients/ocp/stable/openshift-client-$OS.tar.gz" -o $TMP_BIN_DIR/openshift-client-$OS.tar.gz + # tar -zxvf openshift-client-$OS.tar.gz + + echo "opm" + curl -#L "https://mirror.openshift.com/pub/openshift-v4/$MACHINE/clients/ocp/stable/opm-$OS.tar.gz" -o $TMP_BIN_DIR/opm-$OS.tar.gz + tar -zxf $TMP_BIN_DIR/opm-$OS.tar.gz -C $TMP_BIN_DIR/ + chmod +x $TMP_BIN_DIR/opm + + echo "crc" + curl -#L "https://developers.redhat.com/content-gateway/rest/mirror/pub/openshift-v4/clients/crc/latest/crc-$OS-$ARCH.tar.xz" -o $TMP_BIN_DIR/crc-$OS-$ARCH.tar.xz + tar -xJf $TMP_BIN_DIR/crc-$OS-$ARCH.tar.xz -C $TMP_BIN_DIR/ --strip-components=1 + chmod +x $TMP_BIN_DIR/crc + + #echo "operator-sdk" + #curl -#L "https://github.com/operator-framework/operator-sdk/releases/download/$OPERATOR_SDK_VERSION/operator-sdk_${OS}_${ARCH}" -o ${TMP_BIN_DIR}/operator-sdk + #chmod +x $TMP_BIN_DIR/operator-sdk } function remove_temp_binaries() { - echo -e "\e[34mRemoving temporary binaries in: $TMP_BIN_DIR\e[0m" - rm -rf $TMP_BIN_DIR + echo -e "\e[34mRemoving temporary binaries in: $TMP_BIN_DIR\e[0m" + rm -rf $TMP_BIN_DIR } yell() { echo "$0: $*" >&2; } die() { - yell "$*" - destroy_crc && exit 111 + yell "$*" + exit 111 } try() { "$@" || die "cannot $*"; } function setup_crc() { - echo -e "\e[34mConfiguring crc\e[0m" - export PATH="$TMP_BIN_DIR:$PATH" - crc config set consent-telemetry no - crc config set skip-check-root-user true - crc config set kubeadmin-password "crclocal" - crc setup - crc start -c 12 -m 20480 - eval $(crc oc-env) - eval $(crc podman-env) - # this creates a symlink "podman" from the "podman-remote", as a hack to solve the a issue with opm: - # opm has hardcoded the command name "podman" causing the index creation to fail - # https://github.com/operator-framework/operator-registry/blob/67e6777b5f5f9d337b94da98b8c550c231a8b47c/pkg/containertools/factory_podman.go#L32 - ocpath=$(dirname $(which podman-remote)) - ln -sf $ocpath/podman-remote $ocpath/podman - try crc version - echo "Waiting for podman vm come online (5m timeout)" - try timeout 600 bash -c -- 'while ! podman image ls 2> /dev/null; do sleep 1 && printf ".";done' + # Set bundle + # Example options + # crc_libvirt_4.11.7_amd64 + # crc_libvirt_4.12.0_amd64 + # crc_libvirt_4.12.5_amd64 + # crc_libvirt_4.13.0_amd64 + # crc_libvirt_4.13.6_amd64 + + bundle_version="$1" + if [ -z "$bundle_version" ]; then + bundle_version="4.13.6" + fi + bundle="crc_libvirt_${bundle_version}_amd64.crcbundle" + echo -e "\e[34mConfiguring crc\e[0m" + export PATH="$TMP_BIN_DIR:$PATH" + crc config set consent-telemetry no + crc config set skip-check-root-user true + crc config set kubeadmin-password "crclocal" + crc setup + crc start -b $bundle -c 12 -m 20480 + eval $(crc oc-env) + eval $(crc podman-env) + # this creates a symlink "podman" from the "podman-remote", as a hack to solve the a issue with opm: + # opm has hardcoded the command name "podman" causing the index creation to fail + # https://github.com/operator-framework/operator-registry/blob/67e6777b5f5f9d337b94da98b8c550c231a8b47c/pkg/containertools/factory_podman.go#L32 + ocpath=$(dirname $(which podman-remote)) + ln -sf $ocpath/podman-remote $ocpath/podman + try crc version + echo "Waiting for podman vm come online (5m timeout)" + try timeout 600 bash -c -- 'while ! podman image ls 2> /dev/null; do sleep 1 && printf ".";done' + oc login -u kubeadmin -p crclocal https://api.crc.testing:6443 --insecure-skip-tls-verify=true } function destroy_crc() { - echo -e "\e[34mdestroy_crc\e[0m" - - # To allow the execution without killing the cluster at the end of the test - # Use below statement to automatically test and kill cluster at the end: - # `unset OPERATOR_DEV_TEST` - # Use below statement to test and keep cluster alive at the end!: - # `export OPERATOR_DEV_TEST="ON"` - if [[ -z "${OPERATOR_DEV_TEST}" ]]; then - # OPERATOR_DEV_TEST is not defined, hence destroy_kind - echo "Cluster will be destroyed for automated testing" - crc stop - crc delete -f - remove_temp_binaries - else - echo -e "\e[33mCluster will remain alive for manual testing\e[0m" - echo "Use the following env varianbles setup" - echo "export PATH=$TMP_BIN_DIR:\$PATH" - echo "eval \$(crc oc-env)" - echo "eval \$(crc podman-env)" - fi + echo -e "\e[34mdestroy_crc\e[0m" + + # To allow the execution without killing the cluster at the end of the test + # Use below statement to automatically test and kill cluster at the end: + # `unset OPERATOR_DEV_TEST` + # Use below statement to test and keep cluster alive at the end!: + # `export OPERATOR_DEV_TEST="ON"` + if [[ -z "${OPERATOR_DEV_TEST}" ]]; then + # OPERATOR_DEV_TEST is not defined, hence destroy_kind + echo "Cluster will be destroyed for automated testing" + crc stop + crc delete -f + remove_temp_binaries + else + echo -e "\e[33mCluster will remain alive for manual testing\e[0m" + echo "Use the following env varianbles setup" + echo "export PATH=$TMP_BIN_DIR:\$PATH" + echo "eval \$(crc oc-env)" + echo "eval \$(crc podman-env)" + fi } -function create_marketplace_catalog(){ - # https://redhat-connect.gitbook.io/certified-operator-guide/ocp-deployment/openshift-deployment - # https://redhat-connect.gitbook.io/certified-operator-guide/ocp-deployment/operator-metadata/bundle-directory - # https://operatorhub.io/preview - - - # Obtain catalog - catalog="$1" - if [ -z "$catalog" ] - then - die "missing catalog to install" - fi - - echo "Create Marketplace for catalog '$catalog'" - - registry="default-route-openshift-image-registry.apps-crc.testing" - operatorNamespace="openshift-operators" - marketplaceNamespace="openshift-marketplace" - operatorContainerImage="$registry/$operatorNamespace/operator:noop" - bundleContainerImage="$registry/$marketplaceNamespace/operator-bundle:noop" - indexContainerImage="$registry/$marketplaceNamespace/minio-operator-index:noop" - package="minio-operator" - if [[ "$catalog" == "redhat-marketplace" ]] - then - package=minio-operator-rhmp - fi - - echo "Compiling operator in current branch" - (cd "${SCRIPT_DIR}/.." && make operator && podman build --quiet --no-cache -t $operatorContainerImage .) - - echo "push operator image to crc registry" - podman login -u `oc whoami` -p `oc whoami --show-token` $registry/$operatorNamespace --tls-verify=false - podman push $operatorContainerImage --tls-verify=false - - echo "Image Stream for operator:" - oc get is -n $operatorNamespace operator - try oc set image-lookup operator -n $operatorNamespace - - echo "Compiling operator bundle for $catalog" - cp -r "${SCRIPT_DIR}/../$catalog/." ${SCRIPT_DIR}/openshift/bundle - yq -i ".metadata.annotations.containerImage |= (\"${operatorContainerImage}\")" ${SCRIPT_DIR}/openshift/bundle/manifests/$package.clusterserviceversion.yaml - yq -i ".annotations.\"operators.operatorframework.io.bundle.package.v1\" |= (\"${package}-noop\")" ${SCRIPT_DIR}/openshift/bundle/metadata/annotations.yaml - (cd "${SCRIPT_DIR}/.." && podman build --quiet --no-cache -t $bundleContainerImage -f ${SCRIPT_DIR}/openshift/bundle.Dockerfile ${SCRIPT_DIR}/openshift) - podman login -u `oc whoami` -p `oc whoami --show-token` $registry --tls-verify=false - - echo "push operator-bundle to crc registry" - podman push $bundleContainerImage --tls-verify=false - - echo "Image Stream for operator-bundle" - oc get is -n $marketplaceNamespace operator-bundle - try oc set image-lookup -n $marketplaceNamespace operator-bundle - - echo "Compiling marketplace index" - opm index add --bundles $bundleContainerImage --tag $indexContainerImage --skip-tls-verify=true - - echo "push minio-operator-index to crc registry" - podman push $indexContainerImage --tls-verify=false - echo "Image Stream for minio-operator-index" - try oc set image-lookup -n $marketplaceNamespace minio-operator-index - - echo "Wait for ImageStream minio-operator-index to be local available" - try oc wait -n $marketplaceNamespace is \ - --for=jsonpath='{.spec.lookupPolicy.local}'=true \ - --field-selector metadata.name=minio-operator-index \ - --timeout=300s - - echo "Create 'Test Minio Operators' marketplace catalog source" - oc create -f ${SCRIPT_DIR}/openshift/test-operator-catalogsource.yaml - sleep 5 - echo "Catalog Source:" - oc get catalogsource -n $marketplaceNamespace minio-test-operators - - catalogSourcePod=$(oc get pods -n $marketplaceNamespace -ojson| jq -r '.items[] | select(.metadata.name | startswith("minio-test-operators")) | .metadata.name') - - # Hack, for some reason the original catalgosource pod cannot pull the image. - # deleting the pod forces to create a new pod and the newly scheduled pod does have the grants to access the image registry - echo "deleting pod $catalogSourcePod" -n $marketplaceNamespace - oc delete pod $catalogSourcePod -n $marketplaceNamespace - - echo "Waiting for Package manifest to be ready (5m timeout)" - try timeout 300 bash -c -- 'while ! oc get packagemanifests -n '"$marketplaceNamespace"' | grep "Test Minio Operators" 2> /dev/null; do sleep 1 && printf ".";done' +function create_marketplace_catalog() { + # https://redhat-connect.gitbook.io/certified-operator-guide/ocp-deployment/openshift-deployment + # https://redhat-connect.gitbook.io/certified-operator-guide/ocp-deployment/operator-metadata/bundle-directory + # https://operatorhub.io/preview + + # Obtain catalog + catalog="$1" + if [ -z "$catalog" ]; then + die "missing catalog to install" + fi + + echo "Create Marketplace for catalog '$catalog'" + + registry="default-route-openshift-image-registry.apps-crc.testing" + operatorNamespace="openshift-operators" + marketplaceNamespace="openshift-marketplace" + operatorContainerImage="$registry/$operatorNamespace/operator:noop" + bundleContainerImage="$registry/$marketplaceNamespace/operator-bundle:noop" + indexContainerImage="$registry/$marketplaceNamespace/minio-operator-index:noop" + package="minio-operator" + if [[ "$catalog" == "redhat-marketplace" ]]; then + package=minio-operator-rhmp + fi + + echo "Compiling operator in current branch" + (cd "${SCRIPT_DIR}/.." && make operator && podman build --quiet --no-cache -t $operatorContainerImage .) + + echo "push operator image to crc registry" + podman login -u $(oc whoami) -p $(oc whoami --show-token) $registry/$operatorNamespace --tls-verify=false + podman push $operatorContainerImage --tls-verify=false + + echo "Image Stream for operator:" + oc get is -n $operatorNamespace operator + try oc set image-lookup operator -n $operatorNamespace + + echo "Compiling operator bundle for $catalog" + cp -r "${SCRIPT_DIR}/../$catalog/." ${SCRIPT_DIR}/openshift/bundle + yq -i ".metadata.annotations.containerImage |= (\"${operatorContainerImage}\")" ${SCRIPT_DIR}/openshift/bundle/manifests/$package.clusterserviceversion.yaml + yq -i ".spec.install.spec.deployments.[0].spec.template.spec.containers.[0].image |= (\"${operatorContainerImage}\")" ${SCRIPT_DIR}/openshift/bundle/manifests/$package.clusterserviceversion.yaml + yq -i ".spec.install.spec.deployments.[1].spec.template.spec.containers.[0].image |= (\"${operatorContainerImage}\")" ${SCRIPT_DIR}/openshift/bundle/manifests/$package.clusterserviceversion.yaml + yq -i ".annotations.\"operators.operatorframework.io.bundle.package.v1\" |= (\"${package}-noop\")" ${SCRIPT_DIR}/openshift/bundle/metadata/annotations.yaml + (cd "${SCRIPT_DIR}/.." && podman build --quiet --no-cache -t $bundleContainerImage -f ${SCRIPT_DIR}/openshift/bundle.Dockerfile ${SCRIPT_DIR}/openshift) + podman login -u $(oc whoami) -p $(oc whoami --show-token) $registry --tls-verify=false + + echo "push operator-bundle to crc registry" + podman push $bundleContainerImage --tls-verify=false + + echo "Image Stream for operator-bundle" + oc get is -n $marketplaceNamespace operator-bundle + try oc set image-lookup -n $marketplaceNamespace operator-bundle + + echo "Compiling marketplace index" + opm index add --bundles $bundleContainerImage --tag $indexContainerImage --skip-tls-verify=true + + echo "push minio-operator-index to crc registry" + podman push $indexContainerImage --tls-verify=false + echo "Image Stream for minio-operator-index" + try oc set image-lookup -n $marketplaceNamespace minio-operator-index + + echo "Wait for ImageStream minio-operator-index to be local available" + try oc wait -n $marketplaceNamespace is \ + --for=jsonpath='{.spec.lookupPolicy.local}'=true \ + --field-selector metadata.name=minio-operator-index \ + --timeout=300s + + echo "Create 'Test Minio Operators' marketplace catalog source" + oc create -f ${SCRIPT_DIR}/openshift/test-operator-catalogsource.yaml + sleep 5 + echo "Catalog Source:" + oc get catalogsource -n $marketplaceNamespace minio-test-operators + + catalogSourcePod=$(oc get pods -n $marketplaceNamespace -ojson | jq -r '.items[] | select(.metadata.name | startswith("minio-test-operators")) | .metadata.name') + + # Hack, for some reason the original catalgosource pod cannot pull the image. + # deleting the pod forces to create a new pod and the newly scheduled pod does have the grants to access the image registry + echo "deleting pod $catalogSourcePod" -n $marketplaceNamespace + oc delete pod $catalogSourcePod -n $marketplaceNamespace + + echo "Waiting for Package manifest to be ready (5m timeout)" + try timeout 300 bash -c -- 'while ! oc get packagemanifests -n '"$marketplaceNamespace"' | grep "Test Minio Operators" 2> /dev/null; do sleep 1 && printf ".";done' } function install_operator() { - # Obtain catalog - catalog="$1" - if [ -z "$catalog" ] - then - catalog="certified-operators" - fi - - echo -e "\e[34mInstalling Operator from catalog '$catalog'\e[0m" - - try oc create -f ${SCRIPT_DIR}/openshift/test-subscription.yaml - - echo "Subscription:" - try oc get sub -n openshift-operators test-subscription - #we wait a moment for the resource to get a status field - sleep 10s - - echo "Wait subscription to be ready (10m timeout)" - try oc wait -n openshift-operators \ - --for=jsonpath='{.status.state}'=AtLatestKnown subscription\ - --field-selector metadata.name=$(oc get subscription -n openshift-operators -o json | jq -r '.items[0] | .metadata.name') \ - --timeout=600s - - echo "Install plan:" - try oc get installplan -n openshift-operators - - echo "Waiting for install plan to be completed (10m timeout)" - oc wait -n openshift-operators \ - --for=jsonpath='{.status.phase}'=Complete installplan \ - --field-selector metadata.name=$(oc get installplan -n openshift-operators -o json | jq -r '.items[0] | .metadata.name') \ - --timeout=600s - - echo "Deployment:" - oc -n openshift-operators get deployment minio-operator - - echo "Waiting for Operator Deployment to come online (5m timeout)" - try oc wait -n openshift-operators deployment \ - --for=condition=Available \ - --field-selector metadata.name=minio-operator \ - --timeout=300s - - echo "start - get data to verify proper image is being used" - echo "Pods:" - oc get pods --namespace openshift-operators - echo "Images:" - oc describe pods -n openshift-operators | grep Image -} \ No newline at end of file + # Obtain catalog + catalog="$1" + if [ -z "$catalog" ]; then + catalog="certified-operators" + fi + + echo -e "\e[34mInstalling Operator from catalog '$catalog'\e[0m" + + try oc create -f ${SCRIPT_DIR}/openshift/test-subscription.yaml + + echo "Subscription:" + try oc get sub -n openshift-operators test-subscription + #we wait a moment for the resource to get a status field + sleep 10s + + echo "Wait subscription to be ready (10m timeout)" + try oc wait -n openshift-operators \ + --for=jsonpath='{.status.state}'=AtLatestKnown subscription --field-selector metadata.name=$(oc get subscription -n openshift-operators -o json | jq -r '.items[0] | .metadata.name') \ + --timeout=600s + + echo "Install plan:" + try oc get installplan -n openshift-operators + + echo "Waiting for install plan to be completed (10m timeout)" + oc wait -n openshift-operators \ + --for=jsonpath='{.status.phase}'=Complete installplan \ + --field-selector metadata.name=$(oc get installplan -n openshift-operators -o json | jq -r '.items[0] | .metadata.name') \ + --timeout=600s + + echo "Deployment:" + oc -n openshift-operators get deployment minio-operator + + echo "Waiting for Operator Deployment to come online (5m timeout)" + try oc wait -n openshift-operators deployment \ + --for=condition=Available \ + --field-selector metadata.name=minio-operator \ + --timeout=300s + + echo "start - get data to verify proper image is being used" + echo "Pods:" + oc get pods --namespace openshift-operators + echo "Images:" + oc describe pods -n openshift-operators | grep Image +} From ebab8cd4e87513d63679d0d1cf9b0e40e83e61bf Mon Sep 17 00:00:00 2001 From: pjuarezd Date: Mon, 21 Aug 2023 16:46:17 -0600 Subject: [PATCH 2/2] rename variables Signed-off-by: pjuarezd --- pkg/apis/minio.min.io/v2/helper.go | 4 ++-- pkg/controller/tls.go | 12 ++++++------ 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/pkg/apis/minio.min.io/v2/helper.go b/pkg/apis/minio.min.io/v2/helper.go index 01048ff8608..fc7af9f79d0 100644 --- a/pkg/apis/minio.min.io/v2/helper.go +++ b/pkg/apis/minio.min.io/v2/helper.go @@ -127,8 +127,8 @@ func GetPublicCertFilePath(serviceName string) string { // GetPrivateKeyFilePath return the path to the key file based for the serviceName func GetPrivateKeyFilePath(serviceName string) string { - publicKeyPath := fmt.Sprintf("/tmp/%s/private.key", serviceName) - return publicKeyPath + privateKey := fmt.Sprintf("/tmp/%s/private.key", serviceName) + return privateKey } // GetNSFromFile assumes the operator is running inside a k8s pod and extract the diff --git a/pkg/controller/tls.go b/pkg/controller/tls.go index 8375a2f4c2b..6bb46b984e7 100644 --- a/pkg/controller/tls.go +++ b/pkg/controller/tls.go @@ -82,8 +82,8 @@ func (c *Controller) writeCertSecretToFile(tlsCertSecret *corev1.Secret, service } publicCertPath := miniov2.GetPublicCertFilePath(serviceName) - publicKeyPath := miniov2.GetPrivateKeyFilePath(serviceName) - publicCertKey, privateKeyKey := c.getKeyNames(tlsCertSecret) + privateKeyPath := miniov2.GetPrivateKeyFilePath(serviceName) + publicCertKey, privateKey := c.getKeyNames(tlsCertSecret) if val, ok := tlsCertSecret.Data[publicCertKey]; ok { err := os.WriteFile(publicCertPath, val, 0o644) @@ -93,15 +93,15 @@ func (c *Controller) writeCertSecretToFile(tlsCertSecret *corev1.Secret, service } else { panic(fmt.Errorf("missing '%s' in %s/%s", publicCertKey, tlsCertSecret.Namespace, tlsCertSecret.Name)) } - if val, ok := tlsCertSecret.Data[privateKeyKey]; ok { - err := os.WriteFile(publicKeyPath, val, 0o644) + if val, ok := tlsCertSecret.Data[privateKey]; ok { + err := os.WriteFile(privateKeyPath, val, 0o644) if err != nil { panic(err) } } else { - panic(fmt.Errorf("missing '%s' in %s/%s", privateKeyKey, tlsCertSecret.Namespace, tlsCertSecret.Name)) + panic(fmt.Errorf("missing '%s' in %s/%s", privateKey, tlsCertSecret.Namespace, tlsCertSecret.Name)) } - return publicCertPath, publicKeyPath + return publicCertPath, privateKeyPath } // generateTLSCert Generic method to generate TLS Certificartes for different services