[Bug]: Standalone Topic Operator entity-operator.key error

[miclip]

Bug Description

Following documentation on setting up the User and Topic operators with AWS MSK. User operator is working and creating clients/users. Receiving the error below when the Topic pod starts.

kl strimzi-topic-operator-554c6bd48f-nhn9z

Defaulted container "strimzi-topic-operator" out of: strimzi-topic-operator, build-truststore (init)
removed directory '/tmp/topic-operator'
Preparing key store certificates for internal communication
Could not open file or uri for loading private key from -inkey file from /etc/eto-certs/entity-operator.key
806B2B44167F0000:error:16000069:STORE routines:ossl_store_get0_loader_int:unregistered scheme:crypto/store/store_register.c:237:scheme=file
806B2B44167F0000:error:80000002:system library:file_open:No such file or directory:providers/implementations/storemgmt/file_store.c:267:calling stat(/etc/eto-certs/entity-operator.key)

Steps to reproduce

apiVersion: apps/v1
kind: Deployment
metadata:
  name: strimzi-topic-operator
  labels:
    app: strimzi
spec:
  replicas: 1
  selector:
    matchLabels:
      name: strimzi-topic-operator
  template:
    metadata:
      labels:
        name: strimzi-topic-operator
    spec:
      serviceAccountName: strimzi-topic-operator
      initContainers:
      - name: build-truststore
        image: quay.io/strimzi/operator:0.43.0
        imagePullPolicy: IfNotPresent
        command:
        - 'sh'
        - '-c'
        - |
          find_keytool_or_fail_fast() {
            local has_keytool
            ls "${JAVA_HOME}"/bin/keytool
            has_keytool=$?

            if [ ${has_keytool} -ne 0 ]; then
              exit ${has_keytool}
            fi
          }

          import_cert() {
            local pemfile="${1}"
            local alias="${2}"

            echo "Adding ${pemfile} to truststore"

            # Have to use cat instead of -file
            # because keytool won't understand all of the filenames!
            cat "${pemfile}" | "${JAVA_HOME}"/bin/keytool \
              -noprompt \
              -import \
              -trustcacerts \
              -alias "${alias}" \
              -keystore "${TRUSTSTORE_FILE}" \
              -storepass "${TRUSTSTORE_PASSWORD}"
          }

          get_alias() {
            local pemfile="${1}"
            basename "${pemfile}" .crt
          }

          add_ca_certs() {
            local has_ca_certs
            ls /certs/*.crt
            has_ca_certs=$?

            if [ ${has_ca_certs} -eq 0 ]; then
              for cert in /certs/*.crt; do
                import_cert "${cert}" "$(get_alias $cert)"
              done
            fi
          }
          add_system_certs() {
            for cert in $OS_CERTS_DIR/ca-bundle.crt; do
              import_cert "${cert}" "$(get_alias $cert)"
            done
          }

          main() {
            find_keytool_or_fail_fast
            add_ca_certs
            add_system_certs
          }

          main
        env:
        - name: TRUSTSTORE_FILE
          value: /etc/truststore/truststore.jks
        - name: TRUSTSTORE_PASSWORD
          value: changeme
        - name: OS_CERTS_DIR
          value: /etc/ssl/certs
        volumeMounts:
        - mountPath: /certs/cluster-ca.crt
          name: cluster-ca
          subPath: ca.crt
        - mountPath: /certs/ca.crt
          name: user-creds
          subPath: ca.crt
        - mountPath: /certs/user.crt
          name: user-creds
          subPath: user.crt
        - mountPath: /etc/truststore
          name: truststore-dir
      volumes:
        - name: truststore-dir
          emptyDir: {}
        - name: strimzi-tmp
          emptyDir:
            medium: Memory
            sizeLimit: 5Mi
        - name: entity-operator-crt
          secret:
            items:
            - key: entity-operator.crt
              path: entity-operator.crt
            secretName: msk-op
        - name: entity-operator-key
          secret:
            items:
            - key: entity-operator.key
              path: entity-operator.key
            secretName: msk-op
        - name: cluster-ca
          secret:
            items:
            - key: ca.crt
              path: ca.crt
            secretName: msk-ca
        - name: user-creds
          secret:
            items:
            - key: ca.crt
              path: ca.crt
            - key: user.crt
              path: user.crt
            - key: user.key
              path: user.key
            - key: user.p12
              path: user.p12
            secretName: kafka.user.kafka-topic-operator-kafka-topic-operator
      containers:
        - name: strimzi-topic-operator
          image: quay.io/strimzi/operator:0.43.0
          args:
            - /opt/strimzi/bin/topic_operator_run.sh
          volumeMounts:
            - name: strimzi-tmp
              mountPath: /tmp
              #  experimented with providing the entity operator cert/key
              # - mountPath: /etc/eto-certs/entity-operator.key
              #   name: user-key
              #   subPath: user.key
              # - mountPath: /etc/eto-certs/entity-operator.crt
              #   name: user-crt
              #   subPath: user.crt
            - mountPath: /certs/ca.crt
              name: user-creds
              subPath: ca.crt
            - mountPath: /certs/user.crt
              name: user-creds
              subPath: user.crt
            - mountPath: /certs/user.key
              name: user-creds
              subPath: user.key
            - mountPath: /certs/user.p12
              name: user-creds
              subPath: user.p12
            - name: truststore-dir
              mountPath: /etc/truststore
              readOnly: true
            - name: cluster-ca
              mountPath: /etc/tls-sidecar/cluster-ca-certs/cluster-ca.crt
              subPath: ca.crt
          env:
            - name: STRIMZI_RESOURCE_LABELS
              value: "strimzi.io/cluster=msk-poc"
            - name: STRIMZI_KAFKA_BOOTSTRAP_SERVERS
              value: b-2.mskpoc.....kafka.us-west-2.amazonaws.com:9094,b-1.mskpoc.....kafka.us-west-2.amazonaws.com:9094
            - name: STRIMZI_FULL_RECONCILIATION_INTERVAL_MS
              value: "120000"
            - name: STRIMZI_LOG_LEVEL
              value: TRACE
            - name: STRIMZI_TLS_ENABLED
              value: "true"
            - name: STRIMZI_TLS_AUTH_ENABLED
              value: "true"
            - name: STRIMZI_PUBLIC_CA
              value: "true"
            - name: STRIMZI_NAMESPACE
              valueFrom:
                fieldRef:
                  fieldPath: metadata.namespace
            - name: STRIMZI_TRUSTSTORE_LOCATION
              value: /etc/truststore/truststore.jks
            - name: STRIMZI_TRUSTSTORE_PASSWORD
              value: changeme
            - name: STRIMZI_KEYSTORE_LOCATION
              value: /certs/user.p12
            - name: STRIMZI_KEYSTORE_PASSWORD
              valueFrom:
                secretKeyRef:
                  key: user.password
                  name: kafka.user.kafka-topic-operator-kafka-topic-operator
           ....
  strategy:
    type: Recreate

Expected behavior

No response

Strimzi version

0.43.0

Kubernetes version

Client Version: v1.31.0 Kustomize Version: v5.4.2 Server Version: v1.31.0-eks-a737599

Installation method

YAML

Infrastructure

EKS

Configuration files and logs

No response

Additional context

No response

[scholzj]

I guess you misconfigured it and the key is missing? You seem to be mounting the certificates in completely different places, so no wonder it does not work.

[miclip]

I'm providing both the STRIMZI_TRUSTSTORE_LOCATION and STRIMZI_KEYSTORE_LOCATION so not sure why it matters where they're located.

[scholzj]

Because that is what the key is expected to be named when TLS is used.

[scholzj]

I'm providing both the STRIMZI_TRUSTSTORE_LOCATION and STRIMZI_KEYSTORE_LOCATION so not sure why it matters where they're located.

The User and Topic operators are intended to be used as a part of Strimzi Kafka clusters. They are not necessarily as configurable as you might want them because they have a narrow use case. So if you want to use it, you might need to follow the startup logic it has and place the keys into the right places with the right names.

[miclip]

okay, understood that standalone isn't the primary use case, I'll try that. Thanks

[miclip]

thanks for the guidance, I have it working.

apiVersion: apps/v1
kind: Deployment
metadata:
  name: strimzi-topic-operator
  labels:
    app: strimzi
spec:
  replicas: 1
  selector:
    matchLabels:
      name: strimzi-topic-operator
  template:
    metadata:
      labels:
        name: strimzi-topic-operator
    spec:
      serviceAccountName: strimzi-topic-operator
      volumes:
      - emptyDir: {}
        name: entity-operator-creds
      - name: strimzi-tmp
        emptyDir:
          medium: Memory
          sizeLimit: 5Mi
      - name: user-creds
        secret:
          items:
          - key: ca.crt
            path: ca.crt
          - key: user.crt
            path: user.crt
          - key: user.key
            path: user.key
          - key: user.p12
            path: user.p12
          secretName: kafka.user.kafka-topic-operator-kafka-topic-operator
      initContainers:
      - name: entity-operator-cert-helper
        image: quay.io/strimzi/operator:0.43.0
        imagePullPolicy: IfNotPresent
        name: kafka-msk-cert-helper
        command: ["/bin/bash", "-c", "cat /etc/eto-certs/entity-operator-tmp.crt /etc/eto-certs/ca.crt > /etc/eto-certs/entity-operator.crt "]
        volumeMounts:
        - mountPath: /etc/eto-certs
          name: entity-operator-creds
        - mountPath: /etc/eto-certs/ca.crt
          name: user-creds
          subPath: ca.crt
        - mountPath: /etc/eto-certs/entity-operator-tmp.crt
          name: user-creds
          subPath: user.crt
      containers:
        - name: strimzi-topic-operator
          image: quay.io/strimzi/operator:0.43.0
          args:
            - /opt/strimzi/bin/topic_operator_run.sh
          volumeMounts:
            - mountPath: /etc/eto-certs
              name: entity-operator-creds
            - name: strimzi-tmp
              mountPath: /tmp
            - mountPath: /etc/eto-certs/ca.crt
              name: user-creds
              subPath: ca.crt
            - mountPath: /etc/eto-certs/entity-operator.key
              name: user-creds
              subPath: user.key
            - mountPath: /tmp/user.p12
              name: user-creds
              subPath: user.p12
          env:
            - name: STRIMZI_RESOURCE_LABELS
              value: "strimzi.io/cluster=msk-poc"
            - name: STRIMZI_KAFKA_BOOTSTRAP_SERVERS
              value: b-2.mskpoc...kafka.us-west-2.amazonaws.com:9094,b-1.mskpoc....kafka.us-west-2.amazonaws.com:9094
            - name: STRIMZI_FULL_RECONCILIATION_INTERVAL_MS
              value: "120000"
            - name: STRIMZI_LOG_LEVEL
              value: TRACE
            - name: STRIMZI_TLS_ENABLED
              value: "true"
            - name: STRIMZI_TLS_AUTH_ENABLED
              value: "true"
            - name: STRIMZI_PUBLIC_CA
              value: "true"
            - name: STRIMZI_NAMESPACE
              valueFrom:
                fieldRef:
                  fieldPath: metadata.namespace
  ...

[miclip]

@scholzj are the minimum required ACLs for the entity operator user documented? I don't see it in the documentation, MSK doesn't support the superuser and I'd like to only provision what is required.

[scholzj]

I'm not aware of any documentation covering this. Sorry.

I would expect the User Operator to need the rights to manage ACLs, Qoutas, and SCRAM-SHA credentials. The topic operator just Topics. But I do not have a mapping to the Kafka ACLs.

@fvaleri Do you know if TO needs something special in regards of ACLs?

[fvaleri]

hi @miclip, TO user's requirements are described here: https://github.com/strimzi/strimzi-kafka-operator/blob/main/topic-operator/src/main/java/io/strimzi/operator/topic/DESIGN.md#assumptions

[miclip]

Thanks for the info