From fe799d34ffcae1cc07b3167bb68bdc1e7836cea5 Mon Sep 17 00:00:00 2001 From: Gary Tully Date: Mon, 10 Jun 2024 15:01:08 +0100 Subject: [PATCH] [#1038] support restricted option, jolokia agent, rbac, read only root, minimal --- api/v1beta1/activemqartemis_types.go | 4 + api/v1beta1/zz_generated.deepcopy.go | 5 + ...rtemis-operator.clusterserviceversion.yaml | 3 + .../broker.amq.io_activemqartemises.yaml | 3 + .../broker.amq.io_activemqartemises.yaml | 3 + ...rtemis-operator.clusterserviceversion.yaml | 3 + controllers/activemqartemis_controller.go | 2 +- ...emqartemis_controller_cert_manager_test.go | 4 +- .../activemqartemis_controller_test.go | 16 +- controllers/activemqartemis_reconciler.go | 355 +++++++++++++++--- .../activemqartemis_reconciler_test.go | 2 +- controllers/common_util_test.go | 14 +- controllers/controll_plane_test.go | 168 +++++++++ controllers/suite_test.go | 3 + deploy/activemq-artemis-operator.yaml | 3 + deploy/crds/broker_activemqartemis_crd.yaml | 3 + docs/help/operator.md | 3 + pkg/resources/secrets/secret.go | 2 +- pkg/resources/serviceports/service_port.go | 61 +-- pkg/resources/volumes/volume.go | 2 +- pkg/utils/artemis/artemis.go | 12 + pkg/utils/certutil/certutil.go | 16 +- pkg/utils/common/common.go | 208 ++++++++++ pkg/utils/jolokia/jolokia.go | 29 ++ pkg/utils/jolokia/mock_jolokia.go | 4 + pkg/utils/jolokia_client/jolokia_client.go | 28 +- pkg/utils/lsrcrs/lsrcr.go | 2 +- 27 files changed, 842 insertions(+), 116 deletions(-) create mode 100644 controllers/controll_plane_test.go diff --git a/api/v1beta1/activemqartemis_types.go b/api/v1beta1/activemqartemis_types.go index 2e0f56722..0fd4a55ed 100644 --- a/api/v1beta1/activemqartemis_types.go +++ b/api/v1beta1/activemqartemis_types.go @@ -71,6 +71,10 @@ type ActiveMQArtemisSpec struct { // Specifies the template for various resources that the operator controls //+operator-sdk:csv:customresourcedefinitions:type=spec,displayName="Resource Templates" ResourceTemplates []ResourceTemplate `json:"resourceTemplates,omitempty"` + + // Restricted deployment, mtls jolokia agent with RBAC + //+operator-sdk:csv:customresourcedefinitions:type=spec,displayName="Restricted" + Restricted *bool `json:"restricted,omitempty"` } type AddressSettingsType struct { diff --git a/api/v1beta1/zz_generated.deepcopy.go b/api/v1beta1/zz_generated.deepcopy.go index 0df8e2efd..ebcdba923 100644 --- a/api/v1beta1/zz_generated.deepcopy.go +++ b/api/v1beta1/zz_generated.deepcopy.go @@ -494,6 +494,11 @@ func (in *ActiveMQArtemisSpec) DeepCopyInto(out *ActiveMQArtemisSpec) { (*in)[i].DeepCopyInto(&(*out)[i]) } } + if in.Restricted != nil { + in, out := &in.Restricted, &out.Restricted + *out = new(bool) + **out = **in + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ActiveMQArtemisSpec. diff --git a/bundle/manifests/activemq-artemis-operator.clusterserviceversion.yaml b/bundle/manifests/activemq-artemis-operator.clusterserviceversion.yaml index 76ddf7ef7..249145143 100644 --- a/bundle/manifests/activemq-artemis-operator.clusterserviceversion.yaml +++ b/bundle/manifests/activemq-artemis-operator.clusterserviceversion.yaml @@ -1495,6 +1495,9 @@ spec: all resources displayName: Selector path: resourceTemplates[0].selector + - description: Restricted deployment, mtls jolokia agent with RBAC + displayName: Restricted + path: restricted - description: Specifies the upgrades (deprecated in favour of Version) displayName: Upgrades path: upgrades diff --git a/bundle/manifests/broker.amq.io_activemqartemises.yaml b/bundle/manifests/broker.amq.io_activemqartemises.yaml index 6fd9f4117..fdca792f3 100644 --- a/bundle/manifests/broker.amq.io_activemqartemises.yaml +++ b/bundle/manifests/broker.amq.io_activemqartemises.yaml @@ -4823,6 +4823,9 @@ spec: type: object type: object type: array + restricted: + description: Restricted deployment, mtls jolokia agent with RBAC + type: boolean upgrades: description: Specifies the upgrades (deprecated in favour of Version) properties: diff --git a/config/crd/bases/broker.amq.io_activemqartemises.yaml b/config/crd/bases/broker.amq.io_activemqartemises.yaml index 9dd282323..5ab451667 100644 --- a/config/crd/bases/broker.amq.io_activemqartemises.yaml +++ b/config/crd/bases/broker.amq.io_activemqartemises.yaml @@ -4824,6 +4824,9 @@ spec: type: object type: object type: array + restricted: + description: Restricted deployment, mtls jolokia agent with RBAC + type: boolean upgrades: description: Specifies the upgrades (deprecated in favour of Version) properties: diff --git a/config/manifests/bases/activemq-artemis-operator.clusterserviceversion.yaml b/config/manifests/bases/activemq-artemis-operator.clusterserviceversion.yaml index b7d63e483..96be78ac2 100644 --- a/config/manifests/bases/activemq-artemis-operator.clusterserviceversion.yaml +++ b/config/manifests/bases/activemq-artemis-operator.clusterserviceversion.yaml @@ -1227,6 +1227,9 @@ spec: all resources displayName: Selector path: resourceTemplates[0].selector + - description: Restricted deployment, mtls jolokia agent with RBAC + displayName: Restricted + path: restricted - description: Specifies the upgrades (deprecated in favour of Version) displayName: Upgrades path: upgrades diff --git a/controllers/activemqartemis_controller.go b/controllers/activemqartemis_controller.go index 252ebcaae..3ab34c50d 100644 --- a/controllers/activemqartemis_controller.go +++ b/controllers/activemqartemis_controller.go @@ -173,7 +173,7 @@ func (r *ActiveMQArtemisReconciler) Reconcile(ctx context.Context, request ctrl. err = reconciler.Process(customResource, *namer, r.Client, r.Scheme) - if ProcessBrokerStatus(customResource, r.Client, r.Scheme) { + if reconciler.ProcessBrokerStatus(customResource, r.Client, r.Scheme) { requeueRequest = true } } diff --git a/controllers/activemqartemis_controller_cert_manager_test.go b/controllers/activemqartemis_controller_cert_manager_test.go index d24d577a4..4f72e1ff8 100644 --- a/controllers/activemqartemis_controller_cert_manager_test.go +++ b/controllers/activemqartemis_controller_cert_manager_test.go @@ -48,7 +48,7 @@ const ( rootCertNamespce = "cert-manager" rootCertSecretName = "artemis-root-cert-secret" caIssuerName = "broker-ca-issuer" - caPemTrustStoreName = "ca-truststore.pem" + caPemTrustStoreName = "ca.pem" caTrustStorePassword = "changeit" ) @@ -57,7 +57,7 @@ var ( rootIssuer = &cmv1.ClusterIssuer{} rootCert = &cmv1.Certificate{} caIssuer = &cmv1.ClusterIssuer{} - caBundleName = "ca-bundle" + caBundleName = "operator-ca" ) type ConnectorConfig struct { diff --git a/controllers/activemqartemis_controller_test.go b/controllers/activemqartemis_controller_test.go index 3ac36a6bc..72aced0ba 100644 --- a/controllers/activemqartemis_controller_test.go +++ b/controllers/activemqartemis_controller_test.go @@ -7497,7 +7497,7 @@ var _ = Describe("artemis controller", func() { Name: secretName, Namespace: defaultNamespace, } - secret, err := secrets.RetriveSecret(namespaceName, secretName, make(map[string]string), k8sClient) + secret, err := secrets.RetriveSecret(namespaceName, make(map[string]string), k8sClient) g.Expect(err).To(BeNil()) data := secret.Data[envVar.ValueFrom.SecretKeyRef.Key] //the value is a string of acceptors in xml format: @@ -7552,7 +7552,7 @@ var _ = Describe("artemis controller", func() { Name: secretName, Namespace: defaultNamespace, } - secret, err := secrets.RetriveSecret(namespaceName, secretName, make(map[string]string), k8sClient) + secret, err := secrets.RetriveSecret(namespaceName, make(map[string]string), k8sClient) g.Expect(err).To(BeNil()) data := secret.Data[envVar.ValueFrom.SecretKeyRef.Key] //the value is a string of acceptors in xml format: @@ -7616,7 +7616,7 @@ var _ = Describe("artemis controller", func() { Name: secretName, Namespace: defaultNamespace, } - secret, err := secrets.RetriveSecret(namespaceName, secretName, make(map[string]string), k8sClient) + secret, err := secrets.RetriveSecret(namespaceName, make(map[string]string), k8sClient) g.Expect(err).To(BeNil()) data := secret.Data[envVar.ValueFrom.SecretKeyRef.Key] //the value is a string of acceptors in xml format: @@ -7679,12 +7679,12 @@ var _ = Describe("artemis controller", func() { Name: secretName, Namespace: defaultNamespace, } - secret, err := secrets.RetriveSecret(namespaceName, secretName, make(map[string]string), k8sClient) + secret, err := secrets.RetriveSecret(namespaceName, make(map[string]string), k8sClient) g.Expect(err).To(BeNil()) data := secret.Data[envVar.ValueFrom.SecretKeyRef.Key] By("Checking data:" + string(data)) g.Expect(strings.Contains(string(data), "ACCEPTOR_IP:61666")).To(BeTrue()) - checkSecretHasCorrectKeyValue(g, secretName, namespaceName, envVar.ValueFrom.SecretKeyRef.Key, "keyStoreProvider=SunJCE") + checkSecretHasCorrectKeyValue(g, namespaceName, envVar.ValueFrom.SecretKeyRef.Key, "keyStoreProvider=SunJCE") } } @@ -7742,7 +7742,7 @@ var _ = Describe("artemis controller", func() { //...... //we need to locate our target acceptor and do the check //we use the port as a clue - checkSecretHasCorrectKeyValue(g, secretName, namespaceName, envVar.ValueFrom.SecretKeyRef.Key, "trustStoreType=JCEKS") + checkSecretHasCorrectKeyValue(g, namespaceName, envVar.ValueFrom.SecretKeyRef.Key, "trustStoreType=JCEKS") found = true } } @@ -7789,7 +7789,7 @@ var _ = Describe("artemis controller", func() { Namespace: defaultNamespace, } - secret, err := secrets.RetriveSecret(namespaceName, secretName, make(map[string]string), k8sClient) + secret, err := secrets.RetriveSecret(namespaceName, make(map[string]string), k8sClient) g.Expect(err).Should(BeNil()) data := secret.Data[envVar.ValueFrom.SecretKeyRef.Key] @@ -7848,7 +7848,7 @@ var _ = Describe("artemis controller", func() { //...... //we need to locate our target acceptor and do the check //we use the port as a clue - checkSecretHasCorrectKeyValue(g, secretName, namespaceName, envVar.ValueFrom.SecretKeyRef.Key, "trustStoreProvider=SUN") + checkSecretHasCorrectKeyValue(g, namespaceName, envVar.ValueFrom.SecretKeyRef.Key, "trustStoreProvider=SUN") } } }, timeout, interval).Should(Succeed()) diff --git a/controllers/activemqartemis_reconciler.go b/controllers/activemqartemis_reconciler.go index 5416f3afb..d69dfff92 100644 --- a/controllers/activemqartemis_reconciler.go +++ b/controllers/activemqartemis_reconciler.go @@ -3,6 +3,8 @@ package controllers import ( "bytes" "context" + "crypto/tls" + "crypto/x509/pkix" "encoding/binary" "encoding/hex" "encoding/json" @@ -280,7 +282,7 @@ func (reconciler *ActiveMQArtemisReconcilerImpl) ProcessStatefulSet(customResour } labels := namer.LabelBuilder.Labels() - headlessServiceDefinition = svc.NewHeadlessServiceForCR2(client, headlesServiceName, ssNamespacedName.Namespace, serviceports.GetDefaultPorts(), labels, headlessServiceDefinition) + headlessServiceDefinition = svc.NewHeadlessServiceForCR2(client, headlesServiceName, ssNamespacedName.Namespace, serviceports.GetDefaultPorts(isRestricted(customResource)), labels, headlessServiceDefinition) reconciler.trackDesired(headlessServiceDefinition) if isClustered(customResource) { @@ -301,6 +303,10 @@ func (reconciler *ActiveMQArtemisReconcilerImpl) ProcessStatefulSet(customResour } func isClustered(customResource *brokerv1beta1.ActiveMQArtemis) bool { + if isRestricted(customResource) { + return false + } + if customResource.Spec.DeploymentPlan.Clustered != nil { return *customResource.Spec.DeploymentPlan.Clustered } @@ -309,6 +315,9 @@ func isClustered(customResource *brokerv1beta1.ActiveMQArtemis) bool { func (reconciler *ActiveMQArtemisReconcilerImpl) ProcessCredentials(customResource *brokerv1beta1.ActiveMQArtemis, namer common.Namers, client rtclient.Client, scheme *runtime.Scheme, currentStatefulSet *appsv1.StatefulSet) { + if isRestricted(customResource) { + return + } reconciler.log.V(1).Info("ProcessCredentials") envVars := make(map[string]ValueInfo) @@ -407,6 +416,10 @@ func (reconciler *ActiveMQArtemisReconcilerImpl) applyPodDisruptionBudget(custom func (reconciler *ActiveMQArtemisReconcilerImpl) ProcessAcceptorsAndConnectors(customResource *brokerv1beta1.ActiveMQArtemis, namer common.Namers, client rtclient.Client, scheme *runtime.Scheme, currentStatefulSet *appsv1.StatefulSet) error { + if isRestricted(customResource) { + return nil + } + acceptorEntry, err := reconciler.generateAcceptorsString(customResource, client, currentStatefulSet) if err != nil { return err @@ -439,7 +452,7 @@ func (reconciler *ActiveMQArtemisReconcilerImpl) ProcessAcceptorsAndConnectors(c func (reconciler *ActiveMQArtemisReconcilerImpl) ProcessConsole(customResource *brokerv1beta1.ActiveMQArtemis, namer common.Namers, client rtclient.Client, scheme *runtime.Scheme, currentStatefulSet *appsv1.StatefulSet) error { reconciler.configureConsoleExposure(customResource, namer, client) - if !customResource.Spec.Console.SSLEnabled { + if !customResource.Spec.Console.SSLEnabled || isRestricted(customResource) { return nil } @@ -1723,6 +1736,9 @@ func (reconciler *ActiveMQArtemisReconcilerImpl) MakeVolumes(customResource *bro if customResource.Spec.DeploymentPlan.PersistenceEnabled { basicCRVolume := volumes.MakePersistentVolume(customResource.Name) volumeDefinitions = append(volumeDefinitions, basicCRVolume...) + } else if isRestricted(customResource) { + emptyDirData := volumes.MakeEmptyDirVolumeFor(customResource.Name) + volumeDefinitions = append(volumeDefinitions, emptyDirData) } volumeDefinitions = append(volumeDefinitions, customResource.Spec.DeploymentPlan.ExtraVolumes...) @@ -1765,7 +1781,7 @@ func (reconciler *ActiveMQArtemisReconcilerImpl) MakeVolumes(customResource *bro } } - if customResource.Spec.Console.SSLEnabled { + if !isRestricted(customResource) && customResource.Spec.Console.SSLEnabled { reconciler.log.V(1).Info("Make volumes for ssl console exposure on k8s") secretName := namer.SecretsConsoleNameBuilder.Name() addNewVolumes(secretVolumes, &volumeDefinitions, &secretName) @@ -1788,8 +1804,8 @@ func addNewVolumeMounts(existingNames map[string]string, existing *[]corev1.Volu func (reconciler *ActiveMQArtemisReconcilerImpl) MakeVolumeMounts(customResource *brokerv1beta1.ActiveMQArtemis, namer common.Namers) ([]corev1.VolumeMount, error) { volumeMounts := []corev1.VolumeMount{} - if customResource.Spec.DeploymentPlan.PersistenceEnabled { - persistentCRVlMnt := volumes.MakePersistentVolumeMount(customResource.Name, namer.GLOBAL_DATA_PATH) + if customResource.Spec.DeploymentPlan.PersistenceEnabled || isRestricted(customResource) { + persistentCRVlMnt := volumes.MakePersistentVolumeMount(customResource.Name, getDataMountPath(customResource, namer)) volumeMounts = append(volumeMounts, persistentCRVlMnt...) } @@ -1874,6 +1890,12 @@ func (reconciler *ActiveMQArtemisReconcilerImpl) MakeVolumeMounts(customResource return volumeMounts, nil } +func getDataMountPath(cr *brokerv1beta1.ActiveMQArtemis, namer common.Namers) string { + if isRestricted(cr) { + return "/app" + } + return namer.GLOBAL_DATA_PATH +} func MakeContainerPorts(cr *brokerv1beta1.ActiveMQArtemis) []corev1.ContainerPort { containerPorts := []corev1.ContainerPort{} @@ -1937,18 +1959,158 @@ func (reconciler *ActiveMQArtemisReconcilerImpl) PodTemplateSpecForCR(customReso reqLogger.V(2).Info("Checking out extraMounts", "extra config", customResource.Spec.DeploymentPlan.ExtraMounts) - configMapsToCreate := customResource.Spec.DeploymentPlan.ExtraMounts.ConfigMaps - secretsToCreate := customResource.Spec.DeploymentPlan.ExtraMounts.Secrets + configMapsToMount := customResource.Spec.DeploymentPlan.ExtraMounts.ConfigMaps + secretsToMount := customResource.Spec.DeploymentPlan.ExtraMounts.Secrets brokerPropertiesResourceName, isSecret, brokerPropertiesMapData, serr := reconciler.addResourceForBrokerProperties(customResource, namer) if serr != nil { return nil, serr } if isSecret { - secretsToCreate = append(secretsToCreate, brokerPropertiesResourceName) + secretsToMount = append(secretsToMount, brokerPropertiesResourceName) } else { - configMapsToCreate = append(configMapsToCreate, brokerPropertiesResourceName) + configMapsToMount = append(configMapsToMount, brokerPropertiesResourceName) + } + + additionalSystemPropsForRestricted := []string{} + if isRestricted(customResource) { + + mountPathRoot := secretPathBase + getPropertiesResourceNsName(customResource).Name + security_properties := newPropsWithHeader() + fmt.Fprintf(security_properties, "login.config.url.1=file:%s/login.config\n", mountPathRoot) + brokerPropertiesMapData["_security.config"] = security_properties.String() + + additionalSystemPropsForRestricted = append(additionalSystemPropsForRestricted, fmt.Sprintf("-Djava.security.properties=%s/_security.config", mountPathRoot)) + + login_config := newBufferWithHeader("//") + fmt.Fprintln(login_config, "http_server_authenticator {") + fmt.Fprintln(login_config, " org.apache.activemq.artemis.spi.core.security.jaas.TextFileCertificateLoginModule required") + fmt.Fprintln(login_config, " reload=true") + fmt.Fprintln(login_config, " debug=true") + fmt.Fprintln(login_config, " org.apache.activemq.jaas.textfiledn.user=_cert-users") + fmt.Fprintln(login_config, " org.apache.activemq.jaas.textfiledn.role=_cert-roles") + fmt.Fprintf(login_config, " baseDir=\"%v\"\n", mountPathRoot) + fmt.Fprintln(login_config, " ;") + fmt.Fprintln(login_config, "};") + brokerPropertiesMapData["login.config"] = login_config.String() + + operandCertSecretName := common.GetOperandCertSecretName(customResource, client) + operandCertSubject, err := common.ExtractCertSubjectFromSecret(operandCertSecretName, customResource.Namespace, client) + if err != nil { + return nil, err + } + + var caCertSecret *corev1.Secret + if caCertSecret, err = common.GetOperatorCASecret(client); err != nil { + return nil, err + } + + caSecretKey, err := common.GetOperatorCASecretKey(client, caCertSecret) + if err != nil { + return nil, err + } + + var operatorCert *tls.Certificate + if operatorCert, err = common.GetOperatorClientCertificate(client, nil); err != nil { + return nil, err + } + + var operatorCertSubject *pkix.Name + if operatorCertSubject, err = common.ExtractCertSubject(operatorCert); err != nil { + return nil, err + } + + // TODO - make configuable + // support control-plane-auth-secret, maybe a suffix for the http_server_authenticator realm login.config + + cert_user := newPropsWithHeader() + fmt.Fprintln(cert_user, "hawtio=/CN = hawtio-online\\.hawtio\\.svc.*/") + fmt.Fprintf(cert_user, "operator=/.*%s.*/\n", operatorCertSubject.CommonName) // regexp syntax start and with / + // can and should use the full DN after https://issues.apache.org/jira/browse/ARTEMIS-5102 + fmt.Fprintf(cert_user, "probe=/.*%s.*/\n", operandCertSubject.CommonName) + brokerPropertiesMapData["_cert-users"] = cert_user.String() + + cert_roles := newPropsWithHeader() + fmt.Fprintln(cert_roles, "status=operator,probe") + fmt.Fprintln(cert_roles, "hawtio=hawtio") + brokerPropertiesMapData["_cert-roles"] = cert_roles.String() + + foundationalProps := newPropsWithHeader() + fmt.Fprintln(foundationalProps, "name=amq-broker") + fmt.Fprintln(foundationalProps, "criticalAnalyzer=false") + fmt.Fprintln(foundationalProps, "journalDirectory=/app/data") + fmt.Fprintln(foundationalProps, "bindingsDirectory=/app/data/bindings") + fmt.Fprintln(foundationalProps, "largeMessagesDirectory=/app/data/largemessages") + fmt.Fprintln(foundationalProps, "pagingDirectory=/app/data/paging") + + brokerPropertiesMapData["aa_restricted.properties"] = foundationalProps.String() + + rbac := newPropsWithHeader() + fmt.Fprintln(rbac, "securityRoles.\"mops.broker.getStatus\".status.view=true") + + brokerPropertiesMapData["aa_rbac.properties"] = rbac.String() + + secretsToMount = append(secretsToMount, operandCertSecretName) + caSecret := common.GetOperatorCASecretName() + secretsToMount = append(secretsToMount, caSecret) + + jolokia_config := newPropsWithHeader() + fmt.Fprintln(jolokia_config, "protocol=https") + fmt.Fprintln(jolokia_config, "authClass=org.apache.activemq.artemis.spi.core.security.jaas.HttpServerAuthenticator") + fmt.Fprintf(jolokia_config, "caCert=%s%s/%s\n", secretPathBase, caSecret, caSecretKey) + fmt.Fprintf(jolokia_config, "serverCert=%s%s/tls.crt\n", secretPathBase, operandCertSecretName) + fmt.Fprintf(jolokia_config, "serverKey=%s%s/tls.key\n", secretPathBase, operandCertSecretName) + fmt.Fprintln(jolokia_config, "port=8778") + // https://github.com/jolokia/jolokia/issues/751 at some point host=$(env:HOSTNAME), host= is on the command line below + fmt.Fprintln(jolokia_config, "useSslClientAuthentication=true") + fmt.Fprintln(jolokia_config, "disabledServices=org.jolokia.service.history.HistoryMBeanRequestInterceptor") + fmt.Fprintln(jolokia_config, "disableDetectors=true") + fmt.Fprintln(jolokia_config, "debug=false") + + brokerPropertiesMapData["_jolokia.config"] = jolokia_config.String() + + // adapt jolokia authentication + additionalSystemPropsForRestricted = append(additionalSystemPropsForRestricted, "-DhttpServerAuthenticator.requestSubjectAttribute=org.jolokia.jaasSubject") + + // install mbean server guard + additionalSystemPropsForRestricted = append(additionalSystemPropsForRestricted, "-Dlog4j2.disableJmx=true -Djavax.management.builder.initial=org.apache.activemq.artemis.core.server.management.ArtemisRbacMBeanServerBuilder") + + // install jolokia agent + additionalSystemPropsForRestricted = append(additionalSystemPropsForRestricted, fmt.Sprintf("-javaagent:/opt/jolokia/javaagent.jar=host=$HOSTNAME,config=%s/_jolokia.config", mountPathRoot)) + + // non boot jar isolation classpath + additionalSystemPropsForRestricted = append(additionalSystemPropsForRestricted, "-classpath /opt/amq/lib/*:/opt/amq/lib/extra/*") + + // temp volume + additionalSystemPropsForRestricted = append(additionalSystemPropsForRestricted, "-Djava.io.tmpdir=/app/tmp") + + // jvm options + additionalSystemPropsForRestricted = append(additionalSystemPropsForRestricted, "-XX:InitialRAMPercentage=70.0 -XX:MaxRAMPercentage=70.0 -XX:AutoBoxCacheMax=20000 -XX:+PrintClassHistogram -XX:+UseG1GC -XX:+UseStringDeduplication -Djava.net.preferIPv4Stack=true") + + if customResource.Spec.DeploymentPlan.LivenessProbe == nil { + container.LivenessProbe = &corev1.Probe{ + ProbeHandler: corev1.ProbeHandler{ + Exec: &corev1.ExecAction{ + Command: []string{ + "/bin/bash", + "-c", + // use curl with mtls as the broker-cert to pull the status to find start state using dns + fmt.Sprintf(`export STATEFUL_SET_ORDINAL=${HOSTNAME##*-};curl --cacert %s%s/%s --cert %s%s/tls.crt --key %s%s/tls.key https://%s:8778/jolokia/read/org.apache.activemq.artemis:broker=%%22amq-broker%%22/Status | grep -w -P "(START|STOPP)(ED|ING)"`, secretPathBase, caSecret, caSecretKey, secretPathBase, operandCertSecretName, secretPathBase, operandCertSecretName, common.OrdinalStringFQDNS(customResource.Name, customResource.Namespace, "$STATEFUL_SET_ORDINAL")), + }, + }, + }, + InitialDelaySeconds: 1, + TimeoutSeconds: 5, + PeriodSeconds: 5, + SuccessThreshold: 1, + FailureThreshold: 2, + TerminationGracePeriodSeconds: &terminationGracePeriodSeconds, + } + } else { + // use the value from the CR + container.LivenessProbe = reconciler.configureLivenessProbe(container, customResource.Spec.DeploymentPlan.LivenessProbe) + } } - extraVolumes, extraVolumeMounts, err := reconciler.createExtraConfigmapsAndSecretsVolumeMounts(configMapsToCreate, secretsToCreate, brokerPropertiesResourceName, brokerPropertiesMapData, client) + extraVolumes, extraVolumeMounts, err := reconciler.createExtraConfigmapsAndSecretsVolumeMounts(configMapsToMount, secretsToMount, brokerPropertiesResourceName, brokerPropertiesMapData, client) if err != nil { return nil, err } @@ -1965,7 +2127,9 @@ func (reconciler *ActiveMQArtemisReconcilerImpl) PodTemplateSpecForCR(customReso } container.StartupProbe = reconciler.configureStartupProbe(container, customResource.Spec.DeploymentPlan.StartupProbe) - container.LivenessProbe = reconciler.configureLivenessProbe(container, customResource.Spec.DeploymentPlan.LivenessProbe) + if !isRestricted(customResource) { + container.LivenessProbe = reconciler.configureLivenessProbe(container, customResource.Spec.DeploymentPlan.LivenessProbe) + } container.ReadinessProbe = reconciler.configureReadinessProbe(container, customResource.Spec.DeploymentPlan.ReadinessProbe) if len(customResource.Spec.DeploymentPlan.NodeSelector) > 0 { @@ -2012,7 +2176,7 @@ func (reconciler *ActiveMQArtemisReconcilerImpl) PodTemplateSpecForCR(customReso // JAAS Config if jaasConfigPath, found := getJaasConfigExtraMountPath(customResource); found { debugArgs := corev1.EnvVar{ - Name: debugArgsEnvVarName, + Name: getJaasConfigEnvVarName(customResource), Value: fmt.Sprintf("-Djava.security.auth.login.config=%v", jaasConfigPath), } environments.CreateOrAppend(podSpec.Containers, &debugArgs) @@ -2020,22 +2184,31 @@ func (reconciler *ActiveMQArtemisReconcilerImpl) PodTemplateSpecForCR(customReso if loggingConfigPath, found := getLoggingConfigExtraMountPath(customResource); found { loggerOpts := corev1.EnvVar{ - Name: javaArgsAppendEnvVarName, + Name: getLoginConfigEnvVarName(customResource), Value: fmt.Sprintf("-Dlog4j2.configurationFile=%v", loggingConfigPath), } environments.CreateOrAppend(podSpec.Containers, &loggerOpts) + } else if isRestricted(customResource) { + // modify log4j2 default of ERROR + loggerOpts := corev1.EnvVar{ + Name: getLoginConfigEnvVarName(customResource), + Value: "-Dlog4j2.level=INFO", + } + environments.CreateOrAppend(podSpec.Containers, &loggerOpts) } - //add empty-dir volume and volumeMounts to main container - volumeForCfg := volumes.MakeVolumeForCfg(cfgVolumeName) - podSpec.Volumes = append(podSpec.Volumes, volumeForCfg) - // add TopologySpreadConstraints config podSpec.TopologySpreadConstraints = customResource.Spec.DeploymentPlan.TopologySpreadConstraints - volumeMountForCfg := volumes.MakeRwVolumeMountForCfg(cfgVolumeName, brokerConfigRoot) - podSpec.Containers[0].VolumeMounts = append(podSpec.Containers[0].VolumeMounts, volumeMountForCfg) + if !isRestricted(customResource) { + //add empty-dir volume and volumeMounts to main container + volumeForCfg := volumes.MakeEmptyDirVolumeFor(cfgVolumeName) + podSpec.Volumes = append(podSpec.Volumes, volumeForCfg) + + volumeMountForCfg := volumes.MakeRwVolumeMountForCfg(cfgVolumeName, brokerConfigRoot) + podSpec.Containers[0].VolumeMounts = append(podSpec.Containers[0].VolumeMounts, volumeMountForCfg) + } reqLogger.V(2).Info("Creating init container for broker configuration") initContainer := containers.MakeInitContainer(podSpec, customResource.Name, common.ResolveImage(customResource, common.InitImageKey), MakeEnvVarArrayForCR(customResource, namer)) initContainer.Resources = customResource.Spec.DeploymentPlan.Resources @@ -2132,12 +2305,12 @@ func (reconciler *ActiveMQArtemisReconcilerImpl) PodTemplateSpecForCR(customReso volumeMountForCfgRoot := volumes.MakeRwVolumeMountForCfg(cfgVolumeName, brokerConfigRoot) podSpec.InitContainers[0].VolumeMounts = append(podSpec.InitContainers[0].VolumeMounts, volumeMountForCfgRoot) - volumeMountForCfg = volumes.MakeRwVolumeMountForCfg("tool-dir", initCfgRootDir) + volumeMountForCfg := volumes.MakeRwVolumeMountForCfg("tool-dir", initCfgRootDir) podSpec.InitContainers[0].VolumeMounts = append(podSpec.InitContainers[0].VolumeMounts, volumeMountForCfg) //add empty-dir volume - volumeForCfg = volumes.MakeVolumeForCfg("tool-dir") - podSpec.Volumes = append(podSpec.Volumes, volumeForCfg) + volumeForTool := volumes.MakeEmptyDirVolumeFor("tool-dir") + podSpec.Volumes = append(podSpec.Volumes, volumeForTool) reqLogger.V(2).Info("Total volumes ", "volumes", podSpec.Volumes) @@ -2230,13 +2403,77 @@ func (reconciler *ActiveMQArtemisReconcilerImpl) PodTemplateSpecForCR(customReso reconciler.configPodSecurity(podSpec, &customResource.Spec.DeploymentPlan.PodSecurity) reconciler.configurePodSecurityContext(podSpec, customResource.Spec.DeploymentPlan.PodSecurityContext) - reqLogger.V(2).Info("Final Init spec", "Detail", podSpec.InitContainers) - pts.Spec = *podSpec + if isRestricted(customResource) { + pts.Spec.InitContainers = nil + + // restricted env + currentEnv := environments.Retrieve(pts.Spec.Containers, jdkJavaOptionsEnvVarName) + pts.Spec.Containers[0].Env = nil + + var commandLineString string = "" + if currentEnv != nil { + commandLineString += currentEnv.Value + } + for _, v := range additionalSystemPropsForRestricted { + commandLineString += " " + v + } + + // env from CR can override + pts.Spec.Containers[0].Env = append(pts.Spec.Containers[0].Env, customResource.Spec.Env...) + + var reEvalJdkJavaOpts string = "" + currentEnv = environments.Retrieve(pts.Spec.Containers, jdkJavaOptionsEnvVarName) + if currentEnv != nil { + // support STATEFUL_SET_ORDINAL in JDK_JAVA_OPTIONS from CR + reEvalJdkJavaOpts = `export JDK_JAVA_OPTIONS=${JDK_JAVA_OPTIONS//\\$\\{STATEFUL_SET_ORDINAL\\}/${HOSTNAME##*-}};` + } + + pts.Spec.Containers[0].Command = []string{ + "/bin/bash", "-c", + fmt.Sprintf("export STATEFUL_SET_ORDINAL=${HOSTNAME##*-}; %s exec java %s $JAVA_ARGS_APPEND org.apache.activemq.artemis.core.server.embedded.Main", reEvalJdkJavaOpts, commandLineString), + } + } + + reqLogger.V(2).Info("Final Init spec", "Detail", podSpec.InitContainers) + return pts, nil } +func getJaasConfigEnvVarName(customResource *brokerv1beta1.ActiveMQArtemis) string { + if !isRestricted(customResource) { + // legacy + return debugArgsEnvVarName + } + + return jdkJavaOptionsEnvVarName +} + +func getLoginConfigEnvVarName(customResource *brokerv1beta1.ActiveMQArtemis) string { + if !isRestricted(customResource) { + // legacy + return javaArgsAppendEnvVarName + } + + return jdkJavaOptionsEnvVarName +} + +func isRestricted(customResource *brokerv1beta1.ActiveMQArtemis) bool { + return customResource.Spec.Restricted != nil && *customResource.Spec.Restricted +} + +func newPropsWithHeader() *bytes.Buffer { + return newBufferWithHeader("#") +} + +func newBufferWithHeader(commentChars string) *bytes.Buffer { + buf := &bytes.Buffer{} + fmt.Fprintf(buf, "%s generated by crd\n", commentChars) + fmt.Fprintf(buf, "%s\n", commentChars) + return buf +} + func brokerPropertiesConfigSystemPropValue(customResource *brokerv1beta1.ActiveMQArtemis, mountPoint, resourceName string, brokerPropertiesData map[string]string) string { var result = "" if len(brokerPropertiesData) == 1 { @@ -2326,7 +2563,8 @@ func (reconciler *ActiveMQArtemisReconcilerImpl) configureLivenessProbe(containe reconciler.log.V(1).Info("Using user provided Liveness Probe Handler" + probeFromCr.ProbeHandler.String()) livenessProbe.ProbeHandler = probeFromCr.ProbeHandler } - } else { + } else if !isRestricted(reconciler.customResource) { + reconciler.log.V(1).Info("Creating Default Liveness Probe") livenessProbe.InitialDelaySeconds = defaultLivenessProbeInitialDelay @@ -2382,6 +2620,9 @@ func (reconciler *ActiveMQArtemisReconcilerImpl) configureReadinessProbe(contain } else { readinessProbe.ProbeHandler = probeFromCr.ProbeHandler } + } else if isRestricted(reconciler.customResource) { + // liveness probe is sufficient + readinessProbe = nil } else { reconciler.log.V(1).Info("creating default readiness Probe") readinessProbe.InitialDelaySeconds = defaultLivenessProbeInitialDelay @@ -2426,7 +2667,7 @@ func conditionallyApplyValuesToPreserveDefaults(readinessProbe *corev1.Probe, pr } } -func getConfigAppliedConfigMapName(artemis *brokerv1beta1.ActiveMQArtemis) types.NamespacedName { +func getPropertiesResourceNsName(artemis *brokerv1beta1.ActiveMQArtemis) types.NamespacedName { return types.NamespacedName{ Namespace: artemis.Namespace, Name: artemis.Name + "-props", @@ -2456,7 +2697,7 @@ func (reconciler *ActiveMQArtemisReconcilerImpl) addResourceForBrokerProperties( } var desired *corev1.Secret - resourceName = getConfigAppliedConfigMapName(customResource) + resourceName = getPropertiesResourceNsName(customResource) obj = reconciler.cloneOfDeployed(reflect.TypeOf(corev1.Secret{}), resourceName.Name) if obj != nil { @@ -2567,6 +2808,7 @@ func (reconciler *ActiveMQArtemisReconcilerImpl) configureContianerSecurityConte container.SecurityContext = containerSecurityContext } else { reconciler.log.V(2).Info("Incoming Container SecurityContext is nil, creating with default values") + readOnlyRootFilesystem := isRestricted(reconciler.customResource) runAsNonRoot := true allowPrivilegeEscalation := false capabilities := corev1.Capabilities{Drop: []corev1.Capability{"ALL"}} @@ -2576,6 +2818,7 @@ func (reconciler *ActiveMQArtemisReconcilerImpl) configureContianerSecurityConte Capabilities: &capabilities, SeccompProfile: &seccompProfile, RunAsNonRoot: &runAsNonRoot, + ReadOnlyRootFilesystem: &readOnlyRootFilesystem, } container.SecurityContext = &securityContext } @@ -2604,6 +2847,9 @@ func (reconciler *ActiveMQArtemisReconcilerImpl) configPodSecurity(podSpec *core if podSecurity.ServiceAccountName != nil { reconciler.log.V(2).Info("Pod serviceAccountName specified", "existing", podSpec.ServiceAccountName, "new", *podSecurity.ServiceAccountName) podSpec.ServiceAccountName = *podSecurity.ServiceAccountName + } else { + autoMount := !isRestricted(reconciler.customResource) + podSpec.AutomountServiceAccountToken = &autoMount } if podSecurity.RunAsUser != nil { reconciler.log.V(2).Info("Pod runAsUser specified", "runAsUser", *podSecurity.RunAsUser) @@ -2882,7 +3128,7 @@ type applyError struct { Reason string `json:"reason"` } -func ProcessBrokerStatus(cr *brokerv1beta1.ActiveMQArtemis, client rtclient.Client, scheme *runtime.Scheme) (retry bool) { +func (reconciler *ActiveMQArtemisReconcilerImpl) ProcessBrokerStatus(cr *brokerv1beta1.ActiveMQArtemis, client rtclient.Client, scheme *runtime.Scheme) (retry bool) { var condition metav1.Condition err := AssertBrokersAvailable(cr, client, scheme) @@ -2892,7 +3138,7 @@ func ProcessBrokerStatus(cr *brokerv1beta1.ActiveMQArtemis, client rtclient.Clie return err.Requeue() } - err = AssertBrokerImageVersion(cr, client, scheme) + err = reconciler.AssertBrokerImageVersion(cr, client, scheme) if err == nil { condition = metav1.Condition{ Type: brokerv1beta1.BrokerVersionAlignedConditionType, @@ -2905,7 +3151,7 @@ func ProcessBrokerStatus(cr *brokerv1beta1.ActiveMQArtemis, client rtclient.Clie } meta.SetStatusCondition(&cr.Status.Conditions, condition) - err = AssertBrokerPropertiesStatus(cr, client, scheme) + err = reconciler.AssertBrokerPropertiesStatus(cr, client, scheme) if err == nil { condition = metav1.Condition{ Type: brokerv1beta1.ConfigAppliedConditionType, @@ -2919,7 +3165,7 @@ func ProcessBrokerStatus(cr *brokerv1beta1.ActiveMQArtemis, client rtclient.Clie meta.SetStatusCondition(&cr.Status.Conditions, condition) if _, _, found := getConfigExtraMount(cr, jaasConfigSuffix); found { - err = AssertJaasPropertiesStatus(cr, client, scheme) + err = reconciler.AssertJaasPropertiesStatus(cr, client, scheme) if err == nil { condition = metav1.Condition{ Type: brokerv1beta1.JaasConfigAppliedConditionType, @@ -3009,16 +3255,16 @@ func AssertBrokersAvailable(cr *brokerv1beta1.ActiveMQArtemis, client rtclient.C return nil } -func AssertBrokerPropertiesStatus(cr *brokerv1beta1.ActiveMQArtemis, client rtclient.Client, scheme *runtime.Scheme) ArtemisError { +func (reconciler *ActiveMQArtemisReconcilerImpl) AssertBrokerPropertiesStatus(cr *brokerv1beta1.ActiveMQArtemis, client rtclient.Client, scheme *runtime.Scheme) ArtemisError { reqLogger := ctrl.Log.WithValues("ActiveMQArtemis Name", cr.Name) - secretProjection, err := getSecretProjection(getConfigAppliedConfigMapName(cr), client) + secretProjection, err := getSecretProjection(getPropertiesResourceNsName(cr), client) if err != nil { reqLogger.V(2).Info("error retrieving config resources. requeing") return NewUnknownJolokiaError(err) } - errorStatus := checkProjectionStatus(cr, client, secretProjection, func(BrokerStatus *brokerStatus, FileName string) (propertiesStatus, bool) { + errorStatus := reconciler.checkProjectionStatus(cr, client, secretProjection, func(BrokerStatus *brokerStatus, FileName string) (propertiesStatus, bool) { current, present := BrokerStatus.BrokerConfigStatus.PropertiesStatus[FileName] return current, present }) @@ -3031,7 +3277,7 @@ func AssertBrokerPropertiesStatus(cr *brokerv1beta1.ActiveMQArtemis, client rtcl reqLogger.V(2).Info("error retrieving -bp extra mount resource. requeing") return NewUnknownJolokiaError(err) } - errorStatus = checkProjectionStatus(cr, client, secretProjection, func(BrokerStatus *brokerStatus, FileName string) (propertiesStatus, bool) { + errorStatus = reconciler.checkProjectionStatus(cr, client, secretProjection, func(BrokerStatus *brokerStatus, FileName string) (propertiesStatus, bool) { current, present := BrokerStatus.BrokerConfigStatus.PropertiesStatus[FileName] return current, present }) @@ -3048,7 +3294,7 @@ func AssertBrokerPropertiesStatus(cr *brokerv1beta1.ActiveMQArtemis, client rtcl return errorStatus } -func AssertJaasPropertiesStatus(cr *brokerv1beta1.ActiveMQArtemis, client rtclient.Client, scheme *runtime.Scheme) ArtemisError { +func (reconciler *ActiveMQArtemisReconcilerImpl) AssertJaasPropertiesStatus(cr *brokerv1beta1.ActiveMQArtemis, client rtclient.Client, scheme *runtime.Scheme) ArtemisError { reqLogger := ctrl.Log.WithValues("ActiveMQArtemis Name", cr.Name) Projection, err := getConfigMappedJaasProperties(cr, client) @@ -3057,7 +3303,7 @@ func AssertJaasPropertiesStatus(cr *brokerv1beta1.ActiveMQArtemis, client rtclie return NewUnknownJolokiaError(err) } - statusError := checkProjectionStatus(cr, client, Projection, func(BrokerStatus *brokerStatus, FileName string) (propertiesStatus, bool) { + statusError := reconciler.checkProjectionStatus(cr, client, Projection, func(BrokerStatus *brokerStatus, FileName string) (propertiesStatus, bool) { current, present := BrokerStatus.ServerStatus.Jaas.PropertiesStatus[FileName] return current, present }) @@ -3069,13 +3315,13 @@ func AssertJaasPropertiesStatus(cr *brokerv1beta1.ActiveMQArtemis, client rtclie return statusError } -func AssertBrokerImageVersion(cr *brokerv1beta1.ActiveMQArtemis, client rtclient.Client, scheme *runtime.Scheme) ArtemisError { +func (reconciler *ActiveMQArtemisReconcilerImpl) AssertBrokerImageVersion(cr *brokerv1beta1.ActiveMQArtemis, client rtclient.Client, scheme *runtime.Scheme) ArtemisError { reqLogger := ctrl.Log.WithValues("ActiveMQArtemis Name", cr.Name) // The ResolveBrokerVersionFromCR should never fail because validation succeeded resolvedFullVersion, _ := common.ResolveBrokerVersionFromCR(cr) - statusError := checkStatus(cr, client, func(brokerStatus *brokerStatus, jk *jolokia_client.JkInfo) ArtemisError { + statusError := reconciler.checkStatus(cr, client, func(brokerStatus *brokerStatus, jk *jolokia_client.JkInfo) ArtemisError { if brokerStatus.ServerStatus.Version != resolvedFullVersion { err := errors.Errorf("broker version non aligned on pod %s-%s, the detected version [%s] doesn't match the spec.version [%s] resolved as [%s]", @@ -3090,20 +3336,27 @@ func AssertBrokerImageVersion(cr *brokerv1beta1.ActiveMQArtemis, client rtclient return statusError } -func checkStatus(cr *brokerv1beta1.ActiveMQArtemis, client rtclient.Client, checkBrokerStatus func(BrokerStatus *brokerStatus, jk *jolokia_client.JkInfo) ArtemisError) ArtemisError { +func (reconciler *ActiveMQArtemisReconcilerImpl) checkStatus(cr *brokerv1beta1.ActiveMQArtemis, client rtclient.Client, checkBrokerStatus func(BrokerStatus *brokerStatus, jk *jolokia_client.JkInfo) ArtemisError) ArtemisError { reqLogger := ctrl.Log.WithValues("ActiveMQArtemis Name", cr.Name) - resource := types.NamespacedName{ - Name: cr.Name, - Namespace: cr.Namespace, + var jks []*jolokia_client.JkInfo + if isRestricted(cr) { + jks = jolokia_client.GetMinimalJolokiaAgents(cr, client) + } else { + resource := types.NamespacedName{ + Name: cr.Name, + Namespace: cr.Namespace, + } + jks = jolokia_client.GetBrokers(resource, []ss.StatefulSetInfo{ + { + NamespacedName: types.NamespacedName{Name: namer.CrToSS(cr.Name), Namespace: cr.Namespace}, + Replicas: cr.Status.DeploymentPlanSize, + Labels: nil, + }}, client) } - ssInfos := ss.GetDeployedStatefulSetNames(client, cr.Namespace, []types.NamespacedName{resource}) - - jks := jolokia_client.GetBrokers(resource, ssInfos, client) - if len(jks) == 0 { - reqLogger.V(1).Info("not found Jolokia Clients available. requeing") + reqLogger.V(1).Info("no Jolokia Clients available. requeing") return NewJolokiaClientsNotFoundError(errors.New("Waiting for Jolokia Clients to become available")) } @@ -3111,7 +3364,7 @@ func checkStatus(cr *brokerv1beta1.ActiveMQArtemis, client rtclient.Client, chec currentJson, err := jk.Artemis.GetStatus() if err != nil { - reqLogger.V(2).Info("unknown status reported from Jolokia.", "IP", jk.IP, "Ordinal", jk.Ordinal, "error", err) + reqLogger.V(1).Info("unknown status reported from Jolokia.", "IP", jk.IP, "Ordinal", jk.Ordinal, "error", err) return NewUnknownJolokiaError(err) } @@ -3134,12 +3387,12 @@ func checkStatus(cr *brokerv1beta1.ActiveMQArtemis, client rtclient.Client, chec return nil } -func checkProjectionStatus(cr *brokerv1beta1.ActiveMQArtemis, client rtclient.Client, secretProjection *projection, extractStatus func(BrokerStatus *brokerStatus, FileName string) (propertiesStatus, bool)) ArtemisError { +func (reconciler *ActiveMQArtemisReconcilerImpl) checkProjectionStatus(cr *brokerv1beta1.ActiveMQArtemis, client rtclient.Client, secretProjection *projection, extractStatus func(BrokerStatus *brokerStatus, FileName string) (propertiesStatus, bool)) ArtemisError { reqLogger := ctrl.Log.WithValues("ActiveMQArtemis Name", cr.Name) reqLogger.V(2).Info("in sync check", "projection", secretProjection) - checkErr := checkStatus(cr, client, func(brokerStatus *brokerStatus, jk *jolokia_client.JkInfo) ArtemisError { + checkErr := reconciler.checkStatus(cr, client, func(brokerStatus *brokerStatus, jk *jolokia_client.JkInfo) ArtemisError { var current propertiesStatus var present bool diff --git a/controllers/activemqartemis_reconciler_test.go b/controllers/activemqartemis_reconciler_test.go index 9f7e1348c..e0ed64aba 100644 --- a/controllers/activemqartemis_reconciler_test.go +++ b/controllers/activemqartemis_reconciler_test.go @@ -272,7 +272,7 @@ func TestGetConfigAppliedConfigMapName(t *testing.T) { Name: "test", }, } - name := getConfigAppliedConfigMapName(&cr) + name := getPropertiesResourceNsName(&cr) assert.Equal(t, "test-ns", name.Namespace) assert.Equal(t, "test-props", name.Name) } diff --git a/controllers/common_util_test.go b/controllers/common_util_test.go index b95dc4b6f..1cfe80c74 100644 --- a/controllers/common_util_test.go +++ b/controllers/common_util_test.go @@ -119,9 +119,9 @@ func CleanClusterResource(res client.Object, name string, namespace string) { CleanResourceWithTimeouts(res, name, namespace, existingClusterTimeout, existingClusterInterval) } -func checkSecretHasCorrectKeyValue(g Gomega, secName string, ns types.NamespacedName, key string, expectedValue string) { +func checkSecretHasCorrectKeyValue(g Gomega, ns types.NamespacedName, key string, expectedValue string) { g.Eventually(func(g Gomega) { - secret, err := secrets.RetriveSecret(ns, secName, make(map[string]string), k8sClient) + secret, err := secrets.RetriveSecret(ns, make(map[string]string), k8sClient) g.Expect(err).Should(BeNil()) data := secret.Data[key] g.Expect(strings.Contains(string(data), expectedValue)).Should(BeTrue()) @@ -964,10 +964,10 @@ func InstallCert(certName string, namespace string, customFunc func(candidate *c return &cmCert } -func InstallCaBundle(name string, sourceSecret string, caFileName string) *tm.Bundle { +func InstallCaBundle(bundleName string, sourceSecret string, caFileName string) *tm.Bundle { bundle := tm.Bundle{} - if k8sClient.Get(ctx, types.NamespacedName{Name: name, Namespace: defaultNamespace}, &bundle) == nil { - CleanResource(&bundle, name, defaultNamespace) + if k8sClient.Get(ctx, types.NamespacedName{Name: bundleName, Namespace: defaultNamespace}, &bundle) == nil { + CleanResource(&bundle, bundleName, defaultNamespace) } bundle = tm.Bundle{ @@ -976,7 +976,7 @@ func InstallCaBundle(name string, sourceSecret string, caFileName string) *tm.Bu Kind: "Bundle", }, ObjectMeta: metav1.ObjectMeta{ - Name: name, + Name: bundleName, Namespace: "cert-manager", }, Spec: tm.BundleSpec{ @@ -1000,7 +1000,7 @@ func InstallCaBundle(name string, sourceSecret string, caFileName string) *tm.Bu k8sClient.Delete(ctx, &bundle) Expect(k8sClient.Create(ctx, &bundle, &client.CreateOptions{})).To(Succeed()) - bundleKey := types.NamespacedName{Name: name, Namespace: "cert-manager"} + bundleKey := types.NamespacedName{Name: bundleName, Namespace: "cert-manager"} newBundle := &tm.Bundle{} Eventually(func(g Gomega) { g.Expect(k8sClient.Get(ctx, bundleKey, newBundle)).Should(Succeed()) diff --git a/controllers/controll_plane_test.go b/controllers/controll_plane_test.go new file mode 100644 index 000000000..dca81c822 --- /dev/null +++ b/controllers/controll_plane_test.go @@ -0,0 +1,168 @@ +/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +// +kubebuilder:docs-gen:collapse=Apache License + +/* +As usual, we start with the necessary imports. We also define some utility variables. +*/ +package controllers + +import ( + "context" + "fmt" + "os" + "time" + + cmv1 "github.com/cert-manager/cert-manager/pkg/apis/certmanager/v1" + cmmetav1 "github.com/cert-manager/cert-manager/pkg/apis/meta/v1" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + + brokerv1beta1 "github.com/artemiscloud/activemq-artemis-operator/api/v1beta1" + "github.com/artemiscloud/activemq-artemis-operator/pkg/utils/common" +) + +var _ = Describe("minimal", func() { + + var installedCertManager bool = false + + BeforeEach(func() { + BeforeEachSpec() + + if verbose { + fmt.Println("Time with MicroSeconds: ", time.Now().Format("2006-01-02 15:04:05.000000"), " test:", CurrentSpecReport()) + } + + if os.Getenv("USE_EXISTING_CLUSTER") == "true" { + //if cert manager/trust manager is not installed, install it + if !CertManagerInstalled() { + Expect(InstallCertManager()).To(Succeed()) + installedCertManager = true + } + + rootIssuer = InstallClusteredIssuer(rootIssuerName, nil) + + rootCert = InstallCert(rootCertName, rootCertNamespce, func(candidate *cmv1.Certificate) { + candidate.Spec.IsCA = true + candidate.Spec.CommonName = "artemis.root.ca" + candidate.Spec.SecretName = rootCertSecretName + candidate.Spec.IssuerRef = cmmetav1.ObjectReference{ + Name: rootIssuer.Name, + Kind: "ClusterIssuer", + } + }) + + caIssuer = InstallClusteredIssuer(caIssuerName, func(candidate *cmv1.ClusterIssuer) { + candidate.Spec.SelfSigned = nil + candidate.Spec.CA = &cmv1.CAIssuer{ + SecretName: rootCertSecretName, + } + }) + InstallCaBundle(caBundleName, rootCertSecretName, caPemTrustStoreName) + + } + + }) + + AfterEach(func() { + + if false && os.Getenv("USE_EXISTING_CLUSTER") == "true" { + UnInstallCaBundle(caBundleName) + UninstallClusteredIssuer(caIssuerName) + UninstallCert(rootCert.Name, rootCert.Namespace) + UninstallClusteredIssuer(rootIssuerName) + + if installedCertManager { + Expect(UninstallCertManager()).To(Succeed()) + installedCertManager = false + } + } + AfterEachSpec() + }) + + Context("restricted rbac", func() { + + It("operator role access", func() { + + if os.Getenv("USE_EXISTING_CLUSTER") != "true" { + return + } + + By("installing operator cert") + InstallCert("operator-cert", defaultNamespace, func(candidate *cmv1.Certificate) { + candidate.Spec.SecretName = "operator-cert" + candidate.Spec.CommonName = "activemq-artemis-operator" + candidate.Spec.IssuerRef = cmmetav1.ObjectReference{ + Name: caIssuer.Name, + Kind: "ClusterIssuer", + } + }) + + ctx := context.Background() + + // empty CRD, name is used for cert subject to match the headless service + crd := brokerv1beta1.ActiveMQArtemis{ + TypeMeta: metav1.TypeMeta{ + Kind: "ActiveMQArtemis", + APIVersion: brokerv1beta1.GroupVersion.Identifier(), + }, + ObjectMeta: metav1.ObjectMeta{ + Name: NextSpecResourceName(), + Namespace: defaultNamespace, + }, + } + + sharedOperandCertName := common.DefaultOperandCertSecretName + By("installing restricted mtls broker cert") + InstallCert(sharedOperandCertName, defaultNamespace, func(candidate *cmv1.Certificate) { + candidate.Spec.SecretName = sharedOperandCertName + candidate.Spec.CommonName = "activemq-artemis-operand" + candidate.Spec.DNSNames = []string{common.OrdinalFQDNS(crd.Name, defaultNamespace, 0)} + candidate.Spec.IssuerRef = cmmetav1.ObjectReference{ + Name: caIssuer.Name, + Kind: "ClusterIssuer", + } + }) + + crd.Spec.Restricted = common.NewTrue() + + By("Deploying the CRD " + crd.ObjectMeta.Name) + Expect(k8sClient.Create(ctx, &crd)).Should(Succeed()) + + brokerKey := types.NamespacedName{Name: crd.Name, Namespace: crd.Namespace} + createdCrd := &brokerv1beta1.ActiveMQArtemis{} + + By("Checking ready, operator can access broker status via jmx") + Eventually(func(g Gomega) { + g.Expect(k8sClient.Get(ctx, brokerKey, createdCrd)).Should(Succeed()) + + if verbose { + fmt.Printf("STATUS: %v\n\n", createdCrd.Status.Conditions) + } + g.Expect(meta.IsStatusConditionTrue(createdCrd.Status.Conditions, brokerv1beta1.ReadyConditionType)).Should(BeTrue()) + g.Expect(meta.IsStatusConditionTrue(createdCrd.Status.Conditions, brokerv1beta1.ConfigAppliedConditionType)).Should(BeTrue()) + + }, existingClusterTimeout, existingClusterInterval).Should(Succeed()) + + Expect(k8sClient.Delete(ctx, createdCrd)).Should(Succeed()) + + UninstallCert("operator-cert", defaultNamespace) + UninstallCert(sharedOperandCertName, defaultNamespace) + }) + }) +}) diff --git a/controllers/suite_test.go b/controllers/suite_test.go index ac9dbe574..a23a8c91b 100644 --- a/controllers/suite_test.go +++ b/controllers/suite_test.go @@ -792,6 +792,9 @@ var _ = BeforeSuite(func() { // role binding to service account for the drainer pod os.Setenv("OPERATOR_WATCH_NAMESPACE", "SomeValueToCauesEqualitytoFailInIsLocalSoDrainControllerSortsCreds") + // pulled from service account when on a pod + os.Setenv("OPERATOR_NAMESPACE", defaultNamespace) + var err error currentDir, err = os.Getwd() Expect(err).To(Succeed()) diff --git a/deploy/activemq-artemis-operator.yaml b/deploy/activemq-artemis-operator.yaml index c67ea8eb8..94e31570d 100644 --- a/deploy/activemq-artemis-operator.yaml +++ b/deploy/activemq-artemis-operator.yaml @@ -3476,6 +3476,9 @@ spec: type: object type: object type: array + restricted: + description: Restricted deployment, mtls jolokia agent with RBAC + type: boolean upgrades: description: Specifies the upgrades (deprecated in favour of Version) properties: diff --git a/deploy/crds/broker_activemqartemis_crd.yaml b/deploy/crds/broker_activemqartemis_crd.yaml index 7de0a5f30..f34e3f1e5 100644 --- a/deploy/crds/broker_activemqartemis_crd.yaml +++ b/deploy/crds/broker_activemqartemis_crd.yaml @@ -2957,6 +2957,9 @@ spec: type: object type: object type: array + restricted: + description: Restricted deployment, mtls jolokia agent with RBAC + type: boolean upgrades: description: Specifies the upgrades (deprecated in favour of Version) properties: diff --git a/docs/help/operator.md b/docs/help/operator.md index 94ac91ae9..dcd701070 100644 --- a/docs/help/operator.md +++ b/docs/help/operator.md @@ -1045,6 +1045,9 @@ There would be corresponding keys for users.properties and roles.properties, the With the possiblity of configuring arbritary jaas login modules directly, the ArtemisSecurityCR ActiveMQArtemisSecuritySpec.LoginModules and ActiveMQArtemisSecuritySpec.SecurityDomains fields are deprecated. +## restricted mode (experimental) +The CR supports a boolean restricted attribute. For single pod broker deployments this provides an empty broker that is configured through brokerProperties. The broker is secured with PKI, there are no passwords. Cert manager can be used to create the necessary PKI secrets. The end result is a minimal broker deployment; an embedded broker with an mtls endpoint for the jolokia jvm agent and RBAC that allows just the operator to check the broker status. There is no init container, no jetty and no xml. + ## Locking down a broker deployment Often when verificiation is complete it is desirable to lock down the broker images and prevent auto upgrades, which will result in a roll out of images and a restart of your broker. diff --git a/pkg/resources/secrets/secret.go b/pkg/resources/secrets/secret.go index dc827bc82..a3a97a736 100644 --- a/pkg/resources/secrets/secret.go +++ b/pkg/resources/secrets/secret.go @@ -89,7 +89,7 @@ func Delete(namespacedName types.NamespacedName, stringDataMap map[string]string resources.Delete(client, secretDefinition) } -func RetriveSecret(namespacedName types.NamespacedName, secretName string, labels map[string]string, client client.Client) (*corev1.Secret, error) { +func RetriveSecret(namespacedName types.NamespacedName, labels map[string]string, client client.Client) (*corev1.Secret, error) { stringData := make(map[string]string) secretDefinition := MakeSecret(namespacedName, stringData, labels) if err := resources.Retrieve(namespacedName, client, &secretDefinition); err != nil { diff --git a/pkg/resources/serviceports/service_port.go b/pkg/resources/serviceports/service_port.go index d98ba7a5d..5b6fc2c2d 100644 --- a/pkg/resources/serviceports/service_port.go +++ b/pkg/resources/serviceports/service_port.go @@ -6,35 +6,48 @@ import ( corev1 "k8s.io/api/core/v1" ) -var appProtocolHTTP = "http" var appProtocolTCP = "tcp" +var appProtocolHTTP = "http" -func GetDefaultPorts() *[]corev1.ServicePort { +func GetDefaultPorts(restricted bool) *[]corev1.ServicePort { - ports := []corev1.ServicePort{ - { - Name: "jgroups", - Protocol: "TCP", - Port: 7800, - AppProtocol: &appProtocolTCP, - TargetPort: intstr.FromInt(int(7800)), - }, - { - Name: "console-jolokia", - Protocol: "TCP", - Port: 8161, - AppProtocol: &appProtocolHTTP, - TargetPort: intstr.FromInt(int(8161)), - }, - { - Name: "all", - Protocol: "TCP", - Port: 61616, - TargetPort: intstr.FromInt(int(61616)), - }, + var ports *[]corev1.ServicePort + if restricted { + ports = &[]corev1.ServicePort{} + } else { + + ports = &[]corev1.ServicePort{ + { + Name: "jgroups", + Protocol: "TCP", + Port: 7800, + AppProtocol: &appProtocolTCP, + TargetPort: intstr.FromInt(int(7800)), + }, + { + Name: "console-jolokia", + Protocol: "TCP", + Port: 8161, + AppProtocol: &appProtocolHTTP, + TargetPort: intstr.FromInt(int(8161)), + }, + { + Name: "jolokia", + Protocol: "TCP", + Port: 8778, + AppProtocol: &appProtocolHTTP, + TargetPort: intstr.FromInt(int(8778)), + }, + { + Name: "all", + Protocol: "TCP", + Port: 61616, + TargetPort: intstr.FromInt(int(61616)), + }, + } } - return &ports + return ports } func setSSLPorts() []corev1.ServicePort { diff --git a/pkg/resources/volumes/volume.go b/pkg/resources/volumes/volume.go index 651d45c20..f6445c0d8 100644 --- a/pkg/resources/volumes/volume.go +++ b/pkg/resources/volumes/volume.go @@ -65,7 +65,7 @@ func MakePersistentVolumeMount(customResourceName string, mountPath string) []co return volumeMounts } -func MakeVolumeForCfg(name string) corev1.Volume { +func MakeEmptyDirVolumeFor(name string) corev1.Volume { volume := corev1.Volume{ Name: name, VolumeSource: corev1.VolumeSource{ diff --git a/pkg/utils/artemis/artemis.go b/pkg/utils/artemis/artemis.go index ed9281073..2aa6528f1 100644 --- a/pkg/utils/artemis/artemis.go +++ b/pkg/utils/artemis/artemis.go @@ -5,6 +5,7 @@ import ( "strings" "github.com/artemiscloud/activemq-artemis-operator/pkg/utils/jolokia" + rtclient "sigs.k8s.io/controller-runtime/pkg/client" ) const ( @@ -54,6 +55,17 @@ func NewArtemis(_ip string, _jolokiaPort string, _name string, _user string, _pa return GetArtemis(_ip, _jolokiaPort, _name, _user, _password, "http") } +func GetArtemisAgentForRestricted(client rtclient.Client, ordinalFqdn string) *Artemis { + artemis := Artemis{ + ip: ordinalFqdn, + jolokiaPort: jolokia.JOLOKIA_AGENT_PORT, + name: "amq-broker", + jolokia: jolokia.GetRestrictedJolokia(client, ordinalFqdn, jolokia.JOLOKIA_AGENT_PORT, "/jolokia"), + } + return &artemis + +} + func GetArtemis(_ip string, _jolokiaPort string, _name string, _user string, _password string, _protocol string) *Artemis { artemis := Artemis{ diff --git a/pkg/utils/certutil/certutil.go b/pkg/utils/certutil/certutil.go index 321a4e4b4..1166c736d 100644 --- a/pkg/utils/certutil/certutil.go +++ b/pkg/utils/certutil/certutil.go @@ -20,6 +20,7 @@ import ( "fmt" "strings" + "github.com/artemiscloud/activemq-artemis-operator/pkg/utils/common" corev1 "k8s.io/api/core/v1" ) @@ -169,19 +170,6 @@ func isSecretFromBundle(secret *corev1.Secret) bool { return exist } -func getBundleNameFromSecret(secret *corev1.Secret) (string, error) { - //extract the bundle target secret key that ends with .pem - //the bundle target secret could include keys for additional formats jks/pkcs12 - for key := range secret.Data { - //the bundle target secret key must ends with .pem - if strings.HasSuffix(key, ".pem") { - return key, nil - } - } - - return "", fmt.Errorf("no keys with the suffix .pem found in the secret %s", secret.Name) -} - func GetSslArgumentsFromSecret(sslSecret *corev1.Secret, trustStoreType string, trustSecret *corev1.Secret, isConsole bool) (*SslArguments, error) { sslArgs := SslArguments{ IsConsole: isConsole, @@ -265,7 +253,7 @@ func GetSslArgumentsFromSecret(sslSecret *corev1.Secret, trustStoreType string, trustVolumeDir := sep + "etc" + sep + trustSecret.Name + "-volume" if isBundleSecret { - bundleName, bundleErr := getBundleNameFromSecret(trustSecret) + bundleName, bundleErr := common.FindFirstDotPemKey(trustSecret) if bundleErr != nil { return nil, bundleErr } diff --git a/pkg/utils/common/common.go b/pkg/utils/common/common.go index 292e8fc60..ba711eb0a 100644 --- a/pkg/utils/common/common.go +++ b/pkg/utils/common/common.go @@ -3,6 +3,9 @@ package common import ( "bytes" "context" + "crypto/tls" + "crypto/x509" + "crypto/x509/pkix" "encoding/json" "fmt" "net" @@ -18,11 +21,14 @@ import ( "github.com/RHsyseng/operator-utils/pkg/olm" "github.com/RHsyseng/operator-utils/pkg/resource/read" brokerv1beta1 "github.com/artemiscloud/activemq-artemis-operator/api/v1beta1" + "github.com/artemiscloud/activemq-artemis-operator/pkg/resources" + "github.com/artemiscloud/activemq-artemis-operator/pkg/resources/secrets" "github.com/artemiscloud/activemq-artemis-operator/pkg/utils/channels" "github.com/artemiscloud/activemq-artemis-operator/pkg/utils/namer" "github.com/artemiscloud/activemq-artemis-operator/pkg/utils/selectors" "github.com/artemiscloud/activemq-artemis-operator/version" "github.com/blang/semver/v4" + routev1 "github.com/openshift/api/route/v1" "github.com/pkg/errors" appsv1 "k8s.io/api/apps/v1" @@ -58,6 +64,11 @@ const ( //defaultRetryInterval is the interval to wait before retring a resource discovery defaultRetryInterval = 3 * time.Second + + // https://cert-manager.io/docs/trust/trust-manager/#preparing-for-production + DefaultOperatorCertSecretName = "operator-cert" + DefaultOperatorCASecretName = "operator-ca" + DefaultOperandCertSecretName = "broker-cert" // or can be prefixed with `cr.Name-` ) var lastStatusMap map[types.NamespacedName]olm.DeploymentStatus = make(map[types.NamespacedName]olm.DeploymentStatus) @@ -70,6 +81,12 @@ var ClusterDomain *string var isOpenshift *bool +var operatorCertSecretName, operatorCASecretName *string + +// we may want to cache and require operator restart on rotation +//var operatorCert *tls.Certificate +//var operatorCertPool *x509.CertPool + type Namers struct { SsGlobalName string SsNameBuilder namer.NamerData @@ -164,6 +181,11 @@ func NewTrue() *bool { return &b } +func NewFalse() *bool { + b := false + return &b +} + // Given the operator's namespace and a string representation of // it's WATCH_NAMESPACE value, this method returns whether // the operator is watching its own(single) namespace, or watching multiple @@ -706,6 +728,178 @@ func DetectOpenshiftWith(config *rest.Config) (bool, error) { return *isOpenshift, nil } +func GetOperandCertSecretName(cr *brokerv1beta1.ActiveMQArtemis, client rtclient.Client) string { + + customName := cr.Name + "-" + DefaultOperandCertSecretName + + if _, err := secrets.RetriveSecret(types.NamespacedName{Namespace: cr.Namespace, Name: customName}, nil, client); err == nil { + return customName + } + return DefaultOperandCertSecretName +} + +func GetOperatorCertSecretName() string { + if operatorCertSecretName == nil { + operatorCertSecretName = fromEnv("OPERATOR_CERT_SECRET_NAME", DefaultOperatorCertSecretName) + } + return *operatorCertSecretName +} + +func GetOperatorCASecretName() string { + if operatorCASecretName == nil { + operatorCASecretName = fromEnv("OPERATOR_CA_SECRET_NAME", DefaultOperatorCASecretName) + } + return *operatorCASecretName +} + +func GetOperatorCASecretKey(client rtclient.Client, bundleSecret *corev1.Secret) (key string, err error) { + if bundleSecret == nil { + if bundleSecret, err = GetOperatorCASecret(client); err != nil { + ctrl.Log.V(1).Info("ca secret not found", "err", err) + return key, errors.Errorf("failed to get ca bundle secret to find ca key %v", err) + } + } + return FindFirstDotPemKey(bundleSecret) +} + +func FindFirstDotPemKey(secret *corev1.Secret) (string, error) { + //extract the bundle target secret key that ends with .pem + //the bundle target secret could include keys for additional formats jks/pkcs12 + for key := range secret.Data { + //the bundle target secret key must ends with .pem + if strings.HasSuffix(key, ".pem") { + return key, nil + } + } + + return "", fmt.Errorf("no keys with the suffix .pem found in the secret %s", secret.Name) +} + +func fromEnv(envVarName, defaultValue string) *string { + if valueFromEnv, found := os.LookupEnv(envVarName); found { + return &valueFromEnv + } else { + return &defaultValue + } +} + +func GetRootCAs(client rtclient.Client) (pool *x509.CertPool, err error) { + + if client == nil { + return nil, nil + } + + var certSecret *corev1.Secret + if certSecret, err = GetOperatorCASecret(client); err != nil { + return nil, err + } + + var bundleSecretKey string + if bundleSecretKey, err = GetOperatorCASecretKey(client, certSecret); err != nil { + return nil, err + } + + pool = x509.NewCertPool() + if ok := pool.AppendCertsFromPEM([]byte(certSecret.Data[bundleSecretKey])); !ok { + return nil, errors.Errorf("Failed to extact key %s from ca secret %v", bundleSecretKey, certSecret.Name) + } + return pool, nil +} + +var operatorNameSpaceFromEnv *string + +func GetOperatorNamespaceFromEnv() (ns string, err error) { + if operatorNameSpaceFromEnv == nil { + if ns, found := os.LookupEnv("OPERATOR_NAMESPACE"); found { + operatorNameSpaceFromEnv = &ns + } else { + return "", errors.New("failed to get operator namespace from env") + } + } + return *operatorNameSpaceFromEnv, nil +} + +func GetOperatorCASecret(client rtclient.Client) (*corev1.Secret, error) { + return GetOperatorSecret(client, GetOperatorCASecretName()) +} + +func GetOperatorClientCertSecret(client rtclient.Client) (*corev1.Secret, error) { + return GetOperatorSecret(client, GetOperatorCertSecretName()) +} + +func GetOperatorSecret(client rtclient.Client, secretName string) (*corev1.Secret, error) { + + var operatorNamespace string + var err error + operatorNamespace, err = GetOperatorNamespaceFromEnv() + if err != nil { + return nil, err + } + + secretNamespacedName := types.NamespacedName{Name: secretName, Namespace: operatorNamespace} + secret := corev1.Secret{} + if err := resources.Retrieve(secretNamespacedName, client, &secret); err != nil { + ctrl.Log.V(1).Info("operator secret not found", "name", secretNamespacedName, "err", err) + return nil, errors.Errorf("failed to get secret %s, %v", secretNamespacedName, err) + } + + return &secret, nil +} + +func GetOperatorClientCertificate(client rtclient.Client, info *tls.CertificateRequestInfo) (cert *tls.Certificate, err error) { + + var secret *corev1.Secret + if secret, err = GetOperatorClientCertSecret(client); err == nil { + cert, err = ExtractCertFromSecret(secret) + } + return cert, err +} + +func ExtractCertFromSecret(certSecret *corev1.Secret) (*tls.Certificate, error) { + cert, err := tls.X509KeyPair(certSecret.Data["tls.crt"], certSecret.Data["tls.key"]) + if err != nil { + return nil, errors.Errorf("invalid key pair in secret %v, %v", certSecret.Name, err) + } + return &cert, nil +} + +func ExtractCertSubjectFromSecret(certSecretName string, namespace string, client rtclient.Client) (*pkix.Name, error) { + + secret, err := GetOperatorSecret(client, certSecretName) + if err != nil { + return nil, err + } + cert, err := ExtractCertFromSecret(secret) + if err != nil { + return nil, err + } + return ExtractCertSubject(cert) +} + +func ExtractCertSubject(cert *tls.Certificate) (*pkix.Name, error) { + x509Cert, err := x509.ParseCertificate(cert.Certificate[0]) + if err != nil { + return nil, errors.Errorf("failed to parse tls.cert %v", err) + } + return &x509Cert.Subject, nil +} + +func OrdinalFQDNS(crName string, crNamespace string, i int32) string { + return OrdinalStringFQDNS(crName, crNamespace, fmt.Sprintf("%d", i)) +} + +func OrdinalStringFQDNS(crName string, crNamespace string, ordinal string) string { + // from NewHeadlessServiceForCR2 and + // https://kubernetes.io/docs/concepts/services-networking/dns-pod-service/#a-aaaa-records + return fmt.Sprintf("%s-ss-%s.%s-hdls-svc.%s.svc.%s", crName, ordinal, crName, crNamespace, GetClusterDomain()) +} + +func ClusterDNSWildCard(crName string, crNamespace string) string { + // from NewHeadlessServiceForCR2 and + // https://kubernetes.io/docs/concepts/services-networking/dns-pod-service/#a-aaaa-records + return fmt.Sprintf("*.%s-hdls-svc.%s.svc.%s", crName, crNamespace, GetClusterDomain()) +} + func ApplyAnnotations(objectMeta *metav1.ObjectMeta, annotations map[string]string) { if annotations != nil { if objectMeta.Annotations == nil { @@ -744,3 +938,17 @@ func HasVolumeMount(container *corev1.Container, mountName string) bool { } return false } + +func GenerateArtemis(name string, namespace string) *brokerv1beta1.ActiveMQArtemis { + return &brokerv1beta1.ActiveMQArtemis{ + TypeMeta: metav1.TypeMeta{ + Kind: "ActiveMQArtemis", + APIVersion: brokerv1beta1.GroupVersion.Identifier(), + }, + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: namespace, + }, + Spec: brokerv1beta1.ActiveMQArtemisSpec{}, + } +} diff --git a/pkg/utils/jolokia/jolokia.go b/pkg/utils/jolokia/jolokia.go index a48830ede..bcb782ea2 100644 --- a/pkg/utils/jolokia/jolokia.go +++ b/pkg/utils/jolokia/jolokia.go @@ -8,8 +8,13 @@ import ( "net/http" "net/url" "time" + + "github.com/artemiscloud/activemq-artemis-operator/pkg/utils/common" + rtclient "sigs.k8s.io/controller-runtime/pkg/client" ) +const JOLOKIA_AGENT_PORT = "8778" + type IData interface { Print() } @@ -49,12 +54,26 @@ type Jolokia struct { user string password string protocol string + restricted bool + client rtclient.Client } func NewJolokia(_ip string, _port string, _path string, _user string, _password string) *Jolokia { return GetJolokia(_ip, _port, _path, _user, _password, "http") } +func GetRestrictedJolokia(client rtclient.Client, _ip string, _port string, _path string) *Jolokia { + j := Jolokia{ + ip: _ip, + port: _port, + jolokiaURL: _ip + ":" + _port + _path, + protocol: "https", + restricted: true, + client: client, + } + return &j +} + func GetJolokia(_ip string, _port string, _path string, _user string, _password string, _protocol string) *Jolokia { j := Jolokia{ @@ -94,6 +113,16 @@ func (j *Jolokia) getClient() *http.Client { InsecureSkipVerify: true, ServerName: j.ip, } + if j.restricted { + httpClientTransport.TLSClientConfig.InsecureSkipVerify = false + httpClientTransport.TLSClientConfig.GetClientCertificate = + func(cri *tls.CertificateRequestInfo) (*tls.Certificate, error) { + return common.GetOperatorClientCertificate(j.client, cri) + } + } + if rootCas, err := common.GetRootCAs(j.client); err == nil { + httpClientTransport.TLSClientConfig.RootCAs = rootCas + } } return &httpClient diff --git a/pkg/utils/jolokia/mock_jolokia.go b/pkg/utils/jolokia/mock_jolokia.go index 026142489..a383f9976 100644 --- a/pkg/utils/jolokia/mock_jolokia.go +++ b/pkg/utils/jolokia/mock_jolokia.go @@ -5,6 +5,7 @@ package jolokia import ( + "net/http" reflect "reflect" gomock "github.com/golang/mock/gomock" @@ -101,3 +102,6 @@ func (mr *MockIJolokiaMockRecorder) Read(path interface{}) *gomock.Call { mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Read", reflect.TypeOf((*MockIJolokia)(nil).Read), path) } + +func (m *MockIJolokia) SetClient(c *http.Client) { +} diff --git a/pkg/utils/jolokia_client/jolokia_client.go b/pkg/utils/jolokia_client/jolokia_client.go index e6aef64ce..331db27a9 100644 --- a/pkg/utils/jolokia_client/jolokia_client.go +++ b/pkg/utils/jolokia_client/jolokia_client.go @@ -18,10 +18,10 @@ package jolokia_client import ( "context" - "fmt" "strconv" "strings" + "github.com/artemiscloud/activemq-artemis-operator/api/v1beta1" "github.com/artemiscloud/activemq-artemis-operator/pkg/resources" "github.com/artemiscloud/activemq-artemis-operator/pkg/resources/secrets" ss "github.com/artemiscloud/activemq-artemis-operator/pkg/resources/statefulsets" @@ -61,6 +61,27 @@ func GetBrokers(resource types.NamespacedName, ssInfos []ss.StatefulSetInfo, cli return artemisArray } +func GetMinimalJolokiaAgents(cr *v1beta1.ActiveMQArtemis, client rtclient.Client) []*JkInfo { + var artemisArray []*JkInfo = nil + var i int32 = 0 + + for i = 0; i < common.GetDeploymentSize(cr); i++ { + + ordinalFqdn := common.OrdinalFQDNS(cr.Name, cr.Namespace, i) + + artemis := mgmt.GetArtemisAgentForRestricted(client, ordinalFqdn) + + jkInfo := JkInfo{ + Artemis: artemis, + IP: ordinalFqdn, + Ordinal: strconv.FormatInt(int64(i), 10), + } + artemisArray = append(artemisArray, &jkInfo) + } + + return artemisArray +} + // Get brokers Using DNS names in the namespace func GetBrokersFromDNS(crName string, namespace string, size int32, client rtclient.Client) []*JkInfo { reqLogger := ctrl.Log.WithName("jolokia").WithValues("Request.Namespace", namespace, "Request.Name", crName) @@ -68,11 +89,8 @@ func GetBrokersFromDNS(crName string, namespace string, size int32, client rtcli var artemisArray []*JkInfo = nil var i int32 = 0 - clusterDomain := common.GetClusterDomain() for i = 0; i < size; i++ { - // from NewHeadlessServiceForCR2 and - // https://kubernetes.io/docs/concepts/services-networking/dns-pod-service/#a-aaaa-records - ordinalFqdn := fmt.Sprintf("%s-ss-%d.%s-hdls-svc.%s.svc.%s", crName, i, crName, namespace, clusterDomain) + ordinalFqdn := common.OrdinalFQDNS(crName, namespace, i) pod := &corev1.Pod{} podNamespacedName := types.NamespacedName{ diff --git a/pkg/utils/lsrcrs/lsrcr.go b/pkg/utils/lsrcrs/lsrcr.go index c88b00506..30c0eb78b 100644 --- a/pkg/utils/lsrcrs/lsrcr.go +++ b/pkg/utils/lsrcrs/lsrcr.go @@ -78,7 +78,7 @@ func retrieveLastSuccessfulReconciledCR(scr *StoredCR, labels map[string]string) Namespace: scr.Namespace, } log.V(2).Info("trying retriving lsrcr", "ns", secretNn, "sec name", secretName, "client", scr.UpdateClient) - theSecret, err := secrets.RetriveSecret(secretNn, secretName, labels, scr.UpdateClient) + theSecret, err := secrets.RetriveSecret(secretNn, labels, scr.UpdateClient) if err != nil { if !errors.IsNotFound(err) { log.Error(err, "failed to retrieve secret", "secret", secretName, "ns", scr.Namespace)