diff --git a/go.mod b/go.mod index 2d298ab..e86477f 100644 --- a/go.mod +++ b/go.mod @@ -7,6 +7,7 @@ require ( github.com/container-storage-interface/spec v1.10.0 github.com/go-logr/logr v1.4.2 github.com/kubernetes-csi/csi-lib-utils v0.19.0 + github.com/prometheus/client_golang v1.20.4 github.com/stretchr/testify v1.9.0 google.golang.org/grpc v1.66.2 k8s.io/apimachinery v0.31.1 @@ -42,6 +43,7 @@ require ( github.com/josharian/intern v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect github.com/klauspost/compress v1.17.9 // indirect + github.com/kylelemons/godebug v1.1.0 // indirect github.com/mailru/easyjson v0.7.7 // indirect github.com/moby/sys/mountinfo v0.7.2 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect @@ -51,7 +53,6 @@ require ( github.com/opencontainers/runtime-spec v1.2.0 // indirect github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect - github.com/prometheus/client_golang v1.20.4 // indirect github.com/prometheus/client_model v0.6.1 // indirect github.com/prometheus/common v0.59.1 // indirect github.com/prometheus/procfs v0.15.1 // indirect diff --git a/manager/manager.go b/manager/manager.go index 869693d..c8c583e 100644 --- a/manager/manager.go +++ b/manager/manager.go @@ -47,6 +47,7 @@ import ( internalapi "github.com/cert-manager/csi-lib/internal/api" internalapiutil "github.com/cert-manager/csi-lib/internal/api/util" "github.com/cert-manager/csi-lib/metadata" + "github.com/cert-manager/csi-lib/metrics" "github.com/cert-manager/csi-lib/storage" ) @@ -89,6 +90,9 @@ type Options struct { // RenewalBackoffConfig configures the exponential backoff applied to certificate renewal failures. RenewalBackoffConfig *wait.Backoff + + // Metrics is used for exposing Prometheus metrics + Metrics *metrics.Metrics } // NewManager constructs a new manager used to manage volumes containing @@ -126,6 +130,9 @@ func NewManager(opts Options) (*Manager, error) { if opts.Log == nil { return nil, errors.New("log must be set") } + if opts.Metrics == nil { + opts.Metrics = metrics.New(opts.Log) + } if opts.MetadataReader == nil { return nil, errors.New("MetadataReader must be set") } @@ -241,6 +248,7 @@ func NewManager(opts Options) (*Manager, error) { metadataReader: opts.MetadataReader, clock: opts.Clock, log: *opts.Log, + metrics: opts.Metrics, generatePrivateKey: opts.GeneratePrivateKey, generateRequest: opts.GenerateRequest, @@ -375,6 +383,9 @@ type Manager struct { // No thread safety is added around this field, and it MUST NOT be used for any implementation logic. // It should not be used full-stop :). doNotUse_CallOnEachIssue func() + + // metrics is used to expose Prometheus + metrics *metrics.Metrics } // issue will step through the entire issuance flow for a volume. @@ -387,6 +398,9 @@ func (m *Manager) issue(ctx context.Context, volumeID string) error { log := m.log.WithValues("volume_id", volumeID) log.Info("Processing issuance") + // Increase issue count + m.metrics.IncrementIssueCallCount(m.nodeNameHash, volumeID) + if err := m.cleanupStaleRequests(ctx, log, volumeID); err != nil { return fmt.Errorf("cleaning up stale requests: %w", err) } @@ -594,7 +608,7 @@ func (m *Manager) handleRequest(ctx context.Context, volumeID string, meta metad // Calculate the default next issuance time. // The implementation's writeKeypair function may override this value before // writing to the storage layer. - renewalPoint, err := calculateNextIssuanceTime(req.Status.Certificate) + expiryPoint, renewalPoint, err := getExpiryAndDefaultNextIssuanceTime(req.Status.Certificate) if err != nil { return fmt.Errorf("calculating next issuance time: %w", err) } @@ -606,6 +620,10 @@ func (m *Manager) handleRequest(ctx context.Context, volumeID string, meta metad } log.V(2).Info("Wrote new keypair to storage") + // Update the request metrics. + // Using meta.NextIssuanceTime instead of renewalPoint here, in case writeKeypair overrides the value. + m.metrics.UpdateCertificateRequest(req, expiryPoint, *meta.NextIssuanceTime) + // We must explicitly delete the private key from the pending requests map so that the existing Completed // request will not be re-used upon renewal. // Without this, the renewal would pick up the existing issued certificate and re-issue, rather than requesting @@ -657,6 +675,9 @@ func (m *Manager) cleanupStaleRequests(ctx context.Context, log logr.Logger, vol } } + // Remove the CertificateRequest from the metrics. + m.metrics.RemoveCertificateRequest(toDelete.Name, toDelete.Namespace) + log.Info("Deleted CertificateRequest resource", "name", toDelete.Name, "namespace", toDelete.Namespace) } @@ -756,6 +777,8 @@ func (m *Manager) ManageVolumeImmediate(ctx context.Context, volumeID string) (m // If issuance fails, immediately return without retrying so the caller can decide // how to proceed depending on the context this method was called within. if err := m.issue(ctx, volumeID); err != nil { + // Increase issue error count + m.metrics.IncrementIssueErrorCount(m.nodeNameHash, volumeID) return true, err } } @@ -783,6 +806,8 @@ func (m *Manager) manageVolumeIfNotManaged(volumeID string) (managed bool) { // construct a new channel used to stop management of the volume stopCh := make(chan struct{}) m.managedVolumes[volumeID] = stopCh + // Increase managed volume count for this driver + m.metrics.IncrementManagedVolumeCount(m.nodeNameHash) return true } @@ -800,6 +825,10 @@ func (m *Manager) startRenewalRoutine(volumeID string) (started bool) { return false } + // Increase managed certificate count for this driver. + // We assume each volume will have one certificate to be managed. + m.metrics.IncrementManagedCertificateCount(m.nodeNameHash) + // Create a context that will be cancelled when the stopCh is closed ctx, cancel := context.WithCancel(context.Background()) go func() { @@ -835,6 +864,8 @@ func (m *Manager) startRenewalRoutine(volumeID string) (started bool) { defer issueCancel() if err := m.issue(issueCtx, volumeID); err != nil { log.Error(err, "Failed to issue certificate, retrying after applying exponential backoff") + // Increase issue error count + m.metrics.IncrementIssueErrorCount(m.nodeNameHash, volumeID) return false, nil } return true, nil @@ -874,6 +905,14 @@ func (m *Manager) UnmanageVolume(volumeID string) { if stopCh, ok := m.managedVolumes[volumeID]; ok { close(stopCh) delete(m.managedVolumes, volumeID) + if reqs, err := m.listAllRequestsForVolume(volumeID); err == nil { + // Remove the CertificateRequest from the metrics with the best effort. + for _, req := range reqs { + if req != nil { + m.metrics.RemoveCertificateRequest(req.Name, req.Namespace) + } + } + } } } @@ -919,19 +958,19 @@ func (m *Manager) Stop() { } } -// calculateNextIssuanceTime will return the default time at which the certificate -// should be renewed by the driver- 2/3rds through its lifetime (NotAfter - -// NotBefore). -func calculateNextIssuanceTime(chain []byte) (time.Time, error) { +// getExpiryAndDefaultNextIssuanceTime will return the certificate expiry time, together with +// default time at which the certificate should be renewed by the driver- 2/3rds through its +// lifetime (NotAfter - NotBefore). +func getExpiryAndDefaultNextIssuanceTime(chain []byte) (time.Time, time.Time, error) { block, _ := pem.Decode(chain) crt, err := x509.ParseCertificate(block.Bytes) if err != nil { - return time.Time{}, fmt.Errorf("parsing issued certificate: %w", err) + return time.Time{}, time.Time{}, fmt.Errorf("parsing issued certificate: %w", err) } actualDuration := crt.NotAfter.Sub(crt.NotBefore) renewBeforeNotAfter := actualDuration / 3 - return crt.NotAfter.Add(-renewBeforeNotAfter), nil + return crt.NotAfter, crt.NotAfter.Add(-renewBeforeNotAfter), nil } diff --git a/manager/manager_test.go b/manager/manager_test.go index 15927c6..34d5f01 100644 --- a/manager/manager_test.go +++ b/manager/manager_test.go @@ -454,7 +454,7 @@ func TestManager_cleanupStaleRequests(t *testing.T) { } } -func Test_calculateNextIssuanceTime(t *testing.T) { +func Test_getExpiryAndDefaultNextIssuanceTime(t *testing.T) { notBefore := time.Date(1970, time.January, 1, 0, 0, 0, 0, time.UTC) notAfter := time.Date(1970, time.January, 4, 0, 0, 0, 0, time.UTC) pk, err := rsa.GenerateKey(rand.Reader, 2048) @@ -474,20 +474,23 @@ func Test_calculateNextIssuanceTime(t *testing.T) { certPEM := pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: derBytes}) tests := map[string]struct { - expTime time.Time - expErr bool + expTime time.Time + renewTime time.Time + expErr bool }{ "if no attributes given, return 2/3rd certificate lifetime": { - expTime: notBefore.AddDate(0, 0, 2), - expErr: false, + expTime: notAfter, + renewTime: notBefore.AddDate(0, 0, 2), + expErr: false, }, } for name, test := range tests { t.Run(name, func(t *testing.T) { - renewTime, err := calculateNextIssuanceTime(certPEM) + expTime, renewTime, err := getExpiryAndDefaultNextIssuanceTime(certPEM) assert.Equal(t, test.expErr, err != nil) - assert.Equal(t, test.expTime, renewTime) + assert.Equal(t, test.expTime, expTime) + assert.Equal(t, test.renewTime, renewTime) }) } } diff --git a/metrics/certificaterequest.go b/metrics/certificaterequest.go new file mode 100644 index 0000000..40a91fc --- /dev/null +++ b/metrics/certificaterequest.go @@ -0,0 +1,102 @@ +/* +Copyright 2024 The cert-manager Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package metrics + +import ( + "time" + + "github.com/prometheus/client_golang/prometheus" + + cmapi "github.com/cert-manager/cert-manager/pkg/apis/certmanager/v1" + cmmeta "github.com/cert-manager/cert-manager/pkg/apis/meta/v1" +) + +var readyConditionStatuses = [...]cmmeta.ConditionStatus{ + cmmeta.ConditionTrue, + cmmeta.ConditionFalse, + cmmeta.ConditionUnknown, +} + +// UpdateCertificateRequest will update the given CertificateRequest's metrics for its expiry, renewal, and status condition. +func (m *Metrics) UpdateCertificateRequest(cr *cmapi.CertificateRequest, exp, renewal time.Time) { + m.updateCertificateRequestExpiryAndRenewalTime(cr, exp, renewal) + m.updateCertificateRequestStatus(cr) +} + +// updateCertificateRequestExpiryAndRenewalTime updates the expiry and renewal time of a certificate request +func (m *Metrics) updateCertificateRequestExpiryAndRenewalTime(cr *cmapi.CertificateRequest, exp, renewal time.Time) { + expiryTime := 0.0 + if !exp.IsZero() { + expiryTime = float64(exp.Unix()) + } + m.certificateRequestExpiryTimeSeconds.With(prometheus.Labels{ + "name": cr.Name, + "namespace": cr.Namespace, + "issuer_name": cr.Spec.IssuerRef.Name, + "issuer_kind": cr.Spec.IssuerRef.Kind, + "issuer_group": cr.Spec.IssuerRef.Group}).Set(expiryTime) + + renewalTime := 0.0 + if !renewal.IsZero() { + renewalTime = float64(renewal.Unix()) + } + m.certificateRequestRenewalTimeSeconds.With(prometheus.Labels{ + "name": cr.Name, + "namespace": cr.Namespace, + "issuer_name": cr.Spec.IssuerRef.Name, + "issuer_kind": cr.Spec.IssuerRef.Kind, + "issuer_group": cr.Spec.IssuerRef.Group}).Set(renewalTime) +} + +// updateCertificateRequestStatus will update the metric for that Certificate Request +func (m *Metrics) updateCertificateRequestStatus(cr *cmapi.CertificateRequest) { + for _, c := range cr.Status.Conditions { + if c.Type == cmapi.CertificateRequestConditionReady { + m.updateCertificateRequestReadyStatus(cr, c.Status) + return + } + } + + // If no status condition set yet, set to Unknown + m.updateCertificateRequestReadyStatus(cr, cmmeta.ConditionUnknown) +} + +func (m *Metrics) updateCertificateRequestReadyStatus(cr *cmapi.CertificateRequest, current cmmeta.ConditionStatus) { + for _, condition := range readyConditionStatuses { + value := 0.0 + + if current == condition { + value = 1.0 + } + + m.certificateRequestReadyStatus.With(prometheus.Labels{ + "name": cr.Name, + "namespace": cr.Namespace, + "condition": string(condition), + "issuer_name": cr.Spec.IssuerRef.Name, + "issuer_kind": cr.Spec.IssuerRef.Kind, + "issuer_group": cr.Spec.IssuerRef.Group, + }).Set(value) + } +} + +// RemoveCertificateRequest will delete the CertificateRequest metrics from continuing to be exposed. +func (m *Metrics) RemoveCertificateRequest(name, namespace string) { + m.certificateRequestExpiryTimeSeconds.DeletePartialMatch(prometheus.Labels{"name": name, "namespace": namespace}) + m.certificateRequestRenewalTimeSeconds.DeletePartialMatch(prometheus.Labels{"name": name, "namespace": namespace}) + m.certificateRequestReadyStatus.DeletePartialMatch(prometheus.Labels{"name": name, "namespace": namespace}) +} diff --git a/metrics/certificaterequest_test.go b/metrics/certificaterequest_test.go new file mode 100644 index 0000000..201b461 --- /dev/null +++ b/metrics/certificaterequest_test.go @@ -0,0 +1,358 @@ +/* +Copyright 2024 The cert-manager Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package metrics + +import ( + "strings" + "testing" + "time" + + "github.com/go-logr/logr/testr" + "github.com/prometheus/client_golang/prometheus/testutil" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + cmapi "github.com/cert-manager/cert-manager/pkg/apis/certmanager/v1" + cmmeta "github.com/cert-manager/cert-manager/pkg/apis/meta/v1" + testcrypto "github.com/cert-manager/cert-manager/test/unit/crypto" + "github.com/cert-manager/cert-manager/test/unit/gen" +) + +const expiryMetadata = ` + # HELP certmanager_csi_certificate_request_expiration_timestamp_seconds The date after which the certificate request expires. Expressed as a Unix Epoch Time. + # TYPE certmanager_csi_certificate_request_expiration_timestamp_seconds gauge +` + +const renewalTimeMetadata = ` + # HELP certmanager_csi_certificate_request_renewal_timestamp_seconds The number of seconds before expiration time the certificate request should renew. + # TYPE certmanager_csi_certificate_request_renewal_timestamp_seconds gauge +` + +const readyMetadata = ` + # HELP certmanager_csi_certificate_request_ready_status The ready status of the certificate request. + # TYPE certmanager_csi_certificate_request_ready_status gauge +` + +func TestCertificateRequestMetrics(t *testing.T) { + type testT struct { + cr *cmapi.CertificateRequest + notAfter, renewBefore time.Time + expectedExpiry, expectedReady, expectedRenewalTime string + } + tests := map[string]testT{ + "certificate with expiry and ready status": { + cr: gen.CertificateRequest("test-certificate-request", + gen.SetCertificateRequestNamespace("test-ns"), + gen.SetCertificateRequestIssuer(cmmeta.ObjectReference{ + Name: "test-issuer", + Kind: "test-issuer-kind", + Group: "test-issuer-group", + }), + gen.SetCertificateRequestStatusCondition(cmapi.CertificateRequestCondition{ + Type: cmapi.CertificateRequestConditionReady, + Status: cmmeta.ConditionTrue, + }), + ), + notAfter: time.Unix(2208988804, 0), + + expectedExpiry: ` + certmanager_csi_certificate_request_expiration_timestamp_seconds{issuer_group="test-issuer-group",issuer_kind="test-issuer-kind",issuer_name="test-issuer",name="test-certificate-request",namespace="test-ns"} 2.208988804e+09 +`, + expectedReady: ` + certmanager_csi_certificate_request_ready_status{condition="False",issuer_group="test-issuer-group",issuer_kind="test-issuer-kind",issuer_name="test-issuer",name="test-certificate-request",namespace="test-ns"} 0 + certmanager_csi_certificate_request_ready_status{condition="True",issuer_group="test-issuer-group",issuer_kind="test-issuer-kind",issuer_name="test-issuer",name="test-certificate-request",namespace="test-ns"} 1 + certmanager_csi_certificate_request_ready_status{condition="Unknown",issuer_group="test-issuer-group",issuer_kind="test-issuer-kind",issuer_name="test-issuer",name="test-certificate-request",namespace="test-ns"} 0 +`, + expectedRenewalTime: ` + certmanager_csi_certificate_request_renewal_timestamp_seconds{issuer_group="test-issuer-group",issuer_kind="test-issuer-kind",issuer_name="test-issuer",name="test-certificate-request",namespace="test-ns"} 0 +`, + }, + "certificate with no expiry and no status should give an expiry of 0 and Unknown status": { + cr: gen.CertificateRequest("test-certificate-request", + gen.SetCertificateRequestNamespace("test-ns"), + gen.SetCertificateRequestIssuer(cmmeta.ObjectReference{ + Name: "test-issuer", + Kind: "test-issuer-kind", + Group: "test-issuer-group", + }), + ), + + expectedExpiry: ` + certmanager_csi_certificate_request_expiration_timestamp_seconds{issuer_group="test-issuer-group",issuer_kind="test-issuer-kind",issuer_name="test-issuer",name="test-certificate-request",namespace="test-ns"} 0 +`, + expectedReady: ` + certmanager_csi_certificate_request_ready_status{condition="False",issuer_group="test-issuer-group",issuer_kind="test-issuer-kind",issuer_name="test-issuer",name="test-certificate-request",namespace="test-ns"} 0 + certmanager_csi_certificate_request_ready_status{condition="True",issuer_group="test-issuer-group",issuer_kind="test-issuer-kind",issuer_name="test-issuer",name="test-certificate-request",namespace="test-ns"} 0 + certmanager_csi_certificate_request_ready_status{condition="Unknown",issuer_group="test-issuer-group",issuer_kind="test-issuer-kind",issuer_name="test-issuer",name="test-certificate-request",namespace="test-ns"} 1 +`, + expectedRenewalTime: ` + certmanager_csi_certificate_request_renewal_timestamp_seconds{issuer_group="test-issuer-group",issuer_kind="test-issuer-kind",issuer_name="test-issuer",name="test-certificate-request",namespace="test-ns"} 0 +`, + }, + "certificate with expiry and status False should give an expiry and False status": { + cr: gen.CertificateRequest("test-certificate-request", + gen.SetCertificateRequestNamespace("test-ns"), + gen.SetCertificateRequestIssuer(cmmeta.ObjectReference{ + Name: "test-issuer", + Kind: "test-issuer-kind", + Group: "test-issuer-group", + }), + gen.SetCertificateRequestStatusCondition(cmapi.CertificateRequestCondition{ + Type: cmapi.CertificateRequestConditionReady, + Status: cmmeta.ConditionFalse, + }), + ), + notAfter: time.Unix(100, 0), + + expectedExpiry: ` + certmanager_csi_certificate_request_expiration_timestamp_seconds{issuer_group="test-issuer-group",issuer_kind="test-issuer-kind",issuer_name="test-issuer",name="test-certificate-request",namespace="test-ns"} 100 +`, + expectedReady: ` + certmanager_csi_certificate_request_ready_status{condition="False",issuer_group="test-issuer-group",issuer_kind="test-issuer-kind",issuer_name="test-issuer",name="test-certificate-request",namespace="test-ns"} 1 + certmanager_csi_certificate_request_ready_status{condition="True",issuer_group="test-issuer-group",issuer_kind="test-issuer-kind",issuer_name="test-issuer",name="test-certificate-request",namespace="test-ns"} 0 + certmanager_csi_certificate_request_ready_status{condition="Unknown",issuer_group="test-issuer-group",issuer_kind="test-issuer-kind",issuer_name="test-issuer",name="test-certificate-request",namespace="test-ns"} 0 +`, + expectedRenewalTime: ` + certmanager_csi_certificate_request_renewal_timestamp_seconds{issuer_group="test-issuer-group",issuer_kind="test-issuer-kind",issuer_name="test-issuer",name="test-certificate-request",namespace="test-ns"} 0 +`, + }, + "certificate with expiry and status Unknown should give an expiry and Unknown status": { + cr: gen.CertificateRequest("test-certificate-request", + gen.SetCertificateRequestNamespace("test-ns"), + gen.SetCertificateRequestIssuer(cmmeta.ObjectReference{ + Name: "test-issuer", + Kind: "test-issuer-kind", + Group: "test-issuer-group", + }), + gen.SetCertificateRequestStatusCondition(cmapi.CertificateRequestCondition{ + Type: cmapi.CertificateRequestConditionReady, + Status: cmmeta.ConditionUnknown, + }), + ), + notAfter: time.Unix(99999, 0), + + expectedExpiry: ` + certmanager_csi_certificate_request_expiration_timestamp_seconds{issuer_group="test-issuer-group",issuer_kind="test-issuer-kind",issuer_name="test-issuer",name="test-certificate-request",namespace="test-ns"} 99999 +`, + expectedReady: ` + certmanager_csi_certificate_request_ready_status{condition="False",issuer_group="test-issuer-group",issuer_kind="test-issuer-kind",issuer_name="test-issuer",name="test-certificate-request",namespace="test-ns"} 0 + certmanager_csi_certificate_request_ready_status{condition="True",issuer_group="test-issuer-group",issuer_kind="test-issuer-kind",issuer_name="test-issuer",name="test-certificate-request",namespace="test-ns"} 0 + certmanager_csi_certificate_request_ready_status{condition="Unknown",issuer_group="test-issuer-group",issuer_kind="test-issuer-kind",issuer_name="test-issuer",name="test-certificate-request",namespace="test-ns"} 1 +`, + expectedRenewalTime: ` + certmanager_csi_certificate_request_renewal_timestamp_seconds{issuer_group="test-issuer-group",issuer_kind="test-issuer-kind",issuer_name="test-issuer",name="test-certificate-request",namespace="test-ns"} 0 +`, + }, + "certificate with expiry and ready status and renew before": { + cr: gen.CertificateRequest("test-certificate-request", + gen.SetCertificateRequestNamespace("test-ns"), + gen.SetCertificateRequestIssuer(cmmeta.ObjectReference{ + Name: "test-issuer", + Kind: "test-issuer-kind", + Group: "test-issuer-group", + }), + gen.SetCertificateRequestStatusCondition(cmapi.CertificateRequestCondition{ + Type: cmapi.CertificateRequestConditionReady, + Status: cmmeta.ConditionTrue, + }), + ), + notAfter: time.Unix(2208988804, 0), + renewBefore: time.Unix(2108988804, 0), + + expectedExpiry: ` + certmanager_csi_certificate_request_expiration_timestamp_seconds{issuer_group="test-issuer-group",issuer_kind="test-issuer-kind",issuer_name="test-issuer",name="test-certificate-request",namespace="test-ns"} 2.208988804e+09 +`, + expectedReady: ` + certmanager_csi_certificate_request_ready_status{condition="False",issuer_group="test-issuer-group",issuer_kind="test-issuer-kind",issuer_name="test-issuer",name="test-certificate-request",namespace="test-ns"} 0 + certmanager_csi_certificate_request_ready_status{condition="True",issuer_group="test-issuer-group",issuer_kind="test-issuer-kind",issuer_name="test-issuer",name="test-certificate-request",namespace="test-ns"} 1 + certmanager_csi_certificate_request_ready_status{condition="Unknown",issuer_group="test-issuer-group",issuer_kind="test-issuer-kind",issuer_name="test-issuer",name="test-certificate-request",namespace="test-ns"} 0 +`, + expectedRenewalTime: ` + certmanager_csi_certificate_request_renewal_timestamp_seconds{issuer_group="test-issuer-group",issuer_kind="test-issuer-kind",issuer_name="test-issuer",name="test-certificate-request",namespace="test-ns"} 2.108988804e+09 +`, + }, + } + for n, test := range tests { + t.Run(n, func(t *testing.T) { + testLog := testr.New(t) + m := New(&testLog) + m.UpdateCertificateRequest(test.cr, test.notAfter, test.renewBefore) + + if err := testutil.CollectAndCompare(m.certificateRequestExpiryTimeSeconds, + strings.NewReader(expiryMetadata+test.expectedExpiry), + "certmanager_csi_certificate_request_expiration_timestamp_seconds", + ); err != nil { + t.Errorf("unexpected collecting result:\n%s", err) + } + + if err := testutil.CollectAndCompare(m.certificateRequestRenewalTimeSeconds, + strings.NewReader(renewalTimeMetadata+test.expectedRenewalTime), + "certmanager_csi_certificate_request_renewal_timestamp_seconds", + ); err != nil { + t.Errorf("unexpected collecting result:\n%s", err) + } + + if err := testutil.CollectAndCompare(m.certificateRequestReadyStatus, + strings.NewReader(readyMetadata+test.expectedReady), + "certmanager_csi_certificate_request_ready_status", + ); err != nil { + t.Errorf("unexpected collecting result:\n%s", err) + } + }) + } +} + +func TestCertificateRequestCache(t *testing.T) { + testLog := testr.New(t) + m := New(&testLog) + + // private key to be used to generate X509 certificate + privKey := testcrypto.MustCreatePEMPrivateKey(t) + certTemplate := &cmapi.Certificate{ + ObjectMeta: metav1.ObjectMeta{Namespace: "testns", Name: "test"}, + Spec: cmapi.CertificateSpec{ + CommonName: "test.example.com", + }, + } + notBefore := time.Unix(0, 0) + notAfter1, notAfter2, notAfter3 := + time.Unix(100, 0), time.Unix(200, 0), time.Unix(300, 0) + renew1, renew2, renew3 := + time.Unix(50, 0), time.Unix(150, 0), time.Unix(250, 0) + + cr1 := gen.CertificateRequest("cr1", + gen.SetCertificateRequestNamespace("testns"), + gen.SetCertificateRequestIssuer(cmmeta.ObjectReference{ + Name: "test-issuer", + Kind: "test-issuer-kind", + Group: "test-issuer-group", + }), + gen.SetCertificateRequestStatusCondition(cmapi.CertificateRequestCondition{ + Type: cmapi.CertificateRequestConditionReady, + Status: cmmeta.ConditionUnknown, + }), + gen.SetCertificateRequestCertificate( + testcrypto.MustCreateCertWithNotBeforeAfter(t, privKey, certTemplate, notBefore, notAfter1)), + ) + cr2 := gen.CertificateRequest("cr2", + gen.SetCertificateRequestNamespace("testns"), + gen.SetCertificateRequestIssuer(cmmeta.ObjectReference{ + Name: "test-issuer", + Kind: "test-issuer-kind", + Group: "test-issuer-group", + }), + gen.SetCertificateRequestStatusCondition(cmapi.CertificateRequestCondition{ + Type: cmapi.CertificateRequestConditionReady, + Status: cmmeta.ConditionTrue, + }), + gen.SetCertificateRequestCertificate( + testcrypto.MustCreateCertWithNotBeforeAfter(t, privKey, certTemplate, notBefore, notAfter2)), + ) + cr3 := gen.CertificateRequest("cr3", + gen.SetCertificateRequestNamespace("testns"), + gen.SetCertificateRequestIssuer(cmmeta.ObjectReference{ + Name: "test-issuer", + Kind: "test-issuer-kind", + Group: "test-issuer-group", + }), + gen.SetCertificateRequestStatusCondition(cmapi.CertificateRequestCondition{ + Type: cmapi.CertificateRequestConditionReady, + Status: cmmeta.ConditionFalse, + }), + gen.SetCertificateRequestCertificate( + testcrypto.MustCreateCertWithNotBeforeAfter(t, privKey, certTemplate, notBefore, notAfter3)), + ) + + // Observe all three Certificate metrics + m.UpdateCertificateRequest(cr1, notAfter1, renew1) + m.UpdateCertificateRequest(cr2, notAfter2, renew2) + m.UpdateCertificateRequest(cr3, notAfter3, renew3) + + // Check all three metrics exist + if err := testutil.CollectAndCompare(m.certificateRequestReadyStatus, + strings.NewReader(readyMetadata+` + certmanager_csi_certificate_request_ready_status{condition="False",issuer_group="test-issuer-group",issuer_kind="test-issuer-kind",issuer_name="test-issuer",name="cr1",namespace="testns"} 0 + certmanager_csi_certificate_request_ready_status{condition="False",issuer_group="test-issuer-group",issuer_kind="test-issuer-kind",issuer_name="test-issuer",name="cr2",namespace="testns"} 0 + certmanager_csi_certificate_request_ready_status{condition="False",issuer_group="test-issuer-group",issuer_kind="test-issuer-kind",issuer_name="test-issuer",name="cr3",namespace="testns"} 1 + certmanager_csi_certificate_request_ready_status{condition="True",issuer_group="test-issuer-group",issuer_kind="test-issuer-kind",issuer_name="test-issuer",name="cr1",namespace="testns"} 0 + certmanager_csi_certificate_request_ready_status{condition="True",issuer_group="test-issuer-group",issuer_kind="test-issuer-kind",issuer_name="test-issuer",name="cr2",namespace="testns"} 1 + certmanager_csi_certificate_request_ready_status{condition="True",issuer_group="test-issuer-group",issuer_kind="test-issuer-kind",issuer_name="test-issuer",name="cr3",namespace="testns"} 0 + certmanager_csi_certificate_request_ready_status{condition="Unknown",issuer_group="test-issuer-group",issuer_kind="test-issuer-kind",issuer_name="test-issuer",name="cr1",namespace="testns"} 1 + certmanager_csi_certificate_request_ready_status{condition="Unknown",issuer_group="test-issuer-group",issuer_kind="test-issuer-kind",issuer_name="test-issuer",name="cr2",namespace="testns"} 0 + certmanager_csi_certificate_request_ready_status{condition="Unknown",issuer_group="test-issuer-group",issuer_kind="test-issuer-kind",issuer_name="test-issuer",name="cr3",namespace="testns"} 0 +`), + "certmanager_csi_certificate_request_ready_status", + ); err != nil { + t.Errorf("unexpected collecting result:\n%s", err) + } + if err := testutil.CollectAndCompare(m.certificateRequestExpiryTimeSeconds, + strings.NewReader(expiryMetadata+` + certmanager_csi_certificate_request_expiration_timestamp_seconds{issuer_group="test-issuer-group",issuer_kind="test-issuer-kind",issuer_name="test-issuer",name="cr1",namespace="testns"} 100 + certmanager_csi_certificate_request_expiration_timestamp_seconds{issuer_group="test-issuer-group",issuer_kind="test-issuer-kind",issuer_name="test-issuer",name="cr2",namespace="testns"} 200 + certmanager_csi_certificate_request_expiration_timestamp_seconds{issuer_group="test-issuer-group",issuer_kind="test-issuer-kind",issuer_name="test-issuer",name="cr3",namespace="testns"} 300 +`), + "certmanager_csi_certificate_request_expiration_timestamp_seconds", + ); err != nil { + t.Errorf("unexpected collecting result:\n%s", err) + } + + if err := testutil.CollectAndCompare(m.certificateRequestRenewalTimeSeconds, + strings.NewReader(renewalTimeMetadata+` + certmanager_csi_certificate_request_renewal_timestamp_seconds{issuer_group="test-issuer-group",issuer_kind="test-issuer-kind",issuer_name="test-issuer",name="cr1",namespace="testns"} 50 + certmanager_csi_certificate_request_renewal_timestamp_seconds{issuer_group="test-issuer-group",issuer_kind="test-issuer-kind",issuer_name="test-issuer",name="cr2",namespace="testns"} 150 + certmanager_csi_certificate_request_renewal_timestamp_seconds{issuer_group="test-issuer-group",issuer_kind="test-issuer-kind",issuer_name="test-issuer",name="cr3",namespace="testns"} 250 +`), + "certmanager_csi_certificate_request_renewal_timestamp_seconds", + ); err != nil { + t.Errorf("unexpected collecting result:\n%s", err) + } + + // Remove second certificate and check not exists + m.RemoveCertificateRequest("cr2", "testns") + if err := testutil.CollectAndCompare(m.certificateRequestReadyStatus, + strings.NewReader(readyMetadata+` + certmanager_csi_certificate_request_ready_status{condition="False",issuer_group="test-issuer-group",issuer_kind="test-issuer-kind",issuer_name="test-issuer",name="cr1",namespace="testns"} 0 + certmanager_csi_certificate_request_ready_status{condition="False",issuer_group="test-issuer-group",issuer_kind="test-issuer-kind",issuer_name="test-issuer",name="cr3",namespace="testns"} 1 + certmanager_csi_certificate_request_ready_status{condition="True",issuer_group="test-issuer-group",issuer_kind="test-issuer-kind",issuer_name="test-issuer",name="cr1",namespace="testns"} 0 + certmanager_csi_certificate_request_ready_status{condition="True",issuer_group="test-issuer-group",issuer_kind="test-issuer-kind",issuer_name="test-issuer",name="cr3",namespace="testns"} 0 + certmanager_csi_certificate_request_ready_status{condition="Unknown",issuer_group="test-issuer-group",issuer_kind="test-issuer-kind",issuer_name="test-issuer",name="cr1",namespace="testns"} 1 + certmanager_csi_certificate_request_ready_status{condition="Unknown",issuer_group="test-issuer-group",issuer_kind="test-issuer-kind",issuer_name="test-issuer",name="cr3",namespace="testns"} 0 +`), + "certmanager_csi_certificate_request_ready_status", + ); err != nil { + t.Errorf("unexpected collecting result:\n%s", err) + } + if err := testutil.CollectAndCompare(m.certificateRequestExpiryTimeSeconds, + strings.NewReader(expiryMetadata+` + certmanager_csi_certificate_request_expiration_timestamp_seconds{issuer_group="test-issuer-group",issuer_kind="test-issuer-kind",issuer_name="test-issuer",name="cr1",namespace="testns"} 100 + certmanager_csi_certificate_request_expiration_timestamp_seconds{issuer_group="test-issuer-group",issuer_kind="test-issuer-kind",issuer_name="test-issuer",name="cr3",namespace="testns"} 300 +`), + "certmanager_csi_certificate_request_expiration_timestamp_seconds", + ); err != nil { + t.Errorf("unexpected collecting result:\n%s", err) + } + + // Remove all Certificates (even is already removed) and observe no Certificates + m.RemoveCertificateRequest("cr1", "testns") + m.RemoveCertificateRequest("cr2", "testns") + m.RemoveCertificateRequest("cr3", "testns") + if testutil.CollectAndCount(m.certificateRequestReadyStatus, "certmanager_csi_certificate_request_ready_status") != 0 { + t.Errorf("unexpected collecting result") + } + if testutil.CollectAndCount(m.certificateRequestExpiryTimeSeconds, "certmanager_csi_certificate_request_expiration_timestamp_seconds") != 0 { + t.Errorf("unexpected collecting result") + } +} diff --git a/metrics/metrics.go b/metrics/metrics.go new file mode 100644 index 0000000..e2f0b06 --- /dev/null +++ b/metrics/metrics.go @@ -0,0 +1,192 @@ +/* +Copyright 2024 The cert-manager Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package metrics + +import ( + "net" + "net/http" + "time" + + "github.com/go-logr/logr" + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/collectors" + "github.com/prometheus/client_golang/prometheus/promhttp" +) + +const ( + // Namespace is the namespace for csi-lib metric names + namespace = "certmanager" + subsystem = "csi" + prometheusMetricsServerReadTimeout = 8 * time.Second + prometheusMetricsServerWriteTimeout = 8 * time.Second + prometheusMetricsServerMaxHeaderBytes = 1 << 20 // 1 MiB +) + +// Metrics is designed to be a shared object for updating the metrics exposed by csi-lib +type Metrics struct { + log logr.Logger + registry *prometheus.Registry + + certificateRequestExpiryTimeSeconds *prometheus.GaugeVec + certificateRequestRenewalTimeSeconds *prometheus.GaugeVec + certificateRequestReadyStatus *prometheus.GaugeVec + driverIssueCallCount *prometheus.CounterVec + driverIssueErrorCount *prometheus.CounterVec + managedVolumeCount *prometheus.CounterVec + managedCertificateCount *prometheus.CounterVec +} + +// New creates a Metrics struct and populates it with prometheus metric types. +func New(logger *logr.Logger) *Metrics { + var ( + certificateRequestExpiryTimeSeconds = prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Namespace: namespace, + Subsystem: subsystem, + Name: "certificate_request_expiration_timestamp_seconds", + Help: "The date after which the certificate request expires. Expressed as a Unix Epoch Time.", + }, + []string{"name", "namespace", "issuer_name", "issuer_kind", "issuer_group"}, + ) + + certificateRequestRenewalTimeSeconds = prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Namespace: namespace, + Subsystem: subsystem, + Name: "certificate_request_renewal_timestamp_seconds", + Help: "The number of seconds before expiration time the certificate request should renew.", + }, + []string{"name", "namespace", "issuer_name", "issuer_kind", "issuer_group"}, + ) + + certificateRequestReadyStatus = prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Namespace: namespace, + Subsystem: subsystem, + Name: "certificate_request_ready_status", + Help: "The ready status of the certificate request.", + }, + []string{"name", "namespace", "condition", "issuer_name", "issuer_kind", "issuer_group"}, + ) + + driverIssueCallCount = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Namespace: namespace, + Subsystem: subsystem, + Name: "driver_issue_call_count", + Help: "The number of issue() calls made by the driver.", + }, + []string{"node", "volume"}, + ) + + driverIssueErrorCount = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Namespace: namespace, + Subsystem: subsystem, + Name: "driver_issue_error_count", + Help: "The number of errors encountered during the driver issue() calls.", + }, + []string{"node", "volume"}, + ) + + managedVolumeCount = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Namespace: namespace, + Subsystem: subsystem, + Name: "managed_volume_count", + Help: "The number of volume managed by the csi driver.", + }, + []string{"node"}, + ) + + managedCertificateCount = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Namespace: namespace, + Subsystem: subsystem, + Name: "managed_certificate_count", + Help: "The number of certificates managed by the csi driver.", + }, + []string{"node"}, + ) + ) + + // Create Registry and register the recommended collectors + registry := prometheus.NewRegistry() + registry.MustRegister( + collectors.NewProcessCollector(collectors.ProcessCollectorOpts{}), + collectors.NewGoCollector(), + ) + // Create server and register Prometheus metrics handler + m := &Metrics{ + log: logger.WithName("metrics"), + registry: registry, + + certificateRequestExpiryTimeSeconds: certificateRequestExpiryTimeSeconds, + certificateRequestRenewalTimeSeconds: certificateRequestRenewalTimeSeconds, + certificateRequestReadyStatus: certificateRequestReadyStatus, + driverIssueCallCount: driverIssueCallCount, + driverIssueErrorCount: driverIssueErrorCount, + managedVolumeCount: managedVolumeCount, + managedCertificateCount: managedCertificateCount, + } + + return m +} + +// NewServer registers Prometheus metrics and returns a new Prometheus metrics HTTP server. +func (m *Metrics) NewServer(ln net.Listener) *http.Server { + m.registry.MustRegister(m.certificateRequestExpiryTimeSeconds) + m.registry.MustRegister(m.certificateRequestRenewalTimeSeconds) + m.registry.MustRegister(m.certificateRequestReadyStatus) + m.registry.MustRegister(m.driverIssueCallCount) + m.registry.MustRegister(m.driverIssueErrorCount) + m.registry.MustRegister(m.managedVolumeCount) + m.registry.MustRegister(m.managedCertificateCount) + + mux := http.NewServeMux() + mux.Handle("/metrics", promhttp.HandlerFor(m.registry, promhttp.HandlerOpts{})) + + server := &http.Server{ + Addr: ln.Addr().String(), + ReadTimeout: prometheusMetricsServerReadTimeout, + WriteTimeout: prometheusMetricsServerWriteTimeout, + MaxHeaderBytes: prometheusMetricsServerMaxHeaderBytes, + Handler: mux, + } + + return server +} + +// IncrementIssueCallCount will increase the issue call counter for the driver. +func (m *Metrics) IncrementIssueCallCount(nodeNameHash, volumeID string) { + m.driverIssueCallCount.WithLabelValues(nodeNameHash, volumeID).Inc() +} + +// IncrementIssueErrorCount will increase count of errors during issue call of the driver. +func (m *Metrics) IncrementIssueErrorCount(nodeNameHash, volumeID string) { + m.driverIssueErrorCount.WithLabelValues(nodeNameHash, volumeID).Inc() +} + +// IncrementManagedVolumeCount will increase the managed volume counter for the driver. +func (m *Metrics) IncrementManagedVolumeCount(nodeNameHash string) { + m.managedVolumeCount.WithLabelValues(nodeNameHash).Inc() +} + +// IncrementManagedCertificateCount will increase the managed certificate count for the driver. +func (m *Metrics) IncrementManagedCertificateCount(nodeNameHash string) { + m.managedCertificateCount.WithLabelValues(nodeNameHash).Inc() +} diff --git a/test/driver/driver_testing.go b/test/driver/driver_testing.go index bbd4252..02659b2 100644 --- a/test/driver/driver_testing.go +++ b/test/driver/driver_testing.go @@ -36,6 +36,7 @@ import ( "github.com/cert-manager/csi-lib/driver" "github.com/cert-manager/csi-lib/manager" "github.com/cert-manager/csi-lib/metadata" + "github.com/cert-manager/csi-lib/metrics" "github.com/cert-manager/csi-lib/storage" ) @@ -45,6 +46,7 @@ type Options struct { Log *logr.Logger Client cmclient.Interface Mounter mount.Interface + Metrics *metrics.Metrics NodeID string MaxRequestsPerVolume int @@ -109,6 +111,7 @@ func Run(t *testing.T, opts Options) (Options, csi.NodeClient, func()) { Clock: opts.Clock, Log: opts.Log, NodeID: opts.NodeID, + Metrics: opts.Metrics, MaxRequestsPerVolume: opts.MaxRequestsPerVolume, GeneratePrivateKey: opts.GeneratePrivateKey, GenerateRequest: opts.GenerateRequest, diff --git a/test/integration/metrics_test.go b/test/integration/metrics_test.go new file mode 100644 index 0000000..34a3923 --- /dev/null +++ b/test/integration/metrics_test.go @@ -0,0 +1,235 @@ +package integration + +import ( + "context" + "crypto" + "crypto/x509" + "fmt" + "io" + "net" + "net/http" + "os" + "strings" + "testing" + "time" + + "github.com/container-storage-interface/spec/lib/go/csi" + "github.com/go-logr/logr/testr" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/wait" + fakeclock "k8s.io/utils/clock/testing" + + cmapi "github.com/cert-manager/cert-manager/pkg/apis/certmanager/v1" + cmmeta "github.com/cert-manager/cert-manager/pkg/apis/meta/v1" + testcrypto "github.com/cert-manager/cert-manager/test/unit/crypto" + "github.com/cert-manager/csi-lib/manager" + "github.com/cert-manager/csi-lib/metadata" + "github.com/cert-manager/csi-lib/metrics" + "github.com/cert-manager/csi-lib/storage" + testdriver "github.com/cert-manager/csi-lib/test/driver" + testutil "github.com/cert-manager/csi-lib/test/util" +) + +var ( + testMetrics = func(ctx context.Context, metricsEndpoint, expectedOutput string) error { + req, err := http.NewRequestWithContext(ctx, http.MethodGet, metricsEndpoint, nil) + if err != nil { + return err + } + resp, err := http.DefaultClient.Do(req) + if err != nil { + return err + } + defer resp.Body.Close() + + output, err := io.ReadAll(resp.Body) + if err != nil { + return err + } + + trimmedOutput := strings.SplitN(string(output), "# HELP go_gc_duration_seconds", 2)[0] + if strings.TrimSpace(trimmedOutput) != strings.TrimSpace(expectedOutput) { + return fmt.Errorf("got unexpected metrics output\nexp:\n%s\ngot:\n%s\n", + expectedOutput, trimmedOutput) + } + + return nil + } + + waitForMetrics = func(t *testing.T, ctx context.Context, metricsEndpoint, expectedOutput string) { + var lastErr error + err := wait.PollUntilContextCancel(ctx, time.Millisecond*100, true, func(ctx context.Context) (done bool, err error) { + if err := testMetrics(ctx, metricsEndpoint, expectedOutput); err != nil { + lastErr = err + return false, nil + } + + return true, nil + }) + if err != nil { + t.Fatalf("%s: failed to wait for expected metrics to be exposed: %s", err, lastErr) + } + } +) + +func TestMetricsServer(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() + + testLog := testr.New(t) + testNamespace := "test-ns" + + // Build metrics handler, and start metrics server with a random available port + metricsHandler := metrics.New(&testLog) + metricsLn, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + metricsServer := metricsHandler.NewServer(metricsLn) + errCh := make(chan error) + go func() { + defer close(errCh) + testLog.Info("starting metrics server", "address", metricsLn.Addr()) + if err := metricsServer.Serve(metricsLn); err != http.ErrServerClosed { + errCh <- err + } + }() + defer func() { + // allow a timeout for graceful shutdown + shutdownCtx, cancel := context.WithTimeout(ctx, time.Second*5) + defer cancel() + + if err := metricsServer.Shutdown(shutdownCtx); err != nil { + t.Fatal(err) + } + err := <-errCh + if err != nil { + t.Fatal(err) + } + }() + + // Build and start the driver + store := storage.NewMemoryFS() + clock := fakeclock.NewFakeClock(time.Now()) + opts, cl, stop := testdriver.Run(t, testdriver.Options{ + Store: store, + Clock: clock, + Metrics: metricsHandler, + Log: &testLog, + GeneratePrivateKey: func(meta metadata.Metadata) (crypto.PrivateKey, error) { + return nil, nil + }, + GenerateRequest: func(meta metadata.Metadata) (*manager.CertificateRequestBundle, error) { + return &manager.CertificateRequestBundle{ + Namespace: testNamespace, + IssuerRef: cmmeta.ObjectReference{ + Name: "test-issuer", + Kind: "test-issuer-kind", + Group: "test-issuer-group", + }, + }, nil + }, + SignRequest: func(meta metadata.Metadata, key crypto.PrivateKey, request *x509.CertificateRequest) (csr []byte, err error) { + return []byte{}, nil + }, + WriteKeypair: func(meta metadata.Metadata, key crypto.PrivateKey, chain []byte, ca []byte) error { + store.WriteFiles(meta, map[string][]byte{ + "ca": ca, + "cert": chain, + }) + nextIssuanceTime := clock.Now().Add(time.Hour) + meta.NextIssuanceTime = &nextIssuanceTime + return store.WriteMetadata(meta.VolumeID, meta) + }, + }) + defer stop() + + // Should expose no additional metrics + metricsEndpoint := fmt.Sprintf("http://%s/metrics", metricsServer.Addr) + waitForMetrics(t, ctx, metricsEndpoint, "") + + // Create a self-signed Certificate and wait for it to be issued + privKey := testcrypto.MustCreatePEMPrivateKey(t) + certTemplate := &cmapi.Certificate{ + ObjectMeta: metav1.ObjectMeta{Namespace: testNamespace, Name: "test"}, + Spec: cmapi.CertificateSpec{ + CommonName: "test.example.com", + }, + } + notBefore, notAfter := time.Unix(0, 0), time.Unix(300, 0) // renewal time will be 200 + selfSignedCertBytesWithValidity := testcrypto.MustCreateCertWithNotBeforeAfter(t, privKey, certTemplate, notBefore, notAfter) + go testutil.IssueOneRequest(ctx, t, opts.Client, testNamespace, selfSignedCertBytesWithValidity, []byte("ca bytes")) + + // Spin up a test pod + tmpDir, err := os.MkdirTemp("", "*") + if err != nil { + t.Fatal(err) + } + defer os.RemoveAll(tmpDir) + _, err = cl.NodePublishVolume(ctx, &csi.NodePublishVolumeRequest{ + VolumeId: "test-vol", + VolumeContext: map[string]string{ + "csi.storage.k8s.io/ephemeral": "true", + "csi.storage.k8s.io/pod.name": "the-pod-name", + "csi.storage.k8s.io/pod.namespace": testNamespace, + }, + TargetPath: tmpDir, + Readonly: true, + }) + if err != nil { + t.Fatal(err) + } + + // Get the CSR name + req, err := testutil.WaitAndGetOneCertificateRequestInNamespace(ctx, opts.Client, testNamespace) + if err != nil { + t.Fatal(err) + } + + // Should expose that CertificateRequest as ready with expiry and renewal time + // node="f56fd9f8b" is the hash value of "test-node" defined in driver_testing.go + expectedOutputTemplate := `# HELP certmanager_csi_certificate_request_expiration_timestamp_seconds The date after which the certificate request expires. Expressed as a Unix Epoch Time. +# TYPE certmanager_csi_certificate_request_expiration_timestamp_seconds gauge +certmanager_csi_certificate_request_expiration_timestamp_seconds{issuer_group="test-issuer-group",issuer_kind="test-issuer-kind",issuer_name="test-issuer",name="test-cr-name",namespace="test-ns"} 300 +# HELP certmanager_csi_certificate_request_ready_status The ready status of the certificate request. +# TYPE certmanager_csi_certificate_request_ready_status gauge +certmanager_csi_certificate_request_ready_status{condition="False",issuer_group="test-issuer-group",issuer_kind="test-issuer-kind",issuer_name="test-issuer",name="test-cr-name",namespace="test-ns"} 0 +certmanager_csi_certificate_request_ready_status{condition="True",issuer_group="test-issuer-group",issuer_kind="test-issuer-kind",issuer_name="test-issuer",name="test-cr-name",namespace="test-ns"} 1 +certmanager_csi_certificate_request_ready_status{condition="Unknown",issuer_group="test-issuer-group",issuer_kind="test-issuer-kind",issuer_name="test-issuer",name="test-cr-name",namespace="test-ns"} 0 +# HELP certmanager_csi_certificate_request_renewal_timestamp_seconds The number of seconds before expiration time the certificate request should renew. +# TYPE certmanager_csi_certificate_request_renewal_timestamp_seconds gauge +certmanager_csi_certificate_request_renewal_timestamp_seconds{issuer_group="test-issuer-group",issuer_kind="test-issuer-kind",issuer_name="test-issuer",name="test-cr-name",namespace="test-ns"} 200 +# HELP certmanager_csi_driver_issue_call_count The number of issue() calls made by the driver. +# TYPE certmanager_csi_driver_issue_call_count counter +certmanager_csi_driver_issue_call_count{node="f56fd9f8b",volume="test-vol"} 1 +# HELP certmanager_csi_managed_certificate_count The number of certificates managed by the csi driver. +# TYPE certmanager_csi_managed_certificate_count counter +certmanager_csi_managed_certificate_count{node="f56fd9f8b"} 1 +# HELP certmanager_csi_managed_volume_count The number of volume managed by the csi driver. +# TYPE certmanager_csi_managed_volume_count counter +certmanager_csi_managed_volume_count{node="f56fd9f8b"} 1 +` + waitForMetrics(t, ctx, metricsEndpoint, strings.ReplaceAll(expectedOutputTemplate, "test-cr-name", req.Name)) + + // Delete the test pod + _, err = cl.NodeUnpublishVolume(ctx, &csi.NodeUnpublishVolumeRequest{ + VolumeId: "test-vol", + TargetPath: tmpDir, + }) + if err != nil { + t.Fatal(err) + } + + // Should expose no CertificateRequest and only metrics counters + waitForMetrics(t, ctx, metricsEndpoint, `# HELP certmanager_csi_driver_issue_call_count The number of issue() calls made by the driver. +# TYPE certmanager_csi_driver_issue_call_count counter +certmanager_csi_driver_issue_call_count{node="f56fd9f8b",volume="test-vol"} 1 +# HELP certmanager_csi_managed_certificate_count The number of certificates managed by the csi driver. +# TYPE certmanager_csi_managed_certificate_count counter +certmanager_csi_managed_certificate_count{node="f56fd9f8b"} 1 +# HELP certmanager_csi_managed_volume_count The number of volume managed by the csi driver. +# TYPE certmanager_csi_managed_volume_count counter +certmanager_csi_managed_volume_count{node="f56fd9f8b"} 1 +`) + +} diff --git a/test/util/testutil.go b/test/util/testutil.go index d405e4a..6d9dd95 100644 --- a/test/util/testutil.go +++ b/test/util/testutil.go @@ -29,7 +29,7 @@ import ( "k8s.io/apimachinery/pkg/util/wait" ) -func waitAndGetOneCertificateRequestInNamespace(ctx context.Context, client cmclient.Interface, ns string) (*cmapi.CertificateRequest, error) { +func WaitAndGetOneCertificateRequestInNamespace(ctx context.Context, client cmclient.Interface, ns string) (*cmapi.CertificateRequest, error) { var req *cmapi.CertificateRequest if err := wait.PollUntilContextCancel(ctx, time.Millisecond*50, true, func(ctx context.Context) (done bool, err error) { reqs, err := client.CertmanagerV1().CertificateRequests(ns).List(ctx, metav1.ListOptions{}) @@ -53,7 +53,7 @@ func waitAndGetOneCertificateRequestInNamespace(ctx context.Context, client cmcl func IssueOneRequest(ctx context.Context, t *testing.T, client cmclient.Interface, namespace string, cert, ca []byte) { if err := func() error { - req, err := waitAndGetOneCertificateRequestInNamespace(ctx, client, namespace) + req, err := WaitAndGetOneCertificateRequestInNamespace(ctx, client, namespace) if err != nil { return err } @@ -80,7 +80,7 @@ func IssueOneRequest(ctx context.Context, t *testing.T, client cmclient.Interfac func SetCertificateRequestConditions(ctx context.Context, t *testing.T, client cmclient.Interface, namespace string, conditions ...cmapi.CertificateRequestCondition) { if err := func() error { - req, err := waitAndGetOneCertificateRequestInNamespace(ctx, client, namespace) + req, err := WaitAndGetOneCertificateRequestInNamespace(ctx, client, namespace) if err != nil { return err }