diff --git a/internal/metrics/metrics.go b/internal/metrics/metrics.go index 1118a653..b4d05dc1 100644 --- a/internal/metrics/metrics.go +++ b/internal/metrics/metrics.go @@ -22,10 +22,7 @@ var ( ConditionQueued *prometheus.CounterVec ConditionCompleted *prometheus.CounterVec PublishErrors *prometheus.CounterVec - ConditionInKV *prometheus.CounterVec ConditionReconcileStale *prometheus.CounterVec - - NatsKVUpdateEvent *prometheus.CounterVec ) func init() { @@ -79,14 +76,6 @@ func init() { []string{"conditionKind"}, ) - ConditionInKV = promauto.NewCounterVec( - prometheus.CounterOpts{ - Name: "conditionorc_condition_in_kv", - Help: "A count of all conditions listed in the KV.", - }, - []string{"conditionKind", "state"}, - ) - ConditionReconcileStale = promauto.NewCounterVec( prometheus.CounterOpts{ Name: "conditionorc_condition_active_stale", @@ -94,14 +83,6 @@ func init() { }, []string{"conditionKind"}, ) - - NatsKVUpdateEvent = promauto.NewCounterVec( - prometheus.CounterOpts{ - Name: "conditionorc_nats_kv_update_event", - Help: "A count of NATS KV Condition update events.", - }, - []string{"conditionKind", "valid"}, - ) } // ListenAndServeMetrics exposes prometheus metrics as /metrics on port 9090 diff --git a/internal/orchestrator/updates.go b/internal/orchestrator/updates.go index 80d01fe8..c7ff3c6e 100644 --- a/internal/orchestrator/updates.go +++ b/internal/orchestrator/updates.go @@ -477,14 +477,21 @@ func (o *Orchestrator) queueFollowingCondition(ctx context.Context, cond *rctype metrics.DependencyError("nats", "retrieve active condition") - return errors.Wrap(errCompleteEvent, err.Error()) + return fmt.Errorf("%w:retrieving active condition:%w", errCompleteEvent, err) } - - // Publish the next event if that event is in the pending state - // // Conditions for controllers that run inband are not published to the JS, // they are retrieved by the inband controllers themselves through the Orchestrator API. - if active != nil && active.State == rctypes.Pending && active.StreamPublishRequired() { + if !active.StreamPublishRequired() { + o.logger.WithFields(logrus.Fields{ + "condition.id": active.ID, + "server.id": active.Target.String(), + "condition.kind": active.Kind, + }).Debug("not publishing inband condition") + return nil + } + + // If we're here we have to publish the next event, if that event is in the pending state + if active != nil && active.State == rctypes.Pending { byt := active.MustBytes() subject := fmt.Sprintf("%s.servers.%s", o.facility, active.Kind) err := o.streamBroker.Publish(ctx, subject, byt) @@ -496,8 +503,9 @@ func (o *Orchestrator) queueFollowingCondition(ctx context.Context, cond *rctype }).Warn("publishing next active condition") metrics.DependencyError("nats", "publish-condition") + metrics.PublishErrors.WithLabelValues(string(active.Kind)).Inc() - return errors.Wrap(errCompleteEvent, err.Error()) + return fmt.Errorf("%w:publishing next event:%w", errCompleteEvent, err) } metrics.ConditionQueued.With(