Instrument QBFT components

protocol/v2/qbft/controller/controller.go

@@ -8,6 +8,9 @@ import (
 	"fmt"
 
 	"github.com/pkg/errors"
+	"go.opentelemetry.io/otel/attribute"
+	"go.opentelemetry.io/otel/codes"
+	"go.opentelemetry.io/otel/trace"
 	"go.uber.org/zap"
 
 	specqbft "github.com/ssvlabs/ssv-spec/qbft"
@@ -55,24 +58,39 @@ func NewController(
 
 // StartNewInstance will start a new QBFT instance, if can't will return error
 func (c *Controller) StartNewInstance(ctx context.Context, logger *zap.Logger, height specqbft.Height, value []byte) error {
+	ctx, span := tracer.Start(ctx,
+		fmt.Sprintf("%s.controller.start_new_instance", observabilityNamespace),
+		trace.WithAttributes(
+			attribute.Int64("ssv.validator.duty.slot", int64(height))))
+	defer span.End()
 
 	if err := c.GetConfig().GetValueCheckF()(value); err != nil {
-		return errors.Wrap(err, "value invalid")
+		err = errors.Wrap(err, "value invalid")
+		span.SetStatus(codes.Error, err.Error())
+		return err
 	}
 
 	if height < c.Height {
-		return errors.New("attempting to start an instance with a past height")
+		err := errors.New("attempting to start an instance with a past height")
+		span.SetStatus(codes.Error, err.Error())
+		return err
 	}
 
 	if c.StoredInstances.FindInstance(height) != nil {
-		return errors.New("instance already running")
+		err := errors.New("instance already running")
+		span.SetStatus(codes.Error, err.Error())
+		return err
 	}
 
 	c.Height = height
 
 	newInstance := c.addAndStoreNewInstance()
+
+	span.AddEvent("start new instance")
 	newInstance.Start(ctx, logger, value, height)
 	c.forceStopAllInstanceExceptCurrent()
+
+	span.SetStatus(codes.Ok, "")
 	return nil
 }
 

protocol/v2/qbft/controller/observability.go

@@ -0,0 +1,14 @@
+package controller
+
+import (
+	"go.opentelemetry.io/otel"
+)
+
+const (
+	observabilityName      = "github.com/ssvlabs/ssv/protocol/v2/qbft/controller"
+	observabilityNamespace = "ssv.validator"
+)
+
+var (
+	tracer = otel.Tracer(observabilityName)
+)

protocol/v2/qbft/instance/instance.go

@@ -3,12 +3,17 @@ package instance
 import (
 	"context"
 	"encoding/base64"
+	"encoding/hex"
 	"encoding/json"
+	"fmt"
 	"sync"
 
 	"github.com/pkg/errors"
 	specqbft "github.com/ssvlabs/ssv-spec/qbft"
 	spectypes "github.com/ssvlabs/ssv-spec/types"
+	"go.opentelemetry.io/otel/attribute"
+	"go.opentelemetry.io/otel/codes"
+	"go.opentelemetry.io/otel/trace"
 	"go.uber.org/zap"
 
 	"github.com/ssvlabs/ssv/logging/fields"
@@ -72,6 +77,12 @@ func (i *Instance) ForceStop() {
 // Start is an interface implementation
 func (i *Instance) Start(ctx context.Context, logger *zap.Logger, value []byte, height specqbft.Height) {
 	i.startOnce.Do(func() {
+		ctx, span := tracer.Start(ctx,
+			fmt.Sprintf("%s.qbft.instance.start", observabilityNamespace),
+			trace.WithAttributes(
+				attribute.Int64("ssv.validator.duty.slot", int64(height))))
+		defer span.End()
+
 		i.StartValue = value
 		i.bumpToRound(ctx, specqbft.FirstRound)
 		i.State.Height = height
@@ -83,27 +94,31 @@ func (i *Instance) Start(ctx context.Context, logger *zap.Logger, value []byte,
 			fields.Height(i.State.Height))
 
 		proposerID := proposer(i.State, i.GetConfig(), specqbft.FirstRound)
-		logger.Debug("ℹ️ starting QBFT instance", zap.Uint64("leader", proposerID))
+		span.SetAttributes(attribute.Int64("ssv.validator.duty.proposer", int64(proposerID)))
+
+		span.AddEvent("starting QBFT instance")
 
 		// propose if this node is the proposer
 		if proposerID == i.State.CommitteeMember.OperatorID {
 			proposal, err := CreateProposal(i.State, i.signer, i.StartValue, nil, nil)
 			// nolint
 			if err != nil {
-				logger.Warn("❗ failed to create proposal", zap.Error(err))
+				span.RecordError(err)
 				// TODO align spec to add else to avoid broadcast errored proposal
 			} else {
 
 				r, err := specqbft.HashDataRoot(i.StartValue) // @TODO (better than decoding?)
 				if err != nil {
-					logger.Warn("❗ failed to hash input data", zap.Error(err))
+					span.SetStatus(codes.Error, err.Error())
 					return
 				}
 				// nolint
 				logger = logger.With(fields.Root(r))
-				logger.Debug("📢 leader broadcasting proposal message")
+				span.AddEvent(
+					"leader broadcasting proposal message",
+					trace.WithAttributes(attribute.String("root", hex.EncodeToString(r[:]))))
 				if err := i.Broadcast(logger, proposal); err != nil {
-					logger.Warn("❌ failed to broadcast proposal", zap.Error(err))
+					span.SetStatus(codes.Error, err.Error())
 				}
 			}
 		}

protocol/v2/qbft/instance/observability.go

@@ -26,7 +26,8 @@ const (
 )
 
 var (
-	meter = otel.Meter(observabilityName)
+	meter  = otel.Meter(observabilityName)
+	tracer = otel.Tracer(observabilityName)
 
 	validatorStageDurationHistogram = observability.NewMetric(
 		meter.Float64Histogram(