diff --git a/protocol/v2/qbft/controller/controller.go b/protocol/v2/qbft/controller/controller.go index cad2459d5d..8101ab0d5e 100644 --- a/protocol/v2/qbft/controller/controller.go +++ b/protocol/v2/qbft/controller/controller.go @@ -8,6 +8,9 @@ import ( "fmt" "github.com/pkg/errors" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/codes" + "go.opentelemetry.io/otel/trace" "go.uber.org/zap" specqbft "github.com/ssvlabs/ssv-spec/qbft" @@ -55,24 +58,39 @@ func NewController( // StartNewInstance will start a new QBFT instance, if can't will return error func (c *Controller) StartNewInstance(ctx context.Context, logger *zap.Logger, height specqbft.Height, value []byte) error { + ctx, span := tracer.Start(ctx, + fmt.Sprintf("%s.controller.start_new_instance", observabilityNamespace), + trace.WithAttributes( + attribute.Int64("ssv.validator.duty.slot", int64(height)))) + defer span.End() if err := c.GetConfig().GetValueCheckF()(value); err != nil { - return errors.Wrap(err, "value invalid") + err = errors.Wrap(err, "value invalid") + span.SetStatus(codes.Error, err.Error()) + return err } if height < c.Height { - return errors.New("attempting to start an instance with a past height") + err := errors.New("attempting to start an instance with a past height") + span.SetStatus(codes.Error, err.Error()) + return err } if c.StoredInstances.FindInstance(height) != nil { - return errors.New("instance already running") + err := errors.New("instance already running") + span.SetStatus(codes.Error, err.Error()) + return err } c.Height = height newInstance := c.addAndStoreNewInstance() + + span.AddEvent("start new instance") newInstance.Start(ctx, logger, value, height) c.forceStopAllInstanceExceptCurrent() + + span.SetStatus(codes.Ok, "") return nil } diff --git a/protocol/v2/qbft/controller/observability.go b/protocol/v2/qbft/controller/observability.go new file mode 100644 index 0000000000..aa7d8c76f6 --- /dev/null +++ b/protocol/v2/qbft/controller/observability.go @@ -0,0 +1,14 @@ +package controller + +import ( + "go.opentelemetry.io/otel" +) + +const ( + observabilityName = "github.com/ssvlabs/ssv/protocol/v2/qbft/controller" + observabilityNamespace = "ssv.validator" +) + +var ( + tracer = otel.Tracer(observabilityName) +) diff --git a/protocol/v2/qbft/instance/instance.go b/protocol/v2/qbft/instance/instance.go index b6eee6b99b..114e8d3799 100644 --- a/protocol/v2/qbft/instance/instance.go +++ b/protocol/v2/qbft/instance/instance.go @@ -3,12 +3,17 @@ package instance import ( "context" "encoding/base64" + "encoding/hex" "encoding/json" + "fmt" "sync" "github.com/pkg/errors" specqbft "github.com/ssvlabs/ssv-spec/qbft" spectypes "github.com/ssvlabs/ssv-spec/types" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/codes" + "go.opentelemetry.io/otel/trace" "go.uber.org/zap" "github.com/ssvlabs/ssv/logging/fields" @@ -72,6 +77,12 @@ func (i *Instance) ForceStop() { // Start is an interface implementation func (i *Instance) Start(ctx context.Context, logger *zap.Logger, value []byte, height specqbft.Height) { i.startOnce.Do(func() { + ctx, span := tracer.Start(ctx, + fmt.Sprintf("%s.qbft.instance.start", observabilityNamespace), + trace.WithAttributes( + attribute.Int64("ssv.validator.duty.slot", int64(height)))) + defer span.End() + i.StartValue = value i.bumpToRound(ctx, specqbft.FirstRound) i.State.Height = height @@ -83,27 +94,31 @@ func (i *Instance) Start(ctx context.Context, logger *zap.Logger, value []byte, fields.Height(i.State.Height)) proposerID := proposer(i.State, i.GetConfig(), specqbft.FirstRound) - logger.Debug("ℹī¸ starting QBFT instance", zap.Uint64("leader", proposerID)) + span.SetAttributes(attribute.Int64("ssv.validator.duty.proposer", int64(proposerID))) + + span.AddEvent("starting QBFT instance") // propose if this node is the proposer if proposerID == i.State.CommitteeMember.OperatorID { proposal, err := CreateProposal(i.State, i.signer, i.StartValue, nil, nil) // nolint if err != nil { - logger.Warn("❗ failed to create proposal", zap.Error(err)) + span.RecordError(err) // TODO align spec to add else to avoid broadcast errored proposal } else { r, err := specqbft.HashDataRoot(i.StartValue) // @TODO (better than decoding?) if err != nil { - logger.Warn("❗ failed to hash input data", zap.Error(err)) + span.SetStatus(codes.Error, err.Error()) return } // nolint logger = logger.With(fields.Root(r)) - logger.Debug("đŸ“ĸ leader broadcasting proposal message") + span.AddEvent( + "leader broadcasting proposal message", + trace.WithAttributes(attribute.String("root", hex.EncodeToString(r[:])))) if err := i.Broadcast(logger, proposal); err != nil { - logger.Warn("❌ failed to broadcast proposal", zap.Error(err)) + span.SetStatus(codes.Error, err.Error()) } } } diff --git a/protocol/v2/qbft/instance/observability.go b/protocol/v2/qbft/instance/observability.go index 3be5fcd864..cddb56256d 100644 --- a/protocol/v2/qbft/instance/observability.go +++ b/protocol/v2/qbft/instance/observability.go @@ -26,7 +26,8 @@ const ( ) var ( - meter = otel.Meter(observabilityName) + meter = otel.Meter(observabilityName) + tracer = otel.Tracer(observabilityName) validatorStageDurationHistogram = observability.NewMetric( meter.Float64Histogram(