fix failure on validator updates

symapp/app_di.go

@@ -4,10 +4,12 @@ package symapp
 
 import (
 	"cosmossdk.io/x/symStaking/abci"
+	stakingtypes "cosmossdk.io/x/symStaking/types"
 	_ "embed"
 	"fmt"
 	"io"
 	"path/filepath"
+	"strings"
 
 	dbm "github.com/cosmos/cosmos-db"
 	"github.com/spf13/cast"
@@ -226,6 +228,7 @@ func NewSymApp(
 		ba.SetVerifyVoteExtensionHandler(voteExtensionHandler.VerifyVoteExtension())
 		ba.SetPrepareProposal(abciPropHandler.PrepareProposal())
 		ba.SetPreBlocker(abciPropHandler.PreBlocker())
+		ba.AddRunTxRecoveryHandler(RecoverHandler)
 	})
 
 	app.App = appBuilder.Build(db, traceStore, baseAppOptions...)
@@ -434,3 +437,19 @@ func BlockedAddresses() map[string]bool {
 
 	return result
 }
+
+// we use this handler to restart on failure on our errors
+func RecoverHandler(recoveryObj interface{}) error {
+	err, ok := recoveryObj.(error)
+	if !ok || err == nil {
+		return nil
+	}
+	if stakingtypes.ErrSymbioticValUpdate.Is(err) {
+		panic(fmt.Errorf("symbiotic panic: %w", err))
+	}
+	// IDK after this error node stops receiving blocks
+	if strings.Contains(err.Error(), "peer is not sending us data fast enough") {
+		panic(fmt.Errorf("Consensus peer error: %w", err))
+	}
+	return err
+}

x/symStaking/abci/proposal.go

@@ -9,9 +9,14 @@ import (
 	"fmt"
 	abci "github.com/cometbft/cometbft/abci/types"
 	sdk "github.com/cosmos/cosmos-sdk/types"
+	"time"
 )
 
-const SYMBIOTIC_SYNC_PERIOD = 10
+const (
+	SYMBIOTIC_SYNC_PERIOD = 10
+	SLEEP_ON_RETRY        = 200
+	RETRIES               = 5
+)
 
 type ProposalHandler struct {
 	logger        log.Logger
@@ -28,63 +33,99 @@ func NewProposalHandler(logger log.Logger, keeper *keeper2.Keeper) *ProposalHand
 
 func (h *ProposalHandler) PrepareProposal() sdk.PrepareProposalHandler {
 	return func(ctx sdk.Context, req *abci.PrepareProposalRequest) (*abci.PrepareProposalResponse, error) {
-		proposalTxs := req.Txs
-
-		if req.Height%SYMBIOTIC_SYNC_PERIOD != 0 {
-			return &abci.PrepareProposalResponse{
-				Txs: proposalTxs,
-			}, nil
+		var err error
+		var resp *abci.PrepareProposalResponse
+
+		for i := 0; i < RETRIES; i++ {
+			resp, err = h.internalPrepareProposal(ctx, req)
+			if err == nil {
+				break
+			}
+			time.Sleep(time.Millisecond * SLEEP_ON_RETRY)
 		}
 
-		blockHash, err := h.keeper.GetFinalizedBlockHash(ctx)
 		if err != nil {
 			panic(errors.Join(types.ErrSymbioticValUpdate, err))
 		}
 
-		// NOTE: We use stdlib JSON encoding, but an application may choose to use
-		// a performant mechanism. This is for demo purposes only.
-		bz, err := json.Marshal(blockHash)
+		return resp, nil
+	}
+}
+
+func (h *ProposalHandler) PreBlocker() sdk.PreBlocker {
+	return func(context sdk.Context, req *abci.FinalizeBlockRequest) error {
+		var err error
+
+		for i := 0; i < RETRIES; i++ {
+			err = h.internalPreBlocker(context, req)
+			if err == nil {
+				break
+			}
+			time.Sleep(time.Millisecond * SLEEP_ON_RETRY)
+		}
+
 		if err != nil {
-			panic(errors.Join(types.ErrSymbioticValUpdate, errors.New("failed to encode injected vote extension tx")))
+			panic(errors.Join(types.ErrSymbioticValUpdate, err))
 		}
 
-		// Inject a "fake" tx into the proposal s.t. validators can decode, verify,
-		// and store the canonical stake-weighted average prices.
-		proposalTxs = append([][]byte{bz}, proposalTxs...)
+		return nil
+	}
+}
 
+func (h *ProposalHandler) internalPrepareProposal(ctx sdk.Context, req *abci.PrepareProposalRequest) (*abci.PrepareProposalResponse, error) {
+	proposalTxs := req.Txs
+
+	if req.Height%SYMBIOTIC_SYNC_PERIOD != 0 {
 		return &abci.PrepareProposalResponse{
 			Txs: proposalTxs,
 		}, nil
 	}
-}
 
-func (h *ProposalHandler) PreBlocker() sdk.PreBlocker {
-	return func(context sdk.Context, req *abci.FinalizeBlockRequest) error {
+	blockHash, err := h.keeper.GetFinalizedBlockHash(ctx)
+	if err != nil {
+		return nil, err
+	}
 
-		if req.Height%SYMBIOTIC_SYNC_PERIOD != 0 || len(req.Txs) == 0 {
-			return nil
-		}
+	// NOTE: We use stdlib JSON encoding, but an application may choose to use
+	// a performant mechanism. This is for demo purposes only.
+	bz, err := json.Marshal(blockHash)
+	if err != nil {
+		return nil, errors.New("failed to encode injected vote extension tx")
+	}
 
-		var blockHash string
-		if err := json.Unmarshal(req.Txs[0], &blockHash); err != nil {
-			panic(errors.Join(types.ErrSymbioticValUpdate, err))
-		}
+	// Inject a "fake" tx into the proposal s.t. validators can decode, verify,
+	// and store the canonical stake-weighted average prices.
+	proposalTxs = append([][]byte{bz}, proposalTxs...)
 
-		block, err := h.keeper.GetBlockByHash(context, blockHash)
-		if err != nil {
-			panic(errors.Join(types.ErrSymbioticValUpdate, err))
-		}
+	return &abci.PrepareProposalResponse{
+		Txs: proposalTxs,
+	}, nil
+}
 
-		if block.Time() < h.prevBlockTime || int64(block.Time()) >= context.HeaderInfo().Time.Unix() || block.Time() < h.keeper.GetMinBlockTimestamp(context) {
-			panic(errors.Join(types.ErrSymbioticValUpdate, fmt.Errorf("symbiotic invalid proposed block")))
-		}
+func (h *ProposalHandler) internalPreBlocker(context sdk.Context, req *abci.FinalizeBlockRequest) error {
+	if req.Height%SYMBIOTIC_SYNC_PERIOD != 0 || len(req.Txs) == 0 {
+		return nil
+	}
 
-		if err := h.keeper.SymbioticUpdateValidatorsPower(context, blockHash); err != nil {
-			panic(errors.Join(types.ErrSymbioticValUpdate, err))
-		}
+	var blockHash string
+	if err := json.Unmarshal(req.Txs[0], &blockHash); err != nil {
+		return err
+	}
 
-		h.prevBlockTime = block.Time()
+	block, err := h.keeper.GetBlockByHash(context, blockHash)
+	if err != nil {
+		return err
+	}
 
-		return nil
+	if block.Time() < h.prevBlockTime || int64(block.Time()) >= context.HeaderInfo().Time.Unix() || block.Time() < h.keeper.GetMinBlockTimestamp(context) {
+		return fmt.Errorf("symbiotic invalid proposed block")
 	}
+
+	if err := h.keeper.SymbioticUpdateValidatorsPower(context, blockHash); err != nil {
+		return err
+	}
+
+	h.prevBlockTime = block.Time()
+
+	return nil
 }