add voluntary exit signature verification optimisation (#12485)

Co-authored-by: shota.silagadze <[email protected]>

cl/beacon/handler/pool.go

@@ -168,27 +168,26 @@ func (a *ApiHandler) PostEthV1BeaconPoolVoluntaryExits(w http.ResponseWriter, r
 		http.Error(w, err.Error(), http.StatusBadRequest)
 		return
 	}
-	if err := a.voluntaryExitService.ProcessMessage(r.Context(), nil, &req); err != nil && !errors.Is(err, services.ErrIgnore) {
-		http.Error(w, err.Error(), http.StatusBadRequest)
+
+	encodedSSZ, err := req.EncodeSSZ(nil)
+	if err != nil {
+		http.Error(w, err.Error(), http.StatusInternalServerError)
 		return
 	}
 
-	// Broadcast to gossip
-	if a.sentinel != nil {
-		encodedSSZ, err := req.EncodeSSZ(nil)
-		if err != nil {
-			http.Error(w, err.Error(), http.StatusInternalServerError)
-			return
-		}
-		if _, err := a.sentinel.PublishGossip(r.Context(), &sentinel.GossipData{
+	if err := a.voluntaryExitService.ProcessMessage(r.Context(), nil, &cltypes.SignedVoluntaryExitWithGossipData{
+		GossipData: &sentinel.GossipData{
 			Data: encodedSSZ,
 			Name: gossip.TopicNameVoluntaryExit,
-		}); err != nil {
-			http.Error(w, err.Error(), http.StatusInternalServerError)
-			return
-		}
-		a.operationsPool.VoluntaryExitsPool.Insert(req.VoluntaryExit.ValidatorIndex, &req)
+		},
+		SignedVoluntaryExit:   &req,
+		ImmediateVerification: true,
+	}); err != nil && !errors.Is(err, services.ErrIgnore) {
+		http.Error(w, err.Error(), http.StatusBadRequest)
+		return
 	}
+	a.operationsPool.VoluntaryExitsPool.Insert(req.VoluntaryExit.ValidatorIndex, &req)
+
 	// Only write 200
 	w.WriteHeader(http.StatusOK)
 }

cl/beacon/handler/utils_test.go

@@ -130,8 +130,8 @@ func setupTestingHandler(t *testing.T, v clparams.StateVersion, logger log.Logge
 		opPool.AttestationsPool.Insert(msg.SignedAggregateAndProof.Message.Aggregate.Signature, msg.SignedAggregateAndProof.Message.Aggregate)
 		return nil
 	}).AnyTimes()
-	voluntaryExitService.EXPECT().ProcessMessage(gomock.Any(), gomock.Any(), gomock.Any()).DoAndReturn(func(ctx context.Context, subnetID *uint64, msg *cltypes.SignedVoluntaryExit) error {
-		opPool.VoluntaryExitsPool.Insert(msg.VoluntaryExit.ValidatorIndex, msg)
+	voluntaryExitService.EXPECT().ProcessMessage(gomock.Any(), gomock.Any(), gomock.Any()).DoAndReturn(func(ctx context.Context, subnetID *uint64, msg *cltypes.SignedVoluntaryExitWithGossipData) error {
+		opPool.VoluntaryExitsPool.Insert(msg.SignedVoluntaryExit.VoluntaryExit.ValidatorIndex, msg.SignedVoluntaryExit)
 		return nil
 	}).AnyTimes()
 	blsToExecutionChangeService.EXPECT().ProcessMessage(gomock.Any(), gomock.Any(), gomock.Any()).DoAndReturn(func(ctx context.Context, subnetID *uint64, msg *cltypes.SignedBLSToExecutionChangeWithGossipData) error {

cl/cltypes/validator.go

@@ -20,6 +20,7 @@ import (
 	"encoding/json"
 
 	libcommon "github.com/erigontech/erigon-lib/common"
+	sentinel "github.com/erigontech/erigon-lib/gointerfaces/sentinelproto"
 	"github.com/erigontech/erigon-lib/types/clonable"
 	"github.com/erigontech/erigon-lib/types/ssz"
 
@@ -128,6 +129,13 @@ func (*VoluntaryExit) EncodingSizeSSZ() int {
 	return 16
 }
 
+// SignedVoluntaryExitWithGossipData type represents SignedVoluntaryExit with the gossip data where it's coming from.
+type SignedVoluntaryExitWithGossipData struct {
+	SignedVoluntaryExit   *SignedVoluntaryExit
+	GossipData            *sentinel.GossipData
+	ImmediateVerification bool
+}
+
 type SignedVoluntaryExit struct {
 	VoluntaryExit *VoluntaryExit    `json:"message"`
 	Signature     libcommon.Bytes96 `json:"signature"`

cl/phase1/network/gossip_manager.go

@@ -203,8 +203,11 @@ func (g *GossipManager) routeAndProcess(ctx context.Context, data *sentinel.Goss
 		}
 		return g.syncContributionService.ProcessMessage(ctx, data.SubnetId, obj)
 	case gossip.TopicNameVoluntaryExit:
-		obj := &cltypes.SignedVoluntaryExit{}
-		if err := obj.DecodeSSZ(data.Data, int(version)); err != nil {
+		obj := &cltypes.SignedVoluntaryExitWithGossipData{
+			GossipData:          copyOfSentinelData(data),
+			SignedVoluntaryExit: &cltypes.SignedVoluntaryExit{},
+		}
+		if err := obj.SignedVoluntaryExit.DecodeSSZ(data.Data, int(version)); err != nil {
 			return err
 		}
 		return g.voluntaryExitService.ProcessMessage(ctx, data.SubnetId, obj)

cl/phase1/network/services/batch_signature_verification.go

@@ -26,6 +26,7 @@ type BatchSignatureVerifier struct {
 	attVerifyAndExecute        chan *AggregateVerificationData
 	aggregateProofVerify       chan *AggregateVerificationData
 	blsToExecutionChangeVerify chan *AggregateVerificationData
+	voluntaryExitVerify        chan *AggregateVerificationData
 	ctx                        context.Context
 }
 
@@ -49,6 +50,7 @@ func NewBatchSignatureVerifier(ctx context.Context, sentinel sentinel.SentinelCl
 		attVerifyAndExecute:        make(chan *AggregateVerificationData, 1024),
 		aggregateProofVerify:       make(chan *AggregateVerificationData, 1024),
 		blsToExecutionChangeVerify: make(chan *AggregateVerificationData, 1024),
+		voluntaryExitVerify:        make(chan *AggregateVerificationData, 1024),
 	}
 }
 
@@ -65,6 +67,10 @@ func (b *BatchSignatureVerifier) AsyncVerifyBlsToExecutionChange(data *Aggregate
 	b.blsToExecutionChangeVerify <- data
 }
 
+func (b *BatchSignatureVerifier) AsyncVerifyVoluntaryExit(data *AggregateVerificationData) {
+	b.voluntaryExitVerify <- data
+}
+
 func (b *BatchSignatureVerifier) ImmediateVerification(data *AggregateVerificationData) error {
 	return b.processSignatureVerification([]*AggregateVerificationData{data})
 }
@@ -74,6 +80,7 @@ func (b *BatchSignatureVerifier) Start() {
 	go b.start(b.attVerifyAndExecute)
 	go b.start(b.aggregateProofVerify)
 	go b.start(b.blsToExecutionChangeVerify)
+	go b.start(b.voluntaryExitVerify)
 }
 
 // When receiving AggregateVerificationData, we simply collect all the signature verification data

cl/phase1/network/services/interface.go

@@ -47,7 +47,7 @@ type AggregateAndProofService Service[*cltypes.SignedAggregateAndProofData]
 type AttestationService Service[*AttestationWithGossipData]
 
 //go:generate mockgen -typed=true -destination=./mock_services/voluntary_exit_service_mock.go -package=mock_services . VoluntaryExitService
-type VoluntaryExitService Service[*cltypes.SignedVoluntaryExit]
+type VoluntaryExitService Service[*cltypes.SignedVoluntaryExitWithGossipData]
 
 //go:generate mockgen -typed=true -destination=./mock_services/bls_to_execution_change_service_mock.go -package=mock_services . BLSToExecutionChangeService
 type BLSToExecutionChangeService Service[*cltypes.SignedBLSToExecutionChangeWithGossipData]

cl/phase1/network/services/voluntary_exit_service.go

@@ -32,11 +32,12 @@ import (
 )
 
 type voluntaryExitService struct {
-	operationsPool    pool.OperationsPool
-	emitters          *beaconevents.EventEmitter
-	syncedDataManager synced_data.SyncedData
-	beaconCfg         *clparams.BeaconChainConfig
-	ethClock          eth_clock.EthereumClock
+	operationsPool         pool.OperationsPool
+	emitters               *beaconevents.EventEmitter
+	syncedDataManager      synced_data.SyncedData
+	beaconCfg              *clparams.BeaconChainConfig
+	ethClock               eth_clock.EthereumClock
+	batchSignatureVerifier *BatchSignatureVerifier
 }
 
 func NewVoluntaryExitService(
@@ -45,19 +46,21 @@ func NewVoluntaryExitService(
 	syncedDataManager synced_data.SyncedData,
 	beaconCfg *clparams.BeaconChainConfig,
 	ethClock eth_clock.EthereumClock,
+	batchSignatureVerifier *BatchSignatureVerifier,
 ) VoluntaryExitService {
 	return &voluntaryExitService{
-		operationsPool:    operationsPool,
-		emitters:          emitters,
-		syncedDataManager: syncedDataManager,
-		beaconCfg:         beaconCfg,
-		ethClock:          ethClock,
+		operationsPool:         operationsPool,
+		emitters:               emitters,
+		syncedDataManager:      syncedDataManager,
+		beaconCfg:              beaconCfg,
+		ethClock:               ethClock,
+		batchSignatureVerifier: batchSignatureVerifier,
 	}
 }
 
-func (s *voluntaryExitService) ProcessMessage(ctx context.Context, subnet *uint64, msg *cltypes.SignedVoluntaryExit) error {
+func (s *voluntaryExitService) ProcessMessage(ctx context.Context, subnet *uint64, msg *cltypes.SignedVoluntaryExitWithGossipData) error {
 	// ref: https://github.com/ethereum/consensus-specs/blob/dev/specs/phase0/p2p-interface.md#voluntary_exit
-	voluntaryExit := msg.VoluntaryExit
+	voluntaryExit := msg.SignedVoluntaryExit.VoluntaryExit
 
 	// [IGNORE] The voluntary exit is the first valid voluntary exit received for the validator with index signed_voluntary_exit.message.validator_index.
 	if s.operationsPool.VoluntaryExitsPool.Has(voluntaryExit.ValidatorIndex) {
@@ -121,14 +124,30 @@ func (s *voluntaryExitService) ProcessMessage(ctx context.Context, subnet *uint6
 	if err != nil {
 		return err
 	}
+
+	aggregateVerificationData := &AggregateVerificationData{
+		Signatures: [][]byte{msg.SignedVoluntaryExit.Signature[:]},
+		SignRoots:  [][]byte{signingRoot[:]},
+		Pks:        [][]byte{pk[:]},
+		GossipData: msg.GossipData,
+		F: func() {
+			s.operationsPool.VoluntaryExitsPool.Insert(voluntaryExit.ValidatorIndex, msg.SignedVoluntaryExit)
+			s.emitters.Operation().SendVoluntaryExit(msg.SignedVoluntaryExit)
+		},
+	}
+
 	cn()
-	if valid, err := blsVerify(msg.Signature[:], signingRoot[:], pk[:]); err != nil {
-		return err
-	} else if !valid {
-		return errors.New("ProcessVoluntaryExit: BLS verification failed")
+
+	if msg.ImmediateVerification {
+		return s.batchSignatureVerifier.ImmediateVerification(aggregateVerificationData)
 	}
 
-	s.operationsPool.VoluntaryExitsPool.Insert(voluntaryExit.ValidatorIndex, msg)
-	s.emitters.Operation().SendVoluntaryExit(msg)
-	return nil
+	// push the signatures to verify asynchronously and run final functions after that.
+	s.batchSignatureVerifier.AsyncVerifyVoluntaryExit(aggregateVerificationData)
+
+	// As the logic goes, if we return ErrIgnore there will be no peer banning and further publishing
+	// gossip data into the network by the gossip manager. That's what we want because we will be doing that ourselves
+	// in BatchSignatureVerifier service. After validating signatures, if they are valid we will publish the
+	// gossip ourselves or ban the peer which sent that particular invalid signature.
+	return ErrIgnore
 }

cl/phase1/network/services/voluntary_exit_service_test.go

@@ -20,6 +20,7 @@ import (
 	"context"
 	"log"
 	"testing"
+	"time"
 
 	"github.com/erigontech/erigon-lib/types/ssz"
 	mockState "github.com/erigontech/erigon/cl/abstract/mock_services"
@@ -61,12 +62,16 @@ func (t *voluntaryExitTestSuite) SetupTest() {
 	t.syncedData = mockSync.NewMockSyncedData(t.gomockCtrl)
 	t.ethClock = eth_clock.NewMockEthereumClock(t.gomockCtrl)
 	t.beaconCfg = &clparams.BeaconChainConfig{}
-	t.voluntaryExitService = NewVoluntaryExitService(*t.operationsPool, t.emitters, t.syncedData, t.beaconCfg, t.ethClock)
+	batchSignatureVerifier := NewBatchSignatureVerifier(context.TODO(), nil)
+	batchCheckInterval = 1 * time.Millisecond
+	go batchSignatureVerifier.Start()
+	t.voluntaryExitService = NewVoluntaryExitService(*t.operationsPool, t.emitters, t.syncedData, t.beaconCfg, t.ethClock, batchSignatureVerifier)
 	// mock global functions
 	t.mockFuncs = &mockFuncs{
 		ctrl: t.gomockCtrl,
 	}
 	blsVerify = t.mockFuncs.BlsVerify
+	blsVerifyMultipleSignatures = t.mockFuncs.BlsVerifyMultipleSignatures
 }
 
 func (t *voluntaryExitTestSuite) TearDownTest() {
@@ -75,25 +80,29 @@ func (t *voluntaryExitTestSuite) TearDownTest() {
 func (t *voluntaryExitTestSuite) TestProcessMessage() {
 	curEpoch := uint64(100)
 	mockValidatorIndex := uint64(10)
-	mockMsg := &cltypes.SignedVoluntaryExit{
-		VoluntaryExit: &cltypes.VoluntaryExit{
-			Epoch:          1,
-			ValidatorIndex: mockValidatorIndex,
+	mockMsg := &cltypes.SignedVoluntaryExitWithGossipData{
+		SignedVoluntaryExit: &cltypes.SignedVoluntaryExit{
+			VoluntaryExit: &cltypes.VoluntaryExit{
+				Epoch:          1,
+				ValidatorIndex: mockValidatorIndex,
+			},
+			Signature: [96]byte{},
 		},
-		Signature: [96]byte{},
+		GossipData:            nil,
+		ImmediateVerification: true,
 	}
 
 	tests := []struct {
 		name    string
 		mock    func()
-		msg     *cltypes.SignedVoluntaryExit
+		msg     *cltypes.SignedVoluntaryExitWithGossipData
 		wantErr bool
 		err     error
 	}{
 		{
 			name: "validator already in pool",
 			mock: func() {
-				t.operationsPool.VoluntaryExitsPool.Insert(mockValidatorIndex, mockMsg)
+				t.operationsPool.VoluntaryExitsPool.Insert(mockValidatorIndex, mockMsg.SignedVoluntaryExit)
 			},
 			msg:     mockMsg,
 			wantErr: true,
@@ -180,11 +189,11 @@ func (t *voluntaryExitTestSuite) TestProcessMessage() {
 				t.ethClock.EXPECT().GetCurrentEpoch().Return(curEpoch).Times(1)
 				t.beaconCfg.FarFutureEpoch = mockValidator.ExitEpoch()
 				mockState.EXPECT().Version().Return(clparams.AltairVersion).Times(1)
-				mockState.EXPECT().GetDomain(t.beaconCfg.DomainVoluntaryExit, mockMsg.VoluntaryExit.Epoch).Return([]byte{}, nil).Times(1)
+				mockState.EXPECT().GetDomain(t.beaconCfg.DomainVoluntaryExit, mockMsg.SignedVoluntaryExit.VoluntaryExit.Epoch).Return([]byte{}, nil).Times(1)
 				computeSigningRoot = func(_ ssz.HashableSSZ, domain []byte) ([32]byte, error) {
 					return [32]byte{}, nil
 				}
-				t.gomockCtrl.RecordCall(t.mockFuncs, "BlsVerify", gomock.Any(), gomock.Any(), gomock.Any()).Return(false, nil).Times(1)
+				t.gomockCtrl.RecordCall(t.mockFuncs, "BlsVerifyMultipleSignatures", gomock.Any(), gomock.Any(), gomock.Any()).Return(false, nil).Times(2)
 			},
 			msg:     mockMsg,
 			wantErr: true,
@@ -208,13 +217,15 @@ func (t *voluntaryExitTestSuite) TestProcessMessage() {
 				t.ethClock.EXPECT().GetCurrentEpoch().Return(curEpoch).Times(1)
 				t.beaconCfg.FarFutureEpoch = mockValidator.ExitEpoch()
 				mockState.EXPECT().Version().Return(clparams.AltairVersion).Times(1)
-				mockState.EXPECT().GetDomain(t.beaconCfg.DomainVoluntaryExit, mockMsg.VoluntaryExit.Epoch).Return([]byte{}, nil).Times(1)
+				mockState.EXPECT().GetDomain(t.beaconCfg.DomainVoluntaryExit, mockMsg.SignedVoluntaryExit.VoluntaryExit.Epoch).Return([]byte{}, nil).Times(1)
 				computeSigningRoot = func(_ ssz.HashableSSZ, domain []byte) ([32]byte, error) {
 					return [32]byte{}, nil
 				}
-				t.gomockCtrl.RecordCall(t.mockFuncs, "BlsVerify", gomock.Any(), gomock.Any(), gomock.Any()).Return(true, nil).Times(1)
+
+				t.gomockCtrl.RecordCall(t.mockFuncs, "BlsVerifyMultipleSignatures", gomock.Any(), gomock.Any(), gomock.Any()).Return(true, nil).Times(1)
 			},
 			msg:     mockMsg,
+			err:     ErrIgnore,
 			wantErr: false,
 		},
 	}

cmd/caplin/caplin1/run.go

@@ -311,7 +311,7 @@ func RunCaplinService(ctx context.Context, engine execution_client.ExecutionEngi
 	attestationService := services.NewAttestationService(ctx, forkChoice, committeeSub, ethClock, syncedDataManager, beaconConfig, networkConfig, emitters, batchSignatureVerifier)
 	syncContributionService := services.NewSyncContributionService(syncedDataManager, beaconConfig, syncContributionPool, ethClock, emitters, false)
 	aggregateAndProofService := services.NewAggregateAndProofService(ctx, syncedDataManager, forkChoice, beaconConfig, pool, false, batchSignatureVerifier)
-	voluntaryExitService := services.NewVoluntaryExitService(pool, emitters, syncedDataManager, beaconConfig, ethClock)
+	voluntaryExitService := services.NewVoluntaryExitService(pool, emitters, syncedDataManager, beaconConfig, ethClock, batchSignatureVerifier)
 	blsToExecutionChangeService := services.NewBLSToExecutionChangeService(pool, emitters, syncedDataManager, beaconConfig, batchSignatureVerifier)
 	proposerSlashingService := services.NewProposerSlashingService(pool, syncedDataManager, beaconConfig, ethClock, emitters)