From 0e857df486cea911513c6b36974074d602c9f6fd Mon Sep 17 00:00:00 2001 From: y0sher Date: Wed, 22 Nov 2023 15:05:47 +0200 Subject: [PATCH 1/4] enable prater in the gitlab ci --- .gitlab-ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 627b5f0e8c..f12adc8970 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -150,7 +150,7 @@ Deploy nodes to prod: # +---------------------------+ # | 🟠 Deploy SSV Prater nodes | # +---------------------------+ - #- .k8/production/prater/scripts/deploy-cluster-1--4.sh $DOCKER_REPO_INFRA_PROD $CI_COMMIT_SHA ssv $APP_REPLICAS_INFRA_PROD blox-infra-prod kubernetes-admin@blox-infra-prod ssv.network $K8S_API_VERSION $PROD_HEALTH_CHECK_IMAGE $SSV_NODES_CPU_LIMIT_V3 $SSV_NODES_MEM_LIMIT_V3 + - .k8/production/prater/scripts/deploy-cluster-1--4.sh $DOCKER_REPO_INFRA_PROD $CI_COMMIT_SHA ssv $APP_REPLICAS_INFRA_PROD blox-infra-prod kubernetes-admin@blox-infra-prod ssv.network $K8S_API_VERSION $PROD_HEALTH_CHECK_IMAGE $SSV_NODES_CPU_LIMIT_V3 $SSV_NODES_MEM_LIMIT_V3 # # +----------------------------+ # | 🔴 Deploy SSV Mainnet nodes | @@ -196,7 +196,7 @@ Deploy exporter to prod: # +------------------------------+ # | 🟠 Deploy Prater exporter | # +------------------------------+ - # - .k8/production/prater/scripts/deploy-exporters.sh $DOCKER_REPO_INFRA_PROD $CI_COMMIT_SHA ssv $APP_REPLICAS_INFRA_PROD blox-infra-prod kubernetes-admin@blox-infra-prod ssv.network $K8S_API_VERSION $SSV_EXPORTER_CPU_LIMIT $SSV_EXPORTER_MEM_LIMIT + - .k8/production/prater/scripts/deploy-exporters.sh $DOCKER_REPO_INFRA_PROD $CI_COMMIT_SHA ssv $APP_REPLICAS_INFRA_PROD blox-infra-prod kubernetes-admin@blox-infra-prod ssv.network $K8S_API_VERSION $SSV_EXPORTER_CPU_LIMIT $SSV_EXPORTER_MEM_LIMIT # # +------------------------------+ # │ 🔴 Deploy Mainnet exporter | From b789b240149538950697ccff0e634785b3508468 Mon Sep 17 00:00:00 2001 From: moshe-blox <89339422+moshe-blox@users.noreply.github.com> Date: Wed, 22 Nov 2023 16:53:43 +0200 Subject: [PATCH 2/4] fix: check signatures in `validatePartialSigMsgForSlot` (#1215) * fix: check signatures in `validatePartialSigMsgForSlot` * deploy to stage * revert --- protocol/v2/ssv/runner/runner.go | 3 +-- protocol/v2/ssv/runner/runner_signatures.go | 14 ++++++-------- protocol/v2/ssv/spectest/msg_processing_type.go | 2 -- 3 files changed, 7 insertions(+), 12 deletions(-) diff --git a/protocol/v2/ssv/runner/runner.go b/protocol/v2/ssv/runner/runner.go index ff833d8c31..9034d5da84 100644 --- a/protocol/v2/ssv/runner/runner.go +++ b/protocol/v2/ssv/runner/runner.go @@ -54,8 +54,7 @@ type BaseRunner struct { BeaconRoleType spectypes.BeaconRole // implementation vars - TimeoutF TimeoutF `json:"-"` - VerifySignatures bool `json:"-"` + TimeoutF TimeoutF `json:"-"` // highestDecidedSlot holds the highest decided duty slot and gets updated after each decided is reached highestDecidedSlot spec.Slot diff --git a/protocol/v2/ssv/runner/runner_signatures.go b/protocol/v2/ssv/runner/runner_signatures.go index edfc608ea7..54e4d9de1e 100644 --- a/protocol/v2/ssv/runner/runner_signatures.go +++ b/protocol/v2/ssv/runner/runner_signatures.go @@ -58,15 +58,13 @@ func (b *BaseRunner) validatePartialSigMsgForSlot( return errors.New("invalid partial sig slot") } - if b.VerifySignatures { - if err := types.VerifyByOperators(signedMsg.GetSignature(), signedMsg, b.Share.DomainType, spectypes.PartialSignatureType, b.Share.Committee); err != nil { - return errors.Wrap(err, "failed to verify PartialSignature") - } + if err := types.VerifyByOperators(signedMsg.GetSignature(), signedMsg, b.Share.DomainType, spectypes.PartialSignatureType, b.Share.Committee); err != nil { + return errors.Wrap(err, "failed to verify PartialSignature") + } - for _, msg := range signedMsg.Message.Messages { - if err := b.verifyBeaconPartialSignature(msg); err != nil { - return errors.Wrap(err, "could not verify Beacon partial Signature") - } + for _, msg := range signedMsg.Message.Messages { + if err := b.verifyBeaconPartialSignature(msg); err != nil { + return errors.Wrap(err, "could not verify Beacon partial Signature") } } diff --git a/protocol/v2/ssv/spectest/msg_processing_type.go b/protocol/v2/ssv/spectest/msg_processing_type.go index 412b92b8da..b962418894 100644 --- a/protocol/v2/ssv/spectest/msg_processing_type.go +++ b/protocol/v2/ssv/spectest/msg_processing_type.go @@ -48,8 +48,6 @@ func RunMsgProcessing(t *testing.T, test *MsgProcessingSpecTest) { } func (test *MsgProcessingSpecTest) RunAsPartOfMultiTest(t *testing.T, logger *zap.Logger) { - test.Runner.GetBaseRunner().VerifySignatures = true - v := ssvtesting.BaseValidator(logger, spectestingutils.KeySetForShare(test.Runner.GetBaseRunner().Share)) v.DutyRunners[test.Runner.GetBaseRunner().BeaconRoleType] = test.Runner v.Network = test.Runner.GetNetwork().(specqbft.Network) // TODO need to align From 20129b16bc3f755b6d67b6f1bab080052dea0157 Mon Sep 17 00:00:00 2001 From: Nikita Kryuchkov Date: Wed, 22 Nov 2023 21:41:26 +0400 Subject: [PATCH 3/4] Simplify duplicated proposal with different data check (#1207) * More logs for "duplicated proposal with different data" * Deploy to 41-44 * Deploy to 1-12 & 17-40 & 41-44 & 53-69 * Improve logging * Fix logging * Extend logs * Fix logging issues * Log slot and round * Log proposal data setting * Fix consensusData init * No deploy to 1-4 * Add validator and signer to logging * Simplify condition * Attempt to fix proposal data assignment * Revert deployment * Revert logs * Revert "Revert logs" This reverts commit 6871b56f4b0dc2d1405471866787a6043e52367d. * Revert "Revert deployment" This reverts commit 1bed180e576d32d6fa5b9dfa79fe05ac868405cf. * Remove heavy logs * Add root log * Fix root log * deploy to most of stage * deploy to more clusters. * change db for ssv-node-44 * print root as hex * deploy to all to prevent confusions * hack to update proposal before message is arriving. * deploy to explorer as well * remove the hack * add more verbose logs * add flag to not broadcast proposal on second quorum of RC * 1-48 deploy * Fix logging * go fmt * Deploy to 5-12 & 17-48 * Extend logs * Attempt to fix empty proposal data * Release 45-48 * Fix logs * Deploy to all nodes * Remove protocol changes * Disable deployment * Remove debugging leftovers * revert database reset * Revert "Remove debugging leftovers" This reverts commit 8d789d5b83be89fd0cf3a72378ef896e25d954b7. * Revert "Disable deployment" This reverts commit 673c162eb190f1f8f29cef58449d859139f56a41. * Revert the possible fix * track concurrency * validation locks by MessageID * Revert "validation locks by MessageID" This reverts commit a6d729f29279c6f77c8055b50489182709f24caa. * Revert "track concurrency" This reverts commit 628548953ff8e2e3bff986dca449e49d801a1d7f. * Revert "Revert the possible fix" This reverts commit 6513988b429a88134748ea7927bb483c832542f8. * Remove debugging leftovers * Disable deployment * Remove redundant code * Narrow proposal data check --------- Co-authored-by: MatheusFranco99 <48058141+MatheusFranco99@users.noreply.github.com> Co-authored-by: y0sher Co-authored-by: moshe-blox --- message/validation/consensus_validation.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/message/validation/consensus_validation.go b/message/validation/consensus_validation.go index fde979826b..674f148ea6 100644 --- a/message/validation/consensus_validation.go +++ b/message/validation/consensus_validation.go @@ -240,7 +240,7 @@ func (mv *messageValidator) validateSignerBehaviorConsensus( return err } - if !(msgSlot > signerState.Slot || msgSlot == signerState.Slot && msgRound > signerState.Round) { + if msgSlot == signerState.Slot && msgRound == signerState.Round { if mv.hasFullData(signedMsg) && signerState.ProposalData != nil && !bytes.Equal(signerState.ProposalData, signedMsg.FullData) { return ErrDuplicatedProposalWithDifferentData } From 6dafce329ea5772f517238ba3ab3b45163dc4b21 Mon Sep 17 00:00:00 2001 From: moshe-blox <89339422+moshe-blox@users.noreply.github.com> Date: Wed, 22 Nov 2023 19:46:05 +0200 Subject: [PATCH 4/4] fix: race conditions in message validation (#1217) * fix: race conditions in message validation * refactor * fix nil map --- message/validation/validation.go | 22 ++++++++++++++++++++-- 1 file changed, 20 insertions(+), 2 deletions(-) diff --git a/message/validation/validation.go b/message/validation/validation.go index 2a363efe25..c00b2b8ea3 100644 --- a/message/validation/validation.go +++ b/message/validation/validation.go @@ -77,8 +77,14 @@ type messageValidator struct { dutyStore *dutystore.Store ownOperatorID spectypes.OperatorID operatorIDToPubkeyCache *hashmap.Map[spectypes.OperatorID, *rsa.PublicKey] - selfPID peer.ID - selfAccept bool + + // validationLocks is a map of lock per SSV message ID to + // prevent concurrent access to the same state. + validationLocks map[spectypes.MessageID]*sync.Mutex + validationMutex sync.Mutex + + selfPID peer.ID + selfAccept bool } // NewMessageValidator returns a new MessageValidator with the given network configuration and options. @@ -88,6 +94,7 @@ func NewMessageValidator(netCfg networkconfig.NetworkConfig, opts ...Option) Mes metrics: &nopMetrics{}, netCfg: netCfg, operatorIDToPubkeyCache: hashmap.New[spectypes.OperatorID, *rsa.PublicKey](), + validationLocks: make(map[spectypes.MessageID]*sync.Mutex), } for _, opt := range opts { @@ -431,6 +438,17 @@ func (mv *messageValidator) validateSSVMessage(ssvMessage *spectypes.SSVMessage, return nil, descriptor, e } + // Lock this SSV message ID to prevent concurrent access to the same state. + mv.validationMutex.Lock() + mutex, ok := mv.validationLocks[msg.GetID()] + if !ok { + mutex = &sync.Mutex{} + mv.validationLocks[msg.GetID()] = mutex + } + mutex.Lock() + defer mutex.Unlock() + mv.validationMutex.Unlock() + descriptor.SSVMessageType = ssvMessage.MsgType if mv.nodeStorage != nil {