Ccip-4564 fix configure chains to be idempotent (#15614)

* remove deployCCIPContracts

* make configureChain idempotent

* add comments

* go mod tidy

* make test reliable

deployment/ccip/changeset/cs_active_candidate.go

@@ -44,15 +44,17 @@ func (p PromoteAllCandidatesChangesetConfig) Validate(e deployment.Environment,
 		return nil, fmt.Errorf("fetch node info: %w", err)
 	}
 
-	donID, err := internal.DonIDForChain(
+	donID, exists, err := internal.DonIDForChain(
 		state.Chains[p.HomeChainSelector].CapabilityRegistry,
 		state.Chains[p.HomeChainSelector].CCIPHome,
 		p.NewChainSelector,
 	)
 	if err != nil {
 		return nil, fmt.Errorf("fetch don id for chain: %w", err)
 	}
-
+	if !exists {
+		return nil, fmt.Errorf("don id for chain(%d) does not exist", p.NewChainSelector)
+	}
 	// check if the DON ID has a candidate digest set that we can promote
 	for _, pluginType := range []cctypes.PluginType{cctypes.PluginTypeCCIPCommit, cctypes.PluginTypeCCIPExec} {
 		candidateDigest, err := state.Chains[p.HomeChainSelector].CCIPHome.GetCandidateDigest(nil, donID, uint8(pluginType))
@@ -204,10 +206,13 @@ func setCandidateOnExistingDon(
 	nodes deployment.Nodes,
 ) ([]mcms.Operation, error) {
 	// fetch DON ID for the chain
-	donID, err := internal.DonIDForChain(capReg, ccipHome, chainSelector)
+	donID, exists, err := internal.DonIDForChain(capReg, ccipHome, chainSelector)
 	if err != nil {
 		return nil, fmt.Errorf("fetch don id for chain: %w", err)
 	}
+	if !exists {
+		return nil, fmt.Errorf("don id for chain(%d) does not exist", chainSelector)
+	}
 	fmt.Printf("donID: %d", donID)
 	encodedSetCandidateCall, err := internal.CCIPHomeABI.Pack(
 		"setCandidate",
@@ -301,10 +306,13 @@ func promoteAllCandidatesForChainOps(
 	nodes deployment.Nodes,
 ) ([]mcms.Operation, error) {
 	// fetch DON ID for the chain
-	donID, err := internal.DonIDForChain(capReg, ccipHome, chainSelector)
+	donID, exists, err := internal.DonIDForChain(capReg, ccipHome, chainSelector)
 	if err != nil {
 		return nil, fmt.Errorf("fetch don id for chain: %w", err)
 	}
+	if !exists {
+		return nil, fmt.Errorf("don id for chain(%d) does not exist", chainSelector)
+	}
 
 	var mcmsOps []mcms.Operation
 	updateCommitOp, err := promoteCandidateOp(donID, uint8(cctypes.PluginTypeCCIPCommit), capReg, ccipHome, nodes)

deployment/ccip/changeset/cs_active_candidate_test.go

@@ -107,8 +107,9 @@ func TestActiveCandidate(t *testing.T) {
 
 	// [ACTIVE, CANDIDATE] setup by setting candidate through cap reg
 	capReg, ccipHome := state.Chains[tenv.HomeChainSel].CapabilityRegistry, state.Chains[tenv.HomeChainSel].CCIPHome
-	donID, err := internal.DonIDForChain(capReg, ccipHome, tenv.FeedChainSel)
+	donID, exists, err := internal.DonIDForChain(capReg, ccipHome, tenv.FeedChainSel)
 	require.NoError(t, err)
+	require.True(t, exists)
 	donInfo, err := state.Chains[tenv.HomeChainSel].CapabilityRegistry.GetDON(nil, donID)
 	require.NoError(t, err)
 	require.Equal(t, 5, len(donInfo.NodeP2PIds))

deployment/ccip/changeset/cs_initial_add_chain.go

@@ -10,7 +10,6 @@ import (
 
 	"github.com/ethereum/go-ethereum/accounts/abi/bind"
 	"github.com/ethereum/go-ethereum/common"
-	"github.com/imdario/mergo"
 	"github.com/smartcontractkit/ccip-owner-contracts/pkg/proposal/timelock"
 
 	"github.com/smartcontractkit/chainlink-ccip/chainconfig"
@@ -60,16 +59,6 @@ type CCIPOCRParams struct {
 	ExecuteOffChainConfig pluginconfig.ExecuteOffchainConfig
 }
 
-// Override overrides non-empty dst CCIPOCRParams attributes with non-empty src CCIPOCRParams attributes values
-// and returns the updated CCIPOCRParams.
-func (c CCIPOCRParams) Override(overrides CCIPOCRParams) (CCIPOCRParams, error) {
-	err := mergo.Merge(&c, &overrides, mergo.WithOverride)
-	if err != nil {
-		return CCIPOCRParams{}, err
-	}
-	return c, nil
-}
-
 func (c CCIPOCRParams) Validate() error {
 	if err := c.OCRParameters.Validate(); err != nil {
 		return fmt.Errorf("invalid OCR parameters: %w", err)
@@ -253,6 +242,20 @@ func setupConfigInfo(chainSelector uint64, readers [][32]byte, fChain uint8, cfg
 	}
 }
 
+func isChainConfigEqual(a, b ccip_home.CCIPHomeChainConfig) bool {
+	mapReader := make(map[[32]byte]struct{})
+	for i := range a.Readers {
+		mapReader[a.Readers[i]] = struct{}{}
+	}
+	for i := range b.Readers {
+		if _, ok := mapReader[b.Readers[i]]; !ok {
+			return false
+		}
+	}
+	return bytes.Equal(a.Config, b.Config) &&
+		a.FChain == b.FChain
+}
+
 func addChainConfig(
 	lggr logger.Logger,
 	h deployment.Chain,
@@ -270,6 +273,18 @@ func addChainConfig(
 		return ccip_home.CCIPHomeChainConfigArgs{}, err
 	}
 	chainConfig := setupConfigInfo(chainSelector, p2pIDs, uint8(len(p2pIDs)/3), encodedExtraChainConfig)
+	existingCfg, err := ccipConfig.GetChainConfig(nil, chainSelector)
+	if err != nil {
+		return ccip_home.CCIPHomeChainConfigArgs{}, fmt.Errorf("get chain config for selector %d: %w", chainSelector, err)
+	}
+	if isChainConfigEqual(existingCfg, chainConfig.ChainConfig) {
+		lggr.Infow("Chain config already exists, not applying again",
+			"homeChain", h.String(),
+			"addedChain", chainSelector,
+			"chainConfig", chainConfig,
+		)
+		return chainConfig, nil
+	}
 	tx, err := ccipConfig.ApplyChainConfigUpdates(h.DeployerKey, nil, []ccip_home.CCIPHomeChainConfigArgs{
 		chainConfig,
 	})
@@ -293,6 +308,14 @@ func createDON(
 	newChainSel uint64,
 	nodes deployment.Nodes,
 ) error {
+	donID, exists, err := internal.DonIDForChain(capReg, ccipHome, newChainSel)
+	if err != nil {
+		return fmt.Errorf("fetch don id for chain: %w", err)
+	}
+	if exists {
+		lggr.Infow("DON already exists not adding it again", "donID", donID, "chain", newChainSel)
+		return ValidateCCIPHomeConfigSetUp(lggr, capReg, ccipHome, newChainSel)
+	}
 	commitConfig, ok := ocr3Configs[cctypes.PluginTypeCCIPCommit]
 	if !ok {
 		return fmt.Errorf("missing commit plugin in ocr3Configs")
@@ -308,7 +331,7 @@ func createDON(
 		return err
 	}
 
-	donID := latestDon.Id + 1
+	donID = latestDon.Id + 1
 
 	err = internal.SetupCommitDON(lggr, donID, commitConfig, capReg, home, nodes, ccipHome)
 	if err != nil {
@@ -361,11 +384,37 @@ func addDON(
 		"chainSelector", dest.Selector,
 	)
 
+	// check if OCR3 config is already set on offramp
+	ocr3ConfigSet, err := isOCR3ConfigSetOnOffRamp(lggr, dest, offRamp, offrampOCR3Configs)
+	if err != nil {
+		return fmt.Errorf("error checking if OCR3 config is set on offramp: %w", err)
+	}
+	if ocr3ConfigSet {
+		lggr.Infow("OCR3 config already set on offramp, not applying again", "chain", dest.String())
+		return nil
+	}
 	tx, err := offRamp.SetOCR3Configs(dest.DeployerKey, offrampOCR3Configs)
 	if _, err := deployment.ConfirmIfNoError(dest, tx, err); err != nil {
 		return err
 	}
+	lggr.Infow("Set OCR3 Configs", "chain", dest.String())
+	// now check if OCR3 config is set on offramp
+	ocr3ConfigSet, err = isOCR3ConfigSetOnOffRamp(lggr, dest, offRamp, offrampOCR3Configs)
+	if err != nil {
+		return fmt.Errorf("error checking if OCR3 config is set on offramp: %w", err)
+	}
+	if !ocr3ConfigSet {
+		return fmt.Errorf("OCR3 config not set on offramp properly, check logs, chain %s", dest.String())
+	}
+	return nil
+}
 
+func isOCR3ConfigSetOnOffRamp(
+	lggr logger.Logger,
+	chain deployment.Chain,
+	offRamp *offramp.OffRamp,
+	offrampOCR3Configs []offramp.MultiOCR3BaseOCRConfigArgs,
+) (bool, error) {
 	mapOfframpOCR3Configs := make(map[cctypes.PluginType]offramp.MultiOCR3BaseOCRConfigArgs)
 	for _, config := range offrampOCR3Configs {
 		mapOfframpOCR3Configs[cctypes.PluginType(config.OcrPluginType)] = config
@@ -376,7 +425,7 @@ func addDON(
 			Context: context.Background(),
 		}, uint8(pluginType))
 		if err != nil {
-			return err
+			return false, fmt.Errorf("error fetching OCR3 config for plugin %s chain %s: %w", pluginType.String(), chain.String(), err)
 		}
 		lggr.Debugw("Fetched OCR3 Configs",
 			"MultiOCR3BaseOCRConfig.F", ocrConfig.ConfigInfo.F,
@@ -385,35 +434,39 @@ func addDON(
 			"Signers", ocrConfig.Signers,
 			"Transmitters", ocrConfig.Transmitters,
 			"configDigest", hex.EncodeToString(ocrConfig.ConfigInfo.ConfigDigest[:]),
-			"chain", dest.String(),
+			"chain", chain.String(),
 		)
 		// TODO: assertions to be done as part of full state
 		// resprentation validation CCIP-3047
 		if mapOfframpOCR3Configs[pluginType].ConfigDigest != ocrConfig.ConfigInfo.ConfigDigest {
-			return fmt.Errorf("%s OCR3 config digest mismatch", pluginType.String())
+			lggr.Infow("OCR3 config digest mismatch", "pluginType", pluginType.String())
+			return false, nil
 		}
 		if mapOfframpOCR3Configs[pluginType].F != ocrConfig.ConfigInfo.F {
-			return fmt.Errorf("%s OCR3 config F mismatch", pluginType.String())
+			lggr.Infow("OCR3 config F mismatch", "pluginType", pluginType.String())
+			return false, nil
 		}
 		if mapOfframpOCR3Configs[pluginType].IsSignatureVerificationEnabled != ocrConfig.ConfigInfo.IsSignatureVerificationEnabled {
-			return fmt.Errorf("%s OCR3 config signature verification mismatch", pluginType.String())
+			lggr.Infow("OCR3 config signature verification mismatch", "pluginType", pluginType.String())
+			return false, nil
 		}
 		if pluginType == cctypes.PluginTypeCCIPCommit {
 			// only commit will set signers, exec doesn't need them.
 			for i, signer := range mapOfframpOCR3Configs[pluginType].Signers {
 				if !bytes.Equal(signer.Bytes(), ocrConfig.Signers[i].Bytes()) {
-					return fmt.Errorf("%s OCR3 config signer mismatch", pluginType.String())
+					lggr.Infow("OCR3 config signer mismatch", "pluginType", pluginType.String())
+					return false, nil
 				}
 			}
 		}
 		for i, transmitter := range mapOfframpOCR3Configs[pluginType].Transmitters {
 			if !bytes.Equal(transmitter.Bytes(), ocrConfig.Transmitters[i].Bytes()) {
-				return fmt.Errorf("%s OCR3 config transmitter mismatch", pluginType.String())
+				lggr.Infow("OCR3 config transmitter mismatch", "pluginType", pluginType.String())
+				return false, nil
 			}
 		}
 	}
-
-	return nil
+	return true, nil
 }
 
 // ValidateCCIPHomeConfigSetUp checks that the commit and exec active and candidate configs are set up correctly
@@ -424,10 +477,13 @@ func ValidateCCIPHomeConfigSetUp(
 	chainSel uint64,
 ) error {
 	// fetch DONID
-	donID, err := internal.DonIDForChain(capReg, ccipHome, chainSel)
+	donID, exists, err := internal.DonIDForChain(capReg, ccipHome, chainSel)
 	if err != nil {
 		return fmt.Errorf("fetch don id for chain: %w", err)
 	}
+	if !exists {
+		return fmt.Errorf("don id for chain(%d) does not exist", chainSel)
+	}
 	// final sanity checks on configs.
 	commitConfigs, err := ccipHome.GetAllConfigs(&bind.CallOpts{
 		//Pending: true,

deployment/ccip/changeset/cs_initial_add_chain_test.go

@@ -2,29 +2,91 @@ package changeset
 
 import (
 	"testing"
-	"time"
 
-	chainselectors "github.com/smartcontractkit/chain-selectors"
+	"github.com/ethereum/go-ethereum/common"
 	"github.com/smartcontractkit/chainlink-ccip/pluginconfig"
-	"github.com/smartcontractkit/chainlink-common/pkg/config"
+	"github.com/smartcontractkit/chainlink-testing-framework/lib/utils/testcontext"
 	"github.com/stretchr/testify/require"
+
+	commonchangeset "github.com/smartcontractkit/chainlink/deployment/common/changeset"
+	"github.com/smartcontractkit/chainlink/v2/core/gethwrappers/ccip/generated/router"
 )
 
-func TestOverrideCCIPParams(t *testing.T) {
-	params := DefaultOCRParams(chainselectors.ETHEREUM_TESTNET_SEPOLIA.Selector, nil, nil)
-	overrides := CCIPOCRParams{
-		ExecuteOffChainConfig: pluginconfig.ExecuteOffchainConfig{
-			RelativeBoostPerWaitHour: 10,
+func TestInitialAddChainAppliedTwice(t *testing.T) {
+	// This already applies the initial add chain changeset.
+	e := NewMemoryEnvironment(t)
+
+	state, err := LoadOnchainState(e.Env)
+	require.NoError(t, err)
+
+	// now try to apply it again for the second time
+	// Build the per chain config.
+	allChains := e.Env.AllChainSelectors()
+	tokenConfig := NewTestTokenConfig(state.Chains[e.FeedChainSel].USDFeeds)
+	chainConfigs := make(map[uint64]CCIPOCRParams)
+	timelockContractsPerChain := make(map[uint64]*commonchangeset.TimelockExecutionContracts)
+
+	for _, chain := range allChains {
+		timelockContractsPerChain[chain] = &commonchangeset.TimelockExecutionContracts{
+			Timelock:  state.Chains[chain].Timelock,
+			CallProxy: state.Chains[chain].CallProxy,
+		}
+		tokenInfo := tokenConfig.GetTokenInfo(e.Env.Logger, state.Chains[chain].LinkToken, state.Chains[chain].Weth9)
+		ocrParams := DefaultOCRParams(e.FeedChainSel, tokenInfo, []pluginconfig.TokenDataObserverConfig{})
+		chainConfigs[chain] = ocrParams
+	}
+	e.Env, err = commonchangeset.ApplyChangesets(t, e.Env, timelockContractsPerChain, []commonchangeset.ChangesetApplication{
+		{
+			Changeset: commonchangeset.WrapChangeSet(ConfigureNewChains),
+			Config: NewChainsConfig{
+				HomeChainSel:       e.HomeChainSel,
+				FeedChainSel:       e.FeedChainSel,
+				ChainConfigByChain: chainConfigs,
+			},
 		},
-		CommitOffChainConfig: pluginconfig.CommitOffchainConfig{
-			TokenPriceBatchWriteFrequency:     *config.MustNewDuration(1_000_000 * time.Hour),
-			RemoteGasPriceBatchWriteFrequency: *config.MustNewDuration(1_000_000 * time.Hour),
+	})
+	require.NoError(t, err)
+	// send requests
+	chain1, chain2 := allChains[0], allChains[1]
+
+	_, err = AddLanes(e.Env, AddLanesConfig{
+		LaneConfigs: []LaneConfig{
+			{
+				SourceSelector:        chain1,
+				DestSelector:          chain2,
+				InitialPricesBySource: DefaultInitialPrices,
+				FeeQuoterDestChain:    DefaultFeeQuoterDestChainConfig(),
+				TestRouter:            true,
+			},
 		},
-	}
-	newParams, err := params.Override(overrides)
+	})
 	require.NoError(t, err)
-	require.Equal(t, overrides.ExecuteOffChainConfig.RelativeBoostPerWaitHour, newParams.ExecuteOffChainConfig.RelativeBoostPerWaitHour)
-	require.Equal(t, overrides.CommitOffChainConfig.TokenPriceBatchWriteFrequency, newParams.CommitOffChainConfig.TokenPriceBatchWriteFrequency)
-	require.Equal(t, overrides.CommitOffChainConfig.RemoteGasPriceBatchWriteFrequency, newParams.CommitOffChainConfig.RemoteGasPriceBatchWriteFrequency)
-	require.Equal(t, params.OCRParameters, newParams.OCRParameters)
+	ReplayLogs(t, e.Env.Offchain, e.ReplayBlocks)
+	// Need to keep track of the block number for each chain so that event subscription can be done from that block.
+	startBlocks := make(map[uint64]*uint64)
+	// Send a message from each chain to every other chain.
+	expectedSeqNumExec := make(map[SourceDestPair][]uint64)
+	expectedSeqNum := make(map[SourceDestPair]uint64)
+	latesthdr, err := e.Env.Chains[chain2].Client.HeaderByNumber(testcontext.Get(t), nil)
+	require.NoError(t, err)
+	block := latesthdr.Number.Uint64()
+	startBlocks[chain2] = &block
+	msgSentEvent := TestSendRequest(t, e.Env, state, chain1, chain2, true, router.ClientEVM2AnyMessage{
+		Receiver:     common.LeftPadBytes(state.Chains[chain2].Receiver.Address().Bytes(), 32),
+		Data:         []byte("hello"),
+		TokenAmounts: nil,
+		FeeToken:     common.HexToAddress("0x0"),
+		ExtraArgs:    nil,
+	})
+
+	expectedSeqNum[SourceDestPair{
+		SourceChainSelector: chain1,
+		DestChainSelector:   chain2,
+	}] = msgSentEvent.SequenceNumber
+	expectedSeqNumExec[SourceDestPair{
+		SourceChainSelector: chain1,
+		DestChainSelector:   chain2,
+	}] = []uint64{msgSentEvent.SequenceNumber}
+	ConfirmCommitForAllWithExpectedSeqNums(t, e.Env, state, expectedSeqNum, startBlocks)
+	ConfirmExecWithSeqNrsForAll(t, e.Env, state, expectedSeqNumExec, startBlocks)
 }

deployment/ccip/changeset/internal/deploy_home_chain.go

@@ -110,25 +110,25 @@ func LatestCCIPDON(registry *capabilities_registry.CapabilitiesRegistry) (*capab
 
 // DonIDForChain returns the DON ID for the chain with the given selector
 // It looks up with the CCIPHome contract to find the OCR3 configs for the DONs, and returns the DON ID for the chain matching with the given selector from the OCR3 configs
-func DonIDForChain(registry *capabilities_registry.CapabilitiesRegistry, ccipHome *ccip_home.CCIPHome, chainSelector uint64) (uint32, error) {
+func DonIDForChain(registry *capabilities_registry.CapabilitiesRegistry, ccipHome *ccip_home.CCIPHome, chainSelector uint64) (uint32, bool, error) {
 	dons, err := registry.GetDONs(nil)
 	if err != nil {
-		return 0, err
+		return 0, false, err
 	}
 	// TODO: what happens if there are multiple dons for one chain (accidentally?)
 	for _, don := range dons {
 		if len(don.CapabilityConfigurations) == 1 &&
 			don.CapabilityConfigurations[0].CapabilityId == CCIPCapabilityID {
 			configs, err := ccipHome.GetAllConfigs(nil, don.Id, uint8(types.PluginTypeCCIPCommit))
 			if err != nil {
-				return 0, err
+				return 0, false, err
 			}
 			if configs.ActiveConfig.Config.ChainSelector == chainSelector || configs.CandidateConfig.Config.ChainSelector == chainSelector {
-				return don.Id, nil
+				return don.Id, true, nil
 			}
 		}
 	}
-	return 0, fmt.Errorf("no DON found for chain %d", chainSelector)
+	return 0, false, nil
 }
 
 func BuildSetOCR3ConfigArgs(