From 2ca1a098597c1e98d0ceaf0c00e7ecd598a760f4 Mon Sep 17 00:00:00 2001 From: dimitris Date: Fri, 29 Nov 2024 16:51:19 +0200 Subject: [PATCH] RMN integration test updates (#15423) * enable rmn tests * provide double fee to the msgs * 10x the fee * print fees * revert * revert flaky test related changes * e2e test with cursed source chain * include global curse test and enable tests in ci * refactor rmn integration test case runner * gomodtidy * lint fix * lint fix * explicitly defined subjects * fix lint --- .github/e2e-tests.yml | 88 ++-- deployment/ccip/changeset/test_helpers.go | 5 +- integration-tests/go.mod | 2 +- integration-tests/smoke/ccip/ccip_rmn_test.go | 498 ++++++++++++------ 4 files changed, 408 insertions(+), 185 deletions(-) diff --git a/.github/e2e-tests.yml b/.github/e2e-tests.yml index 570ebcf4503..3f412854e1c 100644 --- a/.github/e2e-tests.yml +++ b/.github/e2e-tests.yml @@ -1030,21 +1030,20 @@ runner-test-matrix: E2E_RMN_RAGEPROXY_VERSION: master-f461a9e E2E_RMN_AFN2PROXY_VERSION: master-f461a9e -# Enable after flaking issue is resolved -# - id: smoke/ccip/ccip_rmn_test.go:^TestRMN_NotEnoughObservers$ -# path: integration-tests/smoke/ccip/ccip_rmn_test.go -# test_env_type: docker -# runs_on: ubuntu-latest -# triggers: -# - PR E2E Core Tests -# - Nightly E2E Tests -# test_cmd: cd integration-tests/smoke/ccip && go test -test.run ^TestRMN_NotEnoughObservers$ -timeout 12m -test.parallel=1 -count=1 -json -# pyroscope_env: ci-smoke-ccipv1_6-evm-simulated -# test_env_vars: -# E2E_TEST_SELECTED_NETWORK: SIMULATED_1,SIMULATED_2 -# E2E_JD_VERSION: 0.6.0 -# E2E_RMN_RAGEPROXY_VERSION: master-f461a9e -# E2E_RMN_AFN2PROXY_VERSION: master-f461a9e + - id: smoke/ccip/ccip_rmn_test.go:^TestRMN_NotEnoughObservers$ + path: integration-tests/smoke/ccip/ccip_rmn_test.go + test_env_type: docker + runs_on: ubuntu-latest + triggers: + - PR E2E Core Tests + - Nightly E2E Tests + test_cmd: cd integration-tests/smoke/ccip && go test -test.run ^TestRMN_NotEnoughObservers$ -timeout 12m -test.parallel=1 -count=1 -json + pyroscope_env: ci-smoke-ccipv1_6-evm-simulated + test_env_vars: + E2E_TEST_SELECTED_NETWORK: SIMULATED_1,SIMULATED_2 + E2E_JD_VERSION: 0.6.0 + E2E_RMN_RAGEPROXY_VERSION: master-f461a9e + E2E_RMN_AFN2PROXY_VERSION: master-f461a9e - id: smoke/ccip/ccip_rmn_test.go:^TestRMN_DifferentSigners$ path: integration-tests/smoke/ccip/ccip_rmn_test.go @@ -1061,22 +1060,20 @@ runner-test-matrix: E2E_RMN_RAGEPROXY_VERSION: master-f461a9e E2E_RMN_AFN2PROXY_VERSION: master-f461a9e -# Enable after flaking issue is resolved -# - id: smoke/ccip/ccip_rmn_test.go:^TestRMN_NotEnoughSigners$ -# path: integration-tests/smoke/ccip/ccip_rmn_test.go -# test_env_type: docker -# runs_on: ubuntu-latest -# triggers: -# - PR E2E Core Tests -# - Nightly E2E Tests -# test_cmd: cd integration-tests/smoke/ccip && go test -test.run ^TestRMN_NotEnoughSigners$ -timeout 12m -test.parallel=1 -count=1 -json -# pyroscope_env: ci-smoke-ccipv1_6-evm-simulated -# test_env_vars: -# E2E_TEST_SELECTED_NETWORK: SIMULATED_1,SIMULATED_2 -# E2E_JD_VERSION: 0.6.0 -# E2E_RMN_RAGEPROXY_VERSION: master-f461a9e -# E2E_RMN_AFN2PROXY_VERSION: master-f461a9e - + - id: smoke/ccip/ccip_rmn_test.go:^TestRMN_NotEnoughSigners$ + path: integration-tests/smoke/ccip/ccip_rmn_test.go + test_env_type: docker + runs_on: ubuntu-latest + triggers: + - PR E2E Core Tests + - Nightly E2E Tests + test_cmd: cd integration-tests/smoke/ccip && go test -test.run ^TestRMN_NotEnoughSigners$ -timeout 12m -test.parallel=1 -count=1 -json + pyroscope_env: ci-smoke-ccipv1_6-evm-simulated + test_env_vars: + E2E_TEST_SELECTED_NETWORK: SIMULATED_1,SIMULATED_2 + E2E_JD_VERSION: 0.6.0 + E2E_RMN_RAGEPROXY_VERSION: master-f461a9e + E2E_RMN_AFN2PROXY_VERSION: master-f461a9e - id: smoke/ccip/ccip_rmn_test.go:^TestRMN_DifferentRmnNodesForDifferentChains$ path: integration-tests/smoke/ccip/ccip_rmn_test.go @@ -1093,6 +1090,35 @@ runner-test-matrix: E2E_RMN_RAGEPROXY_VERSION: master-f461a9e E2E_RMN_AFN2PROXY_VERSION: master-f461a9e + - id: smoke/ccip/ccip_rmn_test.go:^TestRMN_TwoMessagesOneSourceChainCursed$ + path: integration-tests/smoke/ccip/ccip_rmn_test.go + test_env_type: docker + runs_on: ubuntu-latest + triggers: + - PR E2E Core Tests + - Nightly E2E Tests + test_cmd: cd integration-tests/smoke/ccip && go test -test.run ^TestRMN_TwoMessagesOneSourceChainCursed$ -timeout 12m -test.parallel=1 -count=1 -json + pyroscope_env: ci-smoke-ccipv1_6-evm-simulated + test_env_vars: + E2E_TEST_SELECTED_NETWORK: SIMULATED_1,SIMULATED_2 + E2E_JD_VERSION: 0.6.0 + E2E_RMN_RAGEPROXY_VERSION: master-f461a9e + E2E_RMN_AFN2PROXY_VERSION: master-f461a9e + + - id: smoke/ccip/ccip_rmn_test.go:^TestRMN_GlobalCurseTwoMessagesOnTwoLanes$ + path: integration-tests/smoke/ccip/ccip_rmn_test.go + test_env_type: docker + runs_on: ubuntu-latest + triggers: + - PR E2E Core Tests + - Nightly E2E Tests + test_cmd: cd integration-tests/smoke/ccip && go test -test.run ^TestRMN_GlobalCurseTwoMessagesOnTwoLanes$ -timeout 12m -test.parallel=1 -count=1 -json + pyroscope_env: ci-smoke-ccipv1_6-evm-simulated + test_env_vars: + E2E_TEST_SELECTED_NETWORK: SIMULATED_1,SIMULATED_2 + E2E_JD_VERSION: 0.6.0 + E2E_RMN_RAGEPROXY_VERSION: master-f461a9e + E2E_RMN_AFN2PROXY_VERSION: master-f461a9e # END: CCIPv1.6 tests diff --git a/deployment/ccip/changeset/test_helpers.go b/deployment/ccip/changeset/test_helpers.go index b205463454e..f67d59517ce 100644 --- a/deployment/ccip/changeset/test_helpers.go +++ b/deployment/ccip/changeset/test_helpers.go @@ -33,11 +33,12 @@ import ( chainsel "github.com/smartcontractkit/chain-selectors" - "github.com/smartcontractkit/chainlink-ccip/pkg/reader" - cciptypes "github.com/smartcontractkit/chainlink-ccip/pkg/types/ccipocr3" jobv1 "github.com/smartcontractkit/chainlink-protos/job-distributor/v1/job" "github.com/smartcontractkit/chainlink-testing-framework/lib/utils/testcontext" + "github.com/smartcontractkit/chainlink-ccip/pkg/reader" + cciptypes "github.com/smartcontractkit/chainlink-ccip/pkg/types/ccipocr3" + "github.com/smartcontractkit/chainlink-common/pkg/logger" commonutils "github.com/smartcontractkit/chainlink-common/pkg/utils" "github.com/smartcontractkit/chainlink-common/pkg/utils/tests" diff --git a/integration-tests/go.mod b/integration-tests/go.mod index 371652fbed7..408780ebdf9 100644 --- a/integration-tests/go.mod +++ b/integration-tests/go.mod @@ -39,6 +39,7 @@ require ( github.com/smartcontractkit/chainlink-automation v0.8.1 github.com/smartcontractkit/chainlink-ccip v0.0.0-20241125151847-c63f5f567fcd github.com/smartcontractkit/chainlink-common v0.3.1-0.20241125150608-97ceadb2072d + github.com/smartcontractkit/chainlink-protos/job-distributor v0.6.0 github.com/smartcontractkit/chainlink-testing-framework/havoc v1.50.2 github.com/smartcontractkit/chainlink-testing-framework/lib v1.50.17 github.com/smartcontractkit/chainlink-testing-framework/lib/grafana v1.50.0 @@ -416,7 +417,6 @@ require ( github.com/smartcontractkit/chainlink-cosmos v0.5.2-0.20241017133723-5277829bd53f // indirect github.com/smartcontractkit/chainlink-data-streams v0.1.1-0.20241114154055-8d29ea018b57 // indirect github.com/smartcontractkit/chainlink-feeds v0.1.1 // indirect - github.com/smartcontractkit/chainlink-protos/job-distributor v0.6.0 // indirect github.com/smartcontractkit/chainlink-protos/orchestrator v0.3.0 // indirect github.com/smartcontractkit/chainlink-solana v1.1.1-0.20241127201057-3c9282e39749 // indirect github.com/smartcontractkit/chainlink-starknet/relayer v0.1.1-0.20241017135645-176a23722fd8 // indirect diff --git a/integration-tests/smoke/ccip/ccip_rmn_test.go b/integration-tests/smoke/ccip/ccip_rmn_test.go index 4083be1c6be..201982e19d1 100644 --- a/integration-tests/smoke/ccip/ccip_rmn_test.go +++ b/integration-tests/smoke/ccip/ccip_rmn_test.go @@ -1,9 +1,14 @@ package smoke import ( + "context" + "encoding/binary" + "errors" "math/big" "os" + "slices" "strconv" + "strings" "testing" "time" @@ -13,8 +18,11 @@ import ( "github.com/rs/zerolog" "github.com/stretchr/testify/require" + "github.com/smartcontractkit/chainlink-ccip/commit/merkleroot/rmn/types" + "github.com/smartcontractkit/chainlink-protos/job-distributor/v1/node" "github.com/smartcontractkit/chainlink-testing-framework/lib/utils/osutil" "github.com/smartcontractkit/chainlink-testing-framework/lib/utils/testcontext" + "github.com/smartcontractkit/chainlink/deployment/environment/devenv" "github.com/smartcontractkit/chainlink/deployment/ccip/changeset" @@ -75,7 +83,7 @@ func TestRMN_MultipleMessagesOnOneLaneNoWaitForExec(t *testing.T) { func TestRMN_NotEnoughObservers(t *testing.T) { runRmnTestCase(t, rmnTestCase{ name: "one message but not enough observers, should not get a commit report", - passIfNoCommitAfter: time.Minute, // wait for a minute and assert that commit report was not delivered + passIfNoCommitAfter: 15 * time.Second, homeChainConfig: homeChainConfig{ f: map[int]int{chain0: 1, chain1: 1}, }, @@ -121,7 +129,7 @@ func TestRMN_DifferentSigners(t *testing.T) { func TestRMN_NotEnoughSigners(t *testing.T) { runRmnTestCase(t, rmnTestCase{ name: "different signers and different observers", - passIfNoCommitAfter: time.Minute, // wait for a minute and assert that commit report was not delivered + passIfNoCommitAfter: 15 * time.Second, homeChainConfig: homeChainConfig{ f: map[int]int{chain0: 1, chain1: 1}, }, @@ -169,70 +177,77 @@ func TestRMN_DifferentRmnNodesForDifferentChains(t *testing.T) { }) } +func TestRMN_TwoMessagesOneSourceChainCursed(t *testing.T) { + runRmnTestCase(t, rmnTestCase{ + name: "two messages, one source chain is cursed", + passIfNoCommitAfter: 15 * time.Second, + cursedSubjectsPerChain: map[int][]int{ + chain1: {chain0}, + }, + homeChainConfig: homeChainConfig{ + f: map[int]int{chain0: 1, chain1: 1}, + }, + remoteChainsConfig: []remoteChainConfig{ + {chainIdx: chain0, f: 1}, + {chainIdx: chain1, f: 1}, + }, + rmnNodes: []rmnNode{ + {id: 0, isSigner: true, observedChainIdxs: []int{chain0, chain1}}, + {id: 1, isSigner: true, observedChainIdxs: []int{chain0, chain1}}, + {id: 2, isSigner: true, observedChainIdxs: []int{chain0, chain1}}, + }, + messagesToSend: []messageToSend{ + {fromChainIdx: chain0, toChainIdx: chain1, count: 1}, // <----- this message should not be committed + {fromChainIdx: chain1, toChainIdx: chain0, count: 1}, + }, + }) +} + +func TestRMN_GlobalCurseTwoMessagesOnTwoLanes(t *testing.T) { + runRmnTestCase(t, rmnTestCase{ + name: "global curse messages on two lanes", + waitForExec: false, + homeChainConfig: homeChainConfig{ + f: map[int]int{chain0: 1, chain1: 1}, + }, + remoteChainsConfig: []remoteChainConfig{ + {chainIdx: chain0, f: 1}, + {chainIdx: chain1, f: 1}, + }, + rmnNodes: []rmnNode{ + {id: 0, isSigner: true, observedChainIdxs: []int{chain0, chain1}}, + {id: 1, isSigner: true, observedChainIdxs: []int{chain0, chain1}}, + {id: 2, isSigner: true, observedChainIdxs: []int{chain0, chain1}}, + }, + messagesToSend: []messageToSend{ + {fromChainIdx: chain0, toChainIdx: chain1, count: 1}, + {fromChainIdx: chain1, toChainIdx: chain0, count: 5}, + }, + cursedSubjectsPerChain: map[int][]int{ + chain1: {globalCurse}, + chain0: {globalCurse}, + }, + passIfNoCommitAfter: 15 * time.Second, + }) +} + const ( - chain0 = 0 - chain1 = 1 + chain0 = 0 + chain1 = 1 + globalCurse = 1000 ) func runRmnTestCase(t *testing.T, tc rmnTestCase) { require.NoError(t, os.Setenv("ENABLE_RMN", "true")) + require.NoError(t, tc.validate()) + ctx := testcontext.Get(t) t.Logf("Running RMN test case: %s", tc.name) envWithRMN, rmnCluster := testsetups.NewLocalDevEnvironmentWithRMN(t, logger.TestLogger(t), len(tc.rmnNodes)) t.Logf("envWithRmn: %#v", envWithRMN) - var chainSelectors []uint64 - for _, chain := range envWithRMN.Env.Chains { - chainSelectors = append(chainSelectors, chain.Selector) - } - require.Greater(t, len(chainSelectors), 1, "There should be at least two chains") - - remoteChainSelectors := make([]uint64, 0, len(envWithRMN.Env.Chains)-1) - for _, chain := range envWithRMN.Env.Chains { - remoteChainSelectors = append(remoteChainSelectors, chain.Selector) - } - require.Greater(t, len(remoteChainSelectors), 0, "There should be at least one remote chain") - - var ( - rmnHomeNodes []rmn_home.RMNHomeNode - rmnRemoteSigners []rmn_remote.RMNRemoteSigner - ) - - for _, rmnNodeInfo := range tc.rmnNodes { - rmn := rmnCluster.Nodes["rmn_"+strconv.Itoa(rmnNodeInfo.id)] - - var offchainPublicKey [32]byte - copy(offchainPublicKey[:], rmn.RMN.OffchainPublicKey) - - rmnHomeNodes = append(rmnHomeNodes, rmn_home.RMNHomeNode{ - PeerId: rmn.Proxy.PeerID, - OffchainPublicKey: offchainPublicKey, - }) - - if rmnNodeInfo.isSigner { - if rmnNodeInfo.id < 0 { - t.Fatalf("node id is negative: %d", rmnNodeInfo.id) - } - rmnRemoteSigners = append(rmnRemoteSigners, rmn_remote.RMNRemoteSigner{ - OnchainPublicKey: rmn.RMN.EVMOnchainPublicKey, - NodeIndex: uint64(rmnNodeInfo.id), - }) - } - } - - var rmnHomeSourceChains []rmn_home.RMNHomeSourceChain - for remoteChainIdx, remoteF := range tc.homeChainConfig.f { - if remoteF < 0 { - t.Fatalf("negative remote F: %d", remoteF) - } - // configure remote chain details on the home contract - rmnHomeSourceChains = append(rmnHomeSourceChains, rmn_home.RMNHomeSourceChain{ - ChainSelector: chainSelectors[remoteChainIdx], - F: uint64(remoteF), - ObserverNodesBitmap: createObserverNodesBitmap(chainSelectors[remoteChainIdx], tc.rmnNodes, chainSelectors), - }) - } + tc.populateFields(t, envWithRMN, rmnCluster) onChainState, err := changeset.LoadOnchainState(envWithRMN.Env) require.NoError(t, err) @@ -244,22 +259,14 @@ func runRmnTestCase(t *testing.T, tc rmnTestCase) { homeChainState, ok := onChainState.Chains[envWithRMN.HomeChainSel] require.True(t, ok) - allDigests, err := homeChainState.RMNHome.GetConfigDigests(&bind.CallOpts{ - Context: testcontext.Get(t), - }) + allDigests, err := homeChainState.RMNHome.GetConfigDigests(&bind.CallOpts{Context: ctx}) require.NoError(t, err) t.Logf("RMNHome candidateDigest before setting new candidate: %x, activeDigest: %x", allDigests.CandidateConfigDigest[:], allDigests.ActiveConfigDigest[:]) - staticConfig := rmn_home.RMNHomeStaticConfig{ - Nodes: rmnHomeNodes, - OffchainConfig: []byte{}, - } - dynamicConfig := rmn_home.RMNHomeDynamicConfig{ - SourceChains: rmnHomeSourceChains, - OffchainConfig: []byte{}, - } + staticConfig := rmn_home.RMNHomeStaticConfig{Nodes: tc.pf.rmnHomeNodes, OffchainConfig: []byte{}} + dynamicConfig := rmn_home.RMNHomeDynamicConfig{SourceChains: tc.pf.rmnHomeSourceChains, OffchainConfig: []byte{}} t.Logf("Setting RMNHome candidate with staticConfig: %+v, dynamicConfig: %+v, current candidateDigest: %x", staticConfig, dynamicConfig, allDigests.CandidateConfigDigest[:]) tx, err := homeChainState.RMNHome.SetCandidate(homeChain.DeployerKey, staticConfig, dynamicConfig, allDigests.CandidateConfigDigest) @@ -268,9 +275,7 @@ func runRmnTestCase(t *testing.T, tc rmnTestCase) { _, err = deployment.ConfirmIfNoError(homeChain, tx, err) require.NoError(t, err) - candidateDigest, err := homeChainState.RMNHome.GetCandidateDigest(&bind.CallOpts{ - Context: testcontext.Get(t), - }) + candidateDigest, err := homeChainState.RMNHome.GetCandidateDigest(&bind.CallOpts{Context: ctx}) require.NoError(t, err) t.Logf("RMNHome candidateDigest after setting new candidate: %x", candidateDigest[:]) @@ -284,105 +289,70 @@ func runRmnTestCase(t *testing.T, tc rmnTestCase) { require.NoError(t, err) // check the active digest is the same as the candidate digest - activeDigest, err := homeChainState.RMNHome.GetActiveDigest(&bind.CallOpts{ - Context: testcontext.Get(t), - }) + activeDigest, err := homeChainState.RMNHome.GetActiveDigest(&bind.CallOpts{Context: ctx}) require.NoError(t, err) require.Equalf(t, candidateDigest, activeDigest, "active digest should be the same as the previously candidate digest after promotion, previous candidate: %x, active: %x", candidateDigest[:], activeDigest[:]) - // Set RMN remote config appropriately - for _, remoteCfg := range tc.remoteChainsConfig { - remoteSel := chainSelectors[remoteCfg.chainIdx] - chState, ok := onChainState.Chains[remoteSel] - require.True(t, ok) - if remoteCfg.f < 0 { - t.Fatalf("negative F: %d", remoteCfg.f) - } - rmnRemoteConfig := rmn_remote.RMNRemoteConfig{ - RmnHomeContractConfigDigest: activeDigest, - Signers: rmnRemoteSigners, - F: uint64(remoteCfg.f), - } + tc.setRmnRemoteConfig(ctx, t, onChainState, activeDigest, envWithRMN) - chain := envWithRMN.Env.Chains[chainSelectors[remoteCfg.chainIdx]] - - t.Logf("Setting RMNRemote config with RMNHome active digest: %x, cfg: %+v", activeDigest[:], rmnRemoteConfig) - tx2, err2 := chState.RMNRemote.SetConfig(chain.DeployerKey, rmnRemoteConfig) - require.NoError(t, err2) - _, err2 = deployment.ConfirmIfNoError(chain, tx2, err2) - require.NoError(t, err2) + tc.killMarkedRmnNodes(t, rmnCluster) - // confirm the config is set correctly - config, err2 := chState.RMNRemote.GetVersionedConfig(&bind.CallOpts{ - Context: testcontext.Get(t), - }) - require.NoError(t, err2) - require.Equalf(t, - activeDigest, - config.Config.RmnHomeContractConfigDigest, - "RMNRemote config digest should be the same as the active digest of RMNHome after setting, RMNHome active: %x, RMNRemote config: %x", - activeDigest[:], config.Config.RmnHomeContractConfigDigest[:]) + changeset.ReplayLogs(t, envWithRMN.Env.Offchain, envWithRMN.ReplayBlocks) + require.NoError(t, changeset.AddLanesForAll(envWithRMN.Env, onChainState)) + disabledNodes := tc.disableOraclesIfThisIsACursingTestCase(ctx, t, envWithRMN) - t.Logf("RMNRemote config digest after setting: %x", config.Config.RmnHomeContractConfigDigest[:]) - } + startBlocks, seqNumCommit, seqNumExec := tc.sendMessages(t, onChainState, envWithRMN) + t.Logf("Sent all messages, seqNumCommit: %v seqNumExec: %v", seqNumCommit, seqNumExec) - // Kill the RMN nodes that are marked for force exit - for _, n := range tc.rmnNodes { - if n.forceExit { - t.Logf("Pausing RMN node %d", n.id) - rmnN := rmnCluster.Nodes["rmn_"+strconv.Itoa(n.id)] - require.NoError(t, osutil.ExecCmd(zerolog.Nop(), "docker kill "+rmnN.Proxy.ContainerName)) - t.Logf("Paused RMN node %d", n.id) - } - } + tc.callContractsToCurseChains(ctx, t, onChainState, envWithRMN) - changeset.ReplayLogs(t, envWithRMN.Env.Offchain, envWithRMN.ReplayBlocks) - // Add all lanes - require.NoError(t, changeset.AddLanesForAll(envWithRMN.Env, onChainState)) + tc.enableOracles(ctx, t, envWithRMN, disabledNodes) - // Need to keep track of the block number for each chain so that event subscription can be done from that block. - startBlocks := make(map[uint64]*uint64) expectedSeqNum := make(map[changeset.SourceDestPair]uint64) - expectedSeqNumExec := make(map[changeset.SourceDestPair][]uint64) - for _, msg := range tc.messagesToSend { - fromChain := chainSelectors[msg.fromChainIdx] - toChain := chainSelectors[msg.toChainIdx] + for k, v := range seqNumCommit { + cursedSubjectsOfDest, exists := tc.pf.cursedSubjectsPerChainSel[k.DestChainSelector] + shouldSkip := exists && (slices.Contains(cursedSubjectsOfDest, globalCurse) || + slices.Contains(cursedSubjectsOfDest, k.SourceChainSelector)) - for i := 0; i < msg.count; i++ { - msgSentEvent := changeset.TestSendRequest(t, envWithRMN.Env, onChainState, fromChain, toChain, false, router.ClientEVM2AnyMessage{ - Receiver: common.LeftPadBytes(onChainState.Chains[toChain].Receiver.Address().Bytes(), 32), - Data: []byte("hello world"), - TokenAmounts: nil, - FeeToken: common.HexToAddress("0x0"), - ExtraArgs: nil, - }) - expectedSeqNum[changeset.SourceDestPair{ - SourceChainSelector: fromChain, - DestChainSelector: toChain, - }] = msgSentEvent.SequenceNumber - expectedSeqNumExec[changeset.SourceDestPair{ - SourceChainSelector: fromChain, - DestChainSelector: toChain, - }] = []uint64{msgSentEvent.SequenceNumber} - t.Logf("Sent message from chain %d to chain %d with seqNum %d", fromChain, toChain, msgSentEvent.SequenceNumber) + if !shouldSkip { + expectedSeqNum[k] = v } + } - zero := uint64(0) - startBlocks[toChain] = &zero + t.Logf("expectedSeqNums: %v", expectedSeqNum) + t.Logf("expectedSeqNums including cursed chains: %v", seqNumCommit) + + if len(tc.cursedSubjectsPerChain) > 0 && len(seqNumCommit) == len(expectedSeqNum) { + t.Fatalf("test case is wrong: no message was sent to non-cursed chains when you " + + "define curse subjects, your test case should have at least one message not expected to be delivered") } - t.Logf("Sent all messages, expectedSeqNum: %v", expectedSeqNum) commitReportReceived := make(chan struct{}) go func() { - changeset.ConfirmCommitForAllWithExpectedSeqNums(t, envWithRMN.Env, onChainState, expectedSeqNum, startBlocks) - commitReportReceived <- struct{}{} + if len(expectedSeqNum) > 0 { + changeset.ConfirmCommitForAllWithExpectedSeqNums(t, envWithRMN.Env, onChainState, expectedSeqNum, startBlocks) + commitReportReceived <- struct{}{} + } + + if len(seqNumCommit) > 0 && len(seqNumCommit) > len(expectedSeqNum) { + // wait for a duration and assert that commit reports were not delivered for cursed source chains + changeset.ConfirmCommitForAllWithExpectedSeqNums(t, envWithRMN.Env, onChainState, seqNumCommit, startBlocks) + commitReportReceived <- struct{}{} + } }() if tc.passIfNoCommitAfter > 0 { // wait for a duration and assert that commit reports were not delivered + if len(expectedSeqNum) > 0 && len(seqNumCommit) > len(expectedSeqNum) { + t.Logf("⌛ Waiting for commit reports of non-cursed chains...") + <-commitReportReceived + t.Logf("✅ Commit reports of non-cursed chains received") + } + tim := time.NewTimer(tc.passIfNoCommitAfter) t.Logf("waiting for %s before asserting that commit report was not received", tc.passIfNoCommitAfter) + select { case <-commitReportReceived: t.Errorf("Commit report was received while it was not expected") @@ -398,7 +368,7 @@ func runRmnTestCase(t *testing.T, tc rmnTestCase) { if tc.waitForExec { t.Logf("⌛ Waiting for exec reports...") - changeset.ConfirmExecWithSeqNrsForAll(t, envWithRMN.Env, onChainState, expectedSeqNumExec, startBlocks) + changeset.ConfirmExecWithSeqNrsForAll(t, envWithRMN.Env, onChainState, seqNumExec, startBlocks) t.Logf("✅ Exec report") } } @@ -447,10 +417,236 @@ type rmnTestCase struct { name string // If set to 0, the test will wait for commit reports. // If set to a positive value, the test will wait for that duration and will assert that commit report was not delivered. - passIfNoCommitAfter time.Duration - waitForExec bool - homeChainConfig homeChainConfig - remoteChainsConfig []remoteChainConfig - rmnNodes []rmnNode - messagesToSend []messageToSend + passIfNoCommitAfter time.Duration + cursedSubjectsPerChain map[int][]int + waitForExec bool + homeChainConfig homeChainConfig + remoteChainsConfig []remoteChainConfig + rmnNodes []rmnNode + messagesToSend []messageToSend + + // populated fields after environment setup + pf testCasePopulatedFields +} + +type testCasePopulatedFields struct { + chainSelectors []uint64 + rmnHomeNodes []rmn_home.RMNHomeNode + rmnRemoteSigners []rmn_remote.RMNRemoteSigner + rmnHomeSourceChains []rmn_home.RMNHomeSourceChain + cursedSubjectsPerChainSel map[uint64][]uint64 +} + +func (tc *rmnTestCase) populateFields(t *testing.T, envWithRMN changeset.DeployedEnv, rmnCluster devenv.RMNCluster) { + require.GreaterOrEqual(t, len(envWithRMN.Env.Chains), 2, "test assumes at least two chains") + for _, chain := range envWithRMN.Env.Chains { + tc.pf.chainSelectors = append(tc.pf.chainSelectors, chain.Selector) + } + + for _, rmnNodeInfo := range tc.rmnNodes { + rmn := rmnCluster.Nodes["rmn_"+strconv.Itoa(rmnNodeInfo.id)] + + var offchainPublicKey [32]byte + copy(offchainPublicKey[:], rmn.RMN.OffchainPublicKey) + + tc.pf.rmnHomeNodes = append(tc.pf.rmnHomeNodes, rmn_home.RMNHomeNode{ + PeerId: rmn.Proxy.PeerID, + OffchainPublicKey: offchainPublicKey, + }) + + if rmnNodeInfo.isSigner { + if rmnNodeInfo.id < 0 { + t.Fatalf("node id is negative: %d", rmnNodeInfo.id) + } + tc.pf.rmnRemoteSigners = append(tc.pf.rmnRemoteSigners, rmn_remote.RMNRemoteSigner{ + OnchainPublicKey: rmn.RMN.EVMOnchainPublicKey, + NodeIndex: uint64(rmnNodeInfo.id), + }) + } + } + + for remoteChainIdx, remoteF := range tc.homeChainConfig.f { + if remoteF < 0 { + t.Fatalf("negative remote F: %d", remoteF) + } + // configure remote chain details on the home contract + tc.pf.rmnHomeSourceChains = append(tc.pf.rmnHomeSourceChains, rmn_home.RMNHomeSourceChain{ + ChainSelector: tc.pf.chainSelectors[remoteChainIdx], + F: uint64(remoteF), + ObserverNodesBitmap: createObserverNodesBitmap(tc.pf.chainSelectors[remoteChainIdx], tc.rmnNodes, tc.pf.chainSelectors), + }) + } + + // populate cursed subjects with actual chain selectors + tc.pf.cursedSubjectsPerChainSel = make(map[uint64][]uint64) + for chainIdx, subjects := range tc.cursedSubjectsPerChain { + chainSel := tc.pf.chainSelectors[chainIdx] + for _, subject := range subjects { + subjSel := uint64(globalCurse) + if subject != globalCurse { + subjSel = tc.pf.chainSelectors[subject] + } + tc.pf.cursedSubjectsPerChainSel[chainSel] = append(tc.pf.cursedSubjectsPerChainSel[chainSel], subjSel) + } + } +} + +func (tc rmnTestCase) validate() error { + if len(tc.cursedSubjectsPerChain) > 0 && tc.passIfNoCommitAfter == 0 { + return errors.New("when you define cursed subjects you also need to define the duration that the " + + "test will wait for non-transmitted roots") + } + return nil +} + +func (tc rmnTestCase) setRmnRemoteConfig( + ctx context.Context, + t *testing.T, + onChainState changeset.CCIPOnChainState, + activeDigest [32]byte, + envWithRMN changeset.DeployedEnv) { + for _, remoteCfg := range tc.remoteChainsConfig { + remoteSel := tc.pf.chainSelectors[remoteCfg.chainIdx] + chState, ok := onChainState.Chains[remoteSel] + require.True(t, ok) + if remoteCfg.f < 0 { + t.Fatalf("negative F: %d", remoteCfg.f) + } + rmnRemoteConfig := rmn_remote.RMNRemoteConfig{ + RmnHomeContractConfigDigest: activeDigest, + Signers: tc.pf.rmnRemoteSigners, + F: uint64(remoteCfg.f), + } + + chain := envWithRMN.Env.Chains[tc.pf.chainSelectors[remoteCfg.chainIdx]] + + t.Logf("Setting RMNRemote config with RMNHome active digest: %x, cfg: %+v", activeDigest[:], rmnRemoteConfig) + tx2, err2 := chState.RMNRemote.SetConfig(chain.DeployerKey, rmnRemoteConfig) + require.NoError(t, err2) + _, err2 = deployment.ConfirmIfNoError(chain, tx2, err2) + require.NoError(t, err2) + + // confirm the config is set correctly + config, err2 := chState.RMNRemote.GetVersionedConfig(&bind.CallOpts{Context: ctx}) + require.NoError(t, err2) + require.Equalf(t, + activeDigest, + config.Config.RmnHomeContractConfigDigest, + "RMNRemote config digest should be the same as the active digest of RMNHome after setting, RMNHome active: %x, RMNRemote config: %x", + activeDigest[:], config.Config.RmnHomeContractConfigDigest[:]) + + t.Logf("RMNRemote config digest after setting: %x", config.Config.RmnHomeContractConfigDigest[:]) + } +} + +func (tc rmnTestCase) killMarkedRmnNodes(t *testing.T, rmnCluster devenv.RMNCluster) { + for _, n := range tc.rmnNodes { + if n.forceExit { + t.Logf("Pausing RMN node %d", n.id) + rmnN := rmnCluster.Nodes["rmn_"+strconv.Itoa(n.id)] + require.NoError(t, osutil.ExecCmd(zerolog.Nop(), "docker kill "+rmnN.Proxy.ContainerName)) + t.Logf("Paused RMN node %d", n.id) + } + } +} + +func (tc rmnTestCase) disableOraclesIfThisIsACursingTestCase(ctx context.Context, t *testing.T, envWithRMN changeset.DeployedEnv) []string { + disabledNodes := make([]string, 0) + + if len(tc.cursedSubjectsPerChain) > 0 { + listNodesResp, err := envWithRMN.Env.Offchain.ListNodes(ctx, &node.ListNodesRequest{}) + require.NoError(t, err) + + for _, n := range listNodesResp.Nodes { + if strings.HasPrefix(n.Name, "bootstrap") { + continue + } + _, err := envWithRMN.Env.Offchain.DisableNode(ctx, &node.DisableNodeRequest{Id: n.Id}) + require.NoError(t, err) + disabledNodes = append(disabledNodes, n.Id) + t.Logf("node %s disabled", n.Id) + } + } + + return disabledNodes +} + +func (tc rmnTestCase) sendMessages(t *testing.T, onChainState changeset.CCIPOnChainState, envWithRMN changeset.DeployedEnv) (map[uint64]*uint64, map[changeset.SourceDestPair]uint64, map[changeset.SourceDestPair][]uint64) { + startBlocks := make(map[uint64]*uint64) + seqNumCommit := make(map[changeset.SourceDestPair]uint64) + seqNumExec := make(map[changeset.SourceDestPair][]uint64) + + for _, msg := range tc.messagesToSend { + fromChain := tc.pf.chainSelectors[msg.fromChainIdx] + toChain := tc.pf.chainSelectors[msg.toChainIdx] + + for i := 0; i < msg.count; i++ { + msgSentEvent := changeset.TestSendRequest(t, envWithRMN.Env, onChainState, fromChain, toChain, false, router.ClientEVM2AnyMessage{ + Receiver: common.LeftPadBytes(onChainState.Chains[toChain].Receiver.Address().Bytes(), 32), + Data: []byte("hello world"), + TokenAmounts: nil, + FeeToken: common.HexToAddress("0x0"), + ExtraArgs: nil, + }) + seqNumCommit[changeset.SourceDestPair{ + SourceChainSelector: fromChain, + DestChainSelector: toChain, + }] = msgSentEvent.SequenceNumber + seqNumExec[changeset.SourceDestPair{ + SourceChainSelector: fromChain, + DestChainSelector: toChain, + }] = []uint64{msgSentEvent.SequenceNumber} + t.Logf("Sent message from chain %d to chain %d with seqNum %d", fromChain, toChain, msgSentEvent.SequenceNumber) + } + + zero := uint64(0) + startBlocks[toChain] = &zero + } + + return startBlocks, seqNumCommit, seqNumExec +} + +func (tc rmnTestCase) callContractsToCurseChains(ctx context.Context, t *testing.T, onChainState changeset.CCIPOnChainState, envWithRMN changeset.DeployedEnv) { + for _, remoteCfg := range tc.remoteChainsConfig { + remoteSel := tc.pf.chainSelectors[remoteCfg.chainIdx] + chState, ok := onChainState.Chains[remoteSel] + require.True(t, ok) + chain, ok := envWithRMN.Env.Chains[remoteSel] + require.True(t, ok) + + cursedSubjects, ok := tc.cursedSubjectsPerChain[remoteCfg.chainIdx] + if !ok { + continue // nothing to curse on this chain + } + + for _, subjectDescription := range cursedSubjects { + subj := types.GlobalCurseSubject + if subjectDescription != globalCurse { + subj = chainSelectorToBytes16(tc.pf.chainSelectors[subjectDescription]) + } + t.Logf("cursing subject %d (%d)", subj, subjectDescription) + txCurse, errCurse := chState.RMNRemote.Curse(chain.DeployerKey, subj) + _, errConfirm := deployment.ConfirmIfNoError(chain, txCurse, errCurse) + require.NoError(t, errConfirm) + } + + cs, err := chState.RMNRemote.GetCursedSubjects(&bind.CallOpts{Context: ctx}) + require.NoError(t, err) + t.Logf("Cursed subjects: %v", cs) + } +} + +func (tc rmnTestCase) enableOracles(ctx context.Context, t *testing.T, envWithRMN changeset.DeployedEnv, nodeIDs []string) { + for _, n := range nodeIDs { + _, err := envWithRMN.Env.Offchain.EnableNode(ctx, &node.EnableNodeRequest{Id: n}) + require.NoError(t, err) + t.Logf("node %s enabled", n) + } +} + +func chainSelectorToBytes16(chainSel uint64) [16]byte { + var result [16]byte + // Convert the uint64 to bytes and place it in the last 8 bytes of the array + binary.BigEndian.PutUint64(result[8:], chainSel) + return result }