From 9647852fe2399ad19ee376b7ee5ef6d24c6aca59 Mon Sep 17 00:00:00 2001 From: Lukasz <120112546+lukaszcl@users.noreply.github.com> Date: Tue, 11 Jun 2024 23:59:48 +0200 Subject: [PATCH] Add test for reorg above finality (#13513) * Add test for reorg above finality * Fix * Bump CTF * Fix replace * Fix goimports --- integration-tests/docker/test_env/cl_node.go | 14 +++- integration-tests/go.mod | 2 +- integration-tests/go.sum | 4 +- integration-tests/load/go.mod | 2 +- integration-tests/load/go.sum | 4 +- .../smoke/reorg_above_finality_test.go | 83 +++++++++++++++++++ 6 files changed, 101 insertions(+), 8 deletions(-) create mode 100644 integration-tests/smoke/reorg_above_finality_test.go diff --git a/integration-tests/docker/test_env/cl_node.go b/integration-tests/docker/test_env/cl_node.go index 90b237ac6a1..73e22c58ab3 100644 --- a/integration-tests/docker/test_env/cl_node.go +++ b/integration-tests/docker/test_env/cl_node.go @@ -352,6 +352,8 @@ func (n *ClNode) containerStartOrRestart(restartDb bool) error { Str("userEmail", n.UserEmail). Str("userPassword", n.UserPassword). Msg("Started Chainlink Node container") + nodeConfig, _ := n.GetNodeConfigStr() + n.l.Info().Str("containerName", n.ContainerName).Msgf("Chainlink Node config:\n%s", nodeConfig) clClient, err := client.NewChainlinkClient(&client.ChainlinkConfig{ URL: clEndpoint, Email: n.UserEmail, @@ -399,17 +401,25 @@ func (n *ClNode) ExecGetVersion() (string, error) { return "", errors.Errorf("could not find chainlink version in command output '%'", output) } +func (n ClNode) GetNodeConfigStr() (string, error) { + data, err := toml.Marshal(n.NodeConfig) + if err != nil { + return "", err + } + return string(data), nil +} + func (n *ClNode) getContainerRequest(secrets string) ( *tc.ContainerRequest, error) { configFile, err := os.CreateTemp("", "node_config") if err != nil { return nil, err } - data, err := toml.Marshal(n.NodeConfig) + configStr, err := n.GetNodeConfigStr() if err != nil { return nil, err } - _, err = configFile.WriteString(string(data)) + _, err = configFile.WriteString(configStr) if err != nil { return nil, err } diff --git a/integration-tests/go.mod b/integration-tests/go.mod index b7758a599c3..25598610c6b 100644 --- a/integration-tests/go.mod +++ b/integration-tests/go.mod @@ -28,7 +28,7 @@ require ( github.com/slack-go/slack v0.12.2 github.com/smartcontractkit/chainlink-automation v1.0.4 github.com/smartcontractkit/chainlink-common v0.1.7-0.20240611144925-2baf0f2a3fef - github.com/smartcontractkit/chainlink-testing-framework v1.30.3 + github.com/smartcontractkit/chainlink-testing-framework v1.30.4 github.com/smartcontractkit/chainlink-vrf v0.0.0-20231120191722-fef03814f868 github.com/smartcontractkit/chainlink/v2 v2.0.0-00010101000000-000000000000 github.com/smartcontractkit/libocr v0.0.0-20240419185742-fd3cab206b2c diff --git a/integration-tests/go.sum b/integration-tests/go.sum index a8ac8556222..376233fc0d2 100644 --- a/integration-tests/go.sum +++ b/integration-tests/go.sum @@ -1524,8 +1524,8 @@ github.com/smartcontractkit/chainlink-solana v1.0.3-0.20240605170242-555ff582f36 github.com/smartcontractkit/chainlink-solana v1.0.3-0.20240605170242-555ff582f36a/go.mod h1:QqcZSwLgEIn7YraAIRmomnBMAuVFephiHrIWVlkWbFI= github.com/smartcontractkit/chainlink-starknet/relayer v0.0.1-beta-test.0.20240531021326-99118e47f696 h1:h1E87+z+JcUEfvbJVF56SnZA/YUFE5ewUE61MaR/Ewg= github.com/smartcontractkit/chainlink-starknet/relayer v0.0.1-beta-test.0.20240531021326-99118e47f696/go.mod h1:OiWUTrrpSLLTMh7FINWjEh6mmDJCVPaC4yEsDCVaWdU= -github.com/smartcontractkit/chainlink-testing-framework v1.30.3 h1:3mL0dBI/+8nTSh9GdxYyhNIAPTjSBThPGJnjcjjDaIg= -github.com/smartcontractkit/chainlink-testing-framework v1.30.3/go.mod h1:E6uNEZhZZid9PHv6/Kq5Vn63GlO61ZcKB+/f0DKo3Q4= +github.com/smartcontractkit/chainlink-testing-framework v1.30.4 h1:kf6zRL6v5D047gynYNNqXGl9QBvnQSa4LMs1iHLRu64= +github.com/smartcontractkit/chainlink-testing-framework v1.30.4/go.mod h1:E6uNEZhZZid9PHv6/Kq5Vn63GlO61ZcKB+/f0DKo3Q4= github.com/smartcontractkit/chainlink-testing-framework/grafana v0.0.0-20240328204215-ac91f55f1449 h1:fX/xmGm1GBsD1ZZnooNT+eWA0hiTAqFlHzOC5CY4dy8= github.com/smartcontractkit/chainlink-testing-framework/grafana v0.0.0-20240328204215-ac91f55f1449/go.mod h1:DC8sQMyTlI/44UCTL8QWFwb0bYNoXCfjwCv2hMivYZU= github.com/smartcontractkit/chainlink-vrf v0.0.0-20231120191722-fef03814f868 h1:FFdvEzlYwcuVHkdZ8YnZR/XomeMGbz5E2F2HZI3I3w8= diff --git a/integration-tests/load/go.mod b/integration-tests/load/go.mod index 08ab4475078..8f70870bcb4 100644 --- a/integration-tests/load/go.mod +++ b/integration-tests/load/go.mod @@ -17,7 +17,7 @@ require ( github.com/slack-go/slack v0.12.2 github.com/smartcontractkit/chainlink-automation v1.0.4 github.com/smartcontractkit/chainlink-common v0.1.7-0.20240611144925-2baf0f2a3fef - github.com/smartcontractkit/chainlink-testing-framework v1.30.3 + github.com/smartcontractkit/chainlink-testing-framework v1.30.4 github.com/smartcontractkit/chainlink/integration-tests v0.0.0-20240214231432-4ad5eb95178c github.com/smartcontractkit/chainlink/v2 v2.9.0-beta0.0.20240216210048-da02459ddad8 github.com/smartcontractkit/libocr v0.0.0-20240419185742-fd3cab206b2c diff --git a/integration-tests/load/go.sum b/integration-tests/load/go.sum index 85ae0860ae2..832c83cb3e6 100644 --- a/integration-tests/load/go.sum +++ b/integration-tests/load/go.sum @@ -1514,8 +1514,8 @@ github.com/smartcontractkit/chainlink-solana v1.0.3-0.20240605170242-555ff582f36 github.com/smartcontractkit/chainlink-solana v1.0.3-0.20240605170242-555ff582f36a/go.mod h1:QqcZSwLgEIn7YraAIRmomnBMAuVFephiHrIWVlkWbFI= github.com/smartcontractkit/chainlink-starknet/relayer v0.0.1-beta-test.0.20240531021326-99118e47f696 h1:h1E87+z+JcUEfvbJVF56SnZA/YUFE5ewUE61MaR/Ewg= github.com/smartcontractkit/chainlink-starknet/relayer v0.0.1-beta-test.0.20240531021326-99118e47f696/go.mod h1:OiWUTrrpSLLTMh7FINWjEh6mmDJCVPaC4yEsDCVaWdU= -github.com/smartcontractkit/chainlink-testing-framework v1.30.3 h1:3mL0dBI/+8nTSh9GdxYyhNIAPTjSBThPGJnjcjjDaIg= -github.com/smartcontractkit/chainlink-testing-framework v1.30.3/go.mod h1:E6uNEZhZZid9PHv6/Kq5Vn63GlO61ZcKB+/f0DKo3Q4= +github.com/smartcontractkit/chainlink-testing-framework v1.30.4 h1:kf6zRL6v5D047gynYNNqXGl9QBvnQSa4LMs1iHLRu64= +github.com/smartcontractkit/chainlink-testing-framework v1.30.4/go.mod h1:E6uNEZhZZid9PHv6/Kq5Vn63GlO61ZcKB+/f0DKo3Q4= github.com/smartcontractkit/chainlink-testing-framework/grafana v0.0.0-20240328204215-ac91f55f1449 h1:fX/xmGm1GBsD1ZZnooNT+eWA0hiTAqFlHzOC5CY4dy8= github.com/smartcontractkit/chainlink-testing-framework/grafana v0.0.0-20240328204215-ac91f55f1449/go.mod h1:DC8sQMyTlI/44UCTL8QWFwb0bYNoXCfjwCv2hMivYZU= github.com/smartcontractkit/chainlink-vrf v0.0.0-20231120191722-fef03814f868 h1:FFdvEzlYwcuVHkdZ8YnZR/XomeMGbz5E2F2HZI3I3w8= diff --git a/integration-tests/smoke/reorg_above_finality_test.go b/integration-tests/smoke/reorg_above_finality_test.go new file mode 100644 index 00000000000..26e909e84e8 --- /dev/null +++ b/integration-tests/smoke/reorg_above_finality_test.go @@ -0,0 +1,83 @@ +package smoke + +import ( + "math/big" + "testing" + "time" + + ctf_client "github.com/smartcontractkit/chainlink-testing-framework/client" + + "github.com/stretchr/testify/require" + + "github.com/smartcontractkit/chainlink-testing-framework/logging" + "github.com/smartcontractkit/chainlink/integration-tests/actions" + "github.com/smartcontractkit/chainlink/integration-tests/docker/test_env" + tc "github.com/smartcontractkit/chainlink/integration-tests/testconfig" +) + +func TestReorgAboveFinality(t *testing.T) { + t.Parallel() + + l := logging.GetTestLogger(t) + config, err := tc.GetConfig("Smoke", tc.OCR2) + require.NoError(t, err, "Error getting config") + + privateNetworkConf, err := actions.EthereumNetworkConfigFromConfig(l, &config) + require.NoError(t, err) + + nodeFinalityDepthInt := int64(10) + + testEnv, err := test_env.NewCLTestEnvBuilder(). + WithTestInstance(t). + WithTestConfig(&config). + WithPrivateEthereumNetwork(privateNetworkConf.EthereumNetworkConfig). + WithMockAdapter(). + WithCLNodes(6). + WithFunding(big.NewFloat(*config.Common.ChainlinkNodeFunding)). + WithoutCleanup(). + WithSeth(). + Build() + require.NoError(t, err) + + network := testEnv.EVMNetworks[0] + client := ctf_client.NewRPCClient(network.HTTPURLs[0]) + + // Wait for chain to progress + targetBlockNumber := nodeFinalityDepthInt * 3 + require.Eventually(t, func() bool { + bn, err := client.BlockNumber() + require.NoError(t, err) + l.Info().Int64("blockNumber", bn).Int64("targetBlockNumber", targetBlockNumber).Msg("Waiting for chain to progress above target block number") + return bn > nodeFinalityDepthInt*3 + }, 3*time.Minute, 3*time.Second, "chain did not progress above the target block number") + + // Run reorg above finality depth + reorgDepth := int(nodeFinalityDepthInt) + 20 + l.Info(). + Str("URL", client.URL). + Int64("nodeFinalityDepth", nodeFinalityDepthInt). + Int("reorgDepth", reorgDepth). + Msg("Starting blockchain reorg on Simulated Geth chain") + err = client.GethSetHead(reorgDepth) + require.NoError(t, err, "Error starting blockchain reorg on Simulated Geth chain") + + l.Info().Msg("Waiting for all nodes to report finality violation") + nodes := testEnv.ClCluster.NodeAPIs() + require.Eventually(t, func() bool { + violatedResponses := 0 + for _, node := range nodes { + resp, _, err := node.Health() + require.NoError(t, err) + for _, d := range resp.Data { + if d.Attributes.Name == "EVM.1337.LogPoller" && d.Attributes.Output == "finality violated" && d.Attributes.Status == "failing" { + violatedResponses++ + } + } + l.Info().Msgf("Resp: %v", resp) + } + + l.Info().Int("violatedResponses", violatedResponses).Int("nodes", len(nodes)).Msg("Checking if all nodes reported finality violation") + return violatedResponses == len(nodes) + }, 3*time.Minute, 5*time.Second, "not all the nodes report finality violation") + l.Info().Msg("All nodes reported finality violation") +}