From 0ecef68e698bd7387254c5171a40c6279d5cb344 Mon Sep 17 00:00:00 2001 From: Bartek Tofel Date: Mon, 20 May 2024 13:35:34 +0200 Subject: [PATCH] [TT-1179] scan cl logs to check plugin activation (#13247) * add support for median plugin in OCR2 tests * add support for median plugin in OCR2 tests * scan CL node logs to check whether plugins were activated * use tagged CTF version --- integration-tests/go.mod | 2 +- integration-tests/go.sum | 4 +-- integration-tests/load/go.mod | 2 +- integration-tests/load/go.sum | 4 +-- integration-tests/smoke/ocr2_test.go | 54 +++++++++++++++++++++++----- 5 files changed, 52 insertions(+), 14 deletions(-) diff --git a/integration-tests/go.mod b/integration-tests/go.mod index 35325dbcb9a..9355e8007bc 100644 --- a/integration-tests/go.mod +++ b/integration-tests/go.mod @@ -28,7 +28,7 @@ require ( github.com/slack-go/slack v0.12.2 github.com/smartcontractkit/chainlink-automation v1.0.3 github.com/smartcontractkit/chainlink-common v0.1.7-0.20240517134904-f4446b816a28 - github.com/smartcontractkit/chainlink-testing-framework v1.28.12 + github.com/smartcontractkit/chainlink-testing-framework v1.28.14 github.com/smartcontractkit/chainlink-vrf v0.0.0-20231120191722-fef03814f868 github.com/smartcontractkit/chainlink/v2 v2.0.0-00010101000000-000000000000 github.com/smartcontractkit/libocr v0.0.0-20240419185742-fd3cab206b2c diff --git a/integration-tests/go.sum b/integration-tests/go.sum index 0fc4df6d613..5fc82cbff5d 100644 --- a/integration-tests/go.sum +++ b/integration-tests/go.sum @@ -1524,8 +1524,8 @@ github.com/smartcontractkit/chainlink-solana v1.0.3-0.20240510181707-46b1311a5a8 github.com/smartcontractkit/chainlink-solana v1.0.3-0.20240510181707-46b1311a5a83/go.mod h1:RdAtOeBUWq2zByw2kEbwPlXaPIb7YlaDOmnn+nVUBJI= github.com/smartcontractkit/chainlink-starknet/relayer v0.0.1-beta-test.0.20240508155030-1024f2b55c69 h1:ssh/w3oXWu+C6bE88GuFRC1+0Bx/4ihsbc80XMLrl2k= github.com/smartcontractkit/chainlink-starknet/relayer v0.0.1-beta-test.0.20240508155030-1024f2b55c69/go.mod h1:VsfjhvWgjxqWja4q+FlXEtX5lu8BSxn10xRo6gi948g= -github.com/smartcontractkit/chainlink-testing-framework v1.28.12 h1:15ssos9DvWekvj6JjmiPjTYsj/uw12HvTWlm1FHdYaA= -github.com/smartcontractkit/chainlink-testing-framework v1.28.12/go.mod h1:x1zDOz8zcLjEvs9fNA9y/DMguLam/2+CJdpxX0+rM8A= +github.com/smartcontractkit/chainlink-testing-framework v1.28.14 h1:LChhbd/dJWf+yainZ/mjbfu6XP8IbGQ9S64MDTg+tt4= +github.com/smartcontractkit/chainlink-testing-framework v1.28.14/go.mod h1:x1zDOz8zcLjEvs9fNA9y/DMguLam/2+CJdpxX0+rM8A= github.com/smartcontractkit/chainlink-testing-framework/grafana v0.0.0-20240328204215-ac91f55f1449 h1:fX/xmGm1GBsD1ZZnooNT+eWA0hiTAqFlHzOC5CY4dy8= github.com/smartcontractkit/chainlink-testing-framework/grafana v0.0.0-20240328204215-ac91f55f1449/go.mod h1:DC8sQMyTlI/44UCTL8QWFwb0bYNoXCfjwCv2hMivYZU= github.com/smartcontractkit/chainlink-vrf v0.0.0-20231120191722-fef03814f868 h1:FFdvEzlYwcuVHkdZ8YnZR/XomeMGbz5E2F2HZI3I3w8= diff --git a/integration-tests/load/go.mod b/integration-tests/load/go.mod index f9289bf1290..1e51191034c 100644 --- a/integration-tests/load/go.mod +++ b/integration-tests/load/go.mod @@ -17,7 +17,7 @@ require ( github.com/slack-go/slack v0.12.2 github.com/smartcontractkit/chainlink-automation v1.0.3 github.com/smartcontractkit/chainlink-common v0.1.7-0.20240517134904-f4446b816a28 - github.com/smartcontractkit/chainlink-testing-framework v1.28.12 + github.com/smartcontractkit/chainlink-testing-framework v1.28.14 github.com/smartcontractkit/chainlink/integration-tests v0.0.0-20240214231432-4ad5eb95178c github.com/smartcontractkit/chainlink/v2 v2.9.0-beta0.0.20240216210048-da02459ddad8 github.com/smartcontractkit/libocr v0.0.0-20240419185742-fd3cab206b2c diff --git a/integration-tests/load/go.sum b/integration-tests/load/go.sum index 860b0d61519..9542f355788 100644 --- a/integration-tests/load/go.sum +++ b/integration-tests/load/go.sum @@ -1514,8 +1514,8 @@ github.com/smartcontractkit/chainlink-solana v1.0.3-0.20240510181707-46b1311a5a8 github.com/smartcontractkit/chainlink-solana v1.0.3-0.20240510181707-46b1311a5a83/go.mod h1:RdAtOeBUWq2zByw2kEbwPlXaPIb7YlaDOmnn+nVUBJI= github.com/smartcontractkit/chainlink-starknet/relayer v0.0.1-beta-test.0.20240508155030-1024f2b55c69 h1:ssh/w3oXWu+C6bE88GuFRC1+0Bx/4ihsbc80XMLrl2k= github.com/smartcontractkit/chainlink-starknet/relayer v0.0.1-beta-test.0.20240508155030-1024f2b55c69/go.mod h1:VsfjhvWgjxqWja4q+FlXEtX5lu8BSxn10xRo6gi948g= -github.com/smartcontractkit/chainlink-testing-framework v1.28.12 h1:15ssos9DvWekvj6JjmiPjTYsj/uw12HvTWlm1FHdYaA= -github.com/smartcontractkit/chainlink-testing-framework v1.28.12/go.mod h1:x1zDOz8zcLjEvs9fNA9y/DMguLam/2+CJdpxX0+rM8A= +github.com/smartcontractkit/chainlink-testing-framework v1.28.14 h1:LChhbd/dJWf+yainZ/mjbfu6XP8IbGQ9S64MDTg+tt4= +github.com/smartcontractkit/chainlink-testing-framework v1.28.14/go.mod h1:x1zDOz8zcLjEvs9fNA9y/DMguLam/2+CJdpxX0+rM8A= github.com/smartcontractkit/chainlink-testing-framework/grafana v0.0.0-20240328204215-ac91f55f1449 h1:fX/xmGm1GBsD1ZZnooNT+eWA0hiTAqFlHzOC5CY4dy8= github.com/smartcontractkit/chainlink-testing-framework/grafana v0.0.0-20240328204215-ac91f55f1449/go.mod h1:DC8sQMyTlI/44UCTL8QWFwb0bYNoXCfjwCv2hMivYZU= github.com/smartcontractkit/chainlink-vrf v0.0.0-20231120191722-fef03814f868 h1:FFdvEzlYwcuVHkdZ8YnZR/XomeMGbz5E2F2HZI3I3w8= diff --git a/integration-tests/smoke/ocr2_test.go b/integration-tests/smoke/ocr2_test.go index 19bf4a5aa07..ba9cb5f5775 100644 --- a/integration-tests/smoke/ocr2_test.go +++ b/integration-tests/smoke/ocr2_test.go @@ -4,6 +4,7 @@ import ( "fmt" "math/big" "net/http" + "strings" "testing" "time" @@ -13,6 +14,7 @@ import ( "github.com/stretchr/testify/require" "github.com/smartcontractkit/chainlink-testing-framework/logging" + "github.com/smartcontractkit/chainlink-testing-framework/logstream" "github.com/smartcontractkit/chainlink-testing-framework/networks" "github.com/smartcontractkit/chainlink-testing-framework/utils/testcontext" "github.com/smartcontractkit/chainlink/v2/core/config/env" @@ -56,9 +58,9 @@ func TestOCRv2Basic(t *testing.T) { t.Parallel() l := logging.GetTestLogger(t) - env, aggregatorContracts, sethClient := prepareORCv2SmokeTestEnv(t, test, l, 5) + testEnv, aggregatorContracts, sethClient := prepareORCv2SmokeTestEnv(t, test, l, 5) - err := env.MockAdapter.SetAdapterBasedIntValuePath("ocr2", []string{http.MethodGet, http.MethodPost}, 10) + err := testEnv.MockAdapter.SetAdapterBasedIntValuePath("ocr2", []string{http.MethodGet, http.MethodPost}, 10) require.NoError(t, err) err = actions_seth.WatchNewOCRRound(l, sethClient, 2, contracts.V2OffChainAgrregatorToOffChainAggregatorWithRounds(aggregatorContracts), time.Minute*5) require.NoError(t, err) @@ -145,7 +147,9 @@ func prepareORCv2SmokeTestEnv(t *testing.T, testData ocr2test, l zerolog.Logger, privateNetwork, err := actions.EthereumNetworkConfigFromConfig(l, &config) require.NoError(t, err, "Error building ethereum network config") - env, err := test_env.NewCLTestEnvBuilder(). + clNodeCount := 6 + + testEnv, err := test_env.NewCLTestEnvBuilder(). WithTestInstance(t). WithTestConfig(&config). WithPrivateEthereumNetwork(privateNetwork.EthereumNetworkConfig). @@ -156,7 +160,7 @@ func prepareORCv2SmokeTestEnv(t *testing.T, testData ocr2test, l zerolog.Logger, node.WithTracing(), )). WithCLNodeOptions(test_env.WithNodeEnvVars(testData.env)). - WithCLNodes(6). + WithCLNodes(clNodeCount). WithFunding(big.NewFloat(.1)). WithStandardCleanup(). WithSeth(). @@ -164,10 +168,10 @@ func prepareORCv2SmokeTestEnv(t *testing.T, testData ocr2test, l zerolog.Logger, require.NoError(t, err) selectedNetwork := networks.MustGetSelectedNetworkConfig(config.Network)[0] - sethClient, err := env.GetSethClient(selectedNetwork.ChainID) + sethClient, err := testEnv.GetSethClient(selectedNetwork.ChainID) require.NoError(t, err, "Error getting seth client") - nodeClients := env.ClCluster.NodeAPIs() + nodeClients := testEnv.ClCluster.NodeAPIs() bootstrapNode, workerNodes := nodeClients[0], nodeClients[1:] linkContract, err := contracts.DeployLinkTokenContract(l, sethClient) @@ -190,7 +194,7 @@ func prepareORCv2SmokeTestEnv(t *testing.T, testData ocr2test, l zerolog.Logger, aggregatorContracts, err := actions_seth.DeployOCRv2Contracts(l, sethClient, 1, common.HexToAddress(linkContract.Address()), transmitters, ocrOffchainOptions) require.NoError(t, err, "Error deploying OCRv2 aggregator contracts") - err = actions.CreateOCRv2JobsLocal(aggregatorContracts, bootstrapNode, workerNodes, env.MockAdapter, "ocr2", 5, uint64(sethClient.ChainID), false, testData.chainReaderAndCodec) + err = actions.CreateOCRv2JobsLocal(aggregatorContracts, bootstrapNode, workerNodes, testEnv.MockAdapter, "ocr2", 5, uint64(sethClient.ChainID), false, testData.chainReaderAndCodec) require.NoError(t, err, "Error creating OCRv2 jobs") ocrv2Config, err := actions.BuildMedianOCR2ConfigLocal(workerNodes, ocrOffchainOptions) @@ -199,6 +203,8 @@ func prepareORCv2SmokeTestEnv(t *testing.T, testData ocr2test, l zerolog.Logger, err = actions_seth.ConfigureOCRv2AggregatorContracts(ocrv2Config, aggregatorContracts) require.NoError(t, err, "Error configuring OCRv2 aggregator contracts") + assertCorrectNodeConfiguration(t, l, clNodeCount, testData, testEnv) + err = actions_seth.WatchNewOCRRound(l, sethClient, 1, contracts.V2OffChainAgrregatorToOffChainAggregatorWithRounds(aggregatorContracts), time.Minute*5) require.NoError(t, err, "Error watching for new OCR2 round") roundData, err := aggregatorContracts[0].GetRound(testcontext.Get(t), big.NewInt(1)) @@ -208,5 +214,37 @@ func prepareORCv2SmokeTestEnv(t *testing.T, testData ocr2test, l zerolog.Logger, roundData.Answer.Int64(), ) - return env, aggregatorContracts, sethClient + return testEnv, aggregatorContracts, sethClient +} + +func assertCorrectNodeConfiguration(t *testing.T, l zerolog.Logger, totalNodeCount int, testData ocr2test, testEnv *test_env.CLClusterTestEnv) { + expectedNodesWithConfiguration := totalNodeCount - 1 // minus bootstrap node + expectedPatterns := []string{} + + if testData.env[string(env.MedianPlugin.Cmd)] != "" { + expectedPatterns = append(expectedPatterns, "Registered loopp.*OCR2.*Median.*") + } + + if testData.chainReaderAndCodec { + expectedPatterns = append(expectedPatterns, "relayConfig\\.chainReader") + } else { + expectedPatterns = append(expectedPatterns, "ChainReader missing from RelayConfig; falling back to internal MedianContract") + } + + // make sure that nodes are correctly configured by scanning the logs + for _, pattern := range expectedPatterns { + l.Info().Msgf("Checking for pattern: '%s' in CL node logs", pattern) + var correctlyConfiguredNodes []string + for i := 1; i < len(testEnv.ClCluster.Nodes); i++ { + logProcessor, processFn, err := logstream.GetRegexMatchingProcessor(testEnv.LogStream, pattern) + require.NoError(t, err, "Error getting regex matching processor") + + count, err := logProcessor.ProcessContainerLogs(testEnv.ClCluster.Nodes[i].ContainerName, processFn) + require.NoError(t, err, "Error processing container logs") + if *count >= 1 { + correctlyConfiguredNodes = append(correctlyConfiguredNodes, testEnv.ClCluster.Nodes[i].ContainerName) + } + } + require.Equal(t, expectedNodesWithConfiguration, len(correctlyConfiguredNodes), "expected correct plugin config to be applied to %d cl-nodes, but only following ones had it: %s; regexp used: %s", expectedNodesWithConfiguration, strings.Join(correctlyConfiguredNodes, ", "), string(pattern)) + } }