[TT-1179] scan cl logs to check plugin activation (#13247)

* add support for median plugin in OCR2 tests

* add support for median plugin in OCR2 tests

* scan CL node logs to check whether plugins were activated

* use tagged CTF version

integration-tests/go.mod

@@ -28,7 +28,7 @@ require (
 	github.com/slack-go/slack v0.12.2
 	github.com/smartcontractkit/chainlink-automation v1.0.3
 	github.com/smartcontractkit/chainlink-common v0.1.7-0.20240517134904-f4446b816a28
-	github.com/smartcontractkit/chainlink-testing-framework v1.28.12
+	github.com/smartcontractkit/chainlink-testing-framework v1.28.14
 	github.com/smartcontractkit/chainlink-vrf v0.0.0-20231120191722-fef03814f868
 	github.com/smartcontractkit/chainlink/v2 v2.0.0-00010101000000-000000000000
 	github.com/smartcontractkit/libocr v0.0.0-20240419185742-fd3cab206b2c

integration-tests/go.sum

@@ -1524,8 +1524,8 @@ github.com/smartcontractkit/chainlink-solana v1.0.3-0.20240510181707-46b1311a5a8
 github.com/smartcontractkit/chainlink-solana v1.0.3-0.20240510181707-46b1311a5a83/go.mod h1:RdAtOeBUWq2zByw2kEbwPlXaPIb7YlaDOmnn+nVUBJI=
 github.com/smartcontractkit/chainlink-starknet/relayer v0.0.1-beta-test.0.20240508155030-1024f2b55c69 h1:ssh/w3oXWu+C6bE88GuFRC1+0Bx/4ihsbc80XMLrl2k=
 github.com/smartcontractkit/chainlink-starknet/relayer v0.0.1-beta-test.0.20240508155030-1024f2b55c69/go.mod h1:VsfjhvWgjxqWja4q+FlXEtX5lu8BSxn10xRo6gi948g=
-github.com/smartcontractkit/chainlink-testing-framework v1.28.12 h1:15ssos9DvWekvj6JjmiPjTYsj/uw12HvTWlm1FHdYaA=
-github.com/smartcontractkit/chainlink-testing-framework v1.28.12/go.mod h1:x1zDOz8zcLjEvs9fNA9y/DMguLam/2+CJdpxX0+rM8A=
+github.com/smartcontractkit/chainlink-testing-framework v1.28.14 h1:LChhbd/dJWf+yainZ/mjbfu6XP8IbGQ9S64MDTg+tt4=
+github.com/smartcontractkit/chainlink-testing-framework v1.28.14/go.mod h1:x1zDOz8zcLjEvs9fNA9y/DMguLam/2+CJdpxX0+rM8A=
 github.com/smartcontractkit/chainlink-testing-framework/grafana v0.0.0-20240328204215-ac91f55f1449 h1:fX/xmGm1GBsD1ZZnooNT+eWA0hiTAqFlHzOC5CY4dy8=
 github.com/smartcontractkit/chainlink-testing-framework/grafana v0.0.0-20240328204215-ac91f55f1449/go.mod h1:DC8sQMyTlI/44UCTL8QWFwb0bYNoXCfjwCv2hMivYZU=
 github.com/smartcontractkit/chainlink-vrf v0.0.0-20231120191722-fef03814f868 h1:FFdvEzlYwcuVHkdZ8YnZR/XomeMGbz5E2F2HZI3I3w8=

integration-tests/load/go.mod

@@ -17,7 +17,7 @@ require (
 	github.com/slack-go/slack v0.12.2
 	github.com/smartcontractkit/chainlink-automation v1.0.3
 	github.com/smartcontractkit/chainlink-common v0.1.7-0.20240517134904-f4446b816a28
-	github.com/smartcontractkit/chainlink-testing-framework v1.28.12
+	github.com/smartcontractkit/chainlink-testing-framework v1.28.14
 	github.com/smartcontractkit/chainlink/integration-tests v0.0.0-20240214231432-4ad5eb95178c
 	github.com/smartcontractkit/chainlink/v2 v2.9.0-beta0.0.20240216210048-da02459ddad8
 	github.com/smartcontractkit/libocr v0.0.0-20240419185742-fd3cab206b2c

integration-tests/load/go.sum

@@ -1514,8 +1514,8 @@ github.com/smartcontractkit/chainlink-solana v1.0.3-0.20240510181707-46b1311a5a8
 github.com/smartcontractkit/chainlink-solana v1.0.3-0.20240510181707-46b1311a5a83/go.mod h1:RdAtOeBUWq2zByw2kEbwPlXaPIb7YlaDOmnn+nVUBJI=
 github.com/smartcontractkit/chainlink-starknet/relayer v0.0.1-beta-test.0.20240508155030-1024f2b55c69 h1:ssh/w3oXWu+C6bE88GuFRC1+0Bx/4ihsbc80XMLrl2k=
 github.com/smartcontractkit/chainlink-starknet/relayer v0.0.1-beta-test.0.20240508155030-1024f2b55c69/go.mod h1:VsfjhvWgjxqWja4q+FlXEtX5lu8BSxn10xRo6gi948g=
-github.com/smartcontractkit/chainlink-testing-framework v1.28.12 h1:15ssos9DvWekvj6JjmiPjTYsj/uw12HvTWlm1FHdYaA=
-github.com/smartcontractkit/chainlink-testing-framework v1.28.12/go.mod h1:x1zDOz8zcLjEvs9fNA9y/DMguLam/2+CJdpxX0+rM8A=
+github.com/smartcontractkit/chainlink-testing-framework v1.28.14 h1:LChhbd/dJWf+yainZ/mjbfu6XP8IbGQ9S64MDTg+tt4=
+github.com/smartcontractkit/chainlink-testing-framework v1.28.14/go.mod h1:x1zDOz8zcLjEvs9fNA9y/DMguLam/2+CJdpxX0+rM8A=
 github.com/smartcontractkit/chainlink-testing-framework/grafana v0.0.0-20240328204215-ac91f55f1449 h1:fX/xmGm1GBsD1ZZnooNT+eWA0hiTAqFlHzOC5CY4dy8=
 github.com/smartcontractkit/chainlink-testing-framework/grafana v0.0.0-20240328204215-ac91f55f1449/go.mod h1:DC8sQMyTlI/44UCTL8QWFwb0bYNoXCfjwCv2hMivYZU=
 github.com/smartcontractkit/chainlink-vrf v0.0.0-20231120191722-fef03814f868 h1:FFdvEzlYwcuVHkdZ8YnZR/XomeMGbz5E2F2HZI3I3w8=

integration-tests/smoke/ocr2_test.go

@@ -4,6 +4,7 @@ import (
 	"fmt"
 	"math/big"
 	"net/http"
+	"strings"
 	"testing"
 	"time"
 
@@ -13,6 +14,7 @@ import (
 	"github.com/stretchr/testify/require"
 
 	"github.com/smartcontractkit/chainlink-testing-framework/logging"
+	"github.com/smartcontractkit/chainlink-testing-framework/logstream"
 	"github.com/smartcontractkit/chainlink-testing-framework/networks"
 	"github.com/smartcontractkit/chainlink-testing-framework/utils/testcontext"
 	"github.com/smartcontractkit/chainlink/v2/core/config/env"
@@ -56,9 +58,9 @@ func TestOCRv2Basic(t *testing.T) {
 			t.Parallel()
 			l := logging.GetTestLogger(t)
 
-			env, aggregatorContracts, sethClient := prepareORCv2SmokeTestEnv(t, test, l, 5)
+			testEnv, aggregatorContracts, sethClient := prepareORCv2SmokeTestEnv(t, test, l, 5)
 
-			err := env.MockAdapter.SetAdapterBasedIntValuePath("ocr2", []string{http.MethodGet, http.MethodPost}, 10)
+			err := testEnv.MockAdapter.SetAdapterBasedIntValuePath("ocr2", []string{http.MethodGet, http.MethodPost}, 10)
 			require.NoError(t, err)
 			err = actions_seth.WatchNewOCRRound(l, sethClient, 2, contracts.V2OffChainAgrregatorToOffChainAggregatorWithRounds(aggregatorContracts), time.Minute*5)
 			require.NoError(t, err)
@@ -145,7 +147,9 @@ func prepareORCv2SmokeTestEnv(t *testing.T, testData ocr2test, l zerolog.Logger,
 	privateNetwork, err := actions.EthereumNetworkConfigFromConfig(l, &config)
 	require.NoError(t, err, "Error building ethereum network config")
 
-	env, err := test_env.NewCLTestEnvBuilder().
+	clNodeCount := 6
+
+	testEnv, err := test_env.NewCLTestEnvBuilder().
 		WithTestInstance(t).
 		WithTestConfig(&config).
 		WithPrivateEthereumNetwork(privateNetwork.EthereumNetworkConfig).
@@ -156,18 +160,18 @@ func prepareORCv2SmokeTestEnv(t *testing.T, testData ocr2test, l zerolog.Logger,
 			node.WithTracing(),
 		)).
 		WithCLNodeOptions(test_env.WithNodeEnvVars(testData.env)).
-		WithCLNodes(6).
+		WithCLNodes(clNodeCount).
 		WithFunding(big.NewFloat(.1)).
 		WithStandardCleanup().
 		WithSeth().
 		Build()
 	require.NoError(t, err)
 
 	selectedNetwork := networks.MustGetSelectedNetworkConfig(config.Network)[0]
-	sethClient, err := env.GetSethClient(selectedNetwork.ChainID)
+	sethClient, err := testEnv.GetSethClient(selectedNetwork.ChainID)
 	require.NoError(t, err, "Error getting seth client")
 
-	nodeClients := env.ClCluster.NodeAPIs()
+	nodeClients := testEnv.ClCluster.NodeAPIs()
 	bootstrapNode, workerNodes := nodeClients[0], nodeClients[1:]
 
 	linkContract, err := contracts.DeployLinkTokenContract(l, sethClient)
@@ -190,7 +194,7 @@ func prepareORCv2SmokeTestEnv(t *testing.T, testData ocr2test, l zerolog.Logger,
 	aggregatorContracts, err := actions_seth.DeployOCRv2Contracts(l, sethClient, 1, common.HexToAddress(linkContract.Address()), transmitters, ocrOffchainOptions)
 	require.NoError(t, err, "Error deploying OCRv2 aggregator contracts")
 
-	err = actions.CreateOCRv2JobsLocal(aggregatorContracts, bootstrapNode, workerNodes, env.MockAdapter, "ocr2", 5, uint64(sethClient.ChainID), false, testData.chainReaderAndCodec)
+	err = actions.CreateOCRv2JobsLocal(aggregatorContracts, bootstrapNode, workerNodes, testEnv.MockAdapter, "ocr2", 5, uint64(sethClient.ChainID), false, testData.chainReaderAndCodec)
 	require.NoError(t, err, "Error creating OCRv2 jobs")
 
 	ocrv2Config, err := actions.BuildMedianOCR2ConfigLocal(workerNodes, ocrOffchainOptions)
@@ -199,6 +203,8 @@ func prepareORCv2SmokeTestEnv(t *testing.T, testData ocr2test, l zerolog.Logger,
 	err = actions_seth.ConfigureOCRv2AggregatorContracts(ocrv2Config, aggregatorContracts)
 	require.NoError(t, err, "Error configuring OCRv2 aggregator contracts")
 
+	assertCorrectNodeConfiguration(t, l, clNodeCount, testData, testEnv)
+
 	err = actions_seth.WatchNewOCRRound(l, sethClient, 1, contracts.V2OffChainAgrregatorToOffChainAggregatorWithRounds(aggregatorContracts), time.Minute*5)
 	require.NoError(t, err, "Error watching for new OCR2 round")
 	roundData, err := aggregatorContracts[0].GetRound(testcontext.Get(t), big.NewInt(1))
@@ -208,5 +214,37 @@ func prepareORCv2SmokeTestEnv(t *testing.T, testData ocr2test, l zerolog.Logger,
 		roundData.Answer.Int64(),
 	)
 
-	return env, aggregatorContracts, sethClient
+	return testEnv, aggregatorContracts, sethClient
+}
+
+func assertCorrectNodeConfiguration(t *testing.T, l zerolog.Logger, totalNodeCount int, testData ocr2test, testEnv *test_env.CLClusterTestEnv) {
+	expectedNodesWithConfiguration := totalNodeCount - 1 // minus bootstrap node
+	expectedPatterns := []string{}
+
+	if testData.env[string(env.MedianPlugin.Cmd)] != "" {
+		expectedPatterns = append(expectedPatterns, "Registered loopp.*OCR2.*Median.*")
+	}
+
+	if testData.chainReaderAndCodec {
+		expectedPatterns = append(expectedPatterns, "relayConfig\\.chainReader")
+	} else {
+		expectedPatterns = append(expectedPatterns, "ChainReader missing from RelayConfig; falling back to internal MedianContract")
+	}
+
+	// make sure that nodes are correctly configured by scanning the logs
+	for _, pattern := range expectedPatterns {
+		l.Info().Msgf("Checking for pattern: '%s' in CL node logs", pattern)
+		var correctlyConfiguredNodes []string
+		for i := 1; i < len(testEnv.ClCluster.Nodes); i++ {
+			logProcessor, processFn, err := logstream.GetRegexMatchingProcessor(testEnv.LogStream, pattern)
+			require.NoError(t, err, "Error getting regex matching processor")
+
+			count, err := logProcessor.ProcessContainerLogs(testEnv.ClCluster.Nodes[i].ContainerName, processFn)
+			require.NoError(t, err, "Error processing container logs")
+			if *count >= 1 {
+				correctlyConfiguredNodes = append(correctlyConfiguredNodes, testEnv.ClCluster.Nodes[i].ContainerName)
+			}
+		}
+		require.Equal(t, expectedNodesWithConfiguration, len(correctlyConfiguredNodes), "expected correct plugin config to be applied to %d cl-nodes, but only following ones had it: %s; regexp used: %s", expectedNodesWithConfiguration, strings.Join(correctlyConfiguredNodes, ", "), string(pattern))
+	}
 }