[TT-1094] add support to compat pipeline for reth

.github/workflows/client-compatibility-tests.yml

@@ -14,10 +14,10 @@ on:
         required: false
         type: string
       evmImplementations:
-        description: comma separated list of EVM implementations to test (ignored if base64TestList is used)
+        description: comma separated list of EVM implementations to test (ignored if base64TestList is used); supports geth,besu,nethermind,erigon,reth
         required: true
         type: string
-        default: "geth,besu,nethermind,erigon"
+        default: "geth,besu,nethermind,erigon,reth"
       latestVersionsNumber:
         description: how many of latest images of EVM implementations to test with (ignored if base64TestList is used)
         required: true
@@ -155,6 +155,12 @@ jobs:
               echo "New nethermind release found: $new_nethermind"
               implementations_arr+=("nethermind")
             fi
+            new_reth=$(ghlatestreleasechecker "paradigmxyz/reth" $RELEASED_DAYS_AGO)
+            if [ "new_reth" != "none" ]; then
+              echo "New reth release found: $new_reth"
+              implementations_arr+=("reth")
+            fi
+
             IFS=','
             eth_implementations="${implementations_arr[*]}"
             if [ -n "$eth_implementations" ]; then
@@ -172,7 +178,7 @@ jobs:
             fi
           else 
             echo "Will test all EVM implementations"
-            echo "evm_implementations=geth,besu,nethermind,erigon" >> $GITHUB_OUTPUT
+            echo "evm_implementations=geth,besu,nethermind,erigon,reth" >> $GITHUB_OUTPUT
           fi
       - name: Select Chainlink version
         id: select-chainlink-version
@@ -302,7 +308,7 @@ jobs:
 
   get-latest-available-images:
     name: Get Latest EVM Implementation's Images
-    if: always() && needs.should-run.outputs.should_run == 'true' && (needs.select-versions.outputs.evm_implementations != '' || github.event.inputs.base64TestList != '')
+    if: always() && needs.should-run.outputs.should_run == 'true' && needs.select-versions.outputs.evm_implementations != '' && github.event.inputs.base64TestList == ''
     environment: integration
     runs-on: ubuntu-latest
     needs: [check-ecr-images-exist, should-run, select-versions]
@@ -316,6 +322,7 @@ jobs:
       nethermind_images: ${{ env.NETHERMIND_IMAGES }}
       besu_images: ${{ env.BESU_IMAGES }}
       erigon_images: ${{ env.ERIGON_IMAGES }}
+      reth_images: ${{ env.RETH_IMAGES }}
     steps:
       # Setup AWS creds
       - name: Configure AWS Credentials
@@ -370,6 +377,12 @@ jobs:
             echo "ERIGON_IMAGES=$erigon_images" >> $GITHUB_ENV
             echo "Erigon latest images: $erigon_images" 
           fi
+
+          if [[ "$ETH_IMPLEMENTATIONS" == *"reth"* ]]; then
+            reth_images=$(ecrimagefetcher 'ghcr.io/paradigmxyz/reth' '^v[0-9]+\.[0-9]+\.[0-9]+$' ${{ env.LATEST_IMAGE_COUNT }})
+            echo "RETH_IMAGES=$reth_images" >> $GITHUB_ENV
+            echo "Reth latest images: $reth_images" 
+          fi          
 
   # End Build Test Dependencies
 
@@ -499,6 +512,23 @@ jobs:
             echo "Will not test compatibility with nethermind"
           fi
 
+          if [[ "$ETH_IMPLEMENTATIONS" == *"reth"* ]]; then
+            echo "Will test compatibility with reth"
+            testlistgenerator -o compatibility_test_list.json -p cron -r TestCronBasic -f './smoke/cron_test.go' -e reth -d "${{ needs.get-latest-available-images.outputs.reth_images }}" -t "evm-implementation-compatibility-test" -n "ubuntu-latest"
+            testlistgenerator -o compatibility_test_list.json -p flux -r TestFluxBasic -f './smoke/flux_test.go' -e reth -d "${{ needs.get-latest-available-images.outputs.reth_images }}" -t "evm-implementation-compatibility-test" -n "ubuntu-latest"
+            testlistgenerator -o compatibility_test_list.json -p runlog -r TestRunLogBasic -f './smoke/runlog_test.go' -e reth -d "${{ needs.get-latest-available-images.outputs.reth_images }}" -t "evm-implementation-compatibility-test" -n "ubuntu-latest"
+            testlistgenerator -o compatibility_test_list.json -p log_poller -r TestLogPollerFewFiltersFixedDepth -f './smoke/log_poller_test.go' -e reth -d "${{ needs.get-latest-available-images.outputs.reth_images }}" -t "evm-implementation-compatibility-test" -n "ubuntu-latest"
+            testlistgenerator -o compatibility_test_list.json -p ocr -r TestOCRBasic -f './smoke/ocr_test.go' -e reth -d "${{ needs.get-latest-available-images.outputs.reth_images }}" -t "evm-implementation-compatibility-test" -n "ubuntu-latest"
+            testlistgenerator -o compatibility_test_list.json -p ocr2 -r '^TestOCRv2Basic/plugins$' -f './smoke/ocr2_test.go' -e reth -d "${{ needs.get-latest-available-images.outputs.reth_images }}" -t "evm-implementation-compatibility-test" -n "ubuntu-latest"
+            testlistgenerator -o compatibility_test_list.json -p automation -r 'TestAutomationBasic/registry_2_1_logtrigger' -f './smoke/automation_test.go' -e reth -d "${{ needs.get-latest-available-images.outputs.reth_images }}" -t "evm-implementation-compatibility-test" -n "ubuntu-latest"
+            testlistgenerator -o compatibility_test_list.json -p keeper -r 'TestKeeperBasicSmoke/registry_1_3' -f './smoke/keeper_test.go' -e reth -d "${{ needs.get-latest-available-images.outputs.reth_images }}" -t "evm-implementation-compatibility-test" -n "ubuntu-latest"
+            testlistgenerator -o compatibility_test_list.json -p vrf -r '^TestVRFBasic/Request_Randomness$' -f './smoke/vrf_test.go' -e reth -d "${{ needs.get-latest-available-images.outputs.reth_images }}" -t "evm-implementation-compatibility-test" -n "ubuntu-latest"
+            testlistgenerator -o compatibility_test_list.json -p vrfv2 -r '^TestVRFv2Basic/Request_Randomness$' -f './smoke/vrfv2_test.go' -e reth -d "${{ needs.get-latest-available-images.outputs.reth_images }}" -t "evm-implementation-compatibility-test" -n "ubuntu-latest"
+            testlistgenerator -o compatibility_test_list.json -p vrfv2plus -r '^TestVRFv2Plus$/^Link_Billing$' -f './smoke/vrfv2plus_test.go' -e reth -d "${{ needs.get-latest-available-images.outputs.reth_images }}" -t "evm-implementation-compatibility-test" -n "ubuntu-latest"
+          else
+            echo "Will not test compatibility with reth"
+          fi          
+
           jq . compatibility_test_list.json
           JOB_MATRIX_JSON=$(jq -c . compatibility_test_list.json)
           echo "JOB_MATRIX_JSON=${JOB_MATRIX_JSON}" >> $GITHUB_ENV           

integration-tests/docker/test_env/test_env_builder.go

@@ -280,6 +280,10 @@ func (b *CLTestEnvBuilder) Build() (*CLClusterTestEnv, error) {
 					// new logs can be added to the log stream, so parallel processing would get stuck on waiting for it to be unlocked
 				LogScanningLoop:
 					for i := 0; i < b.clNodesCount; i++ {
+						// if something went wrong during environment setup we might not have all nodes, and we don't want an NPE
+						if b.te.ClCluster == nil || b.te.ClCluster.Nodes == nil || len(b.te.ClCluster.Nodes) < i || b.te.ClCluster.Nodes[i] == nil {
+							continue
+						}
 						// ignore count return, because we are only interested in the error
 						_, err := logProcessor.ProcessContainerLogs(b.te.ClCluster.Nodes[i].ContainerName, processFn)
 						if err != nil && !strings.Contains(err.Error(), testreporters.MultipleLogsAtLogLevelErr) && !strings.Contains(err.Error(), testreporters.OneLogAtLogLevelErr) {

integration-tests/go.mod

@@ -29,7 +29,7 @@ require (
 	github.com/slack-go/slack v0.12.2
 	github.com/smartcontractkit/chainlink-automation v1.0.4
 	github.com/smartcontractkit/chainlink-common v0.1.7-0.20240703234618-dc1fbe45acc0
-	github.com/smartcontractkit/chainlink-testing-framework v1.31.7
+	github.com/smartcontractkit/chainlink-testing-framework v1.31.9-0.20240708163738-0050e3999514
 	github.com/smartcontractkit/chainlink-testing-framework/grafana v0.0.0-20240405215812-5a72bc9af239
 	github.com/smartcontractkit/chainlink/v2 v2.0.0-00010101000000-000000000000
 	github.com/smartcontractkit/havoc/k8schaos v0.0.0-20240409145249-e78d20847e37

integration-tests/go.sum

@@ -1524,8 +1524,8 @@ github.com/smartcontractkit/chainlink-solana v1.0.3-0.20240708130426-294b81e4afe
 github.com/smartcontractkit/chainlink-solana v1.0.3-0.20240708130426-294b81e4afe7/go.mod h1:NbXXQaNFskVMYRut0MvBlcHu/vDgipGMwYjamvjVB9Y=
 github.com/smartcontractkit/chainlink-starknet/relayer v0.0.1-beta-test.0.20240625074951-06ab5e670dba h1:YNSlhK5mobyAaw02LPGgIEuS3lXyCTXcc6oaV2L6uUI=
 github.com/smartcontractkit/chainlink-starknet/relayer v0.0.1-beta-test.0.20240625074951-06ab5e670dba/go.mod h1:UVFRacRkP7O7TQAzFmR52v5mUlxf+G1ovMlCQAB/cHU=
-github.com/smartcontractkit/chainlink-testing-framework v1.31.7 h1:Vy4ah8VAfj+Y7vVmhjvwyAO6wG+Fp2vzdkSJwJPMQO4=
-github.com/smartcontractkit/chainlink-testing-framework v1.31.7/go.mod h1:E6uNEZhZZid9PHv6/Kq5Vn63GlO61ZcKB+/f0DKo3Q4=
+github.com/smartcontractkit/chainlink-testing-framework v1.31.9-0.20240708163738-0050e3999514 h1:S8ysLZWmTLRA0N+O6Su+7JVB5awcVsFcVAPfqc4QZBw=
+github.com/smartcontractkit/chainlink-testing-framework v1.31.9-0.20240708163738-0050e3999514/go.mod h1:E6uNEZhZZid9PHv6/Kq5Vn63GlO61ZcKB+/f0DKo3Q4=
 github.com/smartcontractkit/chainlink-testing-framework/grafana v0.0.0-20240405215812-5a72bc9af239 h1:Kk5OVlx/5g9q3Z3lhxytZS4/f8ds1MiNM8yaHgK3Oe8=
 github.com/smartcontractkit/chainlink-testing-framework/grafana v0.0.0-20240405215812-5a72bc9af239/go.mod h1:DC8sQMyTlI/44UCTL8QWFwb0bYNoXCfjwCv2hMivYZU=
 github.com/smartcontractkit/go-plugin v0.0.0-20240208201424-b3b91517de16 h1:TFe+FvzxClblt6qRfqEhUfa4kFQx5UobuoFGO2W4mMo=

integration-tests/load/go.mod

@@ -17,7 +17,7 @@ require (
 	github.com/slack-go/slack v0.12.2
 	github.com/smartcontractkit/chainlink-automation v1.0.4
 	github.com/smartcontractkit/chainlink-common v0.1.7-0.20240703234618-dc1fbe45acc0
-	github.com/smartcontractkit/chainlink-testing-framework v1.31.7
+	github.com/smartcontractkit/chainlink-testing-framework v1.31.9-0.20240708163738-0050e3999514
 	github.com/smartcontractkit/chainlink/integration-tests v0.0.0-20240214231432-4ad5eb95178c
 	github.com/smartcontractkit/chainlink/v2 v2.9.0-beta0.0.20240216210048-da02459ddad8
 	github.com/smartcontractkit/libocr v0.0.0-20240702141926-063ceef8c42e

integration-tests/load/go.sum

@@ -1514,8 +1514,8 @@ github.com/smartcontractkit/chainlink-solana v1.0.3-0.20240708130426-294b81e4afe
 github.com/smartcontractkit/chainlink-solana v1.0.3-0.20240708130426-294b81e4afe7/go.mod h1:NbXXQaNFskVMYRut0MvBlcHu/vDgipGMwYjamvjVB9Y=
 github.com/smartcontractkit/chainlink-starknet/relayer v0.0.1-beta-test.0.20240625074951-06ab5e670dba h1:YNSlhK5mobyAaw02LPGgIEuS3lXyCTXcc6oaV2L6uUI=
 github.com/smartcontractkit/chainlink-starknet/relayer v0.0.1-beta-test.0.20240625074951-06ab5e670dba/go.mod h1:UVFRacRkP7O7TQAzFmR52v5mUlxf+G1ovMlCQAB/cHU=
-github.com/smartcontractkit/chainlink-testing-framework v1.31.7 h1:Vy4ah8VAfj+Y7vVmhjvwyAO6wG+Fp2vzdkSJwJPMQO4=
-github.com/smartcontractkit/chainlink-testing-framework v1.31.7/go.mod h1:E6uNEZhZZid9PHv6/Kq5Vn63GlO61ZcKB+/f0DKo3Q4=
+github.com/smartcontractkit/chainlink-testing-framework v1.31.9-0.20240708163738-0050e3999514 h1:S8ysLZWmTLRA0N+O6Su+7JVB5awcVsFcVAPfqc4QZBw=
+github.com/smartcontractkit/chainlink-testing-framework v1.31.9-0.20240708163738-0050e3999514/go.mod h1:E6uNEZhZZid9PHv6/Kq5Vn63GlO61ZcKB+/f0DKo3Q4=
 github.com/smartcontractkit/chainlink-testing-framework/grafana v0.0.0-20240405215812-5a72bc9af239 h1:Kk5OVlx/5g9q3Z3lhxytZS4/f8ds1MiNM8yaHgK3Oe8=
 github.com/smartcontractkit/chainlink-testing-framework/grafana v0.0.0-20240405215812-5a72bc9af239/go.mod h1:DC8sQMyTlI/44UCTL8QWFwb0bYNoXCfjwCv2hMivYZU=
 github.com/smartcontractkit/go-plugin v0.0.0-20240208201424-b3b91517de16 h1:TFe+FvzxClblt6qRfqEhUfa4kFQx5UobuoFGO2W4mMo=