From 8e21ca5c4d274b4afa039e1fcb7374b35b5e61c0 Mon Sep 17 00:00:00 2001
From: lukaszcl <120112546+lukaszcl@users.noreply.github.com>
Date: Fri, 21 Jun 2024 17:48:16 +0200
Subject: [PATCH 001/137] wip nightly pipeline

add nightly pipeline
add check tool that will be executed in PRs if a test from integration-tests is added or modified. It will report a comment or block pipeline requiring test to be added to the nightly workflow
---
 .../check-e2e-tests-in-nightly-workflow.go    | 79 +++++++++++++++++++
 .github/workflows/nightly-e2e-tests.yml       | 40 ++++++++++
 2 files changed, 119 insertions(+)
 create mode 100644 .github/scripts/check-e2e-tests-in-nightly-workflow.go
 create mode 100644 .github/workflows/nightly-e2e-tests.yml

diff --git a/.github/scripts/check-e2e-tests-in-nightly-workflow.go b/.github/scripts/check-e2e-tests-in-nightly-workflow.go
new file mode 100644
index 00000000000..a629c296cef
--- /dev/null
+++ b/.github/scripts/check-e2e-tests-in-nightly-workflow.go
@@ -0,0 +1,79 @@
+package main
+
+import (
+	"fmt"
+	"io/ioutil"
+	"os"
+	"strings"
+
+	"gopkg.in/yaml.v3"
+)
+
+// JobConfig represents the structure of the jobs in the YAML file.
+type JobConfig struct {
+	Jobs map[string]struct {
+		Strategy struct {
+			Matrix struct {
+				Test []struct {
+					Path     string `yaml:"path"`
+					TestOpts string `yaml:"testOpts"`
+				} `yaml:"test"`
+			} `yaml:"matrix"`
+		} `yaml:"strategy"`
+	} `yaml:"jobs"`
+}
+
+func main() {
+	// Check command-line arguments
+	if len(os.Args) != 4 {
+		fmt.Println("Usage: go run check_test_in_pipeline.go <yaml file> <test name> <test file path>")
+		os.Exit(1)
+	}
+	filename := os.Args[1]
+	testName := os.Args[2]
+	testFilePath := os.Args[3]
+
+	// Read the YAML file
+	data, err := ioutil.ReadFile(filename)
+	if err != nil {
+		fmt.Printf("Error reading YAML file: %s\n", err)
+		os.Exit(1)
+	}
+
+	// Parse the YAML
+	var config JobConfig
+	err = yaml.Unmarshal(data, &config)
+	if err != nil {
+		fmt.Printf("Error parsing YAML: %s\n", err)
+		os.Exit(1)
+	}
+
+	// Look for the test in the configuration
+	found := false
+	for _, job := range config.Jobs {
+		for _, test := range job.Strategy.Matrix.Test {
+			if test.Path == testFilePath {
+				// Check if -test.run is specified and if it contains the testName
+				if strings.Contains(test.TestOpts, "-test.run") {
+					if strings.Contains(test.TestOpts, testName) {
+						fmt.Printf("Test '%s' is specifically included in the nightly CI pipeline for file '%s'.\n", testName, testFilePath)
+						found = true
+						break
+					}
+				} else {
+					// If -test.run is not specified, assume all tests in the file are included
+					fmt.Printf("All tests in '%s' are included in the nightly CI pipeline, including '%s'.\n", testFilePath, testName)
+					found = true
+					break
+				}
+			}
+		}
+		if found {
+			break
+		}
+	}
+
+	if !found {
+		fmt.Printf("Test '%s' is NOT included in the nightly CI pipeline for file '%s'.\n", testName, testFilePath)
+	}
+}
diff --git a/.github/workflows/nightly-e2e-tests.yml b/.github/workflows/nightly-e2e-tests.yml
new file mode 100644
index 00000000000..2e09321f759
--- /dev/null
+++ b/.github/workflows/nightly-e2e-tests.yml
@@ -0,0 +1,40 @@
+name: Nightly Smoke Tests
+on:
+  schedule:
+    # Run every night at midnight UTC (0:00 AM)
+    - cron: '0 0 * * *'
+
+jobs:
+  docker-tests:
+    strategy:
+      fail-fast: false
+      matrix:
+        test:
+          # Example of 1 runner for all tests in a file
+          - path: integration-tests/smoke/ocr_test.go
+            runs-on: ubuntu-latest
+            testOpts: -test.parallel=1 -timeout 30m -count=1 -json
+          # Example of 2 separate runners for the same test file
+          - path: integration-tests/smoke/ocr2_test.go
+            runs-on: ubuntu-latest
+            testOpts: -test.run ^TestOCRv2Request$ -test.parallel=1 -timeout 30m -count=1 -json
+          - path: integration-tests/smoke/ocr2_test.go
+            runs-on: ubuntu-latest
+            testOpts: -test.run ^TestOCRv2Basic$ -test.parallel=1 -timeout 30m -count=1 -json
+    runs-on: ${{ matrix.test.runs-on }}
+    environment: integration
+    permissions:
+      actions: read
+      checks: write
+      pull-requests: write
+      id-token: write
+      contents: read
+    env:
+      SELECTED_NETWORKS: SIMULATED
+      CHAINLINK_COMMIT_SHA: ${{ inputs.evm-ref || github.sha }}
+      CHAINLINK_ENV_USER: ${{ github.actor }}
+      TEST_LOG_LEVEL: debug
+              
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v3

From 01d71a36a35a6fc312a24883142e0c44c70f6aa1 Mon Sep 17 00:00:00 2001
From: lukaszcl <120112546+lukaszcl@users.noreply.github.com>
Date: Sat, 22 Jun 2024 11:19:19 +0200
Subject: [PATCH 002/137] add reusable workflow

---
 .github/e2e-tests.yml                        |  34 ++++
 .github/scripts/e2e_tests_tool/main.go       |  91 +++++++++
 .github/scripts/e2e_tests_tool/main_test.go  |  64 ++++++
 .github/workflows/nightly-e2e-tests.yml      |  40 ----
 .github/workflows/run-all-e2e-tests.yml      | 202 +++++++++++++++++++
 .github/workflows/run-selected-e2e-tests.yml | 157 ++++++++++++++
 6 files changed, 548 insertions(+), 40 deletions(-)
 create mode 100644 .github/e2e-tests.yml
 create mode 100644 .github/scripts/e2e_tests_tool/main.go
 create mode 100644 .github/scripts/e2e_tests_tool/main_test.go
 delete mode 100644 .github/workflows/nightly-e2e-tests.yml
 create mode 100644 .github/workflows/run-all-e2e-tests.yml
 create mode 100644 .github/workflows/run-selected-e2e-tests.yml

diff --git a/.github/e2e-tests.yml b/.github/e2e-tests.yml
new file mode 100644
index 00000000000..46e90d4f237
--- /dev/null
+++ b/.github/e2e-tests.yml
@@ -0,0 +1,34 @@
+# This configuration file defines the GitHub runner requirements for each E2E test or set of E2E tests.
+#
+# Each test entry details the runner environment (Docker or Kubernetes), Go test command, and triggers
+# that determine under what conditions the test should be executed (e.g., on push or nightly builds).
+#
+# This file is used by CI workflows to run E2E tests depending on the trigger and the test type. 
+
+# Example of 1 runner for all tests in integration-tests/smoke/ocr_test.go
+- id: run_all_in_ocr_tests_go 
+  path: integration-tests/smoke/ocr_test.go
+  test-type: docker
+  runs-on: ubuntu-latest
+  cmd: go test integration-tests/smoke/ocr_test.go -test.parallel=1 -timeout 30m -count=1 -json
+  # workflow: [nightly-e2e-tests.yml]
+  trigger: ["push", "nightly"]
+
+# Example of 2 separate runners for the same test file but different tests. Can be used if tests if are too heavy to run on the same runner
+- id: run_TestOCRv2Request_in_ocr2_test_go
+  path: integration-tests/smoke/ocr2_test.go
+  test-type: docker
+  runs-on: ubuntu-latest
+  cmd: go test integration-tests/smoke/ocr2_test.go -test.run ^TestOCRv2Request$ -test.parallel=1 -timeout 30m -count=1 -json
+- id: run_TestOCRv2Basic_in_ocr2_test_go
+  path: integration-tests/smoke/ocr2_test.go
+  test-type: docker
+  runs-on: ubuntu-latest
+  cmd: go test integration-tests/smoke/ocr2_test.go -test.run ^TestOCRv2Basic$ -test.parallel=1 -timeout 30m -count=1 -json
+
+# Example of configuration for running a test in Kubernetes Remote Runner
+- id: run_TestOCRSoak_in_ocr_test_go
+  path: integration-tests/smoke/ocr2_test.go
+  test-type: k8s-remote-runner
+  runs-on: ubuntu-latest
+  cmd: go test integration-tests/soak/ocr_test.go -test.run ^TestOCRSoak$ -test.parallel=1 -timeout 30m -count=1 -json
diff --git a/.github/scripts/e2e_tests_tool/main.go b/.github/scripts/e2e_tests_tool/main.go
new file mode 100644
index 00000000000..7554aa5b9fa
--- /dev/null
+++ b/.github/scripts/e2e_tests_tool/main.go
@@ -0,0 +1,91 @@
+package main
+
+import (
+	"encoding/json"
+	"flag"
+	"fmt"
+	"io/ioutil"
+	"log"
+	"strings"
+
+	"gopkg.in/yaml.v2"
+)
+
+// Test defines the structure of a test entry in the YAML file.
+type Test struct {
+	ID       string   `yaml:"id" json:"id"`
+	Name     string   `yaml:"name" json:"name"`
+	Path     string   `yaml:"path" json:"path"`
+	TestType string   `yaml:"test-type" json:"testType"`
+	RunsOn   string   `yaml:"runs-on" json:"runsOn"`
+	Cmd      string   `yaml:"cmd" json:"cmd"`
+	Trigger  []string `yaml:"trigger" json:"trigger"`
+}
+
+// Config represents the tests configuration.
+type Config struct {
+	Tests []Test `yaml:"test-runner-matrix"`
+}
+
+// Filter tests based on name, trigger, test type, and test IDs.
+func FilterTests(tests []Test, names, trigger, testType, ids string) []Test {
+	nameFilter := strings.Split(names, ",")
+	triggerFilter := trigger
+	typeFilter := testType
+	idFilter := strings.Split(ids, ",")
+
+	var filteredTests []Test
+
+	for _, test := range tests {
+		nameMatch := names == "" || contains(nameFilter, test.Name)
+		triggerMatch := trigger == "" || contains(test.Trigger, triggerFilter)
+		typeMatch := testType == "" || test.TestType == typeFilter
+		idMatch := ids == "*" || ids == "" || contains(idFilter, test.ID)
+
+		if nameMatch && triggerMatch && typeMatch && idMatch {
+			filteredTests = append(filteredTests, test)
+		}
+	}
+
+	return filteredTests
+}
+
+// Main function including the new test-ids flag.
+func main() {
+	yamlFile := flag.String("file", ".github/e2e-tests.yml", "Path to the YAML file")
+	names := flag.String("name", "", "Comma-separated list of test names to filter by")
+	trigger := flag.String("trigger", "", "Trigger filter")
+	testType := flag.String("test-type", "", "Type of test to filter by")
+	testIDs := flag.String("test-ids", "*", "Comma-separated list of test IDs to filter by")
+
+	flag.Parse()
+
+	data, err := ioutil.ReadFile(*yamlFile)
+	if err != nil {
+		log.Fatalf("Error reading YAML file: %v", err)
+	}
+
+	var tests []Test
+	err = yaml.Unmarshal(data, &tests)
+	if err != nil {
+		log.Fatalf("Error parsing YAML data: %v", err)
+	}
+
+	filteredTests := FilterTests(tests, *names, *trigger, *testType, *testIDs)
+	testsJSON, err := json.Marshal(filteredTests)
+	if err != nil {
+		log.Fatalf("Error marshaling tests to JSON: %v", err)
+	}
+
+	fmt.Printf("%s", testsJSON)
+}
+
+// Utility function to check if a slice contains a string.
+func contains(slice []string, element string) bool {
+	for _, s := range slice {
+		if s == element {
+			return true
+		}
+	}
+	return false
+}
diff --git a/.github/scripts/e2e_tests_tool/main_test.go b/.github/scripts/e2e_tests_tool/main_test.go
new file mode 100644
index 00000000000..61d2afa37de
--- /dev/null
+++ b/.github/scripts/e2e_tests_tool/main_test.go
@@ -0,0 +1,64 @@
+package main
+
+import (
+	"testing"
+)
+
+func TestFilterTestsByID(t *testing.T) {
+	tests := []Test{
+		{ID: "run_all_in_ocr_tests_go", Name: "Run all ocr_tests.go", TestType: "docker"},
+		{ID: "run_all_in_ocr2_tests_go", Name: "Run TestOCRv2Request in ocr2_test.go", TestType: "docker"},
+		{ID: "run_all_in_ocr3_tests_go", Name: "Run TestOCRv2Basic in ocr2_test.go", TestType: "k8s_remote_runner"},
+	}
+
+	cases := []struct {
+		description string
+		inputIDs    string
+		expectedLen int
+	}{
+		{"Filter by single ID", "run_all_in_ocr_tests_go", 1},
+		{"Filter by multiple IDs", "run_all_in_ocr_tests_go,run_all_in_ocr2_tests_go", 2},
+		{"Wildcard to include all", "*", 3},
+		{"Empty ID string to include all", "", 3},
+	}
+
+	for _, c := range cases {
+		t.Run(c.description, func(t *testing.T) {
+			filtered := FilterTests(tests, "", "", "", c.inputIDs)
+			if len(filtered) != c.expectedLen {
+				t.Errorf("FilterTests(%s) returned %d tests, expected %d", c.description, len(filtered), c.expectedLen)
+			}
+		})
+	}
+}
+
+func TestFilterTestsIntegration(t *testing.T) {
+	tests := []Test{
+		{ID: "run_all_in_ocr_tests_go", Name: "Run all ocr_tests.go", TestType: "docker", Trigger: []string{"nightly"}},
+		{ID: "run_all_in_ocr2_tests_go", Name: "Run TestOCRv2Request in ocr2_test.go", TestType: "docker", Trigger: []string{"push"}},
+		{ID: "run_all_in_ocr3_tests_go", Name: "Run TestOCRv2Basic in ocr2_test.go", TestType: "k8s_remote_runner", Trigger: []string{"push"}},
+	}
+
+	cases := []struct {
+		description   string
+		inputNames    string
+		inputTrigger  string
+		inputTestType string
+		inputIDs      string
+		expectedLen   int
+	}{
+		{"Filter by test type and ID", "", "", "docker", "run_all_in_ocr2_tests_go", 1},
+		{"Filter by trigger and test type", "", "push", "docker", "*", 1},
+		{"No filters applied", "", "", "", "*", 3},
+		{"Filter mismatching all criteria", "", "nightly", "k8s_remote_runner", "run_all_in_ocr_tests_go", 0},
+	}
+
+	for _, c := range cases {
+		t.Run(c.description, func(t *testing.T) {
+			filtered := FilterTests(tests, c.inputNames, c.inputTrigger, c.inputTestType, c.inputIDs)
+			if len(filtered) != c.expectedLen {
+				t.Errorf("FilterTests(%s) returned %d tests, expected %d", c.description, len(filtered), c.expectedLen)
+			}
+		})
+	}
+}
diff --git a/.github/workflows/nightly-e2e-tests.yml b/.github/workflows/nightly-e2e-tests.yml
deleted file mode 100644
index 2e09321f759..00000000000
--- a/.github/workflows/nightly-e2e-tests.yml
+++ /dev/null
@@ -1,40 +0,0 @@
-name: Nightly Smoke Tests
-on:
-  schedule:
-    # Run every night at midnight UTC (0:00 AM)
-    - cron: '0 0 * * *'
-
-jobs:
-  docker-tests:
-    strategy:
-      fail-fast: false
-      matrix:
-        test:
-          # Example of 1 runner for all tests in a file
-          - path: integration-tests/smoke/ocr_test.go
-            runs-on: ubuntu-latest
-            testOpts: -test.parallel=1 -timeout 30m -count=1 -json
-          # Example of 2 separate runners for the same test file
-          - path: integration-tests/smoke/ocr2_test.go
-            runs-on: ubuntu-latest
-            testOpts: -test.run ^TestOCRv2Request$ -test.parallel=1 -timeout 30m -count=1 -json
-          - path: integration-tests/smoke/ocr2_test.go
-            runs-on: ubuntu-latest
-            testOpts: -test.run ^TestOCRv2Basic$ -test.parallel=1 -timeout 30m -count=1 -json
-    runs-on: ${{ matrix.test.runs-on }}
-    environment: integration
-    permissions:
-      actions: read
-      checks: write
-      pull-requests: write
-      id-token: write
-      contents: read
-    env:
-      SELECTED_NETWORKS: SIMULATED
-      CHAINLINK_COMMIT_SHA: ${{ inputs.evm-ref || github.sha }}
-      CHAINLINK_ENV_USER: ${{ github.actor }}
-      TEST_LOG_LEVEL: debug
-              
-    steps:
-      - name: Checkout repository
-        uses: actions/checkout@v3
diff --git a/.github/workflows/run-all-e2e-tests.yml b/.github/workflows/run-all-e2e-tests.yml
new file mode 100644
index 00000000000..ed4482443c5
--- /dev/null
+++ b/.github/workflows/run-all-e2e-tests.yml
@@ -0,0 +1,202 @@
+name: Run All E2E Tests
+on:
+  # schedule:
+  #   # Run every night at midnight UTC (0:00 AM) TODO: create separate workflow that uses this one 
+  #   - cron: '0 0 * * *'
+  workflow_dispatch:
+    inputs:
+      custom-chainlink-image:
+        description: 'Enter custom Chainlink image to use for the tests. Example: "public.ecr.aws/chainlink/chainlink"'
+        required: false
+        type: string
+      chainlink-version:
+        description: 'Enter Chainlink version to use for the tests. Example: "v2.10.0" or sha'
+        required: true
+        type: string
+      test-ids:
+        description: 'Run all tests "*" by default. Or, enter test IDs to run separated by commas. Example: "run_all_in_ocr_tests_go,run_TestOCRv2Request_in_ocr2_test_go". Check all test IDs in .github/e2e-tests.yml'
+        default: "*"
+        required: true
+        type: string                
+env: 
+  CHAINLINK_IMAGE: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com/chainlink
+
+jobs:
+  load-test-configurations:
+    runs-on: ubuntu-latest
+    outputs:
+      docker-matrix: ${{ steps.set-docker-matrix.outputs.matrix }}
+      k8s-runner-matrix: ${{ steps.set-k8s-runner-matrix.outputs.matrix }}
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2
+      - name: Set up Go
+        uses: actions/setup-go@v2
+        with:
+          go-version: '1.21.7'
+      - name: Generate Docker Tests Matrix
+        id: set-docker-matrix
+        run: echo "::set-output name=matrix::$(go run ./scripts/e2e_tests_tool/main.go -file e2e-tests.yml -test-type 'docker' -test-ids '${{ github.event.inputs.test-ids }}')"
+      - name: Generate K8s Tests Matrix
+        id: set-k8s-runner-matrix
+        run: echo "::set-output name=matrix::$(go run ./scripts/e2e_tests_tool/main.go -file e2e-tests.yml -test-type 'k8s_remote_runner' -test-ids '${{ github.event.inputs.test-ids }}')"
+
+  # Run Docker tests
+
+  docker-tests:
+    needs: load-test-configurations
+    runs-on: ${{ matrix.tests.runs-on }}
+    strategy:
+      fail-fast: false
+      matrix: ${{fromJson(needs.load-test-configurations.outputs.docker-matrix)}}
+    environment: integration
+    permissions:
+      actions: read
+      checks: write
+      pull-requests: write
+      id-token: write
+      contents: read
+    env:
+      SELECTED_NETWORKS: SIMULATED
+      CHAINLINK_ENV_USER: ${{ github.actor }}
+      TEST_LOG_LEVEL: debug
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2
+      - name: Run Tests
+        run: echo "Running - ${{ matrix.tests.cmd }}"
+
+  # Run K8s tests using old remote runner
+
+  build-k8s-runner-test-image:
+    if: false
+    runs-on: ubuntu-latest
+    steps:
+      - name: Build Test Runner Image
+        uses: ./.github/actions/build-test-image
+        with:
+          QA_AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }}
+          QA_AWS_REGION: ${{ secrets.QA_AWS_REGION }}
+          QA_AWS_ACCOUNT_NUMBER: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}   
+
+  run-k8s-runner-tests:
+    if: false
+    needs: [load-test-configurations, build-k8s-runner-test-image]
+    runs-on: ${{ matrix.tests.runs-on }}
+    strategy:
+      fail-fast: false
+      matrix: ${{fromJson(needs.load-test-configurations.outputs.k8s-runner-matrix)}}
+    environment: integration
+    permissions:
+      actions: read
+      checks: write
+      pull-requests: write
+      id-token: write
+      contents: read
+    env:
+      SELECTED_NETWORKS: SIMULATED
+      CHAINLINK_COMMIT_SHA: ${{ inputs.evm-ref || github.sha }}
+      CHAINLINK_ENV_USER: ${{ github.actor }}
+      TEST_LOG_LEVEL: debug
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2
+      - name: Run Tests
+        uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/run-tests@af92c5fae8dcf1659201e907db82d221fc304b94 # v2.3.21
+        env:
+          DETACH_RUNNER: true
+          TEST_SUITE: soak
+          TEST_ARGS: -test.timeout 900h -test.memprofile memprofile.out -test.cpuprofile profile.out
+          ENV_JOB_IMAGE: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com/chainlink-tests:${{ github.sha }}
+          # We can comment these out when we have a stable soak test and aren't worried about resource consumption
+          TEST_UPLOAD_CPU_PROFILE: true
+          TEST_UPLOAD_MEM_PROFILE: true
+        with:
+          test_command_to_run: ${{ matrix.tests.cmd }}
+          test_download_vendor_packages_command: make gomod
+          cl_repo: ${{ env.CHAINLINK_IMAGE }}
+          cl_image_tag: ${{ env.CHAINLINK_VERSION }}
+          token: ${{ secrets.GITHUB_TOKEN }}
+          should_cleanup: false
+          go_mod_path: ./integration-tests/go.mod
+          QA_AWS_REGION: ${{ secrets.QA_AWS_REGION }}
+          QA_AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }}
+          QA_KUBECONFIG: ${{ secrets.QA_KUBECONFIG }} 
+
+  # Run K8s tests using new remote runner
+  # remote-runner-k8s-tests:
+  #   runs-on: ubuntu-latest
+  #   container:
+  #     image: golang:1.18
+  #   steps:
+  #     - name: Checkout repository
+  #       uses: actions/checkout@v2
+      
+  #     - name: Set up Go
+  #       uses: actions/setup-go@v2
+  #       with:
+  #         go-version: '1.18'
+      
+  #     - name: Load Runner Config
+  #       run: echo "$RUNNER_CONFIG" > runner.toml
+  #       env:
+  #         RUNNER_CONFIG: |
+  #           # Runner configuration
+  #           detached_mode = true
+  #           debug = false
+            
+  #           [[test_runs]]
+  #           namespace = "dev-env"
+  #           rbac_role_name = "dev-role"
+  #           rbac_service_account_name = "dev-service-account"
+  #           sync_value = "unique-sync-value-1"
+  #           ttl_seconds_after_finished = 300
+  #           image_registry_url = "https://myregistry.dev/"
+  #           image_name = "dev-image"
+  #           image_tag = "v1.0.0"
+  #           test_name = "TestMercuryLoad/all_endpoints"
+  #           test_config_base64_env_name = "CONFIG_ENV_DEV"
+  #           test_config_file_path = "/configs/dev/test-config.toml"
+  #           test_config_base64 = "dGVzdCBjb25maWcgdmFsdWUgZGV2"
+  #           test_timeout = "30m"
+  #           resources_requests_cpu = "500m"
+  #           resources_requests_memory = "1Gi"
+  #           resources_limits_cpu = "1000m"
+  #           resources_limits_memory = "2Gi"
+  #           job_count = 2
+  #           chart_path = "/charts/dev"
+  #           [envs]
+  #           WASP_LOG_LEVEL = "info"
+  #           TEST_LOG_LEVEL = "info"
+  #           MERCURY_TEST_LOG_LEVEL = "info"
+            
+  #           [[test_runs]]
+  #           namespace = "prod-env"
+  #           rbac_role_name = "prod-role"
+  #           rbac_service_account_name = "prod-service-account"
+  #           sync_value = "unique-sync-value-2"
+  #           ttl_seconds_after_finished = 600
+  #           image_registry_url = "https://myregistry.prod/"
+  #           image_name = "prod-image"
+  #           image_tag = "v1.0.1"
+  #           test_name = "TestMercuryLoad/all_endpoints"
+  #           test_config_base64_env_name = "CONFIG_ENV_PROD"
+  #           test_config_file_path = "/configs/prod/test-config.toml"
+  #           test_config_base64 = "dGVzdCBjb25maWcgdmFsdWUgcHJvZA=="
+  #           test_timeout = "45m"
+  #           resources_requests_cpu = "800m"
+  #           resources_requests_memory = "2Gi"
+  #           resources_limits_cpu = "1500m"
+  #           resources_limits_memory = "4Gi"
+  #           job_count = 3
+  #           chart_path = "/charts/prod"
+  #           [envs]
+  #           WASP_LOG_LEVEL = "info"
+  #           TEST_LOG_LEVEL = "info"
+  #           MERCURY_TEST_LOG_LEVEL = "info"
+        
+  #     # Schedule the tests in K8s in remote runner
+  #     - name: Run Kubernetes Tests
+  #       run: go run ./cmd/main.go run -c runner.toml        
\ No newline at end of file
diff --git a/.github/workflows/run-selected-e2e-tests.yml b/.github/workflows/run-selected-e2e-tests.yml
new file mode 100644
index 00000000000..865bc2592e9
--- /dev/null
+++ b/.github/workflows/run-selected-e2e-tests.yml
@@ -0,0 +1,157 @@
+name: Run Selected E2E Tests
+
+on:
+  workflow_dispatch:
+    inputs:
+      testIDs:
+        description: 'Enter test IDs to run separated by commas. Example: "run_all_in_ocr_tests_go,run_TestOCRv2Request_in_ocr2_test_go". Check all test IDs in .github/e2e-tests.yml'
+        required: true
+        type: string
+
+jobs:
+  load-test-configurations:
+    runs-on: ubuntu-latest
+    outputs:
+      docker-matrix: ${{ steps.set-docker-matrix.outputs.matrix }}
+      k8s-runner-matrix: ${{ steps.set-k8s-runner-matrix.outputs.matrix }}
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v3
+      - name: Set up Go
+        uses: actions/setup-go@v2
+        with:
+          go-version: '1.18'
+      - name: Generate Docker Tests Matrix
+        id: set-docker-matrix
+        run: echo "::set-output name=matrix::$(go run ./cmd/parse_e2e_tests.go -file .github/test-runner-matrix.yml -name '${{ github.event.inputs.testIDs }}' -test-type 'docker')"
+      - name: Generate K8s Tests Matrix
+        id: set-k8s-runner-matrix
+        run: echo "::set-output name=matrix::$(go run ./cmd/parse_e2e_tests.go -file .github/test-runner-matrix.yml -name '${{ github.event.inputs.testIDs }}' -test-type 'k8s_remote_runner')"
+
+  docker-tests:
+    needs: load-test-configurations
+    runs-on: ${{ matrix.tests.runs-on }}
+    strategy:
+      fail-fast: false
+      matrix: ${{fromJson(needs.load-test-configurations.outputs.docker-matrix)}}
+    environment: integration
+    permissions:
+      actions: read
+      checks: write
+      pull-requests: write
+      id-token: write
+      contents: read
+    env:
+      SELECTED_NETWORKS: SIMULATED
+      CHAINLINK_COMMIT_SHA: ${{ inputs.evm-ref || github.sha }}
+      CHAINLINK_ENV_USER: ${{ github.actor }}
+      TEST_LOG_LEVEL: debug
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v3
+      - name: Run Tests
+        run: go test ${{ matrix.tests.path }} ${{ matrix.tests.testOpts }}
+
+  # Run K8s tests with old remote runner
+  k8s-runner-tests:
+    needs: load-test-configurations
+    runs-on: ${{ matrix.tests.runs-on }}
+    strategy:
+      fail-fast: false
+      matrix: ${{fromJson(needs.load-test-configurations.outputs.k8s-runner-matrix)}}
+    environment: integration
+    permissions:
+      actions: read
+      checks: write
+      pull-requests: write
+      id-token: write
+      contents: read
+    env:
+      SELECTED_NETWORKS: SIMULATED
+      CHAINLINK_COMMIT_SHA: ${{ inputs.evm-ref || github.sha }}
+      CHAINLINK_ENV_USER: ${{ github.actor }}
+      TEST_LOG_LEVEL: debug
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v3
+      - name: Run Tests
+        run: go test ${{ matrix.tests.path }} ${{ matrix.tests.testOpts }}
+
+  # Run K8s tests
+  # Use new remote runner to run K8s tests
+  remote-runner-k8s-tests:
+    runs-on: ubuntu-latest
+    container:
+      image: golang:1.18
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v2
+      
+      - name: Set up Go
+        uses: actions/setup-go@v2
+        with:
+          go-version: '1.18'
+      
+      - name: Load Runner Config
+        run: echo "$RUNNER_CONFIG" > runner.toml
+        env:
+          RUNNER_CONFIG: |
+            # Runner configuration
+            detached_mode = true
+            debug = false
+            
+            [[test_runs]]
+            namespace = "dev-env"
+            rbac_role_name = "dev-role"
+            rbac_service_account_name = "dev-service-account"
+            sync_value = "unique-sync-value-1"
+            ttl_seconds_after_finished = 300
+            image_registry_url = "https://myregistry.dev/"
+            image_name = "dev-image"
+            image_tag = "v1.0.0"
+            test_name = "TestMercuryLoad/all_endpoints"
+            test_config_base64_env_name = "CONFIG_ENV_DEV"
+            test_config_file_path = "/configs/dev/test-config.toml"
+            test_config_base64 = "dGVzdCBjb25maWcgdmFsdWUgZGV2"
+            test_timeout = "30m"
+            resources_requests_cpu = "500m"
+            resources_requests_memory = "1Gi"
+            resources_limits_cpu = "1000m"
+            resources_limits_memory = "2Gi"
+            job_count = 2
+            chart_path = "/charts/dev"
+            [envs]
+            WASP_LOG_LEVEL = "info"
+            TEST_LOG_LEVEL = "info"
+            MERCURY_TEST_LOG_LEVEL = "info"
+            
+            [[test_runs]]
+            namespace = "prod-env"
+            rbac_role_name = "prod-role"
+            rbac_service_account_name = "prod-service-account"
+            sync_value = "unique-sync-value-2"
+            ttl_seconds_after_finished = 600
+            image_registry_url = "https://myregistry.prod/"
+            image_name = "prod-image"
+            image_tag = "v1.0.1"
+            test_name = "TestMercuryLoad/all_endpoints"
+            test_config_base64_env_name = "CONFIG_ENV_PROD"
+            test_config_file_path = "/configs/prod/test-config.toml"
+            test_config_base64 = "dGVzdCBjb25maWcgdmFsdWUgcHJvZA=="
+            test_timeout = "45m"
+            resources_requests_cpu = "800m"
+            resources_requests_memory = "2Gi"
+            resources_limits_cpu = "1500m"
+            resources_limits_memory = "4Gi"
+            job_count = 3
+            chart_path = "/charts/prod"
+            [envs]
+            WASP_LOG_LEVEL = "info"
+            TEST_LOG_LEVEL = "info"
+            MERCURY_TEST_LOG_LEVEL = "info"
+        
+      # Schedule the tests in K8s in remote runner
+      - name: Run Kubernetes Tests
+        run: go run ./cmd/main.go run -c runner.toml        
\ No newline at end of file

From 08eb8d0b5dee1f06dfafc95c1f2781b3c4e292d7 Mon Sep 17 00:00:00 2001
From: lukaszcl <120112546+lukaszcl@users.noreply.github.com>
Date: Sat, 22 Jun 2024 12:12:49 +0200
Subject: [PATCH 003/137] on push

---
 .github/workflows/run-all-e2e-tests.yml | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/run-all-e2e-tests.yml b/.github/workflows/run-all-e2e-tests.yml
index ed4482443c5..606f50b1e57 100644
--- a/.github/workflows/run-all-e2e-tests.yml
+++ b/.github/workflows/run-all-e2e-tests.yml
@@ -1,5 +1,6 @@
 name: Run All E2E Tests
 on:
+  push:
   # schedule:
   #   # Run every night at midnight UTC (0:00 AM) TODO: create separate workflow that uses this one 
   #   - cron: '0 0 * * *'
@@ -36,10 +37,10 @@ jobs:
           go-version: '1.21.7'
       - name: Generate Docker Tests Matrix
         id: set-docker-matrix
-        run: echo "::set-output name=matrix::$(go run ./scripts/e2e_tests_tool/main.go -file e2e-tests.yml -test-type 'docker' -test-ids '${{ github.event.inputs.test-ids }}')"
+        run: echo "::set-output name=matrix::$(go run .github/scripts/e2e_tests_tool/main.go -file .github/e2e-tests.yml -test-type 'docker' -test-ids '${{ github.event.inputs.test-ids }}')"
       - name: Generate K8s Tests Matrix
         id: set-k8s-runner-matrix
-        run: echo "::set-output name=matrix::$(go run ./scripts/e2e_tests_tool/main.go -file e2e-tests.yml -test-type 'k8s_remote_runner' -test-ids '${{ github.event.inputs.test-ids }}')"
+        run: echo "::set-output name=matrix::$(go run .github/scripts/e2e_tests_tool/main.go -file .github/e2e-tests.yml -test-type 'k8s_remote_runner' -test-ids '${{ github.event.inputs.test-ids }}')"
 
   # Run Docker tests
 

From 3b5a6e73bbb730616d29c700f1cbb443db923d03 Mon Sep 17 00:00:00 2001
From: lukaszcl <120112546+lukaszcl@users.noreply.github.com>
Date: Sat, 22 Jun 2024 12:18:15 +0200
Subject: [PATCH 004/137] echo tests

---
 .github/workflows/run-all-e2e-tests.yml | 16 ++++++++++++----
 1 file changed, 12 insertions(+), 4 deletions(-)

diff --git a/.github/workflows/run-all-e2e-tests.yml b/.github/workflows/run-all-e2e-tests.yml
index 606f50b1e57..9a431453d23 100644
--- a/.github/workflows/run-all-e2e-tests.yml
+++ b/.github/workflows/run-all-e2e-tests.yml
@@ -1,6 +1,5 @@
 name: Run All E2E Tests
 on:
-  push:
   # schedule:
   #   # Run every night at midnight UTC (0:00 AM) TODO: create separate workflow that uses this one 
   #   - cron: '0 0 * * *'
@@ -35,13 +34,22 @@ jobs:
         uses: actions/setup-go@v2
         with:
           go-version: '1.21.7'
+      - name: Install jq
+        run: sudo apt-get install jq
       - name: Generate Docker Tests Matrix
         id: set-docker-matrix
-        run: echo "::set-output name=matrix::$(go run .github/scripts/e2e_tests_tool/main.go -file .github/e2e-tests.yml -test-type 'docker' -test-ids '${{ github.event.inputs.test-ids }}')"
+        run: |
+          MATRIX_JSON=$(go run .github/scripts/e2e_tests_tool/main.go -file .github/e2e-tests.yml -test-type 'docker' -test-ids '${{ github.event.inputs.test-ids }}')
+          echo "Docker Tests Matrix:"
+          echo "$MATRIX_JSON" | jq
+          echo "::set-output name=matrix::$MATRIX_JSON"
       - name: Generate K8s Tests Matrix
         id: set-k8s-runner-matrix
-        run: echo "::set-output name=matrix::$(go run .github/scripts/e2e_tests_tool/main.go -file .github/e2e-tests.yml -test-type 'k8s_remote_runner' -test-ids '${{ github.event.inputs.test-ids }}')"
-
+        run: |
+          MATRIX_JSON=$(go run .github/scripts/e2e_tests_tool/main.go -file .github/e2e-tests.yml -test-type 'k8s_remote_runner' -test-ids '${{ github.event.inputs.test-ids }}')
+          echo "K8s Tests Matrix:"
+          echo "$MATRIX_JSON" | jq
+          echo "::set-output name=matrix::$MATRIX_JSON"
   # Run Docker tests
 
   docker-tests:

From 2bf79bc52933cdac533fd87a9bf65b9d1b6d28dd Mon Sep 17 00:00:00 2001
From: lukaszcl <120112546+lukaszcl@users.noreply.github.com>
Date: Sat, 22 Jun 2024 12:24:24 +0200
Subject: [PATCH 005/137] fix matrix

---
 .github/e2e-tests.yml                   | 52 +++++++++++++------------
 .github/scripts/e2e_tests_tool/main.go  | 13 ++++---
 .github/workflows/run-all-e2e-tests.yml |  4 +-
 3 files changed, 36 insertions(+), 33 deletions(-)

diff --git a/.github/e2e-tests.yml b/.github/e2e-tests.yml
index 46e90d4f237..fb158fa8da7 100644
--- a/.github/e2e-tests.yml
+++ b/.github/e2e-tests.yml
@@ -4,31 +4,33 @@
 # that determine under what conditions the test should be executed (e.g., on push or nightly builds).
 #
 # This file is used by CI workflows to run E2E tests depending on the trigger and the test type. 
+#
+test-runner-matrix:
 
-# Example of 1 runner for all tests in integration-tests/smoke/ocr_test.go
-- id: run_all_in_ocr_tests_go 
-  path: integration-tests/smoke/ocr_test.go
-  test-type: docker
-  runs-on: ubuntu-latest
-  cmd: go test integration-tests/smoke/ocr_test.go -test.parallel=1 -timeout 30m -count=1 -json
-  # workflow: [nightly-e2e-tests.yml]
-  trigger: ["push", "nightly"]
+  # Example of 1 runner for all tests in integration-tests/smoke/ocr_test.go
+  - id: run_all_in_ocr_tests_go 
+    path: integration-tests/smoke/ocr_test.go
+    test-type: docker
+    runs-on: ubuntu-latest
+    cmd: go test integration-tests/smoke/ocr_test.go -test.parallel=1 -timeout 30m -count=1 -json
+    # workflow: [nightly-e2e-tests.yml]
+    trigger: ["push", "nightly"]
 
-# Example of 2 separate runners for the same test file but different tests. Can be used if tests if are too heavy to run on the same runner
-- id: run_TestOCRv2Request_in_ocr2_test_go
-  path: integration-tests/smoke/ocr2_test.go
-  test-type: docker
-  runs-on: ubuntu-latest
-  cmd: go test integration-tests/smoke/ocr2_test.go -test.run ^TestOCRv2Request$ -test.parallel=1 -timeout 30m -count=1 -json
-- id: run_TestOCRv2Basic_in_ocr2_test_go
-  path: integration-tests/smoke/ocr2_test.go
-  test-type: docker
-  runs-on: ubuntu-latest
-  cmd: go test integration-tests/smoke/ocr2_test.go -test.run ^TestOCRv2Basic$ -test.parallel=1 -timeout 30m -count=1 -json
+  # Example of 2 separate runners for the same test file but different tests. Can be used if tests if are too heavy to run on the same runner
+  - id: run_TestOCRv2Request_in_ocr2_test_go
+    path: integration-tests/smoke/ocr2_test.go
+    test-type: docker
+    runs-on: ubuntu-latest
+    cmd: go test integration-tests/smoke/ocr2_test.go -test.run ^TestOCRv2Request$ -test.parallel=1 -timeout 30m -count=1 -json
+  - id: run_TestOCRv2Basic_in_ocr2_test_go
+    path: integration-tests/smoke/ocr2_test.go
+    test-type: docker
+    runs-on: ubuntu-latest
+    cmd: go test integration-tests/smoke/ocr2_test.go -test.run ^TestOCRv2Basic$ -test.parallel=1 -timeout 30m -count=1 -json
 
-# Example of configuration for running a test in Kubernetes Remote Runner
-- id: run_TestOCRSoak_in_ocr_test_go
-  path: integration-tests/smoke/ocr2_test.go
-  test-type: k8s-remote-runner
-  runs-on: ubuntu-latest
-  cmd: go test integration-tests/soak/ocr_test.go -test.run ^TestOCRSoak$ -test.parallel=1 -timeout 30m -count=1 -json
+  # Example of configuration for running a test in Kubernetes Remote Runner
+  - id: run_TestOCRSoak_in_ocr_test_go
+    path: integration-tests/smoke/ocr2_test.go
+    test-type: k8s-remote-runner
+    runs-on: ubuntu-latest
+    cmd: go test integration-tests/soak/ocr_test.go -test.run ^TestOCRSoak$ -test.parallel=1 -timeout 30m -count=1 -json
diff --git a/.github/scripts/e2e_tests_tool/main.go b/.github/scripts/e2e_tests_tool/main.go
index 7554aa5b9fa..939ae4ebb7c 100644
--- a/.github/scripts/e2e_tests_tool/main.go
+++ b/.github/scripts/e2e_tests_tool/main.go
@@ -65,19 +65,20 @@ func main() {
 		log.Fatalf("Error reading YAML file: %v", err)
 	}
 
-	var tests []Test
-	err = yaml.Unmarshal(data, &tests)
+	var config Config
+	err = yaml.Unmarshal(data, &config)
 	if err != nil {
 		log.Fatalf("Error parsing YAML data: %v", err)
 	}
 
-	filteredTests := FilterTests(tests, *names, *trigger, *testType, *testIDs)
-	testsJSON, err := json.Marshal(filteredTests)
+	filteredTests := FilterTests(config.Tests, *names, *trigger, *testType, *testIDs)
+	matrix := map[string][]Test{"tests": filteredTests}
+	matrixJSON, err := json.Marshal(matrix)
 	if err != nil {
-		log.Fatalf("Error marshaling tests to JSON: %v", err)
+		log.Fatalf("Error marshaling matrix to JSON: %v", err)
 	}
 
-	fmt.Printf("%s", testsJSON)
+	fmt.Printf("%s", matrixJSON)
 }
 
 // Utility function to check if a slice contains a string.
diff --git a/.github/workflows/run-all-e2e-tests.yml b/.github/workflows/run-all-e2e-tests.yml
index 9a431453d23..a08fb807f06 100644
--- a/.github/workflows/run-all-e2e-tests.yml
+++ b/.github/workflows/run-all-e2e-tests.yml
@@ -40,14 +40,14 @@ jobs:
         id: set-docker-matrix
         run: |
           MATRIX_JSON=$(go run .github/scripts/e2e_tests_tool/main.go -file .github/e2e-tests.yml -test-type 'docker' -test-ids '${{ github.event.inputs.test-ids }}')
-          echo "Docker Tests Matrix:"
+          echo "Docker tests:"
           echo "$MATRIX_JSON" | jq
           echo "::set-output name=matrix::$MATRIX_JSON"
       - name: Generate K8s Tests Matrix
         id: set-k8s-runner-matrix
         run: |
           MATRIX_JSON=$(go run .github/scripts/e2e_tests_tool/main.go -file .github/e2e-tests.yml -test-type 'k8s_remote_runner' -test-ids '${{ github.event.inputs.test-ids }}')
-          echo "K8s Tests Matrix:"
+          echo "K8s tests:"
           echo "$MATRIX_JSON" | jq
           echo "::set-output name=matrix::$MATRIX_JSON"
   # Run Docker tests

From dfbdecf055394a68e9fc4b4fd5cc2a1debd4b5fd Mon Sep 17 00:00:00 2001
From: lukaszcl <120112546+lukaszcl@users.noreply.github.com>
Date: Sat, 22 Jun 2024 12:25:49 +0200
Subject: [PATCH 006/137] fix

---
 .github/workflows/run-all-e2e-tests.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/run-all-e2e-tests.yml b/.github/workflows/run-all-e2e-tests.yml
index a08fb807f06..19890f0e580 100644
--- a/.github/workflows/run-all-e2e-tests.yml
+++ b/.github/workflows/run-all-e2e-tests.yml
@@ -54,7 +54,7 @@ jobs:
 
   docker-tests:
     needs: load-test-configurations
-    runs-on: ${{ matrix.tests.runs-on }}
+    runs-on: ${{ matrix.tests.runsOn }}
     strategy:
       fail-fast: false
       matrix: ${{fromJson(needs.load-test-configurations.outputs.docker-matrix)}}

From 397ba14e312687537c25c1acefe399183ed45a64 Mon Sep 17 00:00:00 2001
From: lukaszcl <120112546+lukaszcl@users.noreply.github.com>
Date: Sat, 22 Jun 2024 12:28:06 +0200
Subject: [PATCH 007/137] add job name

---
 .github/workflows/run-all-e2e-tests.yml | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/.github/workflows/run-all-e2e-tests.yml b/.github/workflows/run-all-e2e-tests.yml
index 19890f0e580..c2d17b2212e 100644
--- a/.github/workflows/run-all-e2e-tests.yml
+++ b/.github/workflows/run-all-e2e-tests.yml
@@ -23,6 +23,7 @@ env:
 
 jobs:
   load-test-configurations:
+    name: Load Test Configurations
     runs-on: ubuntu-latest
     outputs:
       docker-matrix: ${{ steps.set-docker-matrix.outputs.matrix }}
@@ -53,6 +54,7 @@ jobs:
   # Run Docker tests
 
   docker-tests:
+    name: Run Docker Tests (${{ matrix.tests.id }})
     needs: load-test-configurations
     runs-on: ${{ matrix.tests.runsOn }}
     strategy:
@@ -80,6 +82,7 @@ jobs:
 
   build-k8s-runner-test-image:
     if: false
+    name: Build K8s Runner Test Image
     runs-on: ubuntu-latest
     steps:
       - name: Build Test Runner Image
@@ -91,6 +94,7 @@ jobs:
 
   run-k8s-runner-tests:
     if: false
+    name: Run K8s Tests (${{ matrix.tests.id }})
     needs: [load-test-configurations, build-k8s-runner-test-image]
     runs-on: ${{ matrix.tests.runs-on }}
     strategy:

From 6603e9bcc5fc86e1245433f020d5d6ee5d39d048 Mon Sep 17 00:00:00 2001
From: lukaszcl <120112546+lukaszcl@users.noreply.github.com>
Date: Sat, 22 Jun 2024 12:34:07 +0200
Subject: [PATCH 008/137] run docker tests

---
 .github/e2e-tests.yml                   | 28 +++++++-------
 .github/workflows/run-all-e2e-tests.yml | 49 +++++++++++++++++++++++++
 2 files changed, 63 insertions(+), 14 deletions(-)

diff --git a/.github/e2e-tests.yml b/.github/e2e-tests.yml
index fb158fa8da7..4b20c1caa34 100644
--- a/.github/e2e-tests.yml
+++ b/.github/e2e-tests.yml
@@ -8,29 +8,29 @@
 test-runner-matrix:
 
   # Example of 1 runner for all tests in integration-tests/smoke/ocr_test.go
-  - id: run_all_in_ocr_tests_go 
+  - id: all_tests_in_ocr_tests_go 
     path: integration-tests/smoke/ocr_test.go
     test-type: docker
     runs-on: ubuntu-latest
-    cmd: go test integration-tests/smoke/ocr_test.go -test.parallel=1 -timeout 30m -count=1 -json
+    cmd: cd integration-tests/ && go test smoke/ocr_test.go -test.parallel=1 -timeout 30m -count=1 -json
     # workflow: [nightly-e2e-tests.yml]
     trigger: ["push", "nightly"]
 
   # Example of 2 separate runners for the same test file but different tests. Can be used if tests if are too heavy to run on the same runner
-  - id: run_TestOCRv2Request_in_ocr2_test_go
-    path: integration-tests/smoke/ocr2_test.go
-    test-type: docker
-    runs-on: ubuntu-latest
-    cmd: go test integration-tests/smoke/ocr2_test.go -test.run ^TestOCRv2Request$ -test.parallel=1 -timeout 30m -count=1 -json
-  - id: run_TestOCRv2Basic_in_ocr2_test_go
-    path: integration-tests/smoke/ocr2_test.go
-    test-type: docker
-    runs-on: ubuntu-latest
-    cmd: go test integration-tests/smoke/ocr2_test.go -test.run ^TestOCRv2Basic$ -test.parallel=1 -timeout 30m -count=1 -json
+  # - id: TestOCRv2Request_in_ocr2_test_go
+  #   path: integration-tests/smoke/ocr2_test.go
+  #   test-type: docker
+  #   runs-on: ubuntu-latest
+  #   cmd: cd integration-tests/ && go test smoke/ocr2_test.go -test.run ^TestOCRv2Request$ -test.parallel=1 -timeout 30m -count=1 -json
+  # - id: TestOCRv2Basic_in_ocr2_test_go
+  #   path: integration-tests/smoke/ocr2_test.go
+  #   test-type: docker
+  #   runs-on: ubuntu-latest
+  #   cmd: cd integration-tests/ && go test smoke/ocr2_test.go -test.run ^TestOCRv2Basic$ -test.parallel=1 -timeout 30m -count=1 -json
 
   # Example of configuration for running a test in Kubernetes Remote Runner
-  - id: run_TestOCRSoak_in_ocr_test_go
+  - id: TestOCRSoak_in_ocr_test_go
     path: integration-tests/smoke/ocr2_test.go
     test-type: k8s-remote-runner
     runs-on: ubuntu-latest
-    cmd: go test integration-tests/soak/ocr_test.go -test.run ^TestOCRSoak$ -test.parallel=1 -timeout 30m -count=1 -json
+    cmd: cd integration-tests/ && go test soak/ocr_test.go -test.run ^TestOCRSoak$ -test.parallel=1 -timeout 30m -count=1 -json
diff --git a/.github/workflows/run-all-e2e-tests.yml b/.github/workflows/run-all-e2e-tests.yml
index c2d17b2212e..2f47d3ba665 100644
--- a/.github/workflows/run-all-e2e-tests.yml
+++ b/.github/workflows/run-all-e2e-tests.yml
@@ -75,6 +75,55 @@ jobs:
     steps:
       - name: Checkout repository
         uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2
+      - name: Setup GAP for Grafana
+        uses: smartcontractkit/.github/actions/setup-gap@6c9d62fdad050cfb8b59376ded291f1350705944 # setup-gap@0.2.2
+        with:
+          aws-region: ${{ secrets.AWS_REGION }}
+          aws-role-arn: ${{ secrets.AWS_OIDC_IAM_ROLE_VALIDATION_PROD_ARN }}
+          api-gateway-host: ${{ secrets.AWS_API_GW_HOST_GRAFANA }}
+          duplicate-authorization-header: "true"
+      - name: Prepare Base64 TOML override
+        uses: ./.github/actions/setup-create-base64-config
+        with:
+          runId: ${{ github.run_id }}
+          # testLogCollect: ${{ vars.TEST_LOG_COLLECT }}
+          selectedNetworks: ${{ env.SELECTED_NETWORKS }}
+          chainlinkImage: ${{ env.CHAINLINK_IMAGE }}
+          chainlinkVersion: ${{ inputs.chainlink-version }}
+          pyroscopeServer: ${{ matrix.product.pyroscope_env == '' && '' || !startsWith(github.ref, 'refs/tags/') && '' || secrets.QA_PYROSCOPE_INSTANCE }} # Avoid sending blank envs https://github.com/orgs/community/discussions/25725
+          pyroscopeEnvironment: ${{ matrix.product.pyroscope_env }}
+          pyroscopeKey: ${{ secrets.QA_PYROSCOPE_KEY }}
+          lokiEndpoint: https://${{ secrets.GRAFANA_INTERNAL_HOST }}/loki/api/v1/push
+          lokiTenantId: ${{ secrets.GRAFANA_INTERNAL_TENANT_ID }}
+          lokiBasicAuth: ${{ secrets.GRAFANA_INTERNAL_BASIC_AUTH }}
+          logstreamLogTargets: ${{ vars.LOGSTREAM_LOG_TARGETS }}
+          grafanaUrl: "http://localhost:8080/primary" # This is GAP's address
+          grafanaDashboardUrl: "/d/ddf75041-1e39-42af-aa46-361fe4c36e9e/ci-e2e-tests-logs"
+          grafanaBearerToken: ${{ secrets.GRAFANA_INTERNAL_URL_SHORTENER_TOKEN }}
+      - name: Run Tests
+        uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/run-tests@af92c5fae8dcf1659201e907db82d221fc304b94 # v2.3.21
+        with:
+          test_command_to_run: ${{ matrix.tests.cmd }} 2>&1 | tee /tmp/gotest.log | gotestloghelper -ci -singlepackage -hidepassingtests=false -hidepassinglogs
+          test_download_vendor_packages_command: cd ./integration-tests && go mod download
+          cl_repo: ${{ env.CHAINLINK_IMAGE }}
+          cl_image_tag: ${{ inputs.chainlink-version }}
+          aws_registries: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}
+          artifacts_name: ${{ matrix.product.name }}-test-logs
+          artifacts_location: |
+            ./integration-tests/smoke/logs/
+            /tmp/gotest.log
+          publish_check_name: ${{ matrix.product.name }}
+          token: ${{ secrets.GITHUB_TOKEN }}
+          go_mod_path: ./integration-tests/go.mod
+          cache_key_id: core-e2e-${{ env.MOD_CACHE_VERSION }}
+          cache_restore_only: "true"
+          QA_AWS_REGION: ${{ secrets.QA_AWS_REGION }}
+          QA_AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }}
+          QA_KUBECONFIG: ""
+          should_tidy: "false"
+          go_coverage_src_dir: /var/tmp/go-coverage
+          go_coverage_dest_dir: ${{ github.workspace }}/.covdata
+
       - name: Run Tests
         run: echo "Running - ${{ matrix.tests.cmd }}"
 

From e17499e024d437711ad57ca0a2b2501ce908279c Mon Sep 17 00:00:00 2001
From: lukaszcl <120112546+lukaszcl@users.noreply.github.com>
Date: Sat, 22 Jun 2024 12:43:45 +0200
Subject: [PATCH 009/137] add reusable workflow

---
 .github/workflows/run-all-e2e-tests.yml      |  15 ++
 .github/workflows/run-selected-e2e-tests.yml | 159 ++-----------------
 2 files changed, 27 insertions(+), 147 deletions(-)

diff --git a/.github/workflows/run-all-e2e-tests.yml b/.github/workflows/run-all-e2e-tests.yml
index 2f47d3ba665..97a16cdf2cf 100644
--- a/.github/workflows/run-all-e2e-tests.yml
+++ b/.github/workflows/run-all-e2e-tests.yml
@@ -3,6 +3,21 @@ on:
   # schedule:
   #   # Run every night at midnight UTC (0:00 AM) TODO: create separate workflow that uses this one 
   #   - cron: '0 0 * * *'
+  workflow_call:
+    inputs:
+      custom-chainlink-image:
+        description: 'Enter custom Chainlink image to use for the tests. Example: "public.ecr.aws/chainlink/chainlink"'
+        required: false
+        type: string
+      chainlink-version:
+        description: 'Enter Chainlink version to use for the tests. Example: "v2.10.0" or sha'
+        required: true
+        type: string
+      test-ids:
+        description: 'Run all tests "*" by default. Or, enter test IDs to run separated by commas. Example: "run_all_in_ocr_tests_go,run_TestOCRv2Request_in_ocr2_test_go". Check all test IDs in .github/e2e-tests.yml'
+        default: "*"
+        required: true
+        type: string     
   workflow_dispatch:
     inputs:
       custom-chainlink-image:
diff --git a/.github/workflows/run-selected-e2e-tests.yml b/.github/workflows/run-selected-e2e-tests.yml
index 865bc2592e9..5bedaf09586 100644
--- a/.github/workflows/run-selected-e2e-tests.yml
+++ b/.github/workflows/run-selected-e2e-tests.yml
@@ -3,155 +3,20 @@ name: Run Selected E2E Tests
 on:
   workflow_dispatch:
     inputs:
-      testIDs:
+      chainlink-version:
+        description: 'Enter Chainlink version to use for the tests. Example: "v2.10.0" or sha'
+        default: develop
+        required: false
+        type: string
+      test-ids:
         description: 'Enter test IDs to run separated by commas. Example: "run_all_in_ocr_tests_go,run_TestOCRv2Request_in_ocr2_test_go". Check all test IDs in .github/e2e-tests.yml'
         required: true
         type: string
 
 jobs:
-  load-test-configurations:
-    runs-on: ubuntu-latest
-    outputs:
-      docker-matrix: ${{ steps.set-docker-matrix.outputs.matrix }}
-      k8s-runner-matrix: ${{ steps.set-k8s-runner-matrix.outputs.matrix }}
-    steps:
-      - name: Checkout code
-        uses: actions/checkout@v3
-      - name: Set up Go
-        uses: actions/setup-go@v2
-        with:
-          go-version: '1.18'
-      - name: Generate Docker Tests Matrix
-        id: set-docker-matrix
-        run: echo "::set-output name=matrix::$(go run ./cmd/parse_e2e_tests.go -file .github/test-runner-matrix.yml -name '${{ github.event.inputs.testIDs }}' -test-type 'docker')"
-      - name: Generate K8s Tests Matrix
-        id: set-k8s-runner-matrix
-        run: echo "::set-output name=matrix::$(go run ./cmd/parse_e2e_tests.go -file .github/test-runner-matrix.yml -name '${{ github.event.inputs.testIDs }}' -test-type 'k8s_remote_runner')"
-
-  docker-tests:
-    needs: load-test-configurations
-    runs-on: ${{ matrix.tests.runs-on }}
-    strategy:
-      fail-fast: false
-      matrix: ${{fromJson(needs.load-test-configurations.outputs.docker-matrix)}}
-    environment: integration
-    permissions:
-      actions: read
-      checks: write
-      pull-requests: write
-      id-token: write
-      contents: read
-    env:
-      SELECTED_NETWORKS: SIMULATED
-      CHAINLINK_COMMIT_SHA: ${{ inputs.evm-ref || github.sha }}
-      CHAINLINK_ENV_USER: ${{ github.actor }}
-      TEST_LOG_LEVEL: debug
-
-    steps:
-      - name: Checkout repository
-        uses: actions/checkout@v3
-      - name: Run Tests
-        run: go test ${{ matrix.tests.path }} ${{ matrix.tests.testOpts }}
-
-  # Run K8s tests with old remote runner
-  k8s-runner-tests:
-    needs: load-test-configurations
-    runs-on: ${{ matrix.tests.runs-on }}
-    strategy:
-      fail-fast: false
-      matrix: ${{fromJson(needs.load-test-configurations.outputs.k8s-runner-matrix)}}
-    environment: integration
-    permissions:
-      actions: read
-      checks: write
-      pull-requests: write
-      id-token: write
-      contents: read
-    env:
-      SELECTED_NETWORKS: SIMULATED
-      CHAINLINK_COMMIT_SHA: ${{ inputs.evm-ref || github.sha }}
-      CHAINLINK_ENV_USER: ${{ github.actor }}
-      TEST_LOG_LEVEL: debug
-
-    steps:
-      - name: Checkout repository
-        uses: actions/checkout@v3
-      - name: Run Tests
-        run: go test ${{ matrix.tests.path }} ${{ matrix.tests.testOpts }}
-
-  # Run K8s tests
-  # Use new remote runner to run K8s tests
-  remote-runner-k8s-tests:
-    runs-on: ubuntu-latest
-    container:
-      image: golang:1.18
-    steps:
-      - name: Checkout repository
-        uses: actions/checkout@v2
-      
-      - name: Set up Go
-        uses: actions/setup-go@v2
-        with:
-          go-version: '1.18'
-      
-      - name: Load Runner Config
-        run: echo "$RUNNER_CONFIG" > runner.toml
-        env:
-          RUNNER_CONFIG: |
-            # Runner configuration
-            detached_mode = true
-            debug = false
-            
-            [[test_runs]]
-            namespace = "dev-env"
-            rbac_role_name = "dev-role"
-            rbac_service_account_name = "dev-service-account"
-            sync_value = "unique-sync-value-1"
-            ttl_seconds_after_finished = 300
-            image_registry_url = "https://myregistry.dev/"
-            image_name = "dev-image"
-            image_tag = "v1.0.0"
-            test_name = "TestMercuryLoad/all_endpoints"
-            test_config_base64_env_name = "CONFIG_ENV_DEV"
-            test_config_file_path = "/configs/dev/test-config.toml"
-            test_config_base64 = "dGVzdCBjb25maWcgdmFsdWUgZGV2"
-            test_timeout = "30m"
-            resources_requests_cpu = "500m"
-            resources_requests_memory = "1Gi"
-            resources_limits_cpu = "1000m"
-            resources_limits_memory = "2Gi"
-            job_count = 2
-            chart_path = "/charts/dev"
-            [envs]
-            WASP_LOG_LEVEL = "info"
-            TEST_LOG_LEVEL = "info"
-            MERCURY_TEST_LOG_LEVEL = "info"
-            
-            [[test_runs]]
-            namespace = "prod-env"
-            rbac_role_name = "prod-role"
-            rbac_service_account_name = "prod-service-account"
-            sync_value = "unique-sync-value-2"
-            ttl_seconds_after_finished = 600
-            image_registry_url = "https://myregistry.prod/"
-            image_name = "prod-image"
-            image_tag = "v1.0.1"
-            test_name = "TestMercuryLoad/all_endpoints"
-            test_config_base64_env_name = "CONFIG_ENV_PROD"
-            test_config_file_path = "/configs/prod/test-config.toml"
-            test_config_base64 = "dGVzdCBjb25maWcgdmFsdWUgcHJvZA=="
-            test_timeout = "45m"
-            resources_requests_cpu = "800m"
-            resources_requests_memory = "2Gi"
-            resources_limits_cpu = "1500m"
-            resources_limits_memory = "4Gi"
-            job_count = 3
-            chart_path = "/charts/prod"
-            [envs]
-            WASP_LOG_LEVEL = "info"
-            TEST_LOG_LEVEL = "info"
-            MERCURY_TEST_LOG_LEVEL = "info"
-        
-      # Schedule the tests in K8s in remote runner
-      - name: Run Kubernetes Tests
-        run: go run ./cmd/main.go run -c runner.toml        
\ No newline at end of file
+  call-run-e2e-tests-workflow:
+    name: Run E2E Tests
+    uses: ./.github/workflows/run-all-e2e-tests.yml
+    with:
+      chainlink-version: ${{ github.event.inputs.chainlink-version }}
+      test-ids: ${{ github.event.inputs.test-ids }}
\ No newline at end of file

From f2552212341eee12d65ffdafc4d99acf7983a540 Mon Sep 17 00:00:00 2001
From: lukaszcl <120112546+lukaszcl@users.noreply.github.com>
Date: Sat, 22 Jun 2024 12:51:34 +0200
Subject: [PATCH 010/137] fix

---
 .github/e2e-tests.yml                   |  5 +++++
 .github/scripts/e2e_tests_tool/main.go  | 15 ++++++++-------
 .github/workflows/run-all-e2e-tests.yml |  8 ++++----
 3 files changed, 17 insertions(+), 11 deletions(-)

diff --git a/.github/e2e-tests.yml b/.github/e2e-tests.yml
index 4b20c1caa34..b81c04bfdeb 100644
--- a/.github/e2e-tests.yml
+++ b/.github/e2e-tests.yml
@@ -15,6 +15,7 @@ test-runner-matrix:
     cmd: cd integration-tests/ && go test smoke/ocr_test.go -test.parallel=1 -timeout 30m -count=1 -json
     # workflow: [nightly-e2e-tests.yml]
     trigger: ["push", "nightly"]
+    pyroscope-env: ci-smoke-ocr-evm-simulated-nightly
 
   # Example of 2 separate runners for the same test file but different tests. Can be used if tests if are too heavy to run on the same runner
   # - id: TestOCRv2Request_in_ocr2_test_go
@@ -22,11 +23,14 @@ test-runner-matrix:
   #   test-type: docker
   #   runs-on: ubuntu-latest
   #   cmd: cd integration-tests/ && go test smoke/ocr2_test.go -test.run ^TestOCRv2Request$ -test.parallel=1 -timeout 30m -count=1 -json
+ #    pyroscope-env: ci-smoke-ocr-evm-simulated-nightly
+
   # - id: TestOCRv2Basic_in_ocr2_test_go
   #   path: integration-tests/smoke/ocr2_test.go
   #   test-type: docker
   #   runs-on: ubuntu-latest
   #   cmd: cd integration-tests/ && go test smoke/ocr2_test.go -test.run ^TestOCRv2Basic$ -test.parallel=1 -timeout 30m -count=1 -json
+  #   pyroscope-env: ci-smoke-ocr-evm-simulated-nightly
 
   # Example of configuration for running a test in Kubernetes Remote Runner
   - id: TestOCRSoak_in_ocr_test_go
@@ -34,3 +38,4 @@ test-runner-matrix:
     test-type: k8s-remote-runner
     runs-on: ubuntu-latest
     cmd: cd integration-tests/ && go test soak/ocr_test.go -test.run ^TestOCRSoak$ -test.parallel=1 -timeout 30m -count=1 -json
+    pyroscope-env: ci-smoke-ocr-evm-simulated-nightly
diff --git a/.github/scripts/e2e_tests_tool/main.go b/.github/scripts/e2e_tests_tool/main.go
index 939ae4ebb7c..b07e5dfad52 100644
--- a/.github/scripts/e2e_tests_tool/main.go
+++ b/.github/scripts/e2e_tests_tool/main.go
@@ -13,13 +13,14 @@ import (
 
 // Test defines the structure of a test entry in the YAML file.
 type Test struct {
-	ID       string   `yaml:"id" json:"id"`
-	Name     string   `yaml:"name" json:"name"`
-	Path     string   `yaml:"path" json:"path"`
-	TestType string   `yaml:"test-type" json:"testType"`
-	RunsOn   string   `yaml:"runs-on" json:"runsOn"`
-	Cmd      string   `yaml:"cmd" json:"cmd"`
-	Trigger  []string `yaml:"trigger" json:"trigger"`
+	ID           string   `yaml:"id" json:"id"`
+	Name         string   `yaml:"name" json:"name"`
+	Path         string   `yaml:"path" json:"path"`
+	TestType     string   `yaml:"test-type" json:"testType"`
+	RunsOn       string   `yaml:"runs-on" json:"runsOn"`
+	Cmd          string   `yaml:"cmd" json:"cmd"`
+	PyroscopeEnv string   `yaml:"pyroscope-env" json:"pyroscopeEnv"`
+	Trigger      []string `yaml:"trigger" json:"trigger"`
 }
 
 // Config represents the tests configuration.
diff --git a/.github/workflows/run-all-e2e-tests.yml b/.github/workflows/run-all-e2e-tests.yml
index 97a16cdf2cf..1eb4c74de8f 100644
--- a/.github/workflows/run-all-e2e-tests.yml
+++ b/.github/workflows/run-all-e2e-tests.yml
@@ -105,8 +105,8 @@ jobs:
           selectedNetworks: ${{ env.SELECTED_NETWORKS }}
           chainlinkImage: ${{ env.CHAINLINK_IMAGE }}
           chainlinkVersion: ${{ inputs.chainlink-version }}
-          pyroscopeServer: ${{ matrix.product.pyroscope_env == '' && '' || !startsWith(github.ref, 'refs/tags/') && '' || secrets.QA_PYROSCOPE_INSTANCE }} # Avoid sending blank envs https://github.com/orgs/community/discussions/25725
-          pyroscopeEnvironment: ${{ matrix.product.pyroscope_env }}
+          pyroscopeServer: ${{ secrets.QA_PYROSCOPE_INSTANCE }} # Avoid sending blank envs https://github.com/orgs/community/discussions/25725
+          pyroscopeEnvironment: ${{ matrix.tests.pyroscopeEnv }}
           pyroscopeKey: ${{ secrets.QA_PYROSCOPE_KEY }}
           lokiEndpoint: https://${{ secrets.GRAFANA_INTERNAL_HOST }}/loki/api/v1/push
           lokiTenantId: ${{ secrets.GRAFANA_INTERNAL_TENANT_ID }}
@@ -123,11 +123,11 @@ jobs:
           cl_repo: ${{ env.CHAINLINK_IMAGE }}
           cl_image_tag: ${{ inputs.chainlink-version }}
           aws_registries: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}
-          artifacts_name: ${{ matrix.product.name }}-test-logs
+          artifacts_name: ${{ matrix.tests.id }}-test-logs
           artifacts_location: |
             ./integration-tests/smoke/logs/
             /tmp/gotest.log
-          publish_check_name: ${{ matrix.product.name }}
+          publish_check_name: ${{ matrix.tests.id }}
           token: ${{ secrets.GITHUB_TOKEN }}
           go_mod_path: ./integration-tests/go.mod
           cache_key_id: core-e2e-${{ env.MOD_CACHE_VERSION }}

From 1f3b5ca4cfdf38c20fa2b7e440bd28ca559f1591 Mon Sep 17 00:00:00 2001
From: lukaszcl <120112546+lukaszcl@users.noreply.github.com>
Date: Sat, 22 Jun 2024 12:57:53 +0200
Subject: [PATCH 011/137] fail if no test found

---
 .github/e2e-tests.yml                   | 24 ++++++++++++------------
 .github/workflows/run-all-e2e-tests.yml | 18 ++++++++++++++++++
 2 files changed, 30 insertions(+), 12 deletions(-)

diff --git a/.github/e2e-tests.yml b/.github/e2e-tests.yml
index b81c04bfdeb..1748ecee09e 100644
--- a/.github/e2e-tests.yml
+++ b/.github/e2e-tests.yml
@@ -18,19 +18,19 @@ test-runner-matrix:
     pyroscope-env: ci-smoke-ocr-evm-simulated-nightly
 
   # Example of 2 separate runners for the same test file but different tests. Can be used if tests if are too heavy to run on the same runner
-  # - id: TestOCRv2Request_in_ocr2_test_go
-  #   path: integration-tests/smoke/ocr2_test.go
-  #   test-type: docker
-  #   runs-on: ubuntu-latest
-  #   cmd: cd integration-tests/ && go test smoke/ocr2_test.go -test.run ^TestOCRv2Request$ -test.parallel=1 -timeout 30m -count=1 -json
- #    pyroscope-env: ci-smoke-ocr-evm-simulated-nightly
+  - id: TestOCRv2Request_in_ocr2_test_go
+    path: integration-tests/smoke/ocr2_test.go
+    test-type: docker
+    runs-on: ubuntu-latest
+    cmd: cd integration-tests/ && go test smoke/ocr2_test.go -test.run ^TestOCRv2Request$ -test.parallel=1 -timeout 30m -count=1 -json
+    pyroscope-env: ci-smoke-ocr-evm-simulated-nightly
 
-  # - id: TestOCRv2Basic_in_ocr2_test_go
-  #   path: integration-tests/smoke/ocr2_test.go
-  #   test-type: docker
-  #   runs-on: ubuntu-latest
-  #   cmd: cd integration-tests/ && go test smoke/ocr2_test.go -test.run ^TestOCRv2Basic$ -test.parallel=1 -timeout 30m -count=1 -json
-  #   pyroscope-env: ci-smoke-ocr-evm-simulated-nightly
+  - id: TestOCRv2Basic_in_ocr2_test_go
+    path: integration-tests/smoke/ocr2_test.go
+    test-type: docker
+    runs-on: ubuntu-latest
+    cmd: cd integration-tests/ && go test smoke/ocr2_test.go -test.run ^TestOCRv2Basic$ -test.parallel=1 -timeout 30m -count=1 -json
+    pyroscope-env: ci-smoke-ocr-evm-simulated-nightly
 
   # Example of configuration for running a test in Kubernetes Remote Runner
   - id: TestOCRSoak_in_ocr_test_go
diff --git a/.github/workflows/run-all-e2e-tests.yml b/.github/workflows/run-all-e2e-tests.yml
index 1eb4c74de8f..0cc5fddab36 100644
--- a/.github/workflows/run-all-e2e-tests.yml
+++ b/.github/workflows/run-all-e2e-tests.yml
@@ -66,6 +66,24 @@ jobs:
           echo "K8s tests:"
           echo "$MATRIX_JSON" | jq
           echo "::set-output name=matrix::$MATRIX_JSON"
+      - name: Check Test Matrices
+        run: |
+          DOCKER_MATRIX=$(echo '${{ steps.set-docker-matrix.outputs.matrix }}' | jq '.tests == null or .tests == []')
+          K8S_MATRIX=$(echo '${{ steps.set-k8s-runner-matrix.outputs.matrix }}' | jq '.tests == null or .tests == []')
+
+          # Check if jq commands succeeded
+          if [ $? -ne 0 ]; then
+            echo "JSON parse error occurred."
+            exit 1
+          fi
+      
+          # Check if both matrices are empty
+          if [[ "$DOCKER_MATRIX" == "true" ]] && [[ "$K8S_MATRIX" == "true" ]]; then
+            echo "No tests found for inputs: '${{ toJson(github.event.inputs) }}'. Both Docker and Kubernetes tests matrices are empty"
+            exit 1
+          fi
+        shell: bash   
+
   # Run Docker tests
 
   docker-tests:

From 5901261d9a9569dafd4009ed8512b97795266226 Mon Sep 17 00:00:00 2001
From: lukaszcl <120112546+lukaszcl@users.noreply.github.com>
Date: Sat, 22 Jun 2024 13:45:55 +0200
Subject: [PATCH 012/137] fix run selected workflow

---
 .github/workflows/run-selected-e2e-tests.yml | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/run-selected-e2e-tests.yml b/.github/workflows/run-selected-e2e-tests.yml
index 5bedaf09586..e9009c6ac0c 100644
--- a/.github/workflows/run-selected-e2e-tests.yml
+++ b/.github/workflows/run-selected-e2e-tests.yml
@@ -19,4 +19,5 @@ jobs:
     uses: ./.github/workflows/run-all-e2e-tests.yml
     with:
       chainlink-version: ${{ github.event.inputs.chainlink-version }}
-      test-ids: ${{ github.event.inputs.test-ids }}
\ No newline at end of file
+      test-ids: ${{ github.event.inputs.test-ids }}
+    secrets: inherit

From d550f1df9023ce09b7fe558d1c1028463a1d6492 Mon Sep 17 00:00:00 2001
From: lukaszcl <120112546+lukaszcl@users.noreply.github.com>
Date: Sat, 22 Jun 2024 13:52:04 +0200
Subject: [PATCH 013/137] Rename reusable workflow

---
 ...ml => run-e2e-tests-reusable-workflow.yml} | 20 ++++---------------
 .github/workflows/run-selected-e2e-tests.yml  |  5 +++--
 2 files changed, 7 insertions(+), 18 deletions(-)
 rename .github/workflows/{run-all-e2e-tests.yml => run-e2e-tests-reusable-workflow.yml} (94%)

diff --git a/.github/workflows/run-all-e2e-tests.yml b/.github/workflows/run-e2e-tests-reusable-workflow.yml
similarity index 94%
rename from .github/workflows/run-all-e2e-tests.yml
rename to .github/workflows/run-e2e-tests-reusable-workflow.yml
index 0cc5fddab36..12ac4763adb 100644
--- a/.github/workflows/run-all-e2e-tests.yml
+++ b/.github/workflows/run-e2e-tests-reusable-workflow.yml
@@ -1,4 +1,6 @@
-name: Run All E2E Tests
+# This is a reusable workflow that runs E2E tests for Chainlink. 
+# It is not meant to be run on its own.
+name: Run E2E Tests
 on:
   # schedule:
   #   # Run every night at midnight UTC (0:00 AM) TODO: create separate workflow that uses this one 
@@ -18,21 +20,7 @@ on:
         default: "*"
         required: true
         type: string     
-  workflow_dispatch:
-    inputs:
-      custom-chainlink-image:
-        description: 'Enter custom Chainlink image to use for the tests. Example: "public.ecr.aws/chainlink/chainlink"'
-        required: false
-        type: string
-      chainlink-version:
-        description: 'Enter Chainlink version to use for the tests. Example: "v2.10.0" or sha'
-        required: true
-        type: string
-      test-ids:
-        description: 'Run all tests "*" by default. Or, enter test IDs to run separated by commas. Example: "run_all_in_ocr_tests_go,run_TestOCRv2Request_in_ocr2_test_go". Check all test IDs in .github/e2e-tests.yml'
-        default: "*"
-        required: true
-        type: string                
+               
 env: 
   CHAINLINK_IMAGE: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com/chainlink
 
diff --git a/.github/workflows/run-selected-e2e-tests.yml b/.github/workflows/run-selected-e2e-tests.yml
index e9009c6ac0c..929cf556265 100644
--- a/.github/workflows/run-selected-e2e-tests.yml
+++ b/.github/workflows/run-selected-e2e-tests.yml
@@ -9,14 +9,15 @@ on:
         required: false
         type: string
       test-ids:
-        description: 'Enter test IDs to run separated by commas. Example: "run_all_in_ocr_tests_go,run_TestOCRv2Request_in_ocr2_test_go". Check all test IDs in .github/e2e-tests.yml'
+        description: 'Run all tests "*" by default. Or, enter test IDs to run separated by commas. Example: "run_all_in_ocr_tests_go,run_TestOCRv2Request_in_ocr2_test_go". Check all test IDs in .github/e2e-tests.yml'
+        default: "*"
         required: true
         type: string
 
 jobs:
   call-run-e2e-tests-workflow:
     name: Run E2E Tests
-    uses: ./.github/workflows/run-all-e2e-tests.yml
+    uses: ./.github/workflows/run-e2e-tests-reusable-workflow.yml
     with:
       chainlink-version: ${{ github.event.inputs.chainlink-version }}
       test-ids: ${{ github.event.inputs.test-ids }}

From 04f2b32435839f8ae460a31c417874c698cad776 Mon Sep 17 00:00:00 2001
From: lukaszcl <120112546+lukaszcl@users.noreply.github.com>
Date: Sat, 22 Jun 2024 14:00:00 +0200
Subject: [PATCH 014/137] Add nighly workflow

---
 .../run-e2e-tests-reusable-workflow.yml         |  6 +-----
 .github/workflows/run-nightly-e2e-tests.yml     | 17 +++++++++++++++++
 2 files changed, 18 insertions(+), 5 deletions(-)
 create mode 100644 .github/workflows/run-nightly-e2e-tests.yml

diff --git a/.github/workflows/run-e2e-tests-reusable-workflow.yml b/.github/workflows/run-e2e-tests-reusable-workflow.yml
index 12ac4763adb..1b619c655ce 100644
--- a/.github/workflows/run-e2e-tests-reusable-workflow.yml
+++ b/.github/workflows/run-e2e-tests-reusable-workflow.yml
@@ -7,10 +7,6 @@ on:
   #   - cron: '0 0 * * *'
   workflow_call:
     inputs:
-      custom-chainlink-image:
-        description: 'Enter custom Chainlink image to use for the tests. Example: "public.ecr.aws/chainlink/chainlink"'
-        required: false
-        type: string
       chainlink-version:
         description: 'Enter Chainlink version to use for the tests. Example: "v2.10.0" or sha'
         required: true
@@ -19,7 +15,7 @@ on:
         description: 'Run all tests "*" by default. Or, enter test IDs to run separated by commas. Example: "run_all_in_ocr_tests_go,run_TestOCRv2Request_in_ocr2_test_go". Check all test IDs in .github/e2e-tests.yml'
         default: "*"
         required: true
-        type: string     
+        type: string
                
 env: 
   CHAINLINK_IMAGE: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com/chainlink
diff --git a/.github/workflows/run-nightly-e2e-tests.yml b/.github/workflows/run-nightly-e2e-tests.yml
new file mode 100644
index 00000000000..2b956919d30
--- /dev/null
+++ b/.github/workflows/run-nightly-e2e-tests.yml
@@ -0,0 +1,17 @@
+name: Run Nightly E2E Tests
+
+on:
+  schedule:
+    # Run every night at midnight UTC (0:00 AM) TODO: create separate workflow that uses this one 
+    - cron: '0 0 * * *'
+
+jobs:
+  call-run-e2e-tests-workflow:
+    # TODO: remove when the workflow is ready
+    if: false
+    name: Run E2E Tests
+    uses: ./.github/workflows/run-e2e-tests-reusable-workflow.yml
+    with:
+      chainlink-version: develop
+      test-ids: '*'
+    secrets: inherit

From 06f5922443bdfd0327389a56dce0627df1183016 Mon Sep 17 00:00:00 2001
From: lukaszcl <120112546+lukaszcl@users.noreply.github.com>
Date: Sat, 22 Jun 2024 14:10:25 +0200
Subject: [PATCH 015/137] try running k8s tests

---
 .github/e2e-tests.yml                         | 10 +++---
 .../run-e2e-tests-reusable-workflow.yml       | 34 ++++++++++++++-----
 2 files changed, 31 insertions(+), 13 deletions(-)

diff --git a/.github/e2e-tests.yml b/.github/e2e-tests.yml
index 1748ecee09e..9d5110005b5 100644
--- a/.github/e2e-tests.yml
+++ b/.github/e2e-tests.yml
@@ -23,19 +23,19 @@ test-runner-matrix:
     test-type: docker
     runs-on: ubuntu-latest
     cmd: cd integration-tests/ && go test smoke/ocr2_test.go -test.run ^TestOCRv2Request$ -test.parallel=1 -timeout 30m -count=1 -json
-    pyroscope-env: ci-smoke-ocr-evm-simulated-nightly
+    pyroscope-env: ci-smoke-ocr2-evm-simulated-nightly
 
   - id: TestOCRv2Basic_in_ocr2_test_go
     path: integration-tests/smoke/ocr2_test.go
     test-type: docker
     runs-on: ubuntu-latest
     cmd: cd integration-tests/ && go test smoke/ocr2_test.go -test.run ^TestOCRv2Basic$ -test.parallel=1 -timeout 30m -count=1 -json
-    pyroscope-env: ci-smoke-ocr-evm-simulated-nightly
+    pyroscope-env: ci-smoke-ocr2-evm-simulated-nightly
 
   # Example of configuration for running a test in Kubernetes Remote Runner
-  - id: TestOCRSoak_in_ocr_test_go
-    path: integration-tests/smoke/ocr2_test.go
+  - id: TestOCRSoak_in_soak/ocr_test.go
+    path: integration-tests/soak/ocr_test.go
     test-type: k8s-remote-runner
     runs-on: ubuntu-latest
     cmd: cd integration-tests/ && go test soak/ocr_test.go -test.run ^TestOCRSoak$ -test.parallel=1 -timeout 30m -count=1 -json
-    pyroscope-env: ci-smoke-ocr-evm-simulated-nightly
+    pyroscope-env: ci-smoke-ocr2-evm-simulated-nightly
diff --git a/.github/workflows/run-e2e-tests-reusable-workflow.yml b/.github/workflows/run-e2e-tests-reusable-workflow.yml
index 1b619c655ce..c4bad414d63 100644
--- a/.github/workflows/run-e2e-tests-reusable-workflow.yml
+++ b/.github/workflows/run-e2e-tests-reusable-workflow.yml
@@ -25,6 +25,8 @@ jobs:
     name: Load Test Configurations
     runs-on: ubuntu-latest
     outputs:
+      run-docker-tests: ${{ steps.check-matrices.outputs.run-docker-tests }}
+      run-k8s-tests: ${{ steps.check-matrices.outputs.run-k8s-tests }}
       docker-matrix: ${{ steps.set-docker-matrix.outputs.matrix }}
       k8s-runner-matrix: ${{ steps.set-k8s-runner-matrix.outputs.matrix }}
     steps:
@@ -46,23 +48,35 @@ jobs:
       - name: Generate K8s Tests Matrix
         id: set-k8s-runner-matrix
         run: |
-          MATRIX_JSON=$(go run .github/scripts/e2e_tests_tool/main.go -file .github/e2e-tests.yml -test-type 'k8s_remote_runner' -test-ids '${{ github.event.inputs.test-ids }}')
+          MATRIX_JSON=$(go run .github/scripts/e2e_tests_tool/main.go -file .github/e2e-tests.yml -test-type 'k8s-remote-runner' -test-ids '${{ github.event.inputs.test-ids }}')
           echo "K8s tests:"
           echo "$MATRIX_JSON" | jq
           echo "::set-output name=matrix::$MATRIX_JSON"
       - name: Check Test Matrices
+        id: check-matrices
         run: |
-          DOCKER_MATRIX=$(echo '${{ steps.set-docker-matrix.outputs.matrix }}' | jq '.tests == null or .tests == []')
-          K8S_MATRIX=$(echo '${{ steps.set-k8s-runner-matrix.outputs.matrix }}' | jq '.tests == null or .tests == []')
+          DOCKER_MATRIX_EMPTY=$(echo '${{ steps.set-docker-matrix.outputs.matrix }}' | jq '.tests == null or .tests == []')
+          K8S_MATRIX_EMPTY=$(echo '${{ steps.set-k8s-runner-matrix.outputs.matrix }}' | jq '.tests == null or .tests == []')
 
           # Check if jq commands succeeded
           if [ $? -ne 0 ]; then
             echo "JSON parse error occurred."
             exit 1
           fi
-      
+
+          if [[ "$DOCKER_MATRIX_EMPTY" == "true" ]]; then
+            echo "::set-output name=run-docker-tests::false"
+          else
+            echo "::set-output name=run-docker-tests::true"
+          fi
+          if [[ "$K8S_MATRIX_EMPTY" == "true" ]]; then
+            echo "::set-output name=run-k8s-tests::false"
+          else
+            echo "::set-output name=run-k8s-tests::true"
+          fi
+
           # Check if both matrices are empty
-          if [[ "$DOCKER_MATRIX" == "true" ]] && [[ "$K8S_MATRIX" == "true" ]]; then
+          if [[ "$DOCKER_MATRIX_EMPTY" == "true" ]] && [[ "$K8S_MATRIX_EMPTY" == "true" ]]; then
             echo "No tests found for inputs: '${{ toJson(github.event.inputs) }}'. Both Docker and Kubernetes tests matrices are empty"
             exit 1
           fi
@@ -73,6 +87,7 @@ jobs:
   docker-tests:
     name: Run Docker Tests (${{ matrix.tests.id }})
     needs: load-test-configurations
+    if: ${{ needs.load-test-configurations.outputs.run-docker-tests == 'true' }}
     runs-on: ${{ matrix.tests.runsOn }}
     strategy:
       fail-fast: false
@@ -147,10 +162,13 @@ jobs:
   # Run K8s tests using old remote runner
 
   build-k8s-runner-test-image:
-    if: false
+    needs: load-test-configurations
+    if: ${{ needs.load-test-configurations.outputs.run-k8s-tests == 'true' }}
     name: Build K8s Runner Test Image
     runs-on: ubuntu-latest
     steps:
+      - name: Checkout repository
+        uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2      
       - name: Build Test Runner Image
         uses: ./.github/actions/build-test-image
         with:
@@ -159,9 +177,9 @@ jobs:
           QA_AWS_ACCOUNT_NUMBER: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}   
 
   run-k8s-runner-tests:
-    if: false
-    name: Run K8s Tests (${{ matrix.tests.id }})
     needs: [load-test-configurations, build-k8s-runner-test-image]
+    if: ${{ needs.load-test-configurations.outputs.run-k8s-tests == 'true' }}
+    name: Run K8s Tests (${{ matrix.tests.id }})
     runs-on: ${{ matrix.tests.runs-on }}
     strategy:
       fail-fast: false

From 099d654446e2849f62ab6918b2dc1407b4e87e8e Mon Sep 17 00:00:00 2001
From: lukaszcl <120112546+lukaszcl@users.noreply.github.com>
Date: Sat, 22 Jun 2024 15:33:25 +0200
Subject: [PATCH 016/137] cleanup

---
 .github/workflows/run-e2e-tests-reusable-workflow.yml | 6 ------
 1 file changed, 6 deletions(-)

diff --git a/.github/workflows/run-e2e-tests-reusable-workflow.yml b/.github/workflows/run-e2e-tests-reusable-workflow.yml
index c4bad414d63..ebe82ad775c 100644
--- a/.github/workflows/run-e2e-tests-reusable-workflow.yml
+++ b/.github/workflows/run-e2e-tests-reusable-workflow.yml
@@ -2,9 +2,6 @@
 # It is not meant to be run on its own.
 name: Run E2E Tests
 on:
-  # schedule:
-  #   # Run every night at midnight UTC (0:00 AM) TODO: create separate workflow that uses this one 
-  #   - cron: '0 0 * * *'
   workflow_call:
     inputs:
       chainlink-version:
@@ -156,9 +153,6 @@ jobs:
           go_coverage_src_dir: /var/tmp/go-coverage
           go_coverage_dest_dir: ${{ github.workspace }}/.covdata
 
-      - name: Run Tests
-        run: echo "Running - ${{ matrix.tests.cmd }}"
-
   # Run K8s tests using old remote runner
 
   build-k8s-runner-test-image:

From 53858e0ddd98562ffd817dd1f17a0d1d52e2603a Mon Sep 17 00:00:00 2001
From: lukaszcl <120112546+lukaszcl@users.noreply.github.com>
Date: Sat, 22 Jun 2024 15:44:12 +0200
Subject: [PATCH 017/137] add permissions

---
 .github/workflows/run-e2e-tests-reusable-workflow.yml | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/.github/workflows/run-e2e-tests-reusable-workflow.yml b/.github/workflows/run-e2e-tests-reusable-workflow.yml
index ebe82ad775c..0fbd5f0cf34 100644
--- a/.github/workflows/run-e2e-tests-reusable-workflow.yml
+++ b/.github/workflows/run-e2e-tests-reusable-workflow.yml
@@ -160,6 +160,11 @@ jobs:
     if: ${{ needs.load-test-configurations.outputs.run-k8s-tests == 'true' }}
     name: Build K8s Runner Test Image
     runs-on: ubuntu-latest
+    permissions:
+      checks: write
+      pull-requests: write
+      id-token: write
+      contents: read
     steps:
       - name: Checkout repository
         uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2      

From 7db66128f290cc5ce86198409f64d3b51242b462 Mon Sep 17 00:00:00 2001
From: lukaszcl <120112546+lukaszcl@users.noreply.github.com>
Date: Sat, 22 Jun 2024 15:58:13 +0200
Subject: [PATCH 018/137] fix

---
 .github/workflows/run-e2e-tests-reusable-workflow.yml | 7 ++++++-
 .github/workflows/run-selected-e2e-tests.yml          | 3 ++-
 2 files changed, 8 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/run-e2e-tests-reusable-workflow.yml b/.github/workflows/run-e2e-tests-reusable-workflow.yml
index 0fbd5f0cf34..45d88ea4fe4 100644
--- a/.github/workflows/run-e2e-tests-reusable-workflow.yml
+++ b/.github/workflows/run-e2e-tests-reusable-workflow.yml
@@ -13,6 +13,9 @@ on:
         default: "*"
         required: true
         type: string
+    secrets:
+      QA_AWS_REGION:
+        required: true        
                
 env: 
   CHAINLINK_IMAGE: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com/chainlink
@@ -165,6 +168,8 @@ jobs:
       pull-requests: write
       id-token: write
       contents: read
+    env:
+      ENV_JOB_IMAGE_BASE: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com/chainlink-tests
     steps:
       - name: Checkout repository
         uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2      
@@ -173,7 +178,7 @@ jobs:
         with:
           QA_AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }}
           QA_AWS_REGION: ${{ secrets.QA_AWS_REGION }}
-          QA_AWS_ACCOUNT_NUMBER: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}   
+          QA_AWS_ACCOUNT_NUMBER: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}
 
   run-k8s-runner-tests:
     needs: [load-test-configurations, build-k8s-runner-test-image]
diff --git a/.github/workflows/run-selected-e2e-tests.yml b/.github/workflows/run-selected-e2e-tests.yml
index 929cf556265..ed2d03710a4 100644
--- a/.github/workflows/run-selected-e2e-tests.yml
+++ b/.github/workflows/run-selected-e2e-tests.yml
@@ -21,4 +21,5 @@ jobs:
     with:
       chainlink-version: ${{ github.event.inputs.chainlink-version }}
       test-ids: ${{ github.event.inputs.test-ids }}
-    secrets: inherit
+    secrets:
+      QA_AWS_REGION: ${{ secrets.QA_AWS_REGION }}

From 5611e1711595582ad8ffdb1e77c06d9ac92a00cb Mon Sep 17 00:00:00 2001
From: lukaszcl <120112546+lukaszcl@users.noreply.github.com>
Date: Sat, 22 Jun 2024 16:04:30 +0200
Subject: [PATCH 019/137] test

---
 .../run-e2e-tests-reusable-workflow.yml       |  9 +++++++
 .github/workflows/run-selected-e2e-tests.yml  | 27 +++++++++++++++++++
 2 files changed, 36 insertions(+)

diff --git a/.github/workflows/run-e2e-tests-reusable-workflow.yml b/.github/workflows/run-e2e-tests-reusable-workflow.yml
index 45d88ea4fe4..54012d19729 100644
--- a/.github/workflows/run-e2e-tests-reusable-workflow.yml
+++ b/.github/workflows/run-e2e-tests-reusable-workflow.yml
@@ -163,7 +163,9 @@ jobs:
     if: ${{ needs.load-test-configurations.outputs.run-k8s-tests == 'true' }}
     name: Build K8s Runner Test Image
     runs-on: ubuntu-latest
+    environment: integration
     permissions:
+      actions: read
       checks: write
       pull-requests: write
       id-token: write
@@ -171,6 +173,13 @@ jobs:
     env:
       ENV_JOB_IMAGE_BASE: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com/chainlink-tests
     steps:
+      - name: Check secret presence
+        run: |
+          if [ -z "${{ secrets.QA_AWS_REGION }}" ]; then
+            echo "Secret is empty"
+          else
+            echo "Secret is present"
+          fi
       - name: Checkout repository
         uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2      
       - name: Build Test Runner Image
diff --git a/.github/workflows/run-selected-e2e-tests.yml b/.github/workflows/run-selected-e2e-tests.yml
index ed2d03710a4..17521535414 100644
--- a/.github/workflows/run-selected-e2e-tests.yml
+++ b/.github/workflows/run-selected-e2e-tests.yml
@@ -15,11 +15,38 @@ on:
         type: string
 
 jobs:
+  test:
+    runs-on: ubuntu-latest
+    environment: integration
+    permissions:
+      actions: read
+      checks: write
+      pull-requests: write
+      id-token: write
+      contents: read
+    env:
+      ENV_JOB_IMAGE_BASE: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com/chainlink-tests
+    steps:
+      - name: Check secret presence
+        run: |
+          if [ -z "${{ secrets.QA_AWS_REGION }}" ]; then
+            echo "Secret is empty"
+          else
+            echo "Secret is present"
+          fi
+
   call-run-e2e-tests-workflow:
     name: Run E2E Tests
     uses: ./.github/workflows/run-e2e-tests-reusable-workflow.yml
+    permissions:
+      actions: read
+      checks: write
+      pull-requests: write
+      id-token: write
+      contents: read    
     with:
       chainlink-version: ${{ github.event.inputs.chainlink-version }}
       test-ids: ${{ github.event.inputs.test-ids }}
     secrets:
       QA_AWS_REGION: ${{ secrets.QA_AWS_REGION }}
+

From f54a25bc69dda88399212cb88f52e64a20b19ad3 Mon Sep 17 00:00:00 2001
From: lukaszcl <120112546+lukaszcl@users.noreply.github.com>
Date: Sat, 22 Jun 2024 16:36:51 +0200
Subject: [PATCH 020/137] Fix

---
 .../run-e2e-tests-reusable-workflow.yml       | 26 +++++++++++++++++++
 .github/workflows/run-selected-e2e-tests.yml  | 15 ++++++++++-
 2 files changed, 40 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/run-e2e-tests-reusable-workflow.yml b/.github/workflows/run-e2e-tests-reusable-workflow.yml
index 54012d19729..1feab4f778e 100644
--- a/.github/workflows/run-e2e-tests-reusable-workflow.yml
+++ b/.github/workflows/run-e2e-tests-reusable-workflow.yml
@@ -15,6 +15,32 @@ on:
         type: string
     secrets:
       QA_AWS_REGION:
+        required: true
+      QA_AWS_ROLE_TO_ASSUME:
+        required: true
+      QA_AWS_ACCOUNT_NUMBER:
+        required: true
+      QA_PYROSCOPE_INSTANCE:
+        required: true
+      QA_PYROSCOPE_KEY:
+        required: true
+      QA_KUBECONFIG:
+        required: true
+      GRAFANA_INTERNAL_TENANT_ID:
+        required: true
+      GRAFANA_INTERNAL_BASIC_AUTH:
+        required: true
+      GRAFANA_INTERNAL_HOST:
+        required: true
+      GRAFANA_INTERNAL_URL_SHORTENER_TOKEN:
+        required: true
+      GITHUB_TOKEN:
+        required: true  
+      AWS_REGION:
+        required: true
+      AWS_OIDC_IAM_ROLE_VALIDATION_PROD_ARN:
+        required: true
+      AWS_API_GW_HOST_GRAFANA:
         required: true        
                
 env: 
diff --git a/.github/workflows/run-selected-e2e-tests.yml b/.github/workflows/run-selected-e2e-tests.yml
index 17521535414..4737e55e180 100644
--- a/.github/workflows/run-selected-e2e-tests.yml
+++ b/.github/workflows/run-selected-e2e-tests.yml
@@ -49,4 +49,17 @@ jobs:
       test-ids: ${{ github.event.inputs.test-ids }}
     secrets:
       QA_AWS_REGION: ${{ secrets.QA_AWS_REGION }}
-
+      QA_AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }}
+      QA_AWS_ACCOUNT_NUMBER: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}
+      QA_PYROSCOPE_INSTANCE: ${{ secrets.QA_PYROSCOPE_INSTANCE }}
+      QA_PYROSCOPE_KEY: ${{ secrets.QA_PYROSCOPE_KEY }}
+      QA_KUBECONFIG: ${{ secrets.QA_KUBECONFIG }}
+      GRAFANA_INTERNAL_TENANT_ID: ${{ secrets.GRAFANA_INTERNAL_TENANT_ID }}
+      GRAFANA_INTERNAL_BASIC_AUTH: ${{ secrets.GRAFANA_INTERNAL_BASIC_AUTH }}
+      GRAFANA_INTERNAL_HOST: ${{ secrets.GRAFANA_INTERNAL_HOST }}
+      GRAFANA_INTERNAL_URL_SHORTENER_TOKEN: ${{ secrets.GRAFANA_INTERNAL_URL_SHORTENER_TOKEN }}
+      GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}  
+      AWS_REGION: ${{ secrets.QA_AWS_REGION }}
+      AWS_OIDC_IAM_ROLE_VALIDATION_PROD_ARN: ${{ secrets.AWS_OIDC_IAM_ROLE_VALIDATION_PROD_ARN }}
+      AWS_API_GW_HOST_GRAFANA: ${{ secrets.AWS_API_GW_HOST_GRAFANA }}        
+      

From a6ca1ff648cb3e9dbbe685637e80d5f8ad6cf9e9 Mon Sep 17 00:00:00 2001
From: lukaszcl <120112546+lukaszcl@users.noreply.github.com>
Date: Sat, 22 Jun 2024 16:37:50 +0200
Subject: [PATCH 021/137] fix

---
 .github/workflows/run-e2e-tests-reusable-workflow.yml | 6 +++---
 .github/workflows/run-selected-e2e-tests.yml          | 4 ++--
 2 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/.github/workflows/run-e2e-tests-reusable-workflow.yml b/.github/workflows/run-e2e-tests-reusable-workflow.yml
index 1feab4f778e..48bdbcaff05 100644
--- a/.github/workflows/run-e2e-tests-reusable-workflow.yml
+++ b/.github/workflows/run-e2e-tests-reusable-workflow.yml
@@ -34,7 +34,7 @@ on:
         required: true
       GRAFANA_INTERNAL_URL_SHORTENER_TOKEN:
         required: true
-      GITHUB_TOKEN:
+      GH_TOKEN:
         required: true  
       AWS_REGION:
         required: true
@@ -171,7 +171,7 @@ jobs:
             ./integration-tests/smoke/logs/
             /tmp/gotest.log
           publish_check_name: ${{ matrix.tests.id }}
-          token: ${{ secrets.GITHUB_TOKEN }}
+          token: ${{ secrets.GH_TOKEN }}
           go_mod_path: ./integration-tests/go.mod
           cache_key_id: core-e2e-${{ env.MOD_CACHE_VERSION }}
           cache_restore_only: "true"
@@ -254,7 +254,7 @@ jobs:
           test_download_vendor_packages_command: make gomod
           cl_repo: ${{ env.CHAINLINK_IMAGE }}
           cl_image_tag: ${{ env.CHAINLINK_VERSION }}
-          token: ${{ secrets.GITHUB_TOKEN }}
+          token: ${{ secrets.GH_TOKEN }}
           should_cleanup: false
           go_mod_path: ./integration-tests/go.mod
           QA_AWS_REGION: ${{ secrets.QA_AWS_REGION }}
diff --git a/.github/workflows/run-selected-e2e-tests.yml b/.github/workflows/run-selected-e2e-tests.yml
index 4737e55e180..44e038abba4 100644
--- a/.github/workflows/run-selected-e2e-tests.yml
+++ b/.github/workflows/run-selected-e2e-tests.yml
@@ -58,8 +58,8 @@ jobs:
       GRAFANA_INTERNAL_BASIC_AUTH: ${{ secrets.GRAFANA_INTERNAL_BASIC_AUTH }}
       GRAFANA_INTERNAL_HOST: ${{ secrets.GRAFANA_INTERNAL_HOST }}
       GRAFANA_INTERNAL_URL_SHORTENER_TOKEN: ${{ secrets.GRAFANA_INTERNAL_URL_SHORTENER_TOKEN }}
-      GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}  
+      GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}  
       AWS_REGION: ${{ secrets.QA_AWS_REGION }}
       AWS_OIDC_IAM_ROLE_VALIDATION_PROD_ARN: ${{ secrets.AWS_OIDC_IAM_ROLE_VALIDATION_PROD_ARN }}
       AWS_API_GW_HOST_GRAFANA: ${{ secrets.AWS_API_GW_HOST_GRAFANA }}        
-      
+

From c2b411a664d6d6b4bda710bbcaaba9d8c8d82256 Mon Sep 17 00:00:00 2001
From: lukaszcl <120112546+lukaszcl@users.noreply.github.com>
Date: Sat, 22 Jun 2024 16:44:24 +0200
Subject: [PATCH 022/137] test remote runner

---
 .../run-e2e-tests-reusable-workflow.yml       | 26 ++++++++-----------
 1 file changed, 11 insertions(+), 15 deletions(-)

diff --git a/.github/workflows/run-e2e-tests-reusable-workflow.yml b/.github/workflows/run-e2e-tests-reusable-workflow.yml
index 48bdbcaff05..8559d312369 100644
--- a/.github/workflows/run-e2e-tests-reusable-workflow.yml
+++ b/.github/workflows/run-e2e-tests-reusable-workflow.yml
@@ -199,27 +199,21 @@ jobs:
     env:
       ENV_JOB_IMAGE_BASE: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com/chainlink-tests
     steps:
-      - name: Check secret presence
-        run: |
-          if [ -z "${{ secrets.QA_AWS_REGION }}" ]; then
-            echo "Secret is empty"
-          else
-            echo "Secret is present"
-          fi
       - name: Checkout repository
         uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2      
-      - name: Build Test Runner Image
-        uses: ./.github/actions/build-test-image
-        with:
-          QA_AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }}
-          QA_AWS_REGION: ${{ secrets.QA_AWS_REGION }}
-          QA_AWS_ACCOUNT_NUMBER: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}
+      # TODO: Uncomment this step
+      # - name: Build Test Runner Image
+      #   uses: ./.github/actions/build-test-image
+      #   with:
+      #     QA_AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }}
+      #     QA_AWS_REGION: ${{ secrets.QA_AWS_REGION }}
+      #     QA_AWS_ACCOUNT_NUMBER: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}
 
   run-k8s-runner-tests:
     needs: [load-test-configurations, build-k8s-runner-test-image]
     if: ${{ needs.load-test-configurations.outputs.run-k8s-tests == 'true' }}
     name: Run K8s Tests (${{ matrix.tests.id }})
-    runs-on: ${{ matrix.tests.runs-on }}
+    runs-on: ${{ matrix.tests.runsOn }}
     strategy:
       fail-fast: false
       matrix: ${{fromJson(needs.load-test-configurations.outputs.k8s-runner-matrix)}}
@@ -245,7 +239,9 @@ jobs:
           DETACH_RUNNER: true
           TEST_SUITE: soak
           TEST_ARGS: -test.timeout 900h -test.memprofile memprofile.out -test.cpuprofile profile.out
-          ENV_JOB_IMAGE: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com/chainlink-tests:${{ github.sha }}
+          # TODO: Remove and use sha step below
+          ENV_JOB_IMAGE: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com/chainlink-tests:e7bce59ea616e4f13b8506f1a2414e3df586b130
+          # ENV_JOB_IMAGE: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com/chainlink-tests:${{ github.sha }}
           # We can comment these out when we have a stable soak test and aren't worried about resource consumption
           TEST_UPLOAD_CPU_PROFILE: true
           TEST_UPLOAD_MEM_PROFILE: true

From 2c9ab53e900ece34b4310da5a96ff4df35f00494 Mon Sep 17 00:00:00 2001
From: lukaszcl <120112546+lukaszcl@users.noreply.github.com>
Date: Sat, 22 Jun 2024 16:53:04 +0200
Subject: [PATCH 023/137] Add test suite param

---
 .github/e2e-tests.yml                           |  1 +
 .github/scripts/e2e_tests_tool/main.go          | 17 +++++++++--------
 .../run-e2e-tests-reusable-workflow.yml         |  2 +-
 3 files changed, 11 insertions(+), 9 deletions(-)

diff --git a/.github/e2e-tests.yml b/.github/e2e-tests.yml
index 9d5110005b5..d0b768d4750 100644
--- a/.github/e2e-tests.yml
+++ b/.github/e2e-tests.yml
@@ -36,6 +36,7 @@ test-runner-matrix:
   - id: TestOCRSoak_in_soak/ocr_test.go
     path: integration-tests/soak/ocr_test.go
     test-type: k8s-remote-runner
+    remote-runner-test-suite: soak
     runs-on: ubuntu-latest
     cmd: cd integration-tests/ && go test soak/ocr_test.go -test.run ^TestOCRSoak$ -test.parallel=1 -timeout 30m -count=1 -json
     pyroscope-env: ci-smoke-ocr2-evm-simulated-nightly
diff --git a/.github/scripts/e2e_tests_tool/main.go b/.github/scripts/e2e_tests_tool/main.go
index b07e5dfad52..a3f6e4059eb 100644
--- a/.github/scripts/e2e_tests_tool/main.go
+++ b/.github/scripts/e2e_tests_tool/main.go
@@ -13,14 +13,15 @@ import (
 
 // Test defines the structure of a test entry in the YAML file.
 type Test struct {
-	ID           string   `yaml:"id" json:"id"`
-	Name         string   `yaml:"name" json:"name"`
-	Path         string   `yaml:"path" json:"path"`
-	TestType     string   `yaml:"test-type" json:"testType"`
-	RunsOn       string   `yaml:"runs-on" json:"runsOn"`
-	Cmd          string   `yaml:"cmd" json:"cmd"`
-	PyroscopeEnv string   `yaml:"pyroscope-env" json:"pyroscopeEnv"`
-	Trigger      []string `yaml:"trigger" json:"trigger"`
+	ID                    string   `yaml:"id" json:"id"`
+	Name                  string   `yaml:"name" json:"name"`
+	Path                  string   `yaml:"path" json:"path"`
+	TestType              string   `yaml:"test-type" json:"testType"`
+	RunsOn                string   `yaml:"runs-on" json:"runsOn"`
+	Cmd                   string   `yaml:"cmd" json:"cmd"`
+	RemoteRunnerTestSuite string   `yaml:"remote-runner-test-suite" json:"remoteRunnerTestSuite"`
+	PyroscopeEnv          string   `yaml:"pyroscope-env" json:"pyroscopeEnv"`
+	Trigger               []string `yaml:"trigger" json:"trigger"`
 }
 
 // Config represents the tests configuration.
diff --git a/.github/workflows/run-e2e-tests-reusable-workflow.yml b/.github/workflows/run-e2e-tests-reusable-workflow.yml
index 8559d312369..33bdc9eedd2 100644
--- a/.github/workflows/run-e2e-tests-reusable-workflow.yml
+++ b/.github/workflows/run-e2e-tests-reusable-workflow.yml
@@ -237,7 +237,7 @@ jobs:
         uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/run-tests@af92c5fae8dcf1659201e907db82d221fc304b94 # v2.3.21
         env:
           DETACH_RUNNER: true
-          TEST_SUITE: soak
+          TEST_SUITE: ${{ matrix.tests.remoteRunnerTestSuite }}
           TEST_ARGS: -test.timeout 900h -test.memprofile memprofile.out -test.cpuprofile profile.out
           # TODO: Remove and use sha step below
           ENV_JOB_IMAGE: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com/chainlink-tests:e7bce59ea616e4f13b8506f1a2414e3df586b130

From ada4954f001c832bfdaba12783111cbc9f79914d Mon Sep 17 00:00:00 2001
From: lukaszcl <120112546+lukaszcl@users.noreply.github.com>
Date: Sat, 22 Jun 2024 16:54:02 +0200
Subject: [PATCH 024/137] Run all_tests_in_ocr_tests_go in parallel

---
 .github/e2e-tests.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/e2e-tests.yml b/.github/e2e-tests.yml
index d0b768d4750..1cd0ebff16d 100644
--- a/.github/e2e-tests.yml
+++ b/.github/e2e-tests.yml
@@ -12,7 +12,7 @@ test-runner-matrix:
     path: integration-tests/smoke/ocr_test.go
     test-type: docker
     runs-on: ubuntu-latest
-    cmd: cd integration-tests/ && go test smoke/ocr_test.go -test.parallel=1 -timeout 30m -count=1 -json
+    cmd: cd integration-tests/ && go test smoke/ocr_test.go -timeout 30m -count=1 -json
     # workflow: [nightly-e2e-tests.yml]
     trigger: ["push", "nightly"]
     pyroscope-env: ci-smoke-ocr-evm-simulated-nightly

From 3d587e8888edaa9585ae80f091386fb31548cb2e Mon Sep 17 00:00:00 2001
From: lukaszcl <120112546+lukaszcl@users.noreply.github.com>
Date: Sat, 22 Jun 2024 17:04:06 +0200
Subject: [PATCH 025/137] Update e2e-tests.yml

---
 .github/e2e-tests.yml                  | 10 +++++-----
 .github/scripts/e2e_tests_tool/main.go |  1 -
 2 files changed, 5 insertions(+), 6 deletions(-)

diff --git a/.github/e2e-tests.yml b/.github/e2e-tests.yml
index 1cd0ebff16d..ea3bad6364d 100644
--- a/.github/e2e-tests.yml
+++ b/.github/e2e-tests.yml
@@ -8,7 +8,7 @@
 test-runner-matrix:
 
   # Example of 1 runner for all tests in integration-tests/smoke/ocr_test.go
-  - id: all_tests_in_ocr_tests_go 
+  - id: smoke/ocr_test.go:* 
     path: integration-tests/smoke/ocr_test.go
     test-type: docker
     runs-on: ubuntu-latest
@@ -18,22 +18,22 @@ test-runner-matrix:
     pyroscope-env: ci-smoke-ocr-evm-simulated-nightly
 
   # Example of 2 separate runners for the same test file but different tests. Can be used if tests if are too heavy to run on the same runner
-  - id: TestOCRv2Request_in_ocr2_test_go
+  - id: smoke/ocr2_test.go:^TestOCRv2Request$
     path: integration-tests/smoke/ocr2_test.go
     test-type: docker
     runs-on: ubuntu-latest
     cmd: cd integration-tests/ && go test smoke/ocr2_test.go -test.run ^TestOCRv2Request$ -test.parallel=1 -timeout 30m -count=1 -json
     pyroscope-env: ci-smoke-ocr2-evm-simulated-nightly
 
-  - id: TestOCRv2Basic_in_ocr2_test_go
+  - id: smoke/ocr2_test.go:^TestOCRv2Basic$
     path: integration-tests/smoke/ocr2_test.go
     test-type: docker
     runs-on: ubuntu-latest
     cmd: cd integration-tests/ && go test smoke/ocr2_test.go -test.run ^TestOCRv2Basic$ -test.parallel=1 -timeout 30m -count=1 -json
     pyroscope-env: ci-smoke-ocr2-evm-simulated-nightly
 
-  # Example of configuration for running a test in Kubernetes Remote Runner
-  - id: TestOCRSoak_in_soak/ocr_test.go
+  # Example of configuration for running a single soak test in Kubernetes Remote Runner
+  - id: soak/ocr_test.go:^TestOCRSoak$
     path: integration-tests/soak/ocr_test.go
     test-type: k8s-remote-runner
     remote-runner-test-suite: soak
diff --git a/.github/scripts/e2e_tests_tool/main.go b/.github/scripts/e2e_tests_tool/main.go
index a3f6e4059eb..045c14f796c 100644
--- a/.github/scripts/e2e_tests_tool/main.go
+++ b/.github/scripts/e2e_tests_tool/main.go
@@ -14,7 +14,6 @@ import (
 // Test defines the structure of a test entry in the YAML file.
 type Test struct {
 	ID                    string   `yaml:"id" json:"id"`
-	Name                  string   `yaml:"name" json:"name"`
 	Path                  string   `yaml:"path" json:"path"`
 	TestType              string   `yaml:"test-type" json:"testType"`
 	RunsOn                string   `yaml:"runs-on" json:"runsOn"`

From fb9589084368ef676733e521a9b1115a5e4c1618 Mon Sep 17 00:00:00 2001
From: lukaszcl <120112546+lukaszcl@users.noreply.github.com>
Date: Sat, 22 Jun 2024 19:37:42 +0200
Subject: [PATCH 026/137] Add check-tests cmd

---
 .../e2e_tests_tool_final/cmd/check_tests.go   | 154 ++++++++++++++++++
 .../cmd/check_tests_test.go                   |  47 ++++++
 .../e2e_tests_tool_final/cmd/find_tests.go    |  26 +++
 .../scripts/e2e_tests_tool_final/cmd/root.go  |  31 ++++
 .github/scripts/e2e_tests_tool_final/main.go  |   7 +
 go.mod                                        |   4 +-
 go.sum                                        |   4 +
 7 files changed, 271 insertions(+), 2 deletions(-)
 create mode 100644 .github/scripts/e2e_tests_tool_final/cmd/check_tests.go
 create mode 100644 .github/scripts/e2e_tests_tool_final/cmd/check_tests_test.go
 create mode 100644 .github/scripts/e2e_tests_tool_final/cmd/find_tests.go
 create mode 100644 .github/scripts/e2e_tests_tool_final/cmd/root.go
 create mode 100644 .github/scripts/e2e_tests_tool_final/main.go

diff --git a/.github/scripts/e2e_tests_tool_final/cmd/check_tests.go b/.github/scripts/e2e_tests_tool_final/cmd/check_tests.go
new file mode 100644
index 00000000000..42079288d36
--- /dev/null
+++ b/.github/scripts/e2e_tests_tool_final/cmd/check_tests.go
@@ -0,0 +1,154 @@
+package cmd
+
+import (
+	"fmt"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"regexp"
+	"strings"
+
+	"github.com/spf13/cobra"
+	"gopkg.in/yaml.v3"
+)
+
+type JobConfig struct {
+	Jobs map[string]struct {
+		Strategy struct {
+			Matrix struct {
+				Test []struct {
+					Path     string `yaml:"path"`
+					TestOpts string `yaml:"testOpts"`
+				} `yaml:"test"`
+			} `yaml:"matrix"`
+		} `yaml:"strategy"`
+	} `yaml:"jobs"`
+}
+
+var checkTestsCmd = &cobra.Command{
+	Use:   "check-tests [directory] [yaml file]",
+	Short: "Check if all tests in a directory are included in the YAML file",
+	Args:  cobra.ExactArgs(2),
+	Run: func(cmd *cobra.Command, args []string) {
+		directory := args[0]
+		yamlFile := args[1]
+		tests, err := extractTests(directory)
+		if err != nil {
+			fmt.Println("Error extracting tests:", err)
+			os.Exit(1)
+		}
+
+		checkTestsInPipeline(yamlFile, tests)
+	},
+}
+
+type Test struct {
+	Name string
+	Path string
+}
+
+type TestRun struct {
+	ID                    string   `yaml:"id" json:"id"`
+	Path                  string   `yaml:"path" json:"path"`
+	TestType              string   `yaml:"test-type" json:"testType"`
+	RunsOn                string   `yaml:"runs-on" json:"runsOn"`
+	Cmd                   string   `yaml:"cmd" json:"cmd"`
+	RemoteRunnerTestSuite string   `yaml:"remote-runner-test-suite" json:"remoteRunnerTestSuite"`
+	PyroscopeEnv          string   `yaml:"pyroscope-env" json:"pyroscopeEnv"`
+	Trigger               []string `yaml:"trigger" json:"trigger"`
+}
+
+type Config struct {
+	Tests []TestRun `yaml:"test-runner-matrix"`
+}
+
+func extractTests(dir string) ([]Test, error) {
+	var tests []Test
+	err := filepath.Walk(dir, func(path string, info os.FileInfo, err error) error {
+		if err != nil {
+			return err
+		}
+		if !info.IsDir() && strings.HasSuffix(info.Name(), "_test.go") {
+			content, err := ioutil.ReadFile(path)
+			if err != nil {
+				return err
+			}
+			re := regexp.MustCompile(`func (Test\w+)`)
+			matches := re.FindAllSubmatch(content, -1)
+			for _, match := range matches {
+				tests = append(tests, Test{
+					Name: string(match[1]),
+					Path: path,
+				})
+			}
+		}
+		return nil
+	})
+	return tests, err
+}
+
+func checkTestsInPipeline(yamlFile string, tests []Test) {
+	data, err := ioutil.ReadFile(yamlFile)
+	if err != nil {
+		fmt.Printf("Error reading YAML file: %s\n", err)
+		return
+	}
+
+	var config Config
+	err = yaml.Unmarshal(data, &config)
+	if err != nil {
+		fmt.Printf("Error parsing YAML: %s\n", err)
+		return
+	}
+
+	missingTests := []string{} // Track missing tests
+
+	for _, test := range tests {
+		found := false
+		for _, item := range config.Tests {
+			if item.Path == test.Path {
+				if strings.Contains(item.Cmd, "-test.run") {
+					if matchTestNameInCmd(item.Cmd, test.Name) {
+						found = true
+						break
+					}
+				} else {
+					found = true
+					break
+				}
+			}
+		}
+		if !found {
+			missingTests = append(missingTests, fmt.Sprintf("Test '%s' in file '%s' does not have CI configuration", test.Name, test.Path))
+		}
+	}
+
+	if len(missingTests) > 0 {
+		for _, missing := range missingTests {
+			fmt.Println(missing)
+		}
+		fmt.Printf("\nERROR: These tests must be added to the E2E CI conf in '%s' file.\n", yamlFile)
+		os.Exit(1) // Exit with a failure status
+	}
+}
+
+// matchTestNameInCmd checks if the given test name matches the -test.run pattern in the command string.
+func matchTestNameInCmd(cmd string, testName string) bool {
+	testRunRegex := regexp.MustCompile(`-test\.run ([^\s]+)`)
+	matches := testRunRegex.FindStringSubmatch(cmd)
+	if len(matches) > 1 {
+		// Extract the regex pattern used in the -test.run command
+		pattern := matches[1]
+
+		// Escape regex metacharacters in the testName before matching
+		escapedTestName := regexp.QuoteMeta(testName)
+
+		// Check if the escaped test name matches the extracted pattern
+		return regexp.MustCompile(pattern).MatchString(escapedTestName)
+	}
+	return false
+}
+
+func init() {
+	rootCmd.AddCommand(checkTestsCmd)
+}
diff --git a/.github/scripts/e2e_tests_tool_final/cmd/check_tests_test.go b/.github/scripts/e2e_tests_tool_final/cmd/check_tests_test.go
new file mode 100644
index 00000000000..4b7f50e7f06
--- /dev/null
+++ b/.github/scripts/e2e_tests_tool_final/cmd/check_tests_test.go
@@ -0,0 +1,47 @@
+package cmd
+
+import (
+	"testing"
+)
+
+func TestMatchTestNameInCmd(t *testing.T) {
+	tests := []struct {
+		cmd      string
+		testName string
+		expected bool
+	}{
+		{"go test -test.run ^TestExample$", "TestExample", true},
+		{"go test -test.run ^TestExample$", "TestAnother", false},
+		{"go test -test.run ^TestExample$ -v", "TestExample", true},
+		{"go test -test.run ^TestExamplePart$", "TestExample", false},
+		{"go test -test.run ^TestWithNumbers123$", "TestWithNumbers123", true},
+		{"go test -test.run ^Test_With_Underscores$", "Test_With_Underscores", true},
+		{"go test -test.run ^Test-With-Dash$", "Test-With-Dash", true},
+		{"go test -test.run ^TestWithSpace Space$", "TestWithSpace Space", true},
+		{"go test -test.run ^TestWithNewline\nNewline$", "TestWithNewline\nNewline", true},
+		{"go test -test.run ^TestOne$|^TestTwo$", "TestOne", true},
+		{"go test -test.run ^TestOne$|^TestTwo$", "TestTwo", true},
+		{"go test -test.run ^TestOne$|^TestTwo$", "TestThree", false},
+		{"go test -test.run TestOne|TestTwo", "TestTwo", true},
+		{"go test -test.run TestOne|TestTwo", "TestOne", true},
+		{"go test -test.run TestOne|TestTwo|TestThree", "TestFour", false},
+		{"go test -test.run ^TestOne$|TestTwo$", "TestTwo", true},
+		{"go test -test.run ^TestOne$|TestTwo|TestThree$", "TestThree", true},
+		{"go test -test.run TestOne|TestTwo|TestThree", "TestOne", true},
+		{"go test -test.run TestOne|TestTwo|TestThree", "TestThree", true},
+		{"go test -test.run ^TestA$|^TestB$|^TestC$", "TestA", true},
+		{"go test -test.run ^TestA$|^TestB$|^TestC$", "TestB", true},
+		{"go test -test.run ^TestA$|^TestB$|^TestC$", "TestD", false},
+		{"go test -test.run TestA|^TestB$|TestC", "TestB", true},
+		{"go test -test.run ^TestA|^TestB|TestC$", "TestA", true},
+		{"go test -test.run ^TestA|^TestB|TestC$", "TestC", true},
+		{"go test -test.run ^TestA|^TestB|TestC$", "TestD", false},
+	}
+
+	for _, tt := range tests {
+		result := matchTestNameInCmd(tt.cmd, tt.testName)
+		if result != tt.expected {
+			t.Errorf("matchTestNameInCmd(%s, %s) = %t; expected %t", tt.cmd, tt.testName, result, tt.expected)
+		}
+	}
+}
diff --git a/.github/scripts/e2e_tests_tool_final/cmd/find_tests.go b/.github/scripts/e2e_tests_tool_final/cmd/find_tests.go
new file mode 100644
index 00000000000..e105f90adf1
--- /dev/null
+++ b/.github/scripts/e2e_tests_tool_final/cmd/find_tests.go
@@ -0,0 +1,26 @@
+package cmd
+
+import (
+	"fmt"
+	"os"
+
+	"github.com/spf13/cobra"
+)
+
+// findTestsCmd represents the find-tests command
+var findTestsCmd = &cobra.Command{
+	Use:   "find-tests [path]",
+	Short: "Find all Go test functions in a directory",
+	Args:  cobra.ExactArgs(1),
+	Run: func(cmd *cobra.Command, args []string) {
+		path := args[0]
+		tests, err := extractTests(path)
+		if err != nil {
+			fmt.Println("Error extracting tests:", err)
+			os.Exit(1)
+		}
+		for _, t := range tests {
+			fmt.Printf("%+v", t)
+		}
+	},
+}
diff --git a/.github/scripts/e2e_tests_tool_final/cmd/root.go b/.github/scripts/e2e_tests_tool_final/cmd/root.go
new file mode 100644
index 00000000000..0a04d04f3df
--- /dev/null
+++ b/.github/scripts/e2e_tests_tool_final/cmd/root.go
@@ -0,0 +1,31 @@
+package cmd
+
+import (
+	"os"
+
+	"github.com/spf13/cobra"
+)
+
+// rootCmd represents the base command when called without any subcommands
+var rootCmd = &cobra.Command{
+	Use:   "e2e_tests_ci_tool",
+	Short: "A brief description of your application",
+	Long: `A longer description that spans multiple lines and likely contains
+examples and usage of using your application`,
+}
+
+// Execute adds all child commands to the root command and sets flags appropriately.
+// This is called by main.main(). It only needs to happen once to the rootCmd.
+func Execute() {
+	err := rootCmd.Execute()
+	if err != nil {
+		os.Exit(1)
+	}
+}
+
+func init() {
+	rootCmd.Flags().BoolP("toggle", "t", false, "Help message for toggle")
+
+	rootCmd.AddCommand(findTestsCmd)
+	rootCmd.AddCommand(checkTestsCmd)
+}
diff --git a/.github/scripts/e2e_tests_tool_final/main.go b/.github/scripts/e2e_tests_tool_final/main.go
new file mode 100644
index 00000000000..2cd999ee492
--- /dev/null
+++ b/.github/scripts/e2e_tests_tool_final/main.go
@@ -0,0 +1,7 @@
+package main
+
+import "github.com/smartcontractkit/chainlink/v2/.github/scripts/e2e_tests_tool_final/cmd"
+
+func main() {
+	cmd.Execute()
+}
diff --git a/go.mod b/go.mod
index 6208b5e8762..7ca51538e84 100644
--- a/go.mod
+++ b/go.mod
@@ -165,7 +165,7 @@ require (
 	github.com/cosmos/ibc-go/v7 v7.0.1 // indirect
 	github.com/cosmos/ics23/go v0.9.1-0.20221207100636-b1abd8678aab // indirect
 	github.com/cosmos/ledger-cosmos-go v0.12.1 // indirect
-	github.com/cpuguy83/go-md2man/v2 v2.0.3 // indirect
+	github.com/cpuguy83/go-md2man/v2 v2.0.4 // indirect
 	github.com/crate-crypto/go-ipa v0.0.0-20231025140028-3c0104f4b233 // indirect
 	github.com/crate-crypto/go-kzg-4844 v0.7.0 // indirect
 	github.com/danieljoos/wincred v1.1.2 // indirect
@@ -285,7 +285,7 @@ require (
 	github.com/shirou/gopsutil v3.21.11+incompatible // indirect
 	github.com/sirupsen/logrus v1.9.3 // indirect
 	github.com/spf13/afero v1.9.5 // indirect
-	github.com/spf13/cobra v1.8.0 // indirect
+	github.com/spf13/cobra v1.8.1 // indirect
 	github.com/spf13/jwalterweatherman v1.1.0 // indirect
 	github.com/spf13/pflag v1.0.5 // indirect
 	github.com/spf13/viper v1.15.0 // indirect
diff --git a/go.sum b/go.sum
index 636390f117b..091a218472b 100644
--- a/go.sum
+++ b/go.sum
@@ -298,6 +298,8 @@ github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsr
 github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
 github.com/cpuguy83/go-md2man/v2 v2.0.3 h1:qMCsGGgs+MAzDFyp9LpAe1Lqy/fY/qCovCm0qnXZOBM=
 github.com/cpuguy83/go-md2man/v2 v2.0.3/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
+github.com/cpuguy83/go-md2man/v2 v2.0.4 h1:wfIWP927BUkWJb2NmU/kNDYIBTh/ziUX91+lVfRxZq4=
+github.com/cpuguy83/go-md2man/v2 v2.0.4/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
 github.com/crate-crypto/go-ipa v0.0.0-20231025140028-3c0104f4b233 h1:d28BXYi+wUpz1KBmiF9bWrjEMacUEREV6MBi2ODnrfQ=
 github.com/crate-crypto/go-ipa v0.0.0-20231025140028-3c0104f4b233/go.mod h1:geZJZH3SzKCqnz5VT0q/DyIG/tvu/dZk+VIfXicupJs=
 github.com/crate-crypto/go-kzg-4844 v0.7.0 h1:C0vgZRk4q4EZ/JgPfzuSoxdCq3C3mOZMBShovmncxvA=
@@ -1214,6 +1216,8 @@ github.com/spf13/cobra v0.0.5/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tL
 github.com/spf13/cobra v1.1.1/go.mod h1:WnodtKOvamDL/PwE2M4iKs8aMDBZ5Q5klgD3qfVJQMI=
 github.com/spf13/cobra v1.8.0 h1:7aJaZx1B85qltLMc546zn58BxxfZdR/W22ej9CFoEf0=
 github.com/spf13/cobra v1.8.0/go.mod h1:WXLWApfZ71AjXPya3WOlMsY9yMs7YeiHhFVlvLyhcho=
+github.com/spf13/cobra v1.8.1 h1:e5/vxKd/rZsfSJMUX1agtjeTDf+qv1/JdBF8gg5k9ZM=
+github.com/spf13/cobra v1.8.1/go.mod h1:wHxEcudfqmLYa8iTfL+OuZPbBZkmvliBWKIezN3kD9Y=
 github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo=
 github.com/spf13/jwalterweatherman v1.1.0 h1:ue6voC5bR5F8YxI5S67j9i582FU4Qvo2bmqnqMYADFk=
 github.com/spf13/jwalterweatherman v1.1.0/go.mod h1:aNWZUN0dPAAO/Ljvb5BEdw96iTZ0EXowPYD95IqWIGo=

From a82ad0616ba0e60bcbdf231d1ce3ee2577eb3e76 Mon Sep 17 00:00:00 2001
From: lukaszcl <120112546+lukaszcl@users.noreply.github.com>
Date: Sat, 22 Jun 2024 20:02:00 +0200
Subject: [PATCH 027/137] Unify tools into single cmd

---
 .../cmd/check_tests_cmd.go}                   | 26 +-----
 .../cmd/check_tests_cmd_test.go}              |  0
 .../scripts/e2e_tests_tool/cmd/filter_cmd.go  | 88 ++++++++++++++++++
 .../{main_test.go => cmd/filter_cmd_test.go}  | 22 ++---
 .../cmd/root_cmd.go}                          |  8 +-
 .github/scripts/e2e_tests_tool/cmd/types.go   | 21 +++++
 .github/scripts/e2e_tests_tool/main.go        | 90 +------------------
 .../e2e_tests_tool_final/cmd/find_tests.go    | 26 ------
 .github/scripts/e2e_tests_tool_final/main.go  |  7 --
 .../run-e2e-tests-reusable-workflow.yml       |  4 +-
 10 files changed, 128 insertions(+), 164 deletions(-)
 rename .github/scripts/{e2e_tests_tool_final/cmd/check_tests.go => e2e_tests_tool/cmd/check_tests_cmd.go} (79%)
 rename .github/scripts/{e2e_tests_tool_final/cmd/check_tests_test.go => e2e_tests_tool/cmd/check_tests_cmd_test.go} (100%)
 create mode 100644 .github/scripts/e2e_tests_tool/cmd/filter_cmd.go
 rename .github/scripts/e2e_tests_tool/{main_test.go => cmd/filter_cmd_test.go} (62%)
 rename .github/scripts/{e2e_tests_tool_final/cmd/root.go => e2e_tests_tool/cmd/root_cmd.go} (69%)
 create mode 100644 .github/scripts/e2e_tests_tool/cmd/types.go
 delete mode 100644 .github/scripts/e2e_tests_tool_final/cmd/find_tests.go
 delete mode 100644 .github/scripts/e2e_tests_tool_final/main.go

diff --git a/.github/scripts/e2e_tests_tool_final/cmd/check_tests.go b/.github/scripts/e2e_tests_tool/cmd/check_tests_cmd.go
similarity index 79%
rename from .github/scripts/e2e_tests_tool_final/cmd/check_tests.go
rename to .github/scripts/e2e_tests_tool/cmd/check_tests_cmd.go
index 42079288d36..8c5bd9c85d0 100644
--- a/.github/scripts/e2e_tests_tool_final/cmd/check_tests.go
+++ b/.github/scripts/e2e_tests_tool/cmd/check_tests_cmd.go
@@ -27,7 +27,7 @@ type JobConfig struct {
 
 var checkTestsCmd = &cobra.Command{
 	Use:   "check-tests [directory] [yaml file]",
-	Short: "Check if all tests in a directory are included in the YAML file",
+	Short: "Check if all tests in a directory are included in the test configurations YAML file",
 	Args:  cobra.ExactArgs(2),
 	Run: func(cmd *cobra.Command, args []string) {
 		directory := args[0]
@@ -42,26 +42,6 @@ var checkTestsCmd = &cobra.Command{
 	},
 }
 
-type Test struct {
-	Name string
-	Path string
-}
-
-type TestRun struct {
-	ID                    string   `yaml:"id" json:"id"`
-	Path                  string   `yaml:"path" json:"path"`
-	TestType              string   `yaml:"test-type" json:"testType"`
-	RunsOn                string   `yaml:"runs-on" json:"runsOn"`
-	Cmd                   string   `yaml:"cmd" json:"cmd"`
-	RemoteRunnerTestSuite string   `yaml:"remote-runner-test-suite" json:"remoteRunnerTestSuite"`
-	PyroscopeEnv          string   `yaml:"pyroscope-env" json:"pyroscopeEnv"`
-	Trigger               []string `yaml:"trigger" json:"trigger"`
-}
-
-type Config struct {
-	Tests []TestRun `yaml:"test-runner-matrix"`
-}
-
 func extractTests(dir string) ([]Test, error) {
 	var tests []Test
 	err := filepath.Walk(dir, func(path string, info os.FileInfo, err error) error {
@@ -148,7 +128,3 @@ func matchTestNameInCmd(cmd string, testName string) bool {
 	}
 	return false
 }
-
-func init() {
-	rootCmd.AddCommand(checkTestsCmd)
-}
diff --git a/.github/scripts/e2e_tests_tool_final/cmd/check_tests_test.go b/.github/scripts/e2e_tests_tool/cmd/check_tests_cmd_test.go
similarity index 100%
rename from .github/scripts/e2e_tests_tool_final/cmd/check_tests_test.go
rename to .github/scripts/e2e_tests_tool/cmd/check_tests_cmd_test.go
diff --git a/.github/scripts/e2e_tests_tool/cmd/filter_cmd.go b/.github/scripts/e2e_tests_tool/cmd/filter_cmd.go
new file mode 100644
index 00000000000..15ee6c2634f
--- /dev/null
+++ b/.github/scripts/e2e_tests_tool/cmd/filter_cmd.go
@@ -0,0 +1,88 @@
+package cmd
+
+import (
+	"encoding/json"
+	"fmt"
+	"io/ioutil"
+	"log"
+	"strings"
+
+	"github.com/spf13/cobra"
+	"gopkg.in/yaml.v2"
+)
+
+// Filter tests based on trigger, test type, and test IDs.
+func filterTests(tests []TestConf, names, trigger, testType, ids string) []TestConf {
+	triggerFilter := trigger
+	typeFilter := testType
+	idFilter := strings.Split(ids, ",")
+
+	var filteredTests []TestConf
+
+	for _, test := range tests {
+		triggerMatch := trigger == "" || contains(test.Trigger, triggerFilter)
+		typeMatch := testType == "" || test.TestType == typeFilter
+		idMatch := ids == "*" || ids == "" || contains(idFilter, test.ID)
+
+		if triggerMatch && typeMatch && idMatch {
+			filteredTests = append(filteredTests, test)
+		}
+	}
+
+	return filteredTests
+}
+
+// Utility function to check if a slice contains a string.
+func contains(slice []string, element string) bool {
+	for _, s := range slice {
+		if s == element {
+			return true
+		}
+	}
+	return false
+}
+
+// filterCmd represents the filter command
+var filterCmd = &cobra.Command{
+	Use:   "filter",
+	Short: "Filter test configurations based on specified criteria",
+	Long: `Filters tests from a YAML configuration based on name, trigger, test type, and test IDs.
+Example usage:
+./e2e_tests_tool filter --file .github/e2e-tests.yml --trigger "push" --test-type "docker" --test-ids "test1,test2"`,
+	Run: func(cmd *cobra.Command, args []string) {
+		yamlFile, _ := cmd.Flags().GetString("file")
+		names, _ := cmd.Flags().GetString("name")
+		trigger, _ := cmd.Flags().GetString("trigger")
+		testType, _ := cmd.Flags().GetString("test-type")
+		testIDs, _ := cmd.Flags().GetString("test-ids")
+
+		data, err := ioutil.ReadFile(yamlFile)
+		if err != nil {
+			log.Fatalf("Error reading YAML file: %v", err)
+		}
+
+		var config Config
+		err = yaml.Unmarshal(data, &config)
+		if err != nil {
+			log.Fatalf("Error parsing YAML data: %v", err)
+		}
+
+		filteredTests := filterTests(config.Tests, names, trigger, testType, testIDs)
+		matrix := map[string][]TestConf{"tests": filteredTests}
+		matrixJSON, err := json.Marshal(matrix)
+		if err != nil {
+			log.Fatalf("Error marshaling matrix to JSON: %v", err)
+		}
+
+		fmt.Printf("%s", matrixJSON)
+	},
+}
+
+func init() {
+	filterCmd.Flags().StringP("file", "f", "", "Path to the YAML file")
+	filterCmd.Flags().StringP("name", "n", "", "Comma-separated list of test names to filter by")
+	filterCmd.Flags().StringP("trigger", "t", "", "Trigger filter")
+	filterCmd.Flags().StringP("test-type", "y", "", "Type of test to filter by")
+	filterCmd.Flags().StringP("test-ids", "i", "*", "Comma-separated list of test IDs to filter by")
+	filterCmd.MarkFlagRequired("file")
+}
diff --git a/.github/scripts/e2e_tests_tool/main_test.go b/.github/scripts/e2e_tests_tool/cmd/filter_cmd_test.go
similarity index 62%
rename from .github/scripts/e2e_tests_tool/main_test.go
rename to .github/scripts/e2e_tests_tool/cmd/filter_cmd_test.go
index 61d2afa37de..27b8d09dedf 100644
--- a/.github/scripts/e2e_tests_tool/main_test.go
+++ b/.github/scripts/e2e_tests_tool/cmd/filter_cmd_test.go
@@ -1,14 +1,14 @@
-package main
+package cmd
 
 import (
 	"testing"
 )
 
 func TestFilterTestsByID(t *testing.T) {
-	tests := []Test{
-		{ID: "run_all_in_ocr_tests_go", Name: "Run all ocr_tests.go", TestType: "docker"},
-		{ID: "run_all_in_ocr2_tests_go", Name: "Run TestOCRv2Request in ocr2_test.go", TestType: "docker"},
-		{ID: "run_all_in_ocr3_tests_go", Name: "Run TestOCRv2Basic in ocr2_test.go", TestType: "k8s_remote_runner"},
+	tests := []TestConf{
+		{ID: "run_all_in_ocr_tests_go", TestType: "docker"},
+		{ID: "run_all_in_ocr2_tests_go", TestType: "docker"},
+		{ID: "run_all_in_ocr3_tests_go", TestType: "k8s_remote_runner"},
 	}
 
 	cases := []struct {
@@ -24,7 +24,7 @@ func TestFilterTestsByID(t *testing.T) {
 
 	for _, c := range cases {
 		t.Run(c.description, func(t *testing.T) {
-			filtered := FilterTests(tests, "", "", "", c.inputIDs)
+			filtered := filterTests(tests, "", "", "", c.inputIDs)
 			if len(filtered) != c.expectedLen {
 				t.Errorf("FilterTests(%s) returned %d tests, expected %d", c.description, len(filtered), c.expectedLen)
 			}
@@ -33,10 +33,10 @@ func TestFilterTestsByID(t *testing.T) {
 }
 
 func TestFilterTestsIntegration(t *testing.T) {
-	tests := []Test{
-		{ID: "run_all_in_ocr_tests_go", Name: "Run all ocr_tests.go", TestType: "docker", Trigger: []string{"nightly"}},
-		{ID: "run_all_in_ocr2_tests_go", Name: "Run TestOCRv2Request in ocr2_test.go", TestType: "docker", Trigger: []string{"push"}},
-		{ID: "run_all_in_ocr3_tests_go", Name: "Run TestOCRv2Basic in ocr2_test.go", TestType: "k8s_remote_runner", Trigger: []string{"push"}},
+	tests := []TestConf{
+		{ID: "run_all_in_ocr_tests_go", TestType: "docker", Trigger: []string{"nightly"}},
+		{ID: "run_all_in_ocr2_tests_go", TestType: "docker", Trigger: []string{"push"}},
+		{ID: "run_all_in_ocr3_tests_go", TestType: "k8s_remote_runner", Trigger: []string{"push"}},
 	}
 
 	cases := []struct {
@@ -55,7 +55,7 @@ func TestFilterTestsIntegration(t *testing.T) {
 
 	for _, c := range cases {
 		t.Run(c.description, func(t *testing.T) {
-			filtered := FilterTests(tests, c.inputNames, c.inputTrigger, c.inputTestType, c.inputIDs)
+			filtered := filterTests(tests, c.inputNames, c.inputTrigger, c.inputTestType, c.inputIDs)
 			if len(filtered) != c.expectedLen {
 				t.Errorf("FilterTests(%s) returned %d tests, expected %d", c.description, len(filtered), c.expectedLen)
 			}
diff --git a/.github/scripts/e2e_tests_tool_final/cmd/root.go b/.github/scripts/e2e_tests_tool/cmd/root_cmd.go
similarity index 69%
rename from .github/scripts/e2e_tests_tool_final/cmd/root.go
rename to .github/scripts/e2e_tests_tool/cmd/root_cmd.go
index 0a04d04f3df..88310117536 100644
--- a/.github/scripts/e2e_tests_tool_final/cmd/root.go
+++ b/.github/scripts/e2e_tests_tool/cmd/root_cmd.go
@@ -8,10 +8,8 @@ import (
 
 // rootCmd represents the base command when called without any subcommands
 var rootCmd = &cobra.Command{
-	Use:   "e2e_tests_ci_tool",
-	Short: "A brief description of your application",
-	Long: `A longer description that spans multiple lines and likely contains
-examples and usage of using your application`,
+	Use:   "e2e_tests_tool",
+	Short: "A tool to manage E2E tests on Github CI",
 }
 
 // Execute adds all child commands to the root command and sets flags appropriately.
@@ -26,6 +24,6 @@ func Execute() {
 func init() {
 	rootCmd.Flags().BoolP("toggle", "t", false, "Help message for toggle")
 
-	rootCmd.AddCommand(findTestsCmd)
 	rootCmd.AddCommand(checkTestsCmd)
+	rootCmd.AddCommand(filterCmd)
 }
diff --git a/.github/scripts/e2e_tests_tool/cmd/types.go b/.github/scripts/e2e_tests_tool/cmd/types.go
new file mode 100644
index 00000000000..24908ea2880
--- /dev/null
+++ b/.github/scripts/e2e_tests_tool/cmd/types.go
@@ -0,0 +1,21 @@
+package cmd
+
+type Test struct {
+	Name string
+	Path string
+}
+
+type TestConf struct {
+	ID                    string   `yaml:"id" json:"id"`
+	Path                  string   `yaml:"path" json:"path"`
+	TestType              string   `yaml:"test-type" json:"testType"`
+	RunsOn                string   `yaml:"runs-on" json:"runsOn"`
+	Cmd                   string   `yaml:"cmd" json:"cmd"`
+	RemoteRunnerTestSuite string   `yaml:"remote-runner-test-suite" json:"remoteRunnerTestSuite"`
+	PyroscopeEnv          string   `yaml:"pyroscope-env" json:"pyroscopeEnv"`
+	Trigger               []string `yaml:"trigger" json:"trigger"`
+}
+
+type Config struct {
+	Tests []TestConf `yaml:"test-runner-matrix"`
+}
diff --git a/.github/scripts/e2e_tests_tool/main.go b/.github/scripts/e2e_tests_tool/main.go
index 045c14f796c..c3c185a4fbd 100644
--- a/.github/scripts/e2e_tests_tool/main.go
+++ b/.github/scripts/e2e_tests_tool/main.go
@@ -1,93 +1,7 @@
 package main
 
-import (
-	"encoding/json"
-	"flag"
-	"fmt"
-	"io/ioutil"
-	"log"
-	"strings"
+import "github.com/smartcontractkit/chainlink/v2/.github/scripts/e2e_tests_tool/cmd"
 
-	"gopkg.in/yaml.v2"
-)
-
-// Test defines the structure of a test entry in the YAML file.
-type Test struct {
-	ID                    string   `yaml:"id" json:"id"`
-	Path                  string   `yaml:"path" json:"path"`
-	TestType              string   `yaml:"test-type" json:"testType"`
-	RunsOn                string   `yaml:"runs-on" json:"runsOn"`
-	Cmd                   string   `yaml:"cmd" json:"cmd"`
-	RemoteRunnerTestSuite string   `yaml:"remote-runner-test-suite" json:"remoteRunnerTestSuite"`
-	PyroscopeEnv          string   `yaml:"pyroscope-env" json:"pyroscopeEnv"`
-	Trigger               []string `yaml:"trigger" json:"trigger"`
-}
-
-// Config represents the tests configuration.
-type Config struct {
-	Tests []Test `yaml:"test-runner-matrix"`
-}
-
-// Filter tests based on name, trigger, test type, and test IDs.
-func FilterTests(tests []Test, names, trigger, testType, ids string) []Test {
-	nameFilter := strings.Split(names, ",")
-	triggerFilter := trigger
-	typeFilter := testType
-	idFilter := strings.Split(ids, ",")
-
-	var filteredTests []Test
-
-	for _, test := range tests {
-		nameMatch := names == "" || contains(nameFilter, test.Name)
-		triggerMatch := trigger == "" || contains(test.Trigger, triggerFilter)
-		typeMatch := testType == "" || test.TestType == typeFilter
-		idMatch := ids == "*" || ids == "" || contains(idFilter, test.ID)
-
-		if nameMatch && triggerMatch && typeMatch && idMatch {
-			filteredTests = append(filteredTests, test)
-		}
-	}
-
-	return filteredTests
-}
-
-// Main function including the new test-ids flag.
 func main() {
-	yamlFile := flag.String("file", ".github/e2e-tests.yml", "Path to the YAML file")
-	names := flag.String("name", "", "Comma-separated list of test names to filter by")
-	trigger := flag.String("trigger", "", "Trigger filter")
-	testType := flag.String("test-type", "", "Type of test to filter by")
-	testIDs := flag.String("test-ids", "*", "Comma-separated list of test IDs to filter by")
-
-	flag.Parse()
-
-	data, err := ioutil.ReadFile(*yamlFile)
-	if err != nil {
-		log.Fatalf("Error reading YAML file: %v", err)
-	}
-
-	var config Config
-	err = yaml.Unmarshal(data, &config)
-	if err != nil {
-		log.Fatalf("Error parsing YAML data: %v", err)
-	}
-
-	filteredTests := FilterTests(config.Tests, *names, *trigger, *testType, *testIDs)
-	matrix := map[string][]Test{"tests": filteredTests}
-	matrixJSON, err := json.Marshal(matrix)
-	if err != nil {
-		log.Fatalf("Error marshaling matrix to JSON: %v", err)
-	}
-
-	fmt.Printf("%s", matrixJSON)
-}
-
-// Utility function to check if a slice contains a string.
-func contains(slice []string, element string) bool {
-	for _, s := range slice {
-		if s == element {
-			return true
-		}
-	}
-	return false
+	cmd.Execute()
 }
diff --git a/.github/scripts/e2e_tests_tool_final/cmd/find_tests.go b/.github/scripts/e2e_tests_tool_final/cmd/find_tests.go
deleted file mode 100644
index e105f90adf1..00000000000
--- a/.github/scripts/e2e_tests_tool_final/cmd/find_tests.go
+++ /dev/null
@@ -1,26 +0,0 @@
-package cmd
-
-import (
-	"fmt"
-	"os"
-
-	"github.com/spf13/cobra"
-)
-
-// findTestsCmd represents the find-tests command
-var findTestsCmd = &cobra.Command{
-	Use:   "find-tests [path]",
-	Short: "Find all Go test functions in a directory",
-	Args:  cobra.ExactArgs(1),
-	Run: func(cmd *cobra.Command, args []string) {
-		path := args[0]
-		tests, err := extractTests(path)
-		if err != nil {
-			fmt.Println("Error extracting tests:", err)
-			os.Exit(1)
-		}
-		for _, t := range tests {
-			fmt.Printf("%+v", t)
-		}
-	},
-}
diff --git a/.github/scripts/e2e_tests_tool_final/main.go b/.github/scripts/e2e_tests_tool_final/main.go
deleted file mode 100644
index 2cd999ee492..00000000000
--- a/.github/scripts/e2e_tests_tool_final/main.go
+++ /dev/null
@@ -1,7 +0,0 @@
-package main
-
-import "github.com/smartcontractkit/chainlink/v2/.github/scripts/e2e_tests_tool_final/cmd"
-
-func main() {
-	cmd.Execute()
-}
diff --git a/.github/workflows/run-e2e-tests-reusable-workflow.yml b/.github/workflows/run-e2e-tests-reusable-workflow.yml
index 33bdc9eedd2..3f829cdf9af 100644
--- a/.github/workflows/run-e2e-tests-reusable-workflow.yml
+++ b/.github/workflows/run-e2e-tests-reusable-workflow.yml
@@ -67,14 +67,14 @@ jobs:
       - name: Generate Docker Tests Matrix
         id: set-docker-matrix
         run: |
-          MATRIX_JSON=$(go run .github/scripts/e2e_tests_tool/main.go -file .github/e2e-tests.yml -test-type 'docker' -test-ids '${{ github.event.inputs.test-ids }}')
+          MATRIX_JSON=$(go run .github/scripts/e2e_tests_tool/main.go filter --file .github/e2e-tests.yml --test-type 'docker' --test-ids '${{ github.event.inputs.test-ids }}')
           echo "Docker tests:"
           echo "$MATRIX_JSON" | jq
           echo "::set-output name=matrix::$MATRIX_JSON"
       - name: Generate K8s Tests Matrix
         id: set-k8s-runner-matrix
         run: |
-          MATRIX_JSON=$(go run .github/scripts/e2e_tests_tool/main.go -file .github/e2e-tests.yml -test-type 'k8s-remote-runner' -test-ids '${{ github.event.inputs.test-ids }}')
+          MATRIX_JSON=$(go run .github/scripts/e2e_tests_tool/main.go filter --file .github/e2e-tests.yml --test-type 'k8s-remote-runner' --test-ids '${{ github.event.inputs.test-ids }}')
           echo "K8s tests:"
           echo "$MATRIX_JSON" | jq
           echo "::set-output name=matrix::$MATRIX_JSON"

From 79d31467b9f8eb923e7425c9d60d1130db266816 Mon Sep 17 00:00:00 2001
From: lukaszcl <120112546+lukaszcl@users.noreply.github.com>
Date: Sat, 22 Jun 2024 20:07:29 +0200
Subject: [PATCH 028/137] Add check tests step to workflow

---
 .../workflows/run-e2e-tests-reusable-workflow.yml  | 14 ++++++++++++++
 1 file changed, 14 insertions(+)

diff --git a/.github/workflows/run-e2e-tests-reusable-workflow.yml b/.github/workflows/run-e2e-tests-reusable-workflow.yml
index 3f829cdf9af..066b8f90aac 100644
--- a/.github/workflows/run-e2e-tests-reusable-workflow.yml
+++ b/.github/workflows/run-e2e-tests-reusable-workflow.yml
@@ -47,7 +47,21 @@ env:
   CHAINLINK_IMAGE: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com/chainlink
 
 jobs:
+  run-check-tests:
+    name: Check All E2E Tests Defined in CI Configuration
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2
+      - name: Set up Go
+        uses: actions/setup-go@v2
+        with:
+          go-version: '1.21.7'
+      - name: Run Check Tests Command
+        run: go run .github/scripts/e2e_tests_tool/main.go check-tests ./integration-tests .github/e2e-tests.yml
+
   load-test-configurations:
+    needs: run-check-tests
     name: Load Test Configurations
     runs-on: ubuntu-latest
     outputs:

From 0e6d314567ceb61f782208f04b2b46b961d1a0cf Mon Sep 17 00:00:00 2001
From: lukaszcl <120112546+lukaszcl@users.noreply.github.com>
Date: Sat, 22 Jun 2024 20:21:29 +0200
Subject: [PATCH 029/137] show more info in annotation

---
 .github/e2e-tests.yml                                 | 4 ++--
 .github/scripts/e2e_tests_tool/cmd/check_tests_cmd.go | 3 +--
 .github/workflows/run-e2e-tests-reusable-workflow.yml | 7 +++++--
 3 files changed, 8 insertions(+), 6 deletions(-)

diff --git a/.github/e2e-tests.yml b/.github/e2e-tests.yml
index ea3bad6364d..e9f9ead639d 100644
--- a/.github/e2e-tests.yml
+++ b/.github/e2e-tests.yml
@@ -1,6 +1,6 @@
-# This configuration file defines the GitHub runner requirements for each E2E test or set of E2E tests.
+# This configuration file defines the Github CI configuration for each E2E test or set of E2E tests.
 #
-# Each test entry details the runner environment (Docker or Kubernetes), Go test command, and triggers
+# Each test entry details the Github runner configuration, test environment (Docker or Kubernetes), Go test command, and triggers
 # that determine under what conditions the test should be executed (e.g., on push or nightly builds).
 #
 # This file is used by CI workflows to run E2E tests depending on the trigger and the test type. 
diff --git a/.github/scripts/e2e_tests_tool/cmd/check_tests_cmd.go b/.github/scripts/e2e_tests_tool/cmd/check_tests_cmd.go
index 8c5bd9c85d0..037d1b50099 100644
--- a/.github/scripts/e2e_tests_tool/cmd/check_tests_cmd.go
+++ b/.github/scripts/e2e_tests_tool/cmd/check_tests_cmd.go
@@ -99,7 +99,7 @@ func checkTestsInPipeline(yamlFile string, tests []Test) {
 			}
 		}
 		if !found {
-			missingTests = append(missingTests, fmt.Sprintf("Test '%s' in file '%s' does not have CI configuration", test.Name, test.Path))
+			missingTests = append(missingTests, fmt.Sprintf("ERROR: Test '%s' in file '%s' does not have CI configuration in '%s'", test.Name, test.Path, yamlFile))
 		}
 	}
 
@@ -107,7 +107,6 @@ func checkTestsInPipeline(yamlFile string, tests []Test) {
 		for _, missing := range missingTests {
 			fmt.Println(missing)
 		}
-		fmt.Printf("\nERROR: These tests must be added to the E2E CI conf in '%s' file.\n", yamlFile)
 		os.Exit(1) // Exit with a failure status
 	}
 }
diff --git a/.github/workflows/run-e2e-tests-reusable-workflow.yml b/.github/workflows/run-e2e-tests-reusable-workflow.yml
index 066b8f90aac..a503e24fb55 100644
--- a/.github/workflows/run-e2e-tests-reusable-workflow.yml
+++ b/.github/workflows/run-e2e-tests-reusable-workflow.yml
@@ -48,7 +48,7 @@ env:
 
 jobs:
   run-check-tests:
-    name: Check All E2E Tests Defined in CI Configuration
+    name: Check Test Configurations
     runs-on: ubuntu-latest
     steps:
       - name: Checkout code
@@ -58,7 +58,10 @@ jobs:
         with:
           go-version: '1.21.7'
       - name: Run Check Tests Command
-        run: go run .github/scripts/e2e_tests_tool/main.go check-tests ./integration-tests .github/e2e-tests.yml
+        run: |
+          if ! go run .github/scripts/e2e_tests_tool/main.go check-tests ./integration-tests .github/e2e-tests.yml; then
+            echo "::error::Some E2E test configurations have to be added in .github/e2e-tests.yml. This file defines Github CI configuration for each E2E test or set of E2E tests." && exit 1
+          fi
 
   load-test-configurations:
     needs: run-check-tests

From cad8ebda0885f847235093ecb2cb6d3bd316ff56 Mon Sep 17 00:00:00 2001
From: lukaszcl <120112546+lukaszcl@users.noreply.github.com>
Date: Sat, 22 Jun 2024 20:36:26 +0200
Subject: [PATCH 030/137] fix

---
 .github/e2e-tests.yml                         | 12 +++++------
 .../run-e2e-tests-reusable-workflow.yml       |  2 +-
 .github/workflows/run-selected-e2e-tests.yml  | 20 -------------------
 3 files changed, 7 insertions(+), 27 deletions(-)

diff --git a/.github/e2e-tests.yml b/.github/e2e-tests.yml
index e9f9ead639d..b894d9ae27d 100644
--- a/.github/e2e-tests.yml
+++ b/.github/e2e-tests.yml
@@ -1,10 +1,8 @@
-# This configuration file defines the Github CI configuration for each E2E test or set of E2E tests.
+# This file defines Github CI configuration for each E2E test and is used by CI workflows to run E2E tests.
 #
 # Each test entry details the Github runner configuration, test environment (Docker or Kubernetes), Go test command, and triggers
 # that determine under what conditions the test should be executed (e.g., on push or nightly builds).
 #
-# This file is used by CI workflows to run E2E tests depending on the trigger and the test type. 
-#
 test-runner-matrix:
 
   # Example of 1 runner for all tests in integration-tests/smoke/ocr_test.go
@@ -12,9 +10,8 @@ test-runner-matrix:
     path: integration-tests/smoke/ocr_test.go
     test-type: docker
     runs-on: ubuntu-latest
+    trigger: ["push", "nightly", "pull-request", "release-tag"]
     cmd: cd integration-tests/ && go test smoke/ocr_test.go -timeout 30m -count=1 -json
-    # workflow: [nightly-e2e-tests.yml]
-    trigger: ["push", "nightly"]
     pyroscope-env: ci-smoke-ocr-evm-simulated-nightly
 
   # Example of 2 separate runners for the same test file but different tests. Can be used if tests if are too heavy to run on the same runner
@@ -22,6 +19,7 @@ test-runner-matrix:
     path: integration-tests/smoke/ocr2_test.go
     test-type: docker
     runs-on: ubuntu-latest
+    trigger: ["push", "nightly", "pull-request", "release-tag"]
     cmd: cd integration-tests/ && go test smoke/ocr2_test.go -test.run ^TestOCRv2Request$ -test.parallel=1 -timeout 30m -count=1 -json
     pyroscope-env: ci-smoke-ocr2-evm-simulated-nightly
 
@@ -29,14 +27,16 @@ test-runner-matrix:
     path: integration-tests/smoke/ocr2_test.go
     test-type: docker
     runs-on: ubuntu-latest
+    trigger: ["push", "nightly", "pull-request", "release-tag"]
     cmd: cd integration-tests/ && go test smoke/ocr2_test.go -test.run ^TestOCRv2Basic$ -test.parallel=1 -timeout 30m -count=1 -json
     pyroscope-env: ci-smoke-ocr2-evm-simulated-nightly
 
-  # Example of configuration for running a single soak test in Kubernetes Remote Runner
+  # Example of a configuration for running a single soak test in Kubernetes Remote Runner
   - id: soak/ocr_test.go:^TestOCRSoak$
     path: integration-tests/soak/ocr_test.go
     test-type: k8s-remote-runner
     remote-runner-test-suite: soak
     runs-on: ubuntu-latest
+    trigger: ["nightly", "release-tag"] # Run nightly and on release tags
     cmd: cd integration-tests/ && go test soak/ocr_test.go -test.run ^TestOCRSoak$ -test.parallel=1 -timeout 30m -count=1 -json
     pyroscope-env: ci-smoke-ocr2-evm-simulated-nightly
diff --git a/.github/workflows/run-e2e-tests-reusable-workflow.yml b/.github/workflows/run-e2e-tests-reusable-workflow.yml
index a503e24fb55..80da0e74ab0 100644
--- a/.github/workflows/run-e2e-tests-reusable-workflow.yml
+++ b/.github/workflows/run-e2e-tests-reusable-workflow.yml
@@ -60,7 +60,7 @@ jobs:
       - name: Run Check Tests Command
         run: |
           if ! go run .github/scripts/e2e_tests_tool/main.go check-tests ./integration-tests .github/e2e-tests.yml; then
-            echo "::error::Some E2E test configurations have to be added in .github/e2e-tests.yml. This file defines Github CI configuration for each E2E test or set of E2E tests." && exit 1
+            echo "::error::Some E2E test configurations have to be added to .github/e2e-tests.yml. This file defines Github CI configuration for each E2E test or set of E2E tests." && exit 1
           fi
 
   load-test-configurations:
diff --git a/.github/workflows/run-selected-e2e-tests.yml b/.github/workflows/run-selected-e2e-tests.yml
index 44e038abba4..038e1a0047d 100644
--- a/.github/workflows/run-selected-e2e-tests.yml
+++ b/.github/workflows/run-selected-e2e-tests.yml
@@ -15,26 +15,6 @@ on:
         type: string
 
 jobs:
-  test:
-    runs-on: ubuntu-latest
-    environment: integration
-    permissions:
-      actions: read
-      checks: write
-      pull-requests: write
-      id-token: write
-      contents: read
-    env:
-      ENV_JOB_IMAGE_BASE: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com/chainlink-tests
-    steps:
-      - name: Check secret presence
-        run: |
-          if [ -z "${{ secrets.QA_AWS_REGION }}" ]; then
-            echo "Secret is empty"
-          else
-            echo "Secret is present"
-          fi
-
   call-run-e2e-tests-workflow:
     name: Run E2E Tests
     uses: ./.github/workflows/run-e2e-tests-reusable-workflow.yml

From 880ae3b61eaf2776e0d58bdc73cc6763d4fd9975 Mon Sep 17 00:00:00 2001
From: lukaszcl <120112546+lukaszcl@users.noreply.github.com>
Date: Mon, 24 Jun 2024 12:04:23 +0200
Subject: [PATCH 031/137] Do not require check-tests

---
 .github/workflows/run-e2e-tests-reusable-workflow.yml | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/run-e2e-tests-reusable-workflow.yml b/.github/workflows/run-e2e-tests-reusable-workflow.yml
index 80da0e74ab0..be45c4c8a93 100644
--- a/.github/workflows/run-e2e-tests-reusable-workflow.yml
+++ b/.github/workflows/run-e2e-tests-reusable-workflow.yml
@@ -64,7 +64,8 @@ jobs:
           fi
 
   load-test-configurations:
-    needs: run-check-tests
+    # Uncomment if we want to run check-tests before loading test configurations
+    # needs: run-check-tests 
     name: Load Test Configurations
     runs-on: ubuntu-latest
     outputs:

From 8079dab767b9a200779352eded98f3aebb408f49 Mon Sep 17 00:00:00 2001
From: lukaszcl <120112546+lukaszcl@users.noreply.github.com>
Date: Mon, 24 Jun 2024 12:32:53 +0200
Subject: [PATCH 032/137] fix go mod

---
 integration-tests/go.mod      | 2 +-
 integration-tests/go.sum      | 8 ++++----
 integration-tests/load/go.mod | 2 +-
 integration-tests/load/go.sum | 8 ++++----
 4 files changed, 10 insertions(+), 10 deletions(-)

diff --git a/integration-tests/go.mod b/integration-tests/go.mod
index a700e20d88c..62b7c07ba63 100644
--- a/integration-tests/go.mod
+++ b/integration-tests/go.mod
@@ -37,7 +37,7 @@ require (
 	github.com/smartcontractkit/libocr v0.0.0-20240419185742-fd3cab206b2c
 	github.com/smartcontractkit/seth v1.0.12
 	github.com/smartcontractkit/wasp v0.4.7
-	github.com/spf13/cobra v1.8.0
+	github.com/spf13/cobra v1.8.1
 	github.com/stretchr/testify v1.9.0
 	github.com/test-go/testify v1.1.4
 	github.com/testcontainers/testcontainers-go v0.28.0
diff --git a/integration-tests/go.sum b/integration-tests/go.sum
index 97813f85ea1..c4b175e85fb 100644
--- a/integration-tests/go.sum
+++ b/integration-tests/go.sum
@@ -382,8 +382,8 @@ github.com/cpuguy83/dockercfg v0.3.1/go.mod h1:sugsbF4//dDlL/i+S+rtpIWp+5h0BHJHf
 github.com/cpuguy83/go-md2man v1.0.10 h1:BSKMNlYxDvnunlTymqtgONjNnaRV1sTpcovwwjF22jk=
 github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE=
 github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU=
-github.com/cpuguy83/go-md2man/v2 v2.0.3 h1:qMCsGGgs+MAzDFyp9LpAe1Lqy/fY/qCovCm0qnXZOBM=
-github.com/cpuguy83/go-md2man/v2 v2.0.3/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
+github.com/cpuguy83/go-md2man/v2 v2.0.4 h1:wfIWP927BUkWJb2NmU/kNDYIBTh/ziUX91+lVfRxZq4=
+github.com/cpuguy83/go-md2man/v2 v2.0.4/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
 github.com/crate-crypto/go-ipa v0.0.0-20231025140028-3c0104f4b233 h1:d28BXYi+wUpz1KBmiF9bWrjEMacUEREV6MBi2ODnrfQ=
 github.com/crate-crypto/go-ipa v0.0.0-20231025140028-3c0104f4b233/go.mod h1:geZJZH3SzKCqnz5VT0q/DyIG/tvu/dZk+VIfXicupJs=
 github.com/crate-crypto/go-kzg-4844 v0.7.0 h1:C0vgZRk4q4EZ/JgPfzuSoxdCq3C3mOZMBShovmncxvA=
@@ -1570,8 +1570,8 @@ github.com/spf13/cast v1.6.0/go.mod h1:ancEpBxwJDODSW/UG4rDrAqiKolqNNh2DX3mk86cA
 github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ=
 github.com/spf13/cobra v0.0.5/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tLCHU=
 github.com/spf13/cobra v1.1.1/go.mod h1:WnodtKOvamDL/PwE2M4iKs8aMDBZ5Q5klgD3qfVJQMI=
-github.com/spf13/cobra v1.8.0 h1:7aJaZx1B85qltLMc546zn58BxxfZdR/W22ej9CFoEf0=
-github.com/spf13/cobra v1.8.0/go.mod h1:WXLWApfZ71AjXPya3WOlMsY9yMs7YeiHhFVlvLyhcho=
+github.com/spf13/cobra v1.8.1 h1:e5/vxKd/rZsfSJMUX1agtjeTDf+qv1/JdBF8gg5k9ZM=
+github.com/spf13/cobra v1.8.1/go.mod h1:wHxEcudfqmLYa8iTfL+OuZPbBZkmvliBWKIezN3kD9Y=
 github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo=
 github.com/spf13/jwalterweatherman v1.1.0 h1:ue6voC5bR5F8YxI5S67j9i582FU4Qvo2bmqnqMYADFk=
 github.com/spf13/jwalterweatherman v1.1.0/go.mod h1:aNWZUN0dPAAO/Ljvb5BEdw96iTZ0EXowPYD95IqWIGo=
diff --git a/integration-tests/load/go.mod b/integration-tests/load/go.mod
index f00ba4a35d7..979a4cff1df 100644
--- a/integration-tests/load/go.mod
+++ b/integration-tests/load/go.mod
@@ -377,7 +377,7 @@ require (
 	github.com/sony/gobreaker v0.5.0 // indirect
 	github.com/spf13/afero v1.9.5 // indirect
 	github.com/spf13/cast v1.6.0 // indirect
-	github.com/spf13/cobra v1.8.0 // indirect
+	github.com/spf13/cobra v1.8.1 // indirect
 	github.com/spf13/jwalterweatherman v1.1.0 // indirect
 	github.com/spf13/pflag v1.0.5 // indirect
 	github.com/spf13/viper v1.15.0 // indirect
diff --git a/integration-tests/load/go.sum b/integration-tests/load/go.sum
index e3e9254d67c..708bc075206 100644
--- a/integration-tests/load/go.sum
+++ b/integration-tests/load/go.sum
@@ -376,8 +376,8 @@ github.com/cpuguy83/dockercfg v0.3.1/go.mod h1:sugsbF4//dDlL/i+S+rtpIWp+5h0BHJHf
 github.com/cpuguy83/go-md2man v1.0.10 h1:BSKMNlYxDvnunlTymqtgONjNnaRV1sTpcovwwjF22jk=
 github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE=
 github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU=
-github.com/cpuguy83/go-md2man/v2 v2.0.3 h1:qMCsGGgs+MAzDFyp9LpAe1Lqy/fY/qCovCm0qnXZOBM=
-github.com/cpuguy83/go-md2man/v2 v2.0.3/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
+github.com/cpuguy83/go-md2man/v2 v2.0.4 h1:wfIWP927BUkWJb2NmU/kNDYIBTh/ziUX91+lVfRxZq4=
+github.com/cpuguy83/go-md2man/v2 v2.0.4/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
 github.com/crate-crypto/go-ipa v0.0.0-20231025140028-3c0104f4b233 h1:d28BXYi+wUpz1KBmiF9bWrjEMacUEREV6MBi2ODnrfQ=
 github.com/crate-crypto/go-ipa v0.0.0-20231025140028-3c0104f4b233/go.mod h1:geZJZH3SzKCqnz5VT0q/DyIG/tvu/dZk+VIfXicupJs=
 github.com/crate-crypto/go-kzg-4844 v0.7.0 h1:C0vgZRk4q4EZ/JgPfzuSoxdCq3C3mOZMBShovmncxvA=
@@ -1558,8 +1558,8 @@ github.com/spf13/cast v1.6.0/go.mod h1:ancEpBxwJDODSW/UG4rDrAqiKolqNNh2DX3mk86cA
 github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ=
 github.com/spf13/cobra v0.0.5/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tLCHU=
 github.com/spf13/cobra v1.1.1/go.mod h1:WnodtKOvamDL/PwE2M4iKs8aMDBZ5Q5klgD3qfVJQMI=
-github.com/spf13/cobra v1.8.0 h1:7aJaZx1B85qltLMc546zn58BxxfZdR/W22ej9CFoEf0=
-github.com/spf13/cobra v1.8.0/go.mod h1:WXLWApfZ71AjXPya3WOlMsY9yMs7YeiHhFVlvLyhcho=
+github.com/spf13/cobra v1.8.1 h1:e5/vxKd/rZsfSJMUX1agtjeTDf+qv1/JdBF8gg5k9ZM=
+github.com/spf13/cobra v1.8.1/go.mod h1:wHxEcudfqmLYa8iTfL+OuZPbBZkmvliBWKIezN3kD9Y=
 github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo=
 github.com/spf13/jwalterweatherman v1.1.0 h1:ue6voC5bR5F8YxI5S67j9i582FU4Qvo2bmqnqMYADFk=
 github.com/spf13/jwalterweatherman v1.1.0/go.mod h1:aNWZUN0dPAAO/Ljvb5BEdw96iTZ0EXowPYD95IqWIGo=

From 2e112fcdaceece4ed913a796b2c4d40bfd566897 Mon Sep 17 00:00:00 2001
From: lukaszcl <120112546+lukaszcl@users.noreply.github.com>
Date: Mon, 24 Jun 2024 12:52:17 +0200
Subject: [PATCH 033/137] temp: Disable checking test configuration

---
 .../run-e2e-tests-reusable-workflow.yml       | 30 +++++++++----------
 1 file changed, 15 insertions(+), 15 deletions(-)

diff --git a/.github/workflows/run-e2e-tests-reusable-workflow.yml b/.github/workflows/run-e2e-tests-reusable-workflow.yml
index be45c4c8a93..4a9bf0633b8 100644
--- a/.github/workflows/run-e2e-tests-reusable-workflow.yml
+++ b/.github/workflows/run-e2e-tests-reusable-workflow.yml
@@ -47,21 +47,21 @@ env:
   CHAINLINK_IMAGE: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com/chainlink
 
 jobs:
-  run-check-tests:
-    name: Check Test Configurations
-    runs-on: ubuntu-latest
-    steps:
-      - name: Checkout code
-        uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2
-      - name: Set up Go
-        uses: actions/setup-go@v2
-        with:
-          go-version: '1.21.7'
-      - name: Run Check Tests Command
-        run: |
-          if ! go run .github/scripts/e2e_tests_tool/main.go check-tests ./integration-tests .github/e2e-tests.yml; then
-            echo "::error::Some E2E test configurations have to be added to .github/e2e-tests.yml. This file defines Github CI configuration for each E2E test or set of E2E tests." && exit 1
-          fi
+  # run-check-tests:
+  #   name: Check Test Configurations
+  #   runs-on: ubuntu-latest
+  #   steps:
+  #     - name: Checkout code
+  #       uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2
+  #     - name: Set up Go
+  #       uses: actions/setup-go@v2
+  #       with:
+  #         go-version: '1.21.7'
+  #     - name: Run Check Tests Command
+  #       run: |
+  #         if ! go run .github/scripts/e2e_tests_tool/main.go check-tests ./integration-tests .github/e2e-tests.yml; then
+  #           echo "::error::Some E2E test configurations have to be added to .github/e2e-tests.yml. This file defines Github CI configuration for each E2E test or set of E2E tests." && exit 1
+  #         fi
 
   load-test-configurations:
     # Uncomment if we want to run check-tests before loading test configurations

From aa967ac4a9a447e37d28a32c14fca6d82165212f Mon Sep 17 00:00:00 2001
From: lukaszcl <120112546+lukaszcl@users.noreply.github.com>
Date: Mon, 24 Jun 2024 18:10:05 +0200
Subject: [PATCH 034/137] Add enable-check-test-configurations to reusable
 workflow. False by default

---
 .../run-e2e-tests-reusable-workflow.yml       | 37 ++++++++++---------
 1 file changed, 20 insertions(+), 17 deletions(-)

diff --git a/.github/workflows/run-e2e-tests-reusable-workflow.yml b/.github/workflows/run-e2e-tests-reusable-workflow.yml
index 4a9bf0633b8..84f47d8c760 100644
--- a/.github/workflows/run-e2e-tests-reusable-workflow.yml
+++ b/.github/workflows/run-e2e-tests-reusable-workflow.yml
@@ -13,6 +13,11 @@ on:
         default: "*"
         required: true
         type: string
+      enable-check-test-configurations:
+        description: 'Set to "true" to enable check-test-configurations job'
+        required: false
+        type: boolean
+        default: true        
     secrets:
       QA_AWS_REGION:
         required: true
@@ -47,25 +52,23 @@ env:
   CHAINLINK_IMAGE: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com/chainlink
 
 jobs:
-  # run-check-tests:
-  #   name: Check Test Configurations
-  #   runs-on: ubuntu-latest
-  #   steps:
-  #     - name: Checkout code
-  #       uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2
-  #     - name: Set up Go
-  #       uses: actions/setup-go@v2
-  #       with:
-  #         go-version: '1.21.7'
-  #     - name: Run Check Tests Command
-  #       run: |
-  #         if ! go run .github/scripts/e2e_tests_tool/main.go check-tests ./integration-tests .github/e2e-tests.yml; then
-  #           echo "::error::Some E2E test configurations have to be added to .github/e2e-tests.yml. This file defines Github CI configuration for each E2E test or set of E2E tests." && exit 1
-  #         fi
+  check-test-configurations:
+    name: Check Test Configurations
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2
+      - name: Set up Go
+        uses: actions/setup-go@v2
+        with:
+          go-version: '1.21.7'
+      - name: Run Check Tests Command
+        run: |
+          if ! go run .github/scripts/e2e_tests_tool/main.go check-tests ./integration-tests .github/e2e-tests.yml; then
+            echo "::error::Some E2E test configurations have to be added to .github/e2e-tests.yml. This file defines Github CI configuration for each E2E test or set of E2E tests." && exit 1
+          fi
 
   load-test-configurations:
-    # Uncomment if we want to run check-tests before loading test configurations
-    # needs: run-check-tests 
     name: Load Test Configurations
     runs-on: ubuntu-latest
     outputs:

From acd558aa21b054fe7f9e23f66b0c721c71c8bcf4 Mon Sep 17 00:00:00 2001
From: lukaszcl <120112546+lukaszcl@users.noreply.github.com>
Date: Mon, 24 Jun 2024 18:18:07 +0200
Subject: [PATCH 035/137] Add automation upgrade test to e2e list

---
 .github/e2e-tests.yml | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/.github/e2e-tests.yml b/.github/e2e-tests.yml
index b894d9ae27d..155e3d8dda5 100644
--- a/.github/e2e-tests.yml
+++ b/.github/e2e-tests.yml
@@ -31,6 +31,16 @@ test-runner-matrix:
     cmd: cd integration-tests/ && go test smoke/ocr2_test.go -test.run ^TestOCRv2Basic$ -test.parallel=1 -timeout 30m -count=1 -json
     pyroscope-env: ci-smoke-ocr2-evm-simulated-nightly
 
+  # Automation upgrade tests
+  
+  - id: smoke/automation_upgrade_test.go:^TestAutomationNodeUpgrade/registry_2_0
+    path: integration-tests/smoke/automation_upgrade_test.go
+    test-type: docker
+    runs-on: ubuntu22.04-8cores-32GB
+    trigger: ["push", "nightly", "pull-request", "release-tag"]
+    cmd: cd integration-tests/ && go test smoke/automation_upgrade_test.go -test.run ^TestAutomationNodeUpgrade/registry_2_0 -test.parallel=1 -timeout 60m -count=1 -json
+    pyroscope-env: ci-smoke-automation-upgrade-tests
+
   # Example of a configuration for running a single soak test in Kubernetes Remote Runner
   - id: soak/ocr_test.go:^TestOCRSoak$
     path: integration-tests/soak/ocr_test.go

From f6c50499e5bdcd0942e075a91bfe983756d1be62 Mon Sep 17 00:00:00 2001
From: lukaszcl <120112546+lukaszcl@users.noreply.github.com>
Date: Mon, 24 Jun 2024 18:19:57 +0200
Subject: [PATCH 036/137] Fix enable-check-test-configurations

---
 .github/workflows/run-e2e-tests-reusable-workflow.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.github/workflows/run-e2e-tests-reusable-workflow.yml b/.github/workflows/run-e2e-tests-reusable-workflow.yml
index 84f47d8c760..850ca0bdae6 100644
--- a/.github/workflows/run-e2e-tests-reusable-workflow.yml
+++ b/.github/workflows/run-e2e-tests-reusable-workflow.yml
@@ -54,6 +54,7 @@ env:
 jobs:
   check-test-configurations:
     name: Check Test Configurations
+    if: ${{ inputs.enable-check-test-configurations }}
     runs-on: ubuntu-latest
     steps:
       - name: Checkout code

From 74fb16ddda234b02579d23f04d9defa8b213f244 Mon Sep 17 00:00:00 2001
From: lukaszcl <120112546+lukaszcl@users.noreply.github.com>
Date: Mon, 24 Jun 2024 18:28:38 +0200
Subject: [PATCH 037/137] Fix test cmd

---
 .github/e2e-tests.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/e2e-tests.yml b/.github/e2e-tests.yml
index 155e3d8dda5..e4182efacd5 100644
--- a/.github/e2e-tests.yml
+++ b/.github/e2e-tests.yml
@@ -38,7 +38,7 @@ test-runner-matrix:
     test-type: docker
     runs-on: ubuntu22.04-8cores-32GB
     trigger: ["push", "nightly", "pull-request", "release-tag"]
-    cmd: cd integration-tests/ && go test smoke/automation_upgrade_test.go -test.run ^TestAutomationNodeUpgrade/registry_2_0 -test.parallel=1 -timeout 60m -count=1 -json
+    cmd: cd integration-tests/smoke && go test -test.run ^TestAutomationNodeUpgrade/registry_2_0 -test.parallel=1 -timeout 60m -count=1 -json
     pyroscope-env: ci-smoke-automation-upgrade-tests
 
   # Example of a configuration for running a single soak test in Kubernetes Remote Runner

From 27b320116de534eef8685372babb61bd178f3fe2 Mon Sep 17 00:00:00 2001
From: lukaszcl <120112546+lukaszcl@users.noreply.github.com>
Date: Mon, 24 Jun 2024 18:34:43 +0200
Subject: [PATCH 038/137] Make enable-check-test-configurations false by
 default

---
 .github/workflows/run-e2e-tests-reusable-workflow.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/run-e2e-tests-reusable-workflow.yml b/.github/workflows/run-e2e-tests-reusable-workflow.yml
index 850ca0bdae6..ae270311cbd 100644
--- a/.github/workflows/run-e2e-tests-reusable-workflow.yml
+++ b/.github/workflows/run-e2e-tests-reusable-workflow.yml
@@ -17,7 +17,7 @@ on:
         description: 'Set to "true" to enable check-test-configurations job'
         required: false
         type: boolean
-        default: true        
+        default: false        
     secrets:
       QA_AWS_REGION:
         required: true

From b7375e0956f55b9764125a12c1f720975dda2f8c Mon Sep 17 00:00:00 2001
From: lukaszcl <120112546+lukaszcl@users.noreply.github.com>
Date: Mon, 24 Jun 2024 18:52:55 +0200
Subject: [PATCH 039/137] support test config override

---
 .github/e2e-tests.yml                         | 17 +++--
 .../e2e_tests_tool/cmd/check_tests_cmd.go     |  4 +-
 .../e2e_tests_tool/cmd/envreplace_cmd.go      | 59 +++++++++++++++++
 .../e2e_tests_tool/cmd/envreplace_cmd_test.go | 64 +++++++++++++++++++
 .../scripts/e2e_tests_tool/cmd/root_cmd.go    |  1 +
 .github/scripts/e2e_tests_tool/cmd/types.go   |  3 +-
 .../run-e2e-tests-reusable-workflow.yml       | 20 +++++-
 7 files changed, 158 insertions(+), 10 deletions(-)
 create mode 100644 .github/scripts/e2e_tests_tool/cmd/envreplace_cmd.go
 create mode 100644 .github/scripts/e2e_tests_tool/cmd/envreplace_cmd_test.go

diff --git a/.github/e2e-tests.yml b/.github/e2e-tests.yml
index e4182efacd5..3b69cb8448b 100644
--- a/.github/e2e-tests.yml
+++ b/.github/e2e-tests.yml
@@ -11,7 +11,7 @@ test-runner-matrix:
     test-type: docker
     runs-on: ubuntu-latest
     trigger: ["push", "nightly", "pull-request", "release-tag"]
-    cmd: cd integration-tests/ && go test smoke/ocr_test.go -timeout 30m -count=1 -json
+    test-cmd: cd integration-tests/ && go test smoke/ocr_test.go -timeout 30m -count=1 -json
     pyroscope-env: ci-smoke-ocr-evm-simulated-nightly
 
   # Example of 2 separate runners for the same test file but different tests. Can be used if tests if are too heavy to run on the same runner
@@ -20,7 +20,7 @@ test-runner-matrix:
     test-type: docker
     runs-on: ubuntu-latest
     trigger: ["push", "nightly", "pull-request", "release-tag"]
-    cmd: cd integration-tests/ && go test smoke/ocr2_test.go -test.run ^TestOCRv2Request$ -test.parallel=1 -timeout 30m -count=1 -json
+    test-cmd: cd integration-tests/ && go test smoke/ocr2_test.go -test.run ^TestOCRv2Request$ -test.parallel=1 -timeout 30m -count=1 -json
     pyroscope-env: ci-smoke-ocr2-evm-simulated-nightly
 
   - id: smoke/ocr2_test.go:^TestOCRv2Basic$
@@ -28,7 +28,7 @@ test-runner-matrix:
     test-type: docker
     runs-on: ubuntu-latest
     trigger: ["push", "nightly", "pull-request", "release-tag"]
-    cmd: cd integration-tests/ && go test smoke/ocr2_test.go -test.run ^TestOCRv2Basic$ -test.parallel=1 -timeout 30m -count=1 -json
+    test-cmd: cd integration-tests/ && go test smoke/ocr2_test.go -test.run ^TestOCRv2Basic$ -test.parallel=1 -timeout 30m -count=1 -json
     pyroscope-env: ci-smoke-ocr2-evm-simulated-nightly
 
   # Automation upgrade tests
@@ -38,7 +38,14 @@ test-runner-matrix:
     test-type: docker
     runs-on: ubuntu22.04-8cores-32GB
     trigger: ["push", "nightly", "pull-request", "release-tag"]
-    cmd: cd integration-tests/smoke && go test -test.run ^TestAutomationNodeUpgrade/registry_2_0 -test.parallel=1 -timeout 60m -count=1 -json
+    test-cmd: cd integration-tests/smoke && go test -test.run ^TestAutomationNodeUpgrade/registry_2_0 -test.parallel=1 -timeout 60m -count=1 -json
+    test-config-override: |
+      [ChainlinkUpgradeImage]
+      image="${{ env.CHAINLINK_IMAGE }}"
+      version="${{ env.GITHUB_SHA }}"
+      [ChainlinkImage]
+      image="public.ecr.aws/chainlink/chainlink"
+      version="latest"
     pyroscope-env: ci-smoke-automation-upgrade-tests
 
   # Example of a configuration for running a single soak test in Kubernetes Remote Runner
@@ -48,5 +55,5 @@ test-runner-matrix:
     remote-runner-test-suite: soak
     runs-on: ubuntu-latest
     trigger: ["nightly", "release-tag"] # Run nightly and on release tags
-    cmd: cd integration-tests/ && go test soak/ocr_test.go -test.run ^TestOCRSoak$ -test.parallel=1 -timeout 30m -count=1 -json
+    test-cmd: cd integration-tests/ && go test soak/ocr_test.go -test.run ^TestOCRSoak$ -test.parallel=1 -timeout 30m -count=1 -json
     pyroscope-env: ci-smoke-ocr2-evm-simulated-nightly
diff --git a/.github/scripts/e2e_tests_tool/cmd/check_tests_cmd.go b/.github/scripts/e2e_tests_tool/cmd/check_tests_cmd.go
index 037d1b50099..6c73bbcedd0 100644
--- a/.github/scripts/e2e_tests_tool/cmd/check_tests_cmd.go
+++ b/.github/scripts/e2e_tests_tool/cmd/check_tests_cmd.go
@@ -87,8 +87,8 @@ func checkTestsInPipeline(yamlFile string, tests []Test) {
 		found := false
 		for _, item := range config.Tests {
 			if item.Path == test.Path {
-				if strings.Contains(item.Cmd, "-test.run") {
-					if matchTestNameInCmd(item.Cmd, test.Name) {
+				if strings.Contains(item.TestCmd, "-test.run") {
+					if matchTestNameInCmd(item.TestCmd, test.Name) {
 						found = true
 						break
 					}
diff --git a/.github/scripts/e2e_tests_tool/cmd/envreplace_cmd.go b/.github/scripts/e2e_tests_tool/cmd/envreplace_cmd.go
new file mode 100644
index 00000000000..cd9d3bc7ee2
--- /dev/null
+++ b/.github/scripts/e2e_tests_tool/cmd/envreplace_cmd.go
@@ -0,0 +1,59 @@
+package cmd
+
+import (
+	"fmt"
+	"os"
+	"regexp"
+	"strings"
+
+	"github.com/spf13/cobra"
+)
+
+// envreplaceCmd represents the environment replace command
+var envreplaceCmd = &cobra.Command{
+	Use:   "envreplace",
+	Short: "Replace environment variable placeholders in a string",
+	Long: `Replaces placeholders of the form ${{ env.VAR_NAME }} with the actual environment variable values.
+Example usage:
+./e2e_tests_tool envreplace --input 'Example with ${{ env.PATH }} and ${{ env.HOME }}.'`,
+	Run: func(cmd *cobra.Command, args []string) {
+		input, _ := cmd.Flags().GetString("input")
+
+		output, err := replaceEnvPlaceholders(input)
+		if err != nil {
+			fmt.Printf("Error replacing placeholders: %v\n", err)
+			return
+		}
+
+		fmt.Println(output)
+	},
+}
+
+func replaceEnvPlaceholders(input string) (string, error) {
+	// Regular expression to match ${env.VAR_NAME}
+	r := regexp.MustCompile(`\$\{\{\s*env\.(\w+)\s*\}\}`)
+	var errors []string // Slice to accumulate error messages
+
+	// Replace each match in the input string
+	replaced := r.ReplaceAllStringFunc(input, func(m string) string {
+		varName := r.FindStringSubmatch(m)[1]
+		value := os.Getenv(varName)
+		if value == "" {
+			// If the environment variable is not set or empty, accumulate an error message
+			errors = append(errors, fmt.Sprintf("environment variable '%s' not set or is empty", varName))
+			return m // Return the original placeholder if error occurs
+		}
+		return value
+	})
+
+	// Check if there were any errors
+	if len(errors) > 0 {
+		return replaced, fmt.Errorf("multiple errors occurred: %s", strings.Join(errors, ", "))
+	}
+	return replaced, nil // No error occurred, return the replaced string
+}
+
+func init() {
+	envreplaceCmd.Flags().StringP("input", "i", "", "Input string with placeholders")
+	envreplaceCmd.MarkFlagRequired("input")
+}
diff --git a/.github/scripts/e2e_tests_tool/cmd/envreplace_cmd_test.go b/.github/scripts/e2e_tests_tool/cmd/envreplace_cmd_test.go
new file mode 100644
index 00000000000..a20a6fba5d2
--- /dev/null
+++ b/.github/scripts/e2e_tests_tool/cmd/envreplace_cmd_test.go
@@ -0,0 +1,64 @@
+package cmd
+
+import (
+	"os"
+	"strings"
+	"testing"
+)
+
+func TestReplaceEnvPlaceholders(t *testing.T) {
+	// Define test cases
+	tests := []struct {
+		name       string
+		input      string
+		want       string
+		wantErr    bool
+		wantErrMsg string
+	}{
+		{
+			name:    "All variables set",
+			input:   "Path: ${{ env.PATH }}, Home: ${{ env.HOME }}",
+			want:    "Path: /usr/bin, Home: /home/user",
+			wantErr: false,
+		},
+		{
+			name:       "One variable unset",
+			input:      "Path: ${{ env.PATH }}, Unset: ${{ env.UNSET_VAR }}",
+			want:       "Path: /usr/bin, Unset: ${{ env.UNSET_VAR }}",
+			wantErr:    true,
+			wantErrMsg: "environment variable 'UNSET_VAR' not set or is empty",
+		},
+		{
+			name:       "Multiple variables unset",
+			input:      "Unset1: ${{ env.UNSET_VAR1 }}, Unset2: ${{ env.UNSET_VAR2 }}",
+			want:       "Unset1: ${{ env.UNSET_VAR1 }}, Unset2: ${{ env.UNSET_VAR2 }}",
+			wantErr:    true,
+			wantErrMsg: "environment variable 'UNSET_VAR1' not set or is empty, environment variable 'UNSET_VAR2' not set or is empty",
+		},
+	}
+
+	// Set environment variables for the test
+	os.Setenv("PATH", "/usr/bin")
+	os.Setenv("HOME", "/home/user")
+	defer os.Unsetenv("PATH")
+	defer os.Unsetenv("HOME")
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got, err := replaceEnvPlaceholders(tt.input)
+			if (err != nil) != tt.wantErr {
+				t.Fatalf("replaceEnvPlaceholders() error = %v, wantErr %v", err, tt.wantErr)
+			}
+			if err != nil && !strings.Contains(err.Error(), tt.wantErrMsg) {
+				t.Errorf("replaceEnvPlaceholders() error = %v, wantErrMsg %v", err, tt.wantErrMsg)
+			}
+			if got != tt.want {
+				t.Errorf("replaceEnvPlaceholders() got = %v, want %v", got, tt.want)
+			}
+		})
+	}
+
+	// Unset the variables to clean up after tests
+	os.Unsetenv("UNSET_VAR1")
+	os.Unsetenv("UNSET_VAR2")
+}
diff --git a/.github/scripts/e2e_tests_tool/cmd/root_cmd.go b/.github/scripts/e2e_tests_tool/cmd/root_cmd.go
index 88310117536..ad10523a49e 100644
--- a/.github/scripts/e2e_tests_tool/cmd/root_cmd.go
+++ b/.github/scripts/e2e_tests_tool/cmd/root_cmd.go
@@ -26,4 +26,5 @@ func init() {
 
 	rootCmd.AddCommand(checkTestsCmd)
 	rootCmd.AddCommand(filterCmd)
+	rootCmd.AddCommand(envreplaceCmd)
 }
diff --git a/.github/scripts/e2e_tests_tool/cmd/types.go b/.github/scripts/e2e_tests_tool/cmd/types.go
index 24908ea2880..ee981c23431 100644
--- a/.github/scripts/e2e_tests_tool/cmd/types.go
+++ b/.github/scripts/e2e_tests_tool/cmd/types.go
@@ -10,7 +10,8 @@ type TestConf struct {
 	Path                  string   `yaml:"path" json:"path"`
 	TestType              string   `yaml:"test-type" json:"testType"`
 	RunsOn                string   `yaml:"runs-on" json:"runsOn"`
-	Cmd                   string   `yaml:"cmd" json:"cmd"`
+	TestCmd               string   `yaml:"test-cmd" json:"testCmd"`
+	TestConfigOverride    string   `yaml:"test-config-override" json:"testConfigOverride"`
 	RemoteRunnerTestSuite string   `yaml:"remote-runner-test-suite" json:"remoteRunnerTestSuite"`
 	PyroscopeEnv          string   `yaml:"pyroscope-env" json:"pyroscopeEnv"`
 	Trigger               []string `yaml:"trigger" json:"trigger"`
diff --git a/.github/workflows/run-e2e-tests-reusable-workflow.yml b/.github/workflows/run-e2e-tests-reusable-workflow.yml
index ae270311cbd..bc151a6077b 100644
--- a/.github/workflows/run-e2e-tests-reusable-workflow.yml
+++ b/.github/workflows/run-e2e-tests-reusable-workflow.yml
@@ -180,10 +180,26 @@ jobs:
           grafanaUrl: "http://localhost:8080/primary" # This is GAP's address
           grafanaDashboardUrl: "/d/ddf75041-1e39-42af-aa46-361fe4c36e9e/ci-e2e-tests-logs"
           grafanaBearerToken: ${{ secrets.GRAFANA_INTERNAL_URL_SHORTENER_TOKEN }}
+      - name: Set up Go
+        uses: actions/setup-go@v2
+        with:
+          go-version: '1.21.7'
+      - name: Set Test Config Override
+        if: ${{ matrix.tests.testConfigOverride != '' }}
+        run: |
+          echo "::add-mask::${{ env.CHAINLINK_IMAGE }}"
+
+          # Use the Go tool to replace environment variables
+          replaced_content=$(go run .github/scripts/e2e_tests_tool/main.go envreplace --input '${{ matrix.tests.testConfigOverride }}')
+          
+          # Encode the replaced content to Base64
+          BASE64_CONFIG_OVERRIDE=$(echo "$replaced_content" | base64 -w 0)
+          echo ::add-mask::$BASE64_CONFIG_OVERRIDE
+          echo "BASE64_CONFIG_OVERRIDE=$BASE64_CONFIG_OVERRIDE" >> $GITHUB_ENV
       - name: Run Tests
         uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/run-tests@af92c5fae8dcf1659201e907db82d221fc304b94 # v2.3.21
         with:
-          test_command_to_run: ${{ matrix.tests.cmd }} 2>&1 | tee /tmp/gotest.log | gotestloghelper -ci -singlepackage -hidepassingtests=false -hidepassinglogs
+          test_command_to_run: ${{ matrix.tests.testCmd }} 2>&1 | tee /tmp/gotest.log | gotestloghelper -ci -singlepackage -hidepassingtests=false -hidepassinglogs
           test_download_vendor_packages_command: cd ./integration-tests && go mod download
           cl_repo: ${{ env.CHAINLINK_IMAGE }}
           cl_image_tag: ${{ inputs.chainlink-version }}
@@ -268,7 +284,7 @@ jobs:
           TEST_UPLOAD_CPU_PROFILE: true
           TEST_UPLOAD_MEM_PROFILE: true
         with:
-          test_command_to_run: ${{ matrix.tests.cmd }}
+          test_command_to_run: ${{ matrix.tests.testCmd }}
           test_download_vendor_packages_command: make gomod
           cl_repo: ${{ env.CHAINLINK_IMAGE }}
           cl_image_tag: ${{ env.CHAINLINK_VERSION }}

From be845a27822442c2d55c16c7d30f8d6fb31e3660 Mon Sep 17 00:00:00 2001
From: lukaszcl <120112546+lukaszcl@users.noreply.github.com>
Date: Mon, 24 Jun 2024 20:04:44 +0200
Subject: [PATCH 040/137] add more automation tests

---
 .github/e2e-tests.yml | 32 ++++++++++++++++++++++++++++++++
 1 file changed, 32 insertions(+)

diff --git a/.github/e2e-tests.yml b/.github/e2e-tests.yml
index 3b69cb8448b..617ead2d0ea 100644
--- a/.github/e2e-tests.yml
+++ b/.github/e2e-tests.yml
@@ -48,6 +48,38 @@ test-runner-matrix:
       version="latest"
     pyroscope-env: ci-smoke-automation-upgrade-tests
 
+  - id: smoke/automation_upgrade_test.go:^TestAutomationNodeUpgrade/registry_2_1
+    path: integration-tests/smoke/automation_upgrade_test.go
+    test-type: docker
+    runs-on: ubuntu22.04-8cores-32GB
+    trigger: ["push", "nightly", "pull-request", "release-tag"]
+    test-cmd: cd integration-tests/smoke && go test -test.run ^TestAutomationNodeUpgrade/registry_2_1 -test.parallel=1 -timeout 60m -count=1 -json
+    test-config-override: |
+      [ChainlinkUpgradeImage]
+      image="${{ env.CHAINLINK_IMAGE }}"
+      version="${{ env.GITHUB_SHA }}"
+      [ChainlinkImage]
+      image="public.ecr.aws/chainlink/chainlink"
+      version="latest"
+    pyroscope-env: ci-smoke-automation-upgrade-tests
+
+  - id: smoke/automation_upgrade_test.go:^TestAutomationNodeUpgrade/registry_2_2
+    path: integration-tests/smoke/automation_upgrade_test.go
+    test-type: docker
+    runs-on: ubuntu22.04-8cores-32GB
+    trigger: ["push", "nightly", "pull-request", "release-tag"]
+    test-cmd: cd integration-tests/smoke && go test -test.run ^TestAutomationNodeUpgrade/registry_2_2 -test.parallel=1 -timeout 60m -count=1 -json
+    test-config-override: |
+      [ChainlinkUpgradeImage]
+      image="${{ env.CHAINLINK_IMAGE }}"
+      version="${{ env.GITHUB_SHA }}"
+      [ChainlinkImage]
+      image="public.ecr.aws/chainlink/chainlink"
+      version="latest"
+    pyroscope-env: ci-smoke-automation-upgrade-tests
+
+  # END Automation upgrade tests
+
   # Example of a configuration for running a single soak test in Kubernetes Remote Runner
   - id: soak/ocr_test.go:^TestOCRSoak$
     path: integration-tests/soak/ocr_test.go

From b0acdda44ea4b822961a4fa42de45e9a8faeda4d Mon Sep 17 00:00:00 2001
From: lukaszcl <120112546+lukaszcl@users.noreply.github.com>
Date: Mon, 24 Jun 2024 20:08:52 +0200
Subject: [PATCH 041/137] Run automation upgrade tests on latest develop

---
 .github/e2e-tests.yml | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/.github/e2e-tests.yml b/.github/e2e-tests.yml
index 617ead2d0ea..fa7b4cd5b35 100644
--- a/.github/e2e-tests.yml
+++ b/.github/e2e-tests.yml
@@ -42,7 +42,7 @@ test-runner-matrix:
     test-config-override: |
       [ChainlinkUpgradeImage]
       image="${{ env.CHAINLINK_IMAGE }}"
-      version="${{ env.GITHUB_SHA }}"
+      version="develop"
       [ChainlinkImage]
       image="public.ecr.aws/chainlink/chainlink"
       version="latest"
@@ -57,7 +57,7 @@ test-runner-matrix:
     test-config-override: |
       [ChainlinkUpgradeImage]
       image="${{ env.CHAINLINK_IMAGE }}"
-      version="${{ env.GITHUB_SHA }}"
+      version="develop"
       [ChainlinkImage]
       image="public.ecr.aws/chainlink/chainlink"
       version="latest"
@@ -72,7 +72,7 @@ test-runner-matrix:
     test-config-override: |
       [ChainlinkUpgradeImage]
       image="${{ env.CHAINLINK_IMAGE }}"
-      version="${{ env.GITHUB_SHA }}"
+      version="develop"
       [ChainlinkImage]
       image="public.ecr.aws/chainlink/chainlink"
       version="latest"

From 73ed44ef7a51daf2126e3dbe9bec9b0763dbc7c1 Mon Sep 17 00:00:00 2001
From: lukaszcl <120112546+lukaszcl@users.noreply.github.com>
Date: Mon, 24 Jun 2024 20:24:30 +0200
Subject: [PATCH 042/137] Migrate automation-nightly-tests.yml to reusable
 workflow

---
 .../run-automation-nightly-e2e-tests.yml      | 39 +++++++++++++++++++
 1 file changed, 39 insertions(+)
 create mode 100644 .github/workflows/run-automation-nightly-e2e-tests.yml

diff --git a/.github/workflows/run-automation-nightly-e2e-tests.yml b/.github/workflows/run-automation-nightly-e2e-tests.yml
new file mode 100644
index 00000000000..41b6a6f7a27
--- /dev/null
+++ b/.github/workflows/run-automation-nightly-e2e-tests.yml
@@ -0,0 +1,39 @@
+name: Run Automation Product Nightly E2E Tests
+
+on:
+  # schedule:
+  #   - cron: "0 0 * * *" # Run nightly
+  # push:
+  #   tags:
+  #     - "*"
+  workflow_dispatch:
+
+jobs:
+  call-run-e2e-tests-workflow:
+    name: Run E2E Tests
+    uses: ./.github/workflows/run-e2e-tests-reusable-workflow.yml
+    permissions:
+      actions: read
+      checks: write
+      pull-requests: write
+      id-token: write
+      contents: read    
+    with:
+      chainlink-version: ${{ github.event.inputs.chainlink-version }}
+      test-ids: "smoke/automation_upgrade_test.go:^TestAutomationNodeUpgrade/registry_2_0,smoke/automation_upgrade_test.go:^TestAutomationNodeUpgrade/registry_2_1,smoke/automation_upgrade_test.go:^TestAutomationNodeUpgrade/registry_2_2"
+    secrets:
+      QA_AWS_REGION: ${{ secrets.QA_AWS_REGION }}
+      QA_AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }}
+      QA_AWS_ACCOUNT_NUMBER: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}
+      QA_PYROSCOPE_INSTANCE: ${{ secrets.QA_PYROSCOPE_INSTANCE }}
+      QA_PYROSCOPE_KEY: ${{ secrets.QA_PYROSCOPE_KEY }}
+      QA_KUBECONFIG: ${{ secrets.QA_KUBECONFIG }}
+      GRAFANA_INTERNAL_TENANT_ID: ${{ secrets.GRAFANA_INTERNAL_TENANT_ID }}
+      GRAFANA_INTERNAL_BASIC_AUTH: ${{ secrets.GRAFANA_INTERNAL_BASIC_AUTH }}
+      GRAFANA_INTERNAL_HOST: ${{ secrets.GRAFANA_INTERNAL_HOST }}
+      GRAFANA_INTERNAL_URL_SHORTENER_TOKEN: ${{ secrets.GRAFANA_INTERNAL_URL_SHORTENER_TOKEN }}
+      GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}  
+      AWS_REGION: ${{ secrets.QA_AWS_REGION }}
+      AWS_OIDC_IAM_ROLE_VALIDATION_PROD_ARN: ${{ secrets.AWS_OIDC_IAM_ROLE_VALIDATION_PROD_ARN }}
+      AWS_API_GW_HOST_GRAFANA: ${{ secrets.AWS_API_GW_HOST_GRAFANA }}        
+

From 8c19ba887fb4ecea21a465ca3a9fd4b4f179c3ab Mon Sep 17 00:00:00 2001
From: lukaszcl <120112546+lukaszcl@users.noreply.github.com>
Date: Mon, 24 Jun 2024 20:55:04 +0200
Subject: [PATCH 043/137] Update automation tests

---
 .github/e2e-tests.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/e2e-tests.yml b/.github/e2e-tests.yml
index fa7b4cd5b35..c884154d930 100644
--- a/.github/e2e-tests.yml
+++ b/.github/e2e-tests.yml
@@ -53,7 +53,7 @@ test-runner-matrix:
     test-type: docker
     runs-on: ubuntu22.04-8cores-32GB
     trigger: ["push", "nightly", "pull-request", "release-tag"]
-    test-cmd: cd integration-tests/smoke && go test -test.run ^TestAutomationNodeUpgrade/registry_2_1 -test.parallel=1 -timeout 60m -count=1 -json
+    test-cmd: cd integration-tests/smoke && go test -test.run ^TestAutomationNodeUpgrade/registry_2_1 -test.parallel=5 -timeout 60m -count=1 -json
     test-config-override: |
       [ChainlinkUpgradeImage]
       image="${{ env.CHAINLINK_IMAGE }}"
@@ -68,7 +68,7 @@ test-runner-matrix:
     test-type: docker
     runs-on: ubuntu22.04-8cores-32GB
     trigger: ["push", "nightly", "pull-request", "release-tag"]
-    test-cmd: cd integration-tests/smoke && go test -test.run ^TestAutomationNodeUpgrade/registry_2_2 -test.parallel=1 -timeout 60m -count=1 -json
+    test-cmd: cd integration-tests/smoke && go test -test.run ^TestAutomationNodeUpgrade/registry_2_2 -test.parallel=5 -timeout 60m -count=1 -json
     test-config-override: |
       [ChainlinkUpgradeImage]
       image="${{ env.CHAINLINK_IMAGE }}"

From 092d8b0a737c77e69f906cd51ffc17e7f6dc361d Mon Sep 17 00:00:00 2001
From: lukaszcl <120112546+lukaszcl@users.noreply.github.com>
Date: Mon, 24 Jun 2024 20:55:38 +0200
Subject: [PATCH 044/137] Fix automation workflow

---
 .github/workflows/run-automation-nightly-e2e-tests.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/run-automation-nightly-e2e-tests.yml b/.github/workflows/run-automation-nightly-e2e-tests.yml
index 41b6a6f7a27..2159cd2d929 100644
--- a/.github/workflows/run-automation-nightly-e2e-tests.yml
+++ b/.github/workflows/run-automation-nightly-e2e-tests.yml
@@ -19,7 +19,7 @@ jobs:
       id-token: write
       contents: read    
     with:
-      chainlink-version: ${{ github.event.inputs.chainlink-version }}
+      chainlink-version: develop
       test-ids: "smoke/automation_upgrade_test.go:^TestAutomationNodeUpgrade/registry_2_0,smoke/automation_upgrade_test.go:^TestAutomationNodeUpgrade/registry_2_1,smoke/automation_upgrade_test.go:^TestAutomationNodeUpgrade/registry_2_2"
     secrets:
       QA_AWS_REGION: ${{ secrets.QA_AWS_REGION }}

From 74d55e7a8f7bd56780857876f8ff253cabdce26c Mon Sep 17 00:00:00 2001
From: lukaszcl <120112546+lukaszcl@users.noreply.github.com>
Date: Mon, 24 Jun 2024 20:56:32 +0200
Subject: [PATCH 045/137] Fix

---
 .github/workflows/run-automation-nightly-e2e-tests.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.github/workflows/run-automation-nightly-e2e-tests.yml b/.github/workflows/run-automation-nightly-e2e-tests.yml
index 2159cd2d929..06ff6b0215c 100644
--- a/.github/workflows/run-automation-nightly-e2e-tests.yml
+++ b/.github/workflows/run-automation-nightly-e2e-tests.yml
@@ -1,6 +1,7 @@
 name: Run Automation Product Nightly E2E Tests
 
 on:
+  push:
   # schedule:
   #   - cron: "0 0 * * *" # Run nightly
   # push:

From bbbbfc9beb9c635944beb03a5153d659fc113bcc Mon Sep 17 00:00:00 2001
From: lukaszcl <120112546+lukaszcl@users.noreply.github.com>
Date: Mon, 24 Jun 2024 20:57:00 +0200
Subject: [PATCH 046/137] Fix

---
 .github/workflows/run-automation-nightly-e2e-tests.yml | 1 -
 1 file changed, 1 deletion(-)

diff --git a/.github/workflows/run-automation-nightly-e2e-tests.yml b/.github/workflows/run-automation-nightly-e2e-tests.yml
index 06ff6b0215c..2159cd2d929 100644
--- a/.github/workflows/run-automation-nightly-e2e-tests.yml
+++ b/.github/workflows/run-automation-nightly-e2e-tests.yml
@@ -1,7 +1,6 @@
 name: Run Automation Product Nightly E2E Tests
 
 on:
-  push:
   # schedule:
   #   - cron: "0 0 * * *" # Run nightly
   # push:

From f4feb7fb19fe66858ce8e44b3a78a9997b2d333c Mon Sep 17 00:00:00 2001
From: lukaszcl <120112546+lukaszcl@users.noreply.github.com>
Date: Mon, 24 Jun 2024 21:03:33 +0200
Subject: [PATCH 047/137] Fx

---
 .github/workflows/run-automation-nightly-e2e-tests.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/run-automation-nightly-e2e-tests.yml b/.github/workflows/run-automation-nightly-e2e-tests.yml
index 2159cd2d929..8d9127a064e 100644
--- a/.github/workflows/run-automation-nightly-e2e-tests.yml
+++ b/.github/workflows/run-automation-nightly-e2e-tests.yml
@@ -20,7 +20,7 @@ jobs:
       contents: read    
     with:
       chainlink-version: develop
-      test-ids: "smoke/automation_upgrade_test.go:^TestAutomationNodeUpgrade/registry_2_0,smoke/automation_upgrade_test.go:^TestAutomationNodeUpgrade/registry_2_1,smoke/automation_upgrade_test.go:^TestAutomationNodeUpgrade/registry_2_2"
+      test-ids: 'smoke/automation_upgrade_test.go:^TestAutomationNodeUpgrade/registry_2_0,smoke/automation_upgrade_test.go:^TestAutomationNodeUpgrade/registry_2_1,smoke/automation_upgrade_test.go:^TestAutomationNodeUpgrade/registry_2_2'
     secrets:
       QA_AWS_REGION: ${{ secrets.QA_AWS_REGION }}
       QA_AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }}

From 5f44e12fec7aae387a05dc91308392531efecc10 Mon Sep 17 00:00:00 2001
From: lukaszcl <120112546+lukaszcl@users.noreply.github.com>
Date: Mon, 24 Jun 2024 21:07:18 +0200
Subject: [PATCH 048/137] Fix workflow_call inputs

---
 .github/workflows/run-e2e-tests-reusable-workflow.yml | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/run-e2e-tests-reusable-workflow.yml b/.github/workflows/run-e2e-tests-reusable-workflow.yml
index bc151a6077b..4d2047363a3 100644
--- a/.github/workflows/run-e2e-tests-reusable-workflow.yml
+++ b/.github/workflows/run-e2e-tests-reusable-workflow.yml
@@ -89,14 +89,14 @@ jobs:
       - name: Generate Docker Tests Matrix
         id: set-docker-matrix
         run: |
-          MATRIX_JSON=$(go run .github/scripts/e2e_tests_tool/main.go filter --file .github/e2e-tests.yml --test-type 'docker' --test-ids '${{ github.event.inputs.test-ids }}')
+          MATRIX_JSON=$(go run .github/scripts/e2e_tests_tool/main.go filter --file .github/e2e-tests.yml --test-type 'docker' --test-ids '${{ inputs.test-ids }}')
           echo "Docker tests:"
           echo "$MATRIX_JSON" | jq
           echo "::set-output name=matrix::$MATRIX_JSON"
       - name: Generate K8s Tests Matrix
         id: set-k8s-runner-matrix
         run: |
-          MATRIX_JSON=$(go run .github/scripts/e2e_tests_tool/main.go filter --file .github/e2e-tests.yml --test-type 'k8s-remote-runner' --test-ids '${{ github.event.inputs.test-ids }}')
+          MATRIX_JSON=$(go run .github/scripts/e2e_tests_tool/main.go filter --file .github/e2e-tests.yml --test-type 'k8s-remote-runner' --test-ids '${{ inputs.test-ids }}')
           echo "K8s tests:"
           echo "$MATRIX_JSON" | jq
           echo "::set-output name=matrix::$MATRIX_JSON"
@@ -125,7 +125,7 @@ jobs:
 
           # Check if both matrices are empty
           if [[ "$DOCKER_MATRIX_EMPTY" == "true" ]] && [[ "$K8S_MATRIX_EMPTY" == "true" ]]; then
-            echo "No tests found for inputs: '${{ toJson(github.event.inputs) }}'. Both Docker and Kubernetes tests matrices are empty"
+            echo "No tests found for inputs: '${{ toJson(inputs) }}'. Both Docker and Kubernetes tests matrices are empty"
             exit 1
           fi
         shell: bash   

From c88a7f09e6478d95f20fe17e4752d996b3657bee Mon Sep 17 00:00:00 2001
From: lukaszcl <120112546+lukaszcl@users.noreply.github.com>
Date: Mon, 24 Jun 2024 21:31:04 +0200
Subject: [PATCH 049/137] Select tests by github workflows

---
 .github/e2e-tests.yml                         | 24 +++++++++++++------
 .../scripts/e2e_tests_tool/cmd/filter_cmd.go  | 20 ++++++++--------
 .../e2e_tests_tool/cmd/filter_cmd_test.go     | 14 +++++------
 .github/scripts/e2e_tests_tool/cmd/types.go   |  2 +-
 .../run-automation-nightly-e2e-tests.yml      |  2 +-
 .../run-e2e-tests-reusable-workflow.yml       |  9 ++++---
 .github/workflows/run-nightly-e2e-tests.yml   |  2 +-
 7 files changed, 43 insertions(+), 30 deletions(-)

diff --git a/.github/e2e-tests.yml b/.github/e2e-tests.yml
index c884154d930..3863f1464dc 100644
--- a/.github/e2e-tests.yml
+++ b/.github/e2e-tests.yml
@@ -10,7 +10,9 @@ test-runner-matrix:
     path: integration-tests/smoke/ocr_test.go
     test-type: docker
     runs-on: ubuntu-latest
-    trigger: ["push", "nightly", "pull-request", "release-tag"]
+    workflows:
+      - Run PR E2E Tests
+      - Run Nightly E2E Tests
     test-cmd: cd integration-tests/ && go test smoke/ocr_test.go -timeout 30m -count=1 -json
     pyroscope-env: ci-smoke-ocr-evm-simulated-nightly
 
@@ -19,7 +21,9 @@ test-runner-matrix:
     path: integration-tests/smoke/ocr2_test.go
     test-type: docker
     runs-on: ubuntu-latest
-    trigger: ["push", "nightly", "pull-request", "release-tag"]
+    workflows:
+      - Run PR E2E Tests
+      - Run Nightly E2E Tests
     test-cmd: cd integration-tests/ && go test smoke/ocr2_test.go -test.run ^TestOCRv2Request$ -test.parallel=1 -timeout 30m -count=1 -json
     pyroscope-env: ci-smoke-ocr2-evm-simulated-nightly
 
@@ -27,7 +31,9 @@ test-runner-matrix:
     path: integration-tests/smoke/ocr2_test.go
     test-type: docker
     runs-on: ubuntu-latest
-    trigger: ["push", "nightly", "pull-request", "release-tag"]
+    workflows:
+      - Run PR E2E Tests
+      - Run Nightly E2E Tests
     test-cmd: cd integration-tests/ && go test smoke/ocr2_test.go -test.run ^TestOCRv2Basic$ -test.parallel=1 -timeout 30m -count=1 -json
     pyroscope-env: ci-smoke-ocr2-evm-simulated-nightly
 
@@ -37,7 +43,8 @@ test-runner-matrix:
     path: integration-tests/smoke/automation_upgrade_test.go
     test-type: docker
     runs-on: ubuntu22.04-8cores-32GB
-    trigger: ["push", "nightly", "pull-request", "release-tag"]
+    workflows:
+      - Run Automation Product Nightly E2E Tests
     test-cmd: cd integration-tests/smoke && go test -test.run ^TestAutomationNodeUpgrade/registry_2_0 -test.parallel=1 -timeout 60m -count=1 -json
     test-config-override: |
       [ChainlinkUpgradeImage]
@@ -52,7 +59,8 @@ test-runner-matrix:
     path: integration-tests/smoke/automation_upgrade_test.go
     test-type: docker
     runs-on: ubuntu22.04-8cores-32GB
-    trigger: ["push", "nightly", "pull-request", "release-tag"]
+    workflows:
+      - Run Automation Product Nightly E2E Tests    
     test-cmd: cd integration-tests/smoke && go test -test.run ^TestAutomationNodeUpgrade/registry_2_1 -test.parallel=5 -timeout 60m -count=1 -json
     test-config-override: |
       [ChainlinkUpgradeImage]
@@ -67,7 +75,8 @@ test-runner-matrix:
     path: integration-tests/smoke/automation_upgrade_test.go
     test-type: docker
     runs-on: ubuntu22.04-8cores-32GB
-    trigger: ["push", "nightly", "pull-request", "release-tag"]
+    workflows:
+      - Run Automation Product Nightly E2E Tests
     test-cmd: cd integration-tests/smoke && go test -test.run ^TestAutomationNodeUpgrade/registry_2_2 -test.parallel=5 -timeout 60m -count=1 -json
     test-config-override: |
       [ChainlinkUpgradeImage]
@@ -86,6 +95,7 @@ test-runner-matrix:
     test-type: k8s-remote-runner
     remote-runner-test-suite: soak
     runs-on: ubuntu-latest
-    trigger: ["nightly", "release-tag"] # Run nightly and on release tags
+    workflows:
+      - Run Nightly E2E Tests
     test-cmd: cd integration-tests/ && go test soak/ocr_test.go -test.run ^TestOCRSoak$ -test.parallel=1 -timeout 30m -count=1 -json
     pyroscope-env: ci-smoke-ocr2-evm-simulated-nightly
diff --git a/.github/scripts/e2e_tests_tool/cmd/filter_cmd.go b/.github/scripts/e2e_tests_tool/cmd/filter_cmd.go
index 15ee6c2634f..a571110d6be 100644
--- a/.github/scripts/e2e_tests_tool/cmd/filter_cmd.go
+++ b/.github/scripts/e2e_tests_tool/cmd/filter_cmd.go
@@ -11,20 +11,20 @@ import (
 	"gopkg.in/yaml.v2"
 )
 
-// Filter tests based on trigger, test type, and test IDs.
-func filterTests(tests []TestConf, names, trigger, testType, ids string) []TestConf {
-	triggerFilter := trigger
+// Filter tests based on workflow, test type, and test IDs.
+func filterTests(tests []TestConf, names, workflow, testType, ids string) []TestConf {
+	workflowFilter := workflow
 	typeFilter := testType
 	idFilter := strings.Split(ids, ",")
 
 	var filteredTests []TestConf
 
 	for _, test := range tests {
-		triggerMatch := trigger == "" || contains(test.Trigger, triggerFilter)
+		workflowMatch := workflow == "" || contains(test.Workflows, workflowFilter)
 		typeMatch := testType == "" || test.TestType == typeFilter
 		idMatch := ids == "*" || ids == "" || contains(idFilter, test.ID)
 
-		if triggerMatch && typeMatch && idMatch {
+		if workflowMatch && typeMatch && idMatch {
 			filteredTests = append(filteredTests, test)
 		}
 	}
@@ -46,13 +46,13 @@ func contains(slice []string, element string) bool {
 var filterCmd = &cobra.Command{
 	Use:   "filter",
 	Short: "Filter test configurations based on specified criteria",
-	Long: `Filters tests from a YAML configuration based on name, trigger, test type, and test IDs.
+	Long: `Filters tests from a YAML configuration based on name, workflow, test type, and test IDs.
 Example usage:
-./e2e_tests_tool filter --file .github/e2e-tests.yml --trigger "push" --test-type "docker" --test-ids "test1,test2"`,
+./e2e_tests_tool filter --file .github/e2e-tests.yml --workflow "Run Nightly E2E Tests" --test-type "docker" --test-ids "test1,test2"`,
 	Run: func(cmd *cobra.Command, args []string) {
 		yamlFile, _ := cmd.Flags().GetString("file")
 		names, _ := cmd.Flags().GetString("name")
-		trigger, _ := cmd.Flags().GetString("trigger")
+		workflow, _ := cmd.Flags().GetString("workflow")
 		testType, _ := cmd.Flags().GetString("test-type")
 		testIDs, _ := cmd.Flags().GetString("test-ids")
 
@@ -67,7 +67,7 @@ Example usage:
 			log.Fatalf("Error parsing YAML data: %v", err)
 		}
 
-		filteredTests := filterTests(config.Tests, names, trigger, testType, testIDs)
+		filteredTests := filterTests(config.Tests, names, workflow, testType, testIDs)
 		matrix := map[string][]TestConf{"tests": filteredTests}
 		matrixJSON, err := json.Marshal(matrix)
 		if err != nil {
@@ -81,7 +81,7 @@ Example usage:
 func init() {
 	filterCmd.Flags().StringP("file", "f", "", "Path to the YAML file")
 	filterCmd.Flags().StringP("name", "n", "", "Comma-separated list of test names to filter by")
-	filterCmd.Flags().StringP("trigger", "t", "", "Trigger filter")
+	filterCmd.Flags().StringP("workflow", "t", "", "Workflow filter")
 	filterCmd.Flags().StringP("test-type", "y", "", "Type of test to filter by")
 	filterCmd.Flags().StringP("test-ids", "i", "*", "Comma-separated list of test IDs to filter by")
 	filterCmd.MarkFlagRequired("file")
diff --git a/.github/scripts/e2e_tests_tool/cmd/filter_cmd_test.go b/.github/scripts/e2e_tests_tool/cmd/filter_cmd_test.go
index 27b8d09dedf..b76d03a0eb2 100644
--- a/.github/scripts/e2e_tests_tool/cmd/filter_cmd_test.go
+++ b/.github/scripts/e2e_tests_tool/cmd/filter_cmd_test.go
@@ -34,28 +34,28 @@ func TestFilterTestsByID(t *testing.T) {
 
 func TestFilterTestsIntegration(t *testing.T) {
 	tests := []TestConf{
-		{ID: "run_all_in_ocr_tests_go", TestType: "docker", Trigger: []string{"nightly"}},
-		{ID: "run_all_in_ocr2_tests_go", TestType: "docker", Trigger: []string{"push"}},
-		{ID: "run_all_in_ocr3_tests_go", TestType: "k8s_remote_runner", Trigger: []string{"push"}},
+		{ID: "run_all_in_ocr_tests_go", TestType: "docker", Workflows: []string{"Run Nightly E2E Tests"}},
+		{ID: "run_all_in_ocr2_tests_go", TestType: "docker", Workflows: []string{"Run PR E2E Tests"}},
+		{ID: "run_all_in_ocr3_tests_go", TestType: "k8s_remote_runner", Workflows: []string{"Run PR E2E Tests"}},
 	}
 
 	cases := []struct {
 		description   string
 		inputNames    string
-		inputTrigger  string
+		inputWorkflow string
 		inputTestType string
 		inputIDs      string
 		expectedLen   int
 	}{
 		{"Filter by test type and ID", "", "", "docker", "run_all_in_ocr2_tests_go", 1},
-		{"Filter by trigger and test type", "", "push", "docker", "*", 1},
+		{"Filter by trigger and test type", "", "Run PR E2E Tests", "docker", "*", 1},
 		{"No filters applied", "", "", "", "*", 3},
-		{"Filter mismatching all criteria", "", "nightly", "k8s_remote_runner", "run_all_in_ocr_tests_go", 0},
+		{"Filter mismatching all criteria", "", "Run Nightly E2E Tests", "", "", 1},
 	}
 
 	for _, c := range cases {
 		t.Run(c.description, func(t *testing.T) {
-			filtered := filterTests(tests, c.inputNames, c.inputTrigger, c.inputTestType, c.inputIDs)
+			filtered := filterTests(tests, c.inputNames, c.inputWorkflow, c.inputTestType, c.inputIDs)
 			if len(filtered) != c.expectedLen {
 				t.Errorf("FilterTests(%s) returned %d tests, expected %d", c.description, len(filtered), c.expectedLen)
 			}
diff --git a/.github/scripts/e2e_tests_tool/cmd/types.go b/.github/scripts/e2e_tests_tool/cmd/types.go
index ee981c23431..2b83f8bd193 100644
--- a/.github/scripts/e2e_tests_tool/cmd/types.go
+++ b/.github/scripts/e2e_tests_tool/cmd/types.go
@@ -14,7 +14,7 @@ type TestConf struct {
 	TestConfigOverride    string   `yaml:"test-config-override" json:"testConfigOverride"`
 	RemoteRunnerTestSuite string   `yaml:"remote-runner-test-suite" json:"remoteRunnerTestSuite"`
 	PyroscopeEnv          string   `yaml:"pyroscope-env" json:"pyroscopeEnv"`
-	Trigger               []string `yaml:"trigger" json:"trigger"`
+	Workflows             []string `yaml:"workflows" json:"workflows"`
 }
 
 type Config struct {
diff --git a/.github/workflows/run-automation-nightly-e2e-tests.yml b/.github/workflows/run-automation-nightly-e2e-tests.yml
index 8d9127a064e..6cba34d63d4 100644
--- a/.github/workflows/run-automation-nightly-e2e-tests.yml
+++ b/.github/workflows/run-automation-nightly-e2e-tests.yml
@@ -20,7 +20,7 @@ jobs:
       contents: read    
     with:
       chainlink-version: develop
-      test-ids: 'smoke/automation_upgrade_test.go:^TestAutomationNodeUpgrade/registry_2_0,smoke/automation_upgrade_test.go:^TestAutomationNodeUpgrade/registry_2_1,smoke/automation_upgrade_test.go:^TestAutomationNodeUpgrade/registry_2_2'
+      test-workflow: Run Automation Product Nightly E2E Tests
     secrets:
       QA_AWS_REGION: ${{ secrets.QA_AWS_REGION }}
       QA_AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }}
diff --git a/.github/workflows/run-e2e-tests-reusable-workflow.yml b/.github/workflows/run-e2e-tests-reusable-workflow.yml
index 4d2047363a3..6447bb3f5be 100644
--- a/.github/workflows/run-e2e-tests-reusable-workflow.yml
+++ b/.github/workflows/run-e2e-tests-reusable-workflow.yml
@@ -9,10 +9,13 @@ on:
         required: true
         type: string
       test-ids:
-        description: 'Run all tests "*" by default. Or, enter test IDs to run separated by commas. Example: "run_all_in_ocr_tests_go,run_TestOCRv2Request_in_ocr2_test_go". Check all test IDs in .github/e2e-tests.yml'
-        default: "*"
-        required: true
+        description: 'Run tests by test ids separated by commas. Example: "run_all_in_ocr_tests_go,run_TestOCRv2Request_in_ocr2_test_go". Check all test IDs in .github/e2e-tests.yml'
+        required: false
         type: string
+      test-workflow:
+        description: 'Run tests by workflow name. Example: "Run Nightly E2E Tests"'
+        required: false
+        type: string        
       enable-check-test-configurations:
         description: 'Set to "true" to enable check-test-configurations job'
         required: false
diff --git a/.github/workflows/run-nightly-e2e-tests.yml b/.github/workflows/run-nightly-e2e-tests.yml
index 2b956919d30..258937e0ce5 100644
--- a/.github/workflows/run-nightly-e2e-tests.yml
+++ b/.github/workflows/run-nightly-e2e-tests.yml
@@ -13,5 +13,5 @@ jobs:
     uses: ./.github/workflows/run-e2e-tests-reusable-workflow.yml
     with:
       chainlink-version: develop
-      test-ids: '*'
+      test-workflow: Run Nightly E2E Tests
     secrets: inherit

From e7be8f18dcbaf6ed9ae90f14ab17f3b06294415e Mon Sep 17 00:00:00 2001
From: lukaszcl <120112546+lukaszcl@users.noreply.github.com>
Date: Mon, 24 Jun 2024 21:34:49 +0200
Subject: [PATCH 050/137] Allow either test-ids or test-workflow in reusable
 workflow

---
 .../run-e2e-tests-reusable-workflow.yml         | 17 +++++++++++++++--
 1 file changed, 15 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/run-e2e-tests-reusable-workflow.yml b/.github/workflows/run-e2e-tests-reusable-workflow.yml
index 6447bb3f5be..d37847680c5 100644
--- a/.github/workflows/run-e2e-tests-reusable-workflow.yml
+++ b/.github/workflows/run-e2e-tests-reusable-workflow.yml
@@ -55,9 +55,21 @@ env:
   CHAINLINK_IMAGE: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com/chainlink
 
 jobs:
+  validate-inputs:
+    name: Validate Workflow Inputs
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check input conditions
+        run: |
+          if [[ "${{ inputs.test-ids }}" != "" && "${{ inputs.test-workflow }}" != "" ]]; then
+            echo "Error: Both 'test-ids' and 'test-workflow' are provided. Please specify only one."
+            exit 1
+          fi
+
   check-test-configurations:
     name: Check Test Configurations
     if: ${{ inputs.enable-check-test-configurations }}
+    needs: validate-inputs
     runs-on: ubuntu-latest
     steps:
       - name: Checkout code
@@ -75,6 +87,7 @@ jobs:
   load-test-configurations:
     name: Load Test Configurations
     runs-on: ubuntu-latest
+    needs: validate-inputs
     outputs:
       run-docker-tests: ${{ steps.check-matrices.outputs.run-docker-tests }}
       run-k8s-tests: ${{ steps.check-matrices.outputs.run-k8s-tests }}
@@ -92,14 +105,14 @@ jobs:
       - name: Generate Docker Tests Matrix
         id: set-docker-matrix
         run: |
-          MATRIX_JSON=$(go run .github/scripts/e2e_tests_tool/main.go filter --file .github/e2e-tests.yml --test-type 'docker' --test-ids '${{ inputs.test-ids }}')
+          MATRIX_JSON=$(go run .github/scripts/e2e_tests_tool/main.go filter --file .github/e2e-tests.yml --test-type 'docker' --test-ids '${{ inputs.test-ids }}' --workflow '${{ inputs.test-workflow }}')
           echo "Docker tests:"
           echo "$MATRIX_JSON" | jq
           echo "::set-output name=matrix::$MATRIX_JSON"
       - name: Generate K8s Tests Matrix
         id: set-k8s-runner-matrix
         run: |
-          MATRIX_JSON=$(go run .github/scripts/e2e_tests_tool/main.go filter --file .github/e2e-tests.yml --test-type 'k8s-remote-runner' --test-ids '${{ inputs.test-ids }}')
+          MATRIX_JSON=$(go run .github/scripts/e2e_tests_tool/main.go filter --file .github/e2e-tests.yml --test-type 'k8s-remote-runner' --test-ids '${{ inputs.test-ids }} --workflow '${{ inputs.test-workflow }}')
           echo "K8s tests:"
           echo "$MATRIX_JSON" | jq
           echo "::set-output name=matrix::$MATRIX_JSON"

From ea39b1453836124cc60ee21d524c59d1e0f2ebf8 Mon Sep 17 00:00:00 2001
From: lukaszcl <120112546+lukaszcl@users.noreply.github.com>
Date: Mon, 24 Jun 2024 21:36:39 +0200
Subject: [PATCH 051/137] Fix

---
 .github/workflows/run-e2e-tests-reusable-workflow.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/run-e2e-tests-reusable-workflow.yml b/.github/workflows/run-e2e-tests-reusable-workflow.yml
index d37847680c5..434d572b378 100644
--- a/.github/workflows/run-e2e-tests-reusable-workflow.yml
+++ b/.github/workflows/run-e2e-tests-reusable-workflow.yml
@@ -112,7 +112,7 @@ jobs:
       - name: Generate K8s Tests Matrix
         id: set-k8s-runner-matrix
         run: |
-          MATRIX_JSON=$(go run .github/scripts/e2e_tests_tool/main.go filter --file .github/e2e-tests.yml --test-type 'k8s-remote-runner' --test-ids '${{ inputs.test-ids }} --workflow '${{ inputs.test-workflow }}')
+          MATRIX_JSON=$(go run .github/scripts/e2e_tests_tool/main.go filter --file .github/e2e-tests.yml --test-type 'k8s-remote-runner' --test-ids '${{ inputs.test-ids }}' --workflow '${{ inputs.test-workflow }}')
           echo "K8s tests:"
           echo "$MATRIX_JSON" | jq
           echo "::set-output name=matrix::$MATRIX_JSON"

From a36fcee50e84c28a5f26a445e81039c38418eeef Mon Sep 17 00:00:00 2001
From: lukaszcl <120112546+lukaszcl@users.noreply.github.com>
Date: Mon, 24 Jun 2024 22:32:22 +0200
Subject: [PATCH 052/137] Update workflow

---
 .github/workflows/run-selected-e2e-tests.yml | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/.github/workflows/run-selected-e2e-tests.yml b/.github/workflows/run-selected-e2e-tests.yml
index 038e1a0047d..745d34f2f44 100644
--- a/.github/workflows/run-selected-e2e-tests.yml
+++ b/.github/workflows/run-selected-e2e-tests.yml
@@ -13,6 +13,11 @@ on:
         default: "*"
         required: true
         type: string
+      enable-check-test-configurations:
+        description: 'Set to "true" to enable check-test-configurations job'
+        required: false
+        type: boolean
+        default: false         
 
 jobs:
   call-run-e2e-tests-workflow:
@@ -27,6 +32,7 @@ jobs:
     with:
       chainlink-version: ${{ github.event.inputs.chainlink-version }}
       test-ids: ${{ github.event.inputs.test-ids }}
+      enable-check-test-configurations: ${{ github.event.inputs.enable-check-test-configurations }}
     secrets:
       QA_AWS_REGION: ${{ secrets.QA_AWS_REGION }}
       QA_AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }}

From 0b02624a2540799bbe8fcd4b9abf8be2ba9f3299 Mon Sep 17 00:00:00 2001
From: lukaszcl <120112546+lukaszcl@users.noreply.github.com>
Date: Mon, 24 Jun 2024 22:36:00 +0200
Subject: [PATCH 053/137] Add comment

---
 .github/workflows/run-automation-nightly-e2e-tests.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.github/workflows/run-automation-nightly-e2e-tests.yml b/.github/workflows/run-automation-nightly-e2e-tests.yml
index 6cba34d63d4..55c9648470c 100644
--- a/.github/workflows/run-automation-nightly-e2e-tests.yml
+++ b/.github/workflows/run-automation-nightly-e2e-tests.yml
@@ -20,6 +20,7 @@ jobs:
       contents: read    
     with:
       chainlink-version: develop
+      # Select tests to run from .github/e2e_tests.yml based on workflow name 
       test-workflow: Run Automation Product Nightly E2E Tests
     secrets:
       QA_AWS_REGION: ${{ secrets.QA_AWS_REGION }}

From 8f7e27c4918b9ec5bc6f0c77cadb88344134abb2 Mon Sep 17 00:00:00 2001
From: lukaszcl <120112546+lukaszcl@users.noreply.github.com>
Date: Mon, 24 Jun 2024 22:39:12 +0200
Subject: [PATCH 054/137] Fix

---
 .github/workflows/run-selected-e2e-tests.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/run-selected-e2e-tests.yml b/.github/workflows/run-selected-e2e-tests.yml
index 745d34f2f44..caaa4b57a6d 100644
--- a/.github/workflows/run-selected-e2e-tests.yml
+++ b/.github/workflows/run-selected-e2e-tests.yml
@@ -32,7 +32,7 @@ jobs:
     with:
       chainlink-version: ${{ github.event.inputs.chainlink-version }}
       test-ids: ${{ github.event.inputs.test-ids }}
-      enable-check-test-configurations: ${{ github.event.inputs.enable-check-test-configurations }}
+      enable-check-test-configurations: "${{ github.event.inputs.enable-check-test-configurations }}"
     secrets:
       QA_AWS_REGION: ${{ secrets.QA_AWS_REGION }}
       QA_AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }}

From a4754daef6fd35eabc2817a900aeb0d7a0c73643 Mon Sep 17 00:00:00 2001
From: lukaszcl <120112546+lukaszcl@users.noreply.github.com>
Date: Mon, 24 Jun 2024 22:39:33 +0200
Subject: [PATCH 055/137] Fix

---
 .github/workflows/run-selected-e2e-tests.yml | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/run-selected-e2e-tests.yml b/.github/workflows/run-selected-e2e-tests.yml
index caaa4b57a6d..b959d1beb57 100644
--- a/.github/workflows/run-selected-e2e-tests.yml
+++ b/.github/workflows/run-selected-e2e-tests.yml
@@ -32,7 +32,8 @@ jobs:
     with:
       chainlink-version: ${{ github.event.inputs.chainlink-version }}
       test-ids: ${{ github.event.inputs.test-ids }}
-      enable-check-test-configurations: "${{ github.event.inputs.enable-check-test-configurations }}"
+      # Use fromJSON to convert string to boolean. More info: https://github.com/actions/runner/issues/2206#issuecomment-1532246677
+      enable-check-test-configurations: ${{ fromJSON(github.event.inputs.enable-check-test-configurations) }}
     secrets:
       QA_AWS_REGION: ${{ secrets.QA_AWS_REGION }}
       QA_AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }}

From 3e0dcb72c21cf64361fea9b708442c5fbb54b81d Mon Sep 17 00:00:00 2001
From: lukaszcl <120112546+lukaszcl@users.noreply.github.com>
Date: Mon, 24 Jun 2024 22:56:40 +0200
Subject: [PATCH 056/137] Update e2e tests

---
 .github/e2e-tests.yml | 11 +++++++++++
 1 file changed, 11 insertions(+)

diff --git a/.github/e2e-tests.yml b/.github/e2e-tests.yml
index 3863f1464dc..6ac14cce62a 100644
--- a/.github/e2e-tests.yml
+++ b/.github/e2e-tests.yml
@@ -99,3 +99,14 @@ test-runner-matrix:
       - Run Nightly E2E Tests
     test-cmd: cd integration-tests/ && go test soak/ocr_test.go -test.run ^TestOCRSoak$ -test.parallel=1 -timeout 30m -count=1 -json
     pyroscope-env: ci-smoke-ocr2-evm-simulated-nightly
+
+  - id: integration-tests/benchmark/keeper_test.go:^TestAutomationBenchmark$
+    path: integration-tests/benchmark/keeper_test.go
+    test-type: k8s-remote-runner
+    remote-runner-test-suite: benchmark
+    remote-runner-memory: 4Gi
+    runs-on: ubuntu-latest
+    workflows:
+      - Run Nightly E2E Tests
+    test-cmd: cd integration-tests/benchmark && go test -test.run ^TestAutomationBenchmark$ -test.parallel=1 -timeout 30m -count=1 -json
+    pyroscope-env: ci-benchmark-automation-nightly
\ No newline at end of file

From da3b625d26645c35bc1665aee29701077a505544 Mon Sep 17 00:00:00 2001
From: lukaszcl <120112546+lukaszcl@users.noreply.github.com>
Date: Mon, 24 Jun 2024 23:02:02 +0200
Subject: [PATCH 057/137] Add remote-runner-memory option

---
 .github/scripts/e2e_tests_tool/cmd/types.go           | 1 +
 .github/workflows/run-e2e-tests-reusable-workflow.yml | 2 ++
 2 files changed, 3 insertions(+)

diff --git a/.github/scripts/e2e_tests_tool/cmd/types.go b/.github/scripts/e2e_tests_tool/cmd/types.go
index 2b83f8bd193..f3af171db53 100644
--- a/.github/scripts/e2e_tests_tool/cmd/types.go
+++ b/.github/scripts/e2e_tests_tool/cmd/types.go
@@ -13,6 +13,7 @@ type TestConf struct {
 	TestCmd               string   `yaml:"test-cmd" json:"testCmd"`
 	TestConfigOverride    string   `yaml:"test-config-override" json:"testConfigOverride"`
 	RemoteRunnerTestSuite string   `yaml:"remote-runner-test-suite" json:"remoteRunnerTestSuite"`
+	RemoteRunnerMemory    string   `yaml:"remote-runner-memory" json:"remoteRunnerMemory"`
 	PyroscopeEnv          string   `yaml:"pyroscope-env" json:"pyroscopeEnv"`
 	Workflows             []string `yaml:"workflows" json:"workflows"`
 }
diff --git a/.github/workflows/run-e2e-tests-reusable-workflow.yml b/.github/workflows/run-e2e-tests-reusable-workflow.yml
index 434d572b378..f32a8e95f80 100644
--- a/.github/workflows/run-e2e-tests-reusable-workflow.yml
+++ b/.github/workflows/run-e2e-tests-reusable-workflow.yml
@@ -292,6 +292,8 @@ jobs:
         env:
           DETACH_RUNNER: true
           TEST_SUITE: ${{ matrix.tests.remoteRunnerTestSuite }}
+          TEST_TYPE: ${{ matrix.tests.remoteRunnerTestSuite }}
+          RR_MEM: ${{ matrix.tests.remoteRunnerMemory }}
           TEST_ARGS: -test.timeout 900h -test.memprofile memprofile.out -test.cpuprofile profile.out
           # TODO: Remove and use sha step below
           ENV_JOB_IMAGE: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com/chainlink-tests:e7bce59ea616e4f13b8506f1a2414e3df586b130

From a780c75b24257fe289a03ab78d0c656f0a2ed266 Mon Sep 17 00:00:00 2001
From: lukaszcl <120112546+lukaszcl@users.noreply.github.com>
Date: Mon, 24 Jun 2024 23:04:05 +0200
Subject: [PATCH 058/137] Disable nightly run for now

---
 .github/workflows/run-automation-nightly-e2e-tests.yml | 1 +
 .github/workflows/run-nightly-e2e-tests.yml            | 6 ++++--
 2 files changed, 5 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/run-automation-nightly-e2e-tests.yml b/.github/workflows/run-automation-nightly-e2e-tests.yml
index 55c9648470c..3fa60114e6a 100644
--- a/.github/workflows/run-automation-nightly-e2e-tests.yml
+++ b/.github/workflows/run-automation-nightly-e2e-tests.yml
@@ -1,6 +1,7 @@
 name: Run Automation Product Nightly E2E Tests
 
 on:
+  # TODO: uncomment when the workflow is ready
   # schedule:
   #   - cron: "0 0 * * *" # Run nightly
   # push:
diff --git a/.github/workflows/run-nightly-e2e-tests.yml b/.github/workflows/run-nightly-e2e-tests.yml
index 258937e0ce5..175c6abdfcf 100644
--- a/.github/workflows/run-nightly-e2e-tests.yml
+++ b/.github/workflows/run-nightly-e2e-tests.yml
@@ -1,9 +1,11 @@
 name: Run Nightly E2E Tests
 
 on:
-  schedule:
+  # TODO: uncomment when the workflow is ready
+  # schedule:
     # Run every night at midnight UTC (0:00 AM) TODO: create separate workflow that uses this one 
-    - cron: '0 0 * * *'
+    # - cron: '0 0 * * *'
+    workflow_dispatch:
 
 jobs:
   call-run-e2e-tests-workflow:

From 52016f463deaa298d012d24b836410f15020e794 Mon Sep 17 00:00:00 2001
From: lukaszcl <120112546+lukaszcl@users.noreply.github.com>
Date: Mon, 24 Jun 2024 23:07:25 +0200
Subject: [PATCH 059/137] Fix output in k8s tests

---
 .github/workflows/run-e2e-tests-reusable-workflow.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/run-e2e-tests-reusable-workflow.yml b/.github/workflows/run-e2e-tests-reusable-workflow.yml
index f32a8e95f80..0f574e41ad5 100644
--- a/.github/workflows/run-e2e-tests-reusable-workflow.yml
+++ b/.github/workflows/run-e2e-tests-reusable-workflow.yml
@@ -302,7 +302,7 @@ jobs:
           TEST_UPLOAD_CPU_PROFILE: true
           TEST_UPLOAD_MEM_PROFILE: true
         with:
-          test_command_to_run: ${{ matrix.tests.testCmd }}
+          test_command_to_run: ${{ matrix.tests.testCmd }} 2>&1 | tee /tmp/gotest.log | gotestloghelper -ci -singlepackage -hidepassingtests=false -hidepassinglogs
           test_download_vendor_packages_command: make gomod
           cl_repo: ${{ env.CHAINLINK_IMAGE }}
           cl_image_tag: ${{ env.CHAINLINK_VERSION }}

From f0b4a662e70fce20e597bea88d28c84a81a5e475 Mon Sep 17 00:00:00 2001
From: lukaszcl <120112546+lukaszcl@users.noreply.github.com>
Date: Mon, 24 Jun 2024 23:17:42 +0200
Subject: [PATCH 060/137] Add show test configuration step

---
 .github/workflows/run-e2e-tests-reusable-workflow.yml | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/.github/workflows/run-e2e-tests-reusable-workflow.yml b/.github/workflows/run-e2e-tests-reusable-workflow.yml
index 0f574e41ad5..be24cc7f5ba 100644
--- a/.github/workflows/run-e2e-tests-reusable-workflow.yml
+++ b/.github/workflows/run-e2e-tests-reusable-workflow.yml
@@ -212,6 +212,8 @@ jobs:
           BASE64_CONFIG_OVERRIDE=$(echo "$replaced_content" | base64 -w 0)
           echo ::add-mask::$BASE64_CONFIG_OVERRIDE
           echo "BASE64_CONFIG_OVERRIDE=$BASE64_CONFIG_OVERRIDE" >> $GITHUB_ENV
+      - name: Show Test Configuration
+        run: echo "${{ toJson(matrix.tests) }}"          
       - name: Run Tests
         uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/run-tests@af92c5fae8dcf1659201e907db82d221fc304b94 # v2.3.21
         with:
@@ -287,6 +289,8 @@ jobs:
     steps:
       - name: Checkout repository
         uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2
+      - name: Show Test Configuration
+        run: echo "${{ toJson(matrix.tests) }}"
       - name: Run Tests
         uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/run-tests@af92c5fae8dcf1659201e907db82d221fc304b94 # v2.3.21
         env:

From d3bf5044af33e08be788a2df31c4a745cf69d787 Mon Sep 17 00:00:00 2001
From: lukaszcl <120112546+lukaszcl@users.noreply.github.com>
Date: Mon, 24 Jun 2024 23:26:10 +0200
Subject: [PATCH 061/137] Uncomment building test runner image

---
 .../run-e2e-tests-reusable-workflow.yml       | 21 +++++++++----------
 1 file changed, 10 insertions(+), 11 deletions(-)

diff --git a/.github/workflows/run-e2e-tests-reusable-workflow.yml b/.github/workflows/run-e2e-tests-reusable-workflow.yml
index be24cc7f5ba..4efc8b72422 100644
--- a/.github/workflows/run-e2e-tests-reusable-workflow.yml
+++ b/.github/workflows/run-e2e-tests-reusable-workflow.yml
@@ -257,13 +257,12 @@ jobs:
     steps:
       - name: Checkout repository
         uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2      
-      # TODO: Uncomment this step
-      # - name: Build Test Runner Image
-      #   uses: ./.github/actions/build-test-image
-      #   with:
-      #     QA_AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }}
-      #     QA_AWS_REGION: ${{ secrets.QA_AWS_REGION }}
-      #     QA_AWS_ACCOUNT_NUMBER: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}
+      - name: Build Test Runner Image
+        uses: ./.github/actions/build-test-image
+        with:
+          QA_AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }}
+          QA_AWS_REGION: ${{ secrets.QA_AWS_REGION }}
+          QA_AWS_ACCOUNT_NUMBER: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}
 
   run-k8s-runner-tests:
     needs: [load-test-configurations, build-k8s-runner-test-image]
@@ -299,14 +298,14 @@ jobs:
           TEST_TYPE: ${{ matrix.tests.remoteRunnerTestSuite }}
           RR_MEM: ${{ matrix.tests.remoteRunnerMemory }}
           TEST_ARGS: -test.timeout 900h -test.memprofile memprofile.out -test.cpuprofile profile.out
-          # TODO: Remove and use sha step below
-          ENV_JOB_IMAGE: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com/chainlink-tests:e7bce59ea616e4f13b8506f1a2414e3df586b130
-          # ENV_JOB_IMAGE: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com/chainlink-tests:${{ github.sha }}
+          # ENV_JOB_IMAGE: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com/chainlink-tests:e7bce59ea616e4f13b8506f1a2414e3df586b130
+          ENV_JOB_IMAGE: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com/chainlink-tests:${{ github.sha }}
+          INTERNAL_DOCKER_REPO: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com
           # We can comment these out when we have a stable soak test and aren't worried about resource consumption
           TEST_UPLOAD_CPU_PROFILE: true
           TEST_UPLOAD_MEM_PROFILE: true
         with:
-          test_command_to_run: ${{ matrix.tests.testCmd }} 2>&1 | tee /tmp/gotest.log | gotestloghelper -ci -singlepackage -hidepassingtests=false -hidepassinglogs
+          test_command_to_run: ${{ matrix.tests.testCmd }}
           test_download_vendor_packages_command: make gomod
           cl_repo: ${{ env.CHAINLINK_IMAGE }}
           cl_image_tag: ${{ env.CHAINLINK_VERSION }}

From 20e14098efc4d5534e20f7173a80e76f9abbacc6 Mon Sep 17 00:00:00 2001
From: lukaszcl <120112546+lukaszcl@users.noreply.github.com>
Date: Mon, 24 Jun 2024 23:36:09 +0200
Subject: [PATCH 062/137] Do not use json output for k8s tests

---
 .github/e2e-tests.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/e2e-tests.yml b/.github/e2e-tests.yml
index 6ac14cce62a..149d29d50ce 100644
--- a/.github/e2e-tests.yml
+++ b/.github/e2e-tests.yml
@@ -97,7 +97,7 @@ test-runner-matrix:
     runs-on: ubuntu-latest
     workflows:
       - Run Nightly E2E Tests
-    test-cmd: cd integration-tests/ && go test soak/ocr_test.go -test.run ^TestOCRSoak$ -test.parallel=1 -timeout 30m -count=1 -json
+    test-cmd: cd integration-tests/ && go test soak/ocr_test.go -test.run ^TestOCRSoak$ -test.parallel=1 -timeout 30m -count=1
     pyroscope-env: ci-smoke-ocr2-evm-simulated-nightly
 
   - id: integration-tests/benchmark/keeper_test.go:^TestAutomationBenchmark$
@@ -108,5 +108,5 @@ test-runner-matrix:
     runs-on: ubuntu-latest
     workflows:
       - Run Nightly E2E Tests
-    test-cmd: cd integration-tests/benchmark && go test -test.run ^TestAutomationBenchmark$ -test.parallel=1 -timeout 30m -count=1 -json
+    test-cmd: cd integration-tests/benchmark && go test -test.run ^TestAutomationBenchmark$ -test.parallel=1 -timeout 30m -count=1
     pyroscope-env: ci-benchmark-automation-nightly
\ No newline at end of file

From 973cdff9a6c59cf9e596bb8bad951b0aff893331 Mon Sep 17 00:00:00 2001
From: lukaszcl <120112546+lukaszcl@users.noreply.github.com>
Date: Mon, 24 Jun 2024 23:44:42 +0200
Subject: [PATCH 063/137] test

---
 .../run-e2e-tests-reusable-workflow.yml          | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/.github/workflows/run-e2e-tests-reusable-workflow.yml b/.github/workflows/run-e2e-tests-reusable-workflow.yml
index 4efc8b72422..fb1aa1e95b7 100644
--- a/.github/workflows/run-e2e-tests-reusable-workflow.yml
+++ b/.github/workflows/run-e2e-tests-reusable-workflow.yml
@@ -257,12 +257,12 @@ jobs:
     steps:
       - name: Checkout repository
         uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2      
-      - name: Build Test Runner Image
-        uses: ./.github/actions/build-test-image
-        with:
-          QA_AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }}
-          QA_AWS_REGION: ${{ secrets.QA_AWS_REGION }}
-          QA_AWS_ACCOUNT_NUMBER: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}
+      # - name: Build Test Runner Image
+      #   uses: ./.github/actions/build-test-image
+      #   with:
+      #     QA_AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }}
+      #     QA_AWS_REGION: ${{ secrets.QA_AWS_REGION }}
+      #     QA_AWS_ACCOUNT_NUMBER: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}
 
   run-k8s-runner-tests:
     needs: [load-test-configurations, build-k8s-runner-test-image]
@@ -298,8 +298,8 @@ jobs:
           TEST_TYPE: ${{ matrix.tests.remoteRunnerTestSuite }}
           RR_MEM: ${{ matrix.tests.remoteRunnerMemory }}
           TEST_ARGS: -test.timeout 900h -test.memprofile memprofile.out -test.cpuprofile profile.out
-          # ENV_JOB_IMAGE: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com/chainlink-tests:e7bce59ea616e4f13b8506f1a2414e3df586b130
-          ENV_JOB_IMAGE: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com/chainlink-tests:${{ github.sha }}
+          ENV_JOB_IMAGE: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com/chainlink-tests:d3bf5044af33e08be788a2df31c4a745cf69d787
+          # ENV_JOB_IMAGE: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com/chainlink-tests:${{ github.sha }}
           INTERNAL_DOCKER_REPO: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com
           # We can comment these out when we have a stable soak test and aren't worried about resource consumption
           TEST_UPLOAD_CPU_PROFILE: true

From 104dbdc524f3ae0090af64a6e72fdb4d249b6d7b Mon Sep 17 00:00:00 2001
From: lukaszcl <120112546+lukaszcl@users.noreply.github.com>
Date: Tue, 25 Jun 2024 00:07:52 +0200
Subject: [PATCH 064/137] Support custom runner version

---
 .../run-e2e-tests-reusable-workflow.yml       | 49 +++++++++++++------
 .github/workflows/run-selected-e2e-tests.yml  |  7 ++-
 2 files changed, 41 insertions(+), 15 deletions(-)

diff --git a/.github/workflows/run-e2e-tests-reusable-workflow.yml b/.github/workflows/run-e2e-tests-reusable-workflow.yml
index fb1aa1e95b7..c63c08f4595 100644
--- a/.github/workflows/run-e2e-tests-reusable-workflow.yml
+++ b/.github/workflows/run-e2e-tests-reusable-workflow.yml
@@ -20,7 +20,11 @@ on:
         description: 'Set to "true" to enable check-test-configurations job'
         required: false
         type: boolean
-        default: false        
+        default: false
+      with-existing-remote-runner-version:
+        description: 'Use the existing remote runner version for k8s tests. Example: "d3bf5044af33e08be788a2df31c4a745cf69d787"'
+        required: false
+        type: string
     secrets:
       QA_AWS_REGION:
         required: true
@@ -212,8 +216,10 @@ jobs:
           BASE64_CONFIG_OVERRIDE=$(echo "$replaced_content" | base64 -w 0)
           echo ::add-mask::$BASE64_CONFIG_OVERRIDE
           echo "BASE64_CONFIG_OVERRIDE=$BASE64_CONFIG_OVERRIDE" >> $GITHUB_ENV
+      - name: Install jq
+        run: sudo apt-get install -y jq
       - name: Show Test Configuration
-        run: echo "${{ toJson(matrix.tests) }}"          
+        run: echo '${{ toJson(matrix.tests) }}' | jq .
       - name: Run Tests
         uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/run-tests@af92c5fae8dcf1659201e907db82d221fc304b94 # v2.3.21
         with:
@@ -240,10 +246,10 @@ jobs:
 
   # Run K8s tests using old remote runner
 
-  build-k8s-runner-test-image:
+  prepare-remote-runner-test-image:
     needs: load-test-configurations
     if: ${{ needs.load-test-configurations.outputs.run-k8s-tests == 'true' }}
-    name: Build K8s Runner Test Image
+    name: Prepare Remote Runner Test Image
     runs-on: ubuntu-latest
     environment: integration
     permissions:
@@ -252,20 +258,31 @@ jobs:
       pull-requests: write
       id-token: write
       contents: read
+    outputs:
+      remote-runner-version: ${{ steps.set-remote-runner-version.outputs.remote-runner-version }}
     env:
       ENV_JOB_IMAGE_BASE: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com/chainlink-tests
     steps:
       - name: Checkout repository
         uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2      
-      # - name: Build Test Runner Image
-      #   uses: ./.github/actions/build-test-image
-      #   with:
-      #     QA_AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }}
-      #     QA_AWS_REGION: ${{ secrets.QA_AWS_REGION }}
-      #     QA_AWS_ACCOUNT_NUMBER: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}
+      - name: Build Test Runner Image
+        uses: ./.github/actions/build-test-image
+        if: ${{ inputs.with-existing-remote-runner-version == '' }}
+        with:
+          QA_AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }}
+          QA_AWS_REGION: ${{ secrets.QA_AWS_REGION }}
+          QA_AWS_ACCOUNT_NUMBER: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}
+      - name: Set Remote Runner Version
+        id: set-remote-runner-version
+        run: |
+          if [[ -z "${{ inputs.with-existing-remote-runner-version }}" ]]; then
+            echo "::set-output name=remote-runner-version::${{ github.sha }}"
+          else
+            echo "::set-output name=remote-runner-version::${{ inputs.with-existing-remote-runner-version }}"
+          fi
 
   run-k8s-runner-tests:
-    needs: [load-test-configurations, build-k8s-runner-test-image]
+    needs: [load-test-configurations, prepare-remote-runner-test-image]
     if: ${{ needs.load-test-configurations.outputs.run-k8s-tests == 'true' }}
     name: Run K8s Tests (${{ matrix.tests.id }})
     runs-on: ${{ matrix.tests.runsOn }}
@@ -288,8 +305,13 @@ jobs:
     steps:
       - name: Checkout repository
         uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2
+      - name: Install jq
+        run: sudo apt-get install -y jq
       - name: Show Test Configuration
-        run: echo "${{ toJson(matrix.tests) }}"
+        run: echo '${{ toJson(matrix.tests) }}' | jq .
+      - name: Show Remote Runner Version
+        run: |
+          echo "Remote Runner Version: ${{ needs.prepare-remote-runner-test-image.outputs.remote-runner-version }}"
       - name: Run Tests
         uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/run-tests@af92c5fae8dcf1659201e907db82d221fc304b94 # v2.3.21
         env:
@@ -298,8 +320,7 @@ jobs:
           TEST_TYPE: ${{ matrix.tests.remoteRunnerTestSuite }}
           RR_MEM: ${{ matrix.tests.remoteRunnerMemory }}
           TEST_ARGS: -test.timeout 900h -test.memprofile memprofile.out -test.cpuprofile profile.out
-          ENV_JOB_IMAGE: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com/chainlink-tests:d3bf5044af33e08be788a2df31c4a745cf69d787
-          # ENV_JOB_IMAGE: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com/chainlink-tests:${{ github.sha }}
+          ENV_JOB_IMAGE: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com/chainlink-tests:${{ needs.prepare-remote-runner-test-image.outputs.remote-runner-version }}
           INTERNAL_DOCKER_REPO: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com
           # We can comment these out when we have a stable soak test and aren't worried about resource consumption
           TEST_UPLOAD_CPU_PROFILE: true
diff --git a/.github/workflows/run-selected-e2e-tests.yml b/.github/workflows/run-selected-e2e-tests.yml
index b959d1beb57..3c42b4f8bb1 100644
--- a/.github/workflows/run-selected-e2e-tests.yml
+++ b/.github/workflows/run-selected-e2e-tests.yml
@@ -17,7 +17,11 @@ on:
         description: 'Set to "true" to enable check-test-configurations job'
         required: false
         type: boolean
-        default: false         
+        default: false
+      with-existing-remote-runner-version:
+        description: 'Use the existing remote runner version for k8s tests. Example: "d3bf5044af33e08be788a2df31c4a745cf69d787"'
+        required: false
+        type: string                 
 
 jobs:
   call-run-e2e-tests-workflow:
@@ -32,6 +36,7 @@ jobs:
     with:
       chainlink-version: ${{ github.event.inputs.chainlink-version }}
       test-ids: ${{ github.event.inputs.test-ids }}
+      with-existing-remote-runner-version: ${{ github.event.inputs.with-existing-remote-runner-version }}
       # Use fromJSON to convert string to boolean. More info: https://github.com/actions/runner/issues/2206#issuecomment-1532246677
       enable-check-test-configurations: ${{ fromJSON(github.event.inputs.enable-check-test-configurations) }}
     secrets:

From 1ff30f24061b2526491fefa5fc00d6526e73806b Mon Sep 17 00:00:00 2001
From: lukaszcl <120112546+lukaszcl@users.noreply.github.com>
Date: Tue, 25 Jun 2024 00:18:14 +0200
Subject: [PATCH 065/137] Fix test type env in remote runner

---
 .github/workflows/run-e2e-tests-reusable-workflow.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.github/workflows/run-e2e-tests-reusable-workflow.yml b/.github/workflows/run-e2e-tests-reusable-workflow.yml
index c63c08f4595..9e808279f07 100644
--- a/.github/workflows/run-e2e-tests-reusable-workflow.yml
+++ b/.github/workflows/run-e2e-tests-reusable-workflow.yml
@@ -318,6 +318,7 @@ jobs:
           DETACH_RUNNER: true
           TEST_SUITE: ${{ matrix.tests.remoteRunnerTestSuite }}
           TEST_TYPE: ${{ matrix.tests.remoteRunnerTestSuite }}
+          TEST_TEST_TYPE: ${{ matrix.tests.remoteRunnerTestSuite }}
           RR_MEM: ${{ matrix.tests.remoteRunnerMemory }}
           TEST_ARGS: -test.timeout 900h -test.memprofile memprofile.out -test.cpuprofile profile.out
           ENV_JOB_IMAGE: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com/chainlink-tests:${{ needs.prepare-remote-runner-test-image.outputs.remote-runner-version }}

From 3d9820e289adee6be3042df1c05f1c8869fdcc8d Mon Sep 17 00:00:00 2001
From: lukaszcl <120112546+lukaszcl@users.noreply.github.com>
Date: Tue, 25 Jun 2024 09:03:21 +0200
Subject: [PATCH 066/137] Update names and docs

---
 .github/e2e-tests.yml                                 | 10 ++++++----
 .github/scripts/e2e_tests_tool/cmd/filter_cmd.go      |  6 +++---
 .github/scripts/e2e_tests_tool/cmd/filter_cmd_test.go |  4 ++--
 .github/scripts/e2e_tests_tool/cmd/types.go           |  5 +++--
 4 files changed, 14 insertions(+), 11 deletions(-)

diff --git a/.github/e2e-tests.yml b/.github/e2e-tests.yml
index 149d29d50ce..43031fe663f 100644
--- a/.github/e2e-tests.yml
+++ b/.github/e2e-tests.yml
@@ -1,9 +1,11 @@
-# This file defines Github CI configuration for each E2E test and is used by CI workflows to run E2E tests.
+# This file specifies the GitHub runner for each E2E test and is utilized by all E2E CI workflows.
 #
-# Each test entry details the Github runner configuration, test environment (Docker or Kubernetes), Go test command, and triggers
-# that determine under what conditions the test should be executed (e.g., on push or nightly builds).
+# Each entry in this file includes the following:
+# - The GitHub runner (runs-on field) that will execute tests.
+# - The tests that will be run by the runner.
+# - The workflows (e.g., Run PR E2E Tests, Run Nightly E2E Tests) that should trigger these tests.
 #
-test-runner-matrix:
+runner-test-matrix:
 
   # Example of 1 runner for all tests in integration-tests/smoke/ocr_test.go
   - id: smoke/ocr_test.go:* 
diff --git a/.github/scripts/e2e_tests_tool/cmd/filter_cmd.go b/.github/scripts/e2e_tests_tool/cmd/filter_cmd.go
index a571110d6be..5de3bea3253 100644
--- a/.github/scripts/e2e_tests_tool/cmd/filter_cmd.go
+++ b/.github/scripts/e2e_tests_tool/cmd/filter_cmd.go
@@ -12,12 +12,12 @@ import (
 )
 
 // Filter tests based on workflow, test type, and test IDs.
-func filterTests(tests []TestConf, names, workflow, testType, ids string) []TestConf {
+func filterTests(tests []CITestConf, names, workflow, testType, ids string) []CITestConf {
 	workflowFilter := workflow
 	typeFilter := testType
 	idFilter := strings.Split(ids, ",")
 
-	var filteredTests []TestConf
+	var filteredTests []CITestConf
 
 	for _, test := range tests {
 		workflowMatch := workflow == "" || contains(test.Workflows, workflowFilter)
@@ -68,7 +68,7 @@ Example usage:
 		}
 
 		filteredTests := filterTests(config.Tests, names, workflow, testType, testIDs)
-		matrix := map[string][]TestConf{"tests": filteredTests}
+		matrix := map[string][]CITestConf{"tests": filteredTests}
 		matrixJSON, err := json.Marshal(matrix)
 		if err != nil {
 			log.Fatalf("Error marshaling matrix to JSON: %v", err)
diff --git a/.github/scripts/e2e_tests_tool/cmd/filter_cmd_test.go b/.github/scripts/e2e_tests_tool/cmd/filter_cmd_test.go
index b76d03a0eb2..28871f337ad 100644
--- a/.github/scripts/e2e_tests_tool/cmd/filter_cmd_test.go
+++ b/.github/scripts/e2e_tests_tool/cmd/filter_cmd_test.go
@@ -5,7 +5,7 @@ import (
 )
 
 func TestFilterTestsByID(t *testing.T) {
-	tests := []TestConf{
+	tests := []CITestConf{
 		{ID: "run_all_in_ocr_tests_go", TestType: "docker"},
 		{ID: "run_all_in_ocr2_tests_go", TestType: "docker"},
 		{ID: "run_all_in_ocr3_tests_go", TestType: "k8s_remote_runner"},
@@ -33,7 +33,7 @@ func TestFilterTestsByID(t *testing.T) {
 }
 
 func TestFilterTestsIntegration(t *testing.T) {
-	tests := []TestConf{
+	tests := []CITestConf{
 		{ID: "run_all_in_ocr_tests_go", TestType: "docker", Workflows: []string{"Run Nightly E2E Tests"}},
 		{ID: "run_all_in_ocr2_tests_go", TestType: "docker", Workflows: []string{"Run PR E2E Tests"}},
 		{ID: "run_all_in_ocr3_tests_go", TestType: "k8s_remote_runner", Workflows: []string{"Run PR E2E Tests"}},
diff --git a/.github/scripts/e2e_tests_tool/cmd/types.go b/.github/scripts/e2e_tests_tool/cmd/types.go
index f3af171db53..a297323d8b0 100644
--- a/.github/scripts/e2e_tests_tool/cmd/types.go
+++ b/.github/scripts/e2e_tests_tool/cmd/types.go
@@ -5,7 +5,8 @@ type Test struct {
 	Path string
 }
 
-type TestConf struct {
+// CITestConf defines the configuration for running a test in a CI environment, specifying details like test ID, path, type, runner settings, command, and associated workflows.
+type CITestConf struct {
 	ID                    string   `yaml:"id" json:"id"`
 	Path                  string   `yaml:"path" json:"path"`
 	TestType              string   `yaml:"test-type" json:"testType"`
@@ -19,5 +20,5 @@ type TestConf struct {
 }
 
 type Config struct {
-	Tests []TestConf `yaml:"test-runner-matrix"`
+	Tests []CITestConf `yaml:"runner-test-matrix"`
 }

From 9f0b76debd466edc39a7eb66f749a447ae9e6cd3 Mon Sep 17 00:00:00 2001
From: lukaszcl <120112546+lukaszcl@users.noreply.github.com>
Date: Wed, 26 Jun 2024 12:50:44 +0200
Subject: [PATCH 067/137] Add script to prepare test config override

---
 .github/e2e-tests.yml                         |  36 +--
 .github/scripts/e2e_tests_tool/main.go        |   7 -
 .../run-automation-benchmark-e2e-tests.yml    |  56 ++++
 .../run-e2e-tests-reusable-workflow.yml       |  48 ++-
 go.mod                                        |  89 +++++-
 go.sum                                        | 163 ++++++++++
 .../e2e_tests_ci_tool}/cmd/check_tests_cmd.go |   0
 .../cmd/check_tests_cmd_test.go               |   0
 .../cmd/create_test_config_override.go        | 300 ++++++++++++++++++
 .../e2e_tests_ci_tool}/cmd/envreplace_cmd.go  |   0
 .../cmd/envreplace_cmd_test.go                |   0
 .../e2e_tests_ci_tool}/cmd/filter_cmd.go      |   0
 .../e2e_tests_ci_tool}/cmd/filter_cmd_test.go |   0
 .../e2e_tests_ci_tool}/cmd/root_cmd.go        |   2 +
 .../e2e_tests_ci_tool}/cmd/types.go           |   0
 integration-tests/e2e_tests_ci_tool/main.go   |   9 +
 integration-tests/go.mod                      |   6 +-
 integration-tests/go.sum                      |   4 +-
 integration-tests/load/go.mod                 |   2 +-
 integration-tests/load/go.sum                 |   4 +-
 20 files changed, 677 insertions(+), 49 deletions(-)
 delete mode 100644 .github/scripts/e2e_tests_tool/main.go
 create mode 100644 .github/workflows/run-automation-benchmark-e2e-tests.yml
 rename {.github/scripts/e2e_tests_tool => integration-tests/e2e_tests_ci_tool}/cmd/check_tests_cmd.go (100%)
 rename {.github/scripts/e2e_tests_tool => integration-tests/e2e_tests_ci_tool}/cmd/check_tests_cmd_test.go (100%)
 create mode 100644 integration-tests/e2e_tests_ci_tool/cmd/create_test_config_override.go
 rename {.github/scripts/e2e_tests_tool => integration-tests/e2e_tests_ci_tool}/cmd/envreplace_cmd.go (100%)
 rename {.github/scripts/e2e_tests_tool => integration-tests/e2e_tests_ci_tool}/cmd/envreplace_cmd_test.go (100%)
 rename {.github/scripts/e2e_tests_tool => integration-tests/e2e_tests_ci_tool}/cmd/filter_cmd.go (100%)
 rename {.github/scripts/e2e_tests_tool => integration-tests/e2e_tests_ci_tool}/cmd/filter_cmd_test.go (100%)
 rename {.github/scripts/e2e_tests_tool => integration-tests/e2e_tests_ci_tool}/cmd/root_cmd.go (87%)
 rename {.github/scripts/e2e_tests_tool => integration-tests/e2e_tests_ci_tool}/cmd/types.go (100%)
 create mode 100644 integration-tests/e2e_tests_ci_tool/main.go

diff --git a/.github/e2e-tests.yml b/.github/e2e-tests.yml
index 43031fe663f..960dad5485a 100644
--- a/.github/e2e-tests.yml
+++ b/.github/e2e-tests.yml
@@ -48,13 +48,11 @@ runner-test-matrix:
     workflows:
       - Run Automation Product Nightly E2E Tests
     test-cmd: cd integration-tests/smoke && go test -test.run ^TestAutomationNodeUpgrade/registry_2_0 -test.parallel=1 -timeout 60m -count=1 -json
-    test-config-override: |
-      [ChainlinkUpgradeImage]
-      image="${{ env.CHAINLINK_IMAGE }}"
-      version="develop"
-      [ChainlinkImage]
-      image="public.ecr.aws/chainlink/chainlink"
-      version="latest"
+    test-config-overrides:
+      chainlink-upgrade-image: ${{ env.CHAINLINK_IMAGE }}
+      chainlink-upgrade-version: develop
+      chainlink-image: public.ecr.aws/chainlink/chainlink
+      chainlink-version: latest
     pyroscope-env: ci-smoke-automation-upgrade-tests
 
   - id: smoke/automation_upgrade_test.go:^TestAutomationNodeUpgrade/registry_2_1
@@ -64,13 +62,11 @@ runner-test-matrix:
     workflows:
       - Run Automation Product Nightly E2E Tests    
     test-cmd: cd integration-tests/smoke && go test -test.run ^TestAutomationNodeUpgrade/registry_2_1 -test.parallel=5 -timeout 60m -count=1 -json
-    test-config-override: |
-      [ChainlinkUpgradeImage]
-      image="${{ env.CHAINLINK_IMAGE }}"
-      version="develop"
-      [ChainlinkImage]
-      image="public.ecr.aws/chainlink/chainlink"
-      version="latest"
+    test-config-overrides:
+      chainlink-upgrade-image: ${{ env.CHAINLINK_IMAGE }}
+      chainlink-upgrade-version: develop
+      chainlink-image: public.ecr.aws/chainlink/chainlink
+      chainlink-version: latest
     pyroscope-env: ci-smoke-automation-upgrade-tests
 
   - id: smoke/automation_upgrade_test.go:^TestAutomationNodeUpgrade/registry_2_2
@@ -80,13 +76,11 @@ runner-test-matrix:
     workflows:
       - Run Automation Product Nightly E2E Tests
     test-cmd: cd integration-tests/smoke && go test -test.run ^TestAutomationNodeUpgrade/registry_2_2 -test.parallel=5 -timeout 60m -count=1 -json
-    test-config-override: |
-      [ChainlinkUpgradeImage]
-      image="${{ env.CHAINLINK_IMAGE }}"
-      version="develop"
-      [ChainlinkImage]
-      image="public.ecr.aws/chainlink/chainlink"
-      version="latest"
+    test-config-overrides:
+      chainlink-upgrade-image: ${{ env.CHAINLINK_IMAGE }}
+      chainlink-upgrade-version: develop
+      chainlink-image: public.ecr.aws/chainlink/chainlink
+      chainlink-version: latest
     pyroscope-env: ci-smoke-automation-upgrade-tests
 
   # END Automation upgrade tests
diff --git a/.github/scripts/e2e_tests_tool/main.go b/.github/scripts/e2e_tests_tool/main.go
deleted file mode 100644
index c3c185a4fbd..00000000000
--- a/.github/scripts/e2e_tests_tool/main.go
+++ /dev/null
@@ -1,7 +0,0 @@
-package main
-
-import "github.com/smartcontractkit/chainlink/v2/.github/scripts/e2e_tests_tool/cmd"
-
-func main() {
-	cmd.Execute()
-}
diff --git a/.github/workflows/run-automation-benchmark-e2e-tests.yml b/.github/workflows/run-automation-benchmark-e2e-tests.yml
new file mode 100644
index 00000000000..cf65649ab9d
--- /dev/null
+++ b/.github/workflows/run-automation-benchmark-e2e-tests.yml
@@ -0,0 +1,56 @@
+name: Run Automation Benchmark E2E Tests
+
+on:
+  workflow_dispatch:
+    inputs:
+      test-id:
+        description: Select a test to run
+        required: true
+        default: TestAutomationBenchmark
+        type: choice
+        options:
+          - TestAutomationBenchmark
+          - TestAutomationBenchmark_1000Upkeeps_2_1
+      test-config-base64-override:
+        description: Custom base64 test config
+        required: false
+        type: string
+
+# TODO:
+# 1. list of tests to select 
+# 2. prepare toml config with grafana url for automation benchmark, take it from github secrets
+# 3. Still support customBase64Config if provided 
+# 4. Add slack notification, have slackMemberID para as input 
+
+jobs:
+  call-run-e2e-tests-workflow:
+    name: Run E2E Tests
+    uses: ./.github/workflows/run-e2e-tests-reusable-workflow.yml
+    permissions:
+      actions: read
+      checks: write
+      pull-requests: write
+      id-token: write
+      contents: read    
+    with:
+      chainlink-version: develop
+      test-ids: ${{ github.event.inputs.test-id }}
+      # Select tests to run from .github/e2e_tests.yml based on workflow name 
+      test-workflow: Run Automation Product Nightly E2E Tests
+      test-config-base64-override: ${{ github.event.inputs.test-config-base64-override }}
+    secrets:
+      QA_AWS_REGION: ${{ secrets.QA_AWS_REGION }}
+      QA_AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }}
+      QA_AWS_ACCOUNT_NUMBER: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}
+      QA_PYROSCOPE_INSTANCE: ${{ secrets.QA_PYROSCOPE_INSTANCE }}
+      QA_PYROSCOPE_KEY: ${{ secrets.QA_PYROSCOPE_KEY }}
+      QA_KUBECONFIG: ${{ secrets.QA_KUBECONFIG }}
+      GRAFANA_INTERNAL_TENANT_ID: ${{ secrets.GRAFANA_INTERNAL_TENANT_ID }}
+      GRAFANA_INTERNAL_BASIC_AUTH: ${{ secrets.GRAFANA_INTERNAL_BASIC_AUTH }}
+      GRAFANA_INTERNAL_HOST: ${{ secrets.GRAFANA_INTERNAL_HOST }}
+      GRAFANA_INTERNAL_URL_SHORTENER_TOKEN: ${{ secrets.GRAFANA_INTERNAL_URL_SHORTENER_TOKEN }}
+      GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}  
+      AWS_REGION: ${{ secrets.QA_AWS_REGION }}
+      AWS_OIDC_IAM_ROLE_VALIDATION_PROD_ARN: ${{ secrets.AWS_OIDC_IAM_ROLE_VALIDATION_PROD_ARN }}
+      AWS_API_GW_HOST_GRAFANA: ${{ secrets.AWS_API_GW_HOST_GRAFANA }}        
+
diff --git a/.github/workflows/run-e2e-tests-reusable-workflow.yml b/.github/workflows/run-e2e-tests-reusable-workflow.yml
index 9e808279f07..ad1890276b8 100644
--- a/.github/workflows/run-e2e-tests-reusable-workflow.yml
+++ b/.github/workflows/run-e2e-tests-reusable-workflow.yml
@@ -16,6 +16,10 @@ on:
         description: 'Run tests by workflow name. Example: "Run Nightly E2E Tests"'
         required: false
         type: string        
+      test-config-base64-override:
+        description: 'Custom base64 test config override'
+        required: false
+        type: string        
       enable-check-test-configurations:
         description: 'Set to "true" to enable check-test-configurations job'
         required: false
@@ -204,22 +208,48 @@ jobs:
         uses: actions/setup-go@v2
         with:
           go-version: '1.21.7'
-      - name: Set Test Config Override
-        if: ${{ matrix.tests.testConfigOverride != '' }}
-        run: |
-          echo "::add-mask::${{ env.CHAINLINK_IMAGE }}"
 
-          # Use the Go tool to replace environment variables
-          replaced_content=$(go run .github/scripts/e2e_tests_tool/main.go envreplace --input '${{ matrix.tests.testConfigOverride }}')
-          
-          # Encode the replaced content to Base64
-          BASE64_CONFIG_OVERRIDE=$(echo "$replaced_content" | base64 -w 0)
+      - name: Set default test config override
+        if: ${{ inputs.test-config-base64-override == '' }}
+        run: |      
+          chainlink_version="${{ github.sha || matrix.tests.testConfigOverride.chainlinkVersion }}"
+          chainlink_postgres_version="${{ matrix.tests.testConfigOverride.chainlinkPostgresVersion }}"
+          chainlink_upgrade_version="${{ matrix.tests.testConfigOverride.chainlinkUpgradeVersion }}"
+          selected_networks="${{ env.SELECTED_NETWORKS || matrix.tests.testConfigOverride.selectedNetworks }}"
+
+          cmd_args=""
+          if [ -n "$chainlink_version" ]; then
+            cmd_args+="--chainlink-version=\"$chainlink_version\" "
+          fi
+          if [ -n "$chainlink_postgres_version" ]; then
+            cmd_args+="--chainlink-postgres-version=\"$chainlink_postgres_version\" "
+          fi
+          if [ -n "$chainlink_upgrade_version" ]; then
+            cmd_args+="--chainlink-upgrade-version=\"$chainlink_upgrade_version\" "
+          fi
+          if [ -n "$selected_networks" ]; then
+            cmd_args+="--selected-networks=\"$selected_networks\" "
+          fi    
+
+          config_override=$(go run .github/scripts/e2e_tests_tool/main.go mask-test-config-override $cmd_args)  
+          echo $config_override
+
+          BASE64_CONFIG_OVERRIDE=$(echo "$config_override" | base64 -w 0)
           echo ::add-mask::$BASE64_CONFIG_OVERRIDE
           echo "BASE64_CONFIG_OVERRIDE=$BASE64_CONFIG_OVERRIDE" >> $GITHUB_ENV
+
+      - name: Set custom test config override
+        if: ${{ inputs.test-config-base64-override != '' }}          
+        run: |
+          BASE64_CONFIG_OVERRIDE="${{ inputs.test-config-base64-override }}"
+          echo "::add-mask::$BASE64_CONFIG_OVERRIDE"
+          echo "BASE64_CONFIG_OVERRIDE=$BASE64_CONFIG_OVERRIDE" >> $GITHUB_ENV
+
       - name: Install jq
         run: sudo apt-get install -y jq
       - name: Show Test Configuration
         run: echo '${{ toJson(matrix.tests) }}' | jq .
+
       - name: Run Tests
         uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/run-tests@af92c5fae8dcf1659201e907db82d221fc304b94 # v2.3.21
         with:
diff --git a/go.mod b/go.mod
index 7ca51538e84..e7d53d98be0 100644
--- a/go.mod
+++ b/go.mod
@@ -54,11 +54,12 @@ require (
 	github.com/mitchellh/go-homedir v1.1.0
 	github.com/mitchellh/mapstructure v1.5.0
 	github.com/mr-tron/base58 v1.2.0
+	github.com/naoina/toml v0.1.2-0.20170918210437-9fafd6967416
 	github.com/olekukonko/tablewriter v0.0.5
 	github.com/onsi/gomega v1.30.0
 	github.com/patrickmn/go-cache v2.1.0+incompatible
 	github.com/pelletier/go-toml v1.9.5
-	github.com/pelletier/go-toml/v2 v2.1.1
+	github.com/pelletier/go-toml/v2 v2.2.2
 	github.com/pkg/errors v0.9.1
 	github.com/pressly/goose/v3 v3.16.0
 	github.com/prometheus/client_golang v1.17.0
@@ -78,12 +79,14 @@ require (
 	github.com/smartcontractkit/chainlink-feeds v0.0.0-20240522213638-159fb2d99917
 	github.com/smartcontractkit/chainlink-solana v1.0.3-0.20240605170242-555ff582f36a
 	github.com/smartcontractkit/chainlink-starknet/relayer v0.0.1-beta-test.0.20240531021326-99118e47f696
+	github.com/smartcontractkit/chainlink-testing-framework v1.31.4
 	github.com/smartcontractkit/chainlink-vrf v0.0.0-20231120191722-fef03814f868
 	github.com/smartcontractkit/libocr v0.0.0-20240419185742-fd3cab206b2c
 	github.com/smartcontractkit/tdh2/go/ocr2/decryptionplugin v0.0.0-20230906073235-9e478e5e19f1
 	github.com/smartcontractkit/tdh2/go/tdh2 v0.0.0-20230906073235-9e478e5e19f1
 	github.com/smartcontractkit/wsrpc v0.8.1
 	github.com/spf13/cast v1.6.0
+	github.com/spf13/cobra v1.8.1
 	github.com/stretchr/testify v1.9.0
 	github.com/test-go/testify v1.1.4
 	github.com/theodesp/go-heaps v0.0.0-20190520121037-88e35354fe0a
@@ -113,6 +116,8 @@ require (
 	google.golang.org/protobuf v1.33.0
 	gopkg.in/guregu/null.v4 v4.0.0
 	gopkg.in/natefinch/lumberjack.v2 v2.2.1
+	gopkg.in/yaml.v2 v2.4.0
+	gopkg.in/yaml.v3 v3.0.1
 )
 
 require (
@@ -122,18 +127,26 @@ require (
 	cosmossdk.io/depinject v1.0.0-alpha.3 // indirect
 	cosmossdk.io/errors v1.0.0 // indirect
 	cosmossdk.io/math v1.0.1 // indirect
+	dario.cat/mergo v1.0.0 // indirect
 	filippo.io/edwards25519 v1.0.0 // indirect
 	github.com/99designs/go-keychain v0.0.0-20191008050251-8e49817e8af4 // indirect
 	github.com/99designs/keyring v1.2.1 // indirect
+	github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161 // indirect
 	github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358 // indirect
 	github.com/ChainSafe/go-schnorrkel v0.0.0-20200405005733-88cbf1b4c40d // indirect
 	github.com/CosmWasm/wasmd v0.40.1 // indirect
 	github.com/CosmWasm/wasmvm v1.2.4 // indirect
 	github.com/DataDog/zstd v1.5.2 // indirect
+	github.com/MakeNowJust/heredoc v1.0.0 // indirect
 	github.com/Masterminds/goutils v1.1.1 // indirect
 	github.com/Microsoft/go-winio v0.6.1 // indirect
+	github.com/Microsoft/hcsshim v0.11.4 // indirect
 	github.com/VictoriaMetrics/fastcache v1.12.1 // indirect
+	github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137 // indirect
 	github.com/armon/go-metrics v0.4.1 // indirect
+	github.com/avast/retry-go v3.0.0+incompatible // indirect
+	github.com/aws/constructs-go/constructs/v10 v10.1.255 // indirect
+	github.com/aws/jsii-runtime-go v1.75.0 // indirect
 	github.com/bahlo/generic-list-go v0.2.0 // indirect
 	github.com/benbjohnson/clock v1.3.5 // indirect
 	github.com/beorn7/perks v1.0.1 // indirect
@@ -142,10 +155,13 @@ require (
 	github.com/blendle/zapdriver v1.3.1 // indirect
 	github.com/buger/jsonparser v1.1.1 // indirect
 	github.com/bytedance/sonic v1.10.1 // indirect
+	github.com/cdk8s-team/cdk8s-core-go/cdk8s/v2 v2.7.5 // indirect
 	github.com/cenkalti/backoff v2.2.1+incompatible // indirect
 	github.com/cenkalti/backoff/v4 v4.2.1 // indirect
 	github.com/cespare/xxhash v1.1.0 // indirect
 	github.com/cespare/xxhash/v2 v2.2.0 // indirect
+	github.com/chai2010/gettext-go v1.0.2 // indirect
+	github.com/chaos-mesh/chaos-mesh/api/v1alpha1 v0.0.0-20220226050744-799408773657 // indirect
 	github.com/chenzhuoyu/base64x v0.0.0-20230717121745-296ad89f973d // indirect
 	github.com/chenzhuoyu/iasm v0.9.0 // indirect
 	github.com/cockroachdb/errors v1.9.1 // indirect
@@ -157,6 +173,8 @@ require (
 	github.com/confio/ics23/go v0.9.0 // indirect
 	github.com/consensys/bavard v0.1.13 // indirect
 	github.com/consensys/gnark-crypto v0.12.1 // indirect
+	github.com/containerd/containerd v1.7.12 // indirect
+	github.com/containerd/log v0.1.0 // indirect
 	github.com/cosmos/btcutil v1.0.5 // indirect
 	github.com/cosmos/cosmos-proto v1.0.0-beta.2 // indirect
 	github.com/cosmos/go-bip39 v1.0.0 // indirect
@@ -165,6 +183,7 @@ require (
 	github.com/cosmos/ibc-go/v7 v7.0.1 // indirect
 	github.com/cosmos/ics23/go v0.9.1-0.20221207100636-b1abd8678aab // indirect
 	github.com/cosmos/ledger-cosmos-go v0.12.1 // indirect
+	github.com/cpuguy83/dockercfg v0.3.1 // indirect
 	github.com/cpuguy83/go-md2man/v2 v2.0.4 // indirect
 	github.com/crate-crypto/go-ipa v0.0.0-20231025140028-3c0104f4b233 // indirect
 	github.com/crate-crypto/go-kzg-4844 v0.7.0 // indirect
@@ -176,11 +195,22 @@ require (
 	github.com/dgraph-io/badger/v2 v2.2007.4 // indirect
 	github.com/dgraph-io/ristretto v0.1.1 // indirect
 	github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13 // indirect
+	github.com/distribution/reference v0.5.0 // indirect
 	github.com/docker/distribution v2.8.2+incompatible // indirect
+	github.com/docker/docker v25.0.2+incompatible // indirect
+	github.com/docker/go-connections v0.5.0 // indirect
+	github.com/docker/go-units v0.5.0 // indirect
 	github.com/dustin/go-humanize v1.0.1 // indirect
 	github.com/dvsekhvalnov/jose2go v1.7.0 // indirect
+	github.com/emicklei/go-restful/v3 v3.10.2 // indirect
 	github.com/ethereum/c-kzg-4844 v0.4.0 // indirect
+	github.com/evanphx/json-patch v5.6.0+incompatible // indirect
+	github.com/evanphx/json-patch/v5 v5.6.0 // indirect
+	github.com/exponent-io/jsonpath v0.0.0-20210407135951-1de76d718b3f // indirect
+	github.com/fatih/camelcase v1.0.0 // indirect
+	github.com/felixge/httpsnoop v1.0.3 // indirect
 	github.com/fsnotify/fsnotify v1.6.0 // indirect
+	github.com/fvbommel/sortorder v1.0.2 // indirect
 	github.com/gabriel-vasile/mimetype v1.4.2 // indirect
 	github.com/gagliardetto/binary v0.7.7 // indirect
 	github.com/gagliardetto/treeout v0.1.4 // indirect
@@ -189,6 +219,7 @@ require (
 	github.com/gedex/inflector v0.0.0-20170307190818-16278e9db813 // indirect
 	github.com/gin-contrib/sse v0.1.0 // indirect
 	github.com/go-asn1-ber/asn1-ber v1.5.5 // indirect
+	github.com/go-errors/errors v1.4.2 // indirect
 	github.com/go-json-experiment/json v0.0.0-20231102232822-2e55bd4e08b0 // indirect
 	github.com/go-kit/kit v0.12.0 // indirect
 	github.com/go-kit/log v0.2.1 // indirect
@@ -196,9 +227,13 @@ require (
 	github.com/go-logr/logr v1.4.1 // indirect
 	github.com/go-logr/stdr v1.2.2 // indirect
 	github.com/go-ole/go-ole v1.2.6 // indirect
+	github.com/go-openapi/jsonpointer v0.20.0 // indirect
+	github.com/go-openapi/jsonreference v0.20.2 // indirect
+	github.com/go-openapi/swag v0.22.4 // indirect
 	github.com/go-playground/locales v0.14.1 // indirect
 	github.com/go-playground/universal-translator v0.18.1 // indirect
 	github.com/go-playground/validator/v10 v10.15.5 // indirect
+	github.com/go-resty/resty/v2 v2.7.0 // indirect
 	github.com/go-webauthn/x v0.1.5 // indirect
 	github.com/goccy/go-json v0.10.2 // indirect
 	github.com/godbus/dbus v0.0.0-20190726142602-4481cbc300e2 // indirect
@@ -210,11 +245,15 @@ require (
 	github.com/golang/protobuf v1.5.4 // indirect
 	github.com/golang/snappy v0.0.5-0.20220116011046-fa5810519dcb // indirect
 	github.com/google/btree v1.1.2 // indirect
+	github.com/google/gnostic v0.6.9 // indirect
+	github.com/google/gnostic-models v0.6.8 // indirect
 	github.com/google/go-cmp v0.6.0 // indirect
 	github.com/google/go-tpm v0.9.0 // indirect
 	github.com/google/gofuzz v1.2.0 // indirect
+	github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 // indirect
 	github.com/gorilla/context v1.1.1 // indirect
 	github.com/grafana/pyroscope-go/godeltaprof v0.1.6 // indirect
+	github.com/gregjones/httpcache v0.0.0-20190611155906-901d90724c79 // indirect
 	github.com/grpc-ecosystem/go-grpc-middleware v1.3.0 // indirect
 	github.com/grpc-ecosystem/go-grpc-middleware/providers/prometheus v1.0.0 // indirect
 	github.com/grpc-ecosystem/go-grpc-middleware/v2 v2.0.0-rc.3 // indirect
@@ -245,14 +284,18 @@ require (
 	github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a // indirect
 	github.com/jackpal/go-nat-pmp v1.0.2 // indirect
 	github.com/jmhodges/levigo v1.0.0 // indirect
+	github.com/josharian/intern v1.0.0 // indirect
 	github.com/json-iterator/go v1.1.12 // indirect
+	github.com/kelseyhightower/envconfig v1.4.0 // indirect
 	github.com/klauspost/compress v1.17.3 // indirect
 	github.com/klauspost/cpuid/v2 v2.2.5 // indirect
 	github.com/kr/pretty v0.3.1 // indirect
 	github.com/kr/text v0.2.0 // indirect
 	github.com/leodido/go-urn v1.2.4 // indirect
 	github.com/libp2p/go-buffer-pool v0.1.0 // indirect
+	github.com/liggitt/tabwriter v0.0.0-20181228230101-89fcab3d43de // indirect
 	github.com/logrusorgru/aurora v2.0.3+incompatible // indirect
+	github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 // indirect
 	github.com/magiconair/properties v1.8.7 // indirect
 	github.com/mailru/easyjson v0.7.7 // indirect
 	github.com/mattn/go-colorable v0.1.13 // indirect
@@ -262,30 +305,46 @@ require (
 	github.com/mimoo/StrobeGo v0.0.0-20210601165009-122bf33a46e0 // indirect
 	github.com/mitchellh/copystructure v1.2.0 // indirect
 	github.com/mitchellh/go-testing-interface v1.14.1 // indirect
+	github.com/mitchellh/go-wordwrap v1.0.1 // indirect
 	github.com/mitchellh/reflectwalk v1.0.2 // indirect
 	github.com/mmcloughlin/addchain v0.4.0 // indirect
+	github.com/moby/patternmatcher v0.6.0 // indirect
+	github.com/moby/spdystream v0.2.0 // indirect
+	github.com/moby/sys/sequential v0.5.0 // indirect
+	github.com/moby/sys/user v0.1.0 // indirect
+	github.com/moby/term v0.5.0 // indirect
 	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
 	github.com/modern-go/reflect2 v1.0.2 // indirect
+	github.com/monochromegane/go-gitignore v0.0.0-20200626010858-205db1a8cc00 // indirect
+	github.com/montanaflynn/stats v0.7.1 // indirect
+	github.com/morikuni/aec v1.0.0 // indirect
 	github.com/mostynb/zstdpool-freelist v0.0.0-20201229113212-927304c0c3b1 // indirect
 	github.com/mtibben/percent v0.2.1 // indirect
+	github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
 	github.com/mwitkow/grpc-proxy v0.0.0-20230212185441-f345521cb9c9 // indirect
+	github.com/naoina/go-stringutil v0.1.0 // indirect
 	github.com/oklog/run v1.1.0 // indirect
 	github.com/opencontainers/go-digest v1.0.0 // indirect
+	github.com/opencontainers/image-spec v1.1.0-rc5 // indirect
 	github.com/opentracing/opentracing-go v1.2.0 // indirect
+	github.com/peterbourgon/diskv v2.0.1+incompatible // indirect
 	github.com/petermattis/goid v0.0.0-20230317030725-371a4b8eda08 // indirect
 	github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
 	github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c // indirect
 	github.com/prometheus/procfs v0.12.0 // indirect
 	github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 // indirect
 	github.com/rivo/uniseg v0.4.4 // indirect
+	github.com/rs/zerolog v1.30.0 // indirect
+	github.com/russross/blackfriday v1.6.0 // indirect
 	github.com/russross/blackfriday/v2 v2.1.0 // indirect
 	github.com/santhosh-tekuri/jsonschema/v5 v5.3.1 // indirect
 	github.com/sasha-s/go-deadlock v0.3.1 // indirect
 	github.com/sethvargo/go-retry v0.2.4 // indirect
 	github.com/shirou/gopsutil v3.21.11+incompatible // indirect
+	github.com/shoenig/go-m1cpu v0.1.6 // indirect
 	github.com/sirupsen/logrus v1.9.3 // indirect
+	github.com/smartcontractkit/seth v1.0.11 // indirect
 	github.com/spf13/afero v1.9.5 // indirect
-	github.com/spf13/cobra v1.8.1 // indirect
 	github.com/spf13/jwalterweatherman v1.1.0 // indirect
 	github.com/spf13/pflag v1.0.5 // indirect
 	github.com/spf13/viper v1.15.0 // indirect
@@ -298,6 +357,7 @@ require (
 	github.com/tecbot/gorocksdb v0.0.0-20191217155057-f0fad39f321c // indirect
 	github.com/tendermint/go-amino v0.16.0 // indirect
 	github.com/teris-io/shortid v0.0.0-20201117134242-e59966efd125 // indirect
+	github.com/testcontainers/testcontainers-go v0.28.0 // indirect
 	github.com/tidwall/btree v1.6.0 // indirect
 	github.com/tidwall/match v1.1.1 // indirect
 	github.com/tidwall/pretty v1.2.0 // indirect
@@ -309,6 +369,7 @@ require (
 	github.com/valyala/fastjson v1.4.1 // indirect
 	github.com/wk8/go-ordered-map/v2 v2.1.8 // indirect
 	github.com/x448/float16 v0.8.4 // indirect
+	github.com/xlab/treeprint v1.1.0 // indirect
 	github.com/yusufpapurcu/wmi v1.2.4 // indirect
 	github.com/zondax/hid v0.9.1 // indirect
 	github.com/zondax/ledger-go v0.14.1 // indirect
@@ -317,25 +378,45 @@ require (
 	go.mongodb.org/mongo-driver v1.15.0 // indirect
 	go.opencensus.io v0.24.0 // indirect
 	go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.46.1 // indirect
+	go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.45.0 // indirect
 	go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.21.0 // indirect
 	go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.21.0 // indirect
 	go.opentelemetry.io/otel/metric v1.24.0 // indirect
 	go.opentelemetry.io/otel/sdk v1.21.0 // indirect
 	go.opentelemetry.io/otel/trace v1.24.0 // indirect
 	go.opentelemetry.io/proto/otlp v1.0.0 // indirect
+	go.starlark.net v0.0.0-20220817180228-f738f5508c12 // indirect
+	go.uber.org/atomic v1.11.0 // indirect
 	go.uber.org/ratelimit v0.3.0 // indirect
 	golang.org/x/arch v0.7.0 // indirect
+	golang.org/x/oauth2 v0.17.0 // indirect
 	golang.org/x/sys v0.19.0 // indirect
+	gomodules.xyz/jsonpatch/v2 v2.2.0 // indirect
 	google.golang.org/api v0.149.0 // indirect
+	google.golang.org/appengine v1.6.8 // indirect
 	google.golang.org/genproto v0.0.0-20231030173426-d783a09b4405 // indirect
 	google.golang.org/genproto/googleapis/api v0.0.0-20231016165738-49dd2c1f3d0b // indirect
 	google.golang.org/genproto/googleapis/rpc v0.0.0-20231106174013-bbf56f31fb17 // indirect
 	gopkg.in/guregu/null.v2 v2.1.2 // indirect
+	gopkg.in/inf.v0 v0.9.1 // indirect
 	gopkg.in/ini.v1 v1.67.0 // indirect
-	gopkg.in/yaml.v2 v2.4.0 // indirect
-	gopkg.in/yaml.v3 v3.0.1 // indirect
+	k8s.io/api v0.28.2 // indirect
+	k8s.io/apiextensions-apiserver v0.25.3 // indirect
+	k8s.io/apimachinery v0.28.2 // indirect
+	k8s.io/cli-runtime v0.25.11 // indirect
+	k8s.io/client-go v0.28.2 // indirect
+	k8s.io/component-base v0.26.2 // indirect
+	k8s.io/klog/v2 v2.100.1 // indirect
+	k8s.io/kube-openapi v0.0.0-20230717233707-2695361300d9 // indirect
+	k8s.io/kubectl v0.25.11 // indirect
+	k8s.io/utils v0.0.0-20230711102312-30195339c3c7 // indirect
 	pgregory.net/rapid v0.5.5 // indirect
 	rsc.io/tmplfunc v0.0.3 // indirect
+	sigs.k8s.io/controller-runtime v0.13.0 // indirect
+	sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect
+	sigs.k8s.io/kustomize/api v0.12.1 // indirect
+	sigs.k8s.io/kustomize/kyaml v0.13.9 // indirect
+	sigs.k8s.io/structured-merge-diff/v4 v4.3.0 // indirect
 	sigs.k8s.io/yaml v1.4.0 // indirect
 )
 
diff --git a/go.sum b/go.sum
index 091a218472b..a086b130d83 100644
--- a/go.sum
+++ b/go.sum
@@ -65,6 +65,8 @@ cosmossdk.io/math v1.0.1 h1:Qx3ifyOPaMLNH/89WeZFH268yCvU4xEcnPLu3sJqPPg=
 cosmossdk.io/math v1.0.1/go.mod h1:Ygz4wBHrgc7g0N+8+MrnTfS9LLn9aaTGa9hKopuym5k=
 cosmossdk.io/tools/rosetta v0.2.1 h1:ddOMatOH+pbxWbrGJKRAawdBkPYLfKXutK9IETnjYxw=
 cosmossdk.io/tools/rosetta v0.2.1/go.mod h1:Pqdc1FdvkNV3LcNIkYWt2RQY6IP1ge6YWZk8MhhO9Hw=
+dario.cat/mergo v1.0.0 h1:AGCNq9Evsj31mOgNPcLyXc+4PNABt905YmuqPYYpBWk=
+dario.cat/mergo v1.0.0/go.mod h1:uNxQE+84aUszobStD9th8a29P2fMDhsBdgRYvZOxGmk=
 dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU=
 filippo.io/edwards25519 v1.0.0-rc.1/go.mod h1:N1IkdkCkiLB6tki+MYJoSx2JTY9NUlxZE7eHn5EwJns=
 filippo.io/edwards25519 v1.0.0 h1:0wAIcmJUqRdI8IJ/3eGi5/HwXZWPujYXXlkrQogz0Ek=
@@ -103,6 +105,8 @@ github.com/Depado/ginprom v1.8.0/go.mod h1:XBaKzeNBqPF4vxJpNLincSQZeMDnZp1tIbU0F
 github.com/GeertJohan/go.incremental v1.0.0/go.mod h1:6fAjUhbVuX1KcMD3c8TEgVUqmo4seqhv0i0kdATSkM0=
 github.com/GeertJohan/go.rice v1.0.0/go.mod h1:eH6gbSOAUv07dQuZVnBmoDP8mgsM1rtixis4Tib9if0=
 github.com/Joker/hpp v1.0.0/go.mod h1:8x5n+M1Hp5hC0g8okX3sR3vFQwynaX/UgSOM9MeBKzY=
+github.com/MakeNowJust/heredoc v1.0.0 h1:cXCdzVdstXyiTqTvfqk9SDHpKNjxuom+DOlyEeQ4pzQ=
+github.com/MakeNowJust/heredoc v1.0.0/go.mod h1:mG5amYoWBHf8vpLOuehzbGGw0EHxpZZ6lCpQ4fNJ8LE=
 github.com/Masterminds/goutils v1.1.1 h1:5nUrii3FMTL5diU80unEVvNevw1nH4+ZV4DSLVJLSYI=
 github.com/Masterminds/goutils v1.1.1/go.mod h1:8cTjp+g8YejhMuvIA5y2vz3BpJxksy863GQaJW2MFNU=
 github.com/Masterminds/semver/v3 v3.1.1/go.mod h1:VPu/7SZ7ePZ3QOrcuXROw5FAcLl4a0cBrbBpGY/8hQs=
@@ -113,6 +117,8 @@ github.com/Masterminds/sprig/v3 v3.2.3 h1:eL2fZNezLomi0uOLqjQoN6BfsDD+fyLtgbJMAj
 github.com/Masterminds/sprig/v3 v3.2.3/go.mod h1:rXcFaZ2zZbLRJv/xSysmlgIM1u11eBaRMhvYXJNkGuM=
 github.com/Microsoft/go-winio v0.6.1 h1:9/kr64B9VUZrLm5YYwbGtUJnMgqWVOdUAXu6Migciow=
 github.com/Microsoft/go-winio v0.6.1/go.mod h1:LRdKpFKfdobln8UmuiYcKPot9D2v6svN5+sAH+4kjUM=
+github.com/Microsoft/hcsshim v0.11.4 h1:68vKo2VN8DE9AdN4tnkWnmdhqdbpUFM8OF3Airm7fz8=
+github.com/Microsoft/hcsshim v0.11.4/go.mod h1:smjE4dvqPX9Zldna+t5FG3rnoHhaB7QYxPRqGcpAD9w=
 github.com/NethermindEth/juno v0.3.1 h1:AW72LiAm9gqUeCVJWvepnZcTnpU4Vkl0KzPMxS+42FA=
 github.com/NethermindEth/juno v0.3.1/go.mod h1:SGbTpgGaCsxhFsKOid7Ylnz//WZ8swtILk+NbHGsk/Q=
 github.com/NethermindEth/starknet.go v0.7.1-0.20240401080518-34a506f3cfdb h1:Mv8SscePPyw2ju4igIJAjFgcq5zCQfjgbz53DwYu5mc=
@@ -137,6 +143,8 @@ github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuy
 github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
 github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
 github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
+github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137 h1:s6gZFSlWYmbqAuRjVTiNNhvNRfY2Wxp9nhfyel4rklc=
+github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137/go.mod h1:OMCwj8VM1Kc9e19TLln2VL61YJF0x1XFtfdL4JdbSyE=
 github.com/alexbrainman/sspi v0.0.0-20210105120005-909beea2cc74 h1:Kk6a4nehpJ3UuJRqlA3JxYxBZEqCeOmATOvrbT4p9RA=
 github.com/alexbrainman/sspi v0.0.0-20210105120005-909beea2cc74/go.mod h1:cEWa1LVoE5KvSD9ONXsZrj0z6KqySlCCNKHlLzbqAt4=
 github.com/allegro/bigcache v1.2.1-0.20190218064605-e24eb225f156/go.mod h1:Cb/ax3seSYIx7SuZdm2G2xzfwmv3TPSk2ucNfQESPXM=
@@ -154,12 +162,18 @@ github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmV
 github.com/armon/go-metrics v0.4.1 h1:hR91U9KYmb6bLBYLQjyM+3j+rcd/UhE+G78SFnF8gJA=
 github.com/armon/go-metrics v0.4.1/go.mod h1:E6amYzXo6aW1tqzoZGT755KkbgrJsSdpwZ+3JqfkOG4=
 github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8=
+github.com/avast/retry-go v3.0.0+incompatible h1:4SOWQ7Qs+oroOTQOYnAHqelpCO0biHSxpiH9JdtuBj0=
+github.com/avast/retry-go v3.0.0+incompatible/go.mod h1:XtSnn+n/sHqQIpZ10K1qAevBhOOCWBLXXy3hyiqqBrY=
 github.com/avast/retry-go/v4 v4.5.1 h1:AxIx0HGi4VZ3I02jr78j5lZ3M6x1E0Ivxa6b0pUUh7o=
 github.com/avast/retry-go/v4 v4.5.1/go.mod h1:/sipNsvNB3RRuT5iNcb6h73nw3IBmXJ/H3XrCQYSOpc=
 github.com/aws/aws-sdk-go v1.22.1/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo=
 github.com/aws/aws-sdk-go v1.23.20/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo=
 github.com/aws/aws-sdk-go v1.45.25 h1:c4fLlh5sLdK2DCRTY1z0hyuJZU4ygxX8m1FswL6/nF4=
 github.com/aws/aws-sdk-go v1.45.25/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI=
+github.com/aws/constructs-go/constructs/v10 v10.1.255 h1:5hARfEmhBqHSTQf/C3QLA3sWOxO2Dfja0iA1W7ZcI7g=
+github.com/aws/constructs-go/constructs/v10 v10.1.255/go.mod h1:DCdBSjN04Ck2pajCacTD4RKFqSA7Utya8d62XreYctI=
+github.com/aws/jsii-runtime-go v1.75.0 h1:NhpUfyiL7/wsRuUekFsz8FFBCYLfPD/l61kKg9kL/a4=
+github.com/aws/jsii-runtime-go v1.75.0/go.mod h1:TKCyrtM0pygEPo4rDZzbMSDNCDNTSYSN6/mGyHI6O3I=
 github.com/aymerick/raymond v2.0.3-0.20180322193309-b565731e1464+incompatible/go.mod h1:osfaiScAUVup+UC9Nfq76eWqDhXlp+4UYaA8uhTBO6g=
 github.com/bahlo/generic-list-go v0.2.0 h1:5sz/EEAK+ls5wF+NeqDpk5+iNdMDXrh3z3nPnH1Wvgk=
 github.com/bahlo/generic-list-go v0.2.0/go.mod h1:2KvAjgMlE5NNynlg/5iLrrCCZ2+5xWbdbCW3pNTGyYg=
@@ -195,6 +209,8 @@ github.com/bytedance/sonic v1.5.0/go.mod h1:ED5hyg4y6t3/9Ku1R6dU/4KyJ48DZ4jPhfY1
 github.com/bytedance/sonic v1.10.0-rc/go.mod h1:ElCzW+ufi8qKqNW0FY314xriJhyJhuoJ3gFZdAHF7NM=
 github.com/bytedance/sonic v1.10.1 h1:7a1wuFXL1cMy7a3f7/VFcEtriuXQnUBhtoVfOZiaysc=
 github.com/bytedance/sonic v1.10.1/go.mod h1:iZcSUejdk5aukTND/Eu/ivjQuEL0Cu9/rf50Hi0u/g4=
+github.com/cdk8s-team/cdk8s-core-go/cdk8s/v2 v2.7.5 h1:rvc39Ol6z3MvaBzXkxFC6Nfsnixq/dRypushKDd7Nc0=
+github.com/cdk8s-team/cdk8s-core-go/cdk8s/v2 v2.7.5/go.mod h1:R/pdNYDYFQk+tuuOo7QES1kkv6OLmp5ze2XBZQIVffM=
 github.com/cenkalti/backoff v2.2.1+incompatible h1:tNowT99t7UNflLxfYYSlKYsBpXdEet03Pg2g16Swow4=
 github.com/cenkalti/backoff v2.2.1+incompatible/go.mod h1:90ReRw6GdpyfrHakVjL/QHaoyV4aDUVVkXQJJJ3NXXM=
 github.com/cenkalti/backoff/v4 v4.2.1 h1:y4OZtCnogmCPw98Zjyt5a6+QwPLGkiQsYW5oUqylYbM=
@@ -207,6 +223,10 @@ github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghf
 github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
 github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44=
 github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
+github.com/chai2010/gettext-go v1.0.2 h1:1Lwwip6Q2QGsAdl/ZKPCwTe9fe0CjlUbqj5bFNSjIRk=
+github.com/chai2010/gettext-go v1.0.2/go.mod h1:y+wnP2cHYaVj19NZhYKAwEMH2CI1gNHeQQ+5AjwawxA=
+github.com/chaos-mesh/chaos-mesh/api/v1alpha1 v0.0.0-20220226050744-799408773657 h1:CyuI+igIjadM/GRnE2o0q+WCwipDh0n2cUYFPAvxziM=
+github.com/chaos-mesh/chaos-mesh/api/v1alpha1 v0.0.0-20220226050744-799408773657/go.mod h1:JRiumF+RFsH1mrrP8FUsi9tExPylKkO/oSRWeQEUdLE=
 github.com/chenzhuoyu/base64x v0.0.0-20211019084208-fb5309c8db06/go.mod h1:DH46F32mSOjUmXrMHnKwZdA8wcEefY7UVqBKYGjpdQY=
 github.com/chenzhuoyu/base64x v0.0.0-20221115062448-fe3a3abad311/go.mod h1:b583jCggY9gE99b6G5LEC39OIiVsWj+R97kbl5odCEk=
 github.com/chenzhuoyu/base64x v0.0.0-20230717121745-296ad89f973d h1:77cEq6EriyTZ0g/qfRdp61a3Uu/AWrgIq2s0ClJV1g0=
@@ -224,6 +244,7 @@ github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDk
 github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc=
 github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk=
 github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk=
+github.com/cncf/xds/go v0.0.0-20210312221358-fbca930ec8ed/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs=
 github.com/cncf/xds/go v0.0.0-20230607035331-e9ce68804cb4 h1:/inchEIKaYC1Akx+H+gqO04wryn5h75LSazbRlnya1k=
 github.com/cncf/xds/go v0.0.0-20230607035331-e9ce68804cb4/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs=
 github.com/cockroachdb/apd v1.1.0 h1:3LFP3629v+1aKXU5Q37mxmRxX/pIu1nijXydLShEq5I=
@@ -259,8 +280,12 @@ github.com/consensys/bavard v0.1.13 h1:oLhMLOFGTLdlda/kma4VOJazblc7IM5y5QPd2A/Yj
 github.com/consensys/bavard v0.1.13/go.mod h1:9ItSMtA/dXMAiL7BG6bqW2m3NdSEObYWoH223nGHukI=
 github.com/consensys/gnark-crypto v0.12.1 h1:lHH39WuuFgVHONRl3J0LRBtuYdQTumFSDtJF7HpyG8M=
 github.com/consensys/gnark-crypto v0.12.1/go.mod h1:v2Gy7L/4ZRosZ7Ivs+9SfUDr0f5UlG+EM5t7MPHiLuY=
+github.com/containerd/containerd v1.7.12 h1:+KQsnv4VnzyxWcfO9mlxxELaoztsDEjOuCMPAuPqgU0=
+github.com/containerd/containerd v1.7.12/go.mod h1:/5OMpE1p0ylxtEUGY8kuCYkDRzJm9NO1TFMWjUpdevk=
 github.com/containerd/continuity v0.4.3 h1:6HVkalIp+2u1ZLH1J/pYX2oBVXlJZvh1X1A7bEZ9Su8=
 github.com/containerd/continuity v0.4.3/go.mod h1:F6PTNCKepoxEaXLQp3wDAjygEnImnZ/7o4JzpodfroQ=
+github.com/containerd/log v0.1.0 h1:TCJt7ioM2cr/tfR8GPbGf9/VRAX8D2B4PjzCpfX540I=
+github.com/containerd/log v0.1.0/go.mod h1:VRRf09a7mHDIRezVKTRCrOq78v577GXq3bSa3EhrzVo=
 github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk=
 github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE=
 github.com/coreos/etcd v3.3.13+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE=
@@ -269,6 +294,7 @@ github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3Ee
 github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk=
 github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4=
 github.com/coreos/go-systemd v0.0.0-20190719114852-fd7a80b32e1f/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4=
+github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc=
 github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA=
 github.com/cosmos/btcutil v1.0.5 h1:t+ZFcX77LpKtDBhjucvnOH8C2l2ioGsBNEQ3jef8xFk=
 github.com/cosmos/btcutil v1.0.5/go.mod h1:IyB7iuqZMJlthe2tkIFL33xPyzbFYP0XVdS8P5lUPis=
@@ -293,6 +319,8 @@ github.com/cosmos/ledger-cosmos-go v0.12.1 h1:sMBxza5p/rNK/06nBSNmsI/WDqI0pVJFVN
 github.com/cosmos/ledger-cosmos-go v0.12.1/go.mod h1:dhO6kj+Y+AHIOgAe4L9HL/6NDdyyth4q238I9yFpD2g=
 github.com/cosmos/rosetta-sdk-go v0.10.0 h1:E5RhTruuoA7KTIXUcMicL76cffyeoyvNybzUGSKFTcM=
 github.com/cosmos/rosetta-sdk-go v0.10.0/go.mod h1:SImAZkb96YbwvoRkzSMQB6noNJXFgWl/ENIznEoYQI4=
+github.com/cpuguy83/dockercfg v0.3.1 h1:/FpZ+JaygUR/lZP2NlFI2DVfrOEMAIKP5wWEJdoYe9E=
+github.com/cpuguy83/dockercfg v0.3.1/go.mod h1:sugsbF4//dDlL/i+S+rtpIWp+5h0BHJHfjj5/jFyUJc=
 github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE=
 github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU=
 github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
@@ -343,16 +371,23 @@ github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUn
 github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13 h1:fAjc9m62+UWV/WAFKLNi6ZS0675eEUC9y3AlwSbQu1Y=
 github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw=
 github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no=
+github.com/distribution/reference v0.5.0 h1:/FUIFXtfc/x2gpa5/VGfiGLuOIdYa1t65IKK2OFGvA0=
+github.com/distribution/reference v0.5.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E=
 github.com/docker/cli v24.0.7+incompatible h1:wa/nIwYFW7BVTGa7SWPVyyXU9lgORqUb1xfI36MSkFg=
 github.com/docker/cli v24.0.7+incompatible/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8=
 github.com/docker/distribution v2.8.2+incompatible h1:T3de5rq0dB1j30rp0sA2rER+m322EBzniBPB6ZIzuh8=
 github.com/docker/distribution v2.8.2+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w=
 github.com/docker/docker v24.0.7+incompatible h1:Wo6l37AuwP3JaMnZa226lzVXGA3F9Ig1seQen0cKYlM=
 github.com/docker/docker v24.0.7+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk=
+github.com/docker/docker v25.0.2+incompatible h1:/OaKeauroa10K4Nqavw4zlhcDq/WBcPMc5DbjOGgozY=
+github.com/docker/docker v25.0.2+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk=
 github.com/docker/go-connections v0.4.0 h1:El9xVISelRB7BuFusrZozjnkIM5YnzCViNKohAFqRJQ=
 github.com/docker/go-connections v0.4.0/go.mod h1:Gbd7IOopHjR8Iph03tsViu4nIes5XhDvyHbTtUxmeec=
+github.com/docker/go-connections v0.5.0 h1:USnMq7hx7gwdVZq1L49hLXaFtUdTADjXGp+uj1Br63c=
+github.com/docker/go-connections v0.5.0/go.mod h1:ov60Kzw0kKElRwhNs9UlUHAE/F9Fe6GLaXnqyDdmEXc=
 github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4=
 github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk=
+github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE=
 github.com/dominikbraun/graph v0.23.0 h1:TdZB4pPqCLFxYhdyMFb1TBdFxp8XLcJfTTBQucVPgCo=
 github.com/dominikbraun/graph v0.23.0/go.mod h1:yOjYyogZLY1LSG9E33JWZJiq5k83Qy2C6POAuiViluc=
 github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=
@@ -365,12 +400,15 @@ github.com/elastic/go-sysinfo v1.11.1 h1:g9mwl05njS4r69TisC+vwHWTSKywZFYYUu3so3T
 github.com/elastic/go-sysinfo v1.11.1/go.mod h1:6KQb31j0QeWBDF88jIdWSxE8cwoOB9tO4Y4osN7Q70E=
 github.com/elastic/go-windows v1.0.1 h1:AlYZOldA+UJ0/2nBuqWdo90GFCgG9xuyw9SYzGUtJm0=
 github.com/elastic/go-windows v1.0.1/go.mod h1:FoVvqWSun28vaDQPbj2Elfc0JahhPB7WQEGa3c814Ss=
+github.com/emicklei/go-restful/v3 v3.10.2 h1:hIovbnmBTLjHXkqEBUz3HGpXZdM7ZrE9fJIZIqlJLqE=
+github.com/emicklei/go-restful/v3 v3.10.2/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc=
 github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
 github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
 github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98=
 github.com/envoyproxy/go-control-plane v0.9.7/go.mod h1:cwu0lG7PUMfa9snN8LXBig5ynNVH9qI8YYLbd1fK2po=
 github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk=
 github.com/envoyproxy/go-control-plane v0.9.9-0.20210217033140-668b12f5399d/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk=
+github.com/envoyproxy/go-control-plane v0.9.9-0.20210512163311-63b5d3c536b0/go.mod h1:hliV/p42l8fGbc6Y9bQ70uLwIvmJyVE5k4iMKlh8wCQ=
 github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
 github.com/envoyproxy/protoc-gen-validate v1.0.2 h1:QkIBuU5k+x7/QXPvPPnWXWlCdaBFApVqftFV6k087DA=
 github.com/envoyproxy/protoc-gen-validate v1.0.2/go.mod h1:GpiZQP3dDbg4JouG/NNS7QWXpgx6x8QiMKdmN72jogE=
@@ -381,6 +419,13 @@ github.com/ethereum/c-kzg-4844 v0.4.0 h1:3MS1s4JtA868KpJxroZoepdV0ZKBp3u/O5HcZ7R
 github.com/ethereum/c-kzg-4844 v0.4.0/go.mod h1:VewdlzQmpT5QSrVhbBuGoCdFJkpaJlO1aQputP83wc0=
 github.com/ethereum/go-ethereum v1.13.8 h1:1od+thJel3tM52ZUNQwvpYOeRHlbkVFZ5S8fhi0Lgsg=
 github.com/ethereum/go-ethereum v1.13.8/go.mod h1:sc48XYQxCzH3fG9BcrXCOOgQk2JfZzNAmIKnceogzsA=
+github.com/evanphx/json-patch v0.5.2/go.mod h1:ZWS5hhDbVDyob71nXKNL0+PWn6ToqBHMikGIFbs31qQ=
+github.com/evanphx/json-patch v5.6.0+incompatible h1:jBYDEEiFBPxA0v50tFdvOzQQTCvpL6mnFh5mB2/l16U=
+github.com/evanphx/json-patch v5.6.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk=
+github.com/evanphx/json-patch/v5 v5.6.0 h1:b91NhWfaz02IuVxO9faSllyAtNXHMPkC5J8sJCLunww=
+github.com/evanphx/json-patch/v5 v5.6.0/go.mod h1:G79N1coSVB93tBe7j6PhzjmR3/2VvlbKOFpnXhI9Bw4=
+github.com/exponent-io/jsonpath v0.0.0-20210407135951-1de76d718b3f h1:Wl78ApPPB2Wvf/TIe2xdyJxTlb6obmF18d8QdkxNDu4=
+github.com/exponent-io/jsonpath v0.0.0-20210407135951-1de76d718b3f/go.mod h1:OSYXu++VVOHnXeitef/D8n/6y4QV8uLHSFXX4NeXMGc=
 github.com/facebookgo/ensure v0.0.0-20200202191622-63f1cf65ac4c h1:8ISkoahWXwZR41ois5lSJBSVw4D0OV19Ht/JSTzvSv0=
 github.com/facebookgo/ensure v0.0.0-20200202191622-63f1cf65ac4c/go.mod h1:Yg+htXGokKKdzcwhuNDwVvN+uBxDGXJ7G/VN1d8fa64=
 github.com/facebookgo/stack v0.0.0-20160209184415-751773369052 h1:JWuenKqqX8nojtoVVWjGfOF9635RETekkoH6Cc9SX0A=
@@ -388,6 +433,8 @@ github.com/facebookgo/stack v0.0.0-20160209184415-751773369052/go.mod h1:UbMTZqL
 github.com/facebookgo/subset v0.0.0-20200203212716-c811ad88dec4 h1:7HZCaLC5+BZpmbhCOZJ293Lz68O7PYrF2EzeiFMwCLk=
 github.com/facebookgo/subset v0.0.0-20200203212716-c811ad88dec4/go.mod h1:5tD+neXqOorC30/tWg0LCSkrqj/AR6gu8yY8/fpw1q0=
 github.com/fasthttp-contrib/websocket v0.0.0-20160511215533-1f3b11f56072/go.mod h1:duJ4Jxv5lDcvg4QuQr0oowTf7dz4/CR8NtyCooz9HL8=
+github.com/fatih/camelcase v1.0.0 h1:hxNvNX/xYBp0ovncs8WyWZrOrpBNub/JfaMvbURyft8=
+github.com/fatih/camelcase v1.0.0/go.mod h1:yN2Sb0lFhZJUdVvtELVWefmrXpuZESvPmqwoZc+/fpc=
 github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4=
 github.com/fatih/color v1.9.0/go.mod h1:eQcE1qtQxscV5RaZvpXrrb8Drkc3/DdQ+uUYCNjL+zU=
 github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYFFOfk=
@@ -398,6 +445,7 @@ github.com/felixge/httpsnoop v1.0.3 h1:s/nj+GCswXYzN5v2DpNMuMQYe+0DDwt5WVCU6CWBd
 github.com/felixge/httpsnoop v1.0.3/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U=
 github.com/fjl/memsize v0.0.0-20190710130421-bcb5799ab5e5 h1:FtmdgXiUlNeRsoNMFlKLDt+S+6hbjVMEW6RGQ7aUf7c=
 github.com/fjl/memsize v0.0.0-20190710130421-bcb5799ab5e5/go.mod h1:VvhXpOYNQvB+uIk2RvXzuaQtkQJzzIx6lSBe1xv7hi0=
+github.com/flowstack/go-jsonschema v0.1.1/go.mod h1:yL7fNggx1o8rm9RlgXv7hTBWxdBM0rVwpMwimd3F3N0=
 github.com/fortytw2/leaktest v1.3.0 h1:u8491cBMTQ8ft8aeV+adlcytMZylmA5nnwwkRZjI8vw=
 github.com/fortytw2/leaktest v1.3.0/go.mod h1:jDsjWgpAGjm2CA7WthBh/CdZYEPF31XHquHwclZch5g=
 github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8=
@@ -407,6 +455,8 @@ github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4
 github.com/fsnotify/fsnotify v1.5.4/go.mod h1:OVB6XrOHzAwXMpEM7uPOzcehqUV2UqJxmVXmkdnm1bU=
 github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY=
 github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbSClcnxKAGw=
+github.com/fvbommel/sortorder v1.0.2 h1:mV4o8B2hKboCdkJm+a7uX/SIpZob4JzUpc5GGnM45eo=
+github.com/fvbommel/sortorder v1.0.2/go.mod h1:uk88iVf1ovNn1iLfgUVU2F9o5eO30ui720w+kxuqRs0=
 github.com/fxamacker/cbor/v2 v2.5.0 h1:oHsG0V/Q6E/wqTS2O1Cozzsy69nqCiguo5Q1a1ADivE=
 github.com/fxamacker/cbor/v2 v2.5.0/go.mod h1:TA1xS00nchWmaBnEIxPSE5oHLuJBAVvqrtAnWBwBCVo=
 github.com/gabriel-vasile/mimetype v1.4.2 h1:w5qFW6JKBz9Y393Y4q372O9A7cUSequkh1Q7OhCmWKU=
@@ -474,6 +524,7 @@ github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V
 github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A=
 github.com/go-logfmt/logfmt v0.6.0 h1:wGYYu3uicYdqXVgoYbvnkrPVXkuLM1p1ifugDMEdRi4=
 github.com/go-logfmt/logfmt v0.6.0/go.mod h1:WYhtIu8zTZfxdn5+rREduYbwxfcBr/Vr6KEVveWlfTs=
+github.com/go-logr/logr v1.2.0/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
 github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
 github.com/go-logr/logr v1.4.1 h1:pKouT5E8xu9zeFC39JXRDukb6JFQPXM5p5I91188VAQ=
 github.com/go-logr/logr v1.4.1/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=
@@ -482,6 +533,14 @@ github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre
 github.com/go-martini/martini v0.0.0-20170121215854-22fa46961aab/go.mod h1:/P9AEU963A2AYjv4d1V5eVL1CQbEJq6aCNHDDjibzu8=
 github.com/go-ole/go-ole v1.2.6 h1:/Fpf6oFPoeFik9ty7siob0G6Ke8QvQEuVcuChpwXzpY=
 github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0=
+github.com/go-openapi/jsonpointer v0.19.6/go.mod h1:osyAmYz/mB/C3I+WsTTSgw1ONzaLJoLCyoi6/zppojs=
+github.com/go-openapi/jsonpointer v0.20.0 h1:ESKJdU9ASRfaPNOPRx12IUyA1vn3R9GiE3KYD14BXdQ=
+github.com/go-openapi/jsonpointer v0.20.0/go.mod h1:6PGzBjjIIumbLYysB73Klnms1mwnU4G3YHOECG3CedA=
+github.com/go-openapi/jsonreference v0.20.2 h1:3sVjiK66+uXK/6oQ8xgcRKcFgQ5KXa2KvnJRumpMGbE=
+github.com/go-openapi/jsonreference v0.20.2/go.mod h1:Bl1zwGIM8/wsvqjsOQLJ/SH+En5Ap4rVB5KVcIDZG2k=
+github.com/go-openapi/swag v0.22.3/go.mod h1:UzaqsxGiab7freDnrUUra0MwWfN/q7tE4j+VcZ0yl14=
+github.com/go-openapi/swag v0.22.4 h1:QLMzNJnMGPRNDCbySlcj1x01tzU8/9LTTL9hZZZogBU=
+github.com/go-openapi/swag v0.22.4/go.mod h1:UzaqsxGiab7freDnrUUra0MwWfN/q7tE4j+VcZ0yl14=
 github.com/go-playground/assert/v2 v2.0.1/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4=
 github.com/go-playground/assert/v2 v2.2.0 h1:JvknZsQTYeFEAhQwI4qEt9cyV5ONwRHC+lYKSsYSR8s=
 github.com/go-playground/assert/v2 v2.2.0/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4=
@@ -494,6 +553,8 @@ github.com/go-playground/universal-translator v0.18.1/go.mod h1:xekY+UJKNuX9WP91
 github.com/go-playground/validator/v10 v10.10.0/go.mod h1:74x4gJWsvQexRdW8Pn3dXSGrTK4nAUsbPlLADvpJkos=
 github.com/go-playground/validator/v10 v10.15.5 h1:LEBecTWb/1j5TNY1YYG2RcOUN3R7NLylN+x8TTueE24=
 github.com/go-playground/validator/v10 v10.15.5/go.mod h1:9iXMNT7sEkjXb0I+enO7QXmzG6QCsPWY4zveKFVRSyU=
+github.com/go-resty/resty/v2 v2.7.0 h1:me+K9p3uhSmXtrBZ4k9jcEAfJmuC8IivWHwaLZwPrFY=
+github.com/go-resty/resty/v2 v2.7.0/go.mod h1:9PWDzw47qPphMRFfhsyk0NnSgvluHcljSMVIq3w7q0I=
 github.com/go-sql-driver/mysql v1.6.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg=
 github.com/go-sql-driver/mysql v1.7.1 h1:lUIinVbN1DY0xBg0eMOzmmtGoHwWBbvnWubQUrtU8EI=
 github.com/go-sql-driver/mysql v1.7.1/go.mod h1:OXbVy3sEdcQ2Doequ6Z5BW6fXNQTmx+9S1MCJN5yJMI=
@@ -513,6 +574,7 @@ github.com/goccy/go-json v0.10.2 h1:CrxCmQqYDkv1z7lO7Wbh2HN93uovUHgrECaO5ZrCXAU=
 github.com/goccy/go-json v0.10.2/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I=
 github.com/godbus/dbus v0.0.0-20190726142602-4481cbc300e2 h1:ZpnhV/YsD2/4cESfV5+Hoeu/iUR3ruzNvZ+yQfO03a0=
 github.com/godbus/dbus v0.0.0-20190726142602-4481cbc300e2/go.mod h1:bBOAhwG1umN6/6ZUMtDFBMQR8jRg9O75tm9K00oMsK4=
+github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA=
 github.com/gofrs/flock v0.8.1 h1:+gYjHKf32LDeiEEFhQaotPbLuUXjY5ZqxKgXy7n59aw=
 github.com/gofrs/flock v0.8.1/go.mod h1:F1TvTiK9OcQqauNUHlbJvyl9Qa1QvF/gOUDKA14jxHU=
 github.com/gofrs/uuid v4.0.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM=
@@ -576,6 +638,10 @@ github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Z
 github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
 github.com/google/btree v1.1.2 h1:xf4v41cLI2Z6FxbKm+8Bu+m8ifhj15JuZ9sa0jZCMUU=
 github.com/google/btree v1.1.2/go.mod h1:qOPhT0dTNdNzV6Z/lhRX0YXUafgPLFUh+gZMl761Gm4=
+github.com/google/gnostic v0.6.9 h1:ZK/5VhkoX835RikCHpSUJV9a+S3e1zLh59YnyWeBW+0=
+github.com/google/gnostic v0.6.9/go.mod h1:Nm8234We1lq6iB9OmlgNv3nH91XLLVZHCDayfA3xq+E=
+github.com/google/gnostic-models v0.6.8 h1:yo/ABAfM5IMRsS1VnXjTBvUb61tFIHozhlYvRgGre9I=
+github.com/google/gnostic-models v0.6.8/go.mod h1:5n7qKqH0f5wFt+aWF8CW6pZLLNOfYuF5OpfBSENuI8U=
 github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
 github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
 github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
@@ -658,6 +724,8 @@ github.com/graph-gophers/dataloader v5.0.0+incompatible h1:R+yjsbrNq1Mo3aPG+Z/EK
 github.com/graph-gophers/dataloader v5.0.0+incompatible/go.mod h1:jk4jk0c5ZISbKaMe8WsVopGB5/15GvGHMdMdPtwlRp4=
 github.com/graph-gophers/graphql-go v1.3.0 h1:Eb9x/q6MFpCLz7jBCiP/WTxjSDrYLR1QY41SORZyNJ0=
 github.com/graph-gophers/graphql-go v1.3.0/go.mod h1:9CQHMSxwO4MprSdzoIEobiHpoLtHm77vfxsvsIN5Vuc=
+github.com/gregjones/httpcache v0.0.0-20190611155906-901d90724c79 h1:+ngKgrYPPJrOjhax5N+uePQ0Fh1Z7PheYoUI/0nzkPA=
+github.com/gregjones/httpcache v0.0.0-20190611155906-901d90724c79/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA=
 github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs=
 github.com/grpc-ecosystem/go-grpc-middleware v1.3.0 h1:+9834+KizmvFV7pXQGSXQTsaWhq2GjuNUt0aUU0YBYw=
 github.com/grpc-ecosystem/go-grpc-middleware v1.3.0/go.mod h1:z0ButlSOZa5vEBq9m2m2hlwIgKw+rp3sdCBRoJY+30Y=
@@ -838,6 +906,7 @@ github.com/joho/godotenv v1.4.0/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwA
 github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo=
 github.com/jonboulle/clockwork v0.4.0 h1:p4Cf1aMWXnXAUh8lVfewRBx1zaTSYKrKMF2g3ST4RZ4=
 github.com/jonboulle/clockwork v0.4.0/go.mod h1:xgRqUGwRcjKCO1vbZUEtSLrqKoPSsUpK7fnezOII0kc=
+github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY=
 github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y=
 github.com/jpillora/backoff v1.0.0 h1:uvFg412JmmHBHw7iwprIxkPMI+sGQ4kzOWsMeHnm2EA=
 github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4=
@@ -857,6 +926,8 @@ github.com/kataras/pio v0.0.2/go.mod h1:hAoW0t9UmXi4R5Oyq5Z4irTbaTsOemSrDGUtaTl7
 github.com/kataras/sitemap v0.0.5/go.mod h1:KY2eugMKiPwsJgx7+U103YZehfvNGOXURubcGyk0Bz8=
 github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 h1:Z9n2FFNUXsshfwJMBgNA0RU6/i7WVaAegv3PtuIHPMs=
 github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51/go.mod h1:CzGEWj7cYgsdH8dAjBGEr58BoE7ScuLd+fwFZ44+/x8=
+github.com/kelseyhightower/envconfig v1.4.0 h1:Im6hONhd3pLkfDFsbRgu68RDNkGF1r3dvMUtDTo2cv8=
+github.com/kelseyhightower/envconfig v1.4.0/go.mod h1:cccZRl6mQpaq41TPp5QxidR+Sa3axMbJDNb//FQX6Gg=
 github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8=
 github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
 github.com/klauspost/compress v1.8.2/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A=
@@ -876,6 +947,7 @@ github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxv
 github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg=
 github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc=
 github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
+github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
 github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
 github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk=
 github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
@@ -902,8 +974,11 @@ github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw=
 github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o=
 github.com/libp2p/go-buffer-pool v0.1.0 h1:oK4mSFcQz7cTQIfqbe4MIj9gLW+mnanjyFtc6cdF0Y8=
 github.com/libp2p/go-buffer-pool v0.1.0/go.mod h1:N+vh8gMqimBzdKkSMVuydVDq+UV5QTWy5HSiZacSbPg=
+github.com/liggitt/tabwriter v0.0.0-20181228230101-89fcab3d43de h1:9TO3cAIGXtEhnIaL+V+BEER86oLrvS+kWobKpbJuye0=
+github.com/liggitt/tabwriter v0.0.0-20181228230101-89fcab3d43de/go.mod h1:zAbeS9B/r2mtpb6U+EI2rYA5OAXxsYw6wTamcNW+zcE=
 github.com/logrusorgru/aurora v2.0.3+incompatible h1:tOpm7WcpBTn4fjmVfgpQq0EfczGlG91VSDkswnjF5A8=
 github.com/logrusorgru/aurora v2.0.3+incompatible/go.mod h1:7rIyQOR62GCctdiQpZ/zOJlFyk6y+94wXzv6RNZgaR4=
+github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 h1:6E+4a0GO5zZEnZ81pIr0yLvtUWk2if982qA3F3QD6H4=
 github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0/go.mod h1:zJYVVT2jmtg6P3p1VtQj7WsuWi/y4VnjVBn7F8KPB3I=
 github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
 github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
@@ -965,6 +1040,8 @@ github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrk
 github.com/mitchellh/go-testing-interface v1.0.0/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI=
 github.com/mitchellh/go-testing-interface v1.14.1 h1:jrgshOhYAUVNMAJiKbEu7EqAwgJJ2JqpQmpLJOu07cU=
 github.com/mitchellh/go-testing-interface v1.14.1/go.mod h1:gfgS7OtZj6MA4U1UrDRp04twqAjfvlZyCfX3sDjEym8=
+github.com/mitchellh/go-wordwrap v1.0.1 h1:TLuKupo69TCn6TQSyGxwI1EblZZEsQ0vMlAFQflz0v0=
+github.com/mitchellh/go-wordwrap v1.0.1/go.mod h1:R62XHJLzvMFRBbcrT7m7WgmE1eOyTSsCt+hzestvNj0=
 github.com/mitchellh/gox v0.4.0/go.mod h1:Sd9lOJ0+aimLBi73mGofS1ycjY8lL3uZM3JPS42BGNg=
 github.com/mitchellh/iochan v1.0.0/go.mod h1:JwYml1nuB7xOzsp52dPpHFffvOCDupsG0QubkSMEySY=
 github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
@@ -979,6 +1056,14 @@ github.com/mitchellh/reflectwalk v1.0.2/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx
 github.com/mmcloughlin/addchain v0.4.0 h1:SobOdjm2xLj1KkXN5/n0xTIWyZA2+s99UCY1iPfkHRY=
 github.com/mmcloughlin/addchain v0.4.0/go.mod h1:A86O+tHqZLMNO4w6ZZ4FlVQEadcoqkyU72HC5wJ4RlU=
 github.com/mmcloughlin/profile v0.1.1/go.mod h1:IhHD7q1ooxgwTgjxQYkACGA77oFTDdFVejUS1/tS/qU=
+github.com/moby/patternmatcher v0.6.0 h1:GmP9lR19aU5GqSSFko+5pRqHi+Ohk1O69aFiKkVGiPk=
+github.com/moby/patternmatcher v0.6.0/go.mod h1:hDPoyOpDY7OrrMDLaYoY3hf52gNCR/YOUYxkhApJIxc=
+github.com/moby/spdystream v0.2.0 h1:cjW1zVyyoiM0T7b6UoySUFqzXMoqRckQtXwGPiBhOM8=
+github.com/moby/spdystream v0.2.0/go.mod h1:f7i0iNDQJ059oMTcWxx8MA/zKFIuD/lY+0GqbN2Wy8c=
+github.com/moby/sys/sequential v0.5.0 h1:OPvI35Lzn9K04PBbCLW0g4LcFAJgHsvXsRyewg5lXtc=
+github.com/moby/sys/sequential v0.5.0/go.mod h1:tH2cOOs5V9MlPiXcQzRC+eEyab644PWKGRYaaV5ZZlo=
+github.com/moby/sys/user v0.1.0 h1:WmZ93f5Ux6het5iituh9x2zAG7NFY9Aqi49jjE1PaQg=
+github.com/moby/sys/user v0.1.0/go.mod h1:fKJhFOnsCN6xZ5gSfbM6zaHGgDJMrqt9/reuj4T7MmU=
 github.com/moby/term v0.5.0 h1:xt8Q1nalod/v7BqbG21f8mQPqH+xAaC9C3N3wfWbVP0=
 github.com/moby/term v0.5.0/go.mod h1:8FzsFHVUBGZdbDsJw/ot+X+d5HLUbvklYLJ9uGfcI3Y=
 github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
@@ -988,7 +1073,13 @@ github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lN
 github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
 github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M=
 github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk=
+github.com/monochromegane/go-gitignore v0.0.0-20200626010858-205db1a8cc00 h1:n6/2gBQ3RWajuToeY6ZtZTIKv2v7ThUy5KKusIT0yc0=
+github.com/monochromegane/go-gitignore v0.0.0-20200626010858-205db1a8cc00/go.mod h1:Pm3mSP3c5uWn86xMLZ5Sa7JB9GsEZySvHYXCTK4E9q4=
 github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc=
+github.com/montanaflynn/stats v0.7.1 h1:etflOAAHORrCC44V+aR6Ftzort912ZU+YLiSTuV8eaE=
+github.com/montanaflynn/stats v0.7.1/go.mod h1:etXPPgVO6n31NxCd9KQUMvCM+ve0ruNzt6R8Bnaayow=
+github.com/morikuni/aec v1.0.0 h1:nP9CBfwrvYnBRgY6qfDQkygYDmYwOilePFkwzv4dU8A=
+github.com/morikuni/aec v1.0.0/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7PXmsc=
 github.com/mostynb/zstdpool-freelist v0.0.0-20201229113212-927304c0c3b1 h1:mPMvm6X6tf4w8y7j9YIt6V9jfWhL6QlbEc7CCmeQlWk=
 github.com/mostynb/zstdpool-freelist v0.0.0-20201229113212-927304c0c3b1/go.mod h1:ye2e/VUEtE2BHE+G/QcKkcLQVAEJoYRFj5VUOQatCRE=
 github.com/moul/http2curl v1.0.0/go.mod h1:8UbvGypXm98wA/IqH45anm5Y2Z6ep6O31QGOAZ3H0fQ=
@@ -996,7 +1087,13 @@ github.com/mr-tron/base58 v1.2.0 h1:T/HDJBh4ZCPbU39/+c3rRvE0uKBQlU27+QI8LJ4t64o=
 github.com/mr-tron/base58 v1.2.0/go.mod h1:BinMc/sQntlIE1frQmRFPUoPA1Zkr8VRgBdjWI2mNwc=
 github.com/mtibben/percent v0.2.1 h1:5gssi8Nqo8QU/r2pynCm+hBQHpkB/uNK7BJCFogWdzs=
 github.com/mtibben/percent v0.2.1/go.mod h1:KG9uO+SZkUp+VkRHsCdYQV3XSZrrSpR3O9ibNBTZrns=
+github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA=
+github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
 github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
+github.com/naoina/go-stringutil v0.1.0 h1:rCUeRUHjBjGTSHl0VC00jUPLz8/F9dDzYI70Hzifhks=
+github.com/naoina/go-stringutil v0.1.0/go.mod h1:XJ2SJL9jCtBh+P9q5btrd/Ylo8XwT/h1USek5+NqSA0=
+github.com/naoina/toml v0.1.2-0.20170918210437-9fafd6967416 h1:shk/vn9oCoOTmwcouEdwIeOtOGA/ELRUw/GwvxwfT+0=
+github.com/naoina/toml v0.1.2-0.20170918210437-9fafd6967416/go.mod h1:NBIhNtsFMo3G2szEBne+bO4gS192HuIYRqfvOWb4i1E=
 github.com/nats-io/jwt v0.3.0/go.mod h1:fRYCDE99xlTsqUzISS1Bi75UBJ6ljOJQOAAu5VglpSg=
 github.com/nats-io/nats.go v1.9.1/go.mod h1:ZjDU1L/7fJ09jvUSRVBR2e7+RnLiiIQyqyzEE/Zbp4w=
 github.com/nats-io/nkeys v0.1.0/go.mod h1:xpnFELMwJABBLVhffcfd1MZx6VsNRFpEugbxziKVo7w=
@@ -1054,6 +1151,10 @@ github.com/pelletier/go-toml v1.9.5/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCko
 github.com/pelletier/go-toml/v2 v2.0.1/go.mod h1:r9LEWfGN8R5k0VXJ+0BkIe7MYkRdwZOjgMj2KwnJFUo=
 github.com/pelletier/go-toml/v2 v2.1.1 h1:LWAJwfNvjQZCFIDKWYQaM62NcYeYViCmWIwmOStowAI=
 github.com/pelletier/go-toml/v2 v2.1.1/go.mod h1:tJU2Z3ZkXwnxa4DPO899bsyIoywizdUvyaeZurnPPDc=
+github.com/pelletier/go-toml/v2 v2.2.2 h1:aYUidT7k73Pcl9nb2gScu7NSrKCSHIDE89b3+6Wq+LM=
+github.com/pelletier/go-toml/v2 v2.2.2/go.mod h1:1t835xjRzz80PqgE6HHgN2JOsmgYu/h4qDAS4n929Rs=
+github.com/peterbourgon/diskv v2.0.1+incompatible h1:UBdAOUP5p4RWqPBg048CAvpKN+vxiaj6gdUUzhl4XmI=
+github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU=
 github.com/petermattis/goid v0.0.0-20180202154549-b0b1615b78e5/go.mod h1:jvVRKCrJTQWu0XVbaOlby/2lO20uSCHEMzzplHXte1o=
 github.com/petermattis/goid v0.0.0-20230317030725-371a4b8eda08 h1:hDSdbBuw3Lefr6R18ax0tZ2BJeNB3NehB3trOwYBsdU=
 github.com/petermattis/goid v0.0.0-20230317030725-371a4b8eda08/go.mod h1:pxMtw7cyUw6B2bRH0ZBANSPg+AoSud1I1iyJHI69jH4=
@@ -1129,11 +1230,14 @@ github.com/rogpeppe/go-internal v1.11.0/go.mod h1:ddIwULY96R17DhadqLgMfk9H9tvdUz
 github.com/rs/cors v1.8.3 h1:O+qNyWn7Z+F9M0ILBHgMVPuB1xTOucVd5gtaYyXBpRo=
 github.com/rs/cors v1.8.3/go.mod h1:XyqrcTp5zjWr1wsJ8PIRZssZ8b/WMcMf71DJnit4EMU=
 github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ=
+github.com/rs/xid v1.5.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg=
 github.com/rs/zerolog v1.13.0/go.mod h1:YbFCdg8HfsridGWAh22vktObvhZbQsZXe4/zB0OKkWU=
 github.com/rs/zerolog v1.15.0/go.mod h1:xYTKnLHcpfU2225ny5qZjxnj9NvkumZYjJHlAThCjNc=
 github.com/rs/zerolog v1.30.0 h1:SymVODrcRsaRaSInD9yQtKbtWqwsfoPcRff/oRXLj4c=
 github.com/rs/zerolog v1.30.0/go.mod h1:/tk+P47gFdPXq4QYjvCmT5/Gsug2nagsFWBWhAiSi1w=
 github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g=
+github.com/russross/blackfriday v1.6.0 h1:KqfZb0pUVN2lYqZUYRddxF4OR8ZMURnJIG5Y3VRLtww=
+github.com/russross/blackfriday v1.6.0/go.mod h1:ti0ldHuxg49ri4ksnFxlkCfN+hvslNlmVHqNRXXJNAY=
 github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
 github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk=
 github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
@@ -1157,6 +1261,7 @@ github.com/shirou/gopsutil v3.21.11+incompatible h1:+1+c1VGhc88SSonWP6foOcLhvnKl
 github.com/shirou/gopsutil v3.21.11+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA=
 github.com/shirou/gopsutil/v3 v3.24.3 h1:eoUGJSmdfLzJ3mxIhmOAhgKEKgQkeOwKpz1NbhVnuPE=
 github.com/shirou/gopsutil/v3 v3.24.3/go.mod h1:JpND7O217xa72ewWz9zN2eIIkPWsDN/3pl0H8Qt0uwg=
+github.com/shoenig/go-m1cpu v0.1.6 h1:nxdKQNcEB6vzgA2E2bvzKIYRuNj7XNJ4S/aRSwKzFtM=
 github.com/shoenig/go-m1cpu v0.1.6/go.mod h1:1JJMcUBvfNwpq05QDQVAnx3gUHr9IYF7GNg9SUEw2VQ=
 github.com/shoenig/test v0.6.4/go.mod h1:byHiCGXqrVaflBLAMq/srcZIHynQPQgeyvkvXnjqq0k=
 github.com/shopspring/decimal v0.0.0-20180709203117-cd690d0c9e24/go.mod h1:M+9NzErvs504Cn4c5DxATwIqPbtswREoFCre64PpcG4=
@@ -1185,6 +1290,8 @@ github.com/smartcontractkit/chainlink-solana v1.0.3-0.20240605170242-555ff582f36
 github.com/smartcontractkit/chainlink-solana v1.0.3-0.20240605170242-555ff582f36a/go.mod h1:QqcZSwLgEIn7YraAIRmomnBMAuVFephiHrIWVlkWbFI=
 github.com/smartcontractkit/chainlink-starknet/relayer v0.0.1-beta-test.0.20240531021326-99118e47f696 h1:h1E87+z+JcUEfvbJVF56SnZA/YUFE5ewUE61MaR/Ewg=
 github.com/smartcontractkit/chainlink-starknet/relayer v0.0.1-beta-test.0.20240531021326-99118e47f696/go.mod h1:OiWUTrrpSLLTMh7FINWjEh6mmDJCVPaC4yEsDCVaWdU=
+github.com/smartcontractkit/chainlink-testing-framework v1.31.4 h1:5/zF5Z2w+/HTvUSSJNhWSiHvlQ2O2qxRw4e1fBbVraQ=
+github.com/smartcontractkit/chainlink-testing-framework v1.31.4/go.mod h1:E6uNEZhZZid9PHv6/Kq5Vn63GlO61ZcKB+/f0DKo3Q4=
 github.com/smartcontractkit/chainlink-vrf v0.0.0-20231120191722-fef03814f868 h1:FFdvEzlYwcuVHkdZ8YnZR/XomeMGbz5E2F2HZI3I3w8=
 github.com/smartcontractkit/chainlink-vrf v0.0.0-20231120191722-fef03814f868/go.mod h1:Kn1Hape05UzFZ7bOUnm3GVsHzP0TNrVmpfXYNHdqGGs=
 github.com/smartcontractkit/go-plugin v0.0.0-20240208201424-b3b91517de16 h1:TFe+FvzxClblt6qRfqEhUfa4kFQx5UobuoFGO2W4mMo=
@@ -1193,6 +1300,8 @@ github.com/smartcontractkit/grpc-proxy v0.0.0-20230731113816-f1be6620749f h1:hgJ
 github.com/smartcontractkit/grpc-proxy v0.0.0-20230731113816-f1be6620749f/go.mod h1:MvMXoufZAtqExNexqi4cjrNYE9MefKddKylxjS+//n0=
 github.com/smartcontractkit/libocr v0.0.0-20240419185742-fd3cab206b2c h1:lIyMbTaF2H0Q71vkwZHX/Ew4KF2BxiKhqEXwF8rn+KI=
 github.com/smartcontractkit/libocr v0.0.0-20240419185742-fd3cab206b2c/go.mod h1:fb1ZDVXACvu4frX3APHZaEBp0xi1DIm34DcA0CwTsZM=
+github.com/smartcontractkit/seth v1.0.11 h1:Ct8wSifW2ZXnyHtYRKaawBnpW7Ef0m8zItUcqYPallM=
+github.com/smartcontractkit/seth v1.0.11/go.mod h1:fVCE+8LD6AbU7d8BHZ1uS/ceJ+8JO0iVTUcDdQmGPAc=
 github.com/smartcontractkit/tdh2/go/ocr2/decryptionplugin v0.0.0-20230906073235-9e478e5e19f1 h1:yiKnypAqP8l0OX0P3klzZ7SCcBUxy5KqTAKZmQOvSQE=
 github.com/smartcontractkit/tdh2/go/ocr2/decryptionplugin v0.0.0-20230906073235-9e478e5e19f1/go.mod h1:q6f4fe39oZPdsh1i57WznEZgxd8siidMaSFq3wdPmVg=
 github.com/smartcontractkit/tdh2/go/tdh2 v0.0.0-20230906073235-9e478e5e19f1 h1:Dai1bn+Q5cpeGMQwRdjOdVjG8mmFFROVkSKuUgBErRQ=
@@ -1231,6 +1340,7 @@ github.com/spf13/viper v1.15.0 h1:js3yy885G8xwJa6iOISGFwd+qlUo5AvyXb7CiihdtiU=
 github.com/spf13/viper v1.15.0/go.mod h1:fFcTBJxvhhzSJiZy8n+PeW6t8l+KeT/uTARa0jHOQLA=
 github.com/status-im/keycard-go v0.2.0 h1:QDLFswOQu1r5jsycloeQh3bVU8n/NatHHaZobtDnDzA=
 github.com/status-im/keycard-go v0.2.0/go.mod h1:wlp8ZLbsmrF6g6WjugPAx+IzoLrkdf9+mHxBEeo3Hbg=
+github.com/stoewer/go-strcase v1.2.0/go.mod h1:IBiWB2sKIp3wVVQ3Y035++gc+knqhUQag1KpM8ahLw8=
 github.com/streamingfast/logging v0.0.0-20220405224725-2755dab2ce75 h1:ZqpS7rAhhKD7S7DnrpEdrnW1/gZcv82ytpMviovkli4=
 github.com/streamingfast/logging v0.0.0-20220405224725-2755dab2ce75/go.mod h1:VlduQ80JcGJSargkRU4Sg9Xo63wZD/l8A5NC/Uo1/uU=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
@@ -1270,6 +1380,8 @@ github.com/teris-io/shortid v0.0.0-20201117134242-e59966efd125 h1:3SNcvBmEPE1YlB
 github.com/teris-io/shortid v0.0.0-20201117134242-e59966efd125/go.mod h1:M8agBzgqHIhgj7wEn9/0hJUZcrvt9VY+Ln+S1I5Mha0=
 github.com/test-go/testify v1.1.4 h1:Tf9lntrKUMHiXQ07qBScBTSA0dhYQlu83hswqelv1iE=
 github.com/test-go/testify v1.1.4/go.mod h1:rH7cfJo/47vWGdi4GPj16x3/t1xGOj2YxzmNQzk2ghU=
+github.com/testcontainers/testcontainers-go v0.28.0 h1:1HLm9qm+J5VikzFDYhOd+Zw12NtOl+8drH2E8nTY1r8=
+github.com/testcontainers/testcontainers-go v0.28.0/go.mod h1:COlDpUXbwW3owtpMkEB1zo9gwb1CoKVKlyrVPejF4AU=
 github.com/theodesp/go-heaps v0.0.0-20190520121037-88e35354fe0a h1:YuO+afVc3eqrjiCUizNCxI53bl/BnPiVwXqLzqYTqgU=
 github.com/theodesp/go-heaps v0.0.0-20190520121037-88e35354fe0a/go.mod h1:/sfW47zCZp9FrtGcWyo1VjbgDaodxX9ovZvgLb/MxaA=
 github.com/tidwall/btree v1.6.0 h1:LDZfKfQIBHGHWSwckhXI0RPSXzlo+KYdjK7FWSqOzzg=
@@ -1339,6 +1451,8 @@ github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415/go.mod h1:
 github.com/xeipuuv/gojsonschema v1.2.0 h1:LhYJRs+L4fBtjZUfuSZIKGeVu0QRy8e5Xi7D17UxZ74=
 github.com/xeipuuv/gojsonschema v1.2.0/go.mod h1:anYRn/JVcOK2ZgGU+IjEV4nwlhoK5sQluxsYJ78Id3Y=
 github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU=
+github.com/xlab/treeprint v1.1.0 h1:G/1DjNkPpfZCFt9CSh6b5/nY4VimlbHF3Rh4obvtzDk=
+github.com/xlab/treeprint v1.1.0/go.mod h1:gj5Gd3gPdKtR1ikdDK6fnFLdmIS0X30kTTuNd/WEJu0=
 github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q=
 github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 h1:bAn7/zixMGCfxrRTfdpNzjtPYqr8smhKouy9mxVdGPU=
 github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673/go.mod h1:N3UwUGtsrSj3ccvlPHLoLsHnpR27oXr4ZE984MbSER8=
@@ -1393,6 +1507,8 @@ go.opentelemetry.io/contrib/instrumentation/github.com/gin-gonic/gin/otelgin v0.
 go.opentelemetry.io/contrib/instrumentation/github.com/gin-gonic/gin/otelgin v0.49.0/go.mod h1:1P/02zM3OwkX9uki+Wmxw3a5GVb6KUXRsa7m7bOC9Fg=
 go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.46.1 h1:SpGay3w+nEwMpfVnbqOLH5gY52/foP8RE8UzTZ1pdSE=
 go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.46.1/go.mod h1:4UoMYEZOC0yN/sPGH76KPkkU7zgiEWYWL9vwmbnTJPE=
+go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.45.0 h1:x8Z78aZx8cOF0+Kkazoc7lwUNMGy0LrzEMxTm4BbTxg=
+go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.45.0/go.mod h1:62CPTSry9QZtOaSsE3tOzhx6LzDhHnXJ6xHeMNNiM6Q=
 go.opentelemetry.io/contrib/propagators/b3 v1.24.0 h1:n4xwCdTx3pZqZs2CjS/CUZAs03y3dZcGhC/FepKtEUY=
 go.opentelemetry.io/contrib/propagators/b3 v1.24.0/go.mod h1:k5wRxKRU2uXx2F8uNJ4TaonuEO/V7/5xoz7kdsDACT8=
 go.opentelemetry.io/otel v1.24.0 h1:0LAOdjNmQeSTzGBzduGe/rU4tZhMwL5rWgtp9Ku5Jfo=
@@ -1409,8 +1525,11 @@ go.opentelemetry.io/otel/sdk/metric v1.21.0 h1:smhI5oD714d6jHE6Tie36fPx4WDFIg+Y6
 go.opentelemetry.io/otel/sdk/metric v1.21.0/go.mod h1:FJ8RAsoPGv/wYMgBdUJXOm+6pzFY3YdljnXtv1SBE8Q=
 go.opentelemetry.io/otel/trace v1.24.0 h1:CsKnnL4dUAr/0llH9FKuc698G04IrpWV0MQA/Y1YELI=
 go.opentelemetry.io/otel/trace v1.24.0/go.mod h1:HPc3Xr/cOApsBI154IU0OI0HJexz+aw5uPdbs3UCjNU=
+go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI=
 go.opentelemetry.io/proto/otlp v1.0.0 h1:T0TX0tmXU8a3CbNXzEKGeU5mIVOdf0oykP+u2lIVU/I=
 go.opentelemetry.io/proto/otlp v1.0.0/go.mod h1:Sy6pihPLfYHkr3NkUbEhGHFhINUSI/v80hjKIs5JXpM=
+go.starlark.net v0.0.0-20220817180228-f738f5508c12 h1:xOBJXWGEDwU5xSDxH6macxO11Us0AH2fTa9rmsbbF7g=
+go.starlark.net v0.0.0-20220817180228-f738f5508c12/go.mod h1:VZcBMdr3cT3PnBoWunTabuSEXwVAH+ZJ5zxfs3AdASk=
 go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
 go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
 go.uber.org/atomic v1.5.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ=
@@ -1560,7 +1679,9 @@ golang.org/x/net v0.0.0-20210331212208-0fccb6fa2b5c/go.mod h1:p54w0d4576C0XHj96b
 golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM=
 golang.org/x/net v0.0.0-20210423184538-5f58ad60dda6/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk=
 golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk=
+golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
 golang.org/x/net v0.0.0-20211008194852-3b03d305991f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
+golang.org/x/net v0.0.0-20211029224645-99673261e6eb/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
 golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
 golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
 golang.org/x/net v0.0.0-20220607020251-c690dde0001d/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
@@ -1660,6 +1781,7 @@ golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20210423185535-09eb48e85fd7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210806184541-e5e7981a1069/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210819135213-f52c844e1c1c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
@@ -1703,6 +1825,7 @@ golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
+golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ=
 golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
 golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
 golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
@@ -1791,6 +1914,8 @@ golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8T
 golang.org/x/xerrors v0.0.0-20220517211312-f3a8303e98df/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8=
 golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 h1:+cNy6SZtPcJQH3LJVLOSmiC7MMxXNOb3PU/VUEz+EhU=
 golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028/go.mod h1:NDW/Ps6MPRej6fsCIbMTohpP40sJ/P/vI1MoTEGwX90=
+gomodules.xyz/jsonpatch/v2 v2.2.0 h1:4pT439QV83L+G9FkcCriY6EkpcK6r6bK+A5FBUMI7qY=
+gomodules.xyz/jsonpatch/v2 v2.2.0/go.mod h1:WXp+iVDkoLQqPudfQ9GBlwB2eZ5DKOnjQZCYdOS8GPY=
 gonum.org/v1/gonum v0.14.0 h1:2NiG67LD1tEH0D7kM+ps2V+fXmsAnpUeec7n8tcr4S0=
 gonum.org/v1/gonum v0.14.0/go.mod h1:AoWeoz0becf9QMWtE8iWXNXc27fK4fNeHNf/oMejGfU=
 google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE=
@@ -1870,6 +1995,7 @@ google.golang.org/genproto v0.0.0-20210108203827-ffc7fda8c3d7/go.mod h1:FWY/as6D
 google.golang.org/genproto v0.0.0-20210226172003-ab064af71705/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
 google.golang.org/genproto v0.0.0-20210401141331-865547bb08e2/go.mod h1:9lPAdzaEmUacj36I+k7YKbEc5CXzPIeORRgDAUOu28A=
 google.golang.org/genproto v0.0.0-20210624195500-8bfb893ecb84/go.mod h1:SzzZ/N+nwJDaO1kznhnlzqS8ocJICar6hYhVyhi++24=
+google.golang.org/genproto v0.0.0-20220107163113-42d7afdf6368/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc=
 google.golang.org/genproto v0.0.0-20231030173426-d783a09b4405 h1:I6WNifs6pF9tNdSob2W24JtyxIYjzFB9qDlpUC76q+U=
 google.golang.org/genproto v0.0.0-20231030173426-d783a09b4405/go.mod h1:3WDQMjmJk36UQhjQ89emUzb1mdaHcPeeAh4SCBKznB4=
 google.golang.org/genproto/googleapis/api v0.0.0-20231016165738-49dd2c1f3d0b h1:CIC2YMXmIhYw6evmhPxBKJ4fmLbOFtXQN/GV3XOZR8k=
@@ -1896,9 +2022,11 @@ google.golang.org/grpc v1.33.1/go.mod h1:fr5YgcSWrqhRRxogOsw7RzIpsmvOZ6IcH4kBYTp
 google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc=
 google.golang.org/grpc v1.34.0/go.mod h1:WotjhfgOW/POjDeRt8vscBtXq+2VjORFy659qA51WJ8=
 google.golang.org/grpc v1.35.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU=
+google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU=
 google.golang.org/grpc v1.36.1/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU=
 google.golang.org/grpc v1.37.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM=
 google.golang.org/grpc v1.38.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM=
+google.golang.org/grpc v1.40.0/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34=
 google.golang.org/grpc v1.59.0 h1:Z5Iec2pjwb+LEOqzpB2MR12/eKFhDPhuqW91O+4bwUk=
 google.golang.org/grpc v1.59.0/go.mod h1:aUPDwccQo6OTjy7Hct4AfBPD1GptF4fyUjIkQ9YtF98=
 google.golang.org/grpc/examples v0.0.0-20210424002626-9572fd6faeae/go.mod h1:Ly7ZA/ARzg8fnPU9TyZIxoz33sEUuWX7txiqs8lPTgE=
@@ -1914,6 +2042,7 @@ google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGj
 google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c=
 google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
 google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
+google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
 google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I=
 google.golang.org/protobuf v1.33.0 h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGmI=
 google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos=
@@ -1933,6 +2062,8 @@ gopkg.in/guregu/null.v2 v2.1.2/go.mod h1:XORrx8tyS5ZDcyUboCIxQtta/Aujk/6pfWrn9Xe
 gopkg.in/guregu/null.v4 v4.0.0 h1:1Wm3S1WEA2I26Kq+6vcW+w0gcDo44YKYD7YIEJNHDjg=
 gopkg.in/guregu/null.v4 v4.0.0/go.mod h1:YoQhUrADuG3i9WqesrCmpNRwm1ypAgSHYqoOcTu/JrI=
 gopkg.in/inconshreveable/log15.v2 v2.0.0-20180818164646-67afb5ed74ec/go.mod h1:aPpfJ7XW+gOuirDoZ8gHhLh3kZ1B08FtV2bbmy7Jv3s=
+gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc=
+gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw=
 gopkg.in/ini.v1 v1.51.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
 gopkg.in/ini.v1 v1.51.1/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
 gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA=
@@ -1955,6 +2086,7 @@ gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
 gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
 gopkg.in/yaml.v3 v3.0.0-20191120175047-4206685974f2/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
 gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
@@ -1971,6 +2103,27 @@ honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9
 honnef.co/go/tools v0.1.3/go.mod h1:NgwopIslSNH47DimFoV78dnkksY2EFtX0ajyb3K/las=
 howett.net/plist v1.0.0 h1:7CrbWYbPPO/PyNy38b2EB/+gYbjCe2DXBxgtOOZbSQM=
 howett.net/plist v1.0.0/go.mod h1:lqaXoTrLY4hg8tnEzNru53gicrbv7rrk+2xJA/7hw9g=
+k8s.io/api v0.28.2 h1:9mpl5mOb6vXZvqbQmankOfPIGiudghwCoLl1EYfUZbw=
+k8s.io/api v0.28.2/go.mod h1:RVnJBsjU8tcMq7C3iaRSGMeaKt2TWEUXcpIt/90fjEg=
+k8s.io/apiextensions-apiserver v0.25.3 h1:bfI4KS31w2f9WM1KLGwnwuVlW3RSRPuIsfNF/3HzR0k=
+k8s.io/apiextensions-apiserver v0.25.3/go.mod h1:ZJqwpCkxIx9itilmZek7JgfUAM0dnTsA48I4krPqRmo=
+k8s.io/apimachinery v0.28.2 h1:KCOJLrc6gu+wV1BYgwik4AF4vXOlVJPdiqn0yAWWwXQ=
+k8s.io/apimachinery v0.28.2/go.mod h1:RdzF87y/ngqk9H4z3EL2Rppv5jj95vGS/HaFXrLDApU=
+k8s.io/cli-runtime v0.25.11 h1:GE2yNZm1tN+MJtw1SGMOLesLF7Kp7NVAVqRSTbXfu4o=
+k8s.io/cli-runtime v0.25.11/go.mod h1:r/nEINuHVEpgGhcd2WamU7hD1t/lMnSz8XM44Autltc=
+k8s.io/client-go v0.28.2 h1:DNoYI1vGq0slMBN/SWKMZMw0Rq+0EQW6/AK4v9+3VeY=
+k8s.io/client-go v0.28.2/go.mod h1:sMkApowspLuc7omj1FOSUxSoqjr+d5Q0Yc0LOFnYFJY=
+k8s.io/component-base v0.26.2 h1:IfWgCGUDzrD6wLLgXEstJKYZKAFS2kO+rBRi0p3LqcI=
+k8s.io/component-base v0.26.2/go.mod h1:DxbuIe9M3IZPRxPIzhch2m1eT7uFrSBJUBuVCQEBivs=
+k8s.io/klog v1.0.0 h1:Pt+yjF5aB1xDSVbau4VsWe+dQNzA0qv1LlXdC2dF6Q8=
+k8s.io/klog/v2 v2.100.1 h1:7WCHKK6K8fNhTqfBhISHQ97KrnJNFZMcQvKp7gP/tmg=
+k8s.io/klog/v2 v2.100.1/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0=
+k8s.io/kube-openapi v0.0.0-20230717233707-2695361300d9 h1:LyMgNKD2P8Wn1iAwQU5OhxCKlKJy0sHc+PcDwFB24dQ=
+k8s.io/kube-openapi v0.0.0-20230717233707-2695361300d9/go.mod h1:wZK2AVp1uHCp4VamDVgBP2COHZjqD1T68Rf0CM3YjSM=
+k8s.io/kubectl v0.25.11 h1:6bsft5Gan6BCvQ7cJbDRFjTm4Zfq8GuUYpsWAdVngYE=
+k8s.io/kubectl v0.25.11/go.mod h1:8mIfgkFgT+yJ8/TlmPW1qoRh46H2si9q5nW8id7i9iM=
+k8s.io/utils v0.0.0-20230711102312-30195339c3c7 h1:ZgnF1KZsYxWIifwSNZFZgNtWE89WI5yiP5WwlfDoIyc=
+k8s.io/utils v0.0.0-20230711102312-30195339c3c7/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0=
 lukechampine.com/uint128 v1.3.0 h1:cDdUVfRwDUDovz610ABgFD17nXD4/uDgVHl2sC3+sbo=
 lukechampine.com/uint128 v1.3.0/go.mod h1:c4eWIwlEGaxC/+H1VguhU4PHXNWDCDMUlWdIWl2j1gk=
 modernc.org/cc/v3 v3.41.0 h1:QoR1Sn3YWlmA1T4vLaKZfawdVtSiGx8H+cEojbC7v1Q=
@@ -2002,5 +2155,15 @@ rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0=
 rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA=
 rsc.io/tmplfunc v0.0.3 h1:53XFQh69AfOa8Tw0Jm7t+GV7KZhOi6jzsCzTtKbMvzU=
 rsc.io/tmplfunc v0.0.3/go.mod h1:AG3sTPzElb1Io3Yg4voV9AGZJuleGAwaVRxL9M49PhA=
+sigs.k8s.io/controller-runtime v0.13.0 h1:iqa5RNciy7ADWnIc8QxCbOX5FEKVR3uxVxKHRMc2WIQ=
+sigs.k8s.io/controller-runtime v0.13.0/go.mod h1:Zbz+el8Yg31jubvAEyglRZGdLAjplZl+PgtYNI6WNTI=
+sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd h1:EDPBXCAspyGV4jQlpZSudPeMmr1bNJefnuqLsRAsHZo=
+sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd/go.mod h1:B8JuhiUyNFVKdsE8h686QcCxMaH6HrOAZj4vswFpcB0=
+sigs.k8s.io/kustomize/api v0.12.1 h1:7YM7gW3kYBwtKvoY216ZzY+8hM+lV53LUayghNRJ0vM=
+sigs.k8s.io/kustomize/api v0.12.1/go.mod h1:y3JUhimkZkR6sbLNwfJHxvo1TCLwuwm14sCYnkH6S1s=
+sigs.k8s.io/kustomize/kyaml v0.13.9 h1:Qz53EAaFFANyNgyOEJbT/yoIHygK40/ZcvU3rgry2Tk=
+sigs.k8s.io/kustomize/kyaml v0.13.9/go.mod h1:QsRbD0/KcU+wdk0/L0fIp2KLnohkVzs6fQ85/nOXac4=
+sigs.k8s.io/structured-merge-diff/v4 v4.3.0 h1:UZbZAZfX0wV2zr7YZorDz6GXROfDFj6LvqCRm4VUVKk=
+sigs.k8s.io/structured-merge-diff/v4 v4.3.0/go.mod h1:N8hJocpFajUSSeSJ9bOZ77VzejKZaXsTtZo4/u7Io08=
 sigs.k8s.io/yaml v1.4.0 h1:Mk1wCc2gy/F0THH0TAp1QYyJNzRm2KCLy3o5ASXVI5E=
 sigs.k8s.io/yaml v1.4.0/go.mod h1:Ejl7/uTz7PSA4eKMyQCUTnhZYNmLIl+5c2lQPGR2BPY=
diff --git a/.github/scripts/e2e_tests_tool/cmd/check_tests_cmd.go b/integration-tests/e2e_tests_ci_tool/cmd/check_tests_cmd.go
similarity index 100%
rename from .github/scripts/e2e_tests_tool/cmd/check_tests_cmd.go
rename to integration-tests/e2e_tests_ci_tool/cmd/check_tests_cmd.go
diff --git a/.github/scripts/e2e_tests_tool/cmd/check_tests_cmd_test.go b/integration-tests/e2e_tests_ci_tool/cmd/check_tests_cmd_test.go
similarity index 100%
rename from .github/scripts/e2e_tests_tool/cmd/check_tests_cmd_test.go
rename to integration-tests/e2e_tests_ci_tool/cmd/check_tests_cmd_test.go
diff --git a/integration-tests/e2e_tests_ci_tool/cmd/create_test_config_override.go b/integration-tests/e2e_tests_ci_tool/cmd/create_test_config_override.go
new file mode 100644
index 00000000000..2bb1e33f7b8
--- /dev/null
+++ b/integration-tests/e2e_tests_ci_tool/cmd/create_test_config_override.go
@@ -0,0 +1,300 @@
+package cmd
+
+import (
+	"fmt"
+	"os"
+	"regexp"
+	"strings"
+
+	"github.com/pelletier/go-toml/v2"
+	ctf_config "github.com/smartcontractkit/chainlink-testing-framework/config"
+	"github.com/spf13/cobra"
+)
+
+// OverrideConfig holds the configuration data for overrides
+type OverrideConfig struct {
+	ChainlinkImage                           string
+	ChainlinkVersion                         string
+	ChainlinkPostgresVersion                 string
+	SelectedNetworks                         []string
+	PyroscopeEnabled                         bool
+	PyroscopeServerURL                       string
+	PyroscopeEnvironment                     string
+	PyroscopeKey                             string
+	LoggingTestLogCollect                    bool
+	LoggingRunID                             string
+	LoggingLogTargets                        []string
+	LoggingLokiTenantID                      string
+	LoggingLokiEndpoint                      string
+	LoggingLokiBasicAuth                     string
+	LoggingGrafanaBaseURL                    string
+	LoggingGrafanaDashboardURL               string
+	LoggingGrafanaToken                      string
+	PrivateEthereumNetworkExecutionLayer     string
+	PrivateEthereumNetworkEthereumVersion    string
+	PrivateEthereumNetworkCustomDockerImages string
+}
+
+var oc OverrideConfig
+
+var createTestConfigOverrideCmd = &cobra.Command{
+	Use:   "create-test-config-override",
+	Short: "Processes configurations and outputs templated results for test config overrides",
+	Run: func(cmd *cobra.Command, args []string) {
+		var config ctf_config.TestConfig
+
+		var image, version, postgresVersion *string
+		if cmd.Flags().Changed("chainlink-image") {
+			image = &oc.ChainlinkImage
+		}
+		if cmd.Flags().Changed("chainlink-version") {
+			version = &oc.ChainlinkVersion
+		}
+		if cmd.Flags().Changed("chainlink-postgres-version") {
+			version = &oc.ChainlinkPostgresVersion
+		}
+		if image != nil && version == nil || image == nil && version != nil {
+			fmt.Fprintf(os.Stderr, "Error: both chainlink-image and chainlink-version must be set\n")
+			os.Exit(1)
+		}
+		if image != nil && version != nil {
+			config.ChainlinkImage = &ctf_config.ChainlinkImageConfig{
+				Image:           image,
+				Version:         version,
+				PostgresVersion: postgresVersion,
+			}
+		}
+
+		var selectedNetworks *[]string
+		if cmd.Flags().Changed("selected-networks") {
+			selectedNetworks = &oc.SelectedNetworks
+		}
+		if selectedNetworks != nil {
+			config.Network = &ctf_config.NetworkConfig{
+				SelectedNetworks: oc.SelectedNetworks,
+			}
+		}
+
+		var peryscopeEnabled *bool
+		var pyroscopeServerURL, pyroscopeEnvironment, pyroscopeKey *string
+		if cmd.Flags().Changed("pyroscope-enabled") {
+			peryscopeEnabled = &oc.PyroscopeEnabled
+		}
+		if cmd.Flags().Changed("peryscope-server-url") {
+			pyroscopeServerURL = &oc.PyroscopeServerURL
+		}
+		if cmd.Flags().Changed("peryscope-server-key") {
+			pyroscopeKey = &oc.PyroscopeKey
+		}
+		if cmd.Flags().Changed("peryscope-environment") {
+			pyroscopeEnvironment = &oc.PyroscopeEnvironment
+		}
+		if peryscopeEnabled != nil {
+			config.Pyroscope = &ctf_config.PyroscopeConfig{
+				Enabled:     peryscopeEnabled,
+				ServerUrl:   pyroscopeServerURL,
+				Environment: pyroscopeEnvironment,
+				Key:         pyroscopeKey,
+			}
+		}
+
+		var testLogCollect *bool
+		if cmd.Flags().Changed("logging-test-log-collect") {
+			testLogCollect = &oc.LoggingTestLogCollect
+		}
+		var loggingRunID *string
+		if cmd.Flags().Changed("logging-run-id") {
+			loggingRunID = &oc.LoggingRunID
+		}
+		var loggingLogTargets []string
+		if cmd.Flags().Changed("logging-log-targets") {
+			loggingLogTargets = oc.LoggingLogTargets
+		}
+		var loggingLokiTenantID *string
+		if cmd.Flags().Changed("logging-loki-tenant-id") {
+			loggingLokiTenantID = &oc.LoggingLokiTenantID
+		}
+		var loggingLokiBasicAuth *string
+		if cmd.Flags().Changed("logging-loki-basic-auth") {
+			loggingLokiBasicAuth = &oc.LoggingLokiBasicAuth
+		}
+		var loggingLokiEndpoint *string
+		if cmd.Flags().Changed("logging-loki-endpoint") {
+			loggingLokiEndpoint = &oc.LoggingLokiEndpoint
+		}
+		var loggingGrafanaBaseURL *string
+		if cmd.Flags().Changed("logging-grafana-base-url") {
+			loggingGrafanaBaseURL = &oc.LoggingGrafanaBaseURL
+		}
+		var loggingGrafanaDashboardURL *string
+		if cmd.Flags().Changed("logging-grafana-dashboard-url") {
+			loggingGrafanaDashboardURL = &oc.LoggingGrafanaDashboardURL
+		}
+		var loggingGrafanaToken *string
+		if cmd.Flags().Changed("logging-grafana-token") {
+			loggingGrafanaToken = &oc.LoggingGrafanaToken
+		}
+
+		if testLogCollect != nil || loggingRunID != nil || loggingLogTargets != nil || loggingLokiEndpoint != nil || loggingLokiTenantID != nil || loggingLokiBasicAuth != nil || loggingGrafanaBaseURL != nil || loggingGrafanaDashboardURL != nil || loggingGrafanaToken != nil {
+			config.Logging = &ctf_config.LoggingConfig{}
+			config.Logging.TestLogCollect = testLogCollect
+			config.Logging.RunId = loggingRunID
+			if loggingLogTargets != nil {
+				config.Logging.LogStream = &ctf_config.LogStreamConfig{
+					LogTargets: loggingLogTargets,
+				}
+			}
+			if loggingLokiTenantID != nil || loggingLokiBasicAuth != nil || loggingLokiEndpoint != nil {
+				config.Logging.Loki = &ctf_config.LokiConfig{
+					TenantId:  loggingLokiTenantID,
+					BasicAuth: loggingLokiBasicAuth,
+					Endpoint:  loggingLokiEndpoint,
+				}
+			}
+			if loggingGrafanaBaseURL != nil || loggingGrafanaDashboardURL != nil || loggingGrafanaToken != nil {
+				config.Logging.Grafana = &ctf_config.GrafanaConfig{
+					BaseUrl:      loggingGrafanaBaseURL,
+					DashboardUrl: loggingGrafanaDashboardURL,
+					BearerToken:  loggingGrafanaToken,
+				}
+			}
+		}
+
+		var privateEthereumNetworkExecutionLayer *string
+		if cmd.Flags().Changed("private-ethereum-network-execution-layer") {
+			privateEthereumNetworkExecutionLayer = &oc.PrivateEthereumNetworkExecutionLayer
+		}
+		var privateEthereumNetworkEthereumVersion *string
+		if cmd.Flags().Changed("private-ethereum-network-ethereum-version") {
+			privateEthereumNetworkEthereumVersion = &oc.PrivateEthereumNetworkEthereumVersion
+		}
+		var privateEthereumNetworkCustomDockerImage *string
+		if cmd.Flags().Changed("private-ethereum-network-custom-docker-image") {
+			privateEthereumNetworkCustomDockerImage = &oc.PrivateEthereumNetworkCustomDockerImages
+		}
+		if privateEthereumNetworkExecutionLayer != nil || privateEthereumNetworkEthereumVersion != nil || privateEthereumNetworkCustomDockerImage != nil {
+			var el ctf_config.ExecutionLayer
+			if privateEthereumNetworkExecutionLayer != nil {
+				el = ctf_config.ExecutionLayer(*privateEthereumNetworkExecutionLayer)
+			}
+			var ev ctf_config.EthereumVersion
+			if privateEthereumNetworkEthereumVersion != nil {
+				ev = ctf_config.EthereumVersion(*privateEthereumNetworkEthereumVersion)
+			}
+			var customImages map[ctf_config.ContainerType]string
+			if privateEthereumNetworkCustomDockerImage != nil {
+				customImages = map[ctf_config.ContainerType]string{"execution_layer": *privateEthereumNetworkCustomDockerImage}
+			}
+			config.PrivateEthereumNetwork = &ctf_config.EthereumNetworkConfig{
+				ExecutionLayer:     &el,
+				EthereumVersion:    &ev,
+				CustomDockerImages: customImages,
+			}
+		}
+
+		configToml, err := toml.Marshal(config)
+		if err != nil {
+			fmt.Fprintf(os.Stderr, "Error marshalling TestConfig to TOML: %v\n", err)
+			os.Exit(1)
+		}
+
+		fmt.Println(string(configToml))
+	},
+}
+
+var maskTestConfigOverrideCmd = &cobra.Command{
+	Use:   "mask-test-config-override",
+	Short: "Processes configurations and outputs templated results for test config overrides",
+	Run: func(cmd *cobra.Command, args []string) {
+		maskSecret("chainlink image", oc.ChainlinkImage)
+		maskSecret("pyroscope server url", oc.PyroscopeServerURL)
+		maskSecret("pyroscope key", oc.PyroscopeKey)
+		maskSecret("loki tenant id", oc.LoggingLokiTenantID)
+		maskSecret("loki endpoint", oc.LoggingLokiEndpoint)
+		maskSecret("loki basic auth", oc.LoggingLokiBasicAuth)
+		maskSecret("loki grafana token", oc.LoggingGrafanaToken)
+	},
+}
+
+func maskSecret(description string, secret string) {
+	if secret != "" {
+		fmt.Printf("# Mask '%s' value\n", description)
+		fmt.Printf("echo ::add-mask::%s\n", secret)
+	}
+}
+
+func init() {
+	cmds := []*cobra.Command{createTestConfigOverrideCmd, maskTestConfigOverrideCmd}
+	for _, c := range cmds {
+		c.Flags().StringArrayVar(&oc.SelectedNetworks, "selected-networks", nil, "Selected networks")
+		c.Flags().StringVar(&oc.ChainlinkImage, "chainlink-image", "", "Chainlink image")
+		c.Flags().StringVar(&oc.ChainlinkVersion, "chainlink-version", "", "Chainlink version")
+		c.Flags().StringVar(&oc.ChainlinkPostgresVersion, "chainlink-postgres-version", "", "Chainlink Postgres version")
+		c.Flags().BoolVar(&oc.PyroscopeEnabled, "pyroscope-enabled", false, "Pyroscope enabled")
+		c.Flags().StringVar(&oc.PyroscopeServerURL, "pyroscope-server-url", "", "Pyroscope server URL")
+		c.Flags().StringVar(&oc.PyroscopeKey, "pyroscope-key", "", "Pyroscope key")
+		c.Flags().StringVar(&oc.PyroscopeEnvironment, "pyroscope-environment", "", "Pyroscope environment")
+		c.Flags().BoolVar(&oc.LoggingTestLogCollect, "logging-test-log-collect", false, "Test log collect")
+		c.Flags().StringVar(&oc.LoggingRunID, "logging-run-id", "", "Run ID")
+		c.Flags().StringArrayVar(&oc.LoggingLogTargets, "logging-log-targets", nil, "Logging.LogStream.LogTargets")
+		c.Flags().StringVar(&oc.LoggingLokiEndpoint, "logging-loki-endpoint", "", "")
+		c.Flags().StringVar(&oc.LoggingLokiTenantID, "logging-loki-tenant-id", "", "")
+		c.Flags().StringVar(&oc.LoggingLokiBasicAuth, "logging-loki-basic-auth", "", "")
+		c.Flags().StringVar(&oc.LoggingGrafanaBaseURL, "logging-grafana-base-url", "", "")
+		c.Flags().StringVar(&oc.LoggingGrafanaDashboardURL, "logging-grafana-dashboard-url", "", "")
+		c.Flags().StringVar(&oc.LoggingGrafanaToken, "logging-grafana-token", "", "")
+		c.Flags().StringVar(&oc.PrivateEthereumNetworkExecutionLayer, "private-ethereum-network-execution-layer", "", "")
+		c.Flags().StringVar(&oc.PrivateEthereumNetworkEthereumVersion, "private-ethereum-network-ethereum-version", "", "")
+		c.Flags().StringVar(&oc.PrivateEthereumNetworkCustomDockerImages, "private-ethereum-network-custom-docker-image", "", "")
+
+		c.PreRun = func(cmd *cobra.Command, args []string) {
+			// Resolve selected networks environment variable if set
+			_, hasEnvVar := lookupEnvVarName(oc.SelectedNetworks[0])
+			if hasEnvVar {
+				selectedNetworks := mustResolveEnvPlaceholder(oc.SelectedNetworks[0])
+				oc.SelectedNetworks = strings.Split(selectedNetworks, ",")
+			}
+
+			// Resolve all other environment variables
+			oc.ChainlinkImage = mustResolveEnvPlaceholder(oc.ChainlinkImage)
+			oc.ChainlinkVersion = mustResolveEnvPlaceholder(oc.ChainlinkVersion)
+			oc.ChainlinkPostgresVersion = mustResolveEnvPlaceholder(oc.ChainlinkPostgresVersion)
+			oc.PyroscopeServerURL = mustResolveEnvPlaceholder(oc.PyroscopeServerURL)
+			oc.PyroscopeKey = mustResolveEnvPlaceholder(oc.PyroscopeKey)
+			oc.PyroscopeEnvironment = mustResolveEnvPlaceholder(oc.PyroscopeEnvironment)
+			oc.LoggingRunID = mustResolveEnvPlaceholder(oc.LoggingRunID)
+			oc.LoggingLokiTenantID = mustResolveEnvPlaceholder(oc.LoggingLokiTenantID)
+			oc.LoggingLokiEndpoint = mustResolveEnvPlaceholder(oc.LoggingLokiEndpoint)
+			oc.LoggingLokiBasicAuth = mustResolveEnvPlaceholder(oc.LoggingLokiBasicAuth)
+			oc.LoggingGrafanaBaseURL = mustResolveEnvPlaceholder(oc.LoggingGrafanaBaseURL)
+			oc.LoggingGrafanaDashboardURL = mustResolveEnvPlaceholder(oc.LoggingGrafanaDashboardURL)
+			oc.LoggingGrafanaToken = mustResolveEnvPlaceholder(oc.LoggingGrafanaToken)
+			oc.PrivateEthereumNetworkExecutionLayer = mustResolveEnvPlaceholder(oc.PrivateEthereumNetworkExecutionLayer)
+			oc.PrivateEthereumNetworkEthereumVersion = mustResolveEnvPlaceholder(oc.PrivateEthereumNetworkEthereumVersion)
+			oc.PrivateEthereumNetworkCustomDockerImages = mustResolveEnvPlaceholder(oc.PrivateEthereumNetworkCustomDockerImages)
+		}
+	}
+}
+
+// mustResolveEnvPlaceholder checks if the input string is an environment variable placeholder and resolves it.
+func mustResolveEnvPlaceholder(input string) string {
+	envVarName, hasEnvVar := lookupEnvVarName(input)
+	if hasEnvVar {
+		value, set := os.LookupEnv(envVarName)
+		if !set {
+			fmt.Fprintf(os.Stderr, "Error resolving '%s'. Environment variable '%s' not set or is empty\n", input, envVarName)
+			os.Exit(1)
+		}
+		return value
+	}
+	return input
+}
+
+func lookupEnvVarName(input string) (string, bool) {
+	re := regexp.MustCompile(`^\${{ env\.([a-zA-Z_]+) }}$`)
+	matches := re.FindStringSubmatch(input)
+	if len(matches) > 1 {
+		return matches[1], true
+	}
+	return "", false
+}
diff --git a/.github/scripts/e2e_tests_tool/cmd/envreplace_cmd.go b/integration-tests/e2e_tests_ci_tool/cmd/envreplace_cmd.go
similarity index 100%
rename from .github/scripts/e2e_tests_tool/cmd/envreplace_cmd.go
rename to integration-tests/e2e_tests_ci_tool/cmd/envreplace_cmd.go
diff --git a/.github/scripts/e2e_tests_tool/cmd/envreplace_cmd_test.go b/integration-tests/e2e_tests_ci_tool/cmd/envreplace_cmd_test.go
similarity index 100%
rename from .github/scripts/e2e_tests_tool/cmd/envreplace_cmd_test.go
rename to integration-tests/e2e_tests_ci_tool/cmd/envreplace_cmd_test.go
diff --git a/.github/scripts/e2e_tests_tool/cmd/filter_cmd.go b/integration-tests/e2e_tests_ci_tool/cmd/filter_cmd.go
similarity index 100%
rename from .github/scripts/e2e_tests_tool/cmd/filter_cmd.go
rename to integration-tests/e2e_tests_ci_tool/cmd/filter_cmd.go
diff --git a/.github/scripts/e2e_tests_tool/cmd/filter_cmd_test.go b/integration-tests/e2e_tests_ci_tool/cmd/filter_cmd_test.go
similarity index 100%
rename from .github/scripts/e2e_tests_tool/cmd/filter_cmd_test.go
rename to integration-tests/e2e_tests_ci_tool/cmd/filter_cmd_test.go
diff --git a/.github/scripts/e2e_tests_tool/cmd/root_cmd.go b/integration-tests/e2e_tests_ci_tool/cmd/root_cmd.go
similarity index 87%
rename from .github/scripts/e2e_tests_tool/cmd/root_cmd.go
rename to integration-tests/e2e_tests_ci_tool/cmd/root_cmd.go
index ad10523a49e..07b547173af 100644
--- a/.github/scripts/e2e_tests_tool/cmd/root_cmd.go
+++ b/integration-tests/e2e_tests_ci_tool/cmd/root_cmd.go
@@ -27,4 +27,6 @@ func init() {
 	rootCmd.AddCommand(checkTestsCmd)
 	rootCmd.AddCommand(filterCmd)
 	rootCmd.AddCommand(envreplaceCmd)
+	rootCmd.AddCommand(createTestConfigOverrideCmd)
+	rootCmd.AddCommand(maskTestConfigOverrideCmd)
 }
diff --git a/.github/scripts/e2e_tests_tool/cmd/types.go b/integration-tests/e2e_tests_ci_tool/cmd/types.go
similarity index 100%
rename from .github/scripts/e2e_tests_tool/cmd/types.go
rename to integration-tests/e2e_tests_ci_tool/cmd/types.go
diff --git a/integration-tests/e2e_tests_ci_tool/main.go b/integration-tests/e2e_tests_ci_tool/main.go
new file mode 100644
index 00000000000..595cc085722
--- /dev/null
+++ b/integration-tests/e2e_tests_ci_tool/main.go
@@ -0,0 +1,9 @@
+package main
+
+import (
+	"github.com/smartcontractkit/chainlink/integration-tests/e2e_tests_ci_tool/cmd"
+)
+
+func main() {
+	cmd.Execute()
+}
diff --git a/integration-tests/go.mod b/integration-tests/go.mod
index 62b7c07ba63..15b41fec4d9 100644
--- a/integration-tests/go.mod
+++ b/integration-tests/go.mod
@@ -29,7 +29,7 @@ require (
 	github.com/slack-go/slack v0.12.2
 	github.com/smartcontractkit/chainlink-automation v1.0.4
 	github.com/smartcontractkit/chainlink-common v0.1.7-0.20240620164046-1c03d0a97b20
-	github.com/smartcontractkit/chainlink-testing-framework v1.31.1
+	github.com/smartcontractkit/chainlink-testing-framework v1.31.4
 	github.com/smartcontractkit/chainlink-testing-framework/grafana v0.0.0-20240405215812-5a72bc9af239
 	github.com/smartcontractkit/chainlink-vrf v0.0.0-20231120191722-fef03814f868
 	github.com/smartcontractkit/chainlink/v2 v2.0.0-00010101000000-000000000000
@@ -47,6 +47,8 @@ require (
 	golang.org/x/sync v0.6.0
 	golang.org/x/text v0.14.0
 	gopkg.in/guregu/null.v4 v4.0.0
+	gopkg.in/yaml.v2 v2.4.0
+	gopkg.in/yaml.v3 v3.0.1
 	k8s.io/apimachinery v0.28.2
 )
 
@@ -469,8 +471,6 @@ require (
 	gopkg.in/inf.v0 v0.9.1 // indirect
 	gopkg.in/ini.v1 v1.67.0 // indirect
 	gopkg.in/natefinch/lumberjack.v2 v2.2.1 // indirect
-	gopkg.in/yaml.v2 v2.4.0 // indirect
-	gopkg.in/yaml.v3 v3.0.1 // indirect
 	k8s.io/api v0.28.2 // indirect
 	k8s.io/apiextensions-apiserver v0.25.3 // indirect
 	k8s.io/cli-runtime v0.25.11 // indirect
diff --git a/integration-tests/go.sum b/integration-tests/go.sum
index c4b175e85fb..fce0a83cf12 100644
--- a/integration-tests/go.sum
+++ b/integration-tests/go.sum
@@ -1526,8 +1526,8 @@ github.com/smartcontractkit/chainlink-solana v1.0.3-0.20240605170242-555ff582f36
 github.com/smartcontractkit/chainlink-solana v1.0.3-0.20240605170242-555ff582f36a/go.mod h1:QqcZSwLgEIn7YraAIRmomnBMAuVFephiHrIWVlkWbFI=
 github.com/smartcontractkit/chainlink-starknet/relayer v0.0.1-beta-test.0.20240531021326-99118e47f696 h1:h1E87+z+JcUEfvbJVF56SnZA/YUFE5ewUE61MaR/Ewg=
 github.com/smartcontractkit/chainlink-starknet/relayer v0.0.1-beta-test.0.20240531021326-99118e47f696/go.mod h1:OiWUTrrpSLLTMh7FINWjEh6mmDJCVPaC4yEsDCVaWdU=
-github.com/smartcontractkit/chainlink-testing-framework v1.31.1 h1:Qqo5VngCqbHQPfQKZneAN0L1dWXOWWd074Oo9Bex/Q0=
-github.com/smartcontractkit/chainlink-testing-framework v1.31.1/go.mod h1:E6uNEZhZZid9PHv6/Kq5Vn63GlO61ZcKB+/f0DKo3Q4=
+github.com/smartcontractkit/chainlink-testing-framework v1.31.4 h1:5/zF5Z2w+/HTvUSSJNhWSiHvlQ2O2qxRw4e1fBbVraQ=
+github.com/smartcontractkit/chainlink-testing-framework v1.31.4/go.mod h1:E6uNEZhZZid9PHv6/Kq5Vn63GlO61ZcKB+/f0DKo3Q4=
 github.com/smartcontractkit/chainlink-testing-framework/grafana v0.0.0-20240405215812-5a72bc9af239 h1:Kk5OVlx/5g9q3Z3lhxytZS4/f8ds1MiNM8yaHgK3Oe8=
 github.com/smartcontractkit/chainlink-testing-framework/grafana v0.0.0-20240405215812-5a72bc9af239/go.mod h1:DC8sQMyTlI/44UCTL8QWFwb0bYNoXCfjwCv2hMivYZU=
 github.com/smartcontractkit/chainlink-vrf v0.0.0-20231120191722-fef03814f868 h1:FFdvEzlYwcuVHkdZ8YnZR/XomeMGbz5E2F2HZI3I3w8=
diff --git a/integration-tests/load/go.mod b/integration-tests/load/go.mod
index 979a4cff1df..1adf5681116 100644
--- a/integration-tests/load/go.mod
+++ b/integration-tests/load/go.mod
@@ -17,7 +17,7 @@ require (
 	github.com/slack-go/slack v0.12.2
 	github.com/smartcontractkit/chainlink-automation v1.0.4
 	github.com/smartcontractkit/chainlink-common v0.1.7-0.20240620164046-1c03d0a97b20
-	github.com/smartcontractkit/chainlink-testing-framework v1.31.1
+	github.com/smartcontractkit/chainlink-testing-framework v1.31.4
 	github.com/smartcontractkit/chainlink/integration-tests v0.0.0-20240214231432-4ad5eb95178c
 	github.com/smartcontractkit/chainlink/v2 v2.9.0-beta0.0.20240216210048-da02459ddad8
 	github.com/smartcontractkit/libocr v0.0.0-20240419185742-fd3cab206b2c
diff --git a/integration-tests/load/go.sum b/integration-tests/load/go.sum
index 708bc075206..5d2ffc169e0 100644
--- a/integration-tests/load/go.sum
+++ b/integration-tests/load/go.sum
@@ -1516,8 +1516,8 @@ github.com/smartcontractkit/chainlink-solana v1.0.3-0.20240605170242-555ff582f36
 github.com/smartcontractkit/chainlink-solana v1.0.3-0.20240605170242-555ff582f36a/go.mod h1:QqcZSwLgEIn7YraAIRmomnBMAuVFephiHrIWVlkWbFI=
 github.com/smartcontractkit/chainlink-starknet/relayer v0.0.1-beta-test.0.20240531021326-99118e47f696 h1:h1E87+z+JcUEfvbJVF56SnZA/YUFE5ewUE61MaR/Ewg=
 github.com/smartcontractkit/chainlink-starknet/relayer v0.0.1-beta-test.0.20240531021326-99118e47f696/go.mod h1:OiWUTrrpSLLTMh7FINWjEh6mmDJCVPaC4yEsDCVaWdU=
-github.com/smartcontractkit/chainlink-testing-framework v1.31.1 h1:Qqo5VngCqbHQPfQKZneAN0L1dWXOWWd074Oo9Bex/Q0=
-github.com/smartcontractkit/chainlink-testing-framework v1.31.1/go.mod h1:E6uNEZhZZid9PHv6/Kq5Vn63GlO61ZcKB+/f0DKo3Q4=
+github.com/smartcontractkit/chainlink-testing-framework v1.31.4 h1:5/zF5Z2w+/HTvUSSJNhWSiHvlQ2O2qxRw4e1fBbVraQ=
+github.com/smartcontractkit/chainlink-testing-framework v1.31.4/go.mod h1:E6uNEZhZZid9PHv6/Kq5Vn63GlO61ZcKB+/f0DKo3Q4=
 github.com/smartcontractkit/chainlink-testing-framework/grafana v0.0.0-20240405215812-5a72bc9af239 h1:Kk5OVlx/5g9q3Z3lhxytZS4/f8ds1MiNM8yaHgK3Oe8=
 github.com/smartcontractkit/chainlink-testing-framework/grafana v0.0.0-20240405215812-5a72bc9af239/go.mod h1:DC8sQMyTlI/44UCTL8QWFwb0bYNoXCfjwCv2hMivYZU=
 github.com/smartcontractkit/chainlink-vrf v0.0.0-20231120191722-fef03814f868 h1:FFdvEzlYwcuVHkdZ8YnZR/XomeMGbz5E2F2HZI3I3w8=

From b1f617be4b5862ed4ceaaccae804c4614948017a Mon Sep 17 00:00:00 2001
From: lukaszcl <120112546+lukaszcl@users.noreply.github.com>
Date: Wed, 26 Jun 2024 12:56:32 +0200
Subject: [PATCH 068/137] Add CI test config override tools

---
 .github/e2e-tests.yml                         |   8 +-
 .../run-e2e-tests-reusable-workflow.yml       | 136 +++++---
 .github/workflows/run-selected-e2e-tests.yml  |   5 +
 .../cmd/create_test_config_cmd.go             | 175 ++++++++++
 .../cmd/create_test_config_override.go        | 300 -----------------
 .../e2e_tests_ci_tool/cmd/filter_cmd.go       |   6 +-
 .../e2e_tests_ci_tool/cmd/filter_cmd_test.go  |   4 +-
 .../cmd/mask_test_config_cmd.go               |  40 +++
 .../cmd/override_test_config_cmd.go           | 304 ++++++++++++++++++
 .../e2e_tests_ci_tool/cmd/root_cmd.go         |   6 +-
 .../e2e_tests_ci_tool/cmd/test_config_cmd.go  | 143 ++++++++
 .../e2e_tests_ci_tool/cmd/types.go            |  27 +-
 12 files changed, 788 insertions(+), 366 deletions(-)
 create mode 100644 integration-tests/e2e_tests_ci_tool/cmd/create_test_config_cmd.go
 delete mode 100644 integration-tests/e2e_tests_ci_tool/cmd/create_test_config_override.go
 create mode 100644 integration-tests/e2e_tests_ci_tool/cmd/mask_test_config_cmd.go
 create mode 100644 integration-tests/e2e_tests_ci_tool/cmd/override_test_config_cmd.go
 create mode 100644 integration-tests/e2e_tests_ci_tool/cmd/test_config_cmd.go

diff --git a/.github/e2e-tests.yml b/.github/e2e-tests.yml
index 960dad5485a..ca701dfee44 100644
--- a/.github/e2e-tests.yml
+++ b/.github/e2e-tests.yml
@@ -12,6 +12,8 @@ runner-test-matrix:
     path: integration-tests/smoke/ocr_test.go
     test-type: docker
     runs-on: ubuntu-latest
+    test-config-override:
+      chainlink-version: mycustomversion    
     workflows:
       - Run PR E2E Tests
       - Run Nightly E2E Tests
@@ -48,7 +50,7 @@ runner-test-matrix:
     workflows:
       - Run Automation Product Nightly E2E Tests
     test-cmd: cd integration-tests/smoke && go test -test.run ^TestAutomationNodeUpgrade/registry_2_0 -test.parallel=1 -timeout 60m -count=1 -json
-    test-config-overrides:
+    test-config-override:
       chainlink-upgrade-image: ${{ env.CHAINLINK_IMAGE }}
       chainlink-upgrade-version: develop
       chainlink-image: public.ecr.aws/chainlink/chainlink
@@ -62,7 +64,7 @@ runner-test-matrix:
     workflows:
       - Run Automation Product Nightly E2E Tests    
     test-cmd: cd integration-tests/smoke && go test -test.run ^TestAutomationNodeUpgrade/registry_2_1 -test.parallel=5 -timeout 60m -count=1 -json
-    test-config-overrides:
+    test-config-override:
       chainlink-upgrade-image: ${{ env.CHAINLINK_IMAGE }}
       chainlink-upgrade-version: develop
       chainlink-image: public.ecr.aws/chainlink/chainlink
@@ -76,7 +78,7 @@ runner-test-matrix:
     workflows:
       - Run Automation Product Nightly E2E Tests
     test-cmd: cd integration-tests/smoke && go test -test.run ^TestAutomationNodeUpgrade/registry_2_2 -test.parallel=5 -timeout 60m -count=1 -json
-    test-config-overrides:
+    test-config-override:
       chainlink-upgrade-image: ${{ env.CHAINLINK_IMAGE }}
       chainlink-upgrade-version: develop
       chainlink-image: public.ecr.aws/chainlink/chainlink
diff --git a/.github/workflows/run-e2e-tests-reusable-workflow.yml b/.github/workflows/run-e2e-tests-reusable-workflow.yml
index ad1890276b8..0191b1fffb4 100644
--- a/.github/workflows/run-e2e-tests-reusable-workflow.yml
+++ b/.github/workflows/run-e2e-tests-reusable-workflow.yml
@@ -17,7 +17,7 @@ on:
         required: false
         type: string        
       test-config-base64-override:
-        description: 'Custom base64 test config override'
+        description: 'Custom base64 test config'
         required: false
         type: string        
       enable-check-test-configurations:
@@ -88,7 +88,7 @@ jobs:
           go-version: '1.21.7'
       - name: Run Check Tests Command
         run: |
-          if ! go run .github/scripts/e2e_tests_tool/main.go check-tests ./integration-tests .github/e2e-tests.yml; then
+          if ! go run integration-tests/e2e_tests_ci_tool/main.go check-tests ./integration-tests .github/e2e-tests.yml; then
             echo "::error::Some E2E test configurations have to be added to .github/e2e-tests.yml. This file defines Github CI configuration for each E2E test or set of E2E tests." && exit 1
           fi
 
@@ -113,14 +113,16 @@ jobs:
       - name: Generate Docker Tests Matrix
         id: set-docker-matrix
         run: |
-          MATRIX_JSON=$(go run .github/scripts/e2e_tests_tool/main.go filter --file .github/e2e-tests.yml --test-type 'docker' --test-ids '${{ inputs.test-ids }}' --workflow '${{ inputs.test-workflow }}')
+          cd integration-tests/e2e_tests_ci_tool/
+          MATRIX_JSON=$(go run main.go filter --file ../../.github/e2e-tests.yml --test-type 'docker' --test-ids '${{ inputs.test-ids }}' --workflow '${{ inputs.test-workflow }}')
           echo "Docker tests:"
           echo "$MATRIX_JSON" | jq
           echo "::set-output name=matrix::$MATRIX_JSON"
       - name: Generate K8s Tests Matrix
         id: set-k8s-runner-matrix
         run: |
-          MATRIX_JSON=$(go run .github/scripts/e2e_tests_tool/main.go filter --file .github/e2e-tests.yml --test-type 'k8s-remote-runner' --test-ids '${{ inputs.test-ids }}' --workflow '${{ inputs.test-workflow }}')
+          cd integration-tests/e2e_tests_ci_tool/
+          MATRIX_JSON=$(go run main.go filter --file ../../.github/e2e-tests.yml --test-type 'k8s-remote-runner' --test-ids '${{ inputs.test-ids }}' --workflow '${{ inputs.test-workflow }}')
           echo "K8s tests:"
           echo "$MATRIX_JSON" | jq
           echo "::set-output name=matrix::$MATRIX_JSON"
@@ -155,7 +157,6 @@ jobs:
         shell: bash   
 
   # Run Docker tests
-
   docker-tests:
     name: Run Docker Tests (${{ matrix.tests.id }})
     needs: load-test-configurations
@@ -179,31 +180,10 @@ jobs:
     steps:
       - name: Checkout repository
         uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2
-      - name: Setup GAP for Grafana
-        uses: smartcontractkit/.github/actions/setup-gap@6c9d62fdad050cfb8b59376ded291f1350705944 # setup-gap@0.2.2
-        with:
-          aws-region: ${{ secrets.AWS_REGION }}
-          aws-role-arn: ${{ secrets.AWS_OIDC_IAM_ROLE_VALIDATION_PROD_ARN }}
-          api-gateway-host: ${{ secrets.AWS_API_GW_HOST_GRAFANA }}
-          duplicate-authorization-header: "true"
-      - name: Prepare Base64 TOML override
-        uses: ./.github/actions/setup-create-base64-config
-        with:
-          runId: ${{ github.run_id }}
-          # testLogCollect: ${{ vars.TEST_LOG_COLLECT }}
-          selectedNetworks: ${{ env.SELECTED_NETWORKS }}
-          chainlinkImage: ${{ env.CHAINLINK_IMAGE }}
-          chainlinkVersion: ${{ inputs.chainlink-version }}
-          pyroscopeServer: ${{ secrets.QA_PYROSCOPE_INSTANCE }} # Avoid sending blank envs https://github.com/orgs/community/discussions/25725
-          pyroscopeEnvironment: ${{ matrix.tests.pyroscopeEnv }}
-          pyroscopeKey: ${{ secrets.QA_PYROSCOPE_KEY }}
-          lokiEndpoint: https://${{ secrets.GRAFANA_INTERNAL_HOST }}/loki/api/v1/push
-          lokiTenantId: ${{ secrets.GRAFANA_INTERNAL_TENANT_ID }}
-          lokiBasicAuth: ${{ secrets.GRAFANA_INTERNAL_BASIC_AUTH }}
-          logstreamLogTargets: ${{ vars.LOGSTREAM_LOG_TARGETS }}
-          grafanaUrl: "http://localhost:8080/primary" # This is GAP's address
-          grafanaDashboardUrl: "/d/ddf75041-1e39-42af-aa46-361fe4c36e9e/ci-e2e-tests-logs"
-          grafanaBearerToken: ${{ secrets.GRAFANA_INTERNAL_URL_SHORTENER_TOKEN }}
+      - name: Install jq
+        run: sudo apt-get install -y jq
+      - name: Show test configuration
+        run: echo '${{ toJson(matrix.tests) }}' | jq .
       - name: Set up Go
         uses: actions/setup-go@v2
         with:
@@ -212,27 +192,47 @@ jobs:
       - name: Set default test config override
         if: ${{ inputs.test-config-base64-override == '' }}
         run: |      
-          chainlink_version="${{ github.sha || matrix.tests.testConfigOverride.chainlinkVersion }}"
+          # Mask secret args before using them
+          # For more, see https://docs.github.com/en/actions/using-workflows/workflow-commands-for-github-actions#masking-a-value-in-a-log
+          echo ::add-mask::${{ env.CHAINLINK_IMAGE }}
+          echo "cl image: ${{ env.CHAINLINK_IMAGE }}"
+
+          # Use overrides from e2e-tests.yml or defaults 
+          chainlink_version="${{ matrix.tests.testConfigOverride.chainlinkVersion || github.sha }}"
+          chainlink_image="${{ matrix.tests.testConfigOverride.chainlinkImage || env.CHAINLINK_IMAGE }}"
           chainlink_postgres_version="${{ matrix.tests.testConfigOverride.chainlinkPostgresVersion }}"
           chainlink_upgrade_version="${{ matrix.tests.testConfigOverride.chainlinkUpgradeVersion }}"
-          selected_networks="${{ env.SELECTED_NETWORKS || matrix.tests.testConfigOverride.selectedNetworks }}"
+          selected_networks="${{ matrix.tests.testConfigOverride.selectedNetworks || env.SELECTED_NETWORKS}}"
 
+          # Build the command line arguments
+          # Need to check if the variable is set before adding it to the command
+          # Otherwise "" will be passed as an argument which is not what we want
           cmd_args=""
+          if [ -n "$chainlink_image" ]; then
+            cmd_args+="--chainlink-image=$chainlink_image "
+          fi
           if [ -n "$chainlink_version" ]; then
-            cmd_args+="--chainlink-version=\"$chainlink_version\" "
+            cmd_args+="--chainlink-version=$chainlink_version "
           fi
           if [ -n "$chainlink_postgres_version" ]; then
-            cmd_args+="--chainlink-postgres-version=\"$chainlink_postgres_version\" "
+            cmd_args+="--chainlink-postgres-version=$chainlink_postgres_version "
           fi
           if [ -n "$chainlink_upgrade_version" ]; then
-            cmd_args+="--chainlink-upgrade-version=\"$chainlink_upgrade_version\" "
+            cmd_args+="--chainlink-upgrade-version=$chainlink_upgrade_version "
           fi
           if [ -n "$selected_networks" ]; then
-            cmd_args+="--selected-networks=\"$selected_networks\" "
+            cmd_args+="--selected-networks=$selected_networks "
           fi    
+          
+          cd integration-tests/e2e_tests_ci_tool/
+
+          # Mask secrets inside the test config override (like resolved env vars)
+          go run main.go test-config mask-secrets $cmd_args
+
+          echo $chainlink_version
 
-          config_override=$(go run .github/scripts/e2e_tests_tool/main.go mask-test-config-override $cmd_args)  
-          echo $config_override
+          # Create a base64 encoded string of the test config override
+          config_override=$(go run main.go test-config create $cmd_args)
 
           BASE64_CONFIG_OVERRIDE=$(echo "$config_override" | base64 -w 0)
           echo ::add-mask::$BASE64_CONFIG_OVERRIDE
@@ -241,15 +241,61 @@ jobs:
       - name: Set custom test config override
         if: ${{ inputs.test-config-base64-override != '' }}          
         run: |
-          BASE64_CONFIG_OVERRIDE="${{ inputs.test-config-base64-override }}"
-          echo "::add-mask::$BASE64_CONFIG_OVERRIDE"
-          echo "BASE64_CONFIG_OVERRIDE=$BASE64_CONFIG_OVERRIDE" >> $GITHUB_ENV
+          # Show what parts of the test config are being overridden
+          go run main.go test-config override \
+            --dry-run \
+            --from-base64-config ${{ inputs.test-config-base64-override }} \
+            --perycope-server-url ${{ secrets.QA_PYROSCOPE_INSTANCE }} \
+            --peryscope-key ${{ secrets.QA_PYROSCOPE_KEY }} \
+            --loki-endpoint https://${{ secrets.GRAFANA_INTERNAL_HOST }}/loki/api/v1/push \
+            --loki-tenant-id ${{ secrets.GRAFANA_INTERNAL_TENANT_ID }} \
+            --loki-basic-auth ${{ secrets.GRAFANA_INTERNAL_BASIC_AUTH }} \
+            --grafana-url "http://localhost:8080/primary" \
+            --grafana-dashboard-url "/d/ddf75041-1e39-42af-aa46-361fe4c36e9e/ci-e2e-tests-logs" \
+            --grafana-bearer-token ${{ secrets.GRAFANA_INTERNAL_URL_SHORTENER_TOKEN }} \
 
-      - name: Install jq
-        run: sudo apt-get install -y jq
-      - name: Show Test Configuration
-        run: echo '${{ toJson(matrix.tests) }}' | jq .
+          # Override custom test config with CI secrets
+          config_override=$(go run main.go test-config override \
+            --from-base64-config ${{ inputs.test-config-base64-override }} \
+            --perycope-server-url ${{ secrets.QA_PYROSCOPE_INSTANCE }} \
+            --peryscope-key ${{ secrets.QA_PYROSCOPE_KEY }} \
+            --loki-endpoint https://${{ secrets.GRAFANA_INTERNAL_HOST }}/loki/api/v1/push \
+            --loki-tenant-id ${{ secrets.GRAFANA_INTERNAL_TENANT_ID }} \
+            --loki-basic-auth ${{ secrets.GRAFANA_INTERNAL_BASIC_AUTH }} \
+            --grafana-url "http://localhost:8080/primary" \
+            --grafana-dashboard-url "/d/ddf75041-1e39-42af-aa46-361fe4c36e9e/ci-e2e-tests-logs" \
+            --grafana-bearer-token ${{ secrets.GRAFANA_INTERNAL_URL_SHORTENER_TOKEN }} \
+          )
+
+          BASE64_CONFIG_OVERRIDE=$(echo "$config_override" | base64 -w 0)
+          echo ::add-mask::$BASE64_CONFIG_OVERRIDE
+          echo "BASE64_CONFIG_OVERRIDE=$BASE64_CONFIG_OVERRIDE" >> $GITHUB_ENV
 
+      - name: Setup GAP for Grafana
+        uses: smartcontractkit/.github/actions/setup-gap@6c9d62fdad050cfb8b59376ded291f1350705944 # setup-gap@0.2.2
+        with:
+          aws-region: ${{ secrets.AWS_REGION }}
+          aws-role-arn: ${{ secrets.AWS_OIDC_IAM_ROLE_VALIDATION_PROD_ARN }}
+          api-gateway-host: ${{ secrets.AWS_API_GW_HOST_GRAFANA }}
+          duplicate-authorization-header: "true"
+      # - name: Prepare Base64 TOML override
+      #   uses: ./.github/actions/setup-create-base64-config
+      #   with:
+      #     runId: ${{ github.run_id }}
+      #     # testLogCollect: ${{ vars.TEST_LOG_COLLECT }}
+      #     selectedNetworks: ${{ env.SELECTED_NETWORKS }}
+      #     chainlinkImage: ${{ env.CHAINLINK_IMAGE }}
+      #     chainlinkVersion: ${{ inputs.chainlink-version }}
+      #     pyroscopeServer: ${{ secrets.QA_PYROSCOPE_INSTANCE }} # Avoid sending blank envs https://github.com/orgs/community/discussions/25725
+      #     pyroscopeEnvironment: ${{ matrix.tests.pyroscopeEnv }}
+      #     pyroscopeKey: ${{ secrets.QA_PYROSCOPE_KEY }}
+      #     lokiEndpoint: https://${{ secrets.GRAFANA_INTERNAL_HOST }}/loki/api/v1/push
+      #     lokiTenantId: ${{ secrets.GRAFANA_INTERNAL_TENANT_ID }}
+      #     lokiBasicAuth: ${{ secrets.GRAFANA_INTERNAL_BASIC_AUTH }}
+      #     logstreamLogTargets: ${{ vars.LOGSTREAM_LOG_TARGETS }}
+      #     grafanaUrl: "http://localhost:8080/primary" # This is GAP's address
+      #     grafanaDashboardUrl: "/d/ddf75041-1e39-42af-aa46-361fe4c36e9e/ci-e2e-tests-logs"
+      #     grafanaBearerToken: ${{ secrets.GRAFANA_INTERNAL_URL_SHORTENER_TOKEN }}
       - name: Run Tests
         uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/run-tests@af92c5fae8dcf1659201e907db82d221fc304b94 # v2.3.21
         with:
diff --git a/.github/workflows/run-selected-e2e-tests.yml b/.github/workflows/run-selected-e2e-tests.yml
index 3c42b4f8bb1..04d73fd43ce 100644
--- a/.github/workflows/run-selected-e2e-tests.yml
+++ b/.github/workflows/run-selected-e2e-tests.yml
@@ -13,6 +13,10 @@ on:
         default: "*"
         required: true
         type: string
+      test-config-base64-override:
+        description: 'Custom base64 test config'
+        required: false
+        type: string          
       enable-check-test-configurations:
         description: 'Set to "true" to enable check-test-configurations job'
         required: false
@@ -36,6 +40,7 @@ jobs:
     with:
       chainlink-version: ${{ github.event.inputs.chainlink-version }}
       test-ids: ${{ github.event.inputs.test-ids }}
+      test-config-base64-override: ${{ github.event.inputs.test-config-base64-override }}
       with-existing-remote-runner-version: ${{ github.event.inputs.with-existing-remote-runner-version }}
       # Use fromJSON to convert string to boolean. More info: https://github.com/actions/runner/issues/2206#issuecomment-1532246677
       enable-check-test-configurations: ${{ fromJSON(github.event.inputs.enable-check-test-configurations) }}
diff --git a/integration-tests/e2e_tests_ci_tool/cmd/create_test_config_cmd.go b/integration-tests/e2e_tests_ci_tool/cmd/create_test_config_cmd.go
new file mode 100644
index 00000000000..bc8792757e1
--- /dev/null
+++ b/integration-tests/e2e_tests_ci_tool/cmd/create_test_config_cmd.go
@@ -0,0 +1,175 @@
+package cmd
+
+import (
+	"fmt"
+	"os"
+
+	"github.com/pelletier/go-toml/v2"
+	ctf_config "github.com/smartcontractkit/chainlink-testing-framework/config"
+	"github.com/spf13/cobra"
+)
+
+var createTestConfigCmd = &cobra.Command{
+	Use:   "create",
+	Short: "Create a test config from the provided flags",
+	Run: func(cmd *cobra.Command, args []string) {
+		var tc ctf_config.TestConfig
+
+		var image, version, postgresVersion *string
+		if cmd.Flags().Changed(ChainlinkImageFlag) {
+			image = &oc.ChainlinkImage
+		}
+		if cmd.Flags().Changed(ChainlinkVersionFlag) {
+			version = &oc.ChainlinkVersion
+		}
+		if cmd.Flags().Changed(ChainlinkPostgresVersionFlag) {
+			version = &oc.ChainlinkPostgresVersion
+		}
+		if image != nil && version == nil || image == nil && version != nil {
+			fmt.Fprintf(os.Stderr, "Error: both chainlink-image and chainlink-version must be set\n")
+			os.Exit(1)
+		}
+		if image != nil && version != nil {
+			tc.ChainlinkImage = &ctf_config.ChainlinkImageConfig{
+				Image:           image,
+				Version:         version,
+				PostgresVersion: postgresVersion,
+			}
+		}
+
+		var selectedNetworks *[]string
+		if cmd.Flags().Changed(SelectedNetworksFlag) {
+			selectedNetworks = &oc.SelectedNetworks
+		}
+		if selectedNetworks != nil {
+			tc.Network = &ctf_config.NetworkConfig{
+				SelectedNetworks: oc.SelectedNetworks,
+			}
+		}
+
+		var peryscopeEnabled *bool
+		var pyroscopeServerURL, pyroscopeEnvironment, pyroscopeKey *string
+		if cmd.Flags().Changed(PyroscopeEnabledFlag) {
+			peryscopeEnabled = &oc.PyroscopeEnabled
+		}
+		if cmd.Flags().Changed(PyroscopeServerURLFlag) {
+			pyroscopeServerURL = &oc.PyroscopeServerURL
+		}
+		if cmd.Flags().Changed(PyroscopeKeyFlag) {
+			pyroscopeKey = &oc.PyroscopeKey
+		}
+		if cmd.Flags().Changed(PyroscopeEnvironmentFlag) {
+			pyroscopeEnvironment = &oc.PyroscopeEnvironment
+		}
+		if peryscopeEnabled != nil {
+			tc.Pyroscope = &ctf_config.PyroscopeConfig{
+				Enabled:     peryscopeEnabled,
+				ServerUrl:   pyroscopeServerURL,
+				Environment: pyroscopeEnvironment,
+				Key:         pyroscopeKey,
+			}
+		}
+
+		var testLogCollect *bool
+		if cmd.Flags().Changed(LoggingTestLogCollectFlag) {
+			testLogCollect = &oc.LoggingTestLogCollect
+		}
+		var loggingRunID *string
+		if cmd.Flags().Changed(LoggingRunIDFlag) {
+			loggingRunID = &oc.LoggingRunID
+		}
+		var loggingLogTargets []string
+		if cmd.Flags().Changed(LoggingLogTargetsFlag) {
+			loggingLogTargets = oc.LoggingLogTargets
+		}
+		var loggingLokiTenantID *string
+		if cmd.Flags().Changed(LoggingLokiTenantIDFlag) {
+			loggingLokiTenantID = &oc.LoggingLokiTenantID
+		}
+		var loggingLokiBasicAuth *string
+		if cmd.Flags().Changed(LoggingLokiBasicAuthFlag) {
+			loggingLokiBasicAuth = &oc.LoggingLokiBasicAuth
+		}
+		var loggingLokiEndpoint *string
+		if cmd.Flags().Changed(LoggingLokiEndpointFlag) {
+			loggingLokiEndpoint = &oc.LoggingLokiEndpoint
+		}
+		var loggingGrafanaBaseURL *string
+		if cmd.Flags().Changed(LoggingGrafanaBaseURLFlag) {
+			loggingGrafanaBaseURL = &oc.LoggingGrafanaBaseURL
+		}
+		var loggingGrafanaDashboardURL *string
+		if cmd.Flags().Changed(LoggingGrafanaDashboardURLFlag) {
+			loggingGrafanaDashboardURL = &oc.LoggingGrafanaDashboardURL
+		}
+		var loggingGrafanaToken *string
+		if cmd.Flags().Changed(LoggingGrafanaTokenFlag) {
+			loggingGrafanaToken = &oc.LoggingGrafanaToken
+		}
+
+		if testLogCollect != nil || loggingRunID != nil || loggingLogTargets != nil || loggingLokiEndpoint != nil || loggingLokiTenantID != nil || loggingLokiBasicAuth != nil || loggingGrafanaBaseURL != nil || loggingGrafanaDashboardURL != nil || loggingGrafanaToken != nil {
+			tc.Logging = &ctf_config.LoggingConfig{}
+			tc.Logging.TestLogCollect = testLogCollect
+			tc.Logging.RunId = loggingRunID
+			if loggingLogTargets != nil {
+				tc.Logging.LogStream = &ctf_config.LogStreamConfig{
+					LogTargets: loggingLogTargets,
+				}
+			}
+			if loggingLokiTenantID != nil || loggingLokiBasicAuth != nil || loggingLokiEndpoint != nil {
+				tc.Logging.Loki = &ctf_config.LokiConfig{
+					TenantId:  loggingLokiTenantID,
+					BasicAuth: loggingLokiBasicAuth,
+					Endpoint:  loggingLokiEndpoint,
+				}
+			}
+			if loggingGrafanaBaseURL != nil || loggingGrafanaDashboardURL != nil || loggingGrafanaToken != nil {
+				tc.Logging.Grafana = &ctf_config.GrafanaConfig{
+					BaseUrl:      loggingGrafanaBaseURL,
+					DashboardUrl: loggingGrafanaDashboardURL,
+					BearerToken:  loggingGrafanaToken,
+				}
+			}
+		}
+
+		var privateEthereumNetworkExecutionLayer *string
+		if cmd.Flags().Changed(PrivateEthereumNetworkExecutionLayerFlag) {
+			privateEthereumNetworkExecutionLayer = &oc.PrivateEthereumNetworkExecutionLayer
+		}
+		var privateEthereumNetworkEthereumVersion *string
+		if cmd.Flags().Changed(PrivateEthereumNetworkEthereumVersionFlag) {
+			privateEthereumNetworkEthereumVersion = &oc.PrivateEthereumNetworkEthereumVersion
+		}
+		var privateEthereumNetworkCustomDockerImage *string
+		if cmd.Flags().Changed(PrivateEthereumNetworkCustomDockerImageFlag) {
+			privateEthereumNetworkCustomDockerImage = &oc.PrivateEthereumNetworkCustomDockerImages
+		}
+		if privateEthereumNetworkExecutionLayer != nil || privateEthereumNetworkEthereumVersion != nil || privateEthereumNetworkCustomDockerImage != nil {
+			var el ctf_config.ExecutionLayer
+			if privateEthereumNetworkExecutionLayer != nil {
+				el = ctf_config.ExecutionLayer(*privateEthereumNetworkExecutionLayer)
+			}
+			var ev ctf_config.EthereumVersion
+			if privateEthereumNetworkEthereumVersion != nil {
+				ev = ctf_config.EthereumVersion(*privateEthereumNetworkEthereumVersion)
+			}
+			var customImages map[ctf_config.ContainerType]string
+			if privateEthereumNetworkCustomDockerImage != nil {
+				customImages = map[ctf_config.ContainerType]string{"execution_layer": *privateEthereumNetworkCustomDockerImage}
+			}
+			tc.PrivateEthereumNetwork = &ctf_config.EthereumNetworkConfig{
+				ExecutionLayer:     &el,
+				EthereumVersion:    &ev,
+				CustomDockerImages: customImages,
+			}
+		}
+
+		configToml, err := toml.Marshal(tc)
+		if err != nil {
+			fmt.Fprintf(os.Stderr, "Error marshalling TestConfig to TOML: %v\n", err)
+			os.Exit(1)
+		}
+
+		fmt.Println(string(configToml))
+	},
+}
diff --git a/integration-tests/e2e_tests_ci_tool/cmd/create_test_config_override.go b/integration-tests/e2e_tests_ci_tool/cmd/create_test_config_override.go
deleted file mode 100644
index 2bb1e33f7b8..00000000000
--- a/integration-tests/e2e_tests_ci_tool/cmd/create_test_config_override.go
+++ /dev/null
@@ -1,300 +0,0 @@
-package cmd
-
-import (
-	"fmt"
-	"os"
-	"regexp"
-	"strings"
-
-	"github.com/pelletier/go-toml/v2"
-	ctf_config "github.com/smartcontractkit/chainlink-testing-framework/config"
-	"github.com/spf13/cobra"
-)
-
-// OverrideConfig holds the configuration data for overrides
-type OverrideConfig struct {
-	ChainlinkImage                           string
-	ChainlinkVersion                         string
-	ChainlinkPostgresVersion                 string
-	SelectedNetworks                         []string
-	PyroscopeEnabled                         bool
-	PyroscopeServerURL                       string
-	PyroscopeEnvironment                     string
-	PyroscopeKey                             string
-	LoggingTestLogCollect                    bool
-	LoggingRunID                             string
-	LoggingLogTargets                        []string
-	LoggingLokiTenantID                      string
-	LoggingLokiEndpoint                      string
-	LoggingLokiBasicAuth                     string
-	LoggingGrafanaBaseURL                    string
-	LoggingGrafanaDashboardURL               string
-	LoggingGrafanaToken                      string
-	PrivateEthereumNetworkExecutionLayer     string
-	PrivateEthereumNetworkEthereumVersion    string
-	PrivateEthereumNetworkCustomDockerImages string
-}
-
-var oc OverrideConfig
-
-var createTestConfigOverrideCmd = &cobra.Command{
-	Use:   "create-test-config-override",
-	Short: "Processes configurations and outputs templated results for test config overrides",
-	Run: func(cmd *cobra.Command, args []string) {
-		var config ctf_config.TestConfig
-
-		var image, version, postgresVersion *string
-		if cmd.Flags().Changed("chainlink-image") {
-			image = &oc.ChainlinkImage
-		}
-		if cmd.Flags().Changed("chainlink-version") {
-			version = &oc.ChainlinkVersion
-		}
-		if cmd.Flags().Changed("chainlink-postgres-version") {
-			version = &oc.ChainlinkPostgresVersion
-		}
-		if image != nil && version == nil || image == nil && version != nil {
-			fmt.Fprintf(os.Stderr, "Error: both chainlink-image and chainlink-version must be set\n")
-			os.Exit(1)
-		}
-		if image != nil && version != nil {
-			config.ChainlinkImage = &ctf_config.ChainlinkImageConfig{
-				Image:           image,
-				Version:         version,
-				PostgresVersion: postgresVersion,
-			}
-		}
-
-		var selectedNetworks *[]string
-		if cmd.Flags().Changed("selected-networks") {
-			selectedNetworks = &oc.SelectedNetworks
-		}
-		if selectedNetworks != nil {
-			config.Network = &ctf_config.NetworkConfig{
-				SelectedNetworks: oc.SelectedNetworks,
-			}
-		}
-
-		var peryscopeEnabled *bool
-		var pyroscopeServerURL, pyroscopeEnvironment, pyroscopeKey *string
-		if cmd.Flags().Changed("pyroscope-enabled") {
-			peryscopeEnabled = &oc.PyroscopeEnabled
-		}
-		if cmd.Flags().Changed("peryscope-server-url") {
-			pyroscopeServerURL = &oc.PyroscopeServerURL
-		}
-		if cmd.Flags().Changed("peryscope-server-key") {
-			pyroscopeKey = &oc.PyroscopeKey
-		}
-		if cmd.Flags().Changed("peryscope-environment") {
-			pyroscopeEnvironment = &oc.PyroscopeEnvironment
-		}
-		if peryscopeEnabled != nil {
-			config.Pyroscope = &ctf_config.PyroscopeConfig{
-				Enabled:     peryscopeEnabled,
-				ServerUrl:   pyroscopeServerURL,
-				Environment: pyroscopeEnvironment,
-				Key:         pyroscopeKey,
-			}
-		}
-
-		var testLogCollect *bool
-		if cmd.Flags().Changed("logging-test-log-collect") {
-			testLogCollect = &oc.LoggingTestLogCollect
-		}
-		var loggingRunID *string
-		if cmd.Flags().Changed("logging-run-id") {
-			loggingRunID = &oc.LoggingRunID
-		}
-		var loggingLogTargets []string
-		if cmd.Flags().Changed("logging-log-targets") {
-			loggingLogTargets = oc.LoggingLogTargets
-		}
-		var loggingLokiTenantID *string
-		if cmd.Flags().Changed("logging-loki-tenant-id") {
-			loggingLokiTenantID = &oc.LoggingLokiTenantID
-		}
-		var loggingLokiBasicAuth *string
-		if cmd.Flags().Changed("logging-loki-basic-auth") {
-			loggingLokiBasicAuth = &oc.LoggingLokiBasicAuth
-		}
-		var loggingLokiEndpoint *string
-		if cmd.Flags().Changed("logging-loki-endpoint") {
-			loggingLokiEndpoint = &oc.LoggingLokiEndpoint
-		}
-		var loggingGrafanaBaseURL *string
-		if cmd.Flags().Changed("logging-grafana-base-url") {
-			loggingGrafanaBaseURL = &oc.LoggingGrafanaBaseURL
-		}
-		var loggingGrafanaDashboardURL *string
-		if cmd.Flags().Changed("logging-grafana-dashboard-url") {
-			loggingGrafanaDashboardURL = &oc.LoggingGrafanaDashboardURL
-		}
-		var loggingGrafanaToken *string
-		if cmd.Flags().Changed("logging-grafana-token") {
-			loggingGrafanaToken = &oc.LoggingGrafanaToken
-		}
-
-		if testLogCollect != nil || loggingRunID != nil || loggingLogTargets != nil || loggingLokiEndpoint != nil || loggingLokiTenantID != nil || loggingLokiBasicAuth != nil || loggingGrafanaBaseURL != nil || loggingGrafanaDashboardURL != nil || loggingGrafanaToken != nil {
-			config.Logging = &ctf_config.LoggingConfig{}
-			config.Logging.TestLogCollect = testLogCollect
-			config.Logging.RunId = loggingRunID
-			if loggingLogTargets != nil {
-				config.Logging.LogStream = &ctf_config.LogStreamConfig{
-					LogTargets: loggingLogTargets,
-				}
-			}
-			if loggingLokiTenantID != nil || loggingLokiBasicAuth != nil || loggingLokiEndpoint != nil {
-				config.Logging.Loki = &ctf_config.LokiConfig{
-					TenantId:  loggingLokiTenantID,
-					BasicAuth: loggingLokiBasicAuth,
-					Endpoint:  loggingLokiEndpoint,
-				}
-			}
-			if loggingGrafanaBaseURL != nil || loggingGrafanaDashboardURL != nil || loggingGrafanaToken != nil {
-				config.Logging.Grafana = &ctf_config.GrafanaConfig{
-					BaseUrl:      loggingGrafanaBaseURL,
-					DashboardUrl: loggingGrafanaDashboardURL,
-					BearerToken:  loggingGrafanaToken,
-				}
-			}
-		}
-
-		var privateEthereumNetworkExecutionLayer *string
-		if cmd.Flags().Changed("private-ethereum-network-execution-layer") {
-			privateEthereumNetworkExecutionLayer = &oc.PrivateEthereumNetworkExecutionLayer
-		}
-		var privateEthereumNetworkEthereumVersion *string
-		if cmd.Flags().Changed("private-ethereum-network-ethereum-version") {
-			privateEthereumNetworkEthereumVersion = &oc.PrivateEthereumNetworkEthereumVersion
-		}
-		var privateEthereumNetworkCustomDockerImage *string
-		if cmd.Flags().Changed("private-ethereum-network-custom-docker-image") {
-			privateEthereumNetworkCustomDockerImage = &oc.PrivateEthereumNetworkCustomDockerImages
-		}
-		if privateEthereumNetworkExecutionLayer != nil || privateEthereumNetworkEthereumVersion != nil || privateEthereumNetworkCustomDockerImage != nil {
-			var el ctf_config.ExecutionLayer
-			if privateEthereumNetworkExecutionLayer != nil {
-				el = ctf_config.ExecutionLayer(*privateEthereumNetworkExecutionLayer)
-			}
-			var ev ctf_config.EthereumVersion
-			if privateEthereumNetworkEthereumVersion != nil {
-				ev = ctf_config.EthereumVersion(*privateEthereumNetworkEthereumVersion)
-			}
-			var customImages map[ctf_config.ContainerType]string
-			if privateEthereumNetworkCustomDockerImage != nil {
-				customImages = map[ctf_config.ContainerType]string{"execution_layer": *privateEthereumNetworkCustomDockerImage}
-			}
-			config.PrivateEthereumNetwork = &ctf_config.EthereumNetworkConfig{
-				ExecutionLayer:     &el,
-				EthereumVersion:    &ev,
-				CustomDockerImages: customImages,
-			}
-		}
-
-		configToml, err := toml.Marshal(config)
-		if err != nil {
-			fmt.Fprintf(os.Stderr, "Error marshalling TestConfig to TOML: %v\n", err)
-			os.Exit(1)
-		}
-
-		fmt.Println(string(configToml))
-	},
-}
-
-var maskTestConfigOverrideCmd = &cobra.Command{
-	Use:   "mask-test-config-override",
-	Short: "Processes configurations and outputs templated results for test config overrides",
-	Run: func(cmd *cobra.Command, args []string) {
-		maskSecret("chainlink image", oc.ChainlinkImage)
-		maskSecret("pyroscope server url", oc.PyroscopeServerURL)
-		maskSecret("pyroscope key", oc.PyroscopeKey)
-		maskSecret("loki tenant id", oc.LoggingLokiTenantID)
-		maskSecret("loki endpoint", oc.LoggingLokiEndpoint)
-		maskSecret("loki basic auth", oc.LoggingLokiBasicAuth)
-		maskSecret("loki grafana token", oc.LoggingGrafanaToken)
-	},
-}
-
-func maskSecret(description string, secret string) {
-	if secret != "" {
-		fmt.Printf("# Mask '%s' value\n", description)
-		fmt.Printf("echo ::add-mask::%s\n", secret)
-	}
-}
-
-func init() {
-	cmds := []*cobra.Command{createTestConfigOverrideCmd, maskTestConfigOverrideCmd}
-	for _, c := range cmds {
-		c.Flags().StringArrayVar(&oc.SelectedNetworks, "selected-networks", nil, "Selected networks")
-		c.Flags().StringVar(&oc.ChainlinkImage, "chainlink-image", "", "Chainlink image")
-		c.Flags().StringVar(&oc.ChainlinkVersion, "chainlink-version", "", "Chainlink version")
-		c.Flags().StringVar(&oc.ChainlinkPostgresVersion, "chainlink-postgres-version", "", "Chainlink Postgres version")
-		c.Flags().BoolVar(&oc.PyroscopeEnabled, "pyroscope-enabled", false, "Pyroscope enabled")
-		c.Flags().StringVar(&oc.PyroscopeServerURL, "pyroscope-server-url", "", "Pyroscope server URL")
-		c.Flags().StringVar(&oc.PyroscopeKey, "pyroscope-key", "", "Pyroscope key")
-		c.Flags().StringVar(&oc.PyroscopeEnvironment, "pyroscope-environment", "", "Pyroscope environment")
-		c.Flags().BoolVar(&oc.LoggingTestLogCollect, "logging-test-log-collect", false, "Test log collect")
-		c.Flags().StringVar(&oc.LoggingRunID, "logging-run-id", "", "Run ID")
-		c.Flags().StringArrayVar(&oc.LoggingLogTargets, "logging-log-targets", nil, "Logging.LogStream.LogTargets")
-		c.Flags().StringVar(&oc.LoggingLokiEndpoint, "logging-loki-endpoint", "", "")
-		c.Flags().StringVar(&oc.LoggingLokiTenantID, "logging-loki-tenant-id", "", "")
-		c.Flags().StringVar(&oc.LoggingLokiBasicAuth, "logging-loki-basic-auth", "", "")
-		c.Flags().StringVar(&oc.LoggingGrafanaBaseURL, "logging-grafana-base-url", "", "")
-		c.Flags().StringVar(&oc.LoggingGrafanaDashboardURL, "logging-grafana-dashboard-url", "", "")
-		c.Flags().StringVar(&oc.LoggingGrafanaToken, "logging-grafana-token", "", "")
-		c.Flags().StringVar(&oc.PrivateEthereumNetworkExecutionLayer, "private-ethereum-network-execution-layer", "", "")
-		c.Flags().StringVar(&oc.PrivateEthereumNetworkEthereumVersion, "private-ethereum-network-ethereum-version", "", "")
-		c.Flags().StringVar(&oc.PrivateEthereumNetworkCustomDockerImages, "private-ethereum-network-custom-docker-image", "", "")
-
-		c.PreRun = func(cmd *cobra.Command, args []string) {
-			// Resolve selected networks environment variable if set
-			_, hasEnvVar := lookupEnvVarName(oc.SelectedNetworks[0])
-			if hasEnvVar {
-				selectedNetworks := mustResolveEnvPlaceholder(oc.SelectedNetworks[0])
-				oc.SelectedNetworks = strings.Split(selectedNetworks, ",")
-			}
-
-			// Resolve all other environment variables
-			oc.ChainlinkImage = mustResolveEnvPlaceholder(oc.ChainlinkImage)
-			oc.ChainlinkVersion = mustResolveEnvPlaceholder(oc.ChainlinkVersion)
-			oc.ChainlinkPostgresVersion = mustResolveEnvPlaceholder(oc.ChainlinkPostgresVersion)
-			oc.PyroscopeServerURL = mustResolveEnvPlaceholder(oc.PyroscopeServerURL)
-			oc.PyroscopeKey = mustResolveEnvPlaceholder(oc.PyroscopeKey)
-			oc.PyroscopeEnvironment = mustResolveEnvPlaceholder(oc.PyroscopeEnvironment)
-			oc.LoggingRunID = mustResolveEnvPlaceholder(oc.LoggingRunID)
-			oc.LoggingLokiTenantID = mustResolveEnvPlaceholder(oc.LoggingLokiTenantID)
-			oc.LoggingLokiEndpoint = mustResolveEnvPlaceholder(oc.LoggingLokiEndpoint)
-			oc.LoggingLokiBasicAuth = mustResolveEnvPlaceholder(oc.LoggingLokiBasicAuth)
-			oc.LoggingGrafanaBaseURL = mustResolveEnvPlaceholder(oc.LoggingGrafanaBaseURL)
-			oc.LoggingGrafanaDashboardURL = mustResolveEnvPlaceholder(oc.LoggingGrafanaDashboardURL)
-			oc.LoggingGrafanaToken = mustResolveEnvPlaceholder(oc.LoggingGrafanaToken)
-			oc.PrivateEthereumNetworkExecutionLayer = mustResolveEnvPlaceholder(oc.PrivateEthereumNetworkExecutionLayer)
-			oc.PrivateEthereumNetworkEthereumVersion = mustResolveEnvPlaceholder(oc.PrivateEthereumNetworkEthereumVersion)
-			oc.PrivateEthereumNetworkCustomDockerImages = mustResolveEnvPlaceholder(oc.PrivateEthereumNetworkCustomDockerImages)
-		}
-	}
-}
-
-// mustResolveEnvPlaceholder checks if the input string is an environment variable placeholder and resolves it.
-func mustResolveEnvPlaceholder(input string) string {
-	envVarName, hasEnvVar := lookupEnvVarName(input)
-	if hasEnvVar {
-		value, set := os.LookupEnv(envVarName)
-		if !set {
-			fmt.Fprintf(os.Stderr, "Error resolving '%s'. Environment variable '%s' not set or is empty\n", input, envVarName)
-			os.Exit(1)
-		}
-		return value
-	}
-	return input
-}
-
-func lookupEnvVarName(input string) (string, bool) {
-	re := regexp.MustCompile(`^\${{ env\.([a-zA-Z_]+) }}$`)
-	matches := re.FindStringSubmatch(input)
-	if len(matches) > 1 {
-		return matches[1], true
-	}
-	return "", false
-}
diff --git a/integration-tests/e2e_tests_ci_tool/cmd/filter_cmd.go b/integration-tests/e2e_tests_ci_tool/cmd/filter_cmd.go
index 5de3bea3253..bd0024053f6 100644
--- a/integration-tests/e2e_tests_ci_tool/cmd/filter_cmd.go
+++ b/integration-tests/e2e_tests_ci_tool/cmd/filter_cmd.go
@@ -12,7 +12,7 @@ import (
 )
 
 // Filter tests based on workflow, test type, and test IDs.
-func filterTests(tests []CITestConf, names, workflow, testType, ids string) []CITestConf {
+func filterTests(tests []CITestConf, workflow, testType, ids string) []CITestConf {
 	workflowFilter := workflow
 	typeFilter := testType
 	idFilter := strings.Split(ids, ",")
@@ -51,7 +51,6 @@ Example usage:
 ./e2e_tests_tool filter --file .github/e2e-tests.yml --workflow "Run Nightly E2E Tests" --test-type "docker" --test-ids "test1,test2"`,
 	Run: func(cmd *cobra.Command, args []string) {
 		yamlFile, _ := cmd.Flags().GetString("file")
-		names, _ := cmd.Flags().GetString("name")
 		workflow, _ := cmd.Flags().GetString("workflow")
 		testType, _ := cmd.Flags().GetString("test-type")
 		testIDs, _ := cmd.Flags().GetString("test-ids")
@@ -67,7 +66,7 @@ Example usage:
 			log.Fatalf("Error parsing YAML data: %v", err)
 		}
 
-		filteredTests := filterTests(config.Tests, names, workflow, testType, testIDs)
+		filteredTests := filterTests(config.Tests, workflow, testType, testIDs)
 		matrix := map[string][]CITestConf{"tests": filteredTests}
 		matrixJSON, err := json.Marshal(matrix)
 		if err != nil {
@@ -80,7 +79,6 @@ Example usage:
 
 func init() {
 	filterCmd.Flags().StringP("file", "f", "", "Path to the YAML file")
-	filterCmd.Flags().StringP("name", "n", "", "Comma-separated list of test names to filter by")
 	filterCmd.Flags().StringP("workflow", "t", "", "Workflow filter")
 	filterCmd.Flags().StringP("test-type", "y", "", "Type of test to filter by")
 	filterCmd.Flags().StringP("test-ids", "i", "*", "Comma-separated list of test IDs to filter by")
diff --git a/integration-tests/e2e_tests_ci_tool/cmd/filter_cmd_test.go b/integration-tests/e2e_tests_ci_tool/cmd/filter_cmd_test.go
index 28871f337ad..a94a55a2c11 100644
--- a/integration-tests/e2e_tests_ci_tool/cmd/filter_cmd_test.go
+++ b/integration-tests/e2e_tests_ci_tool/cmd/filter_cmd_test.go
@@ -24,7 +24,7 @@ func TestFilterTestsByID(t *testing.T) {
 
 	for _, c := range cases {
 		t.Run(c.description, func(t *testing.T) {
-			filtered := filterTests(tests, "", "", "", c.inputIDs)
+			filtered := filterTests(tests, "", "", c.inputIDs)
 			if len(filtered) != c.expectedLen {
 				t.Errorf("FilterTests(%s) returned %d tests, expected %d", c.description, len(filtered), c.expectedLen)
 			}
@@ -55,7 +55,7 @@ func TestFilterTestsIntegration(t *testing.T) {
 
 	for _, c := range cases {
 		t.Run(c.description, func(t *testing.T) {
-			filtered := filterTests(tests, c.inputNames, c.inputWorkflow, c.inputTestType, c.inputIDs)
+			filtered := filterTests(tests, c.inputWorkflow, c.inputTestType, c.inputIDs)
 			if len(filtered) != c.expectedLen {
 				t.Errorf("FilterTests(%s) returned %d tests, expected %d", c.description, len(filtered), c.expectedLen)
 			}
diff --git a/integration-tests/e2e_tests_ci_tool/cmd/mask_test_config_cmd.go b/integration-tests/e2e_tests_ci_tool/cmd/mask_test_config_cmd.go
new file mode 100644
index 00000000000..0d3ea6f23b5
--- /dev/null
+++ b/integration-tests/e2e_tests_ci_tool/cmd/mask_test_config_cmd.go
@@ -0,0 +1,40 @@
+package cmd
+
+import (
+	"fmt"
+	"os"
+	"os/exec"
+
+	"github.com/spf13/cobra"
+)
+
+var maskTestConfigCmd = &cobra.Command{
+	Use:   "mask-secrets",
+	Short: "Run 'echo ::add-mask::${secret}' for all secrets in the test config",
+	Run: func(cmd *cobra.Command, args []string) {
+		mustMaskSecret("chainlink image", oc.ChainlinkImage)
+		mustMaskSecret("chainlink version", oc.ChainlinkVersion)
+		mustMaskSecret("pyroscope server url", oc.PyroscopeServerURL)
+		mustMaskSecret("pyroscope key", oc.PyroscopeKey)
+		mustMaskSecret("loki tenant id", oc.LoggingLokiTenantID)
+		mustMaskSecret("loki endpoint", oc.LoggingLokiEndpoint)
+		mustMaskSecret("loki basic auth", oc.LoggingLokiBasicAuth)
+		mustMaskSecret("loki grafana token", oc.LoggingGrafanaToken)
+		mustMaskSecret("pyroscope environment", oc.PyroscopeEnvironment)
+		mustMaskSecret("pyroscope server url", oc.PyroscopeServerURL)
+		mustMaskSecret("pyroscope key", oc.PyroscopeKey)
+	},
+}
+
+func mustMaskSecret(description string, secret string) {
+	if secret != "" {
+		fmt.Printf("Mask '%s'\n", description)
+		fmt.Printf("::add-mask::%s\n", secret)
+		cmd := exec.Command("bash", "-c", fmt.Sprintf("echo ::add-mask::%s", secret))
+		err := cmd.Run()
+		if err != nil {
+			fmt.Fprintf(os.Stderr, "Failed to mask secret '%s'", description)
+			os.Exit(1)
+		}
+	}
+}
diff --git a/integration-tests/e2e_tests_ci_tool/cmd/override_test_config_cmd.go b/integration-tests/e2e_tests_ci_tool/cmd/override_test_config_cmd.go
new file mode 100644
index 00000000000..1c60cfce312
--- /dev/null
+++ b/integration-tests/e2e_tests_ci_tool/cmd/override_test_config_cmd.go
@@ -0,0 +1,304 @@
+package cmd
+
+import (
+	"encoding/base64"
+	"fmt"
+	"os"
+
+	"github.com/pelletier/go-toml/v2"
+	ctf_config "github.com/smartcontractkit/chainlink-testing-framework/config"
+	"github.com/spf13/cobra"
+	"github.com/spf13/pflag"
+)
+
+const DryRunFlag = "dry-run"
+
+// fromBase64TOML is a base64 encoded TOML config to override
+var fromBase64TOML string
+
+var overrideTestConfigCmd = &cobra.Command{
+	Use:   "override",
+	Short: "Override base64 encoded TOML config with provided flags",
+	Run: func(cmd *cobra.Command, args []string) {
+		dryRun, _ := cmd.Flags().GetBool(DryRunFlag)
+
+		decoded, err := base64.StdEncoding.DecodeString(fromBase64TOML)
+		if err != nil {
+			fmt.Fprintf(os.Stderr, "Error decoding base64 TOML: %v\n", err)
+			os.Exit(1)
+		}
+		var baseConfig ctf_config.TestConfig
+		err = toml.Unmarshal(decoded, &baseConfig)
+		if err != nil {
+			fmt.Fprintf(os.Stderr, "Error unmarshalling base64 TOML: %v\n", err)
+			os.Exit(1)
+		}
+
+		cmd.Flags().Visit(func(flag *pflag.Flag) {
+			switch flag.Name {
+			case ChainlinkImageFlag:
+				if baseConfig.ChainlinkImage != nil && baseConfig.ChainlinkImage.Image != nil {
+					logIfDryRun(dryRun, "Found 'ChainlinkImage.Image' in config. Will override it with %s\n", ChainlinkImageFlag)
+					baseConfig.ChainlinkImage.Image = &oc.ChainlinkImage
+				} else {
+					logIfDryRun(dryRun, "No 'ChainlinkImage.Image' in config. Will NOT OVERRIDE with %s\n", ChainlinkImageFlag)
+				}
+			case ChainlinkVersionFlag:
+				if baseConfig.ChainlinkImage != nil && baseConfig.ChainlinkImage.Version != nil {
+					logIfDryRun(dryRun, "Found 'ChainlinkImage.Version' in config. Will override it with %s\n", ChainlinkVersionFlag)
+					baseConfig.ChainlinkImage.Version = &oc.ChainlinkVersion
+				} else {
+					logIfDryRun(dryRun, "No 'ChainlinkImage.Version' found in config. Will NOT OVERRIDE with %s\n", ChainlinkVersionFlag)
+				}
+			case ChainlinkPostgresVersionFlag:
+				if baseConfig.ChainlinkImage != nil && baseConfig.ChainlinkImage.PostgresVersion != nil {
+					logIfDryRun(dryRun, "Found 'ChainlinkImage.PostgresVersion' with %s\n", ChainlinkPostgresVersionFlag)
+					baseConfig.ChainlinkImage.PostgresVersion = &oc.ChainlinkPostgresVersion
+				} else {
+					logIfDryRun(dryRun, "No 'ChainlinkImage.PostgresVersion' found in config. Will NOT OVERRIDE with %s\n", ChainlinkPostgresVersionFlag)
+				}
+			case SelectedNetworksFlag:
+				if baseConfig.Network != nil && baseConfig.Network.SelectedNetworks != nil {
+					logIfDryRun(dryRun, "Found 'Network.SelectedNetworks' in config. Will override it with %s", SelectedNetworksFlag)
+					baseConfig.Network.SelectedNetworks = oc.SelectedNetworks
+				} else {
+					logIfDryRun(dryRun, "No 'Network.SelectedNetworks' found in config. Will NOT OVERRIDE with %s\n", SelectedNetworksFlag)
+				}
+			case LoggingLokiBasicAuthFlag:
+				if baseConfig.Logging != nil && baseConfig.Logging.Loki != nil && baseConfig.Logging.Loki.BasicAuth != nil {
+					logIfDryRun(dryRun, "Found 'Logging.Loki.BasicAuth' in config. Will override it with %s", LoggingLokiBasicAuthFlag)
+					baseConfig.Logging.Loki.BasicAuth = &oc.LoggingLokiBasicAuth
+				} else {
+					logIfDryRun(dryRun, "No 'Logging.Loki' found in config. Will NOT OVERRIDE with %s\n", LoggingLokiBasicAuthFlag)
+				}
+			case LoggingLokiEndpointFlag:
+				if baseConfig.Logging != nil && baseConfig.Logging.Loki != nil && baseConfig.Logging.Loki.Endpoint != nil {
+					logIfDryRun(dryRun, "Found 'Logging.Loki.Endpoint' in config. Will override it with %s", LoggingLokiEndpointFlag)
+					baseConfig.Logging.Loki.Endpoint = &oc.LoggingLokiEndpoint
+				} else {
+					logIfDryRun(dryRun, "No 'Logging.Loki' found in config. Will NOT OVERRIDE with %s\n", LoggingLokiEndpointFlag)
+				}
+			case LoggingLokiTenantIDFlag:
+				if baseConfig.Logging != nil && baseConfig.Logging.Loki != nil && baseConfig.Logging.Loki.TenantId != nil {
+					logIfDryRun(dryRun, "Found 'Logging.Loki.TenantId' in config. Will override it with %s", LoggingLokiTenantIDFlag)
+					baseConfig.Logging.Loki.TenantId = &oc.LoggingLokiTenantID
+				} else {
+					logIfDryRun(dryRun, "No 'Logging.Loki' found in config. Will NOT OVERRIDE with %s\n", LoggingLokiTenantIDFlag)
+				}
+			case LoggingRunIDFlag:
+				if baseConfig.Logging != nil && baseConfig.Logging.RunId != nil {
+					logIfDryRun(dryRun, "Found 'Logging.Loki.RunId' in config. Will override it with %s", LoggingRunIDFlag)
+					baseConfig.Logging.RunId = &oc.LoggingRunID
+				} else {
+					logIfDryRun(dryRun, "No 'Logging' found in config. Will NOT OVERRIDE with %s\n", LoggingRunIDFlag)
+				}
+			case LoggingGrafanaBaseURLFlag:
+				if baseConfig.Logging != nil && baseConfig.Logging.Grafana != nil && baseConfig.Logging.Grafana.BaseUrl != nil {
+					logIfDryRun(dryRun, "Found 'Logging.Grafana.BaseUrl' in config. Will override it with %s", LoggingGrafanaBaseURLFlag)
+					baseConfig.Logging.Grafana.BaseUrl = &oc.LoggingGrafanaBaseURL
+				} else {
+					logIfDryRun(dryRun, "No 'Logging.Grafana' found in config. Will NOT OVERRIDE with %s\n", LoggingGrafanaBaseURLFlag)
+				}
+			case LoggingGrafanaDashboardURLFlag:
+				if baseConfig.Logging != nil && baseConfig.Logging.Grafana != nil && baseConfig.Logging.Grafana.DashboardUrl != nil {
+					logIfDryRun(dryRun, "Found 'Logging.Grafana.BaseUrl' in config. Will override it with %s", LoggingGrafanaBaseURLFlag)
+					baseConfig.Logging.Grafana.DashboardUrl = &oc.LoggingGrafanaDashboardURL
+				} else {
+					logIfDryRun(dryRun, "No 'Logging.Grafana' found in config. Will NOT OVERRIDE with %s\n", LoggingGrafanaDashboardURLFlag)
+				}
+			case LoggingGrafanaTokenFlag:
+				if baseConfig.Logging != nil && baseConfig.Logging.Grafana != nil && baseConfig.Logging.Grafana.BearerToken != nil {
+					logIfDryRun(dryRun, "Found 'Logging.Grafana.BearerToken' in config. Will override it with %s", LoggingGrafanaTokenFlag)
+					baseConfig.Logging.Grafana.BearerToken = &oc.LoggingGrafanaToken
+				} else {
+					logIfDryRun(dryRun, "No 'Logging.Grafana' found in config. Will NOT OVERRIDE with %s\n", LoggingGrafanaTokenFlag)
+				}
+			case LoggingLogTargetsFlag:
+				if baseConfig.Logging != nil && baseConfig.Logging.LogStream != nil && baseConfig.Logging.LogStream.LogTargets != nil {
+					logIfDryRun(dryRun, "Found 'Logging.LogStream.LogTargets' in config. Will override it with %s", LoggingLogTargetsFlag)
+					baseConfig.Logging.LogStream.LogTargets = oc.LoggingLogTargets
+				} else {
+					logIfDryRun(dryRun, "No 'Logging.LogStream' found in config. Will NOT OVERRIDE with %s\n", LoggingLogTargetsFlag)
+				}
+			case PyroscopeEnabledFlag:
+				if baseConfig.Pyroscope != nil && baseConfig.Pyroscope.Enabled != nil {
+					logIfDryRun(dryRun, "Found 'Pyroscope.Enabled' in config. Will override it with %s", PyroscopeEnabledFlag)
+					baseConfig.Pyroscope.Enabled = &oc.PyroscopeEnabled
+				} else {
+					logIfDryRun(dryRun, "No 'Pyroscope' found in config. Will NOT OVERRIDE with %s\n", PyroscopeEnabledFlag)
+				}
+			case PyroscopeServerURLFlag:
+				if baseConfig.Pyroscope != nil {
+					logIfDryRun(dryRun, "Found 'Pyroscope.ServerUrl' in config. Will override it with %s", PyroscopeServerURLFlag)
+					baseConfig.Pyroscope.ServerUrl = &oc.PyroscopeServerURL
+				} else {
+					logIfDryRun(dryRun, "No 'Pyroscope' found in config. Will NOT OVERRIDE with %s\n", PyroscopeServerURLFlag)
+				}
+			case PyroscopeEnvironmentFlag:
+				if baseConfig.Pyroscope != nil {
+					logIfDryRun(dryRun, "Found 'Pyroscope.Environment' in config. Will override it with %s", PyroscopeEnvironmentFlag)
+					baseConfig.Pyroscope.Environment = &oc.PyroscopeEnvironment
+				} else {
+					logIfDryRun(dryRun, "No 'Pyroscope' found in config. Will NOT OVERRIDE with %s\n", PyroscopeEnvironmentFlag)
+				}
+			case PyroscopeKeyFlag:
+				if baseConfig.Pyroscope != nil {
+					logIfDryRun(dryRun, "Found 'Pyroscope.Key' in config. Will override it with %s", PyroscopeKeyFlag)
+					baseConfig.Pyroscope.Key = &oc.PyroscopeKey
+				} else {
+					logIfDryRun(dryRun, "No 'Pyroscope' found in config. Will NOT OVERRIDE with %s\n", PyroscopeKeyFlag)
+				}
+			case PrivateEthereumNetworkExecutionLayerFlag:
+				if baseConfig.PrivateEthereumNetwork != nil {
+					logIfDryRun(dryRun, "Found 'PrivateEthereumNetwork.ExecutionLayer' in config. Will override it with %s", PrivateEthereumNetworkExecutionLayerFlag)
+					el := ctf_config.ExecutionLayer(oc.PrivateEthereumNetworkExecutionLayer)
+					baseConfig.PrivateEthereumNetwork.ExecutionLayer = &el
+				} else {
+					logIfDryRun(dryRun, "No 'PrivateEthereumNetwork' found in config. Will NOT OVERRIDE with %s\n", PrivateEthereumNetworkExecutionLayerFlag)
+				}
+			case PrivateEthereumNetworkEthereumVersionFlag:
+				if baseConfig.PrivateEthereumNetwork != nil {
+					logIfDryRun(dryRun, "Found 'PrivateEthereumNetwork.EthereumVersion' in config. Will override it with %s", PrivateEthereumNetworkEthereumVersionFlag)
+					ev := ctf_config.EthereumVersion(oc.PrivateEthereumNetworkEthereumVersion)
+					baseConfig.PrivateEthereumNetwork.EthereumVersion = &ev
+				} else {
+					logIfDryRun(dryRun, "No 'PrivateEthereumNetwork' found in config. Will NOT OVERRIDE with %s\n", PrivateEthereumNetworkEthereumVersionFlag)
+				}
+			case PrivateEthereumNetworkCustomDockerImageFlag:
+				if baseConfig.PrivateEthereumNetwork != nil {
+					logIfDryRun(dryRun, "Found 'PrivateEthereumNetwork.CustomDockerImages' in config. Will override it with %s", PrivateEthereumNetworkCustomDockerImageFlag)
+					customImages := map[ctf_config.ContainerType]string{"execution_layer": oc.PrivateEthereumNetworkCustomDockerImages}
+					baseConfig.PrivateEthereumNetwork.CustomDockerImages = customImages
+				} else {
+					logIfDryRun(dryRun, "No 'PrivateEthereumNetwork' found in config. Will NOT OVERRIDE with %s\n", PrivateEthereumNetworkCustomDockerImageFlag)
+				}
+			default:
+				fmt.Printf("Override not supported for flag: %s\n", flag.Name)
+				os.Exit(1)
+			}
+
+		})
+
+		// // Override base config with flags
+		// if cmd.Flags().Changed(ChainlinkImageFlag) {
+		// 	if baseConfig.ChainlinkImage != nil {
+		// 		baseConfig.ChainlinkImage.Image = &oc.ChainlinkImage
+		// 	}
+		// }
+		// if cmd.Flags().Changed(ChainlinkVersionFlag) {
+		// 	if baseConfig.ChainlinkImage != nil {
+		// 		baseConfig.ChainlinkImage.Version = &oc.ChainlinkVersion
+		// 	}
+		// }
+		// if cmd.Flags().Changed(ChainlinkPostgresVersionFlag) {
+		// 	if baseConfig.ChainlinkImage != nil {
+		// 		baseConfig.ChainlinkImage.PostgresVersion = &oc.ChainlinkPostgresVersion
+		// 	}
+		// }
+		// if cmd.Flags().Changed(SelectedNetworksFlag) {
+		// 	if baseConfig.Network != nil {
+		// 		baseConfig.Network.SelectedNetworks = oc.SelectedNetworks
+		// 	}
+		// }
+		// if cmd.Flags().Changed(LoggingLokiBasicAuthFlag) {
+		// 	if baseConfig.Logging != nil && baseConfig.Logging.Loki != nil {
+		// 		baseConfig.Logging.Loki.BasicAuth = &oc.LoggingLokiBasicAuth
+		// 	}
+		// }
+		// if cmd.Flags().Changed(LoggingLokiEndpointFlag) {
+		// 	if baseConfig.Logging != nil && baseConfig.Logging.Loki != nil {
+		// 		baseConfig.Logging.Loki.Endpoint = &oc.LoggingLokiEndpoint
+		// 	}
+		// }
+		// if cmd.Flags().Changed(LoggingLokiTenantIDFlag) {
+		// 	if baseConfig.Logging != nil && baseConfig.Logging.Loki != nil {
+		// 		baseConfig.Logging.Loki.TenantId = &oc.LoggingLokiTenantID
+		// 	}
+		// }
+		// if cmd.Flags().Changed(LoggingRunIDFlag) {
+		// 	if baseConfig.Logging != nil {
+		// 		baseConfig.Logging.RunId = &oc.LoggingRunID
+		// 	}
+		// }
+		// if cmd.Flags().Changed(LoggingGrafanaBaseURLFlag) {
+		// 	if baseConfig.Logging != nil && baseConfig.Logging.Grafana != nil {
+		// 		baseConfig.Logging.Grafana.BaseUrl = &oc.LoggingGrafanaBaseURL
+		// 	}
+		// }
+		// if cmd.Flags().Changed(LoggingGrafanaDashboardURLFlag) {
+		// 	if baseConfig.Logging != nil && baseConfig.Logging.Grafana != nil {
+		// 		baseConfig.Logging.Grafana.DashboardUrl = &oc.LoggingGrafanaDashboardURL
+		// 	}
+		// }
+		// if cmd.Flags().Changed(LoggingGrafanaTokenFlag) {
+		// 	if baseConfig.Logging != nil && baseConfig.Logging.Grafana != nil {
+		// 		baseConfig.Logging.Grafana.BearerToken = &oc.LoggingGrafanaToken
+		// 	}
+		// }
+		// if cmd.Flags().Changed(LoggingLogTargetsFlag) {
+		// 	if baseConfig.Logging != nil && baseConfig.Logging.LogStream != nil {
+		// 		baseConfig.Logging.LogStream.LogTargets = oc.LoggingLogTargets
+		// 	}
+		// }
+		// if cmd.Flags().Changed(PyroscopeEnabledFlag) {
+		// 	if baseConfig.Pyroscope != nil {
+		// 		baseConfig.Pyroscope.Enabled = &oc.PyroscopeEnabled
+		// 	}
+		// }
+		// if cmd.Flags().Changed(PyroscopeServerURLFlag) {
+		// 	if baseConfig.Pyroscope != nil {
+		// 		baseConfig.Pyroscope.ServerUrl = &oc.PyroscopeServerURL
+		// 	}
+		// }
+		// if cmd.Flags().Changed(PyroscopeEnvironmentFlag) {
+		// 	if baseConfig.Pyroscope != nil {
+		// 		baseConfig.Pyroscope.Environment = &oc.PyroscopeEnvironment
+		// 	}
+		// }
+		// if cmd.Flags().Changed(PyroscopeKeyFlag) {
+		// 	if baseConfig.Pyroscope != nil {
+		// 		baseConfig.Pyroscope.Key = &oc.PyroscopeKey
+		// 	}
+		// }
+		// if cmd.Flags().Changed(PrivateEthereumNetworkExecutionLayerFlag) {
+		// 	if baseConfig.PrivateEthereumNetwork != nil {
+		// 		el := ctf_config.ExecutionLayer(oc.PrivateEthereumNetworkExecutionLayer)
+		// 		baseConfig.PrivateEthereumNetwork.ExecutionLayer = &el
+		// 	}
+		// }
+		// if cmd.Flags().Changed(PrivateEthereumNetworkEthereumVersionFlag) {
+		// 	if baseConfig.PrivateEthereumNetwork != nil {
+		// 		ev := ctf_config.EthereumVersion(oc.PrivateEthereumNetworkEthereumVersion)
+		// 		baseConfig.PrivateEthereumNetwork.EthereumVersion = &ev
+		// 	}
+		// }
+		// if cmd.Flags().Changed(PrivateEthereumNetworkCustomDockerImageFlag) {
+		// 	if baseConfig.PrivateEthereumNetwork != nil {
+		// 		customImages := map[ctf_config.ContainerType]string{"execution_layer": oc.PrivateEthereumNetworkCustomDockerImages}
+		// 		baseConfig.PrivateEthereumNetwork.CustomDockerImages = customImages
+		// 	}
+		// }
+
+		if !dryRun {
+			configToml, err := toml.Marshal(baseConfig)
+			if err != nil {
+				fmt.Fprintf(os.Stderr, "Error marshalling TestConfig to TOML: %v\n", err)
+				os.Exit(1)
+			}
+			fmt.Println(string(configToml))
+		}
+	},
+}
+
+func init() {
+	overrideTestConfigCmd.Flags().StringVar(&fromBase64TOML, FromBase64ConfigFlag, "", "Base64 encoded TOML config to override")
+	overrideTestConfigCmd.MarkFlagRequired(FromBase64ConfigFlag)
+
+	overrideTestConfigCmd.Flags().Bool(DryRunFlag, false, "Dry run mode")
+}
+
+func logIfDryRun(dryRun bool, format string, a ...interface{}) {
+	if dryRun {
+		fmt.Printf(format, a...)
+	}
+}
diff --git a/integration-tests/e2e_tests_ci_tool/cmd/root_cmd.go b/integration-tests/e2e_tests_ci_tool/cmd/root_cmd.go
index 07b547173af..789d67f28df 100644
--- a/integration-tests/e2e_tests_ci_tool/cmd/root_cmd.go
+++ b/integration-tests/e2e_tests_ci_tool/cmd/root_cmd.go
@@ -27,6 +27,8 @@ func init() {
 	rootCmd.AddCommand(checkTestsCmd)
 	rootCmd.AddCommand(filterCmd)
 	rootCmd.AddCommand(envreplaceCmd)
-	rootCmd.AddCommand(createTestConfigOverrideCmd)
-	rootCmd.AddCommand(maskTestConfigOverrideCmd)
+	rootCmd.AddCommand(testConfigCmd)
+	testConfigCmd.AddCommand(maskTestConfigCmd)
+	testConfigCmd.AddCommand(createTestConfigCmd)
+	testConfigCmd.AddCommand(overrideTestConfigCmd)
 }
diff --git a/integration-tests/e2e_tests_ci_tool/cmd/test_config_cmd.go b/integration-tests/e2e_tests_ci_tool/cmd/test_config_cmd.go
new file mode 100644
index 00000000000..e5d68b8e40e
--- /dev/null
+++ b/integration-tests/e2e_tests_ci_tool/cmd/test_config_cmd.go
@@ -0,0 +1,143 @@
+package cmd
+
+import (
+	"fmt"
+	"os"
+	"regexp"
+	"strings"
+
+	"github.com/spf13/cobra"
+)
+
+var testConfigCmd = &cobra.Command{
+	Use:   "test-config",
+	Short: "Manage test config",
+}
+
+// OverrideConfig holds the configuration data for overrides
+type OverrideConfig struct {
+	ChainlinkImage                           string
+	ChainlinkVersion                         string
+	ChainlinkPostgresVersion                 string
+	SelectedNetworks                         []string
+	PyroscopeEnabled                         bool
+	PyroscopeServerURL                       string
+	PyroscopeEnvironment                     string
+	PyroscopeKey                             string
+	LoggingTestLogCollect                    bool
+	LoggingRunID                             string
+	LoggingLogTargets                        []string
+	LoggingLokiTenantID                      string
+	LoggingLokiEndpoint                      string
+	LoggingLokiBasicAuth                     string
+	LoggingGrafanaBaseURL                    string
+	LoggingGrafanaDashboardURL               string
+	LoggingGrafanaToken                      string
+	PrivateEthereumNetworkExecutionLayer     string
+	PrivateEthereumNetworkEthereumVersion    string
+	PrivateEthereumNetworkCustomDockerImages string
+}
+
+const (
+	ChainlinkImageFlag                          = "chainlink-image"
+	ChainlinkVersionFlag                        = "chainlink-version"
+	ChainlinkPostgresVersionFlag                = "chainlink-postgres-version"
+	SelectedNetworksFlag                        = "selected-networks"
+	FromBase64ConfigFlag                        = "from-base64-config"
+	LoggingLokiBasicAuthFlag                    = "logging-loki-basic-auth"
+	LoggingLokiEndpointFlag                     = "logging-loki-endpoint"
+	LoggingRunIDFlag                            = "logging-run-id"
+	LoggingLokiTenantIDFlag                     = "logging-loki-tenant-id"
+	LoggingGrafanaBaseURLFlag                   = "logging-grafana-base-url"
+	LoggingGrafanaDashboardURLFlag              = "logging-grafana-dashboard-url"
+	LoggingGrafanaTokenFlag                     = "logging-grafana-token"
+	LoggingLogTargetsFlag                       = "logging-log-targets"
+	LoggingTestLogCollectFlag                   = "logging-test-log-collect"
+	PyroscopeEnabledFlag                        = "pyroscope-enabled"
+	PyroscopeServerURLFlag                      = "pyroscope-server-url"
+	PyroscopeKeyFlag                            = "pyroscope-key"
+	PyroscopeEnvironmentFlag                    = "pyroscope-environment"
+	PrivateEthereumNetworkExecutionLayerFlag    = "private-ethereum-network-execution-layer"
+	PrivateEthereumNetworkEthereumVersionFlag   = "private-ethereum-network-ethereum-version"
+	PrivateEthereumNetworkCustomDockerImageFlag = "private-ethereum-network-custom-docker-image"
+)
+
+var oc OverrideConfig
+
+func init() {
+	cmds := []*cobra.Command{maskTestConfigCmd, createTestConfigCmd, overrideTestConfigCmd}
+	for _, c := range cmds {
+		c.Flags().StringArrayVar(&oc.SelectedNetworks, SelectedNetworksFlag, nil, "Selected networks")
+		c.Flags().StringVar(&oc.ChainlinkImage, ChainlinkImageFlag, "", "Chainlink image")
+		c.Flags().StringVar(&oc.ChainlinkVersion, ChainlinkVersionFlag, "", "Chainlink version")
+		c.Flags().StringVar(&oc.ChainlinkPostgresVersion, ChainlinkPostgresVersionFlag, "", "Chainlink Postgres version")
+		c.Flags().BoolVar(&oc.PyroscopeEnabled, PyroscopeEnabledFlag, false, "Pyroscope enabled")
+		c.Flags().StringVar(&oc.PyroscopeServerURL, PyroscopeServerURLFlag, "", "Pyroscope server URL")
+		c.Flags().StringVar(&oc.PyroscopeKey, PyroscopeKeyFlag, "", "Pyroscope key")
+		c.Flags().StringVar(&oc.PyroscopeEnvironment, PyroscopeEnvironmentFlag, "", "Pyroscope environment")
+		c.Flags().BoolVar(&oc.LoggingTestLogCollect, LoggingTestLogCollectFlag, false, "Test log collect")
+		c.Flags().StringVar(&oc.LoggingRunID, LoggingRunIDFlag, "", "Run ID")
+		c.Flags().StringArrayVar(&oc.LoggingLogTargets, LoggingLogTargetsFlag, nil, "Logging.LogStream.LogTargets")
+		c.Flags().StringVar(&oc.LoggingLokiEndpoint, LoggingLokiEndpointFlag, "", "")
+		c.Flags().StringVar(&oc.LoggingLokiTenantID, LoggingLokiTenantIDFlag, "", "")
+		c.Flags().StringVar(&oc.LoggingLokiBasicAuth, LoggingLokiBasicAuthFlag, "", "")
+		c.Flags().StringVar(&oc.LoggingGrafanaBaseURL, LoggingGrafanaBaseURLFlag, "", "")
+		c.Flags().StringVar(&oc.LoggingGrafanaDashboardURL, LoggingGrafanaDashboardURLFlag, "", "")
+		c.Flags().StringVar(&oc.LoggingGrafanaToken, LoggingGrafanaTokenFlag, "", "")
+		c.Flags().StringVar(&oc.PrivateEthereumNetworkExecutionLayer, PrivateEthereumNetworkExecutionLayerFlag, "", "")
+		c.Flags().StringVar(&oc.PrivateEthereumNetworkEthereumVersion, PrivateEthereumNetworkEthereumVersionFlag, "", "")
+		c.Flags().StringVar(&oc.PrivateEthereumNetworkCustomDockerImages, PrivateEthereumNetworkCustomDockerImageFlag, "", "")
+
+		c.PreRun = func(cmd *cobra.Command, args []string) {
+			// Resolve selected networks environment variable if set
+			if len(oc.SelectedNetworks) > 0 {
+				_, hasEnvVar := lookupEnvVarName(oc.SelectedNetworks[0])
+				if hasEnvVar {
+					selectedNetworks := mustResolveEnvPlaceholder(oc.SelectedNetworks[0])
+					oc.SelectedNetworks = strings.Split(selectedNetworks, ",")
+				}
+			}
+
+			// Resolve all other environment variables
+			oc.ChainlinkImage = mustResolveEnvPlaceholder(oc.ChainlinkImage)
+			oc.ChainlinkVersion = mustResolveEnvPlaceholder(oc.ChainlinkVersion)
+			oc.ChainlinkPostgresVersion = mustResolveEnvPlaceholder(oc.ChainlinkPostgresVersion)
+			oc.PyroscopeServerURL = mustResolveEnvPlaceholder(oc.PyroscopeServerURL)
+			oc.PyroscopeKey = mustResolveEnvPlaceholder(oc.PyroscopeKey)
+			oc.PyroscopeEnvironment = mustResolveEnvPlaceholder(oc.PyroscopeEnvironment)
+			oc.LoggingRunID = mustResolveEnvPlaceholder(oc.LoggingRunID)
+			oc.LoggingLokiTenantID = mustResolveEnvPlaceholder(oc.LoggingLokiTenantID)
+			oc.LoggingLokiEndpoint = mustResolveEnvPlaceholder(oc.LoggingLokiEndpoint)
+			oc.LoggingLokiBasicAuth = mustResolveEnvPlaceholder(oc.LoggingLokiBasicAuth)
+			oc.LoggingGrafanaBaseURL = mustResolveEnvPlaceholder(oc.LoggingGrafanaBaseURL)
+			oc.LoggingGrafanaDashboardURL = mustResolveEnvPlaceholder(oc.LoggingGrafanaDashboardURL)
+			oc.LoggingGrafanaToken = mustResolveEnvPlaceholder(oc.LoggingGrafanaToken)
+			oc.PrivateEthereumNetworkExecutionLayer = mustResolveEnvPlaceholder(oc.PrivateEthereumNetworkExecutionLayer)
+			oc.PrivateEthereumNetworkEthereumVersion = mustResolveEnvPlaceholder(oc.PrivateEthereumNetworkEthereumVersion)
+			oc.PrivateEthereumNetworkCustomDockerImages = mustResolveEnvPlaceholder(oc.PrivateEthereumNetworkCustomDockerImages)
+		}
+	}
+}
+
+// mustResolveEnvPlaceholder checks if the input string is an environment variable placeholder and resolves it.
+func mustResolveEnvPlaceholder(input string) string {
+	envVarName, hasEnvVar := lookupEnvVarName(input)
+	if hasEnvVar {
+		value, set := os.LookupEnv(envVarName)
+		if !set {
+			fmt.Fprintf(os.Stderr, "Error resolving '%s'. Environment variable '%s' not set or is empty\n", input, envVarName)
+			os.Exit(1)
+		}
+		return value
+	}
+	return input
+}
+
+func lookupEnvVarName(input string) (string, bool) {
+	re := regexp.MustCompile(`^\${{ env\.([a-zA-Z_]+) }}$`)
+	matches := re.FindStringSubmatch(input)
+	if len(matches) > 1 {
+		return matches[1], true
+	}
+	return "", false
+}
diff --git a/integration-tests/e2e_tests_ci_tool/cmd/types.go b/integration-tests/e2e_tests_ci_tool/cmd/types.go
index a297323d8b0..6b2c05c3096 100644
--- a/integration-tests/e2e_tests_ci_tool/cmd/types.go
+++ b/integration-tests/e2e_tests_ci_tool/cmd/types.go
@@ -7,16 +7,23 @@ type Test struct {
 
 // CITestConf defines the configuration for running a test in a CI environment, specifying details like test ID, path, type, runner settings, command, and associated workflows.
 type CITestConf struct {
-	ID                    string   `yaml:"id" json:"id"`
-	Path                  string   `yaml:"path" json:"path"`
-	TestType              string   `yaml:"test-type" json:"testType"`
-	RunsOn                string   `yaml:"runs-on" json:"runsOn"`
-	TestCmd               string   `yaml:"test-cmd" json:"testCmd"`
-	TestConfigOverride    string   `yaml:"test-config-override" json:"testConfigOverride"`
-	RemoteRunnerTestSuite string   `yaml:"remote-runner-test-suite" json:"remoteRunnerTestSuite"`
-	RemoteRunnerMemory    string   `yaml:"remote-runner-memory" json:"remoteRunnerMemory"`
-	PyroscopeEnv          string   `yaml:"pyroscope-env" json:"pyroscopeEnv"`
-	Workflows             []string `yaml:"workflows" json:"workflows"`
+	ID                    string             `yaml:"id" json:"id"`
+	Path                  string             `yaml:"path" json:"path"`
+	TestType              string             `yaml:"test-type" json:"testType"`
+	RunsOn                string             `yaml:"runs-on" json:"runsOn"`
+	TestCmd               string             `yaml:"test-cmd" json:"testCmd"`
+	TestConfigOverride    TestConfigOverride `yaml:"test-config-override" json:"testConfigOverride"`
+	RemoteRunnerTestSuite string             `yaml:"remote-runner-test-suite" json:"remoteRunnerTestSuite"`
+	RemoteRunnerMemory    string             `yaml:"remote-runner-memory" json:"remoteRunnerMemory"`
+	PyroscopeEnv          string             `yaml:"pyroscope-env" json:"pyroscopeEnv"`
+	Workflows             []string           `yaml:"workflows" json:"workflows"`
+}
+
+type TestConfigOverride struct {
+	ChainlinkUpgradeImage   string `yaml:"chainlink-upgrade-image" json:"chainlinkUpgradeImage"`
+	ChainlinkUpgradeVersion string `yaml:"chainlink-upgrade-version" json:"chainlinkUpgradeVersion"`
+	ChainlinkImage          string `yaml:"chainlink-image" json:"chainlinkImage"`
+	ChainlinkVersion        string `yaml:"chainlink-version" json:"chainlinkVersion"`
 }
 
 type Config struct {

From d3cdd001d50b4dc4ed63df57ba7603b15cfdb8b4 Mon Sep 17 00:00:00 2001
From: lukaszcl <120112546+lukaszcl@users.noreply.github.com>
Date: Thu, 27 Jun 2024 15:15:43 +0200
Subject: [PATCH 069/137] Add mask-secrets cmd and stop using reusable workflow

---
 .../actions/run-e2e-docker-test/action.yml    | 111 ++++++++++
 .github/actions/setup-e2e-tests/action.yml    |  98 +++++++++
 .../run-e2e-tests-reusable-workflow.yml       |  78 ++++---
 .github/workflows/run-selected-e2e-tests.yml  | 206 +++++++++++++++---
 go.mod                                        |  86 +-------
 go.sum                                        | 171 +--------------
 .../cmd/mask_test_config_cmd.go               | 150 +++++++++++--
 .../e2e_tests_ci_tool/cmd/test_config_cmd.go  |   2 +-
 integration-tests/go.mod                      |   4 +-
 9 files changed, 569 insertions(+), 337 deletions(-)
 create mode 100644 .github/actions/run-e2e-docker-test/action.yml
 create mode 100644 .github/actions/setup-e2e-tests/action.yml

diff --git a/.github/actions/run-e2e-docker-test/action.yml b/.github/actions/run-e2e-docker-test/action.yml
new file mode 100644
index 00000000000..a95224b3c36
--- /dev/null
+++ b/.github/actions/run-e2e-docker-test/action.yml
@@ -0,0 +1,111 @@
+name: 'Run E2E Docker Test'
+inputs:
+  test:
+    required: true
+    description: 'Test to run'
+  test_config_base64_override_unsafe:
+    description: 'This is an unsafe way to pass custom base64 test config. It cannot contain any secrets as this value is exposed in Github logs (Set up job -> Inputs) when the workflow is called! The caller workflow must make sure to remove secrets from this input.'
+    required: false
+    type: string        
+
+runs:
+  using: 'composite'
+  steps:
+    - name: Install jq
+      run: sudo apt-get install -y jq
+    - name: Show test configuration
+      run: echo '${{ toJson(inputs.test) }}' | jq .
+    - name: Set up Go
+      uses: actions/setup-go@v2
+      with:
+        go-version: '1.21.7'
+
+    - name: Set default test config override
+      if: ${{ inputs.test_config_base64_override_unsafe == '' }}
+      run: |      
+        # Mask secret args before using them
+        # For more, see https://docs.github.com/en/actions/using-workflows/workflow-commands-for-github-actions#masking-a-value-in-a-log
+        echo ::add-mask::${{ env.CHAINLINK_IMAGE }}
+        echo "cl image: ${{ env.CHAINLINK_IMAGE }}"
+
+        # Use overrides from e2e-tests.yml or defaults 
+        chainlink_version="${{ inputs.test.testConfigOverride.chainlinkVersion || github.sha }}"
+        chainlink_image="${{ inputs.test.testConfigOverride.chainlinkImage || env.CHAINLINK_IMAGE }}"
+        chainlink_postgres_version="${{ inputs.test.testConfigOverride.chainlinkPostgresVersion }}"
+        chainlink_upgrade_version="${{ inputs.test.testConfigOverride.chainlinkUpgradeVersion }}"
+        selected_networks="${{ inputs.test.testConfigOverride.selectedNetworks || env.SELECTED_NETWORKS}}"
+
+        # Build the command line arguments
+        # Need to check if the variable is set before adding it to the command
+        # Otherwise "" will be passed as an argument which is not what we want
+        cmd_args=""
+        if [ -n "$chainlink_image" ]; then
+          cmd_args+="--chainlink-image=$chainlink_image "
+        fi
+        if [ -n "$chainlink_version" ]; then
+          cmd_args+="--chainlink-version=$chainlink_version "
+        fi
+        if [ -n "$chainlink_postgres_version" ]; then
+          cmd_args+="--chainlink-postgres-version=$chainlink_postgres_version "
+        fi
+        if [ -n "$chainlink_upgrade_version" ]; then
+          cmd_args+="--chainlink-upgrade-version=$chainlink_upgrade_version "
+        fi
+        if [ -n "$selected_networks" ]; then
+          cmd_args+="--selected-networks=$selected_networks "
+        fi    
+        
+        cd integration-tests/e2e_tests_ci_tool/
+
+        # Mask secrets inside the test config override (like resolved env vars)
+        go run main.go test-config mask-secrets $cmd_args
+
+        echo $chainlink_version
+
+        # Create a base64 encoded string of the test config override
+        config_override=$(go run main.go test-config create $cmd_args)
+
+        BASE64_CONFIG_OVERRIDE=$(echo "$config_override" | base64 -w 0)
+        echo ::add-mask::$BASE64_CONFIG_OVERRIDE
+        echo "BASE64_CONFIG_OVERRIDE=$BASE64_CONFIG_OVERRIDE" >> $GITHUB_ENV
+
+    - name: Mask secrets in test config base64 override
+      id: mask-secrets-in-test-config-base64-override
+      if: ${{ inputs.test_config_base64_override_unsafe != '' }}
+      run: |
+        # Mask secrets inside the test config override
+        cd integration-tests/e2e_tests_ci_tool/
+        go run main.go test-config mask-secrets --from-base64-config ${{ inputs.test_config_base64_override_unsafe }}
+
+        exit 1 # Fail the job
+
+    - name: Setup GAP for Grafana
+      uses: smartcontractkit/.github/actions/setup-gap@6c9d62fdad050cfb8b59376ded291f1350705944 # setup-gap@0.2.2
+      with:
+        aws-region: ${{ secrets.AWS_REGION }}
+        aws-role-arn: ${{ secrets.AWS_OIDC_IAM_ROLE_VALIDATION_PROD_ARN }}
+        api-gateway-host: ${{ secrets.AWS_API_GW_HOST_GRAFANA }}
+        duplicate-authorization-header: "true"
+    - name: Run Tests
+      uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/run-tests@af92c5fae8dcf1659201e907db82d221fc304b94 # v2.3.21
+      with:
+        test_command_to_run: ${{ inputs.test.testCmd }} 2>&1 | tee /tmp/gotest.log | gotestloghelper -ci -singlepackage -hidepassingtests=false -hidepassinglogs
+        test_download_vendor_packages_command: cd ./integration-tests && go mod download
+        cl_repo: ${{ env.CHAINLINK_IMAGE }}
+        cl_image_tag: ${{ inputs.chainlink-version }}
+        aws_registries: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}
+        artifacts_name: ${{ inputs.test.id }}-test-logs
+        artifacts_location: |
+          ./integration-tests/smoke/logs/
+          /tmp/gotest.log
+        publish_check_name: ${{ inputs.test.id }}
+        token: ${{ secrets.GITHUB_TOKEN }}
+        go_mod_path: ./integration-tests/go.mod
+        cache_key_id: core-e2e-${{ env.MOD_CACHE_VERSION }}
+        cache_restore_only: "true"
+        QA_AWS_REGION: ${{ secrets.QA_AWS_REGION }}
+        QA_AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }}
+        QA_KUBECONFIG: ""
+        should_tidy: "false"
+        go_coverage_src_dir: /var/tmp/go-coverage
+        go_coverage_dest_dir: ${{ github.workspace }}/.covdata
\ No newline at end of file
diff --git a/.github/actions/setup-e2e-tests/action.yml b/.github/actions/setup-e2e-tests/action.yml
new file mode 100644
index 00000000000..feab30cbf81
--- /dev/null
+++ b/.github/actions/setup-e2e-tests/action.yml
@@ -0,0 +1,98 @@
+name: 'Setup E2E Tests'
+description: 'Setup and generate test matrices for Docker and Kubernetes tests.'
+inputs:
+  config-file-path:
+    required: true
+    description: 'Path to the configuration file for E2E tests'
+  enable-check-test-configurations:
+    required: true
+    description: 'Enable checking of test configurations'
+  test-ids:
+    required: false
+    description: 'Test IDs to filter on'
+  test-workflow:
+    required: false
+    description: 'Workflow name for filtering tests'
+outputs:
+  run-docker-tests:
+    description: 'Indicator to run Docker tests'
+    value: ${{ steps.check-matrices.outputs.run-docker-tests }}
+  run-k8s-tests:
+    description: 'Indicator to run Kubernetes tests'
+    value: ${{ steps.check-matrices.outputs.run-k8s-tests }}
+  docker-matrix:
+    description: 'JSON matrix for Docker tests'
+    value: ${{ steps.set-docker-matrix.outputs.matrix }}
+  k8s-runner-matrix:
+    description: 'JSON matrix for Kubernetes tests'
+    value: ${{ steps.set-k8s-runner-matrix.outputs.matrix }}
+
+runs:
+  using: 'composite'
+  steps:
+    - name: Install jq
+      run: sudo apt-get install jq
+      shell: bash
+
+    - name: Check Test Configurations
+      if: ${{ fromJSON(inputs.enable-check-test-configurations) }}
+      run: |
+        if ! go run integration-tests/e2e_tests_ci_tool/main.go check-tests ./integration-tests .github/e2e-tests.yml; then
+          echo "::error::Some E2E test configurations have to be added to .github/e2e-tests.yml." && exit 1
+        fi
+      shell: bash
+
+    - name: Set up Go
+      uses: actions/setup-go@v2
+      with:
+        go-version: '1.21.7'
+
+    - name: Generate Docker Tests Matrix
+      id: set-docker-matrix
+      run: |
+        cd integration-tests/e2e_tests_ci_tool/
+        MATRIX_JSON=$(go run main.go filter --file ${{ inputs.config-file-path }} --test-type 'docker' --test-ids '${{ inputs.test-ids }}' --workflow '${{ inputs.test-workflow }}')
+        echo "Docker tests:"
+        echo "$MATRIX_JSON" | jq
+        echo "matrix=$MATRIX_JSON" >> $GITHUB_ENV
+      shell: bash
+
+    - name: Generate K8s Tests Matrix
+      id: set-k8s-runner-matrix
+      run: |
+        cd integration-tests/e2e_tests_ci_tool/
+        MATRIX_JSON=$(go run main.go filter --file ${{ inputs.config-file-path }} --test-type 'k8s-remote-runner' --test-ids '${{ inputs.test-ids }}' --workflow '${{ inputs.test-workflow }}')
+        echo "K8s tests:"
+        echo "$MATRIX_JSON" | jq
+        echo "matrix=$MATRIX_JSON" >> $GITHUB_ENV
+      shell: bash
+
+    - name: Check Test Matrices
+      id: check-matrices
+      run: |
+        DOCKER_MATRIX_EMPTY=$(echo '${{ steps.set-docker-matrix.outputs.matrix }}' | jq '.tests == null or .tests == []')
+        K8S_MATRIX_EMPTY=$(echo '${{ steps.set-k8s-runner-matrix.outputs.matrix }}' | jq '.tests == null or .tests == []')
+
+        # Check if jq commands succeeded
+        if [ $? -ne 0 ]; then
+          echo "JSON parse error occurred."
+          exit 1
+        fi
+
+        if [[ "$DOCKER_MATRIX_EMPTY" == "true" ]]; then
+          echo "run-docker-tests=false" >> $GITHUB_ENV
+        else
+          echo "run-docker-tests=true" >> $GITHUB_ENV
+        fi
+        if [[ "$K8S_MATRIX_EMPTY" == "true" ]]; then
+          echo "run-k8s-tests=false" >> $GITHUB_ENV
+        else
+          echo "run-k8s-tests=true" >> $GITHUB_ENV
+        fi
+
+        # Check if both matrices are empty
+        if [[ "$DOCKER_MATRIX_EMPTY" == "true" ]] && [[ "$K8S_MATRIX_EMPTY" == "true" ]]; then
+          echo "No tests found for inputs: '${{ toJson(inputs) }}'. Both Docker and Kubernetes tests matrices are empty"
+          exit 1
+        fi
+      shell: bash   
\ No newline at end of file
diff --git a/.github/workflows/run-e2e-tests-reusable-workflow.yml b/.github/workflows/run-e2e-tests-reusable-workflow.yml
index 0191b1fffb4..c589a90c0cd 100644
--- a/.github/workflows/run-e2e-tests-reusable-workflow.yml
+++ b/.github/workflows/run-e2e-tests-reusable-workflow.yml
@@ -16,8 +16,8 @@ on:
         description: 'Run tests by workflow name. Example: "Run Nightly E2E Tests"'
         required: false
         type: string        
-      test-config-base64-override:
-        description: 'Custom base64 test config'
+      test-config-base64-override-unsafe:
+        description: 'This is an unsafe way to pass custom base64 test config. It cannot contain any secrets as this value is exposed in Github logs (Set up job -> Inputs) when the workflow is called! The caller workflow must make sure to remove secrets from this input.'
         required: false
         type: string        
       enable-check-test-configurations:
@@ -190,7 +190,7 @@ jobs:
           go-version: '1.21.7'
 
       - name: Set default test config override
-        if: ${{ inputs.test-config-base64-override == '' }}
+        if: ${{ inputs.test-config-base64-override-unsafe == '' }}
         run: |      
           # Mask secret args before using them
           # For more, see https://docs.github.com/en/actions/using-workflows/workflow-commands-for-github-actions#masking-a-value-in-a-log
@@ -238,38 +238,50 @@ jobs:
           echo ::add-mask::$BASE64_CONFIG_OVERRIDE
           echo "BASE64_CONFIG_OVERRIDE=$BASE64_CONFIG_OVERRIDE" >> $GITHUB_ENV
 
-      - name: Set custom test config override
-        if: ${{ inputs.test-config-base64-override != '' }}          
-        run: |
-          # Show what parts of the test config are being overridden
-          go run main.go test-config override \
-            --dry-run \
-            --from-base64-config ${{ inputs.test-config-base64-override }} \
-            --perycope-server-url ${{ secrets.QA_PYROSCOPE_INSTANCE }} \
-            --peryscope-key ${{ secrets.QA_PYROSCOPE_KEY }} \
-            --loki-endpoint https://${{ secrets.GRAFANA_INTERNAL_HOST }}/loki/api/v1/push \
-            --loki-tenant-id ${{ secrets.GRAFANA_INTERNAL_TENANT_ID }} \
-            --loki-basic-auth ${{ secrets.GRAFANA_INTERNAL_BASIC_AUTH }} \
-            --grafana-url "http://localhost:8080/primary" \
-            --grafana-dashboard-url "/d/ddf75041-1e39-42af-aa46-361fe4c36e9e/ci-e2e-tests-logs" \
-            --grafana-bearer-token ${{ secrets.GRAFANA_INTERNAL_URL_SHORTENER_TOKEN }} \
+      - name: Mask secrets in test config base64 override
+        id: mask-secrets-in-test-config-base64-override
+        if: ${{ inputs.test-config-base64-override-unsafe != '' }}
+        run: "to do"
+          # - name: Remove secrets from test config base64 override
+          #   id: mask-in
+          # - name: Set outputs
+          #   id: set-outputs
+          #   run: |
+          #     echo "::set-output name=base64ConfigNoSecrets::$BASE64_CONFIG_OVERRIDE"
 
-          # Override custom test config with CI secrets
-          config_override=$(go run main.go test-config override \
-            --from-base64-config ${{ inputs.test-config-base64-override }} \
-            --perycope-server-url ${{ secrets.QA_PYROSCOPE_INSTANCE }} \
-            --peryscope-key ${{ secrets.QA_PYROSCOPE_KEY }} \
-            --loki-endpoint https://${{ secrets.GRAFANA_INTERNAL_HOST }}/loki/api/v1/push \
-            --loki-tenant-id ${{ secrets.GRAFANA_INTERNAL_TENANT_ID }} \
-            --loki-basic-auth ${{ secrets.GRAFANA_INTERNAL_BASIC_AUTH }} \
-            --grafana-url "http://localhost:8080/primary" \
-            --grafana-dashboard-url "/d/ddf75041-1e39-42af-aa46-361fe4c36e9e/ci-e2e-tests-logs" \
-            --grafana-bearer-token ${{ secrets.GRAFANA_INTERNAL_URL_SHORTENER_TOKEN }} \
-          )
+      # TODO: Remove this step since we allow custom secrets in the test config but we mask them    
+      # - name: Override secrets in custom test config
+      #   if: ${{ inputs.test-config-base64-override-unsafe != '' }}          
+      #   run: |
+      #     # Show what parts of the test config are being overridden
+      #     go run main.go test-config override \
+      #       --dry-run \
+      #       --from-base64-config ${{ inputs.test-config-base64-override-unsafe }} \
+      #       --perycope-server-url ${{ secrets.QA_PYROSCOPE_INSTANCE }} \
+      #       --peryscope-key ${{ secrets.QA_PYROSCOPE_KEY }} \
+      #       --loki-endpoint https://${{ secrets.GRAFANA_INTERNAL_HOST }}/loki/api/v1/push \
+      #       --loki-tenant-id ${{ secrets.GRAFANA_INTERNAL_TENANT_ID }} \
+      #       --loki-basic-auth ${{ secrets.GRAFANA_INTERNAL_BASIC_AUTH }} \
+      #       --grafana-url "http://localhost:8080/primary" \
+      #       --grafana-dashboard-url "/d/ddf75041-1e39-42af-aa46-361fe4c36e9e/ci-e2e-tests-logs" \
+      #       --grafana-bearer-token ${{ secrets.GRAFANA_INTERNAL_URL_SHORTENER_TOKEN }}
 
-          BASE64_CONFIG_OVERRIDE=$(echo "$config_override" | base64 -w 0)
-          echo ::add-mask::$BASE64_CONFIG_OVERRIDE
-          echo "BASE64_CONFIG_OVERRIDE=$BASE64_CONFIG_OVERRIDE" >> $GITHUB_ENV
+      #     # Override custom test config with CI secrets
+      #     config_override=$(go run main.go test-config override \
+      #       --from-base64-config ${{ inputs.test-config-base64-override-unsafe }} \
+      #       --perycope-server-url ${{ secrets.QA_PYROSCOPE_INSTANCE }} \
+      #       --peryscope-key ${{ secrets.QA_PYROSCOPE_KEY }} \
+      #       --loki-endpoint https://${{ secrets.GRAFANA_INTERNAL_HOST }}/loki/api/v1/push \
+      #       --loki-tenant-id ${{ secrets.GRAFANA_INTERNAL_TENANT_ID }} \
+      #       --loki-basic-auth ${{ secrets.GRAFANA_INTERNAL_BASIC_AUTH }} \
+      #       --grafana-url "http://localhost:8080/primary" \
+      #       --grafana-dashboard-url "/d/ddf75041-1e39-42af-aa46-361fe4c36e9e/ci-e2e-tests-logs" \
+      #       --grafana-bearer-token ${{ secrets.GRAFANA_INTERNAL_URL_SHORTENER_TOKEN }}
+      #     )
+
+      #     BASE64_CONFIG_OVERRIDE=$(echo "$config_override" | base64 -w 0)
+      #     echo ::add-mask::$BASE64_CONFIG_OVERRIDE
+      #     echo "BASE64_CONFIG_OVERRIDE=$BASE64_CONFIG_OVERRIDE" >> $GITHUB_ENV
 
       - name: Setup GAP for Grafana
         uses: smartcontractkit/.github/actions/setup-gap@6c9d62fdad050cfb8b59376ded291f1350705944 # setup-gap@0.2.2
diff --git a/.github/workflows/run-selected-e2e-tests.yml b/.github/workflows/run-selected-e2e-tests.yml
index 04d73fd43ce..0cf91068e58 100644
--- a/.github/workflows/run-selected-e2e-tests.yml
+++ b/.github/workflows/run-selected-e2e-tests.yml
@@ -5,18 +5,20 @@ on:
     inputs:
       chainlink-version:
         description: 'Enter Chainlink version to use for the tests. Example: "v2.10.0" or sha'
-        default: develop
-        required: false
+        required: true
         type: string
       test-ids:
-        description: 'Run all tests "*" by default. Or, enter test IDs to run separated by commas. Example: "run_all_in_ocr_tests_go,run_TestOCRv2Request_in_ocr2_test_go". Check all test IDs in .github/e2e-tests.yml'
-        default: "*"
-        required: true
+        description: 'Run tests by test ids separated by commas. Example: "run_all_in_ocr_tests_go,run_TestOCRv2Request_in_ocr2_test_go". Check all test IDs in .github/e2e-tests.yml'
+        required: false
         type: string
-      test-config-base64-override:
-        description: 'Custom base64 test config'
+      test-workflow:
+        description: 'Run tests by workflow name. Example: "Run Nightly E2E Tests"'
         required: false
-        type: string          
+        type: string        
+      test_config_base64_override_unsafe:
+        description: 'This is an unsafe way to pass custom base64 test config. It cannot contain any secrets as this value is exposed in Github logs (Set up job -> Inputs) when the workflow is called! The caller workflow must make sure to remove secrets from this input.'
+        required: false
+        type: string        
       enable-check-test-configurations:
         description: 'Set to "true" to enable check-test-configurations job'
         required: false
@@ -25,38 +27,172 @@ on:
       with-existing-remote-runner-version:
         description: 'Use the existing remote runner version for k8s tests. Example: "d3bf5044af33e08be788a2df31c4a745cf69d787"'
         required: false
-        type: string                 
+        type: string
+
+env: 
+  CHAINLINK_IMAGE: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com/chainlink
 
 jobs:
-  call-run-e2e-tests-workflow:
-    name: Run E2E Tests
-    uses: ./.github/workflows/run-e2e-tests-reusable-workflow.yml
+  validate-inputs:
+    name: Validate Workflow Inputs
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check input conditions
+        run: |
+          if [[ "${{ inputs.test-ids }}" != "" && "${{ inputs.test-workflow }}" != "" ]]; then
+            echo "Error: Both 'test-ids' and 'test-workflow' are provided. Please specify only one."
+            exit 1
+          fi
+
+  setup-e2e-tests:
+    name: Setup E2E Tests
+    runs-on: ubuntu-latest
+    outputs:
+      run-docker-tests: ${{ steps.setup.outputs.run-docker-tests }}
+      run-k8s-tests: ${{ steps.setup.outputs.run-k8s-tests }}
+      docker-matrix: ${{ steps.setup.outputs.docker-matrix }}
+      k8s-runner-matrix: ${{ steps.setup.outputs.k8s-runner-matrix }}
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2
+      - name: Setup E2E Test Configurations
+        id: setup
+        uses: ./.github/actions/setup-e2e-tests
+        with:
+          config-file-path: ${{ github.workspace }}/.github/e2e-tests.yml
+          enable-check-test-configurations: ${{ inputs.enable-check-test-configurations }}
+          test-ids: ${{ inputs.test-ids }}
+          test-workflow: ${{ inputs.test-workflow }}
+
+  # Run Docker tests
+  docker-tests:
+    name: Run Docker Tests (${{ matrix.tests.id }})
+    needs: setup-e2e-tests
+    if: ${{ needs.setup-e2e-tests.outputs.run-docker-tests == 'true' }}
+    runs-on: ${{ matrix.tests.runsOn }}
+    strategy:
+      fail-fast: false
+      matrix: ${{fromJson(needs.setup-e2e-tests.outputs.docker-matrix)}}
+    environment: integration
+    permissions:
+      actions: read
+      checks: write
+      pull-requests: write
+      id-token: write
+      contents: read
+    env:
+      SELECTED_NETWORKS: SIMULATED
+      CHAINLINK_ENV_USER: ${{ github.actor }}
+      TEST_LOG_LEVEL: debug
+
+    steps:
+      - name: Mask workflow inputs that are secrets
+        if: ${{ inputs.test_config_base64_override_unsafe != '' }}
+        run: |
+          # Mask test_config_base64_override_unsafe input
+          TEST_CONFIG_INPUT=$(jq -r '.inputs.test_config_base64_override_unsafe' $GITHUB_EVENT_PATH)
+          echo ::add-mask::$TEST_CONFIG_INPUT
+
+      - name: Checkout repository
+        uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2
+
+      - name: Run Docker Test
+        id: setup
+        uses: ./.github/actions/run-e2e-docker-test
+        with: 
+          test: ${{ matrix.tests }}
+          test_config_base64_override_unsafe: ${{ inputs.test_config_base64_override_unsafe }}
+
+  # Run K8s tests using old remote runner
+
+  prepare-remote-runner-test-image:
+    needs: setup-e2e-tests
+    if: ${{ needs.setup-e2e-tests.outputs.run-k8s-tests == 'true' }}
+    name: Prepare Remote Runner Test Image
+    runs-on: ubuntu-latest
+    environment: integration
+    permissions:
+      actions: read
+      checks: write
+      pull-requests: write
+      id-token: write
+      contents: read
+    outputs:
+      remote-runner-version: ${{ steps.set-remote-runner-version.outputs.remote-runner-version }}
+    env:
+      ENV_JOB_IMAGE_BASE: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com/chainlink-tests
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2      
+      - name: Build Test Runner Image
+        uses: ./.github/actions/build-test-image
+        if: ${{ inputs.with-existing-remote-runner-version == '' }}
+        with:
+          QA_AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }}
+          QA_AWS_REGION: ${{ secrets.QA_AWS_REGION }}
+          QA_AWS_ACCOUNT_NUMBER: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}
+      - name: Set Remote Runner Version
+        id: set-remote-runner-version
+        run: |
+          if [[ -z "${{ inputs.with-existing-remote-runner-version }}" ]]; then
+            echo "::set-output name=remote-runner-version::${{ github.sha }}"
+          else
+            echo "::set-output name=remote-runner-version::${{ inputs.with-existing-remote-runner-version }}"
+          fi
+
+  run-k8s-runner-tests:
+    needs: [setup-e2e-tests, prepare-remote-runner-test-image]
+    if: ${{ needs.setup-e2e-tests.outputs.run-k8s-tests == 'true' }}
+    name: Run K8s Tests (${{ matrix.tests.id }})
+    runs-on: ${{ matrix.tests.runsOn }}
+    strategy:
+      fail-fast: false
+      matrix: ${{fromJson(needs.setup-e2e-tests.outputs.k8s-runner-matrix)}}
+    environment: integration
     permissions:
       actions: read
       checks: write
       pull-requests: write
       id-token: write
-      contents: read    
-    with:
-      chainlink-version: ${{ github.event.inputs.chainlink-version }}
-      test-ids: ${{ github.event.inputs.test-ids }}
-      test-config-base64-override: ${{ github.event.inputs.test-config-base64-override }}
-      with-existing-remote-runner-version: ${{ github.event.inputs.with-existing-remote-runner-version }}
-      # Use fromJSON to convert string to boolean. More info: https://github.com/actions/runner/issues/2206#issuecomment-1532246677
-      enable-check-test-configurations: ${{ fromJSON(github.event.inputs.enable-check-test-configurations) }}
-    secrets:
-      QA_AWS_REGION: ${{ secrets.QA_AWS_REGION }}
-      QA_AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }}
-      QA_AWS_ACCOUNT_NUMBER: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}
-      QA_PYROSCOPE_INSTANCE: ${{ secrets.QA_PYROSCOPE_INSTANCE }}
-      QA_PYROSCOPE_KEY: ${{ secrets.QA_PYROSCOPE_KEY }}
-      QA_KUBECONFIG: ${{ secrets.QA_KUBECONFIG }}
-      GRAFANA_INTERNAL_TENANT_ID: ${{ secrets.GRAFANA_INTERNAL_TENANT_ID }}
-      GRAFANA_INTERNAL_BASIC_AUTH: ${{ secrets.GRAFANA_INTERNAL_BASIC_AUTH }}
-      GRAFANA_INTERNAL_HOST: ${{ secrets.GRAFANA_INTERNAL_HOST }}
-      GRAFANA_INTERNAL_URL_SHORTENER_TOKEN: ${{ secrets.GRAFANA_INTERNAL_URL_SHORTENER_TOKEN }}
-      GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}  
-      AWS_REGION: ${{ secrets.QA_AWS_REGION }}
-      AWS_OIDC_IAM_ROLE_VALIDATION_PROD_ARN: ${{ secrets.AWS_OIDC_IAM_ROLE_VALIDATION_PROD_ARN }}
-      AWS_API_GW_HOST_GRAFANA: ${{ secrets.AWS_API_GW_HOST_GRAFANA }}        
+      contents: read
+    env:
+      SELECTED_NETWORKS: SIMULATED
+      CHAINLINK_COMMIT_SHA: ${{ inputs.evm-ref || github.sha }}
+      CHAINLINK_ENV_USER: ${{ github.actor }}
+      TEST_LOG_LEVEL: debug
 
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2
+      - name: Install jq
+        run: sudo apt-get install -y jq
+      - name: Show Test Configuration
+        run: echo '${{ toJson(matrix.tests) }}' | jq .
+      - name: Show Remote Runner Version
+        run: |
+          echo "Remote Runner Version: ${{ needs.prepare-remote-runner-test-image.outputs.remote-runner-version }}"
+      - name: Run Tests
+        uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/run-tests@af92c5fae8dcf1659201e907db82d221fc304b94 # v2.3.21
+        env:
+          DETACH_RUNNER: true
+          TEST_SUITE: ${{ matrix.tests.remoteRunnerTestSuite }}
+          TEST_TYPE: ${{ matrix.tests.remoteRunnerTestSuite }}
+          TEST_TEST_TYPE: ${{ matrix.tests.remoteRunnerTestSuite }}
+          RR_MEM: ${{ matrix.tests.remoteRunnerMemory }}
+          TEST_ARGS: -test.timeout 900h -test.memprofile memprofile.out -test.cpuprofile profile.out
+          ENV_JOB_IMAGE: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com/chainlink-tests:${{ needs.prepare-remote-runner-test-image.outputs.remote-runner-version }}
+          INTERNAL_DOCKER_REPO: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com
+          # We can comment these out when we have a stable soak test and aren't worried about resource consumption
+          TEST_UPLOAD_CPU_PROFILE: true
+          TEST_UPLOAD_MEM_PROFILE: true
+        with:
+          test_command_to_run: ${{ matrix.tests.testCmd }}
+          test_download_vendor_packages_command: make gomod
+          cl_repo: ${{ env.CHAINLINK_IMAGE }}
+          cl_image_tag: ${{ env.CHAINLINK_VERSION }}
+          token: ${{ secrets.GH_TOKEN }}
+          should_cleanup: false
+          go_mod_path: ./integration-tests/go.mod
+          QA_AWS_REGION: ${{ secrets.QA_AWS_REGION }}
+          QA_AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }}
+          QA_KUBECONFIG: ${{ secrets.QA_KUBECONFIG }} 
diff --git a/go.mod b/go.mod
index e7d53d98be0..2bb02a8fa4d 100644
--- a/go.mod
+++ b/go.mod
@@ -54,7 +54,6 @@ require (
 	github.com/mitchellh/go-homedir v1.1.0
 	github.com/mitchellh/mapstructure v1.5.0
 	github.com/mr-tron/base58 v1.2.0
-	github.com/naoina/toml v0.1.2-0.20170918210437-9fafd6967416
 	github.com/olekukonko/tablewriter v0.0.5
 	github.com/onsi/gomega v1.30.0
 	github.com/patrickmn/go-cache v2.1.0+incompatible
@@ -79,14 +78,12 @@ require (
 	github.com/smartcontractkit/chainlink-feeds v0.0.0-20240522213638-159fb2d99917
 	github.com/smartcontractkit/chainlink-solana v1.0.3-0.20240605170242-555ff582f36a
 	github.com/smartcontractkit/chainlink-starknet/relayer v0.0.1-beta-test.0.20240531021326-99118e47f696
-	github.com/smartcontractkit/chainlink-testing-framework v1.31.4
 	github.com/smartcontractkit/chainlink-vrf v0.0.0-20231120191722-fef03814f868
 	github.com/smartcontractkit/libocr v0.0.0-20240419185742-fd3cab206b2c
 	github.com/smartcontractkit/tdh2/go/ocr2/decryptionplugin v0.0.0-20230906073235-9e478e5e19f1
 	github.com/smartcontractkit/tdh2/go/tdh2 v0.0.0-20230906073235-9e478e5e19f1
 	github.com/smartcontractkit/wsrpc v0.8.1
 	github.com/spf13/cast v1.6.0
-	github.com/spf13/cobra v1.8.1
 	github.com/stretchr/testify v1.9.0
 	github.com/test-go/testify v1.1.4
 	github.com/theodesp/go-heaps v0.0.0-20190520121037-88e35354fe0a
@@ -116,8 +113,6 @@ require (
 	google.golang.org/protobuf v1.33.0
 	gopkg.in/guregu/null.v4 v4.0.0
 	gopkg.in/natefinch/lumberjack.v2 v2.2.1
-	gopkg.in/yaml.v2 v2.4.0
-	gopkg.in/yaml.v3 v3.0.1
 )
 
 require (
@@ -127,26 +122,18 @@ require (
 	cosmossdk.io/depinject v1.0.0-alpha.3 // indirect
 	cosmossdk.io/errors v1.0.0 // indirect
 	cosmossdk.io/math v1.0.1 // indirect
-	dario.cat/mergo v1.0.0 // indirect
 	filippo.io/edwards25519 v1.0.0 // indirect
 	github.com/99designs/go-keychain v0.0.0-20191008050251-8e49817e8af4 // indirect
 	github.com/99designs/keyring v1.2.1 // indirect
-	github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161 // indirect
 	github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358 // indirect
 	github.com/ChainSafe/go-schnorrkel v0.0.0-20200405005733-88cbf1b4c40d // indirect
 	github.com/CosmWasm/wasmd v0.40.1 // indirect
 	github.com/CosmWasm/wasmvm v1.2.4 // indirect
 	github.com/DataDog/zstd v1.5.2 // indirect
-	github.com/MakeNowJust/heredoc v1.0.0 // indirect
 	github.com/Masterminds/goutils v1.1.1 // indirect
 	github.com/Microsoft/go-winio v0.6.1 // indirect
-	github.com/Microsoft/hcsshim v0.11.4 // indirect
 	github.com/VictoriaMetrics/fastcache v1.12.1 // indirect
-	github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137 // indirect
 	github.com/armon/go-metrics v0.4.1 // indirect
-	github.com/avast/retry-go v3.0.0+incompatible // indirect
-	github.com/aws/constructs-go/constructs/v10 v10.1.255 // indirect
-	github.com/aws/jsii-runtime-go v1.75.0 // indirect
 	github.com/bahlo/generic-list-go v0.2.0 // indirect
 	github.com/benbjohnson/clock v1.3.5 // indirect
 	github.com/beorn7/perks v1.0.1 // indirect
@@ -155,13 +142,10 @@ require (
 	github.com/blendle/zapdriver v1.3.1 // indirect
 	github.com/buger/jsonparser v1.1.1 // indirect
 	github.com/bytedance/sonic v1.10.1 // indirect
-	github.com/cdk8s-team/cdk8s-core-go/cdk8s/v2 v2.7.5 // indirect
 	github.com/cenkalti/backoff v2.2.1+incompatible // indirect
 	github.com/cenkalti/backoff/v4 v4.2.1 // indirect
 	github.com/cespare/xxhash v1.1.0 // indirect
 	github.com/cespare/xxhash/v2 v2.2.0 // indirect
-	github.com/chai2010/gettext-go v1.0.2 // indirect
-	github.com/chaos-mesh/chaos-mesh/api/v1alpha1 v0.0.0-20220226050744-799408773657 // indirect
 	github.com/chenzhuoyu/base64x v0.0.0-20230717121745-296ad89f973d // indirect
 	github.com/chenzhuoyu/iasm v0.9.0 // indirect
 	github.com/cockroachdb/errors v1.9.1 // indirect
@@ -173,8 +157,6 @@ require (
 	github.com/confio/ics23/go v0.9.0 // indirect
 	github.com/consensys/bavard v0.1.13 // indirect
 	github.com/consensys/gnark-crypto v0.12.1 // indirect
-	github.com/containerd/containerd v1.7.12 // indirect
-	github.com/containerd/log v0.1.0 // indirect
 	github.com/cosmos/btcutil v1.0.5 // indirect
 	github.com/cosmos/cosmos-proto v1.0.0-beta.2 // indirect
 	github.com/cosmos/go-bip39 v1.0.0 // indirect
@@ -183,7 +165,6 @@ require (
 	github.com/cosmos/ibc-go/v7 v7.0.1 // indirect
 	github.com/cosmos/ics23/go v0.9.1-0.20221207100636-b1abd8678aab // indirect
 	github.com/cosmos/ledger-cosmos-go v0.12.1 // indirect
-	github.com/cpuguy83/dockercfg v0.3.1 // indirect
 	github.com/cpuguy83/go-md2man/v2 v2.0.4 // indirect
 	github.com/crate-crypto/go-ipa v0.0.0-20231025140028-3c0104f4b233 // indirect
 	github.com/crate-crypto/go-kzg-4844 v0.7.0 // indirect
@@ -195,22 +176,13 @@ require (
 	github.com/dgraph-io/badger/v2 v2.2007.4 // indirect
 	github.com/dgraph-io/ristretto v0.1.1 // indirect
 	github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13 // indirect
-	github.com/distribution/reference v0.5.0 // indirect
 	github.com/docker/distribution v2.8.2+incompatible // indirect
 	github.com/docker/docker v25.0.2+incompatible // indirect
 	github.com/docker/go-connections v0.5.0 // indirect
-	github.com/docker/go-units v0.5.0 // indirect
 	github.com/dustin/go-humanize v1.0.1 // indirect
 	github.com/dvsekhvalnov/jose2go v1.7.0 // indirect
-	github.com/emicklei/go-restful/v3 v3.10.2 // indirect
 	github.com/ethereum/c-kzg-4844 v0.4.0 // indirect
-	github.com/evanphx/json-patch v5.6.0+incompatible // indirect
-	github.com/evanphx/json-patch/v5 v5.6.0 // indirect
-	github.com/exponent-io/jsonpath v0.0.0-20210407135951-1de76d718b3f // indirect
-	github.com/fatih/camelcase v1.0.0 // indirect
-	github.com/felixge/httpsnoop v1.0.3 // indirect
 	github.com/fsnotify/fsnotify v1.6.0 // indirect
-	github.com/fvbommel/sortorder v1.0.2 // indirect
 	github.com/gabriel-vasile/mimetype v1.4.2 // indirect
 	github.com/gagliardetto/binary v0.7.7 // indirect
 	github.com/gagliardetto/treeout v0.1.4 // indirect
@@ -219,7 +191,6 @@ require (
 	github.com/gedex/inflector v0.0.0-20170307190818-16278e9db813 // indirect
 	github.com/gin-contrib/sse v0.1.0 // indirect
 	github.com/go-asn1-ber/asn1-ber v1.5.5 // indirect
-	github.com/go-errors/errors v1.4.2 // indirect
 	github.com/go-json-experiment/json v0.0.0-20231102232822-2e55bd4e08b0 // indirect
 	github.com/go-kit/kit v0.12.0 // indirect
 	github.com/go-kit/log v0.2.1 // indirect
@@ -227,13 +198,9 @@ require (
 	github.com/go-logr/logr v1.4.1 // indirect
 	github.com/go-logr/stdr v1.2.2 // indirect
 	github.com/go-ole/go-ole v1.2.6 // indirect
-	github.com/go-openapi/jsonpointer v0.20.0 // indirect
-	github.com/go-openapi/jsonreference v0.20.2 // indirect
-	github.com/go-openapi/swag v0.22.4 // indirect
 	github.com/go-playground/locales v0.14.1 // indirect
 	github.com/go-playground/universal-translator v0.18.1 // indirect
 	github.com/go-playground/validator/v10 v10.15.5 // indirect
-	github.com/go-resty/resty/v2 v2.7.0 // indirect
 	github.com/go-webauthn/x v0.1.5 // indirect
 	github.com/goccy/go-json v0.10.2 // indirect
 	github.com/godbus/dbus v0.0.0-20190726142602-4481cbc300e2 // indirect
@@ -245,15 +212,11 @@ require (
 	github.com/golang/protobuf v1.5.4 // indirect
 	github.com/golang/snappy v0.0.5-0.20220116011046-fa5810519dcb // indirect
 	github.com/google/btree v1.1.2 // indirect
-	github.com/google/gnostic v0.6.9 // indirect
-	github.com/google/gnostic-models v0.6.8 // indirect
 	github.com/google/go-cmp v0.6.0 // indirect
 	github.com/google/go-tpm v0.9.0 // indirect
 	github.com/google/gofuzz v1.2.0 // indirect
-	github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 // indirect
 	github.com/gorilla/context v1.1.1 // indirect
 	github.com/grafana/pyroscope-go/godeltaprof v0.1.6 // indirect
-	github.com/gregjones/httpcache v0.0.0-20190611155906-901d90724c79 // indirect
 	github.com/grpc-ecosystem/go-grpc-middleware v1.3.0 // indirect
 	github.com/grpc-ecosystem/go-grpc-middleware/providers/prometheus v1.0.0 // indirect
 	github.com/grpc-ecosystem/go-grpc-middleware/v2 v2.0.0-rc.3 // indirect
@@ -284,18 +247,14 @@ require (
 	github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a // indirect
 	github.com/jackpal/go-nat-pmp v1.0.2 // indirect
 	github.com/jmhodges/levigo v1.0.0 // indirect
-	github.com/josharian/intern v1.0.0 // indirect
 	github.com/json-iterator/go v1.1.12 // indirect
-	github.com/kelseyhightower/envconfig v1.4.0 // indirect
 	github.com/klauspost/compress v1.17.3 // indirect
 	github.com/klauspost/cpuid/v2 v2.2.5 // indirect
 	github.com/kr/pretty v0.3.1 // indirect
 	github.com/kr/text v0.2.0 // indirect
 	github.com/leodido/go-urn v1.2.4 // indirect
 	github.com/libp2p/go-buffer-pool v0.1.0 // indirect
-	github.com/liggitt/tabwriter v0.0.0-20181228230101-89fcab3d43de // indirect
 	github.com/logrusorgru/aurora v2.0.3+incompatible // indirect
-	github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 // indirect
 	github.com/magiconair/properties v1.8.7 // indirect
 	github.com/mailru/easyjson v0.7.7 // indirect
 	github.com/mattn/go-colorable v0.1.13 // indirect
@@ -305,46 +264,31 @@ require (
 	github.com/mimoo/StrobeGo v0.0.0-20210601165009-122bf33a46e0 // indirect
 	github.com/mitchellh/copystructure v1.2.0 // indirect
 	github.com/mitchellh/go-testing-interface v1.14.1 // indirect
-	github.com/mitchellh/go-wordwrap v1.0.1 // indirect
 	github.com/mitchellh/reflectwalk v1.0.2 // indirect
 	github.com/mmcloughlin/addchain v0.4.0 // indirect
-	github.com/moby/patternmatcher v0.6.0 // indirect
-	github.com/moby/spdystream v0.2.0 // indirect
-	github.com/moby/sys/sequential v0.5.0 // indirect
-	github.com/moby/sys/user v0.1.0 // indirect
-	github.com/moby/term v0.5.0 // indirect
 	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
 	github.com/modern-go/reflect2 v1.0.2 // indirect
-	github.com/monochromegane/go-gitignore v0.0.0-20200626010858-205db1a8cc00 // indirect
-	github.com/montanaflynn/stats v0.7.1 // indirect
-	github.com/morikuni/aec v1.0.0 // indirect
 	github.com/mostynb/zstdpool-freelist v0.0.0-20201229113212-927304c0c3b1 // indirect
 	github.com/mtibben/percent v0.2.1 // indirect
-	github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
 	github.com/mwitkow/grpc-proxy v0.0.0-20230212185441-f345521cb9c9 // indirect
-	github.com/naoina/go-stringutil v0.1.0 // indirect
 	github.com/oklog/run v1.1.0 // indirect
 	github.com/opencontainers/go-digest v1.0.0 // indirect
-	github.com/opencontainers/image-spec v1.1.0-rc5 // indirect
 	github.com/opentracing/opentracing-go v1.2.0 // indirect
-	github.com/peterbourgon/diskv v2.0.1+incompatible // indirect
 	github.com/petermattis/goid v0.0.0-20230317030725-371a4b8eda08 // indirect
 	github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
 	github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c // indirect
 	github.com/prometheus/procfs v0.12.0 // indirect
 	github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 // indirect
 	github.com/rivo/uniseg v0.4.4 // indirect
-	github.com/rs/zerolog v1.30.0 // indirect
-	github.com/russross/blackfriday v1.6.0 // indirect
+	github.com/rs/cors v1.9.0 // indirect
 	github.com/russross/blackfriday/v2 v2.1.0 // indirect
 	github.com/santhosh-tekuri/jsonschema/v5 v5.3.1 // indirect
 	github.com/sasha-s/go-deadlock v0.3.1 // indirect
 	github.com/sethvargo/go-retry v0.2.4 // indirect
 	github.com/shirou/gopsutil v3.21.11+incompatible // indirect
-	github.com/shoenig/go-m1cpu v0.1.6 // indirect
 	github.com/sirupsen/logrus v1.9.3 // indirect
-	github.com/smartcontractkit/seth v1.0.11 // indirect
 	github.com/spf13/afero v1.9.5 // indirect
+	github.com/spf13/cobra v1.8.1 // indirect
 	github.com/spf13/jwalterweatherman v1.1.0 // indirect
 	github.com/spf13/pflag v1.0.5 // indirect
 	github.com/spf13/viper v1.15.0 // indirect
@@ -357,7 +301,6 @@ require (
 	github.com/tecbot/gorocksdb v0.0.0-20191217155057-f0fad39f321c // indirect
 	github.com/tendermint/go-amino v0.16.0 // indirect
 	github.com/teris-io/shortid v0.0.0-20201117134242-e59966efd125 // indirect
-	github.com/testcontainers/testcontainers-go v0.28.0 // indirect
 	github.com/tidwall/btree v1.6.0 // indirect
 	github.com/tidwall/match v1.1.1 // indirect
 	github.com/tidwall/pretty v1.2.0 // indirect
@@ -369,7 +312,6 @@ require (
 	github.com/valyala/fastjson v1.4.1 // indirect
 	github.com/wk8/go-ordered-map/v2 v2.1.8 // indirect
 	github.com/x448/float16 v0.8.4 // indirect
-	github.com/xlab/treeprint v1.1.0 // indirect
 	github.com/yusufpapurcu/wmi v1.2.4 // indirect
 	github.com/zondax/hid v0.9.1 // indirect
 	github.com/zondax/ledger-go v0.14.1 // indirect
@@ -378,45 +320,25 @@ require (
 	go.mongodb.org/mongo-driver v1.15.0 // indirect
 	go.opencensus.io v0.24.0 // indirect
 	go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.46.1 // indirect
-	go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.45.0 // indirect
 	go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.21.0 // indirect
 	go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.21.0 // indirect
 	go.opentelemetry.io/otel/metric v1.24.0 // indirect
 	go.opentelemetry.io/otel/sdk v1.21.0 // indirect
 	go.opentelemetry.io/otel/trace v1.24.0 // indirect
 	go.opentelemetry.io/proto/otlp v1.0.0 // indirect
-	go.starlark.net v0.0.0-20220817180228-f738f5508c12 // indirect
-	go.uber.org/atomic v1.11.0 // indirect
 	go.uber.org/ratelimit v0.3.0 // indirect
 	golang.org/x/arch v0.7.0 // indirect
-	golang.org/x/oauth2 v0.17.0 // indirect
 	golang.org/x/sys v0.19.0 // indirect
-	gomodules.xyz/jsonpatch/v2 v2.2.0 // indirect
 	google.golang.org/api v0.149.0 // indirect
-	google.golang.org/appengine v1.6.8 // indirect
 	google.golang.org/genproto v0.0.0-20231030173426-d783a09b4405 // indirect
 	google.golang.org/genproto/googleapis/api v0.0.0-20231016165738-49dd2c1f3d0b // indirect
 	google.golang.org/genproto/googleapis/rpc v0.0.0-20231106174013-bbf56f31fb17 // indirect
 	gopkg.in/guregu/null.v2 v2.1.2 // indirect
-	gopkg.in/inf.v0 v0.9.1 // indirect
 	gopkg.in/ini.v1 v1.67.0 // indirect
-	k8s.io/api v0.28.2 // indirect
-	k8s.io/apiextensions-apiserver v0.25.3 // indirect
-	k8s.io/apimachinery v0.28.2 // indirect
-	k8s.io/cli-runtime v0.25.11 // indirect
-	k8s.io/client-go v0.28.2 // indirect
-	k8s.io/component-base v0.26.2 // indirect
-	k8s.io/klog/v2 v2.100.1 // indirect
-	k8s.io/kube-openapi v0.0.0-20230717233707-2695361300d9 // indirect
-	k8s.io/kubectl v0.25.11 // indirect
-	k8s.io/utils v0.0.0-20230711102312-30195339c3c7 // indirect
+	gopkg.in/yaml.v2 v2.4.0 // indirect
+	gopkg.in/yaml.v3 v3.0.1 // indirect
 	pgregory.net/rapid v0.5.5 // indirect
 	rsc.io/tmplfunc v0.0.3 // indirect
-	sigs.k8s.io/controller-runtime v0.13.0 // indirect
-	sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect
-	sigs.k8s.io/kustomize/api v0.12.1 // indirect
-	sigs.k8s.io/kustomize/kyaml v0.13.9 // indirect
-	sigs.k8s.io/structured-merge-diff/v4 v4.3.0 // indirect
 	sigs.k8s.io/yaml v1.4.0 // indirect
 )
 
diff --git a/go.sum b/go.sum
index a086b130d83..278ece81ab5 100644
--- a/go.sum
+++ b/go.sum
@@ -65,8 +65,6 @@ cosmossdk.io/math v1.0.1 h1:Qx3ifyOPaMLNH/89WeZFH268yCvU4xEcnPLu3sJqPPg=
 cosmossdk.io/math v1.0.1/go.mod h1:Ygz4wBHrgc7g0N+8+MrnTfS9LLn9aaTGa9hKopuym5k=
 cosmossdk.io/tools/rosetta v0.2.1 h1:ddOMatOH+pbxWbrGJKRAawdBkPYLfKXutK9IETnjYxw=
 cosmossdk.io/tools/rosetta v0.2.1/go.mod h1:Pqdc1FdvkNV3LcNIkYWt2RQY6IP1ge6YWZk8MhhO9Hw=
-dario.cat/mergo v1.0.0 h1:AGCNq9Evsj31mOgNPcLyXc+4PNABt905YmuqPYYpBWk=
-dario.cat/mergo v1.0.0/go.mod h1:uNxQE+84aUszobStD9th8a29P2fMDhsBdgRYvZOxGmk=
 dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU=
 filippo.io/edwards25519 v1.0.0-rc.1/go.mod h1:N1IkdkCkiLB6tki+MYJoSx2JTY9NUlxZE7eHn5EwJns=
 filippo.io/edwards25519 v1.0.0 h1:0wAIcmJUqRdI8IJ/3eGi5/HwXZWPujYXXlkrQogz0Ek=
@@ -105,8 +103,6 @@ github.com/Depado/ginprom v1.8.0/go.mod h1:XBaKzeNBqPF4vxJpNLincSQZeMDnZp1tIbU0F
 github.com/GeertJohan/go.incremental v1.0.0/go.mod h1:6fAjUhbVuX1KcMD3c8TEgVUqmo4seqhv0i0kdATSkM0=
 github.com/GeertJohan/go.rice v1.0.0/go.mod h1:eH6gbSOAUv07dQuZVnBmoDP8mgsM1rtixis4Tib9if0=
 github.com/Joker/hpp v1.0.0/go.mod h1:8x5n+M1Hp5hC0g8okX3sR3vFQwynaX/UgSOM9MeBKzY=
-github.com/MakeNowJust/heredoc v1.0.0 h1:cXCdzVdstXyiTqTvfqk9SDHpKNjxuom+DOlyEeQ4pzQ=
-github.com/MakeNowJust/heredoc v1.0.0/go.mod h1:mG5amYoWBHf8vpLOuehzbGGw0EHxpZZ6lCpQ4fNJ8LE=
 github.com/Masterminds/goutils v1.1.1 h1:5nUrii3FMTL5diU80unEVvNevw1nH4+ZV4DSLVJLSYI=
 github.com/Masterminds/goutils v1.1.1/go.mod h1:8cTjp+g8YejhMuvIA5y2vz3BpJxksy863GQaJW2MFNU=
 github.com/Masterminds/semver/v3 v3.1.1/go.mod h1:VPu/7SZ7ePZ3QOrcuXROw5FAcLl4a0cBrbBpGY/8hQs=
@@ -117,8 +113,6 @@ github.com/Masterminds/sprig/v3 v3.2.3 h1:eL2fZNezLomi0uOLqjQoN6BfsDD+fyLtgbJMAj
 github.com/Masterminds/sprig/v3 v3.2.3/go.mod h1:rXcFaZ2zZbLRJv/xSysmlgIM1u11eBaRMhvYXJNkGuM=
 github.com/Microsoft/go-winio v0.6.1 h1:9/kr64B9VUZrLm5YYwbGtUJnMgqWVOdUAXu6Migciow=
 github.com/Microsoft/go-winio v0.6.1/go.mod h1:LRdKpFKfdobln8UmuiYcKPot9D2v6svN5+sAH+4kjUM=
-github.com/Microsoft/hcsshim v0.11.4 h1:68vKo2VN8DE9AdN4tnkWnmdhqdbpUFM8OF3Airm7fz8=
-github.com/Microsoft/hcsshim v0.11.4/go.mod h1:smjE4dvqPX9Zldna+t5FG3rnoHhaB7QYxPRqGcpAD9w=
 github.com/NethermindEth/juno v0.3.1 h1:AW72LiAm9gqUeCVJWvepnZcTnpU4Vkl0KzPMxS+42FA=
 github.com/NethermindEth/juno v0.3.1/go.mod h1:SGbTpgGaCsxhFsKOid7Ylnz//WZ8swtILk+NbHGsk/Q=
 github.com/NethermindEth/starknet.go v0.7.1-0.20240401080518-34a506f3cfdb h1:Mv8SscePPyw2ju4igIJAjFgcq5zCQfjgbz53DwYu5mc=
@@ -143,8 +137,6 @@ github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuy
 github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
 github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
 github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
-github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137 h1:s6gZFSlWYmbqAuRjVTiNNhvNRfY2Wxp9nhfyel4rklc=
-github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137/go.mod h1:OMCwj8VM1Kc9e19TLln2VL61YJF0x1XFtfdL4JdbSyE=
 github.com/alexbrainman/sspi v0.0.0-20210105120005-909beea2cc74 h1:Kk6a4nehpJ3UuJRqlA3JxYxBZEqCeOmATOvrbT4p9RA=
 github.com/alexbrainman/sspi v0.0.0-20210105120005-909beea2cc74/go.mod h1:cEWa1LVoE5KvSD9ONXsZrj0z6KqySlCCNKHlLzbqAt4=
 github.com/allegro/bigcache v1.2.1-0.20190218064605-e24eb225f156/go.mod h1:Cb/ax3seSYIx7SuZdm2G2xzfwmv3TPSk2ucNfQESPXM=
@@ -162,18 +154,12 @@ github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmV
 github.com/armon/go-metrics v0.4.1 h1:hR91U9KYmb6bLBYLQjyM+3j+rcd/UhE+G78SFnF8gJA=
 github.com/armon/go-metrics v0.4.1/go.mod h1:E6amYzXo6aW1tqzoZGT755KkbgrJsSdpwZ+3JqfkOG4=
 github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8=
-github.com/avast/retry-go v3.0.0+incompatible h1:4SOWQ7Qs+oroOTQOYnAHqelpCO0biHSxpiH9JdtuBj0=
-github.com/avast/retry-go v3.0.0+incompatible/go.mod h1:XtSnn+n/sHqQIpZ10K1qAevBhOOCWBLXXy3hyiqqBrY=
 github.com/avast/retry-go/v4 v4.5.1 h1:AxIx0HGi4VZ3I02jr78j5lZ3M6x1E0Ivxa6b0pUUh7o=
 github.com/avast/retry-go/v4 v4.5.1/go.mod h1:/sipNsvNB3RRuT5iNcb6h73nw3IBmXJ/H3XrCQYSOpc=
 github.com/aws/aws-sdk-go v1.22.1/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo=
 github.com/aws/aws-sdk-go v1.23.20/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo=
 github.com/aws/aws-sdk-go v1.45.25 h1:c4fLlh5sLdK2DCRTY1z0hyuJZU4ygxX8m1FswL6/nF4=
 github.com/aws/aws-sdk-go v1.45.25/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI=
-github.com/aws/constructs-go/constructs/v10 v10.1.255 h1:5hARfEmhBqHSTQf/C3QLA3sWOxO2Dfja0iA1W7ZcI7g=
-github.com/aws/constructs-go/constructs/v10 v10.1.255/go.mod h1:DCdBSjN04Ck2pajCacTD4RKFqSA7Utya8d62XreYctI=
-github.com/aws/jsii-runtime-go v1.75.0 h1:NhpUfyiL7/wsRuUekFsz8FFBCYLfPD/l61kKg9kL/a4=
-github.com/aws/jsii-runtime-go v1.75.0/go.mod h1:TKCyrtM0pygEPo4rDZzbMSDNCDNTSYSN6/mGyHI6O3I=
 github.com/aymerick/raymond v2.0.3-0.20180322193309-b565731e1464+incompatible/go.mod h1:osfaiScAUVup+UC9Nfq76eWqDhXlp+4UYaA8uhTBO6g=
 github.com/bahlo/generic-list-go v0.2.0 h1:5sz/EEAK+ls5wF+NeqDpk5+iNdMDXrh3z3nPnH1Wvgk=
 github.com/bahlo/generic-list-go v0.2.0/go.mod h1:2KvAjgMlE5NNynlg/5iLrrCCZ2+5xWbdbCW3pNTGyYg=
@@ -209,8 +195,6 @@ github.com/bytedance/sonic v1.5.0/go.mod h1:ED5hyg4y6t3/9Ku1R6dU/4KyJ48DZ4jPhfY1
 github.com/bytedance/sonic v1.10.0-rc/go.mod h1:ElCzW+ufi8qKqNW0FY314xriJhyJhuoJ3gFZdAHF7NM=
 github.com/bytedance/sonic v1.10.1 h1:7a1wuFXL1cMy7a3f7/VFcEtriuXQnUBhtoVfOZiaysc=
 github.com/bytedance/sonic v1.10.1/go.mod h1:iZcSUejdk5aukTND/Eu/ivjQuEL0Cu9/rf50Hi0u/g4=
-github.com/cdk8s-team/cdk8s-core-go/cdk8s/v2 v2.7.5 h1:rvc39Ol6z3MvaBzXkxFC6Nfsnixq/dRypushKDd7Nc0=
-github.com/cdk8s-team/cdk8s-core-go/cdk8s/v2 v2.7.5/go.mod h1:R/pdNYDYFQk+tuuOo7QES1kkv6OLmp5ze2XBZQIVffM=
 github.com/cenkalti/backoff v2.2.1+incompatible h1:tNowT99t7UNflLxfYYSlKYsBpXdEet03Pg2g16Swow4=
 github.com/cenkalti/backoff v2.2.1+incompatible/go.mod h1:90ReRw6GdpyfrHakVjL/QHaoyV4aDUVVkXQJJJ3NXXM=
 github.com/cenkalti/backoff/v4 v4.2.1 h1:y4OZtCnogmCPw98Zjyt5a6+QwPLGkiQsYW5oUqylYbM=
@@ -223,10 +207,6 @@ github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghf
 github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
 github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44=
 github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
-github.com/chai2010/gettext-go v1.0.2 h1:1Lwwip6Q2QGsAdl/ZKPCwTe9fe0CjlUbqj5bFNSjIRk=
-github.com/chai2010/gettext-go v1.0.2/go.mod h1:y+wnP2cHYaVj19NZhYKAwEMH2CI1gNHeQQ+5AjwawxA=
-github.com/chaos-mesh/chaos-mesh/api/v1alpha1 v0.0.0-20220226050744-799408773657 h1:CyuI+igIjadM/GRnE2o0q+WCwipDh0n2cUYFPAvxziM=
-github.com/chaos-mesh/chaos-mesh/api/v1alpha1 v0.0.0-20220226050744-799408773657/go.mod h1:JRiumF+RFsH1mrrP8FUsi9tExPylKkO/oSRWeQEUdLE=
 github.com/chenzhuoyu/base64x v0.0.0-20211019084208-fb5309c8db06/go.mod h1:DH46F32mSOjUmXrMHnKwZdA8wcEefY7UVqBKYGjpdQY=
 github.com/chenzhuoyu/base64x v0.0.0-20221115062448-fe3a3abad311/go.mod h1:b583jCggY9gE99b6G5LEC39OIiVsWj+R97kbl5odCEk=
 github.com/chenzhuoyu/base64x v0.0.0-20230717121745-296ad89f973d h1:77cEq6EriyTZ0g/qfRdp61a3Uu/AWrgIq2s0ClJV1g0=
@@ -244,7 +224,6 @@ github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDk
 github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc=
 github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk=
 github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk=
-github.com/cncf/xds/go v0.0.0-20210312221358-fbca930ec8ed/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs=
 github.com/cncf/xds/go v0.0.0-20230607035331-e9ce68804cb4 h1:/inchEIKaYC1Akx+H+gqO04wryn5h75LSazbRlnya1k=
 github.com/cncf/xds/go v0.0.0-20230607035331-e9ce68804cb4/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs=
 github.com/cockroachdb/apd v1.1.0 h1:3LFP3629v+1aKXU5Q37mxmRxX/pIu1nijXydLShEq5I=
@@ -280,12 +259,8 @@ github.com/consensys/bavard v0.1.13 h1:oLhMLOFGTLdlda/kma4VOJazblc7IM5y5QPd2A/Yj
 github.com/consensys/bavard v0.1.13/go.mod h1:9ItSMtA/dXMAiL7BG6bqW2m3NdSEObYWoH223nGHukI=
 github.com/consensys/gnark-crypto v0.12.1 h1:lHH39WuuFgVHONRl3J0LRBtuYdQTumFSDtJF7HpyG8M=
 github.com/consensys/gnark-crypto v0.12.1/go.mod h1:v2Gy7L/4ZRosZ7Ivs+9SfUDr0f5UlG+EM5t7MPHiLuY=
-github.com/containerd/containerd v1.7.12 h1:+KQsnv4VnzyxWcfO9mlxxELaoztsDEjOuCMPAuPqgU0=
-github.com/containerd/containerd v1.7.12/go.mod h1:/5OMpE1p0ylxtEUGY8kuCYkDRzJm9NO1TFMWjUpdevk=
 github.com/containerd/continuity v0.4.3 h1:6HVkalIp+2u1ZLH1J/pYX2oBVXlJZvh1X1A7bEZ9Su8=
 github.com/containerd/continuity v0.4.3/go.mod h1:F6PTNCKepoxEaXLQp3wDAjygEnImnZ/7o4JzpodfroQ=
-github.com/containerd/log v0.1.0 h1:TCJt7ioM2cr/tfR8GPbGf9/VRAX8D2B4PjzCpfX540I=
-github.com/containerd/log v0.1.0/go.mod h1:VRRf09a7mHDIRezVKTRCrOq78v577GXq3bSa3EhrzVo=
 github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk=
 github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE=
 github.com/coreos/etcd v3.3.13+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE=
@@ -294,7 +269,6 @@ github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3Ee
 github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk=
 github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4=
 github.com/coreos/go-systemd v0.0.0-20190719114852-fd7a80b32e1f/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4=
-github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc=
 github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA=
 github.com/cosmos/btcutil v1.0.5 h1:t+ZFcX77LpKtDBhjucvnOH8C2l2ioGsBNEQ3jef8xFk=
 github.com/cosmos/btcutil v1.0.5/go.mod h1:IyB7iuqZMJlthe2tkIFL33xPyzbFYP0XVdS8P5lUPis=
@@ -319,13 +293,9 @@ github.com/cosmos/ledger-cosmos-go v0.12.1 h1:sMBxza5p/rNK/06nBSNmsI/WDqI0pVJFVN
 github.com/cosmos/ledger-cosmos-go v0.12.1/go.mod h1:dhO6kj+Y+AHIOgAe4L9HL/6NDdyyth4q238I9yFpD2g=
 github.com/cosmos/rosetta-sdk-go v0.10.0 h1:E5RhTruuoA7KTIXUcMicL76cffyeoyvNybzUGSKFTcM=
 github.com/cosmos/rosetta-sdk-go v0.10.0/go.mod h1:SImAZkb96YbwvoRkzSMQB6noNJXFgWl/ENIznEoYQI4=
-github.com/cpuguy83/dockercfg v0.3.1 h1:/FpZ+JaygUR/lZP2NlFI2DVfrOEMAIKP5wWEJdoYe9E=
-github.com/cpuguy83/dockercfg v0.3.1/go.mod h1:sugsbF4//dDlL/i+S+rtpIWp+5h0BHJHfjj5/jFyUJc=
 github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE=
 github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU=
 github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
-github.com/cpuguy83/go-md2man/v2 v2.0.3 h1:qMCsGGgs+MAzDFyp9LpAe1Lqy/fY/qCovCm0qnXZOBM=
-github.com/cpuguy83/go-md2man/v2 v2.0.3/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
 github.com/cpuguy83/go-md2man/v2 v2.0.4 h1:wfIWP927BUkWJb2NmU/kNDYIBTh/ziUX91+lVfRxZq4=
 github.com/cpuguy83/go-md2man/v2 v2.0.4/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
 github.com/crate-crypto/go-ipa v0.0.0-20231025140028-3c0104f4b233 h1:d28BXYi+wUpz1KBmiF9bWrjEMacUEREV6MBi2ODnrfQ=
@@ -371,23 +341,16 @@ github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUn
 github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13 h1:fAjc9m62+UWV/WAFKLNi6ZS0675eEUC9y3AlwSbQu1Y=
 github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw=
 github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no=
-github.com/distribution/reference v0.5.0 h1:/FUIFXtfc/x2gpa5/VGfiGLuOIdYa1t65IKK2OFGvA0=
-github.com/distribution/reference v0.5.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E=
 github.com/docker/cli v24.0.7+incompatible h1:wa/nIwYFW7BVTGa7SWPVyyXU9lgORqUb1xfI36MSkFg=
 github.com/docker/cli v24.0.7+incompatible/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8=
 github.com/docker/distribution v2.8.2+incompatible h1:T3de5rq0dB1j30rp0sA2rER+m322EBzniBPB6ZIzuh8=
 github.com/docker/distribution v2.8.2+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w=
-github.com/docker/docker v24.0.7+incompatible h1:Wo6l37AuwP3JaMnZa226lzVXGA3F9Ig1seQen0cKYlM=
-github.com/docker/docker v24.0.7+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk=
 github.com/docker/docker v25.0.2+incompatible h1:/OaKeauroa10K4Nqavw4zlhcDq/WBcPMc5DbjOGgozY=
 github.com/docker/docker v25.0.2+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk=
-github.com/docker/go-connections v0.4.0 h1:El9xVISelRB7BuFusrZozjnkIM5YnzCViNKohAFqRJQ=
-github.com/docker/go-connections v0.4.0/go.mod h1:Gbd7IOopHjR8Iph03tsViu4nIes5XhDvyHbTtUxmeec=
 github.com/docker/go-connections v0.5.0 h1:USnMq7hx7gwdVZq1L49hLXaFtUdTADjXGp+uj1Br63c=
 github.com/docker/go-connections v0.5.0/go.mod h1:ov60Kzw0kKElRwhNs9UlUHAE/F9Fe6GLaXnqyDdmEXc=
 github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4=
 github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk=
-github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE=
 github.com/dominikbraun/graph v0.23.0 h1:TdZB4pPqCLFxYhdyMFb1TBdFxp8XLcJfTTBQucVPgCo=
 github.com/dominikbraun/graph v0.23.0/go.mod h1:yOjYyogZLY1LSG9E33JWZJiq5k83Qy2C6POAuiViluc=
 github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=
@@ -400,15 +363,12 @@ github.com/elastic/go-sysinfo v1.11.1 h1:g9mwl05njS4r69TisC+vwHWTSKywZFYYUu3so3T
 github.com/elastic/go-sysinfo v1.11.1/go.mod h1:6KQb31j0QeWBDF88jIdWSxE8cwoOB9tO4Y4osN7Q70E=
 github.com/elastic/go-windows v1.0.1 h1:AlYZOldA+UJ0/2nBuqWdo90GFCgG9xuyw9SYzGUtJm0=
 github.com/elastic/go-windows v1.0.1/go.mod h1:FoVvqWSun28vaDQPbj2Elfc0JahhPB7WQEGa3c814Ss=
-github.com/emicklei/go-restful/v3 v3.10.2 h1:hIovbnmBTLjHXkqEBUz3HGpXZdM7ZrE9fJIZIqlJLqE=
-github.com/emicklei/go-restful/v3 v3.10.2/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc=
 github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
 github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
 github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98=
 github.com/envoyproxy/go-control-plane v0.9.7/go.mod h1:cwu0lG7PUMfa9snN8LXBig5ynNVH9qI8YYLbd1fK2po=
 github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk=
 github.com/envoyproxy/go-control-plane v0.9.9-0.20210217033140-668b12f5399d/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk=
-github.com/envoyproxy/go-control-plane v0.9.9-0.20210512163311-63b5d3c536b0/go.mod h1:hliV/p42l8fGbc6Y9bQ70uLwIvmJyVE5k4iMKlh8wCQ=
 github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
 github.com/envoyproxy/protoc-gen-validate v1.0.2 h1:QkIBuU5k+x7/QXPvPPnWXWlCdaBFApVqftFV6k087DA=
 github.com/envoyproxy/protoc-gen-validate v1.0.2/go.mod h1:GpiZQP3dDbg4JouG/NNS7QWXpgx6x8QiMKdmN72jogE=
@@ -419,13 +379,6 @@ github.com/ethereum/c-kzg-4844 v0.4.0 h1:3MS1s4JtA868KpJxroZoepdV0ZKBp3u/O5HcZ7R
 github.com/ethereum/c-kzg-4844 v0.4.0/go.mod h1:VewdlzQmpT5QSrVhbBuGoCdFJkpaJlO1aQputP83wc0=
 github.com/ethereum/go-ethereum v1.13.8 h1:1od+thJel3tM52ZUNQwvpYOeRHlbkVFZ5S8fhi0Lgsg=
 github.com/ethereum/go-ethereum v1.13.8/go.mod h1:sc48XYQxCzH3fG9BcrXCOOgQk2JfZzNAmIKnceogzsA=
-github.com/evanphx/json-patch v0.5.2/go.mod h1:ZWS5hhDbVDyob71nXKNL0+PWn6ToqBHMikGIFbs31qQ=
-github.com/evanphx/json-patch v5.6.0+incompatible h1:jBYDEEiFBPxA0v50tFdvOzQQTCvpL6mnFh5mB2/l16U=
-github.com/evanphx/json-patch v5.6.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk=
-github.com/evanphx/json-patch/v5 v5.6.0 h1:b91NhWfaz02IuVxO9faSllyAtNXHMPkC5J8sJCLunww=
-github.com/evanphx/json-patch/v5 v5.6.0/go.mod h1:G79N1coSVB93tBe7j6PhzjmR3/2VvlbKOFpnXhI9Bw4=
-github.com/exponent-io/jsonpath v0.0.0-20210407135951-1de76d718b3f h1:Wl78ApPPB2Wvf/TIe2xdyJxTlb6obmF18d8QdkxNDu4=
-github.com/exponent-io/jsonpath v0.0.0-20210407135951-1de76d718b3f/go.mod h1:OSYXu++VVOHnXeitef/D8n/6y4QV8uLHSFXX4NeXMGc=
 github.com/facebookgo/ensure v0.0.0-20200202191622-63f1cf65ac4c h1:8ISkoahWXwZR41ois5lSJBSVw4D0OV19Ht/JSTzvSv0=
 github.com/facebookgo/ensure v0.0.0-20200202191622-63f1cf65ac4c/go.mod h1:Yg+htXGokKKdzcwhuNDwVvN+uBxDGXJ7G/VN1d8fa64=
 github.com/facebookgo/stack v0.0.0-20160209184415-751773369052 h1:JWuenKqqX8nojtoVVWjGfOF9635RETekkoH6Cc9SX0A=
@@ -433,8 +386,6 @@ github.com/facebookgo/stack v0.0.0-20160209184415-751773369052/go.mod h1:UbMTZqL
 github.com/facebookgo/subset v0.0.0-20200203212716-c811ad88dec4 h1:7HZCaLC5+BZpmbhCOZJ293Lz68O7PYrF2EzeiFMwCLk=
 github.com/facebookgo/subset v0.0.0-20200203212716-c811ad88dec4/go.mod h1:5tD+neXqOorC30/tWg0LCSkrqj/AR6gu8yY8/fpw1q0=
 github.com/fasthttp-contrib/websocket v0.0.0-20160511215533-1f3b11f56072/go.mod h1:duJ4Jxv5lDcvg4QuQr0oowTf7dz4/CR8NtyCooz9HL8=
-github.com/fatih/camelcase v1.0.0 h1:hxNvNX/xYBp0ovncs8WyWZrOrpBNub/JfaMvbURyft8=
-github.com/fatih/camelcase v1.0.0/go.mod h1:yN2Sb0lFhZJUdVvtELVWefmrXpuZESvPmqwoZc+/fpc=
 github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4=
 github.com/fatih/color v1.9.0/go.mod h1:eQcE1qtQxscV5RaZvpXrrb8Drkc3/DdQ+uUYCNjL+zU=
 github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYFFOfk=
@@ -445,7 +396,6 @@ github.com/felixge/httpsnoop v1.0.3 h1:s/nj+GCswXYzN5v2DpNMuMQYe+0DDwt5WVCU6CWBd
 github.com/felixge/httpsnoop v1.0.3/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U=
 github.com/fjl/memsize v0.0.0-20190710130421-bcb5799ab5e5 h1:FtmdgXiUlNeRsoNMFlKLDt+S+6hbjVMEW6RGQ7aUf7c=
 github.com/fjl/memsize v0.0.0-20190710130421-bcb5799ab5e5/go.mod h1:VvhXpOYNQvB+uIk2RvXzuaQtkQJzzIx6lSBe1xv7hi0=
-github.com/flowstack/go-jsonschema v0.1.1/go.mod h1:yL7fNggx1o8rm9RlgXv7hTBWxdBM0rVwpMwimd3F3N0=
 github.com/fortytw2/leaktest v1.3.0 h1:u8491cBMTQ8ft8aeV+adlcytMZylmA5nnwwkRZjI8vw=
 github.com/fortytw2/leaktest v1.3.0/go.mod h1:jDsjWgpAGjm2CA7WthBh/CdZYEPF31XHquHwclZch5g=
 github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8=
@@ -455,8 +405,6 @@ github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4
 github.com/fsnotify/fsnotify v1.5.4/go.mod h1:OVB6XrOHzAwXMpEM7uPOzcehqUV2UqJxmVXmkdnm1bU=
 github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY=
 github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbSClcnxKAGw=
-github.com/fvbommel/sortorder v1.0.2 h1:mV4o8B2hKboCdkJm+a7uX/SIpZob4JzUpc5GGnM45eo=
-github.com/fvbommel/sortorder v1.0.2/go.mod h1:uk88iVf1ovNn1iLfgUVU2F9o5eO30ui720w+kxuqRs0=
 github.com/fxamacker/cbor/v2 v2.5.0 h1:oHsG0V/Q6E/wqTS2O1Cozzsy69nqCiguo5Q1a1ADivE=
 github.com/fxamacker/cbor/v2 v2.5.0/go.mod h1:TA1xS00nchWmaBnEIxPSE5oHLuJBAVvqrtAnWBwBCVo=
 github.com/gabriel-vasile/mimetype v1.4.2 h1:w5qFW6JKBz9Y393Y4q372O9A7cUSequkh1Q7OhCmWKU=
@@ -524,7 +472,6 @@ github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V
 github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A=
 github.com/go-logfmt/logfmt v0.6.0 h1:wGYYu3uicYdqXVgoYbvnkrPVXkuLM1p1ifugDMEdRi4=
 github.com/go-logfmt/logfmt v0.6.0/go.mod h1:WYhtIu8zTZfxdn5+rREduYbwxfcBr/Vr6KEVveWlfTs=
-github.com/go-logr/logr v1.2.0/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
 github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
 github.com/go-logr/logr v1.4.1 h1:pKouT5E8xu9zeFC39JXRDukb6JFQPXM5p5I91188VAQ=
 github.com/go-logr/logr v1.4.1/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=
@@ -533,14 +480,6 @@ github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre
 github.com/go-martini/martini v0.0.0-20170121215854-22fa46961aab/go.mod h1:/P9AEU963A2AYjv4d1V5eVL1CQbEJq6aCNHDDjibzu8=
 github.com/go-ole/go-ole v1.2.6 h1:/Fpf6oFPoeFik9ty7siob0G6Ke8QvQEuVcuChpwXzpY=
 github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0=
-github.com/go-openapi/jsonpointer v0.19.6/go.mod h1:osyAmYz/mB/C3I+WsTTSgw1ONzaLJoLCyoi6/zppojs=
-github.com/go-openapi/jsonpointer v0.20.0 h1:ESKJdU9ASRfaPNOPRx12IUyA1vn3R9GiE3KYD14BXdQ=
-github.com/go-openapi/jsonpointer v0.20.0/go.mod h1:6PGzBjjIIumbLYysB73Klnms1mwnU4G3YHOECG3CedA=
-github.com/go-openapi/jsonreference v0.20.2 h1:3sVjiK66+uXK/6oQ8xgcRKcFgQ5KXa2KvnJRumpMGbE=
-github.com/go-openapi/jsonreference v0.20.2/go.mod h1:Bl1zwGIM8/wsvqjsOQLJ/SH+En5Ap4rVB5KVcIDZG2k=
-github.com/go-openapi/swag v0.22.3/go.mod h1:UzaqsxGiab7freDnrUUra0MwWfN/q7tE4j+VcZ0yl14=
-github.com/go-openapi/swag v0.22.4 h1:QLMzNJnMGPRNDCbySlcj1x01tzU8/9LTTL9hZZZogBU=
-github.com/go-openapi/swag v0.22.4/go.mod h1:UzaqsxGiab7freDnrUUra0MwWfN/q7tE4j+VcZ0yl14=
 github.com/go-playground/assert/v2 v2.0.1/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4=
 github.com/go-playground/assert/v2 v2.2.0 h1:JvknZsQTYeFEAhQwI4qEt9cyV5ONwRHC+lYKSsYSR8s=
 github.com/go-playground/assert/v2 v2.2.0/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4=
@@ -553,8 +492,6 @@ github.com/go-playground/universal-translator v0.18.1/go.mod h1:xekY+UJKNuX9WP91
 github.com/go-playground/validator/v10 v10.10.0/go.mod h1:74x4gJWsvQexRdW8Pn3dXSGrTK4nAUsbPlLADvpJkos=
 github.com/go-playground/validator/v10 v10.15.5 h1:LEBecTWb/1j5TNY1YYG2RcOUN3R7NLylN+x8TTueE24=
 github.com/go-playground/validator/v10 v10.15.5/go.mod h1:9iXMNT7sEkjXb0I+enO7QXmzG6QCsPWY4zveKFVRSyU=
-github.com/go-resty/resty/v2 v2.7.0 h1:me+K9p3uhSmXtrBZ4k9jcEAfJmuC8IivWHwaLZwPrFY=
-github.com/go-resty/resty/v2 v2.7.0/go.mod h1:9PWDzw47qPphMRFfhsyk0NnSgvluHcljSMVIq3w7q0I=
 github.com/go-sql-driver/mysql v1.6.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg=
 github.com/go-sql-driver/mysql v1.7.1 h1:lUIinVbN1DY0xBg0eMOzmmtGoHwWBbvnWubQUrtU8EI=
 github.com/go-sql-driver/mysql v1.7.1/go.mod h1:OXbVy3sEdcQ2Doequ6Z5BW6fXNQTmx+9S1MCJN5yJMI=
@@ -574,7 +511,6 @@ github.com/goccy/go-json v0.10.2 h1:CrxCmQqYDkv1z7lO7Wbh2HN93uovUHgrECaO5ZrCXAU=
 github.com/goccy/go-json v0.10.2/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I=
 github.com/godbus/dbus v0.0.0-20190726142602-4481cbc300e2 h1:ZpnhV/YsD2/4cESfV5+Hoeu/iUR3ruzNvZ+yQfO03a0=
 github.com/godbus/dbus v0.0.0-20190726142602-4481cbc300e2/go.mod h1:bBOAhwG1umN6/6ZUMtDFBMQR8jRg9O75tm9K00oMsK4=
-github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA=
 github.com/gofrs/flock v0.8.1 h1:+gYjHKf32LDeiEEFhQaotPbLuUXjY5ZqxKgXy7n59aw=
 github.com/gofrs/flock v0.8.1/go.mod h1:F1TvTiK9OcQqauNUHlbJvyl9Qa1QvF/gOUDKA14jxHU=
 github.com/gofrs/uuid v4.0.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM=
@@ -638,10 +574,6 @@ github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Z
 github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
 github.com/google/btree v1.1.2 h1:xf4v41cLI2Z6FxbKm+8Bu+m8ifhj15JuZ9sa0jZCMUU=
 github.com/google/btree v1.1.2/go.mod h1:qOPhT0dTNdNzV6Z/lhRX0YXUafgPLFUh+gZMl761Gm4=
-github.com/google/gnostic v0.6.9 h1:ZK/5VhkoX835RikCHpSUJV9a+S3e1zLh59YnyWeBW+0=
-github.com/google/gnostic v0.6.9/go.mod h1:Nm8234We1lq6iB9OmlgNv3nH91XLLVZHCDayfA3xq+E=
-github.com/google/gnostic-models v0.6.8 h1:yo/ABAfM5IMRsS1VnXjTBvUb61tFIHozhlYvRgGre9I=
-github.com/google/gnostic-models v0.6.8/go.mod h1:5n7qKqH0f5wFt+aWF8CW6pZLLNOfYuF5OpfBSENuI8U=
 github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
 github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
 github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
@@ -724,8 +656,6 @@ github.com/graph-gophers/dataloader v5.0.0+incompatible h1:R+yjsbrNq1Mo3aPG+Z/EK
 github.com/graph-gophers/dataloader v5.0.0+incompatible/go.mod h1:jk4jk0c5ZISbKaMe8WsVopGB5/15GvGHMdMdPtwlRp4=
 github.com/graph-gophers/graphql-go v1.3.0 h1:Eb9x/q6MFpCLz7jBCiP/WTxjSDrYLR1QY41SORZyNJ0=
 github.com/graph-gophers/graphql-go v1.3.0/go.mod h1:9CQHMSxwO4MprSdzoIEobiHpoLtHm77vfxsvsIN5Vuc=
-github.com/gregjones/httpcache v0.0.0-20190611155906-901d90724c79 h1:+ngKgrYPPJrOjhax5N+uePQ0Fh1Z7PheYoUI/0nzkPA=
-github.com/gregjones/httpcache v0.0.0-20190611155906-901d90724c79/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA=
 github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs=
 github.com/grpc-ecosystem/go-grpc-middleware v1.3.0 h1:+9834+KizmvFV7pXQGSXQTsaWhq2GjuNUt0aUU0YBYw=
 github.com/grpc-ecosystem/go-grpc-middleware v1.3.0/go.mod h1:z0ButlSOZa5vEBq9m2m2hlwIgKw+rp3sdCBRoJY+30Y=
@@ -906,7 +836,6 @@ github.com/joho/godotenv v1.4.0/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwA
 github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo=
 github.com/jonboulle/clockwork v0.4.0 h1:p4Cf1aMWXnXAUh8lVfewRBx1zaTSYKrKMF2g3ST4RZ4=
 github.com/jonboulle/clockwork v0.4.0/go.mod h1:xgRqUGwRcjKCO1vbZUEtSLrqKoPSsUpK7fnezOII0kc=
-github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY=
 github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y=
 github.com/jpillora/backoff v1.0.0 h1:uvFg412JmmHBHw7iwprIxkPMI+sGQ4kzOWsMeHnm2EA=
 github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4=
@@ -926,8 +855,6 @@ github.com/kataras/pio v0.0.2/go.mod h1:hAoW0t9UmXi4R5Oyq5Z4irTbaTsOemSrDGUtaTl7
 github.com/kataras/sitemap v0.0.5/go.mod h1:KY2eugMKiPwsJgx7+U103YZehfvNGOXURubcGyk0Bz8=
 github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 h1:Z9n2FFNUXsshfwJMBgNA0RU6/i7WVaAegv3PtuIHPMs=
 github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51/go.mod h1:CzGEWj7cYgsdH8dAjBGEr58BoE7ScuLd+fwFZ44+/x8=
-github.com/kelseyhightower/envconfig v1.4.0 h1:Im6hONhd3pLkfDFsbRgu68RDNkGF1r3dvMUtDTo2cv8=
-github.com/kelseyhightower/envconfig v1.4.0/go.mod h1:cccZRl6mQpaq41TPp5QxidR+Sa3axMbJDNb//FQX6Gg=
 github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8=
 github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
 github.com/klauspost/compress v1.8.2/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A=
@@ -947,7 +874,6 @@ github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxv
 github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg=
 github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc=
 github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
-github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
 github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
 github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk=
 github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
@@ -974,11 +900,8 @@ github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw=
 github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o=
 github.com/libp2p/go-buffer-pool v0.1.0 h1:oK4mSFcQz7cTQIfqbe4MIj9gLW+mnanjyFtc6cdF0Y8=
 github.com/libp2p/go-buffer-pool v0.1.0/go.mod h1:N+vh8gMqimBzdKkSMVuydVDq+UV5QTWy5HSiZacSbPg=
-github.com/liggitt/tabwriter v0.0.0-20181228230101-89fcab3d43de h1:9TO3cAIGXtEhnIaL+V+BEER86oLrvS+kWobKpbJuye0=
-github.com/liggitt/tabwriter v0.0.0-20181228230101-89fcab3d43de/go.mod h1:zAbeS9B/r2mtpb6U+EI2rYA5OAXxsYw6wTamcNW+zcE=
 github.com/logrusorgru/aurora v2.0.3+incompatible h1:tOpm7WcpBTn4fjmVfgpQq0EfczGlG91VSDkswnjF5A8=
 github.com/logrusorgru/aurora v2.0.3+incompatible/go.mod h1:7rIyQOR62GCctdiQpZ/zOJlFyk6y+94wXzv6RNZgaR4=
-github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 h1:6E+4a0GO5zZEnZ81pIr0yLvtUWk2if982qA3F3QD6H4=
 github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0/go.mod h1:zJYVVT2jmtg6P3p1VtQj7WsuWi/y4VnjVBn7F8KPB3I=
 github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
 github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
@@ -1040,8 +963,6 @@ github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrk
 github.com/mitchellh/go-testing-interface v1.0.0/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI=
 github.com/mitchellh/go-testing-interface v1.14.1 h1:jrgshOhYAUVNMAJiKbEu7EqAwgJJ2JqpQmpLJOu07cU=
 github.com/mitchellh/go-testing-interface v1.14.1/go.mod h1:gfgS7OtZj6MA4U1UrDRp04twqAjfvlZyCfX3sDjEym8=
-github.com/mitchellh/go-wordwrap v1.0.1 h1:TLuKupo69TCn6TQSyGxwI1EblZZEsQ0vMlAFQflz0v0=
-github.com/mitchellh/go-wordwrap v1.0.1/go.mod h1:R62XHJLzvMFRBbcrT7m7WgmE1eOyTSsCt+hzestvNj0=
 github.com/mitchellh/gox v0.4.0/go.mod h1:Sd9lOJ0+aimLBi73mGofS1ycjY8lL3uZM3JPS42BGNg=
 github.com/mitchellh/iochan v1.0.0/go.mod h1:JwYml1nuB7xOzsp52dPpHFffvOCDupsG0QubkSMEySY=
 github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
@@ -1056,14 +977,6 @@ github.com/mitchellh/reflectwalk v1.0.2/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx
 github.com/mmcloughlin/addchain v0.4.0 h1:SobOdjm2xLj1KkXN5/n0xTIWyZA2+s99UCY1iPfkHRY=
 github.com/mmcloughlin/addchain v0.4.0/go.mod h1:A86O+tHqZLMNO4w6ZZ4FlVQEadcoqkyU72HC5wJ4RlU=
 github.com/mmcloughlin/profile v0.1.1/go.mod h1:IhHD7q1ooxgwTgjxQYkACGA77oFTDdFVejUS1/tS/qU=
-github.com/moby/patternmatcher v0.6.0 h1:GmP9lR19aU5GqSSFko+5pRqHi+Ohk1O69aFiKkVGiPk=
-github.com/moby/patternmatcher v0.6.0/go.mod h1:hDPoyOpDY7OrrMDLaYoY3hf52gNCR/YOUYxkhApJIxc=
-github.com/moby/spdystream v0.2.0 h1:cjW1zVyyoiM0T7b6UoySUFqzXMoqRckQtXwGPiBhOM8=
-github.com/moby/spdystream v0.2.0/go.mod h1:f7i0iNDQJ059oMTcWxx8MA/zKFIuD/lY+0GqbN2Wy8c=
-github.com/moby/sys/sequential v0.5.0 h1:OPvI35Lzn9K04PBbCLW0g4LcFAJgHsvXsRyewg5lXtc=
-github.com/moby/sys/sequential v0.5.0/go.mod h1:tH2cOOs5V9MlPiXcQzRC+eEyab644PWKGRYaaV5ZZlo=
-github.com/moby/sys/user v0.1.0 h1:WmZ93f5Ux6het5iituh9x2zAG7NFY9Aqi49jjE1PaQg=
-github.com/moby/sys/user v0.1.0/go.mod h1:fKJhFOnsCN6xZ5gSfbM6zaHGgDJMrqt9/reuj4T7MmU=
 github.com/moby/term v0.5.0 h1:xt8Q1nalod/v7BqbG21f8mQPqH+xAaC9C3N3wfWbVP0=
 github.com/moby/term v0.5.0/go.mod h1:8FzsFHVUBGZdbDsJw/ot+X+d5HLUbvklYLJ9uGfcI3Y=
 github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
@@ -1073,13 +986,7 @@ github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lN
 github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
 github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M=
 github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk=
-github.com/monochromegane/go-gitignore v0.0.0-20200626010858-205db1a8cc00 h1:n6/2gBQ3RWajuToeY6ZtZTIKv2v7ThUy5KKusIT0yc0=
-github.com/monochromegane/go-gitignore v0.0.0-20200626010858-205db1a8cc00/go.mod h1:Pm3mSP3c5uWn86xMLZ5Sa7JB9GsEZySvHYXCTK4E9q4=
 github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc=
-github.com/montanaflynn/stats v0.7.1 h1:etflOAAHORrCC44V+aR6Ftzort912ZU+YLiSTuV8eaE=
-github.com/montanaflynn/stats v0.7.1/go.mod h1:etXPPgVO6n31NxCd9KQUMvCM+ve0ruNzt6R8Bnaayow=
-github.com/morikuni/aec v1.0.0 h1:nP9CBfwrvYnBRgY6qfDQkygYDmYwOilePFkwzv4dU8A=
-github.com/morikuni/aec v1.0.0/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7PXmsc=
 github.com/mostynb/zstdpool-freelist v0.0.0-20201229113212-927304c0c3b1 h1:mPMvm6X6tf4w8y7j9YIt6V9jfWhL6QlbEc7CCmeQlWk=
 github.com/mostynb/zstdpool-freelist v0.0.0-20201229113212-927304c0c3b1/go.mod h1:ye2e/VUEtE2BHE+G/QcKkcLQVAEJoYRFj5VUOQatCRE=
 github.com/moul/http2curl v1.0.0/go.mod h1:8UbvGypXm98wA/IqH45anm5Y2Z6ep6O31QGOAZ3H0fQ=
@@ -1087,13 +994,7 @@ github.com/mr-tron/base58 v1.2.0 h1:T/HDJBh4ZCPbU39/+c3rRvE0uKBQlU27+QI8LJ4t64o=
 github.com/mr-tron/base58 v1.2.0/go.mod h1:BinMc/sQntlIE1frQmRFPUoPA1Zkr8VRgBdjWI2mNwc=
 github.com/mtibben/percent v0.2.1 h1:5gssi8Nqo8QU/r2pynCm+hBQHpkB/uNK7BJCFogWdzs=
 github.com/mtibben/percent v0.2.1/go.mod h1:KG9uO+SZkUp+VkRHsCdYQV3XSZrrSpR3O9ibNBTZrns=
-github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA=
-github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
 github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
-github.com/naoina/go-stringutil v0.1.0 h1:rCUeRUHjBjGTSHl0VC00jUPLz8/F9dDzYI70Hzifhks=
-github.com/naoina/go-stringutil v0.1.0/go.mod h1:XJ2SJL9jCtBh+P9q5btrd/Ylo8XwT/h1USek5+NqSA0=
-github.com/naoina/toml v0.1.2-0.20170918210437-9fafd6967416 h1:shk/vn9oCoOTmwcouEdwIeOtOGA/ELRUw/GwvxwfT+0=
-github.com/naoina/toml v0.1.2-0.20170918210437-9fafd6967416/go.mod h1:NBIhNtsFMo3G2szEBne+bO4gS192HuIYRqfvOWb4i1E=
 github.com/nats-io/jwt v0.3.0/go.mod h1:fRYCDE99xlTsqUzISS1Bi75UBJ6ljOJQOAAu5VglpSg=
 github.com/nats-io/nats.go v1.9.1/go.mod h1:ZjDU1L/7fJ09jvUSRVBR2e7+RnLiiIQyqyzEE/Zbp4w=
 github.com/nats-io/nkeys v0.1.0/go.mod h1:xpnFELMwJABBLVhffcfd1MZx6VsNRFpEugbxziKVo7w=
@@ -1149,12 +1050,8 @@ github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/9
 github.com/pelletier/go-toml v1.9.5 h1:4yBQzkHv+7BHq2PQUZF3Mx0IYxG7LsP222s7Agd3ve8=
 github.com/pelletier/go-toml v1.9.5/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c=
 github.com/pelletier/go-toml/v2 v2.0.1/go.mod h1:r9LEWfGN8R5k0VXJ+0BkIe7MYkRdwZOjgMj2KwnJFUo=
-github.com/pelletier/go-toml/v2 v2.1.1 h1:LWAJwfNvjQZCFIDKWYQaM62NcYeYViCmWIwmOStowAI=
-github.com/pelletier/go-toml/v2 v2.1.1/go.mod h1:tJU2Z3ZkXwnxa4DPO899bsyIoywizdUvyaeZurnPPDc=
 github.com/pelletier/go-toml/v2 v2.2.2 h1:aYUidT7k73Pcl9nb2gScu7NSrKCSHIDE89b3+6Wq+LM=
 github.com/pelletier/go-toml/v2 v2.2.2/go.mod h1:1t835xjRzz80PqgE6HHgN2JOsmgYu/h4qDAS4n929Rs=
-github.com/peterbourgon/diskv v2.0.1+incompatible h1:UBdAOUP5p4RWqPBg048CAvpKN+vxiaj6gdUUzhl4XmI=
-github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU=
 github.com/petermattis/goid v0.0.0-20180202154549-b0b1615b78e5/go.mod h1:jvVRKCrJTQWu0XVbaOlby/2lO20uSCHEMzzplHXte1o=
 github.com/petermattis/goid v0.0.0-20230317030725-371a4b8eda08 h1:hDSdbBuw3Lefr6R18ax0tZ2BJeNB3NehB3trOwYBsdU=
 github.com/petermattis/goid v0.0.0-20230317030725-371a4b8eda08/go.mod h1:pxMtw7cyUw6B2bRH0ZBANSPg+AoSud1I1iyJHI69jH4=
@@ -1227,17 +1124,14 @@ github.com/rogpeppe/go-internal v1.8.1/go.mod h1:JeRgkft04UBgHMgCIwADu4Pn6Mtm5d4
 github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs=
 github.com/rogpeppe/go-internal v1.11.0 h1:cWPaGQEPrBb5/AsnsZesgZZ9yb1OQ+GOISoDNXVBh4M=
 github.com/rogpeppe/go-internal v1.11.0/go.mod h1:ddIwULY96R17DhadqLgMfk9H9tvdUzkipdSkR5nkCZA=
-github.com/rs/cors v1.8.3 h1:O+qNyWn7Z+F9M0ILBHgMVPuB1xTOucVd5gtaYyXBpRo=
-github.com/rs/cors v1.8.3/go.mod h1:XyqrcTp5zjWr1wsJ8PIRZssZ8b/WMcMf71DJnit4EMU=
+github.com/rs/cors v1.9.0 h1:l9HGsTsHJcvW14Nk7J9KFz8bzeAWXn3CG6bgt7LsrAE=
+github.com/rs/cors v1.9.0/go.mod h1:XyqrcTp5zjWr1wsJ8PIRZssZ8b/WMcMf71DJnit4EMU=
 github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ=
-github.com/rs/xid v1.5.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg=
 github.com/rs/zerolog v1.13.0/go.mod h1:YbFCdg8HfsridGWAh22vktObvhZbQsZXe4/zB0OKkWU=
 github.com/rs/zerolog v1.15.0/go.mod h1:xYTKnLHcpfU2225ny5qZjxnj9NvkumZYjJHlAThCjNc=
 github.com/rs/zerolog v1.30.0 h1:SymVODrcRsaRaSInD9yQtKbtWqwsfoPcRff/oRXLj4c=
 github.com/rs/zerolog v1.30.0/go.mod h1:/tk+P47gFdPXq4QYjvCmT5/Gsug2nagsFWBWhAiSi1w=
 github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g=
-github.com/russross/blackfriday v1.6.0 h1:KqfZb0pUVN2lYqZUYRddxF4OR8ZMURnJIG5Y3VRLtww=
-github.com/russross/blackfriday v1.6.0/go.mod h1:ti0ldHuxg49ri4ksnFxlkCfN+hvslNlmVHqNRXXJNAY=
 github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
 github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk=
 github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
@@ -1261,7 +1155,6 @@ github.com/shirou/gopsutil v3.21.11+incompatible h1:+1+c1VGhc88SSonWP6foOcLhvnKl
 github.com/shirou/gopsutil v3.21.11+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA=
 github.com/shirou/gopsutil/v3 v3.24.3 h1:eoUGJSmdfLzJ3mxIhmOAhgKEKgQkeOwKpz1NbhVnuPE=
 github.com/shirou/gopsutil/v3 v3.24.3/go.mod h1:JpND7O217xa72ewWz9zN2eIIkPWsDN/3pl0H8Qt0uwg=
-github.com/shoenig/go-m1cpu v0.1.6 h1:nxdKQNcEB6vzgA2E2bvzKIYRuNj7XNJ4S/aRSwKzFtM=
 github.com/shoenig/go-m1cpu v0.1.6/go.mod h1:1JJMcUBvfNwpq05QDQVAnx3gUHr9IYF7GNg9SUEw2VQ=
 github.com/shoenig/test v0.6.4/go.mod h1:byHiCGXqrVaflBLAMq/srcZIHynQPQgeyvkvXnjqq0k=
 github.com/shopspring/decimal v0.0.0-20180709203117-cd690d0c9e24/go.mod h1:M+9NzErvs504Cn4c5DxATwIqPbtswREoFCre64PpcG4=
@@ -1290,8 +1183,6 @@ github.com/smartcontractkit/chainlink-solana v1.0.3-0.20240605170242-555ff582f36
 github.com/smartcontractkit/chainlink-solana v1.0.3-0.20240605170242-555ff582f36a/go.mod h1:QqcZSwLgEIn7YraAIRmomnBMAuVFephiHrIWVlkWbFI=
 github.com/smartcontractkit/chainlink-starknet/relayer v0.0.1-beta-test.0.20240531021326-99118e47f696 h1:h1E87+z+JcUEfvbJVF56SnZA/YUFE5ewUE61MaR/Ewg=
 github.com/smartcontractkit/chainlink-starknet/relayer v0.0.1-beta-test.0.20240531021326-99118e47f696/go.mod h1:OiWUTrrpSLLTMh7FINWjEh6mmDJCVPaC4yEsDCVaWdU=
-github.com/smartcontractkit/chainlink-testing-framework v1.31.4 h1:5/zF5Z2w+/HTvUSSJNhWSiHvlQ2O2qxRw4e1fBbVraQ=
-github.com/smartcontractkit/chainlink-testing-framework v1.31.4/go.mod h1:E6uNEZhZZid9PHv6/Kq5Vn63GlO61ZcKB+/f0DKo3Q4=
 github.com/smartcontractkit/chainlink-vrf v0.0.0-20231120191722-fef03814f868 h1:FFdvEzlYwcuVHkdZ8YnZR/XomeMGbz5E2F2HZI3I3w8=
 github.com/smartcontractkit/chainlink-vrf v0.0.0-20231120191722-fef03814f868/go.mod h1:Kn1Hape05UzFZ7bOUnm3GVsHzP0TNrVmpfXYNHdqGGs=
 github.com/smartcontractkit/go-plugin v0.0.0-20240208201424-b3b91517de16 h1:TFe+FvzxClblt6qRfqEhUfa4kFQx5UobuoFGO2W4mMo=
@@ -1300,8 +1191,6 @@ github.com/smartcontractkit/grpc-proxy v0.0.0-20230731113816-f1be6620749f h1:hgJ
 github.com/smartcontractkit/grpc-proxy v0.0.0-20230731113816-f1be6620749f/go.mod h1:MvMXoufZAtqExNexqi4cjrNYE9MefKddKylxjS+//n0=
 github.com/smartcontractkit/libocr v0.0.0-20240419185742-fd3cab206b2c h1:lIyMbTaF2H0Q71vkwZHX/Ew4KF2BxiKhqEXwF8rn+KI=
 github.com/smartcontractkit/libocr v0.0.0-20240419185742-fd3cab206b2c/go.mod h1:fb1ZDVXACvu4frX3APHZaEBp0xi1DIm34DcA0CwTsZM=
-github.com/smartcontractkit/seth v1.0.11 h1:Ct8wSifW2ZXnyHtYRKaawBnpW7Ef0m8zItUcqYPallM=
-github.com/smartcontractkit/seth v1.0.11/go.mod h1:fVCE+8LD6AbU7d8BHZ1uS/ceJ+8JO0iVTUcDdQmGPAc=
 github.com/smartcontractkit/tdh2/go/ocr2/decryptionplugin v0.0.0-20230906073235-9e478e5e19f1 h1:yiKnypAqP8l0OX0P3klzZ7SCcBUxy5KqTAKZmQOvSQE=
 github.com/smartcontractkit/tdh2/go/ocr2/decryptionplugin v0.0.0-20230906073235-9e478e5e19f1/go.mod h1:q6f4fe39oZPdsh1i57WznEZgxd8siidMaSFq3wdPmVg=
 github.com/smartcontractkit/tdh2/go/tdh2 v0.0.0-20230906073235-9e478e5e19f1 h1:Dai1bn+Q5cpeGMQwRdjOdVjG8mmFFROVkSKuUgBErRQ=
@@ -1323,8 +1212,6 @@ github.com/spf13/cast v1.6.0 h1:GEiTHELF+vaR5dhz3VqZfFSzZjYbgeKDpBxQVS4GYJ0=
 github.com/spf13/cast v1.6.0/go.mod h1:ancEpBxwJDODSW/UG4rDrAqiKolqNNh2DX3mk86cAdo=
 github.com/spf13/cobra v0.0.5/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tLCHU=
 github.com/spf13/cobra v1.1.1/go.mod h1:WnodtKOvamDL/PwE2M4iKs8aMDBZ5Q5klgD3qfVJQMI=
-github.com/spf13/cobra v1.8.0 h1:7aJaZx1B85qltLMc546zn58BxxfZdR/W22ej9CFoEf0=
-github.com/spf13/cobra v1.8.0/go.mod h1:WXLWApfZ71AjXPya3WOlMsY9yMs7YeiHhFVlvLyhcho=
 github.com/spf13/cobra v1.8.1 h1:e5/vxKd/rZsfSJMUX1agtjeTDf+qv1/JdBF8gg5k9ZM=
 github.com/spf13/cobra v1.8.1/go.mod h1:wHxEcudfqmLYa8iTfL+OuZPbBZkmvliBWKIezN3kD9Y=
 github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo=
@@ -1340,7 +1227,6 @@ github.com/spf13/viper v1.15.0 h1:js3yy885G8xwJa6iOISGFwd+qlUo5AvyXb7CiihdtiU=
 github.com/spf13/viper v1.15.0/go.mod h1:fFcTBJxvhhzSJiZy8n+PeW6t8l+KeT/uTARa0jHOQLA=
 github.com/status-im/keycard-go v0.2.0 h1:QDLFswOQu1r5jsycloeQh3bVU8n/NatHHaZobtDnDzA=
 github.com/status-im/keycard-go v0.2.0/go.mod h1:wlp8ZLbsmrF6g6WjugPAx+IzoLrkdf9+mHxBEeo3Hbg=
-github.com/stoewer/go-strcase v1.2.0/go.mod h1:IBiWB2sKIp3wVVQ3Y035++gc+knqhUQag1KpM8ahLw8=
 github.com/streamingfast/logging v0.0.0-20220405224725-2755dab2ce75 h1:ZqpS7rAhhKD7S7DnrpEdrnW1/gZcv82ytpMviovkli4=
 github.com/streamingfast/logging v0.0.0-20220405224725-2755dab2ce75/go.mod h1:VlduQ80JcGJSargkRU4Sg9Xo63wZD/l8A5NC/Uo1/uU=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
@@ -1380,8 +1266,6 @@ github.com/teris-io/shortid v0.0.0-20201117134242-e59966efd125 h1:3SNcvBmEPE1YlB
 github.com/teris-io/shortid v0.0.0-20201117134242-e59966efd125/go.mod h1:M8agBzgqHIhgj7wEn9/0hJUZcrvt9VY+Ln+S1I5Mha0=
 github.com/test-go/testify v1.1.4 h1:Tf9lntrKUMHiXQ07qBScBTSA0dhYQlu83hswqelv1iE=
 github.com/test-go/testify v1.1.4/go.mod h1:rH7cfJo/47vWGdi4GPj16x3/t1xGOj2YxzmNQzk2ghU=
-github.com/testcontainers/testcontainers-go v0.28.0 h1:1HLm9qm+J5VikzFDYhOd+Zw12NtOl+8drH2E8nTY1r8=
-github.com/testcontainers/testcontainers-go v0.28.0/go.mod h1:COlDpUXbwW3owtpMkEB1zo9gwb1CoKVKlyrVPejF4AU=
 github.com/theodesp/go-heaps v0.0.0-20190520121037-88e35354fe0a h1:YuO+afVc3eqrjiCUizNCxI53bl/BnPiVwXqLzqYTqgU=
 github.com/theodesp/go-heaps v0.0.0-20190520121037-88e35354fe0a/go.mod h1:/sfW47zCZp9FrtGcWyo1VjbgDaodxX9ovZvgLb/MxaA=
 github.com/tidwall/btree v1.6.0 h1:LDZfKfQIBHGHWSwckhXI0RPSXzlo+KYdjK7FWSqOzzg=
@@ -1451,8 +1335,6 @@ github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415/go.mod h1:
 github.com/xeipuuv/gojsonschema v1.2.0 h1:LhYJRs+L4fBtjZUfuSZIKGeVu0QRy8e5Xi7D17UxZ74=
 github.com/xeipuuv/gojsonschema v1.2.0/go.mod h1:anYRn/JVcOK2ZgGU+IjEV4nwlhoK5sQluxsYJ78Id3Y=
 github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU=
-github.com/xlab/treeprint v1.1.0 h1:G/1DjNkPpfZCFt9CSh6b5/nY4VimlbHF3Rh4obvtzDk=
-github.com/xlab/treeprint v1.1.0/go.mod h1:gj5Gd3gPdKtR1ikdDK6fnFLdmIS0X30kTTuNd/WEJu0=
 github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q=
 github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 h1:bAn7/zixMGCfxrRTfdpNzjtPYqr8smhKouy9mxVdGPU=
 github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673/go.mod h1:N3UwUGtsrSj3ccvlPHLoLsHnpR27oXr4ZE984MbSER8=
@@ -1507,8 +1389,6 @@ go.opentelemetry.io/contrib/instrumentation/github.com/gin-gonic/gin/otelgin v0.
 go.opentelemetry.io/contrib/instrumentation/github.com/gin-gonic/gin/otelgin v0.49.0/go.mod h1:1P/02zM3OwkX9uki+Wmxw3a5GVb6KUXRsa7m7bOC9Fg=
 go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.46.1 h1:SpGay3w+nEwMpfVnbqOLH5gY52/foP8RE8UzTZ1pdSE=
 go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.46.1/go.mod h1:4UoMYEZOC0yN/sPGH76KPkkU7zgiEWYWL9vwmbnTJPE=
-go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.45.0 h1:x8Z78aZx8cOF0+Kkazoc7lwUNMGy0LrzEMxTm4BbTxg=
-go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.45.0/go.mod h1:62CPTSry9QZtOaSsE3tOzhx6LzDhHnXJ6xHeMNNiM6Q=
 go.opentelemetry.io/contrib/propagators/b3 v1.24.0 h1:n4xwCdTx3pZqZs2CjS/CUZAs03y3dZcGhC/FepKtEUY=
 go.opentelemetry.io/contrib/propagators/b3 v1.24.0/go.mod h1:k5wRxKRU2uXx2F8uNJ4TaonuEO/V7/5xoz7kdsDACT8=
 go.opentelemetry.io/otel v1.24.0 h1:0LAOdjNmQeSTzGBzduGe/rU4tZhMwL5rWgtp9Ku5Jfo=
@@ -1525,11 +1405,8 @@ go.opentelemetry.io/otel/sdk/metric v1.21.0 h1:smhI5oD714d6jHE6Tie36fPx4WDFIg+Y6
 go.opentelemetry.io/otel/sdk/metric v1.21.0/go.mod h1:FJ8RAsoPGv/wYMgBdUJXOm+6pzFY3YdljnXtv1SBE8Q=
 go.opentelemetry.io/otel/trace v1.24.0 h1:CsKnnL4dUAr/0llH9FKuc698G04IrpWV0MQA/Y1YELI=
 go.opentelemetry.io/otel/trace v1.24.0/go.mod h1:HPc3Xr/cOApsBI154IU0OI0HJexz+aw5uPdbs3UCjNU=
-go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI=
 go.opentelemetry.io/proto/otlp v1.0.0 h1:T0TX0tmXU8a3CbNXzEKGeU5mIVOdf0oykP+u2lIVU/I=
 go.opentelemetry.io/proto/otlp v1.0.0/go.mod h1:Sy6pihPLfYHkr3NkUbEhGHFhINUSI/v80hjKIs5JXpM=
-go.starlark.net v0.0.0-20220817180228-f738f5508c12 h1:xOBJXWGEDwU5xSDxH6macxO11Us0AH2fTa9rmsbbF7g=
-go.starlark.net v0.0.0-20220817180228-f738f5508c12/go.mod h1:VZcBMdr3cT3PnBoWunTabuSEXwVAH+ZJ5zxfs3AdASk=
 go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
 go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
 go.uber.org/atomic v1.5.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ=
@@ -1679,9 +1556,7 @@ golang.org/x/net v0.0.0-20210331212208-0fccb6fa2b5c/go.mod h1:p54w0d4576C0XHj96b
 golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM=
 golang.org/x/net v0.0.0-20210423184538-5f58ad60dda6/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk=
 golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk=
-golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
 golang.org/x/net v0.0.0-20211008194852-3b03d305991f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
-golang.org/x/net v0.0.0-20211029224645-99673261e6eb/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
 golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
 golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
 golang.org/x/net v0.0.0-20220607020251-c690dde0001d/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
@@ -1781,7 +1656,6 @@ golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20210423185535-09eb48e85fd7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210806184541-e5e7981a1069/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210819135213-f52c844e1c1c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
@@ -1825,7 +1699,6 @@ golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
-golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ=
 golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
 golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
 golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
@@ -1914,8 +1787,6 @@ golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8T
 golang.org/x/xerrors v0.0.0-20220517211312-f3a8303e98df/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8=
 golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 h1:+cNy6SZtPcJQH3LJVLOSmiC7MMxXNOb3PU/VUEz+EhU=
 golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028/go.mod h1:NDW/Ps6MPRej6fsCIbMTohpP40sJ/P/vI1MoTEGwX90=
-gomodules.xyz/jsonpatch/v2 v2.2.0 h1:4pT439QV83L+G9FkcCriY6EkpcK6r6bK+A5FBUMI7qY=
-gomodules.xyz/jsonpatch/v2 v2.2.0/go.mod h1:WXp+iVDkoLQqPudfQ9GBlwB2eZ5DKOnjQZCYdOS8GPY=
 gonum.org/v1/gonum v0.14.0 h1:2NiG67LD1tEH0D7kM+ps2V+fXmsAnpUeec7n8tcr4S0=
 gonum.org/v1/gonum v0.14.0/go.mod h1:AoWeoz0becf9QMWtE8iWXNXc27fK4fNeHNf/oMejGfU=
 google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE=
@@ -1995,7 +1866,6 @@ google.golang.org/genproto v0.0.0-20210108203827-ffc7fda8c3d7/go.mod h1:FWY/as6D
 google.golang.org/genproto v0.0.0-20210226172003-ab064af71705/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
 google.golang.org/genproto v0.0.0-20210401141331-865547bb08e2/go.mod h1:9lPAdzaEmUacj36I+k7YKbEc5CXzPIeORRgDAUOu28A=
 google.golang.org/genproto v0.0.0-20210624195500-8bfb893ecb84/go.mod h1:SzzZ/N+nwJDaO1kznhnlzqS8ocJICar6hYhVyhi++24=
-google.golang.org/genproto v0.0.0-20220107163113-42d7afdf6368/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc=
 google.golang.org/genproto v0.0.0-20231030173426-d783a09b4405 h1:I6WNifs6pF9tNdSob2W24JtyxIYjzFB9qDlpUC76q+U=
 google.golang.org/genproto v0.0.0-20231030173426-d783a09b4405/go.mod h1:3WDQMjmJk36UQhjQ89emUzb1mdaHcPeeAh4SCBKznB4=
 google.golang.org/genproto/googleapis/api v0.0.0-20231016165738-49dd2c1f3d0b h1:CIC2YMXmIhYw6evmhPxBKJ4fmLbOFtXQN/GV3XOZR8k=
@@ -2022,11 +1892,9 @@ google.golang.org/grpc v1.33.1/go.mod h1:fr5YgcSWrqhRRxogOsw7RzIpsmvOZ6IcH4kBYTp
 google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc=
 google.golang.org/grpc v1.34.0/go.mod h1:WotjhfgOW/POjDeRt8vscBtXq+2VjORFy659qA51WJ8=
 google.golang.org/grpc v1.35.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU=
-google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU=
 google.golang.org/grpc v1.36.1/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU=
 google.golang.org/grpc v1.37.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM=
 google.golang.org/grpc v1.38.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM=
-google.golang.org/grpc v1.40.0/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34=
 google.golang.org/grpc v1.59.0 h1:Z5Iec2pjwb+LEOqzpB2MR12/eKFhDPhuqW91O+4bwUk=
 google.golang.org/grpc v1.59.0/go.mod h1:aUPDwccQo6OTjy7Hct4AfBPD1GptF4fyUjIkQ9YtF98=
 google.golang.org/grpc/examples v0.0.0-20210424002626-9572fd6faeae/go.mod h1:Ly7ZA/ARzg8fnPU9TyZIxoz33sEUuWX7txiqs8lPTgE=
@@ -2042,7 +1910,6 @@ google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGj
 google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c=
 google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
 google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
-google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
 google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I=
 google.golang.org/protobuf v1.33.0 h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGmI=
 google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos=
@@ -2062,8 +1929,6 @@ gopkg.in/guregu/null.v2 v2.1.2/go.mod h1:XORrx8tyS5ZDcyUboCIxQtta/Aujk/6pfWrn9Xe
 gopkg.in/guregu/null.v4 v4.0.0 h1:1Wm3S1WEA2I26Kq+6vcW+w0gcDo44YKYD7YIEJNHDjg=
 gopkg.in/guregu/null.v4 v4.0.0/go.mod h1:YoQhUrADuG3i9WqesrCmpNRwm1ypAgSHYqoOcTu/JrI=
 gopkg.in/inconshreveable/log15.v2 v2.0.0-20180818164646-67afb5ed74ec/go.mod h1:aPpfJ7XW+gOuirDoZ8gHhLh3kZ1B08FtV2bbmy7Jv3s=
-gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc=
-gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw=
 gopkg.in/ini.v1 v1.51.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
 gopkg.in/ini.v1 v1.51.1/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
 gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA=
@@ -2086,7 +1951,6 @@ gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
 gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
 gopkg.in/yaml.v3 v3.0.0-20191120175047-4206685974f2/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
-gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
 gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
@@ -2103,27 +1967,6 @@ honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9
 honnef.co/go/tools v0.1.3/go.mod h1:NgwopIslSNH47DimFoV78dnkksY2EFtX0ajyb3K/las=
 howett.net/plist v1.0.0 h1:7CrbWYbPPO/PyNy38b2EB/+gYbjCe2DXBxgtOOZbSQM=
 howett.net/plist v1.0.0/go.mod h1:lqaXoTrLY4hg8tnEzNru53gicrbv7rrk+2xJA/7hw9g=
-k8s.io/api v0.28.2 h1:9mpl5mOb6vXZvqbQmankOfPIGiudghwCoLl1EYfUZbw=
-k8s.io/api v0.28.2/go.mod h1:RVnJBsjU8tcMq7C3iaRSGMeaKt2TWEUXcpIt/90fjEg=
-k8s.io/apiextensions-apiserver v0.25.3 h1:bfI4KS31w2f9WM1KLGwnwuVlW3RSRPuIsfNF/3HzR0k=
-k8s.io/apiextensions-apiserver v0.25.3/go.mod h1:ZJqwpCkxIx9itilmZek7JgfUAM0dnTsA48I4krPqRmo=
-k8s.io/apimachinery v0.28.2 h1:KCOJLrc6gu+wV1BYgwik4AF4vXOlVJPdiqn0yAWWwXQ=
-k8s.io/apimachinery v0.28.2/go.mod h1:RdzF87y/ngqk9H4z3EL2Rppv5jj95vGS/HaFXrLDApU=
-k8s.io/cli-runtime v0.25.11 h1:GE2yNZm1tN+MJtw1SGMOLesLF7Kp7NVAVqRSTbXfu4o=
-k8s.io/cli-runtime v0.25.11/go.mod h1:r/nEINuHVEpgGhcd2WamU7hD1t/lMnSz8XM44Autltc=
-k8s.io/client-go v0.28.2 h1:DNoYI1vGq0slMBN/SWKMZMw0Rq+0EQW6/AK4v9+3VeY=
-k8s.io/client-go v0.28.2/go.mod h1:sMkApowspLuc7omj1FOSUxSoqjr+d5Q0Yc0LOFnYFJY=
-k8s.io/component-base v0.26.2 h1:IfWgCGUDzrD6wLLgXEstJKYZKAFS2kO+rBRi0p3LqcI=
-k8s.io/component-base v0.26.2/go.mod h1:DxbuIe9M3IZPRxPIzhch2m1eT7uFrSBJUBuVCQEBivs=
-k8s.io/klog v1.0.0 h1:Pt+yjF5aB1xDSVbau4VsWe+dQNzA0qv1LlXdC2dF6Q8=
-k8s.io/klog/v2 v2.100.1 h1:7WCHKK6K8fNhTqfBhISHQ97KrnJNFZMcQvKp7gP/tmg=
-k8s.io/klog/v2 v2.100.1/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0=
-k8s.io/kube-openapi v0.0.0-20230717233707-2695361300d9 h1:LyMgNKD2P8Wn1iAwQU5OhxCKlKJy0sHc+PcDwFB24dQ=
-k8s.io/kube-openapi v0.0.0-20230717233707-2695361300d9/go.mod h1:wZK2AVp1uHCp4VamDVgBP2COHZjqD1T68Rf0CM3YjSM=
-k8s.io/kubectl v0.25.11 h1:6bsft5Gan6BCvQ7cJbDRFjTm4Zfq8GuUYpsWAdVngYE=
-k8s.io/kubectl v0.25.11/go.mod h1:8mIfgkFgT+yJ8/TlmPW1qoRh46H2si9q5nW8id7i9iM=
-k8s.io/utils v0.0.0-20230711102312-30195339c3c7 h1:ZgnF1KZsYxWIifwSNZFZgNtWE89WI5yiP5WwlfDoIyc=
-k8s.io/utils v0.0.0-20230711102312-30195339c3c7/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0=
 lukechampine.com/uint128 v1.3.0 h1:cDdUVfRwDUDovz610ABgFD17nXD4/uDgVHl2sC3+sbo=
 lukechampine.com/uint128 v1.3.0/go.mod h1:c4eWIwlEGaxC/+H1VguhU4PHXNWDCDMUlWdIWl2j1gk=
 modernc.org/cc/v3 v3.41.0 h1:QoR1Sn3YWlmA1T4vLaKZfawdVtSiGx8H+cEojbC7v1Q=
@@ -2155,15 +1998,5 @@ rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0=
 rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA=
 rsc.io/tmplfunc v0.0.3 h1:53XFQh69AfOa8Tw0Jm7t+GV7KZhOi6jzsCzTtKbMvzU=
 rsc.io/tmplfunc v0.0.3/go.mod h1:AG3sTPzElb1Io3Yg4voV9AGZJuleGAwaVRxL9M49PhA=
-sigs.k8s.io/controller-runtime v0.13.0 h1:iqa5RNciy7ADWnIc8QxCbOX5FEKVR3uxVxKHRMc2WIQ=
-sigs.k8s.io/controller-runtime v0.13.0/go.mod h1:Zbz+el8Yg31jubvAEyglRZGdLAjplZl+PgtYNI6WNTI=
-sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd h1:EDPBXCAspyGV4jQlpZSudPeMmr1bNJefnuqLsRAsHZo=
-sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd/go.mod h1:B8JuhiUyNFVKdsE8h686QcCxMaH6HrOAZj4vswFpcB0=
-sigs.k8s.io/kustomize/api v0.12.1 h1:7YM7gW3kYBwtKvoY216ZzY+8hM+lV53LUayghNRJ0vM=
-sigs.k8s.io/kustomize/api v0.12.1/go.mod h1:y3JUhimkZkR6sbLNwfJHxvo1TCLwuwm14sCYnkH6S1s=
-sigs.k8s.io/kustomize/kyaml v0.13.9 h1:Qz53EAaFFANyNgyOEJbT/yoIHygK40/ZcvU3rgry2Tk=
-sigs.k8s.io/kustomize/kyaml v0.13.9/go.mod h1:QsRbD0/KcU+wdk0/L0fIp2KLnohkVzs6fQ85/nOXac4=
-sigs.k8s.io/structured-merge-diff/v4 v4.3.0 h1:UZbZAZfX0wV2zr7YZorDz6GXROfDFj6LvqCRm4VUVKk=
-sigs.k8s.io/structured-merge-diff/v4 v4.3.0/go.mod h1:N8hJocpFajUSSeSJ9bOZ77VzejKZaXsTtZo4/u7Io08=
 sigs.k8s.io/yaml v1.4.0 h1:Mk1wCc2gy/F0THH0TAp1QYyJNzRm2KCLy3o5ASXVI5E=
 sigs.k8s.io/yaml v1.4.0/go.mod h1:Ejl7/uTz7PSA4eKMyQCUTnhZYNmLIl+5c2lQPGR2BPY=
diff --git a/integration-tests/e2e_tests_ci_tool/cmd/mask_test_config_cmd.go b/integration-tests/e2e_tests_ci_tool/cmd/mask_test_config_cmd.go
index 0d3ea6f23b5..886ba32f7b5 100644
--- a/integration-tests/e2e_tests_ci_tool/cmd/mask_test_config_cmd.go
+++ b/integration-tests/e2e_tests_ci_tool/cmd/mask_test_config_cmd.go
@@ -1,10 +1,16 @@
 package cmd
 
 import (
+	"encoding/base64"
 	"fmt"
 	"os"
 	"os/exec"
 
+	"github.com/olekukonko/tablewriter"
+	"github.com/pelletier/go-toml/v2"
+	"github.com/smartcontractkit/chainlink-testing-framework/blockchain"
+	ctf_config "github.com/smartcontractkit/chainlink-testing-framework/config"
+	"github.com/smartcontractkit/seth"
 	"github.com/spf13/cobra"
 )
 
@@ -12,25 +18,109 @@ var maskTestConfigCmd = &cobra.Command{
 	Use:   "mask-secrets",
 	Short: "Run 'echo ::add-mask::${secret}' for all secrets in the test config",
 	Run: func(cmd *cobra.Command, args []string) {
-		mustMaskSecret("chainlink image", oc.ChainlinkImage)
-		mustMaskSecret("chainlink version", oc.ChainlinkVersion)
-		mustMaskSecret("pyroscope server url", oc.PyroscopeServerURL)
-		mustMaskSecret("pyroscope key", oc.PyroscopeKey)
-		mustMaskSecret("loki tenant id", oc.LoggingLokiTenantID)
-		mustMaskSecret("loki endpoint", oc.LoggingLokiEndpoint)
-		mustMaskSecret("loki basic auth", oc.LoggingLokiBasicAuth)
-		mustMaskSecret("loki grafana token", oc.LoggingGrafanaToken)
-		mustMaskSecret("pyroscope environment", oc.PyroscopeEnvironment)
-		mustMaskSecret("pyroscope server url", oc.PyroscopeServerURL)
-		mustMaskSecret("pyroscope key", oc.PyroscopeKey)
+		decoded, err := base64.StdEncoding.DecodeString(fromBase64TOML)
+		if err != nil {
+			fmt.Fprintf(os.Stderr, "Error decoding base64 TOML: %v\n", err)
+			os.Exit(1)
+		}
+		var config ctf_config.TestConfig
+		err = toml.Unmarshal(decoded, &config)
+		if err != nil {
+			fmt.Fprintf(os.Stderr, "Error unmarshalling base64 TOML: %v\n", err)
+			os.Exit(1)
+		}
+
+		showCmdInfo()
+
+		if config.ChainlinkImage != nil {
+			mustMaskSecret("chainlink image", config.ChainlinkImage.Image)
+			mustMaskSecret("chainlink version", config.ChainlinkImage.Version)
+		}
+		if config.Pyroscope != nil {
+			mustMaskSecret("pyroscope server url", config.Pyroscope.ServerUrl)
+			mustMaskSecret("pyroscope key", config.Pyroscope.Key)
+		}
+		if config.Logging != nil && config.Logging.Loki != nil {
+			// mustMaskSecret("loki tenant id", config.Logging.RunId)
+			mustMaskSecret("loki endpoint", config.Logging.Loki.Endpoint)
+			mustMaskSecret("loki tenant id", config.Logging.Loki.TenantId)
+			mustMaskSecret("loki basic auth", config.Logging.Loki.BasicAuth)
+			mustMaskSecret("loki bearer token", config.Logging.Loki.BearerToken)
+		}
+		if config.Logging != nil && config.Logging.Grafana != nil {
+			mustMaskSecret("loki grafana token", config.Logging.Grafana.BearerToken)
+		}
+		if config.Network != nil && config.Network.RpcHttpUrls != nil {
+			for _, urls := range config.Network.RpcHttpUrls {
+				for _, url := range urls {
+					mustMaskSecret("rpc http url", &url)
+				}
+			}
+		}
+		if config.Network != nil && config.Network.RpcWsUrls != nil {
+			for _, urls := range config.Network.RpcWsUrls {
+				for _, url := range urls {
+					mustMaskSecret("rpc ws url", &url)
+				}
+			}
+		}
+		if config.Network != nil && config.Network.WalletKeys != nil {
+			for _, keys := range config.Network.WalletKeys {
+				for _, key := range keys {
+					mustMaskSecret("wallet key", &key)
+				}
+			}
+		}
+		// Mask EVMNetworks config
+		if config.Network != nil && config.Network.EVMNetworks != nil {
+			for _, network := range config.Network.EVMNetworks {
+				mustMaskEVMNetworkConfig(network)
+			}
+		}
+		if config.WaspConfig != nil {
+			mustMaskSecret("wasp repo image version uri", config.WaspConfig.RepoImageVersionURI)
+		}
+		// Mask Seth config
+		if config.Seth != nil {
+			mustMaskSethNetworkConfig(config.Seth.Network)
+			for _, network := range config.Seth.Networks {
+				mustMaskSethNetworkConfig(network)
+			}
+		}
 	},
 }
 
-func mustMaskSecret(description string, secret string) {
-	if secret != "" {
+func mustMaskEVMNetworkConfig(network *blockchain.EVMNetwork) {
+	if network != nil {
+		for _, url := range network.HTTPURLs {
+			mustMaskSecret("network rpc url", &url)
+		}
+		for _, url := range network.URLs {
+			mustMaskSecret("network ws url", &url)
+		}
+		for _, key := range network.PrivateKeys {
+			mustMaskSecret("network private key", &key)
+		}
+		mustMaskSecret("network url", &network.URL)
+	}
+}
+
+func mustMaskSethNetworkConfig(network *seth.Network) {
+	if network != nil {
+		for _, url := range network.URLs {
+			mustMaskSecret("network url", &url)
+		}
+		for _, key := range network.PrivateKeys {
+			mustMaskSecret("network private key", &key)
+		}
+	}
+}
+
+func mustMaskSecret(description string, secret *string) {
+	if secret != nil && *secret != "" {
 		fmt.Printf("Mask '%s'\n", description)
-		fmt.Printf("::add-mask::%s\n", secret)
-		cmd := exec.Command("bash", "-c", fmt.Sprintf("echo ::add-mask::%s", secret))
+		fmt.Printf("::add-mask::%s\n", *secret)
+		cmd := exec.Command("bash", "-c", fmt.Sprintf("echo ::add-mask::%s", *secret))
 		err := cmd.Run()
 		if err != nil {
 			fmt.Fprintf(os.Stderr, "Failed to mask secret '%s'", description)
@@ -38,3 +128,33 @@ func mustMaskSecret(description string, secret string) {
 		}
 	}
 }
+
+func showCmdInfo() {
+	data := [][]string{
+		{"This command masks ONLY SELECTED secrets in the test config!"},
+		{"Secrets inside Chainlink Node config are NOT masked"},
+		{"Please ensure that are not present in the test config!"},
+	}
+
+	table := tablewriter.NewWriter(os.Stdout)
+	table.SetHeader([]string{"WARNING!"})
+	table.SetHeaderLine(true)
+	table.SetBorder(true)
+	table.SetColWidth(100)
+	table.SetRowLine(true)
+	table.SetAutoFormatHeaders(true)
+	table.SetAlignment(tablewriter.ALIGN_LEFT)
+	table.SetTablePadding("\t")
+
+	for _, v := range data {
+		table.Append(v)
+	}
+
+	table.Render()
+	fmt.Println()
+}
+
+func init() {
+	maskTestConfigCmd.Flags().StringVar(&fromBase64TOML, FromBase64ConfigFlag, "", "Base64 encoded TOML config to override")
+	maskTestConfigCmd.MarkFlagRequired(FromBase64ConfigFlag)
+}
diff --git a/integration-tests/e2e_tests_ci_tool/cmd/test_config_cmd.go b/integration-tests/e2e_tests_ci_tool/cmd/test_config_cmd.go
index e5d68b8e40e..2426f9a5795 100644
--- a/integration-tests/e2e_tests_ci_tool/cmd/test_config_cmd.go
+++ b/integration-tests/e2e_tests_ci_tool/cmd/test_config_cmd.go
@@ -65,7 +65,7 @@ const (
 var oc OverrideConfig
 
 func init() {
-	cmds := []*cobra.Command{maskTestConfigCmd, createTestConfigCmd, overrideTestConfigCmd}
+	cmds := []*cobra.Command{createTestConfigCmd, overrideTestConfigCmd}
 	for _, c := range cmds {
 		c.Flags().StringArrayVar(&oc.SelectedNetworks, SelectedNetworksFlag, nil, "Selected networks")
 		c.Flags().StringVar(&oc.ChainlinkImage, ChainlinkImageFlag, "", "Chainlink image")
diff --git a/integration-tests/go.mod b/integration-tests/go.mod
index 15b41fec4d9..7fe6cd33527 100644
--- a/integration-tests/go.mod
+++ b/integration-tests/go.mod
@@ -19,6 +19,7 @@ require (
 	github.com/lib/pq v1.10.9
 	github.com/manifoldco/promptui v0.9.0
 	github.com/montanaflynn/stats v0.7.1
+	github.com/olekukonko/tablewriter v0.0.5
 	github.com/onsi/gomega v1.30.0
 	github.com/pelletier/go-toml/v2 v2.2.2
 	github.com/pkg/errors v0.9.1
@@ -38,6 +39,7 @@ require (
 	github.com/smartcontractkit/seth v1.0.12
 	github.com/smartcontractkit/wasp v0.4.7
 	github.com/spf13/cobra v1.8.1
+	github.com/spf13/pflag v1.0.5
 	github.com/stretchr/testify v1.9.0
 	github.com/test-go/testify v1.1.4
 	github.com/testcontainers/testcontainers-go v0.28.0
@@ -343,7 +345,6 @@ require (
 	github.com/mwitkow/grpc-proxy v0.0.0-20230212185441-f345521cb9c9 // indirect
 	github.com/oklog/run v1.1.0 // indirect
 	github.com/oklog/ulid v1.3.1 // indirect
-	github.com/olekukonko/tablewriter v0.0.5 // indirect
 	github.com/opencontainers/go-digest v1.0.0 // indirect
 	github.com/opencontainers/image-spec v1.1.0-rc5 // indirect
 	github.com/opencontainers/runc v1.1.7 // indirect
@@ -394,7 +395,6 @@ require (
 	github.com/spf13/afero v1.9.5 // indirect
 	github.com/spf13/cast v1.6.0 // indirect
 	github.com/spf13/jwalterweatherman v1.1.0 // indirect
-	github.com/spf13/pflag v1.0.5 // indirect
 	github.com/spf13/viper v1.15.0 // indirect
 	github.com/status-im/keycard-go v0.2.0 // indirect
 	github.com/streamingfast/logging v0.0.0-20220405224725-2755dab2ce75 // indirect

From d38858072c7394f2c8cf62bffced3a10a7e1eabb Mon Sep 17 00:00:00 2001
From: lukaszcl <120112546+lukaszcl@users.noreply.github.com>
Date: Fri, 28 Jun 2024 10:30:21 +0200
Subject: [PATCH 070/137] Use reusable workflow in run-selected-e2e-tests.yml

---
 .../actions/run-e2e-docker-test/action.yml    |  10 +-
 .../run-automation-benchmark-e2e-tests.yml    |   4 +-
 .../run-e2e-tests-reusable-workflow.yml       |  14 +-
 .github/workflows/run-selected-e2e-tests.yml  | 206 +++---------------
 4 files changed, 49 insertions(+), 185 deletions(-)

diff --git a/.github/actions/run-e2e-docker-test/action.yml b/.github/actions/run-e2e-docker-test/action.yml
index a95224b3c36..ba09d40a759 100644
--- a/.github/actions/run-e2e-docker-test/action.yml
+++ b/.github/actions/run-e2e-docker-test/action.yml
@@ -3,7 +3,7 @@ inputs:
   test:
     required: true
     description: 'Test to run'
-  test_config_base64_override_unsafe:
+  test_config_base64_override:
     description: 'This is an unsafe way to pass custom base64 test config. It cannot contain any secrets as this value is exposed in Github logs (Set up job -> Inputs) when the workflow is called! The caller workflow must make sure to remove secrets from this input.'
     required: false
     type: string        
@@ -21,7 +21,7 @@ runs:
         go-version: '1.21.7'
 
     - name: Set default test config override
-      if: ${{ inputs.test_config_base64_override_unsafe == '' }}
+      if: ${{ inputs.test_config_base64_override == '' }}
       run: |      
         # Mask secret args before using them
         # For more, see https://docs.github.com/en/actions/using-workflows/workflow-commands-for-github-actions#masking-a-value-in-a-log
@@ -70,12 +70,12 @@ runs:
         echo "BASE64_CONFIG_OVERRIDE=$BASE64_CONFIG_OVERRIDE" >> $GITHUB_ENV
 
     - name: Mask secrets in test config base64 override
-      id: mask-secrets-in-test-config-base64-override
-      if: ${{ inputs.test_config_base64_override_unsafe != '' }}
+      id: mask-secrets-in-test_config_base64_override
+      if: ${{ inputs.test_config_base64_override != '' }}
       run: |
         # Mask secrets inside the test config override
         cd integration-tests/e2e_tests_ci_tool/
-        go run main.go test-config mask-secrets --from-base64-config ${{ inputs.test_config_base64_override_unsafe }}
+        go run main.go test-config mask-secrets --from-base64-config ${{ inputs.test_config_base64_override }}
 
         exit 1 # Fail the job
 
diff --git a/.github/workflows/run-automation-benchmark-e2e-tests.yml b/.github/workflows/run-automation-benchmark-e2e-tests.yml
index cf65649ab9d..ad0c0d10d4b 100644
--- a/.github/workflows/run-automation-benchmark-e2e-tests.yml
+++ b/.github/workflows/run-automation-benchmark-e2e-tests.yml
@@ -11,7 +11,7 @@ on:
         options:
           - TestAutomationBenchmark
           - TestAutomationBenchmark_1000Upkeeps_2_1
-      test-config-base64-override:
+      test_config_base64_override:
         description: Custom base64 test config
         required: false
         type: string
@@ -37,7 +37,7 @@ jobs:
       test-ids: ${{ github.event.inputs.test-id }}
       # Select tests to run from .github/e2e_tests.yml based on workflow name 
       test-workflow: Run Automation Product Nightly E2E Tests
-      test-config-base64-override: ${{ github.event.inputs.test-config-base64-override }}
+      test_config_base64_override: ${{ github.event.inputs.test_config_base64_override }}
     secrets:
       QA_AWS_REGION: ${{ secrets.QA_AWS_REGION }}
       QA_AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }}
diff --git a/.github/workflows/run-e2e-tests-reusable-workflow.yml b/.github/workflows/run-e2e-tests-reusable-workflow.yml
index c589a90c0cd..729be537529 100644
--- a/.github/workflows/run-e2e-tests-reusable-workflow.yml
+++ b/.github/workflows/run-e2e-tests-reusable-workflow.yml
@@ -16,7 +16,7 @@ on:
         description: 'Run tests by workflow name. Example: "Run Nightly E2E Tests"'
         required: false
         type: string        
-      test-config-base64-override-unsafe:
+      test_config_base64_override:
         description: 'This is an unsafe way to pass custom base64 test config. It cannot contain any secrets as this value is exposed in Github logs (Set up job -> Inputs) when the workflow is called! The caller workflow must make sure to remove secrets from this input.'
         required: false
         type: string        
@@ -190,7 +190,7 @@ jobs:
           go-version: '1.21.7'
 
       - name: Set default test config override
-        if: ${{ inputs.test-config-base64-override-unsafe == '' }}
+        if: ${{ inputs.test_config_base64_override == '' }}
         run: |      
           # Mask secret args before using them
           # For more, see https://docs.github.com/en/actions/using-workflows/workflow-commands-for-github-actions#masking-a-value-in-a-log
@@ -239,8 +239,8 @@ jobs:
           echo "BASE64_CONFIG_OVERRIDE=$BASE64_CONFIG_OVERRIDE" >> $GITHUB_ENV
 
       - name: Mask secrets in test config base64 override
-        id: mask-secrets-in-test-config-base64-override
-        if: ${{ inputs.test-config-base64-override-unsafe != '' }}
+        id: mask-secrets-in-test_config_base64_override
+        if: ${{ inputs.test_config_base64_override != '' }}
         run: "to do"
           # - name: Remove secrets from test config base64 override
           #   id: mask-in
@@ -251,12 +251,12 @@ jobs:
 
       # TODO: Remove this step since we allow custom secrets in the test config but we mask them    
       # - name: Override secrets in custom test config
-      #   if: ${{ inputs.test-config-base64-override-unsafe != '' }}          
+      #   if: ${{ inputs.test_config_base64_override != '' }}          
       #   run: |
       #     # Show what parts of the test config are being overridden
       #     go run main.go test-config override \
       #       --dry-run \
-      #       --from-base64-config ${{ inputs.test-config-base64-override-unsafe }} \
+      #       --from-base64-config ${{ inputs.test_config_base64_override }} \
       #       --perycope-server-url ${{ secrets.QA_PYROSCOPE_INSTANCE }} \
       #       --peryscope-key ${{ secrets.QA_PYROSCOPE_KEY }} \
       #       --loki-endpoint https://${{ secrets.GRAFANA_INTERNAL_HOST }}/loki/api/v1/push \
@@ -268,7 +268,7 @@ jobs:
 
       #     # Override custom test config with CI secrets
       #     config_override=$(go run main.go test-config override \
-      #       --from-base64-config ${{ inputs.test-config-base64-override-unsafe }} \
+      #       --from-base64-config ${{ inputs.test_config_base64_override }} \
       #       --perycope-server-url ${{ secrets.QA_PYROSCOPE_INSTANCE }} \
       #       --peryscope-key ${{ secrets.QA_PYROSCOPE_KEY }} \
       #       --loki-endpoint https://${{ secrets.GRAFANA_INTERNAL_HOST }}/loki/api/v1/push \
diff --git a/.github/workflows/run-selected-e2e-tests.yml b/.github/workflows/run-selected-e2e-tests.yml
index 0cf91068e58..cac553e25c0 100644
--- a/.github/workflows/run-selected-e2e-tests.yml
+++ b/.github/workflows/run-selected-e2e-tests.yml
@@ -5,20 +5,18 @@ on:
     inputs:
       chainlink-version:
         description: 'Enter Chainlink version to use for the tests. Example: "v2.10.0" or sha'
-        required: true
+        default: develop
+        required: false
         type: string
       test-ids:
-        description: 'Run tests by test ids separated by commas. Example: "run_all_in_ocr_tests_go,run_TestOCRv2Request_in_ocr2_test_go". Check all test IDs in .github/e2e-tests.yml'
-        required: false
+        description: 'Run all tests "*" by default. Or, enter test IDs to run separated by commas. Example: "run_all_in_ocr_tests_go,run_TestOCRv2Request_in_ocr2_test_go". Check all test IDs in .github/e2e-tests.yml'
+        default: "*"
+        required: true
         type: string
-      test-workflow:
-        description: 'Run tests by workflow name. Example: "Run Nightly E2E Tests"'
+      test_config_base64_override:
+        description: 'Custom base64 test config'
         required: false
-        type: string        
-      test_config_base64_override_unsafe:
-        description: 'This is an unsafe way to pass custom base64 test config. It cannot contain any secrets as this value is exposed in Github logs (Set up job -> Inputs) when the workflow is called! The caller workflow must make sure to remove secrets from this input.'
-        required: false
-        type: string        
+        type: string          
       enable-check-test-configurations:
         description: 'Set to "true" to enable check-test-configurations job'
         required: false
@@ -27,172 +25,38 @@ on:
       with-existing-remote-runner-version:
         description: 'Use the existing remote runner version for k8s tests. Example: "d3bf5044af33e08be788a2df31c4a745cf69d787"'
         required: false
-        type: string
-
-env: 
-  CHAINLINK_IMAGE: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com/chainlink
+        type: string                 
 
 jobs:
-  validate-inputs:
-    name: Validate Workflow Inputs
-    runs-on: ubuntu-latest
-    steps:
-      - name: Check input conditions
-        run: |
-          if [[ "${{ inputs.test-ids }}" != "" && "${{ inputs.test-workflow }}" != "" ]]; then
-            echo "Error: Both 'test-ids' and 'test-workflow' are provided. Please specify only one."
-            exit 1
-          fi
-
-  setup-e2e-tests:
-    name: Setup E2E Tests
-    runs-on: ubuntu-latest
-    outputs:
-      run-docker-tests: ${{ steps.setup.outputs.run-docker-tests }}
-      run-k8s-tests: ${{ steps.setup.outputs.run-k8s-tests }}
-      docker-matrix: ${{ steps.setup.outputs.docker-matrix }}
-      k8s-runner-matrix: ${{ steps.setup.outputs.k8s-runner-matrix }}
-    steps:
-      - name: Checkout code
-        uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2
-      - name: Setup E2E Test Configurations
-        id: setup
-        uses: ./.github/actions/setup-e2e-tests
-        with:
-          config-file-path: ${{ github.workspace }}/.github/e2e-tests.yml
-          enable-check-test-configurations: ${{ inputs.enable-check-test-configurations }}
-          test-ids: ${{ inputs.test-ids }}
-          test-workflow: ${{ inputs.test-workflow }}
-
-  # Run Docker tests
-  docker-tests:
-    name: Run Docker Tests (${{ matrix.tests.id }})
-    needs: setup-e2e-tests
-    if: ${{ needs.setup-e2e-tests.outputs.run-docker-tests == 'true' }}
-    runs-on: ${{ matrix.tests.runsOn }}
-    strategy:
-      fail-fast: false
-      matrix: ${{fromJson(needs.setup-e2e-tests.outputs.docker-matrix)}}
-    environment: integration
-    permissions:
-      actions: read
-      checks: write
-      pull-requests: write
-      id-token: write
-      contents: read
-    env:
-      SELECTED_NETWORKS: SIMULATED
-      CHAINLINK_ENV_USER: ${{ github.actor }}
-      TEST_LOG_LEVEL: debug
-
-    steps:
-      - name: Mask workflow inputs that are secrets
-        if: ${{ inputs.test_config_base64_override_unsafe != '' }}
-        run: |
-          # Mask test_config_base64_override_unsafe input
-          TEST_CONFIG_INPUT=$(jq -r '.inputs.test_config_base64_override_unsafe' $GITHUB_EVENT_PATH)
-          echo ::add-mask::$TEST_CONFIG_INPUT
-
-      - name: Checkout repository
-        uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2
-
-      - name: Run Docker Test
-        id: setup
-        uses: ./.github/actions/run-e2e-docker-test
-        with: 
-          test: ${{ matrix.tests }}
-          test_config_base64_override_unsafe: ${{ inputs.test_config_base64_override_unsafe }}
-
-  # Run K8s tests using old remote runner
-
-  prepare-remote-runner-test-image:
-    needs: setup-e2e-tests
-    if: ${{ needs.setup-e2e-tests.outputs.run-k8s-tests == 'true' }}
-    name: Prepare Remote Runner Test Image
-    runs-on: ubuntu-latest
-    environment: integration
-    permissions:
-      actions: read
-      checks: write
-      pull-requests: write
-      id-token: write
-      contents: read
-    outputs:
-      remote-runner-version: ${{ steps.set-remote-runner-version.outputs.remote-runner-version }}
-    env:
-      ENV_JOB_IMAGE_BASE: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com/chainlink-tests
-    steps:
-      - name: Checkout repository
-        uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2      
-      - name: Build Test Runner Image
-        uses: ./.github/actions/build-test-image
-        if: ${{ inputs.with-existing-remote-runner-version == '' }}
-        with:
-          QA_AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }}
-          QA_AWS_REGION: ${{ secrets.QA_AWS_REGION }}
-          QA_AWS_ACCOUNT_NUMBER: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}
-      - name: Set Remote Runner Version
-        id: set-remote-runner-version
-        run: |
-          if [[ -z "${{ inputs.with-existing-remote-runner-version }}" ]]; then
-            echo "::set-output name=remote-runner-version::${{ github.sha }}"
-          else
-            echo "::set-output name=remote-runner-version::${{ inputs.with-existing-remote-runner-version }}"
-          fi
-
-  run-k8s-runner-tests:
-    needs: [setup-e2e-tests, prepare-remote-runner-test-image]
-    if: ${{ needs.setup-e2e-tests.outputs.run-k8s-tests == 'true' }}
-    name: Run K8s Tests (${{ matrix.tests.id }})
-    runs-on: ${{ matrix.tests.runsOn }}
-    strategy:
-      fail-fast: false
-      matrix: ${{fromJson(needs.setup-e2e-tests.outputs.k8s-runner-matrix)}}
-    environment: integration
+  call-run-e2e-tests-workflow:
+    name: Run E2E Tests
+    uses: ./.github/workflows/run-e2e-tests-reusable-workflow.yml
     permissions:
       actions: read
       checks: write
       pull-requests: write
       id-token: write
-      contents: read
-    env:
-      SELECTED_NETWORKS: SIMULATED
-      CHAINLINK_COMMIT_SHA: ${{ inputs.evm-ref || github.sha }}
-      CHAINLINK_ENV_USER: ${{ github.actor }}
-      TEST_LOG_LEVEL: debug
+      contents: read    
+    with:
+      chainlink-version: ${{ github.event.inputs.chainlink-version }}
+      test-ids: ${{ github.event.inputs.test-ids }}
+      test_config_base64_override: ${{ github.event.inputs.test_config_base64_override }}
+      with-existing-remote-runner-version: ${{ github.event.inputs.with-existing-remote-runner-version }}
+      # Use fromJSON to convert string to boolean. More info: https://github.com/actions/runner/issues/2206#issuecomment-1532246677
+      enable-check-test-configurations: ${{ fromJSON(github.event.inputs.enable-check-test-configurations) }}
+    secrets:
+      QA_AWS_REGION: ${{ secrets.QA_AWS_REGION }}
+      QA_AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }}
+      QA_AWS_ACCOUNT_NUMBER: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}
+      QA_PYROSCOPE_INSTANCE: ${{ secrets.QA_PYROSCOPE_INSTANCE }}
+      QA_PYROSCOPE_KEY: ${{ secrets.QA_PYROSCOPE_KEY }}
+      QA_KUBECONFIG: ${{ secrets.QA_KUBECONFIG }}
+      GRAFANA_INTERNAL_TENANT_ID: ${{ secrets.GRAFANA_INTERNAL_TENANT_ID }}
+      GRAFANA_INTERNAL_BASIC_AUTH: ${{ secrets.GRAFANA_INTERNAL_BASIC_AUTH }}
+      GRAFANA_INTERNAL_HOST: ${{ secrets.GRAFANA_INTERNAL_HOST }}
+      GRAFANA_INTERNAL_URL_SHORTENER_TOKEN: ${{ secrets.GRAFANA_INTERNAL_URL_SHORTENER_TOKEN }}
+      GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}  
+      AWS_REGION: ${{ secrets.QA_AWS_REGION }}
+      AWS_OIDC_IAM_ROLE_VALIDATION_PROD_ARN: ${{ secrets.AWS_OIDC_IAM_ROLE_VALIDATION_PROD_ARN }}
+      AWS_API_GW_HOST_GRAFANA: ${{ secrets.AWS_API_GW_HOST_GRAFANA }}        
 
-    steps:
-      - name: Checkout repository
-        uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2
-      - name: Install jq
-        run: sudo apt-get install -y jq
-      - name: Show Test Configuration
-        run: echo '${{ toJson(matrix.tests) }}' | jq .
-      - name: Show Remote Runner Version
-        run: |
-          echo "Remote Runner Version: ${{ needs.prepare-remote-runner-test-image.outputs.remote-runner-version }}"
-      - name: Run Tests
-        uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/run-tests@af92c5fae8dcf1659201e907db82d221fc304b94 # v2.3.21
-        env:
-          DETACH_RUNNER: true
-          TEST_SUITE: ${{ matrix.tests.remoteRunnerTestSuite }}
-          TEST_TYPE: ${{ matrix.tests.remoteRunnerTestSuite }}
-          TEST_TEST_TYPE: ${{ matrix.tests.remoteRunnerTestSuite }}
-          RR_MEM: ${{ matrix.tests.remoteRunnerMemory }}
-          TEST_ARGS: -test.timeout 900h -test.memprofile memprofile.out -test.cpuprofile profile.out
-          ENV_JOB_IMAGE: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com/chainlink-tests:${{ needs.prepare-remote-runner-test-image.outputs.remote-runner-version }}
-          INTERNAL_DOCKER_REPO: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com
-          # We can comment these out when we have a stable soak test and aren't worried about resource consumption
-          TEST_UPLOAD_CPU_PROFILE: true
-          TEST_UPLOAD_MEM_PROFILE: true
-        with:
-          test_command_to_run: ${{ matrix.tests.testCmd }}
-          test_download_vendor_packages_command: make gomod
-          cl_repo: ${{ env.CHAINLINK_IMAGE }}
-          cl_image_tag: ${{ env.CHAINLINK_VERSION }}
-          token: ${{ secrets.GH_TOKEN }}
-          should_cleanup: false
-          go_mod_path: ./integration-tests/go.mod
-          QA_AWS_REGION: ${{ secrets.QA_AWS_REGION }}
-          QA_AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }}
-          QA_KUBECONFIG: ${{ secrets.QA_KUBECONFIG }} 

From f9175318dddbe60cb8572e46d4f4bf334fe06bee Mon Sep 17 00:00:00 2001
From: lukaszcl <120112546+lukaszcl@users.noreply.github.com>
Date: Fri, 28 Jun 2024 10:34:55 +0200
Subject: [PATCH 071/137] Remove unnecessary permissions

---
 .github/workflows/run-selected-e2e-tests.yml | 6 ------
 1 file changed, 6 deletions(-)

diff --git a/.github/workflows/run-selected-e2e-tests.yml b/.github/workflows/run-selected-e2e-tests.yml
index cac553e25c0..fcea08906c3 100644
--- a/.github/workflows/run-selected-e2e-tests.yml
+++ b/.github/workflows/run-selected-e2e-tests.yml
@@ -31,12 +31,6 @@ jobs:
   call-run-e2e-tests-workflow:
     name: Run E2E Tests
     uses: ./.github/workflows/run-e2e-tests-reusable-workflow.yml
-    permissions:
-      actions: read
-      checks: write
-      pull-requests: write
-      id-token: write
-      contents: read    
     with:
       chainlink-version: ${{ github.event.inputs.chainlink-version }}
       test-ids: ${{ github.event.inputs.test-ids }}

From d8a330b2f311788f82875b86d50684ee8393c7a3 Mon Sep 17 00:00:00 2001
From: lukaszcl <120112546+lukaszcl@users.noreply.github.com>
Date: Fri, 28 Jun 2024 10:39:26 +0200
Subject: [PATCH 072/137] Add a sample test to quickly check CI workflows

---
 .github/e2e-tests.yml                          |  9 +++++++++
 integration-tests/smoke/quick_ci_check_test.go | 13 +++++++++++++
 2 files changed, 22 insertions(+)
 create mode 100644 integration-tests/smoke/quick_ci_check_test.go

diff --git a/.github/e2e-tests.yml b/.github/e2e-tests.yml
index ca701dfee44..093363eaac3 100644
--- a/.github/e2e-tests.yml
+++ b/.github/e2e-tests.yml
@@ -7,6 +7,15 @@
 #
 runner-test-matrix:
 
+  # TODO: REMOVE WHEN THE CI WORKFLOWS TESTING IS DONE
+  - id: integration-tests/smoke/quick_ci_check_test.go
+    path: integration-tests/smoke/quick_ci_check_test.go
+    test-type: docker
+    runs-on: ubuntu-latest
+    workflows:
+      - Run Nightly E2E Tests
+    test-cmd: cd integration-tests/ && go test smoke/quick_ci_check_test.go -timeout 1m -count=1 -json
+
   # Example of 1 runner for all tests in integration-tests/smoke/ocr_test.go
   - id: smoke/ocr_test.go:* 
     path: integration-tests/smoke/ocr_test.go
diff --git a/integration-tests/smoke/quick_ci_check_test.go b/integration-tests/smoke/quick_ci_check_test.go
new file mode 100644
index 00000000000..b784e1f5d27
--- /dev/null
+++ b/integration-tests/smoke/quick_ci_check_test.go
@@ -0,0 +1,13 @@
+package smoke
+
+import (
+	"testing"
+
+	"github.com/smartcontractkit/chainlink-testing-framework/logging"
+)
+
+// TODO: REMOVE WHEN THE CI WORKFLOWS TESTING IS DONE
+func TestCICheck_Passing(t *testing.T) {
+	l := logging.GetTestLogger(t)
+	l.Info().Msg("Quick CI check. Always passing")
+}

From 48a40ae66a4fb83d3bf8df32f418c856e01516b8 Mon Sep 17 00:00:00 2001
From: lukaszcl <120112546+lukaszcl@users.noreply.github.com>
Date: Fri, 28 Jun 2024 10:43:32 +0200
Subject: [PATCH 073/137] Do not use mask cmd in reusable workflow

---
 .../run-e2e-tests-reusable-workflow.yml          | 16 ++++++----------
 1 file changed, 6 insertions(+), 10 deletions(-)

diff --git a/.github/workflows/run-e2e-tests-reusable-workflow.yml b/.github/workflows/run-e2e-tests-reusable-workflow.yml
index 729be537529..252fa8065f3 100644
--- a/.github/workflows/run-e2e-tests-reusable-workflow.yml
+++ b/.github/workflows/run-e2e-tests-reusable-workflow.yml
@@ -226,11 +226,6 @@ jobs:
           
           cd integration-tests/e2e_tests_ci_tool/
 
-          # Mask secrets inside the test config override (like resolved env vars)
-          go run main.go test-config mask-secrets $cmd_args
-
-          echo $chainlink_version
-
           # Create a base64 encoded string of the test config override
           config_override=$(go run main.go test-config create $cmd_args)
 
@@ -238,10 +233,11 @@ jobs:
           echo ::add-mask::$BASE64_CONFIG_OVERRIDE
           echo "BASE64_CONFIG_OVERRIDE=$BASE64_CONFIG_OVERRIDE" >> $GITHUB_ENV
 
-      - name: Mask secrets in test config base64 override
-        id: mask-secrets-in-test_config_base64_override
-        if: ${{ inputs.test_config_base64_override != '' }}
-        run: "to do"
+      # TODO: Remove this step since we do not have to mask secrets anymore
+      # - name: Mask secrets in test config base64 override
+      #   id: mask-secrets-in-test_config_base64_override
+      #   if: ${{ inputs.test_config_base64_override != '' }}
+      #   run: "to do"
           # - name: Remove secrets from test config base64 override
           #   id: mask-in
           # - name: Set outputs
@@ -249,7 +245,7 @@ jobs:
           #   run: |
           #     echo "::set-output name=base64ConfigNoSecrets::$BASE64_CONFIG_OVERRIDE"
 
-      # TODO: Remove this step since we allow custom secrets in the test config but we mask them    
+      # TODO: Remove this step since we do not have to mask or override secrets in custom test config anymore 
       # - name: Override secrets in custom test config
       #   if: ${{ inputs.test_config_base64_override != '' }}          
       #   run: |

From 776f5a0c37cb882bf1e4a870548e382fa35ce6a6 Mon Sep 17 00:00:00 2001
From: lukaszcl <120112546+lukaszcl@users.noreply.github.com>
Date: Fri, 28 Jun 2024 10:49:23 +0200
Subject: [PATCH 074/137] Clean up override test config cmd

---
 .../cmd/override_test_config_cmd.go           | 103 +-----------------
 1 file changed, 1 insertion(+), 102 deletions(-)

diff --git a/integration-tests/e2e_tests_ci_tool/cmd/override_test_config_cmd.go b/integration-tests/e2e_tests_ci_tool/cmd/override_test_config_cmd.go
index 1c60cfce312..f90281e8154 100644
--- a/integration-tests/e2e_tests_ci_tool/cmd/override_test_config_cmd.go
+++ b/integration-tests/e2e_tests_ci_tool/cmd/override_test_config_cmd.go
@@ -18,7 +18,7 @@ var fromBase64TOML string
 
 var overrideTestConfigCmd = &cobra.Command{
 	Use:   "override",
-	Short: "Override base64 encoded TOML config with provided flags",
+	Short: "Override base64 encoded TOML config with provided flags. Overrides only existing fields in the base config.",
 	Run: func(cmd *cobra.Command, args []string) {
 		dryRun, _ := cmd.Flags().GetBool(DryRunFlag)
 
@@ -176,109 +176,8 @@ var overrideTestConfigCmd = &cobra.Command{
 				fmt.Printf("Override not supported for flag: %s\n", flag.Name)
 				os.Exit(1)
 			}
-
 		})
 
-		// // Override base config with flags
-		// if cmd.Flags().Changed(ChainlinkImageFlag) {
-		// 	if baseConfig.ChainlinkImage != nil {
-		// 		baseConfig.ChainlinkImage.Image = &oc.ChainlinkImage
-		// 	}
-		// }
-		// if cmd.Flags().Changed(ChainlinkVersionFlag) {
-		// 	if baseConfig.ChainlinkImage != nil {
-		// 		baseConfig.ChainlinkImage.Version = &oc.ChainlinkVersion
-		// 	}
-		// }
-		// if cmd.Flags().Changed(ChainlinkPostgresVersionFlag) {
-		// 	if baseConfig.ChainlinkImage != nil {
-		// 		baseConfig.ChainlinkImage.PostgresVersion = &oc.ChainlinkPostgresVersion
-		// 	}
-		// }
-		// if cmd.Flags().Changed(SelectedNetworksFlag) {
-		// 	if baseConfig.Network != nil {
-		// 		baseConfig.Network.SelectedNetworks = oc.SelectedNetworks
-		// 	}
-		// }
-		// if cmd.Flags().Changed(LoggingLokiBasicAuthFlag) {
-		// 	if baseConfig.Logging != nil && baseConfig.Logging.Loki != nil {
-		// 		baseConfig.Logging.Loki.BasicAuth = &oc.LoggingLokiBasicAuth
-		// 	}
-		// }
-		// if cmd.Flags().Changed(LoggingLokiEndpointFlag) {
-		// 	if baseConfig.Logging != nil && baseConfig.Logging.Loki != nil {
-		// 		baseConfig.Logging.Loki.Endpoint = &oc.LoggingLokiEndpoint
-		// 	}
-		// }
-		// if cmd.Flags().Changed(LoggingLokiTenantIDFlag) {
-		// 	if baseConfig.Logging != nil && baseConfig.Logging.Loki != nil {
-		// 		baseConfig.Logging.Loki.TenantId = &oc.LoggingLokiTenantID
-		// 	}
-		// }
-		// if cmd.Flags().Changed(LoggingRunIDFlag) {
-		// 	if baseConfig.Logging != nil {
-		// 		baseConfig.Logging.RunId = &oc.LoggingRunID
-		// 	}
-		// }
-		// if cmd.Flags().Changed(LoggingGrafanaBaseURLFlag) {
-		// 	if baseConfig.Logging != nil && baseConfig.Logging.Grafana != nil {
-		// 		baseConfig.Logging.Grafana.BaseUrl = &oc.LoggingGrafanaBaseURL
-		// 	}
-		// }
-		// if cmd.Flags().Changed(LoggingGrafanaDashboardURLFlag) {
-		// 	if baseConfig.Logging != nil && baseConfig.Logging.Grafana != nil {
-		// 		baseConfig.Logging.Grafana.DashboardUrl = &oc.LoggingGrafanaDashboardURL
-		// 	}
-		// }
-		// if cmd.Flags().Changed(LoggingGrafanaTokenFlag) {
-		// 	if baseConfig.Logging != nil && baseConfig.Logging.Grafana != nil {
-		// 		baseConfig.Logging.Grafana.BearerToken = &oc.LoggingGrafanaToken
-		// 	}
-		// }
-		// if cmd.Flags().Changed(LoggingLogTargetsFlag) {
-		// 	if baseConfig.Logging != nil && baseConfig.Logging.LogStream != nil {
-		// 		baseConfig.Logging.LogStream.LogTargets = oc.LoggingLogTargets
-		// 	}
-		// }
-		// if cmd.Flags().Changed(PyroscopeEnabledFlag) {
-		// 	if baseConfig.Pyroscope != nil {
-		// 		baseConfig.Pyroscope.Enabled = &oc.PyroscopeEnabled
-		// 	}
-		// }
-		// if cmd.Flags().Changed(PyroscopeServerURLFlag) {
-		// 	if baseConfig.Pyroscope != nil {
-		// 		baseConfig.Pyroscope.ServerUrl = &oc.PyroscopeServerURL
-		// 	}
-		// }
-		// if cmd.Flags().Changed(PyroscopeEnvironmentFlag) {
-		// 	if baseConfig.Pyroscope != nil {
-		// 		baseConfig.Pyroscope.Environment = &oc.PyroscopeEnvironment
-		// 	}
-		// }
-		// if cmd.Flags().Changed(PyroscopeKeyFlag) {
-		// 	if baseConfig.Pyroscope != nil {
-		// 		baseConfig.Pyroscope.Key = &oc.PyroscopeKey
-		// 	}
-		// }
-		// if cmd.Flags().Changed(PrivateEthereumNetworkExecutionLayerFlag) {
-		// 	if baseConfig.PrivateEthereumNetwork != nil {
-		// 		el := ctf_config.ExecutionLayer(oc.PrivateEthereumNetworkExecutionLayer)
-		// 		baseConfig.PrivateEthereumNetwork.ExecutionLayer = &el
-		// 	}
-		// }
-		// if cmd.Flags().Changed(PrivateEthereumNetworkEthereumVersionFlag) {
-		// 	if baseConfig.PrivateEthereumNetwork != nil {
-		// 		ev := ctf_config.EthereumVersion(oc.PrivateEthereumNetworkEthereumVersion)
-		// 		baseConfig.PrivateEthereumNetwork.EthereumVersion = &ev
-		// 	}
-		// }
-		// if cmd.Flags().Changed(PrivateEthereumNetworkCustomDockerImageFlag) {
-		// 	if baseConfig.PrivateEthereumNetwork != nil {
-		// 		customImages := map[ctf_config.ContainerType]string{"execution_layer": oc.PrivateEthereumNetworkCustomDockerImages}
-		// 		baseConfig.PrivateEthereumNetwork.CustomDockerImages = customImages
-		// 	}
-		// }
-
 		if !dryRun {
 			configToml, err := toml.Marshal(baseConfig)
 			if err != nil {

From 265aac708831e0dfd819246d2c403299d3201cec Mon Sep 17 00:00:00 2001
From: lukaszcl <120112546+lukaszcl@users.noreply.github.com>
Date: Fri, 28 Jun 2024 10:52:47 +0200
Subject: [PATCH 075/137] Fix masking of full chainlink image

---
 .github/workflows/run-e2e-tests-reusable-workflow.yml | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/run-e2e-tests-reusable-workflow.yml b/.github/workflows/run-e2e-tests-reusable-workflow.yml
index 252fa8065f3..701c63cbd4c 100644
--- a/.github/workflows/run-e2e-tests-reusable-workflow.yml
+++ b/.github/workflows/run-e2e-tests-reusable-workflow.yml
@@ -194,7 +194,8 @@ jobs:
         run: |      
           # Mask secret args before using them
           # For more, see https://docs.github.com/en/actions/using-workflows/workflow-commands-for-github-actions#masking-a-value-in-a-log
-          echo ::add-mask::${{ env.CHAINLINK_IMAGE }}
+          CHAINLINK_IMAGE=${{ env.CHAINLINK_IMAGE }}
+          echo ::add-mask::$CHAINLINK_IMAGE
           echo "cl image: ${{ env.CHAINLINK_IMAGE }}"
 
           # Use overrides from e2e-tests.yml or defaults 

From 535a49fae52b3e7f8e762c5f72733d40100b913a Mon Sep 17 00:00:00 2001
From: lukaszcl <120112546+lukaszcl@users.noreply.github.com>
Date: Fri, 28 Jun 2024 11:01:30 +0200
Subject: [PATCH 076/137] Finish "Set default test config override" step in
 reusable workflow

---
 .../run-e2e-tests-reusable-workflow.yml       | 67 +++++++++++++------
 .../cmd/create_test_config_cmd.go             | 12 ++--
 .../cmd/override_test_config_cmd.go           |  8 +--
 .../e2e_tests_ci_tool/cmd/test_config_cmd.go  |  8 +--
 4 files changed, 61 insertions(+), 34 deletions(-)

diff --git a/.github/workflows/run-e2e-tests-reusable-workflow.yml b/.github/workflows/run-e2e-tests-reusable-workflow.yml
index 701c63cbd4c..84122a0bb5d 100644
--- a/.github/workflows/run-e2e-tests-reusable-workflow.yml
+++ b/.github/workflows/run-e2e-tests-reusable-workflow.yml
@@ -204,10 +204,22 @@ jobs:
           chainlink_postgres_version="${{ matrix.tests.testConfigOverride.chainlinkPostgresVersion }}"
           chainlink_upgrade_version="${{ matrix.tests.testConfigOverride.chainlinkUpgradeVersion }}"
           selected_networks="${{ matrix.tests.testConfigOverride.selectedNetworks || env.SELECTED_NETWORKS}}"
+          pyroscope_server="${{ secrets.QA_PYROSCOPE_INSTANCE }}"
+          pyroscope_environment="${{ matrix.tests.pyroscopeEnv }}"
+          pyroscope_key="${{ secrets.QA_PYROSCOPE_KEY }}"
+          loki_run_id="${{ github.run_id }}"
+          loki_endpoint="https://${{ secrets.GRAFANA_INTERNAL_HOST }}/loki/api/v1/push"
+          loki_tenant_id="${{ secrets.GRAFANA_INTERNAL_TENANT_ID }}"
+          loki_basic_auth="${{ secrets.GRAFANA_INTERNAL_BASIC_AUTH }}"
+          logstream_log_targets="${{ vars.LOGSTREAM_LOG_TARGETS }}"
+          # This is Grafana GAP's address
+          grafana_url="http://localhost:8080/primary"
+          grafana_dashboard_url="/d/ddf75041-1e39-42af-aa46-361fe4c36e9e/ci-e2e-tests-logs"
+          grafana_bearer_token="${{ secrets.GRAFANA_INTERNAL_URL_SHORTENER_TOKEN }}"
 
           # Build the command line arguments
           # Need to check if the variable is set before adding it to the command
-          # Otherwise "" will be passed as an argument which is not what we want
+          # Otherwise "" will be passed as an argument value and set in the config which is not what we want
           cmd_args=""
           if [ -n "$chainlink_image" ]; then
             cmd_args+="--chainlink-image=$chainlink_image "
@@ -223,7 +235,40 @@ jobs:
           fi
           if [ -n "$selected_networks" ]; then
             cmd_args+="--selected-networks=$selected_networks "
-          fi    
+          fi
+          if [ -n "$pyroscope_server" ]; then
+            cmd_args+="--pyroscope-server-url=$pyroscope_server "
+          fi
+          if [ -n "$pyroscope_environment" ]; then
+            cmd_args+="--pyroscope-environment=$pyroscope_environment "
+          fi
+          if [ -n "$pyroscope_key" ]; then
+            cmd_args+="--pyroscope-key=$pyroscope_key "
+          fi
+          if [ -n "$loki_endpoint" ]; then
+            cmd_args+="--logging-loki-endpoint=$loki_endpoint "
+          fi
+          if [ -n "$loki_tenant_id" ]; then
+            cmd_args+="--logging-loki-tenant-id=$loki_tenant_id "
+          fi
+          if [ -n "$loki_basic_auth" ]; then
+            cmd_args+="--logging-loki-basic-auth=$loki_basic_auth "
+          fi
+          of [ -n "$loki_run_id" ]; then
+            cmd_args+="--logging-run-id=$loki_run_id "
+          fi
+          if [ -n "$logstream_log_targets" ]; then
+            cmd_args+="--logging-log-targets=$logstream_log_targets "
+          fi  
+          if [ -n "$grafana_url" ]; then
+            cmd_args+="--logging-grafana-base-url=$grafana_url "
+          fi
+          if [ -n "$grafana_dashboard_url" ]; then
+            cmd_args+="--logging-grafana-dashboard-url=$grafana_dashboard_url "
+          fi
+          if [ -n "$grafana_bearer_token" ]; then
+            cmd_args+="--logging-grafana-bearer-token=$grafana_bearer_token "
+          fi  
           
           cd integration-tests/e2e_tests_ci_tool/
 
@@ -287,24 +332,6 @@ jobs:
           aws-role-arn: ${{ secrets.AWS_OIDC_IAM_ROLE_VALIDATION_PROD_ARN }}
           api-gateway-host: ${{ secrets.AWS_API_GW_HOST_GRAFANA }}
           duplicate-authorization-header: "true"
-      # - name: Prepare Base64 TOML override
-      #   uses: ./.github/actions/setup-create-base64-config
-      #   with:
-      #     runId: ${{ github.run_id }}
-      #     # testLogCollect: ${{ vars.TEST_LOG_COLLECT }}
-      #     selectedNetworks: ${{ env.SELECTED_NETWORKS }}
-      #     chainlinkImage: ${{ env.CHAINLINK_IMAGE }}
-      #     chainlinkVersion: ${{ inputs.chainlink-version }}
-      #     pyroscopeServer: ${{ secrets.QA_PYROSCOPE_INSTANCE }} # Avoid sending blank envs https://github.com/orgs/community/discussions/25725
-      #     pyroscopeEnvironment: ${{ matrix.tests.pyroscopeEnv }}
-      #     pyroscopeKey: ${{ secrets.QA_PYROSCOPE_KEY }}
-      #     lokiEndpoint: https://${{ secrets.GRAFANA_INTERNAL_HOST }}/loki/api/v1/push
-      #     lokiTenantId: ${{ secrets.GRAFANA_INTERNAL_TENANT_ID }}
-      #     lokiBasicAuth: ${{ secrets.GRAFANA_INTERNAL_BASIC_AUTH }}
-      #     logstreamLogTargets: ${{ vars.LOGSTREAM_LOG_TARGETS }}
-      #     grafanaUrl: "http://localhost:8080/primary" # This is GAP's address
-      #     grafanaDashboardUrl: "/d/ddf75041-1e39-42af-aa46-361fe4c36e9e/ci-e2e-tests-logs"
-      #     grafanaBearerToken: ${{ secrets.GRAFANA_INTERNAL_URL_SHORTENER_TOKEN }}
       - name: Run Tests
         uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/run-tests@af92c5fae8dcf1659201e907db82d221fc304b94 # v2.3.21
         with:
diff --git a/integration-tests/e2e_tests_ci_tool/cmd/create_test_config_cmd.go b/integration-tests/e2e_tests_ci_tool/cmd/create_test_config_cmd.go
index bc8792757e1..90da9b408fa 100644
--- a/integration-tests/e2e_tests_ci_tool/cmd/create_test_config_cmd.go
+++ b/integration-tests/e2e_tests_ci_tool/cmd/create_test_config_cmd.go
@@ -102,12 +102,12 @@ var createTestConfigCmd = &cobra.Command{
 		if cmd.Flags().Changed(LoggingGrafanaDashboardURLFlag) {
 			loggingGrafanaDashboardURL = &oc.LoggingGrafanaDashboardURL
 		}
-		var loggingGrafanaToken *string
-		if cmd.Flags().Changed(LoggingGrafanaTokenFlag) {
-			loggingGrafanaToken = &oc.LoggingGrafanaToken
+		var loggingGrafanaBearerToken *string
+		if cmd.Flags().Changed(LoggingGrafanaBearerTokenFlag) {
+			loggingGrafanaBearerToken = &oc.LoggingGrafanaBearerToken
 		}
 
-		if testLogCollect != nil || loggingRunID != nil || loggingLogTargets != nil || loggingLokiEndpoint != nil || loggingLokiTenantID != nil || loggingLokiBasicAuth != nil || loggingGrafanaBaseURL != nil || loggingGrafanaDashboardURL != nil || loggingGrafanaToken != nil {
+		if testLogCollect != nil || loggingRunID != nil || loggingLogTargets != nil || loggingLokiEndpoint != nil || loggingLokiTenantID != nil || loggingLokiBasicAuth != nil || loggingGrafanaBaseURL != nil || loggingGrafanaDashboardURL != nil || loggingGrafanaBearerToken != nil {
 			tc.Logging = &ctf_config.LoggingConfig{}
 			tc.Logging.TestLogCollect = testLogCollect
 			tc.Logging.RunId = loggingRunID
@@ -123,11 +123,11 @@ var createTestConfigCmd = &cobra.Command{
 					Endpoint:  loggingLokiEndpoint,
 				}
 			}
-			if loggingGrafanaBaseURL != nil || loggingGrafanaDashboardURL != nil || loggingGrafanaToken != nil {
+			if loggingGrafanaBaseURL != nil || loggingGrafanaDashboardURL != nil || loggingGrafanaBearerToken != nil {
 				tc.Logging.Grafana = &ctf_config.GrafanaConfig{
 					BaseUrl:      loggingGrafanaBaseURL,
 					DashboardUrl: loggingGrafanaDashboardURL,
-					BearerToken:  loggingGrafanaToken,
+					BearerToken:  loggingGrafanaBearerToken,
 				}
 			}
 		}
diff --git a/integration-tests/e2e_tests_ci_tool/cmd/override_test_config_cmd.go b/integration-tests/e2e_tests_ci_tool/cmd/override_test_config_cmd.go
index f90281e8154..a0fbb133e00 100644
--- a/integration-tests/e2e_tests_ci_tool/cmd/override_test_config_cmd.go
+++ b/integration-tests/e2e_tests_ci_tool/cmd/override_test_config_cmd.go
@@ -106,12 +106,12 @@ var overrideTestConfigCmd = &cobra.Command{
 				} else {
 					logIfDryRun(dryRun, "No 'Logging.Grafana' found in config. Will NOT OVERRIDE with %s\n", LoggingGrafanaDashboardURLFlag)
 				}
-			case LoggingGrafanaTokenFlag:
+			case LoggingGrafanaBearerTokenFlag:
 				if baseConfig.Logging != nil && baseConfig.Logging.Grafana != nil && baseConfig.Logging.Grafana.BearerToken != nil {
-					logIfDryRun(dryRun, "Found 'Logging.Grafana.BearerToken' in config. Will override it with %s", LoggingGrafanaTokenFlag)
-					baseConfig.Logging.Grafana.BearerToken = &oc.LoggingGrafanaToken
+					logIfDryRun(dryRun, "Found 'Logging.Grafana.BearerToken' in config. Will override it with %s", LoggingGrafanaBearerTokenFlag)
+					baseConfig.Logging.Grafana.BearerToken = &oc.LoggingGrafanaBearerToken
 				} else {
-					logIfDryRun(dryRun, "No 'Logging.Grafana' found in config. Will NOT OVERRIDE with %s\n", LoggingGrafanaTokenFlag)
+					logIfDryRun(dryRun, "No 'Logging.Grafana' found in config. Will NOT OVERRIDE with %s\n", LoggingGrafanaBearerTokenFlag)
 				}
 			case LoggingLogTargetsFlag:
 				if baseConfig.Logging != nil && baseConfig.Logging.LogStream != nil && baseConfig.Logging.LogStream.LogTargets != nil {
diff --git a/integration-tests/e2e_tests_ci_tool/cmd/test_config_cmd.go b/integration-tests/e2e_tests_ci_tool/cmd/test_config_cmd.go
index 2426f9a5795..7f10f1cab4e 100644
--- a/integration-tests/e2e_tests_ci_tool/cmd/test_config_cmd.go
+++ b/integration-tests/e2e_tests_ci_tool/cmd/test_config_cmd.go
@@ -32,7 +32,7 @@ type OverrideConfig struct {
 	LoggingLokiBasicAuth                     string
 	LoggingGrafanaBaseURL                    string
 	LoggingGrafanaDashboardURL               string
-	LoggingGrafanaToken                      string
+	LoggingGrafanaBearerToken                string
 	PrivateEthereumNetworkExecutionLayer     string
 	PrivateEthereumNetworkEthereumVersion    string
 	PrivateEthereumNetworkCustomDockerImages string
@@ -50,7 +50,7 @@ const (
 	LoggingLokiTenantIDFlag                     = "logging-loki-tenant-id"
 	LoggingGrafanaBaseURLFlag                   = "logging-grafana-base-url"
 	LoggingGrafanaDashboardURLFlag              = "logging-grafana-dashboard-url"
-	LoggingGrafanaTokenFlag                     = "logging-grafana-token"
+	LoggingGrafanaBearerTokenFlag               = "logging-grafana-bearer-token"
 	LoggingLogTargetsFlag                       = "logging-log-targets"
 	LoggingTestLogCollectFlag                   = "logging-test-log-collect"
 	PyroscopeEnabledFlag                        = "pyroscope-enabled"
@@ -83,7 +83,7 @@ func init() {
 		c.Flags().StringVar(&oc.LoggingLokiBasicAuth, LoggingLokiBasicAuthFlag, "", "")
 		c.Flags().StringVar(&oc.LoggingGrafanaBaseURL, LoggingGrafanaBaseURLFlag, "", "")
 		c.Flags().StringVar(&oc.LoggingGrafanaDashboardURL, LoggingGrafanaDashboardURLFlag, "", "")
-		c.Flags().StringVar(&oc.LoggingGrafanaToken, LoggingGrafanaTokenFlag, "", "")
+		c.Flags().StringVar(&oc.LoggingGrafanaBearerToken, LoggingGrafanaBearerTokenFlag, "", "")
 		c.Flags().StringVar(&oc.PrivateEthereumNetworkExecutionLayer, PrivateEthereumNetworkExecutionLayerFlag, "", "")
 		c.Flags().StringVar(&oc.PrivateEthereumNetworkEthereumVersion, PrivateEthereumNetworkEthereumVersionFlag, "", "")
 		c.Flags().StringVar(&oc.PrivateEthereumNetworkCustomDockerImages, PrivateEthereumNetworkCustomDockerImageFlag, "", "")
@@ -111,7 +111,7 @@ func init() {
 			oc.LoggingLokiBasicAuth = mustResolveEnvPlaceholder(oc.LoggingLokiBasicAuth)
 			oc.LoggingGrafanaBaseURL = mustResolveEnvPlaceholder(oc.LoggingGrafanaBaseURL)
 			oc.LoggingGrafanaDashboardURL = mustResolveEnvPlaceholder(oc.LoggingGrafanaDashboardURL)
-			oc.LoggingGrafanaToken = mustResolveEnvPlaceholder(oc.LoggingGrafanaToken)
+			oc.LoggingGrafanaBearerToken = mustResolveEnvPlaceholder(oc.LoggingGrafanaBearerToken)
 			oc.PrivateEthereumNetworkExecutionLayer = mustResolveEnvPlaceholder(oc.PrivateEthereumNetworkExecutionLayer)
 			oc.PrivateEthereumNetworkEthereumVersion = mustResolveEnvPlaceholder(oc.PrivateEthereumNetworkEthereumVersion)
 			oc.PrivateEthereumNetworkCustomDockerImages = mustResolveEnvPlaceholder(oc.PrivateEthereumNetworkCustomDockerImages)

From bb8f8512526226769e8cea9bcf0bec1d7f1095bc Mon Sep 17 00:00:00 2001
From: lukaszcl <120112546+lukaszcl@users.noreply.github.com>
Date: Fri, 28 Jun 2024 11:04:10 +0200
Subject: [PATCH 077/137] Remove env.CHAINLINK_IMAGE mask as it is not a scret

---
 .github/workflows/run-e2e-tests-reusable-workflow.yml | 6 ------
 1 file changed, 6 deletions(-)

diff --git a/.github/workflows/run-e2e-tests-reusable-workflow.yml b/.github/workflows/run-e2e-tests-reusable-workflow.yml
index 84122a0bb5d..06975fed908 100644
--- a/.github/workflows/run-e2e-tests-reusable-workflow.yml
+++ b/.github/workflows/run-e2e-tests-reusable-workflow.yml
@@ -192,12 +192,6 @@ jobs:
       - name: Set default test config override
         if: ${{ inputs.test_config_base64_override == '' }}
         run: |      
-          # Mask secret args before using them
-          # For more, see https://docs.github.com/en/actions/using-workflows/workflow-commands-for-github-actions#masking-a-value-in-a-log
-          CHAINLINK_IMAGE=${{ env.CHAINLINK_IMAGE }}
-          echo ::add-mask::$CHAINLINK_IMAGE
-          echo "cl image: ${{ env.CHAINLINK_IMAGE }}"
-
           # Use overrides from e2e-tests.yml or defaults 
           chainlink_version="${{ matrix.tests.testConfigOverride.chainlinkVersion || github.sha }}"
           chainlink_image="${{ matrix.tests.testConfigOverride.chainlinkImage || env.CHAINLINK_IMAGE }}"

From 17425751ca5f431196832c7f015682eef664b039 Mon Sep 17 00:00:00 2001
From: lukaszcl <120112546+lukaszcl@users.noreply.github.com>
Date: Fri, 28 Jun 2024 11:10:16 +0200
Subject: [PATCH 078/137] Fix reusable workflow

---
 .github/workflows/run-e2e-tests-reusable-workflow.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/run-e2e-tests-reusable-workflow.yml b/.github/workflows/run-e2e-tests-reusable-workflow.yml
index 06975fed908..b4d56c6f566 100644
--- a/.github/workflows/run-e2e-tests-reusable-workflow.yml
+++ b/.github/workflows/run-e2e-tests-reusable-workflow.yml
@@ -248,7 +248,7 @@ jobs:
           if [ -n "$loki_basic_auth" ]; then
             cmd_args+="--logging-loki-basic-auth=$loki_basic_auth "
           fi
-          of [ -n "$loki_run_id" ]; then
+          if [ -n "$loki_run_id" ]; then
             cmd_args+="--logging-run-id=$loki_run_id "
           fi
           if [ -n "$logstream_log_targets" ]; then

From c7fa80ed2b4f848492a577296782eba8f064ecbd Mon Sep 17 00:00:00 2001
From: lukaszcl <120112546+lukaszcl@users.noreply.github.com>
Date: Fri, 28 Jun 2024 11:22:51 +0200
Subject: [PATCH 079/137] Create create-default-e2e-config-override action

---
 .../action.yml                                | 112 ++++++++++++++++++
 .../run-e2e-tests-reusable-workflow.yml       | 101 +++-------------
 2 files changed, 131 insertions(+), 82 deletions(-)
 create mode 100644 .github/actions/create-default-e2e-config-override/action.yml

diff --git a/.github/actions/create-default-e2e-config-override/action.yml b/.github/actions/create-default-e2e-config-override/action.yml
new file mode 100644
index 00000000000..0ac1d160d45
--- /dev/null
+++ b/.github/actions/create-default-e2e-config-override/action.yml
@@ -0,0 +1,112 @@
+name: 'Create Default E2E Test Config Override From Inputs'
+inputs:
+  chainlink_image:
+    required: false
+    description: 'Chainlink image to use for the tests. Example: "chainlink/chainlink:develop"'
+  chainlink_version:
+    required: false
+    description: 'Chainlink version to use for the tests. Example: "v2.10.0" or sha'
+  chainlink_postgres_version:
+    required: false
+    description: 'Chainlink Postgres version to use for the tests'
+  chainlink_upgrade_version:
+    required: false
+    description: 'Chainlink upgrade version to use for the tests'
+  selected_networks:
+    required: false
+    description: 'Selected networks for the tests'
+  pyroscope_server:
+    required: false
+    description: 'Pyroscope server for the tests'
+  pyroscope_environment:
+    required: false
+    description: 'Pyroscope environment for the tests'
+  pyroscope_key:
+    required: false
+    description: 'Pyroscope key for the tests'
+  loki_run_id:
+    required: false
+    description: 'Loki run ID for the tests'
+  loki_endpoint:
+    required: false
+    description: 'Loki endpoint for the tests'
+  loki_tenant_id:
+    required: false
+    description: 'Loki tenant ID for the tests'
+  loki_basic_auth:
+    required: false
+    description: 'Loki basic authentication for the tests'
+  logstream_log_targets:
+    required: false
+    description: 'Logstream log targets for the tests'
+  grafana_url:
+    required: false
+    description: 'Grafana URL for the tests'
+  grafana_dashboard_url:
+    required: false
+    description: 'Grafana dashboard URL for the tests'
+  grafana_bearer_token:
+    required: false
+    description: 'Grafana bearer token for the tests'
+
+runs:
+  using: 'composite'
+  steps:
+    - name: Build the command line arguments
+      shell: bash
+      run: |
+        cmd_args=""
+        if [ -n "${{ inputs.chainlink_image }}" ]; then
+          cmd_args+="--chainlink-image=${{ inputs.chainlink_image }} "
+        fi
+        if [ -n "${{ inputs.chainlink_version }}" ]; then
+          cmd_args+="--chainlink-version=${{ inputs.chainlink_version }} "
+        fi
+        if [ -n "${{ inputs.chainlink_postgres_version }}" ]; then
+          cmd_args+="--chainlink-postgres-version=${{ inputs.chainlink_postgres_version }} "
+        fi
+        if [ -n "${{ inputs.chainlink_upgrade_version }}" ]; then
+          cmd_args+="--chainlink-upgrade-version=${{ inputs.chainlink_upgrade_version }} "
+        fi
+        if [ -n "${{ inputs.selected_networks }}" ]; then
+          cmd_args+="--selected-networks=${{ inputs.selected_networks }} "
+        fi
+        if [ -n "${{ inputs.pyroscope_server }}" ]; then
+          cmd_args+="--pyroscope-server-url=${{ inputs.pyroscope_server }} "
+        fi
+        if [ -n "${{ inputs.pyroscope_environment }}" ]; then
+          cmd_args+="--pyroscope-environment=${{ inputs.pyroscope_environment }} "
+        fi
+        if [ -n "${{ inputs.pyroscope_key }}" ]; then
+          cmd_args+="--pyroscope-key=${{ inputs.pyroscope_key }} "
+        fi
+        if [ -n "${{ inputs.loki_endpoint }}" ]; then
+          cmd_args+="--logging-loki-endpoint=${{ inputs.loki_endpoint }} "
+        fi
+        if [ -n "${{ inputs.loki_tenant_id }}" ]; then
+          cmd_args+="--logging-loki-tenant-id=${{ inputs.loki_tenant_id }} "
+        fi
+        if [ -n "${{ inputs.loki_basic_auth }}" ]; then
+          cmd_args+="--logging-loki-basic-auth=${{ inputs.loki_basic_auth }} "
+        fi
+        if [ -n "${{ inputs.loki_run_id }}" ]; then
+          cmd_args+="--logging-run-id=${{ inputs.loki_run_id }} "
+        fi
+        if [ -n "${{ inputs.logstream_log_targets }}" ]; then
+          cmd_args+="--logging-log-targets=${{ inputs.logstream_log_targets }} "
+        fi
+        echo "Command line arguments: $cmd_args" 
+        echo "CMD_ARGS=$cmd_args" >> $GITHUB_ENV
+
+    - name: Run the command to create the test config and set it as BASE64_CONFIG_OVERRIDE env variable
+      shell: bash
+      run: |      
+        cd integration-tests/e2e_tests_ci_tool/
+
+        # Create a base64 encoded string of the test config override
+        config_override=$(go run main.go test-config create ${{ env.CMD_ARGS }})
+
+        BASE64_CONFIG_OVERRIDE=$(echo "$config_override" | base64 -w 0)
+        echo ::add-mask::$BASE64_CONFIG_OVERRIDE
+        echo "BASE64_CONFIG_OVERRIDE=$BASE64_CONFIG_OVERRIDE" >> $GITHUB_ENV
+    
\ No newline at end of file
diff --git a/.github/workflows/run-e2e-tests-reusable-workflow.yml b/.github/workflows/run-e2e-tests-reusable-workflow.yml
index b4d56c6f566..62f5351dd1c 100644
--- a/.github/workflows/run-e2e-tests-reusable-workflow.yml
+++ b/.github/workflows/run-e2e-tests-reusable-workflow.yml
@@ -189,89 +189,26 @@ jobs:
         with:
           go-version: '1.21.7'
 
-      - name: Set default test config override
+      - name: Set default test config override (use overrides from e2e-tests.yml or defaults)
         if: ${{ inputs.test_config_base64_override == '' }}
-        run: |      
-          # Use overrides from e2e-tests.yml or defaults 
-          chainlink_version="${{ matrix.tests.testConfigOverride.chainlinkVersion || github.sha }}"
-          chainlink_image="${{ matrix.tests.testConfigOverride.chainlinkImage || env.CHAINLINK_IMAGE }}"
-          chainlink_postgres_version="${{ matrix.tests.testConfigOverride.chainlinkPostgresVersion }}"
-          chainlink_upgrade_version="${{ matrix.tests.testConfigOverride.chainlinkUpgradeVersion }}"
-          selected_networks="${{ matrix.tests.testConfigOverride.selectedNetworks || env.SELECTED_NETWORKS}}"
-          pyroscope_server="${{ secrets.QA_PYROSCOPE_INSTANCE }}"
-          pyroscope_environment="${{ matrix.tests.pyroscopeEnv }}"
-          pyroscope_key="${{ secrets.QA_PYROSCOPE_KEY }}"
-          loki_run_id="${{ github.run_id }}"
-          loki_endpoint="https://${{ secrets.GRAFANA_INTERNAL_HOST }}/loki/api/v1/push"
-          loki_tenant_id="${{ secrets.GRAFANA_INTERNAL_TENANT_ID }}"
-          loki_basic_auth="${{ secrets.GRAFANA_INTERNAL_BASIC_AUTH }}"
-          logstream_log_targets="${{ vars.LOGSTREAM_LOG_TARGETS }}"
-          # This is Grafana GAP's address
-          grafana_url="http://localhost:8080/primary"
-          grafana_dashboard_url="/d/ddf75041-1e39-42af-aa46-361fe4c36e9e/ci-e2e-tests-logs"
-          grafana_bearer_token="${{ secrets.GRAFANA_INTERNAL_URL_SHORTENER_TOKEN }}"
-
-          # Build the command line arguments
-          # Need to check if the variable is set before adding it to the command
-          # Otherwise "" will be passed as an argument value and set in the config which is not what we want
-          cmd_args=""
-          if [ -n "$chainlink_image" ]; then
-            cmd_args+="--chainlink-image=$chainlink_image "
-          fi
-          if [ -n "$chainlink_version" ]; then
-            cmd_args+="--chainlink-version=$chainlink_version "
-          fi
-          if [ -n "$chainlink_postgres_version" ]; then
-            cmd_args+="--chainlink-postgres-version=$chainlink_postgres_version "
-          fi
-          if [ -n "$chainlink_upgrade_version" ]; then
-            cmd_args+="--chainlink-upgrade-version=$chainlink_upgrade_version "
-          fi
-          if [ -n "$selected_networks" ]; then
-            cmd_args+="--selected-networks=$selected_networks "
-          fi
-          if [ -n "$pyroscope_server" ]; then
-            cmd_args+="--pyroscope-server-url=$pyroscope_server "
-          fi
-          if [ -n "$pyroscope_environment" ]; then
-            cmd_args+="--pyroscope-environment=$pyroscope_environment "
-          fi
-          if [ -n "$pyroscope_key" ]; then
-            cmd_args+="--pyroscope-key=$pyroscope_key "
-          fi
-          if [ -n "$loki_endpoint" ]; then
-            cmd_args+="--logging-loki-endpoint=$loki_endpoint "
-          fi
-          if [ -n "$loki_tenant_id" ]; then
-            cmd_args+="--logging-loki-tenant-id=$loki_tenant_id "
-          fi
-          if [ -n "$loki_basic_auth" ]; then
-            cmd_args+="--logging-loki-basic-auth=$loki_basic_auth "
-          fi
-          if [ -n "$loki_run_id" ]; then
-            cmd_args+="--logging-run-id=$loki_run_id "
-          fi
-          if [ -n "$logstream_log_targets" ]; then
-            cmd_args+="--logging-log-targets=$logstream_log_targets "
-          fi  
-          if [ -n "$grafana_url" ]; then
-            cmd_args+="--logging-grafana-base-url=$grafana_url "
-          fi
-          if [ -n "$grafana_dashboard_url" ]; then
-            cmd_args+="--logging-grafana-dashboard-url=$grafana_dashboard_url "
-          fi
-          if [ -n "$grafana_bearer_token" ]; then
-            cmd_args+="--logging-grafana-bearer-token=$grafana_bearer_token "
-          fi  
-          
-          cd integration-tests/e2e_tests_ci_tool/
-
-          # Create a base64 encoded string of the test config override
-          config_override=$(go run main.go test-config create $cmd_args)
-
-          BASE64_CONFIG_OVERRIDE=$(echo "$config_override" | base64 -w 0)
-          echo ::add-mask::$BASE64_CONFIG_OVERRIDE
-          echo "BASE64_CONFIG_OVERRIDE=$BASE64_CONFIG_OVERRIDE" >> $GITHUB_ENV
+        uses: ./.github/actions/create-default-e2e-config-override
+        with:
+          chainlink_version: ${{ matrix.tests.testConfigOverride.chainlinkVersion || github.sha }}
+          chainlink_image: ${{ matrix.tests.testConfigOverride.chainlinkImage || env.CHAINLINK_IMAGE }}
+          chainlink_postgres_version: ${{ matrix.tests.testConfigOverride.chainlinkPostgresVersion }}
+          chainlink_upgrade_version: ${{ matrix.tests.testConfigOverride.chainlinkUpgradeVersion }}
+          selected_networks: ${{ matrix.tests.testConfigOverride.selectedNetworks || env.SELECTED_NETWORKS}}
+          pyroscope_server: ${{ secrets.QA_PYROSCOPE_INSTANCE }}
+          pyroscope_environment: ${{ matrix.tests.pyroscopeEnv }}
+          pyroscope_key: ${{ secrets.QA_PYROSCOPE_KEY }}
+          loki_run_id: ${{ github.run_id }}
+          loki_endpoint: https://${{ secrets.GRAFANA_INTERNAL_HOST }}/loki/api/v1/push
+          loki_tenant_id: ${{ secrets.GRAFANA_INTERNAL_TENANT_ID }}
+          loki_basic_auth: ${{ secrets.GRAFANA_INTERNAL_BASIC_AUTH }}
+          logstream_log_targets: ${{ vars.LOGSTREAM_LOG_TARGETS }}
+          grafana_url: http://localhost:8080/primary # This is Grafana GAP's address
+          grafana_dashboard_url: /d/ddf75041-1e39-42af-aa46-361fe4c36e9e/ci-e2e-tests-logs
+          grafana_bearer_token: ${{ secrets.GRAFANA_INTERNAL_URL_SHORTENER_TOKEN }}
 
       # TODO: Remove this step since we do not have to mask secrets anymore
       # - name: Mask secrets in test config base64 override

From 9861b6e095bf2c9eca37397d0b0282be137a37ea Mon Sep 17 00:00:00 2001
From: lukaszcl <120112546+lukaszcl@users.noreply.github.com>
Date: Fri, 28 Jun 2024 11:38:16 +0200
Subject: [PATCH 080/137] Fix

---
 .../action.yml                                | 28 +++++++++----------
 .../run-e2e-tests-reusable-workflow.yml       | 18 ++++++------
 2 files changed, 22 insertions(+), 24 deletions(-)

diff --git a/.github/actions/create-default-e2e-config-override/action.yml b/.github/actions/create-default-e2e-config-override/action.yml
index 0ac1d160d45..f961f7fd239 100644
--- a/.github/actions/create-default-e2e-config-override/action.yml
+++ b/.github/actions/create-default-e2e-config-override/action.yml
@@ -57,43 +57,43 @@ runs:
       run: |
         cmd_args=""
         if [ -n "${{ inputs.chainlink_image }}" ]; then
-          cmd_args+="--chainlink-image=${{ inputs.chainlink_image }} "
+          cmd_args+="--chainlink-image=\"${{ inputs.chainlink_image }}\" "
         fi
         if [ -n "${{ inputs.chainlink_version }}" ]; then
-          cmd_args+="--chainlink-version=${{ inputs.chainlink_version }} "
+          cmd_args+="--chainlink-version=\"${{ inputs.chainlink_version }}\" "
         fi
         if [ -n "${{ inputs.chainlink_postgres_version }}" ]; then
-          cmd_args+="--chainlink-postgres-version=${{ inputs.chainlink_postgres_version }} "
+          cmd_args+="--chainlink-postgres-version=\"${{ inputs.chainlink_postgres_version }}\" "
         fi
         if [ -n "${{ inputs.chainlink_upgrade_version }}" ]; then
-          cmd_args+="--chainlink-upgrade-version=${{ inputs.chainlink_upgrade_version }} "
+          cmd_args+="--chainlink-upgrade-version=\"${{ inputs.chainlink_upgrade_version }}\" "
         fi
         if [ -n "${{ inputs.selected_networks }}" ]; then
-          cmd_args+="--selected-networks=${{ inputs.selected_networks }} "
+          cmd_args+="--selected-networks=\"${{ inputs.selected_networks }}\" "
         fi
         if [ -n "${{ inputs.pyroscope_server }}" ]; then
-          cmd_args+="--pyroscope-server-url=${{ inputs.pyroscope_server }} "
+          cmd_args+="--pyroscope-server-url=\"${{ inputs.pyroscope_server }}\" "
         fi
         if [ -n "${{ inputs.pyroscope_environment }}" ]; then
-          cmd_args+="--pyroscope-environment=${{ inputs.pyroscope_environment }} "
+          cmd_args+="--pyroscope-environment=\"${{ inputs.pyroscope_environment }}\" "
         fi
         if [ -n "${{ inputs.pyroscope_key }}" ]; then
-          cmd_args+="--pyroscope-key=${{ inputs.pyroscope_key }} "
+          cmd_args+="--pyroscope-key=\"${{ inputs.pyroscope_key }}\" "
         fi
         if [ -n "${{ inputs.loki_endpoint }}" ]; then
-          cmd_args+="--logging-loki-endpoint=${{ inputs.loki_endpoint }} "
+          cmd_args+="--logging-loki-endpoint=\"${{ inputs.loki_endpoint }}\" "
         fi
         if [ -n "${{ inputs.loki_tenant_id }}" ]; then
-          cmd_args+="--logging-loki-tenant-id=${{ inputs.loki_tenant_id }} "
+          cmd_args+="--logging-loki-tenant-id=\"${{ inputs.loki_tenant_id }}\" "
         fi
         if [ -n "${{ inputs.loki_basic_auth }}" ]; then
-          cmd_args+="--logging-loki-basic-auth=${{ inputs.loki_basic_auth }} "
+          cmd_args+="--logging-loki-basic-auth=\"${{ inputs.loki_basic_auth }}\" "
         fi
         if [ -n "${{ inputs.loki_run_id }}" ]; then
-          cmd_args+="--logging-run-id=${{ inputs.loki_run_id }} "
+          cmd_args+="--logging-run-id=\"${{ inputs.loki_run_id }}\" "
         fi
         if [ -n "${{ inputs.logstream_log_targets }}" ]; then
-          cmd_args+="--logging-log-targets=${{ inputs.logstream_log_targets }} "
+          cmd_args+="--logging-log-targets=\"${{ inputs.logstream_log_targets }}\" "
         fi
         echo "Command line arguments: $cmd_args" 
         echo "CMD_ARGS=$cmd_args" >> $GITHUB_ENV
@@ -102,8 +102,6 @@ runs:
       shell: bash
       run: |      
         cd integration-tests/e2e_tests_ci_tool/
-
-        # Create a base64 encoded string of the test config override
         config_override=$(go run main.go test-config create ${{ env.CMD_ARGS }})
 
         BASE64_CONFIG_OVERRIDE=$(echo "$config_override" | base64 -w 0)
diff --git a/.github/workflows/run-e2e-tests-reusable-workflow.yml b/.github/workflows/run-e2e-tests-reusable-workflow.yml
index 62f5351dd1c..3baf794bff0 100644
--- a/.github/workflows/run-e2e-tests-reusable-workflow.yml
+++ b/.github/workflows/run-e2e-tests-reusable-workflow.yml
@@ -64,7 +64,7 @@ env:
 
 jobs:
   validate-inputs:
-    name: Validate Workflow Inputs
+    name: Validate workflow inputs
     runs-on: ubuntu-latest
     steps:
       - name: Check input conditions
@@ -75,7 +75,7 @@ jobs:
           fi
 
   check-test-configurations:
-    name: Check Test Configurations
+    name: Check test configurations
     if: ${{ inputs.enable-check-test-configurations }}
     needs: validate-inputs
     runs-on: ubuntu-latest
@@ -93,7 +93,7 @@ jobs:
           fi
 
   load-test-configurations:
-    name: Load Test Configurations
+    name: Load test configurations
     runs-on: ubuntu-latest
     needs: validate-inputs
     outputs:
@@ -158,7 +158,7 @@ jobs:
 
   # Run Docker tests
   docker-tests:
-    name: Run Docker Tests (${{ matrix.tests.id }})
+    name: Run docker tests (${{ matrix.tests.id }})
     needs: load-test-configurations
     if: ${{ needs.load-test-configurations.outputs.run-docker-tests == 'true' }}
     runs-on: ${{ matrix.tests.runsOn }}
@@ -189,7 +189,7 @@ jobs:
         with:
           go-version: '1.21.7'
 
-      - name: Set default test config override (use overrides from e2e-tests.yml or defaults)
+      - name: Create default test config override (use overrides from e2e-tests.yml or defaults)
         if: ${{ inputs.test_config_base64_override == '' }}
         uses: ./.github/actions/create-default-e2e-config-override
         with:
@@ -263,7 +263,7 @@ jobs:
           aws-role-arn: ${{ secrets.AWS_OIDC_IAM_ROLE_VALIDATION_PROD_ARN }}
           api-gateway-host: ${{ secrets.AWS_API_GW_HOST_GRAFANA }}
           duplicate-authorization-header: "true"
-      - name: Run Tests
+      - name: Run tests
         uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/run-tests@af92c5fae8dcf1659201e907db82d221fc304b94 # v2.3.21
         with:
           test_command_to_run: ${{ matrix.tests.testCmd }} 2>&1 | tee /tmp/gotest.log | gotestloghelper -ci -singlepackage -hidepassingtests=false -hidepassinglogs
@@ -292,7 +292,7 @@ jobs:
   prepare-remote-runner-test-image:
     needs: load-test-configurations
     if: ${{ needs.load-test-configurations.outputs.run-k8s-tests == 'true' }}
-    name: Prepare Remote Runner Test Image
+    name: Prepare remote runner test image
     runs-on: ubuntu-latest
     environment: integration
     permissions:
@@ -327,7 +327,7 @@ jobs:
   run-k8s-runner-tests:
     needs: [load-test-configurations, prepare-remote-runner-test-image]
     if: ${{ needs.load-test-configurations.outputs.run-k8s-tests == 'true' }}
-    name: Run K8s Tests (${{ matrix.tests.id }})
+    name: Run k8s tests (${{ matrix.tests.id }})
     runs-on: ${{ matrix.tests.runsOn }}
     strategy:
       fail-fast: false
@@ -355,7 +355,7 @@ jobs:
       - name: Show Remote Runner Version
         run: |
           echo "Remote Runner Version: ${{ needs.prepare-remote-runner-test-image.outputs.remote-runner-version }}"
-      - name: Run Tests
+      - name: Run tests
         uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/run-tests@af92c5fae8dcf1659201e907db82d221fc304b94 # v2.3.21
         env:
           DETACH_RUNNER: true

From c67c8546713853d38296f47df8a1fad124055037 Mon Sep 17 00:00:00 2001
From: lukaszcl <120112546+lukaszcl@users.noreply.github.com>
Date: Fri, 28 Jun 2024 12:39:08 +0200
Subject: [PATCH 081/137] Fix log targets in GHA and add test

---
 .../action.yml                                | 10 +++--
 .../cmd/create_test_config_cmd.go             |  2 +-
 .../cmd/test_config_cmd_test.go               | 44 +++++++++++++++++++
 3 files changed, 51 insertions(+), 5 deletions(-)
 create mode 100644 integration-tests/e2e_tests_ci_tool/cmd/test_config_cmd_test.go

diff --git a/.github/actions/create-default-e2e-config-override/action.yml b/.github/actions/create-default-e2e-config-override/action.yml
index f961f7fd239..267d2f2c991 100644
--- a/.github/actions/create-default-e2e-config-override/action.yml
+++ b/.github/actions/create-default-e2e-config-override/action.yml
@@ -38,7 +38,7 @@ inputs:
     description: 'Loki basic authentication for the tests'
   logstream_log_targets:
     required: false
-    description: 'Logstream log targets for the tests'
+    description: 'Logstream log targets for the tests (comma separated)'
   grafana_url:
     required: false
     description: 'Grafana URL for the tests'
@@ -92,9 +92,11 @@ runs:
         if [ -n "${{ inputs.loki_run_id }}" ]; then
           cmd_args+="--logging-run-id=\"${{ inputs.loki_run_id }}\" "
         fi
-        if [ -n "${{ inputs.logstream_log_targets }}" ]; then
-          cmd_args+="--logging-log-targets=\"${{ inputs.logstream_log_targets }}\" "
-        fi
+        # Split the log targets input by comma and add them to the command line arguments
+        IFS=',' read -ra ADDR <<< "${{ inputs.logstream_log_targets }}"
+        for i in "${ADDR[@]}"; do
+          cmd_args+="--logging-log-targets=\"$i\" "
+        done        
         echo "Command line arguments: $cmd_args" 
         echo "CMD_ARGS=$cmd_args" >> $GITHUB_ENV
 
diff --git a/integration-tests/e2e_tests_ci_tool/cmd/create_test_config_cmd.go b/integration-tests/e2e_tests_ci_tool/cmd/create_test_config_cmd.go
index 90da9b408fa..4894049644d 100644
--- a/integration-tests/e2e_tests_ci_tool/cmd/create_test_config_cmd.go
+++ b/integration-tests/e2e_tests_ci_tool/cmd/create_test_config_cmd.go
@@ -170,6 +170,6 @@ var createTestConfigCmd = &cobra.Command{
 			os.Exit(1)
 		}
 
-		fmt.Println(string(configToml))
+		fmt.Fprintln(cmd.OutOrStdout(), string(configToml))
 	},
 }
diff --git a/integration-tests/e2e_tests_ci_tool/cmd/test_config_cmd_test.go b/integration-tests/e2e_tests_ci_tool/cmd/test_config_cmd_test.go
new file mode 100644
index 00000000000..9293bc8e295
--- /dev/null
+++ b/integration-tests/e2e_tests_ci_tool/cmd/test_config_cmd_test.go
@@ -0,0 +1,44 @@
+package cmd
+
+import (
+	"bytes"
+	"testing"
+
+	"github.com/pelletier/go-toml/v2"
+	ctf_config "github.com/smartcontractkit/chainlink-testing-framework/config"
+	"github.com/spf13/cobra"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestCreateTestConfigCmd_LoggingLogTargets(t *testing.T) {
+	// Setup
+	rootCmd := &cobra.Command{}
+	rootCmd.AddCommand(createTestConfigCmd)
+
+	// Set the flag
+	args := []string{"create", "--logging-log-targets=target1", "--logging-log-targets=target2"}
+	rootCmd.SetArgs(args)
+
+	// Capture the output
+	var out bytes.Buffer
+	rootCmd.SetOutput(&out)
+	// createTestConfigCmd.SetOutput(&out)
+
+	// Run the command
+	err := rootCmd.Execute()
+	if err != nil {
+		t.Fatalf("Execute failed: %v", err)
+	}
+
+	// Check the output
+	var tc ctf_config.TestConfig
+	err = toml.Unmarshal(out.Bytes(), &tc)
+	if err != nil {
+		t.Fatalf("Failed to unmarshal output: %v", err)
+	}
+
+	// Assertions
+	assert.NotNil(t, tc.Logging)
+	assert.NotNil(t, tc.Logging.LogStream)
+	assert.Equal(t, []string{"target1", "target2"}, tc.Logging.LogStream.LogTargets)
+}

From 494267c639d1e16e75716bf187b012ec64248368 Mon Sep 17 00:00:00 2001
From: lukaszcl <120112546+lukaszcl@users.noreply.github.com>
Date: Fri, 28 Jun 2024 12:43:10 +0200
Subject: [PATCH 082/137] Add create-default-e2e-config step to k8s tests

---
 .../run-e2e-tests-reusable-workflow.yml       | 22 +++++++++++++++++++
 1 file changed, 22 insertions(+)

diff --git a/.github/workflows/run-e2e-tests-reusable-workflow.yml b/.github/workflows/run-e2e-tests-reusable-workflow.yml
index 3baf794bff0..a591ac77e97 100644
--- a/.github/workflows/run-e2e-tests-reusable-workflow.yml
+++ b/.github/workflows/run-e2e-tests-reusable-workflow.yml
@@ -355,6 +355,28 @@ jobs:
       - name: Show Remote Runner Version
         run: |
           echo "Remote Runner Version: ${{ needs.prepare-remote-runner-test-image.outputs.remote-runner-version }}"
+
+      - name: Create default test config override (use overrides from e2e-tests.yml or defaults)
+        if: ${{ inputs.test_config_base64_override == '' }}
+        uses: ./.github/actions/create-default-e2e-config-override
+        with:
+          chainlink_version: ${{ matrix.tests.testConfigOverride.chainlinkVersion || github.sha }}
+          chainlink_image: ${{ matrix.tests.testConfigOverride.chainlinkImage || env.CHAINLINK_IMAGE }}
+          chainlink_postgres_version: ${{ matrix.tests.testConfigOverride.chainlinkPostgresVersion }}
+          chainlink_upgrade_version: ${{ matrix.tests.testConfigOverride.chainlinkUpgradeVersion }}
+          selected_networks: ${{ matrix.tests.testConfigOverride.selectedNetworks || env.SELECTED_NETWORKS}}
+          pyroscope_server: ${{ secrets.QA_PYROSCOPE_INSTANCE }}
+          pyroscope_environment: ${{ matrix.tests.pyroscopeEnv }}
+          pyroscope_key: ${{ secrets.QA_PYROSCOPE_KEY }}
+          loki_run_id: ${{ github.run_id }}
+          loki_endpoint: https://${{ secrets.GRAFANA_INTERNAL_HOST }}/loki/api/v1/push
+          loki_tenant_id: ${{ secrets.GRAFANA_INTERNAL_TENANT_ID }}
+          loki_basic_auth: ${{ secrets.GRAFANA_INTERNAL_BASIC_AUTH }}
+          logstream_log_targets: ${{ vars.LOGSTREAM_LOG_TARGETS }}
+          grafana_url: http://localhost:8080/primary # This is Grafana GAP's address
+          grafana_dashboard_url: /d/ddf75041-1e39-42af-aa46-361fe4c36e9e/ci-e2e-tests-logs
+          grafana_bearer_token: ${{ secrets.GRAFANA_INTERNAL_URL_SHORTENER_TOKEN }}          
+
       - name: Run tests
         uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/run-tests@af92c5fae8dcf1659201e907db82d221fc304b94 # v2.3.21
         env:

From 9194ff268587907cceb0ac9b2de3e0de488c967f Mon Sep 17 00:00:00 2001
From: lukaszcl <120112546+lukaszcl@users.noreply.github.com>
Date: Fri, 28 Jun 2024 12:48:43 +0200
Subject: [PATCH 083/137] Fix ocr tests

---
 .github/e2e-tests.yml | 2 --
 1 file changed, 2 deletions(-)

diff --git a/.github/e2e-tests.yml b/.github/e2e-tests.yml
index 093363eaac3..c1aec8548a3 100644
--- a/.github/e2e-tests.yml
+++ b/.github/e2e-tests.yml
@@ -21,8 +21,6 @@ runner-test-matrix:
     path: integration-tests/smoke/ocr_test.go
     test-type: docker
     runs-on: ubuntu-latest
-    test-config-override:
-      chainlink-version: mycustomversion    
     workflows:
       - Run PR E2E Tests
       - Run Nightly E2E Tests

From a84b76a52416dba98b980f7834302e93a2f61bcb Mon Sep 17 00:00:00 2001
From: lukaszcl <120112546+lukaszcl@users.noreply.github.com>
Date: Fri, 28 Jun 2024 12:55:56 +0200
Subject: [PATCH 084/137] Use sanitized id for artifact name

---
 .github/workflows/run-e2e-tests-reusable-workflow.yml | 4 ++--
 integration-tests/e2e_tests_ci_tool/cmd/filter_cmd.go | 9 +++++++++
 integration-tests/e2e_tests_ci_tool/cmd/types.go      | 1 +
 3 files changed, 12 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/run-e2e-tests-reusable-workflow.yml b/.github/workflows/run-e2e-tests-reusable-workflow.yml
index a591ac77e97..4086e4fe452 100644
--- a/.github/workflows/run-e2e-tests-reusable-workflow.yml
+++ b/.github/workflows/run-e2e-tests-reusable-workflow.yml
@@ -271,11 +271,11 @@ jobs:
           cl_repo: ${{ env.CHAINLINK_IMAGE }}
           cl_image_tag: ${{ inputs.chainlink-version }}
           aws_registries: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}
-          artifacts_name: ${{ matrix.tests.id }}-test-logs
+          artifacts_name: ${{ matrix.tests.idSanitized }}-test-logs
           artifacts_location: |
             ./integration-tests/smoke/logs/
             /tmp/gotest.log
-          publish_check_name: ${{ matrix.tests.id }}
+          publish_check_name: ${{ matrix.tests.idSanitized }}
           token: ${{ secrets.GH_TOKEN }}
           go_mod_path: ./integration-tests/go.mod
           cache_key_id: core-e2e-${{ env.MOD_CACHE_VERSION }}
diff --git a/integration-tests/e2e_tests_ci_tool/cmd/filter_cmd.go b/integration-tests/e2e_tests_ci_tool/cmd/filter_cmd.go
index bd0024053f6..3559f633395 100644
--- a/integration-tests/e2e_tests_ci_tool/cmd/filter_cmd.go
+++ b/integration-tests/e2e_tests_ci_tool/cmd/filter_cmd.go
@@ -5,6 +5,7 @@ import (
 	"fmt"
 	"io/ioutil"
 	"log"
+	"regexp"
 	"strings"
 
 	"github.com/spf13/cobra"
@@ -25,6 +26,7 @@ func filterTests(tests []CITestConf, workflow, testType, ids string) []CITestCon
 		idMatch := ids == "*" || ids == "" || contains(idFilter, test.ID)
 
 		if workflowMatch && typeMatch && idMatch {
+			test.IDSanitized = sanitizeTestID(test.ID)
 			filteredTests = append(filteredTests, test)
 		}
 	}
@@ -32,6 +34,13 @@ func filterTests(tests []CITestConf, workflow, testType, ids string) []CITestCon
 	return filteredTests
 }
 
+func sanitizeTestID(id string) string {
+	// Define a regular expression that matches any character not a letter, digit, hyphen
+	re := regexp.MustCompile(`[^a-zA-Z0-9-_]+`)
+	// Replace all occurrences of disallowed characters with "_"
+	return re.ReplaceAllString(id, "_")
+}
+
 // Utility function to check if a slice contains a string.
 func contains(slice []string, element string) bool {
 	for _, s := range slice {
diff --git a/integration-tests/e2e_tests_ci_tool/cmd/types.go b/integration-tests/e2e_tests_ci_tool/cmd/types.go
index 6b2c05c3096..c41012a70ec 100644
--- a/integration-tests/e2e_tests_ci_tool/cmd/types.go
+++ b/integration-tests/e2e_tests_ci_tool/cmd/types.go
@@ -8,6 +8,7 @@ type Test struct {
 // CITestConf defines the configuration for running a test in a CI environment, specifying details like test ID, path, type, runner settings, command, and associated workflows.
 type CITestConf struct {
 	ID                    string             `yaml:"id" json:"id"`
+	IDSanitized           string             `json:"idSanitized"`
 	Path                  string             `yaml:"path" json:"path"`
 	TestType              string             `yaml:"test-type" json:"testType"`
 	RunsOn                string             `yaml:"runs-on" json:"runsOn"`

From 45b49acafa6cb66874de2703b73f5823ea9b263f Mon Sep 17 00:00:00 2001
From: lukaszcl <120112546+lukaszcl@users.noreply.github.com>
Date: Fri, 28 Jun 2024 12:59:10 +0200
Subject: [PATCH 085/137] Update nightly workflow to test it

---
 .github/e2e-tests.yml                       |  4 ++--
 .github/workflows/run-nightly-e2e-tests.yml | 18 +++++++++++++++---
 2 files changed, 17 insertions(+), 5 deletions(-)

diff --git a/.github/e2e-tests.yml b/.github/e2e-tests.yml
index c1aec8548a3..a3e3e11b030 100644
--- a/.github/e2e-tests.yml
+++ b/.github/e2e-tests.yml
@@ -111,7 +111,7 @@ runner-test-matrix:
     remote-runner-test-suite: benchmark
     remote-runner-memory: 4Gi
     runs-on: ubuntu-latest
-    workflows:
-      - Run Nightly E2E Tests
+    # workflows:
+    #   - Run Nightly E2E Tests
     test-cmd: cd integration-tests/benchmark && go test -test.run ^TestAutomationBenchmark$ -test.parallel=1 -timeout 30m -count=1
     pyroscope-env: ci-benchmark-automation-nightly
\ No newline at end of file
diff --git a/.github/workflows/run-nightly-e2e-tests.yml b/.github/workflows/run-nightly-e2e-tests.yml
index 175c6abdfcf..4d1d03b9785 100644
--- a/.github/workflows/run-nightly-e2e-tests.yml
+++ b/.github/workflows/run-nightly-e2e-tests.yml
@@ -9,11 +9,23 @@ on:
 
 jobs:
   call-run-e2e-tests-workflow:
-    # TODO: remove when the workflow is ready
-    if: false
     name: Run E2E Tests
     uses: ./.github/workflows/run-e2e-tests-reusable-workflow.yml
     with:
       chainlink-version: develop
       test-workflow: Run Nightly E2E Tests
-    secrets: inherit
+    secrets:
+      QA_AWS_REGION: ${{ secrets.QA_AWS_REGION }}
+      QA_AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }}
+      QA_AWS_ACCOUNT_NUMBER: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}
+      QA_PYROSCOPE_INSTANCE: ${{ secrets.QA_PYROSCOPE_INSTANCE }}
+      QA_PYROSCOPE_KEY: ${{ secrets.QA_PYROSCOPE_KEY }}
+      QA_KUBECONFIG: ${{ secrets.QA_KUBECONFIG }}
+      GRAFANA_INTERNAL_TENANT_ID: ${{ secrets.GRAFANA_INTERNAL_TENANT_ID }}
+      GRAFANA_INTERNAL_BASIC_AUTH: ${{ secrets.GRAFANA_INTERNAL_BASIC_AUTH }}
+      GRAFANA_INTERNAL_HOST: ${{ secrets.GRAFANA_INTERNAL_HOST }}
+      GRAFANA_INTERNAL_URL_SHORTENER_TOKEN: ${{ secrets.GRAFANA_INTERNAL_URL_SHORTENER_TOKEN }}
+      GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}  
+      AWS_REGION: ${{ secrets.QA_AWS_REGION }}
+      AWS_OIDC_IAM_ROLE_VALIDATION_PROD_ARN: ${{ secrets.AWS_OIDC_IAM_ROLE_VALIDATION_PROD_ARN }}
+      AWS_API_GW_HOST_GRAFANA: ${{ secrets.AWS_API_GW_HOST_GRAFANA }}    

From ff41f2eb626e924bd45e34a33e0bfc5578fdbdb6 Mon Sep 17 00:00:00 2001
From: lukaszcl <120112546+lukaszcl@users.noreply.github.com>
Date: Fri, 28 Jun 2024 13:06:10 +0200
Subject: [PATCH 086/137] Fail CI if test_config_base64_override is used

---
 .github/workflows/run-e2e-tests-reusable-workflow.yml | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/run-e2e-tests-reusable-workflow.yml b/.github/workflows/run-e2e-tests-reusable-workflow.yml
index 4086e4fe452..ee83a1396e4 100644
--- a/.github/workflows/run-e2e-tests-reusable-workflow.yml
+++ b/.github/workflows/run-e2e-tests-reusable-workflow.yml
@@ -73,6 +73,10 @@ jobs:
             echo "Error: Both 'test-ids' and 'test-workflow' are provided. Please specify only one."
             exit 1
           fi
+          if [[ ${{ inputs.test_config_base64_override }} != "" ]]; then
+            echo "Error: 'test_config_base64_override' is not supported until we are sure that it does not contain any secrets. Find more info in https://smartcontract-it.atlassian.net/browse/TT-1283"
+            exit 1
+          fi
 
   check-test-configurations:
     name: Check test configurations
@@ -357,7 +361,7 @@ jobs:
           echo "Remote Runner Version: ${{ needs.prepare-remote-runner-test-image.outputs.remote-runner-version }}"
 
       - name: Create default test config override (use overrides from e2e-tests.yml or defaults)
-        if: ${{ inputs.test_config_base64_override == '' }}
+        # if: ${{ inputs.test_config_base64_override == '' }}
         uses: ./.github/actions/create-default-e2e-config-override
         with:
           chainlink_version: ${{ matrix.tests.testConfigOverride.chainlinkVersion || github.sha }}

From 596e0144b0f8671e9c615722230bcf985dbfa9f8 Mon Sep 17 00:00:00 2001
From: lukaszcl <120112546+lukaszcl@users.noreply.github.com>
Date: Fri, 28 Jun 2024 13:10:15 +0200
Subject: [PATCH 087/137] Use underscore for github inputs

---
 .../action.yml                                |  2 +-
 .../actions/run-e2e-docker-test/action.yml    |  4 +--
 .github/actions/setup-e2e-tests/action.yml    | 12 ++++----
 .../run-automation-benchmark-e2e-tests.yml    |  6 ++--
 .../run-automation-nightly-e2e-tests.yml      |  4 +--
 .../run-e2e-tests-reusable-workflow.yml       | 28 +++++++++----------
 .github/workflows/run-nightly-e2e-tests.yml   |  4 +--
 .github/workflows/run-selected-e2e-tests.yml  | 16 +++++------
 8 files changed, 38 insertions(+), 38 deletions(-)

diff --git a/.github/actions/create-default-e2e-config-override/action.yml b/.github/actions/create-default-e2e-config-override/action.yml
index 267d2f2c991..a29300c5379 100644
--- a/.github/actions/create-default-e2e-config-override/action.yml
+++ b/.github/actions/create-default-e2e-config-override/action.yml
@@ -60,7 +60,7 @@ runs:
           cmd_args+="--chainlink-image=\"${{ inputs.chainlink_image }}\" "
         fi
         if [ -n "${{ inputs.chainlink_version }}" ]; then
-          cmd_args+="--chainlink-version=\"${{ inputs.chainlink_version }}\" "
+          cmd_args+="--chainlink_version=\"${{ inputs.chainlink_version }}\" "
         fi
         if [ -n "${{ inputs.chainlink_postgres_version }}" ]; then
           cmd_args+="--chainlink-postgres-version=\"${{ inputs.chainlink_postgres_version }}\" "
diff --git a/.github/actions/run-e2e-docker-test/action.yml b/.github/actions/run-e2e-docker-test/action.yml
index ba09d40a759..a30bfb73297 100644
--- a/.github/actions/run-e2e-docker-test/action.yml
+++ b/.github/actions/run-e2e-docker-test/action.yml
@@ -43,7 +43,7 @@ runs:
           cmd_args+="--chainlink-image=$chainlink_image "
         fi
         if [ -n "$chainlink_version" ]; then
-          cmd_args+="--chainlink-version=$chainlink_version "
+          cmd_args+="--chainlink_version=$chainlink_version "
         fi
         if [ -n "$chainlink_postgres_version" ]; then
           cmd_args+="--chainlink-postgres-version=$chainlink_postgres_version "
@@ -92,7 +92,7 @@ runs:
         test_command_to_run: ${{ inputs.test.testCmd }} 2>&1 | tee /tmp/gotest.log | gotestloghelper -ci -singlepackage -hidepassingtests=false -hidepassinglogs
         test_download_vendor_packages_command: cd ./integration-tests && go mod download
         cl_repo: ${{ env.CHAINLINK_IMAGE }}
-        cl_image_tag: ${{ inputs.chainlink-version }}
+        cl_image_tag: ${{ inputs.chainlink_version }}
         aws_registries: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}
         artifacts_name: ${{ inputs.test.id }}-test-logs
         artifacts_location: |
diff --git a/.github/actions/setup-e2e-tests/action.yml b/.github/actions/setup-e2e-tests/action.yml
index feab30cbf81..2703a0670f4 100644
--- a/.github/actions/setup-e2e-tests/action.yml
+++ b/.github/actions/setup-e2e-tests/action.yml
@@ -4,13 +4,13 @@ inputs:
   config-file-path:
     required: true
     description: 'Path to the configuration file for E2E tests'
-  enable-check-test-configurations:
+  enable_check_test_configurations:
     required: true
     description: 'Enable checking of test configurations'
-  test-ids:
+  test_ids:
     required: false
     description: 'Test IDs to filter on'
-  test-workflow:
+  test_workflow:
     required: false
     description: 'Workflow name for filtering tests'
 outputs:
@@ -35,7 +35,7 @@ runs:
       shell: bash
 
     - name: Check Test Configurations
-      if: ${{ fromJSON(inputs.enable-check-test-configurations) }}
+      if: ${{ fromJSON(inputs.enable_check_test_configurations) }}
       run: |
         if ! go run integration-tests/e2e_tests_ci_tool/main.go check-tests ./integration-tests .github/e2e-tests.yml; then
           echo "::error::Some E2E test configurations have to be added to .github/e2e-tests.yml." && exit 1
@@ -51,7 +51,7 @@ runs:
       id: set-docker-matrix
       run: |
         cd integration-tests/e2e_tests_ci_tool/
-        MATRIX_JSON=$(go run main.go filter --file ${{ inputs.config-file-path }} --test-type 'docker' --test-ids '${{ inputs.test-ids }}' --workflow '${{ inputs.test-workflow }}')
+        MATRIX_JSON=$(go run main.go filter --file ${{ inputs.config-file-path }} --test-type 'docker' --test_ids '${{ inputs.test_ids }}' --workflow '${{ inputs.test_workflow }}')
         echo "Docker tests:"
         echo "$MATRIX_JSON" | jq
         echo "matrix=$MATRIX_JSON" >> $GITHUB_ENV
@@ -61,7 +61,7 @@ runs:
       id: set-k8s-runner-matrix
       run: |
         cd integration-tests/e2e_tests_ci_tool/
-        MATRIX_JSON=$(go run main.go filter --file ${{ inputs.config-file-path }} --test-type 'k8s-remote-runner' --test-ids '${{ inputs.test-ids }}' --workflow '${{ inputs.test-workflow }}')
+        MATRIX_JSON=$(go run main.go filter --file ${{ inputs.config-file-path }} --test-type 'k8s-remote-runner' --test_ids '${{ inputs.test_ids }}' --workflow '${{ inputs.test_workflow }}')
         echo "K8s tests:"
         echo "$MATRIX_JSON" | jq
         echo "matrix=$MATRIX_JSON" >> $GITHUB_ENV
diff --git a/.github/workflows/run-automation-benchmark-e2e-tests.yml b/.github/workflows/run-automation-benchmark-e2e-tests.yml
index ad0c0d10d4b..d676a5c14a4 100644
--- a/.github/workflows/run-automation-benchmark-e2e-tests.yml
+++ b/.github/workflows/run-automation-benchmark-e2e-tests.yml
@@ -33,10 +33,10 @@ jobs:
       id-token: write
       contents: read    
     with:
-      chainlink-version: develop
-      test-ids: ${{ github.event.inputs.test-id }}
+      chainlink_version: develop
+      test_ids: ${{ github.event.inputs.test-id }}
       # Select tests to run from .github/e2e_tests.yml based on workflow name 
-      test-workflow: Run Automation Product Nightly E2E Tests
+      test_workflow: Run Automation Product Nightly E2E Tests
       test_config_base64_override: ${{ github.event.inputs.test_config_base64_override }}
     secrets:
       QA_AWS_REGION: ${{ secrets.QA_AWS_REGION }}
diff --git a/.github/workflows/run-automation-nightly-e2e-tests.yml b/.github/workflows/run-automation-nightly-e2e-tests.yml
index 3fa60114e6a..859012016ff 100644
--- a/.github/workflows/run-automation-nightly-e2e-tests.yml
+++ b/.github/workflows/run-automation-nightly-e2e-tests.yml
@@ -20,9 +20,9 @@ jobs:
       id-token: write
       contents: read    
     with:
-      chainlink-version: develop
+      chainlink_version: develop
       # Select tests to run from .github/e2e_tests.yml based on workflow name 
-      test-workflow: Run Automation Product Nightly E2E Tests
+      test_workflow: Run Automation Product Nightly E2E Tests
     secrets:
       QA_AWS_REGION: ${{ secrets.QA_AWS_REGION }}
       QA_AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }}
diff --git a/.github/workflows/run-e2e-tests-reusable-workflow.yml b/.github/workflows/run-e2e-tests-reusable-workflow.yml
index ee83a1396e4..d3afb661d69 100644
--- a/.github/workflows/run-e2e-tests-reusable-workflow.yml
+++ b/.github/workflows/run-e2e-tests-reusable-workflow.yml
@@ -4,15 +4,15 @@ name: Run E2E Tests
 on:
   workflow_call:
     inputs:
-      chainlink-version:
+      chainlink_version:
         description: 'Enter Chainlink version to use for the tests. Example: "v2.10.0" or sha'
         required: true
         type: string
-      test-ids:
+      test_ids:
         description: 'Run tests by test ids separated by commas. Example: "run_all_in_ocr_tests_go,run_TestOCRv2Request_in_ocr2_test_go". Check all test IDs in .github/e2e-tests.yml'
         required: false
         type: string
-      test-workflow:
+      test_workflow:
         description: 'Run tests by workflow name. Example: "Run Nightly E2E Tests"'
         required: false
         type: string        
@@ -20,12 +20,12 @@ on:
         description: 'This is an unsafe way to pass custom base64 test config. It cannot contain any secrets as this value is exposed in Github logs (Set up job -> Inputs) when the workflow is called! The caller workflow must make sure to remove secrets from this input.'
         required: false
         type: string        
-      enable-check-test-configurations:
+      enable_check_test_configurations:
         description: 'Set to "true" to enable check-test-configurations job'
         required: false
         type: boolean
         default: false
-      with-existing-remote-runner-version:
+      with_existing_remote_runner_version:
         description: 'Use the existing remote runner version for k8s tests. Example: "d3bf5044af33e08be788a2df31c4a745cf69d787"'
         required: false
         type: string
@@ -69,8 +69,8 @@ jobs:
     steps:
       - name: Check input conditions
         run: |
-          if [[ "${{ inputs.test-ids }}" != "" && "${{ inputs.test-workflow }}" != "" ]]; then
-            echo "Error: Both 'test-ids' and 'test-workflow' are provided. Please specify only one."
+          if [[ "${{ inputs.test_ids }}" != "" && "${{ inputs.test_workflow }}" != "" ]]; then
+            echo "Error: Both 'test_ids' and 'test_workflow' are provided. Please specify only one."
             exit 1
           fi
           if [[ ${{ inputs.test_config_base64_override }} != "" ]]; then
@@ -80,7 +80,7 @@ jobs:
 
   check-test-configurations:
     name: Check test configurations
-    if: ${{ inputs.enable-check-test-configurations }}
+    if: ${{ inputs.enable_check_test_configurations }}
     needs: validate-inputs
     runs-on: ubuntu-latest
     steps:
@@ -118,7 +118,7 @@ jobs:
         id: set-docker-matrix
         run: |
           cd integration-tests/e2e_tests_ci_tool/
-          MATRIX_JSON=$(go run main.go filter --file ../../.github/e2e-tests.yml --test-type 'docker' --test-ids '${{ inputs.test-ids }}' --workflow '${{ inputs.test-workflow }}')
+          MATRIX_JSON=$(go run main.go filter --file ../../.github/e2e-tests.yml --test-type 'docker' --test_ids '${{ inputs.test_ids }}' --workflow '${{ inputs.test_workflow }}')
           echo "Docker tests:"
           echo "$MATRIX_JSON" | jq
           echo "::set-output name=matrix::$MATRIX_JSON"
@@ -126,7 +126,7 @@ jobs:
         id: set-k8s-runner-matrix
         run: |
           cd integration-tests/e2e_tests_ci_tool/
-          MATRIX_JSON=$(go run main.go filter --file ../../.github/e2e-tests.yml --test-type 'k8s-remote-runner' --test-ids '${{ inputs.test-ids }}' --workflow '${{ inputs.test-workflow }}')
+          MATRIX_JSON=$(go run main.go filter --file ../../.github/e2e-tests.yml --test-type 'k8s-remote-runner' --test_ids '${{ inputs.test_ids }}' --workflow '${{ inputs.test_workflow }}')
           echo "K8s tests:"
           echo "$MATRIX_JSON" | jq
           echo "::set-output name=matrix::$MATRIX_JSON"
@@ -273,7 +273,7 @@ jobs:
           test_command_to_run: ${{ matrix.tests.testCmd }} 2>&1 | tee /tmp/gotest.log | gotestloghelper -ci -singlepackage -hidepassingtests=false -hidepassinglogs
           test_download_vendor_packages_command: cd ./integration-tests && go mod download
           cl_repo: ${{ env.CHAINLINK_IMAGE }}
-          cl_image_tag: ${{ inputs.chainlink-version }}
+          cl_image_tag: ${{ inputs.chainlink_version }}
           aws_registries: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}
           artifacts_name: ${{ matrix.tests.idSanitized }}-test-logs
           artifacts_location: |
@@ -314,7 +314,7 @@ jobs:
         uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2      
       - name: Build Test Runner Image
         uses: ./.github/actions/build-test-image
-        if: ${{ inputs.with-existing-remote-runner-version == '' }}
+        if: ${{ inputs.with_existing_remote_runner_version == '' }}
         with:
           QA_AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }}
           QA_AWS_REGION: ${{ secrets.QA_AWS_REGION }}
@@ -322,10 +322,10 @@ jobs:
       - name: Set Remote Runner Version
         id: set-remote-runner-version
         run: |
-          if [[ -z "${{ inputs.with-existing-remote-runner-version }}" ]]; then
+          if [[ -z "${{ inputs.with_existing_remote_runner_version }}" ]]; then
             echo "::set-output name=remote-runner-version::${{ github.sha }}"
           else
-            echo "::set-output name=remote-runner-version::${{ inputs.with-existing-remote-runner-version }}"
+            echo "::set-output name=remote-runner-version::${{ inputs.with_existing_remote_runner_version }}"
           fi
 
   run-k8s-runner-tests:
diff --git a/.github/workflows/run-nightly-e2e-tests.yml b/.github/workflows/run-nightly-e2e-tests.yml
index 4d1d03b9785..cf5205208ee 100644
--- a/.github/workflows/run-nightly-e2e-tests.yml
+++ b/.github/workflows/run-nightly-e2e-tests.yml
@@ -12,8 +12,8 @@ jobs:
     name: Run E2E Tests
     uses: ./.github/workflows/run-e2e-tests-reusable-workflow.yml
     with:
-      chainlink-version: develop
-      test-workflow: Run Nightly E2E Tests
+      chainlink_version: develop
+      test_workflow: Run Nightly E2E Tests
     secrets:
       QA_AWS_REGION: ${{ secrets.QA_AWS_REGION }}
       QA_AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }}
diff --git a/.github/workflows/run-selected-e2e-tests.yml b/.github/workflows/run-selected-e2e-tests.yml
index fcea08906c3..e0da9a2ac7e 100644
--- a/.github/workflows/run-selected-e2e-tests.yml
+++ b/.github/workflows/run-selected-e2e-tests.yml
@@ -3,12 +3,12 @@ name: Run Selected E2E Tests
 on:
   workflow_dispatch:
     inputs:
-      chainlink-version:
+      chainlink_version:
         description: 'Enter Chainlink version to use for the tests. Example: "v2.10.0" or sha'
         default: develop
         required: false
         type: string
-      test-ids:
+      test_ids:
         description: 'Run all tests "*" by default. Or, enter test IDs to run separated by commas. Example: "run_all_in_ocr_tests_go,run_TestOCRv2Request_in_ocr2_test_go". Check all test IDs in .github/e2e-tests.yml'
         default: "*"
         required: true
@@ -17,12 +17,12 @@ on:
         description: 'Custom base64 test config'
         required: false
         type: string          
-      enable-check-test-configurations:
+      enable_check_test_configurations:
         description: 'Set to "true" to enable check-test-configurations job'
         required: false
         type: boolean
         default: false
-      with-existing-remote-runner-version:
+      with_existing_remote_runner_version:
         description: 'Use the existing remote runner version for k8s tests. Example: "d3bf5044af33e08be788a2df31c4a745cf69d787"'
         required: false
         type: string                 
@@ -32,12 +32,12 @@ jobs:
     name: Run E2E Tests
     uses: ./.github/workflows/run-e2e-tests-reusable-workflow.yml
     with:
-      chainlink-version: ${{ github.event.inputs.chainlink-version }}
-      test-ids: ${{ github.event.inputs.test-ids }}
+      chainlink_version: ${{ github.event.inputs.chainlink_version }}
+      test_ids: ${{ github.event.inputs.test_ids }}
       test_config_base64_override: ${{ github.event.inputs.test_config_base64_override }}
-      with-existing-remote-runner-version: ${{ github.event.inputs.with-existing-remote-runner-version }}
+      with_existing_remote_runner_version: ${{ github.event.inputs.with_existing_remote_runner_version }}
       # Use fromJSON to convert string to boolean. More info: https://github.com/actions/runner/issues/2206#issuecomment-1532246677
-      enable-check-test-configurations: ${{ fromJSON(github.event.inputs.enable-check-test-configurations) }}
+      enable_check_test_configurations: ${{ fromJSON(github.event.inputs.enable_check_test_configurations) }}
     secrets:
       QA_AWS_REGION: ${{ secrets.QA_AWS_REGION }}
       QA_AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }}

From 3d1abd0235c6825d4a3da4a9cd3ea0cf2330c289 Mon Sep 17 00:00:00 2001
From: lukaszcl <120112546+lukaszcl@users.noreply.github.com>
Date: Fri, 28 Jun 2024 13:11:38 +0200
Subject: [PATCH 088/137] Fix

---
 .github/workflows/run-e2e-tests-reusable-workflow.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/run-e2e-tests-reusable-workflow.yml b/.github/workflows/run-e2e-tests-reusable-workflow.yml
index d3afb661d69..489e85c3f5e 100644
--- a/.github/workflows/run-e2e-tests-reusable-workflow.yml
+++ b/.github/workflows/run-e2e-tests-reusable-workflow.yml
@@ -70,11 +70,11 @@ jobs:
       - name: Check input conditions
         run: |
           if [[ "${{ inputs.test_ids }}" != "" && "${{ inputs.test_workflow }}" != "" ]]; then
-            echo "Error: Both 'test_ids' and 'test_workflow' are provided. Please specify only one."
+            echo "::error::Error: Both 'test_ids' and 'test_workflow' are provided. Please specify only one."
             exit 1
           fi
           if [[ ${{ inputs.test_config_base64_override }} != "" ]]; then
-            echo "Error: 'test_config_base64_override' is not supported until we are sure that it does not contain any secrets. Find more info in https://smartcontract-it.atlassian.net/browse/TT-1283"
+            echo "::error::Error: 'test_config_base64_override' is not supported until we are sure that it does not contain any secrets. Find more info in https://smartcontract-it.atlassian.net/browse/TT-1283"
             exit 1
           fi
 

From 2cae0dab6183d183c7932c8147df9cd2f09e6ffe Mon Sep 17 00:00:00 2001
From: lukaszcl <120112546+lukaszcl@users.noreply.github.com>
Date: Fri, 28 Jun 2024 13:13:14 +0200
Subject: [PATCH 089/137] Fix inputs.chainlink_version

---
 .github/workflows/run-e2e-tests-reusable-workflow.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/run-e2e-tests-reusable-workflow.yml b/.github/workflows/run-e2e-tests-reusable-workflow.yml
index 489e85c3f5e..a2da63cbebb 100644
--- a/.github/workflows/run-e2e-tests-reusable-workflow.yml
+++ b/.github/workflows/run-e2e-tests-reusable-workflow.yml
@@ -197,7 +197,7 @@ jobs:
         if: ${{ inputs.test_config_base64_override == '' }}
         uses: ./.github/actions/create-default-e2e-config-override
         with:
-          chainlink_version: ${{ matrix.tests.testConfigOverride.chainlinkVersion || github.sha }}
+          chainlink_version: ${{ inputs.chainlink_version || matrix.tests.testConfigOverride.chainlinkVersion || github.sha }}
           chainlink_image: ${{ matrix.tests.testConfigOverride.chainlinkImage || env.CHAINLINK_IMAGE }}
           chainlink_postgres_version: ${{ matrix.tests.testConfigOverride.chainlinkPostgresVersion }}
           chainlink_upgrade_version: ${{ matrix.tests.testConfigOverride.chainlinkUpgradeVersion }}

From 700f2f92cbc8c1174f390a9130e6b721fdece168 Mon Sep 17 00:00:00 2001
From: lukaszcl <120112546+lukaszcl@users.noreply.github.com>
Date: Fri, 28 Jun 2024 13:16:25 +0200
Subject: [PATCH 090/137] Remove test_config_base64_override from inputs until
 we are sure it does not have secrets

This will be uncommented when https://smartcontract-it.atlassian.net/browse/TT-1283 is done
---
 .../run-automation-benchmark-e2e-tests.yml         | 10 +++++-----
 .../workflows/run-e2e-tests-reusable-workflow.yml  | 14 +++++---------
 .github/workflows/run-selected-e2e-tests.yml       | 10 +++++-----
 3 files changed, 15 insertions(+), 19 deletions(-)

diff --git a/.github/workflows/run-automation-benchmark-e2e-tests.yml b/.github/workflows/run-automation-benchmark-e2e-tests.yml
index d676a5c14a4..7bcc6839d0c 100644
--- a/.github/workflows/run-automation-benchmark-e2e-tests.yml
+++ b/.github/workflows/run-automation-benchmark-e2e-tests.yml
@@ -11,10 +11,10 @@ on:
         options:
           - TestAutomationBenchmark
           - TestAutomationBenchmark_1000Upkeeps_2_1
-      test_config_base64_override:
-        description: Custom base64 test config
-        required: false
-        type: string
+      # test_config_base64_override:
+      #   description: Custom base64 test config
+      #   required: false
+      #   type: string
 
 # TODO:
 # 1. list of tests to select 
@@ -37,7 +37,7 @@ jobs:
       test_ids: ${{ github.event.inputs.test-id }}
       # Select tests to run from .github/e2e_tests.yml based on workflow name 
       test_workflow: Run Automation Product Nightly E2E Tests
-      test_config_base64_override: ${{ github.event.inputs.test_config_base64_override }}
+      # test_config_base64_override: ${{ github.event.inputs.test_config_base64_override }}
     secrets:
       QA_AWS_REGION: ${{ secrets.QA_AWS_REGION }}
       QA_AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }}
diff --git a/.github/workflows/run-e2e-tests-reusable-workflow.yml b/.github/workflows/run-e2e-tests-reusable-workflow.yml
index a2da63cbebb..1239e123328 100644
--- a/.github/workflows/run-e2e-tests-reusable-workflow.yml
+++ b/.github/workflows/run-e2e-tests-reusable-workflow.yml
@@ -16,10 +16,10 @@ on:
         description: 'Run tests by workflow name. Example: "Run Nightly E2E Tests"'
         required: false
         type: string        
-      test_config_base64_override:
-        description: 'This is an unsafe way to pass custom base64 test config. It cannot contain any secrets as this value is exposed in Github logs (Set up job -> Inputs) when the workflow is called! The caller workflow must make sure to remove secrets from this input.'
-        required: false
-        type: string        
+      # test_config_base64_override:
+      #   description: 'This is an unsafe way to pass custom base64 test config. It cannot contain any secrets as this value is exposed in Github logs (Set up job -> Inputs) when the workflow is called! The caller workflow must make sure to remove secrets from this input.'
+      #   required: false
+      #   type: string        
       enable_check_test_configurations:
         description: 'Set to "true" to enable check-test-configurations job'
         required: false
@@ -73,10 +73,6 @@ jobs:
             echo "::error::Error: Both 'test_ids' and 'test_workflow' are provided. Please specify only one."
             exit 1
           fi
-          if [[ ${{ inputs.test_config_base64_override }} != "" ]]; then
-            echo "::error::Error: 'test_config_base64_override' is not supported until we are sure that it does not contain any secrets. Find more info in https://smartcontract-it.atlassian.net/browse/TT-1283"
-            exit 1
-          fi
 
   check-test-configurations:
     name: Check test configurations
@@ -194,7 +190,7 @@ jobs:
           go-version: '1.21.7'
 
       - name: Create default test config override (use overrides from e2e-tests.yml or defaults)
-        if: ${{ inputs.test_config_base64_override == '' }}
+        # if: ${{ inputs.test_config_base64_override == '' }}
         uses: ./.github/actions/create-default-e2e-config-override
         with:
           chainlink_version: ${{ inputs.chainlink_version || matrix.tests.testConfigOverride.chainlinkVersion || github.sha }}
diff --git a/.github/workflows/run-selected-e2e-tests.yml b/.github/workflows/run-selected-e2e-tests.yml
index e0da9a2ac7e..cb04fafddd8 100644
--- a/.github/workflows/run-selected-e2e-tests.yml
+++ b/.github/workflows/run-selected-e2e-tests.yml
@@ -13,10 +13,10 @@ on:
         default: "*"
         required: true
         type: string
-      test_config_base64_override:
-        description: 'Custom base64 test config'
-        required: false
-        type: string          
+      # test_config_base64_override:
+      #   description: 'Custom base64 test config'
+      #   required: false
+      #   type: string          
       enable_check_test_configurations:
         description: 'Set to "true" to enable check-test-configurations job'
         required: false
@@ -34,7 +34,7 @@ jobs:
     with:
       chainlink_version: ${{ github.event.inputs.chainlink_version }}
       test_ids: ${{ github.event.inputs.test_ids }}
-      test_config_base64_override: ${{ github.event.inputs.test_config_base64_override }}
+      # test_config_base64_override: ${{ github.event.inputs.test_config_base64_override }}
       with_existing_remote_runner_version: ${{ github.event.inputs.with_existing_remote_runner_version }}
       # Use fromJSON to convert string to boolean. More info: https://github.com/actions/runner/issues/2206#issuecomment-1532246677
       enable_check_test_configurations: ${{ fromJSON(github.event.inputs.enable_check_test_configurations) }}

From 9b57657c09d0e41c90ce551d6dd6e87b4760dc39 Mon Sep 17 00:00:00 2001
From: lukaszcl <120112546+lukaszcl@users.noreply.github.com>
Date: Fri, 28 Jun 2024 13:21:49 +0200
Subject: [PATCH 091/137] Fix

---
 .github/workflows/run-e2e-tests-reusable-workflow.yml | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/run-e2e-tests-reusable-workflow.yml b/.github/workflows/run-e2e-tests-reusable-workflow.yml
index 1239e123328..40f6a6bff6d 100644
--- a/.github/workflows/run-e2e-tests-reusable-workflow.yml
+++ b/.github/workflows/run-e2e-tests-reusable-workflow.yml
@@ -114,7 +114,7 @@ jobs:
         id: set-docker-matrix
         run: |
           cd integration-tests/e2e_tests_ci_tool/
-          MATRIX_JSON=$(go run main.go filter --file ../../.github/e2e-tests.yml --test-type 'docker' --test_ids '${{ inputs.test_ids }}' --workflow '${{ inputs.test_workflow }}')
+          MATRIX_JSON=$(go run main.go filter --file ../../.github/e2e-tests.yml --test-type 'docker' --test-ids '${{ inputs.test_ids }}' --workflow '${{ inputs.test_workflow }}')
           echo "Docker tests:"
           echo "$MATRIX_JSON" | jq
           echo "::set-output name=matrix::$MATRIX_JSON"
@@ -122,7 +122,7 @@ jobs:
         id: set-k8s-runner-matrix
         run: |
           cd integration-tests/e2e_tests_ci_tool/
-          MATRIX_JSON=$(go run main.go filter --file ../../.github/e2e-tests.yml --test-type 'k8s-remote-runner' --test_ids '${{ inputs.test_ids }}' --workflow '${{ inputs.test_workflow }}')
+          MATRIX_JSON=$(go run main.go filter --file ../../.github/e2e-tests.yml --test-type 'k8s-remote-runner' --test-ids '${{ inputs.test_ids }}' --workflow '${{ inputs.test_workflow }}')
           echo "K8s tests:"
           echo "$MATRIX_JSON" | jq
           echo "::set-output name=matrix::$MATRIX_JSON"
@@ -360,7 +360,7 @@ jobs:
         # if: ${{ inputs.test_config_base64_override == '' }}
         uses: ./.github/actions/create-default-e2e-config-override
         with:
-          chainlink_version: ${{ matrix.tests.testConfigOverride.chainlinkVersion || github.sha }}
+          chainlink_version: ${{ inputs.chainlink_version || matrix.tests.testConfigOverride.chainlinkVersion || github.sha }}
           chainlink_image: ${{ matrix.tests.testConfigOverride.chainlinkImage || env.CHAINLINK_IMAGE }}
           chainlink_postgres_version: ${{ matrix.tests.testConfigOverride.chainlinkPostgresVersion }}
           chainlink_upgrade_version: ${{ matrix.tests.testConfigOverride.chainlinkUpgradeVersion }}

From 0c88794f7d2f6d6816f20025d8429d2bdff55c8e Mon Sep 17 00:00:00 2001
From: lukaszcl <120112546+lukaszcl@users.noreply.github.com>
Date: Fri, 28 Jun 2024 13:23:20 +0200
Subject: [PATCH 092/137] Remove unused action

---
 .github/actions/setup-e2e-tests/action.yml | 98 ----------------------
 1 file changed, 98 deletions(-)
 delete mode 100644 .github/actions/setup-e2e-tests/action.yml

diff --git a/.github/actions/setup-e2e-tests/action.yml b/.github/actions/setup-e2e-tests/action.yml
deleted file mode 100644
index 2703a0670f4..00000000000
--- a/.github/actions/setup-e2e-tests/action.yml
+++ /dev/null
@@ -1,98 +0,0 @@
-name: 'Setup E2E Tests'
-description: 'Setup and generate test matrices for Docker and Kubernetes tests.'
-inputs:
-  config-file-path:
-    required: true
-    description: 'Path to the configuration file for E2E tests'
-  enable_check_test_configurations:
-    required: true
-    description: 'Enable checking of test configurations'
-  test_ids:
-    required: false
-    description: 'Test IDs to filter on'
-  test_workflow:
-    required: false
-    description: 'Workflow name for filtering tests'
-outputs:
-  run-docker-tests:
-    description: 'Indicator to run Docker tests'
-    value: ${{ steps.check-matrices.outputs.run-docker-tests }}
-  run-k8s-tests:
-    description: 'Indicator to run Kubernetes tests'
-    value: ${{ steps.check-matrices.outputs.run-k8s-tests }}
-  docker-matrix:
-    description: 'JSON matrix for Docker tests'
-    value: ${{ steps.set-docker-matrix.outputs.matrix }}
-  k8s-runner-matrix:
-    description: 'JSON matrix for Kubernetes tests'
-    value: ${{ steps.set-k8s-runner-matrix.outputs.matrix }}
-
-runs:
-  using: 'composite'
-  steps:
-    - name: Install jq
-      run: sudo apt-get install jq
-      shell: bash
-
-    - name: Check Test Configurations
-      if: ${{ fromJSON(inputs.enable_check_test_configurations) }}
-      run: |
-        if ! go run integration-tests/e2e_tests_ci_tool/main.go check-tests ./integration-tests .github/e2e-tests.yml; then
-          echo "::error::Some E2E test configurations have to be added to .github/e2e-tests.yml." && exit 1
-        fi
-      shell: bash
-
-    - name: Set up Go
-      uses: actions/setup-go@v2
-      with:
-        go-version: '1.21.7'
-
-    - name: Generate Docker Tests Matrix
-      id: set-docker-matrix
-      run: |
-        cd integration-tests/e2e_tests_ci_tool/
-        MATRIX_JSON=$(go run main.go filter --file ${{ inputs.config-file-path }} --test-type 'docker' --test_ids '${{ inputs.test_ids }}' --workflow '${{ inputs.test_workflow }}')
-        echo "Docker tests:"
-        echo "$MATRIX_JSON" | jq
-        echo "matrix=$MATRIX_JSON" >> $GITHUB_ENV
-      shell: bash
-
-    - name: Generate K8s Tests Matrix
-      id: set-k8s-runner-matrix
-      run: |
-        cd integration-tests/e2e_tests_ci_tool/
-        MATRIX_JSON=$(go run main.go filter --file ${{ inputs.config-file-path }} --test-type 'k8s-remote-runner' --test_ids '${{ inputs.test_ids }}' --workflow '${{ inputs.test_workflow }}')
-        echo "K8s tests:"
-        echo "$MATRIX_JSON" | jq
-        echo "matrix=$MATRIX_JSON" >> $GITHUB_ENV
-      shell: bash
-
-    - name: Check Test Matrices
-      id: check-matrices
-      run: |
-        DOCKER_MATRIX_EMPTY=$(echo '${{ steps.set-docker-matrix.outputs.matrix }}' | jq '.tests == null or .tests == []')
-        K8S_MATRIX_EMPTY=$(echo '${{ steps.set-k8s-runner-matrix.outputs.matrix }}' | jq '.tests == null or .tests == []')
-
-        # Check if jq commands succeeded
-        if [ $? -ne 0 ]; then
-          echo "JSON parse error occurred."
-          exit 1
-        fi
-
-        if [[ "$DOCKER_MATRIX_EMPTY" == "true" ]]; then
-          echo "run-docker-tests=false" >> $GITHUB_ENV
-        else
-          echo "run-docker-tests=true" >> $GITHUB_ENV
-        fi
-        if [[ "$K8S_MATRIX_EMPTY" == "true" ]]; then
-          echo "run-k8s-tests=false" >> $GITHUB_ENV
-        else
-          echo "run-k8s-tests=true" >> $GITHUB_ENV
-        fi
-
-        # Check if both matrices are empty
-        if [[ "$DOCKER_MATRIX_EMPTY" == "true" ]] && [[ "$K8S_MATRIX_EMPTY" == "true" ]]; then
-          echo "No tests found for inputs: '${{ toJson(inputs) }}'. Both Docker and Kubernetes tests matrices are empty"
-          exit 1
-        fi
-      shell: bash   
\ No newline at end of file

From 9e04190b54465147f5cf915d4ebd5555a0dc668b Mon Sep 17 00:00:00 2001
From: lukaszcl <120112546+lukaszcl@users.noreply.github.com>
Date: Fri, 28 Jun 2024 13:58:16 +0200
Subject: [PATCH 093/137] Fix

---
 .github/actions/create-default-e2e-config-override/action.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/actions/create-default-e2e-config-override/action.yml b/.github/actions/create-default-e2e-config-override/action.yml
index a29300c5379..267d2f2c991 100644
--- a/.github/actions/create-default-e2e-config-override/action.yml
+++ b/.github/actions/create-default-e2e-config-override/action.yml
@@ -60,7 +60,7 @@ runs:
           cmd_args+="--chainlink-image=\"${{ inputs.chainlink_image }}\" "
         fi
         if [ -n "${{ inputs.chainlink_version }}" ]; then
-          cmd_args+="--chainlink_version=\"${{ inputs.chainlink_version }}\" "
+          cmd_args+="--chainlink-version=\"${{ inputs.chainlink_version }}\" "
         fi
         if [ -n "${{ inputs.chainlink_postgres_version }}" ]; then
           cmd_args+="--chainlink-postgres-version=\"${{ inputs.chainlink_postgres_version }}\" "

From 0c2979cec58e94a46a70c3b819862d8b50c1fd5a Mon Sep 17 00:00:00 2001
From: lukaszcl <120112546+lukaszcl@users.noreply.github.com>
Date: Fri, 28 Jun 2024 14:27:04 +0200
Subject: [PATCH 094/137] Update create config override cmd and action

---
 .../action.yml                                | 18 ++++
 .../cmd/test_config_cmd_test.go               | 83 +++++++++++++------
 2 files changed, 74 insertions(+), 27 deletions(-)

diff --git a/.github/actions/create-default-e2e-config-override/action.yml b/.github/actions/create-default-e2e-config-override/action.yml
index 267d2f2c991..b210374540a 100644
--- a/.github/actions/create-default-e2e-config-override/action.yml
+++ b/.github/actions/create-default-e2e-config-override/action.yml
@@ -24,6 +24,9 @@ inputs:
   pyroscope_key:
     required: false
     description: 'Pyroscope key for the tests'
+  test_log_collect:
+    required: false
+    description: 'Whether to always collect test logs'
   loki_run_id:
     required: false
     description: 'Loki run ID for the tests'
@@ -48,6 +51,12 @@ inputs:
   grafana_bearer_token:
     required: false
     description: 'Grafana bearer token for the tests'
+  private_ethereum_network_execution_layer:
+    required: false
+    description: 'Private Ethereum network execution layer for the tests'    
+  private_ethereum_network_custom_docker_image:
+    required: false
+    description: 'Private Ethereum network custom Docker image for the tests'
 
 runs:
   using: 'composite'
@@ -92,6 +101,15 @@ runs:
         if [ -n "${{ inputs.loki_run_id }}" ]; then
           cmd_args+="--logging-run-id=\"${{ inputs.loki_run_id }}\" "
         fi
+        if [ -n "${{ inputs.test_log_collect }}" ]; then
+          cmd_args+="--logging-test-log-collect=\"${{ inputs.test_log_collect }}\" "
+        fi
+        if [ -n "${{ inputs.private_ethereum_network_execution_layer }}" ]; then
+          cmd_args+="--private-ethereum-network-execution-layer=\"${{ inputs.private_ethereum_network_execution_layer }}\" "
+        fi
+        if [ -n "${{ inputs.private_ethereum_network_custom_docker_image }}" ]; then
+          cmd_args+="--private-ethereum-network-custom-docker-image=\"${{ inputs.private_ethereum_network_custom_docker_image }}\" "
+        fi
         # Split the log targets input by comma and add them to the command line arguments
         IFS=',' read -ra ADDR <<< "${{ inputs.logstream_log_targets }}"
         for i in "${ADDR[@]}"; do
diff --git a/integration-tests/e2e_tests_ci_tool/cmd/test_config_cmd_test.go b/integration-tests/e2e_tests_ci_tool/cmd/test_config_cmd_test.go
index 9293bc8e295..34cf3c0ac5b 100644
--- a/integration-tests/e2e_tests_ci_tool/cmd/test_config_cmd_test.go
+++ b/integration-tests/e2e_tests_ci_tool/cmd/test_config_cmd_test.go
@@ -10,35 +10,64 @@ import (
 	"github.com/stretchr/testify/assert"
 )
 
-func TestCreateTestConfigCmd_LoggingLogTargets(t *testing.T) {
-	// Setup
+func TestCreateTestConfigCmd(t *testing.T) {
+	tests := []struct {
+		name    string
+		args    []string
+		want    interface{}
+		check   func(t *testing.T, tc *ctf_config.TestConfig)
+		wantErr bool
+	}{
+		{
+			name: "LoggingLogTargets",
+			args: []string{"create", "--logging-log-targets=target1", "--logging-log-targets=target2"},
+			check: func(t *testing.T, tc *ctf_config.TestConfig) {
+				assert.NotNil(t, tc.Logging)
+				assert.NotNil(t, tc.Logging.LogStream)
+				assert.Equal(t, []string{"target1", "target2"}, tc.Logging.LogStream.LogTargets)
+			},
+		},
+		{
+			name: "PrivateEthereumNetworkExecutionLayerFlag",
+			args: []string{"create", "--private-ethereum-network-execution-layer=geth", "--private-ethereum-network-ethereum-version=1.10.0"},
+			check: func(t *testing.T, tc *ctf_config.TestConfig) {
+				assert.NotNil(t, tc.PrivateEthereumNetwork)
+				assert.NotNil(t, tc.PrivateEthereumNetwork.ExecutionLayer)
+				assert.Equal(t, ctf_config.ExecutionLayer("geth"), *tc.PrivateEthereumNetwork.ExecutionLayer)
+				assert.Equal(t, ctf_config.EthereumVersion("1.10.0"), *tc.PrivateEthereumNetwork.EthereumVersion)
+			},
+		},
+		{
+			name: "PrivateEthereumNetworkCustomDockerImageFlag",
+			args: []string{"create", "--private-ethereum-network-execution-layer=geth", "--private-ethereum-network-ethereum-version=1.10.0", "--private-ethereum-network-custom-docker-image=custom-image:v1.0"},
+			check: func(t *testing.T, tc *ctf_config.TestConfig) {
+				assert.NotNil(t, tc.PrivateEthereumNetwork)
+				assert.NotNil(t, tc.PrivateEthereumNetwork.ExecutionLayer)
+				assert.Equal(t, map[ctf_config.ContainerType]string{"execution_layer": "custom-image:v1.0"}, tc.PrivateEthereumNetwork.CustomDockerImages)
+			},
+		},
+	}
+
 	rootCmd := &cobra.Command{}
 	rootCmd.AddCommand(createTestConfigCmd)
 
-	// Set the flag
-	args := []string{"create", "--logging-log-targets=target1", "--logging-log-targets=target2"}
-	rootCmd.SetArgs(args)
-
-	// Capture the output
-	var out bytes.Buffer
-	rootCmd.SetOutput(&out)
-	// createTestConfigCmd.SetOutput(&out)
-
-	// Run the command
-	err := rootCmd.Execute()
-	if err != nil {
-		t.Fatalf("Execute failed: %v", err)
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			rootCmd.SetArgs(tt.args)
+			var out bytes.Buffer
+			rootCmd.SetOutput(&out)
+			err := rootCmd.Execute()
+			if (err != nil) != tt.wantErr {
+				t.Fatalf("Execute() error = %v, wantErr %v", err, tt.wantErr)
+			}
+			var tc ctf_config.TestConfig
+			err = toml.Unmarshal(out.Bytes(), &tc)
+			if err != nil {
+				t.Fatalf("Failed to unmarshal output: %v", err)
+			}
+			if tt.check != nil {
+				tt.check(t, &tc)
+			}
+		})
 	}
-
-	// Check the output
-	var tc ctf_config.TestConfig
-	err = toml.Unmarshal(out.Bytes(), &tc)
-	if err != nil {
-		t.Fatalf("Failed to unmarshal output: %v", err)
-	}
-
-	// Assertions
-	assert.NotNil(t, tc.Logging)
-	assert.NotNil(t, tc.Logging.LogStream)
-	assert.Equal(t, []string{"target1", "target2"}, tc.Logging.LogStream.LogTargets)
 }

From 50818094b13ec8b71e50c8e356cda6a5e054a2cf Mon Sep 17 00:00:00 2001
From: lukaszcl <120112546+lukaszcl@users.noreply.github.com>
Date: Fri, 28 Jun 2024 14:33:59 +0200
Subject: [PATCH 095/137] Refactor client-compatibility-tests.yml to use
 actions/create-default-e2e-config-override

---
 .../workflows/client-compatibility-tests.yml  | 36 +++++++++----------
 1 file changed, 18 insertions(+), 18 deletions(-)

diff --git a/.github/workflows/client-compatibility-tests.yml b/.github/workflows/client-compatibility-tests.yml
index 75553412725..a272c3a5e4a 100644
--- a/.github/workflows/client-compatibility-tests.yml
+++ b/.github/workflows/client-compatibility-tests.yml
@@ -535,25 +535,25 @@ jobs:
           # other inputs
           duplicate-authorization-header: "true"
       - name: Prepare Base64 TOML override
-        uses: ./.github/actions/setup-create-base64-config
+        uses: ./.github/actions/create-default-e2e-config-override
         with:
-          runId: ${{ github.run_id }}
-          testLogCollect: ${{ vars.TEST_LOG_COLLECT }}
-          selectedNetworks: ${{ env.SELECTED_NETWORKS }}
-          chainlinkImage: ${{ env.CHAINLINK_IMAGE }}
-          chainlinkVersion: ${{ needs.select-versions.outputs.chainlink_version }}
-          lokiEndpoint: https://${{ secrets.GRAFANA_INTERNAL_HOST }}/loki/api/v1/push
-          lokiTenantId: ${{ secrets.GRAFANA_INTERNAL_TENANT_ID }}
-          lokiBasicAuth:  ${{ secrets.GRAFANA_INTERNAL_BASIC_AUTH }}
-          logstreamLogTargets: ${{ vars.LOGSTREAM_LOG_TARGETS }}
-          grafanaUrl: "http://localhost:8080/primary"
-          grafanaDashboardUrl: "/d/ddf75041-1e39-42af-aa46-361fe4c36e9e/ci-e2e-tests-logs"
-          grafanaBearerToken: ${{ secrets.GRAFANA_INTERNAL_URL_SHORTENER_TOKEN }}
-          ethExecutionClient: ${{ matrix.evm_node.eth_implementation }}
-          customEthClientDockerImage: ${{ matrix.evm_node.docker_image }}
-          pyroscopeServer: ${{ secrets.QA_PYROSCOPE_INSTANCE }}
-          pyroscopeEnvironment: ci-client-compatability-${{ matrix.eth_client }}-testnet
-          pyroscopeKey: ${{ secrets.QA_PYROSCOPE_KEY }}
+          chainlink_image: ${{ env.CHAINLINK_IMAGE }}
+          chainlink_version: ${{ needs.select-versions.outputs.chainlink_version }}
+          selected_networks: ${{ env.SELECTED_NETWORKS}}
+          pyroscope_server: ${{ secrets.QA_PYROSCOPE_INSTANCE }}
+          pyroscope_environment: ci-client-compatability-${{ matrix.eth_client }}-testnet
+          pyroscope_key: ${{ secrets.QA_PYROSCOPE_KEY }}
+          loki_run_id: ${{ github.run_id }}
+          loki_endpoint: https://${{ secrets.GRAFANA_INTERNAL_HOST }}/loki/api/v1/push
+          loki_tenant_id: ${{ secrets.GRAFANA_INTERNAL_TENANT_ID }}
+          loki_basic_auth: ${{ secrets.GRAFANA_INTERNAL_BASIC_AUTH }}
+          logstream_log_targets: ${{ vars.LOGSTREAM_LOG_TARGETS }}
+          grafana_url: http://localhost:8080/primary # This is Grafana GAP's address
+          grafana_dashboard_url: /d/ddf75041-1e39-42af-aa46-361fe4c36e9e/ci-e2e-tests-logs
+          grafana_bearer_token: ${{ secrets.GRAFANA_INTERNAL_URL_SHORTENER_TOKEN }}    
+          test_log_collect: ${{ vars.TEST_LOG_COLLECT }}
+          private_ethereum_network_custom_docker_image: ${{ matrix.evm_node.docker_image }}
+          private_ethereum_network_execution_layer: ${{ matrix.evm_node.eth_implementation || 'geth' }}
       - name: Prepare test log name
         run: |
           replace_special_chars() {

From e5a5782b2d20698183c7fce2a1e3936f671443d6 Mon Sep 17 00:00:00 2001
From: lukaszcl <120112546+lukaszcl@users.noreply.github.com>
Date: Fri, 28 Jun 2024 14:40:57 +0200
Subject: [PATCH 096/137] Refactor on-demand-keeper-smoke-tests.yml

---
 .../on-demand-keeper-smoke-tests.yml          | 31 ++++++++++---------
 1 file changed, 16 insertions(+), 15 deletions(-)

diff --git a/.github/workflows/on-demand-keeper-smoke-tests.yml b/.github/workflows/on-demand-keeper-smoke-tests.yml
index f6fa8f4467a..7d7378557e0 100644
--- a/.github/workflows/on-demand-keeper-smoke-tests.yml
+++ b/.github/workflows/on-demand-keeper-smoke-tests.yml
@@ -132,22 +132,23 @@ jobs:
             echo "run_command=./smoke/${{ matrix.product.name }}_test.go" >> "$GITHUB_OUTPUT"
           fi
       - name: Prepare Base64 TOML override
-        uses: ./.github/actions/setup-create-base64-config
+        uses: ./.github/actions/create-default-e2e-config-override
         with:
-          runId: ${{ github.run_id }}
-          testLogCollect: ${{ vars.TEST_LOG_COLLECT }}
-          selectedNetworks: ${{ env.SELECTED_NETWORKS }}
-          chainlinkImage: ${{ env.CHAINLINK_IMAGE }}
-          chainlinkVersion: ${{ inputs.evm-ref || github.sha }}
-          pyroscopeServer: ${{ matrix.product.pyroscope_env == '' && '' || !startsWith(github.ref, 'refs/tags/') && '' || secrets.QA_PYROSCOPE_INSTANCE }} # Avoid sending blank envs https://github.com/orgs/community/discussions/25725
-          pyroscopeEnvironment: ${{ matrix.product.pyroscope_env }}
-          pyroscopeKey: ${{ secrets.QA_PYROSCOPE_KEY }}
-          lokiEndpoint: https://${{ secrets.GRAFANA_INTERNAL_HOST }}/loki/api/v1/push
-          lokiTenantId: ${{ secrets.GRAFANA_INTERNAL_TENANT_ID }}
-          lokiBasicAuth: ${{ secrets.GRAFANA_INTERNAL_BASIC_AUTH }}
-          logstreamLogTargets: ${{ vars.LOGSTREAM_LOG_TARGETS }}
-          grafanaUrl: ${{ vars.GRAFANA_URL }}
-          grafanaDashboardUrl: "/d/ddf75041-1e39-42af-aa46-361fe4c36e9e/ci-e2e-tests-logs"
+          chainlink_version: ${{ inputs.evm-ref || github.sha }}
+          chainlink_image: ${{ env.CHAINLINK_IMAGE }}
+          selected_networks: ${{ env.SELECTED_NETWORKS}}
+          pyroscope_server: ${{ matrix.product.pyroscope_env == '' && '' || !startsWith(github.ref, 'refs/tags/') && '' || secrets.QA_PYROSCOPE_INSTANCE }} # Avoid sending blank envs https://github.com/orgs/community/discussions/25725
+          pyroscope_environment: ${{ matrix.product.pyroscope_env }}
+          pyroscope_key: ${{ secrets.QA_PYROSCOPE_KEY }}
+          loki_run_id: ${{ github.run_id }}
+          loki_endpoint: https://${{ secrets.GRAFANA_INTERNAL_HOST }}/loki/api/v1/push
+          loki_tenant_id: ${{ secrets.GRAFANA_INTERNAL_TENANT_ID }}
+          loki_basic_auth: ${{ secrets.GRAFANA_INTERNAL_BASIC_AUTH }}
+          logstream_log_targets: ${{ vars.LOGSTREAM_LOG_TARGETS }}
+          grafana_url: ${{ vars.GRAFANA_URL }}
+          grafana_dashboard_url: /d/ddf75041-1e39-42af-aa46-361fe4c36e9e/ci-e2e-tests-logs
+          grafana_bearer_token: ${{ secrets.GRAFANA_INTERNAL_URL_SHORTENER_TOKEN }}   
+          test_log_collect: ${{ vars.TEST_LOG_COLLECT }}
 
       ## Run this step when changes that require tests to be run are made
       - name: Run Tests

From cd90826098737812681b4b4a49ffeb11f1f45a9c Mon Sep 17 00:00:00 2001
From: lukaszcl <120112546+lukaszcl@users.noreply.github.com>
Date: Fri, 28 Jun 2024 14:56:23 +0200
Subject: [PATCH 097/137] Replace setup-create-base64-config with the new
 action in integration-tests.yml workflow

---
 .github/workflows/integration-tests.yml       | 104 +++++++++---------
 .../run-e2e-tests-reusable-workflow.yml       |   1 +
 2 files changed, 53 insertions(+), 52 deletions(-)

diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml
index a29f553f045..86e7602bdd0 100644
--- a/.github/workflows/integration-tests.yml
+++ b/.github/workflows/integration-tests.yml
@@ -331,23 +331,23 @@ jobs:
           # other inputs
           duplicate-authorization-header: "true"
       - name: Prepare Base64 TOML override
-        uses: ./.github/actions/setup-create-base64-config
-        with:
-          runId: ${{ github.run_id }}
-          testLogCollect: ${{ vars.TEST_LOG_COLLECT }}
-          selectedNetworks: ${{ env.SELECTED_NETWORKS }}
-          chainlinkImage: ${{ env.CHAINLINK_IMAGE }}
-          chainlinkVersion: ${{ inputs.evm-ref || github.sha }}
-          pyroscopeServer: ${{ matrix.product.pyroscope_env == '' && '' || !startsWith(github.ref, 'refs/tags/') && '' || secrets.QA_PYROSCOPE_INSTANCE }} # Avoid sending blank envs https://github.com/orgs/community/discussions/25725
-          pyroscopeEnvironment: ${{ matrix.product.pyroscope_env }}
-          pyroscopeKey: ${{ secrets.QA_PYROSCOPE_KEY }}
-          lokiEndpoint: https://${{ secrets.GRAFANA_INTERNAL_HOST }}/loki/api/v1/push
-          lokiTenantId: ${{ secrets.GRAFANA_INTERNAL_TENANT_ID }}
-          lokiBasicAuth: ${{ secrets.GRAFANA_INTERNAL_BASIC_AUTH }}
-          logstreamLogTargets: ${{ vars.LOGSTREAM_LOG_TARGETS }}
-          grafanaUrl: "http://localhost:8080/primary" # This is GAP's address
-          grafanaDashboardUrl: "/d/ddf75041-1e39-42af-aa46-361fe4c36e9e/ci-e2e-tests-logs"
-          grafanaBearerToken: ${{ secrets.GRAFANA_INTERNAL_URL_SHORTENER_TOKEN }}
+        uses: ./.github/actions/create-default-e2e-config-override
+        with:
+          chainlink_version: ${{ inputs.evm-ref || github.sha }}
+          chainlink_image: ${{ env.CHAINLINK_IMAGE }}
+          selected_networks: ${{ env.SELECTED_NETWORKS}}
+          pyroscope_server: ${{ matrix.product.pyroscope_env == '' && '' || !startsWith(github.ref, 'refs/tags/') && '' || secrets.QA_PYROSCOPE_INSTANCE }} # Avoid sending blank envs https://github.com/orgs/community/discussions/25725
+          pyroscope_environment: ${{ matrix.product.pyroscope_env }}
+          pyroscope_key: ${{ secrets.QA_PYROSCOPE_KEY }}
+          loki_run_id: ${{ github.run_id }}
+          loki_endpoint: https://${{ secrets.GRAFANA_INTERNAL_HOST }}/loki/api/v1/push
+          loki_tenant_id: ${{ secrets.GRAFANA_INTERNAL_TENANT_ID }}
+          loki_basic_auth: ${{ secrets.GRAFANA_INTERNAL_BASIC_AUTH }}
+          logstream_log_targets: ${{ vars.LOGSTREAM_LOG_TARGETS }}
+          grafana_url: http://localhost:8080/primary # This is Grafana GAP's address
+          grafana_dashboard_url: /d/ddf75041-1e39-42af-aa46-361fe4c36e9e/ci-e2e-tests-logs
+          grafana_bearer_token: ${{ secrets.GRAFANA_INTERNAL_URL_SHORTENER_TOKEN }}   
+          test_log_collect: ${{ vars.TEST_LOG_COLLECT }}
 
       ## Run this step when changes that require tests to be run are made
       - name: Run Tests
@@ -446,23 +446,24 @@ jobs:
           # other inputs
           duplicate-authorization-header: "true"
       - name: Prepare Base64 TOML override
-        uses: ./.github/actions/setup-create-base64-config
-        with:
-          runId: ${{ github.run_id }}
-          testLogCollect: ${{ vars.TEST_LOG_COLLECT }}
-          selectedNetworks: ${{ env.SELECTED_NETWORKS }}
-          chainlinkImage: ${{ env.CHAINLINK_IMAGE }}
-          chainlinkVersion: ${{ inputs.evm-ref || github.sha }}
-          pyroscopeServer: ${{ matrix.product.pyroscope_env == '' && '' || !startsWith(github.ref, 'refs/tags/') && '' || secrets.QA_PYROSCOPE_INSTANCE }} # Avoid sending blank envs https://github.com/orgs/community/discussions/25725
-          pyroscopeEnvironment: ${{ matrix.product.pyroscope_env }}
-          pyroscopeKey: ${{ secrets.QA_PYROSCOPE_KEY }}
-          lokiEndpoint: https://${{ secrets.GRAFANA_INTERNAL_HOST }}/loki/api/v1/push
-          lokiTenantId: ${{ secrets.GRAFANA_INTERNAL_TENANT_ID }}
-          lokiBasicAuth: ${{ secrets.GRAFANA_INTERNAL_BASIC_AUTH }}
-          logstreamLogTargets: ${{ vars.LOGSTREAM_LOG_TARGETS }}
-          grafanaUrl: "http://localhost:8080/primary" # This is GAP's address
-          grafanaDashboardUrl: "/d/ddf75041-1e39-42af-aa46-361fe4c36e9e/ci-e2e-tests-logs"
-          grafanaBearerToken: ${{ secrets.GRAFANA_INTERNAL_URL_SHORTENER_TOKEN }}
+        uses: ./.github/actions/create-default-e2e-config-override
+        with:
+          loki_run_id: ${{ github.run_id }}
+          test_log_collect: ${{ vars.TEST_LOG_COLLECT }}    
+          chainlink_version: ${{ inputs.evm-ref || github.sha }}
+          chainlink_image: ${{ env.CHAINLINK_IMAGE }}
+          selected_networks: ${{ env.SELECTED_NETWORKS}}
+          pyroscope_server: ${{ matrix.product.pyroscope_env == '' && '' || !startsWith(github.ref, 'refs/tags/') && '' || secrets.QA_PYROSCOPE_INSTANCE }} # Avoid sending blank envs https://github.com/orgs/community/discussions/25725
+          pyroscope_environment: ${{ matrix.product.pyroscope_env }}
+          pyroscope_key: ${{ secrets.QA_PYROSCOPE_KEY }}
+          loki_endpoint: https://${{ secrets.GRAFANA_INTERNAL_HOST }}/loki/api/v1/push
+          loki_tenant_id: ${{ secrets.GRAFANA_INTERNAL_TENANT_ID }}
+          loki_basic_auth: ${{ secrets.GRAFANA_INTERNAL_BASIC_AUTH }}
+          logstream_log_targets: ${{ vars.LOGSTREAM_LOG_TARGETS }}
+          grafana_url: http://localhost:8080/primary # This is Grafana GAP's address
+          grafana_dashboard_url: /d/ddf75041-1e39-42af-aa46-361fe4c36e9e/ci-e2e-tests-logs
+          grafana_bearer_token: ${{ secrets.GRAFANA_INTERNAL_URL_SHORTENER_TOKEN }}   
+
       ## Run this step when changes that require tests to be run are made
       - name: Run Tests
         if: needs.changes.outputs.src == 'true'
@@ -675,24 +676,23 @@ jobs:
           # other inputs
           duplicate-authorization-header: "true"
       - name: Prepare Base64 TOML override
-        uses: ./.github/actions/setup-create-base64-config
-        with:
-          runId: ${{ github.run_id }}
-          testLogCollect: ${{ vars.TEST_LOG_COLLECT }}
-          selectedNetworks: ${{ env.SELECTED_NETWORKS }}
-          chainlinkImage: ${{ env.CHAINLINK_IMAGE }}
-          chainlinkVersion: ${{ inputs.evm-ref || github.sha }}
-          pyroscopeServer: ${{ matrix.product.pyroscope_env == '' && '' || !startsWith(github.ref, 'refs/tags/') && '' || secrets.QA_PYROSCOPE_INSTANCE }} # Avoid sending blank envs https://github.com/orgs/community/discussions/25725
-          pyroscopeEnvironment: ${{ matrix.product.pyroscope_env }}
-          pyroscopeKey: ${{ secrets.QA_PYROSCOPE_KEY }}
-          lokiEndpoint: https://${{ secrets.GRAFANA_INTERNAL_HOST }}/loki/api/v1/push
-          lokiTenantId: ${{ secrets.GRAFANA_INTERNAL_TENANT_ID }}
-          lokiBasicAuth: ${{ secrets.GRAFANA_INTERNAL_BASIC_AUTH }}
-          logstreamLogTargets: ${{ vars.LOGSTREAM_LOG_TARGETS }}
-          grafanaUrl: "http://localhost:8080/primary"
-          grafanaDashboardUrl: "/d/ddf75041-1e39-42af-aa46-361fe4c36e9e/ci-e2e-tests-logs"
-          grafanaBearerToken: ${{ secrets.GRAFANA_INTERNAL_URL_SHORTENER_TOKEN }}
-      ## Run this step when changes that require tests to be run are made
+        uses: ./.github/actions/create-default-e2e-config-override
+        with:
+          loki_run_id: ${{ github.run_id }}
+          test_log_collect: ${{ vars.TEST_LOG_COLLECT }}    
+          chainlink_version: ${{ inputs.evm-ref || github.sha }}
+          chainlink_image: ${{ env.CHAINLINK_IMAGE }}
+          selected_networks: ${{ env.SELECTED_NETWORKS}}
+          pyroscope_server: ${{ matrix.product.pyroscope_env == '' && '' || !startsWith(github.ref, 'refs/tags/') && '' || secrets.QA_PYROSCOPE_INSTANCE }} # Avoid sending blank envs https://github.com/orgs/community/discussions/25725
+          pyroscope_environment: ${{ matrix.product.pyroscope_env }}
+          pyroscope_key: ${{ secrets.QA_PYROSCOPE_KEY }}
+          loki_endpoint: https://${{ secrets.GRAFANA_INTERNAL_HOST }}/loki/api/v1/push
+          loki_tenant_id: ${{ secrets.GRAFANA_INTERNAL_TENANT_ID }}
+          loki_basic_auth: ${{ secrets.GRAFANA_INTERNAL_BASIC_AUTH }}
+          logstream_log_targets: ${{ vars.LOGSTREAM_LOG_TARGETS }}
+          grafana_url: http://localhost:8080/primary # This is Grafana GAP's address
+          grafana_dashboard_url: /d/ddf75041-1e39-42af-aa46-361fe4c36e9e/ci-e2e-tests-logs
+          grafana_bearer_token: ${{ secrets.GRAFANA_INTERNAL_URL_SHORTENER_TOKEN }}  
       - name: Run Tests
         if: needs.changes.outputs.src == 'true' || github.event_name == 'workflow_dispatch'
         uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/run-tests@af92c5fae8dcf1659201e907db82d221fc304b94 # v2.3.21
diff --git a/.github/workflows/run-e2e-tests-reusable-workflow.yml b/.github/workflows/run-e2e-tests-reusable-workflow.yml
index 40f6a6bff6d..b4e03dc6d89 100644
--- a/.github/workflows/run-e2e-tests-reusable-workflow.yml
+++ b/.github/workflows/run-e2e-tests-reusable-workflow.yml
@@ -376,6 +376,7 @@ jobs:
           grafana_url: http://localhost:8080/primary # This is Grafana GAP's address
           grafana_dashboard_url: /d/ddf75041-1e39-42af-aa46-361fe4c36e9e/ci-e2e-tests-logs
           grafana_bearer_token: ${{ secrets.GRAFANA_INTERNAL_URL_SHORTENER_TOKEN }}          
+          test_log_collect: ${{ vars.TEST_LOG_COLLECT }}
 
       - name: Run tests
         uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/run-tests@af92c5fae8dcf1659201e907db82d221fc304b94 # v2.3.21

From e4af6e0f53bacbf91069bab5dc908cc093a02fa3 Mon Sep 17 00:00:00 2001
From: lukaszcl <120112546+lukaszcl@users.noreply.github.com>
Date: Fri, 28 Jun 2024 14:58:29 +0200
Subject: [PATCH 098/137] Remove unused action

---
 .../setup-create-base64-config/action.yml     | 161 ------------------
 1 file changed, 161 deletions(-)
 delete mode 100644 .github/actions/setup-create-base64-config/action.yml

diff --git a/.github/actions/setup-create-base64-config/action.yml b/.github/actions/setup-create-base64-config/action.yml
deleted file mode 100644
index d54554278d1..00000000000
--- a/.github/actions/setup-create-base64-config/action.yml
+++ /dev/null
@@ -1,161 +0,0 @@
-name: Create Base64 Config
-description: A composite action that creates a base64-encoded config to be used by integration tests
-
-inputs:
-  runId:
-    description: The run id
-  testLogCollect:
-    description: Whether to always collect logs, even for passing tests
-    default: "false"
-  selectedNetworks:
-    description: The networks to run tests against
-  chainlinkImage:
-    description: The chainlink image to use
-    default: "public.ecr.aws/chainlink/chainlink"
-  chainlinkPostgresVersion:
-    description: The postgres version to use with the chainlink node
-    default: "15.6"
-  chainlinkVersion:
-    description: The git commit sha to use for the image tag
-  pyroscopeServer:
-    description: URL of Pyroscope server
-  pyroscopeEnvironment:
-    description: Name of Pyroscope environment
-  pyroscopeKey:
-    description: Pyroscope server key
-  lokiEndpoint:
-    description: Loki push endpoint
-  lokiTenantId:
-    description: Loki tenant id
-  lokiBasicAuth:
-    description: Loki basic auth
-  logstreamLogTargets:
-    description: Where to send logs (e.g. file, loki)
-  grafanaUrl:
-    description: Grafana URL
-  grafanaDashboardUrl:
-    description: Grafana dashboard URL
-  grafanaBearerToken:
-    description: Grafana bearer token
-  ethExecutionClient:
-    description: Ethereum execution client to use (geth, besu, nethermind or erigon)
-  customEthClientDockerImage:
-    description: custom docker image to use for eth client (e.g. hyperledger/besu:21.10.0)
-
-runs:
-  using: composite
-  steps:
-    - name: Prepare Base64 TOML override
-      shell: bash
-      id: base64-config-override
-      env:
-        RUN_ID: ${{ inputs.runId }}
-        TEST_LOG_COLLECT: ${{ inputs.testLogCollect }}
-        SELECTED_NETWORKS: ${{ inputs.selectedNetworks }}
-        PYROSCOPE_SERVER: ${{ inputs.pyroscopeServer }}
-        PYROSCOPE_ENVIRONMENT: ${{ inputs.pyroscopeEnvironment }}
-        PYROSCOPE_KEY: ${{ inputs.pyroscopeKey }}
-        CHAINLINK_IMAGE: ${{ inputs.chainlinkImage }}
-        CHAINLINK_VERSION: ${{ inputs.chainlinkVersion }}
-        CHAINLINK_POSTGRES_VERSION: ${{ inputs.chainlinkPostgresVersion }}
-        LOKI_ENDPOINT: ${{ inputs.lokiEndpoint }}
-        LOKI_TENANT_ID: ${{ inputs.lokiTenantId }}
-        LOKI_BASIC_AUTH: ${{ inputs.lokiBasicAuth }}
-        LOGSTREAM_LOG_TARGETS: ${{ inputs.logstreamLogTargets }}
-        GRAFANA_URL: ${{ inputs.grafanaUrl }}
-        GRAFANA_DASHBOARD_URL: ${{ inputs.grafanaDashboardUrl }}
-        GRAFANA_BEARER_TOKEN: ${{ inputs.grafanaBearerToken }}
-        ETH_EXECUTION_CLIENT: ${{ inputs.ethExecutionClient }}
-        CUSTOM_ETH_CLIENT_DOCKER_IMAGE: ${{ inputs.customEthClientDockerImage }}
-      run: |
-        echo ::add-mask::$CHAINLINK_IMAGE
-        function convert_to_toml_array() {
-          local IFS=','
-          local input_array=($1)
-          local toml_array_format="["
-
-          for element in "${input_array[@]}"; do
-              toml_array_format+="\"$element\","
-          done
-
-          toml_array_format="${toml_array_format%,}]"
-          echo "$toml_array_format"
-        }
-
-        selected_networks=$(convert_to_toml_array "$SELECTED_NETWORKS")
-        log_targets=$(convert_to_toml_array "$LOGSTREAM_LOG_TARGETS")
-
-        if [ -n "$PYROSCOPE_SERVER" ]; then
-          pyroscope_enabled=true
-        else
-          pyroscope_enabled=false
-        fi
-
-        if [ -n "$TEST_LOG_COLLECT" ]; then
-          test_log_collect=true
-        else
-          test_log_collect=false
-        fi
-
-        custom_images=""
-        ethereum_version=""
-
-        if [ -n "$CUSTOM_ETH_CLIENT_DOCKER_IMAGE" ]; then
-            ethereum_version="ethereum_version=\"\""
-            custom_images+="[PrivateEthereumNetwork.CustomDockerImages]"
-            custom_images+=$'\n'"execution_layer=\"$CUSTOM_ETH_CLIENT_DOCKER_IMAGE\""
-        fi
-
-        if [ -n "$ETH_EXECUTION_CLIENT" ]; then
-            execution_layer="$ETH_EXECUTION_CLIENT"
-          else
-            execution_layer="geth"
-        fi
-        
-        grafana_bearer_token=""
-        if [ -n "$GRAFANA_BEARER_TOKEN" ]; then
-          grafana_bearer_token="bearer_token_secret=\"$GRAFANA_BEARER_TOKEN\""
-        fi 
-
-        cat << EOF > config.toml
-        [Network]
-        selected_networks=$selected_networks
-
-        [ChainlinkImage]
-        image="$CHAINLINK_IMAGE"
-        version="$CHAINLINK_VERSION"
-        postgres_version="$CHAINLINK_POSTGRES_VERSION"
-
-        [Pyroscope]
-        enabled=$pyroscope_enabled
-        server_url="$PYROSCOPE_SERVER"
-        environment="$PYROSCOPE_ENVIRONMENT"
-        key_secret="$PYROSCOPE_KEY"
-
-        [Logging]
-        test_log_collect=$test_log_collect
-        run_id="$RUN_ID"
-
-        [Logging.LogStream]
-        log_targets=$log_targets
-
-        [Logging.Loki]
-        tenant_id="$LOKI_TENANT_ID"
-        endpoint="$LOKI_ENDPOINT"
-        basic_auth_secret="$LOKI_BASIC_AUTH"
-
-        [Logging.Grafana]
-        base_url="$GRAFANA_URL"
-        dashboard_url="$GRAFANA_DASHBOARD_URL"
-        $grafana_bearer_token
-
-        [PrivateEthereumNetwork]
-        execution_layer="$execution_layer"
-        $ethereum_version
-
-        $custom_images
-        EOF
-
-        BASE64_CONFIG_OVERRIDE=$(cat config.toml | base64 -w 0)
-        echo ::add-mask::$BASE64_CONFIG_OVERRIDE
-        echo "BASE64_CONFIG_OVERRIDE=$BASE64_CONFIG_OVERRIDE" >> $GITHUB_ENV

From 70493a856ea40f80f79a750a689e88707cc6fd61 Mon Sep 17 00:00:00 2001
From: lukaszcl <120112546+lukaszcl@users.noreply.github.com>
Date: Fri, 28 Jun 2024 15:18:49 +0200
Subject: [PATCH 099/137] Fix lint

---
 .../e2e_tests_ci_tool/cmd/check_tests_cmd.go  |  7 +-
 .../cmd/create_test_config_cmd.go             |  5 +-
 .../e2e_tests_ci_tool/cmd/envreplace_cmd.go   |  8 ++-
 .../e2e_tests_ci_tool/cmd/filter_cmd.go       | 12 ++--
 .../cmd/mask_test_config_cmd.go               | 66 +++++++++++--------
 .../cmd/override_test_config_cmd.go           | 12 ++--
 .../e2e_tests_ci_tool/cmd/test_config_cmd.go  |  2 +-
 .../cmd/test_config_cmd_test.go               |  3 +-
 8 files changed, 70 insertions(+), 45 deletions(-)

diff --git a/integration-tests/e2e_tests_ci_tool/cmd/check_tests_cmd.go b/integration-tests/e2e_tests_ci_tool/cmd/check_tests_cmd.go
index 6c73bbcedd0..285413e98d7 100644
--- a/integration-tests/e2e_tests_ci_tool/cmd/check_tests_cmd.go
+++ b/integration-tests/e2e_tests_ci_tool/cmd/check_tests_cmd.go
@@ -2,7 +2,6 @@ package cmd
 
 import (
 	"fmt"
-	"io/ioutil"
 	"os"
 	"path/filepath"
 	"regexp"
@@ -29,7 +28,7 @@ var checkTestsCmd = &cobra.Command{
 	Use:   "check-tests [directory] [yaml file]",
 	Short: "Check if all tests in a directory are included in the test configurations YAML file",
 	Args:  cobra.ExactArgs(2),
-	Run: func(cmd *cobra.Command, args []string) {
+	Run: func(_ *cobra.Command, args []string) {
 		directory := args[0]
 		yamlFile := args[1]
 		tests, err := extractTests(directory)
@@ -49,7 +48,7 @@ func extractTests(dir string) ([]Test, error) {
 			return err
 		}
 		if !info.IsDir() && strings.HasSuffix(info.Name(), "_test.go") {
-			content, err := ioutil.ReadFile(path)
+			content, err := os.ReadFile(path)
 			if err != nil {
 				return err
 			}
@@ -68,7 +67,7 @@ func extractTests(dir string) ([]Test, error) {
 }
 
 func checkTestsInPipeline(yamlFile string, tests []Test) {
-	data, err := ioutil.ReadFile(yamlFile)
+	data, err := os.ReadFile(yamlFile)
 	if err != nil {
 		fmt.Printf("Error reading YAML file: %s\n", err)
 		return
diff --git a/integration-tests/e2e_tests_ci_tool/cmd/create_test_config_cmd.go b/integration-tests/e2e_tests_ci_tool/cmd/create_test_config_cmd.go
index 4894049644d..f422f03e80b 100644
--- a/integration-tests/e2e_tests_ci_tool/cmd/create_test_config_cmd.go
+++ b/integration-tests/e2e_tests_ci_tool/cmd/create_test_config_cmd.go
@@ -5,14 +5,15 @@ import (
 	"os"
 
 	"github.com/pelletier/go-toml/v2"
-	ctf_config "github.com/smartcontractkit/chainlink-testing-framework/config"
 	"github.com/spf13/cobra"
+
+	ctf_config "github.com/smartcontractkit/chainlink-testing-framework/config"
 )
 
 var createTestConfigCmd = &cobra.Command{
 	Use:   "create",
 	Short: "Create a test config from the provided flags",
-	Run: func(cmd *cobra.Command, args []string) {
+	Run: func(cmd *cobra.Command, _ []string) {
 		var tc ctf_config.TestConfig
 
 		var image, version, postgresVersion *string
diff --git a/integration-tests/e2e_tests_ci_tool/cmd/envreplace_cmd.go b/integration-tests/e2e_tests_ci_tool/cmd/envreplace_cmd.go
index cd9d3bc7ee2..1c2caf6c565 100644
--- a/integration-tests/e2e_tests_ci_tool/cmd/envreplace_cmd.go
+++ b/integration-tests/e2e_tests_ci_tool/cmd/envreplace_cmd.go
@@ -16,7 +16,7 @@ var envreplaceCmd = &cobra.Command{
 	Long: `Replaces placeholders of the form ${{ env.VAR_NAME }} with the actual environment variable values.
 Example usage:
 ./e2e_tests_tool envreplace --input 'Example with ${{ env.PATH }} and ${{ env.HOME }}.'`,
-	Run: func(cmd *cobra.Command, args []string) {
+	Run: func(cmd *cobra.Command, _ []string) {
 		input, _ := cmd.Flags().GetString("input")
 
 		output, err := replaceEnvPlaceholders(input)
@@ -55,5 +55,9 @@ func replaceEnvPlaceholders(input string) (string, error) {
 
 func init() {
 	envreplaceCmd.Flags().StringP("input", "i", "", "Input string with placeholders")
-	envreplaceCmd.MarkFlagRequired("input")
+	err := envreplaceCmd.MarkFlagRequired("input")
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "Error marking flag as required: %v\n", err)
+		os.Exit(1)
+	}
 }
diff --git a/integration-tests/e2e_tests_ci_tool/cmd/filter_cmd.go b/integration-tests/e2e_tests_ci_tool/cmd/filter_cmd.go
index 3559f633395..56a88702738 100644
--- a/integration-tests/e2e_tests_ci_tool/cmd/filter_cmd.go
+++ b/integration-tests/e2e_tests_ci_tool/cmd/filter_cmd.go
@@ -3,8 +3,8 @@ package cmd
 import (
 	"encoding/json"
 	"fmt"
-	"io/ioutil"
 	"log"
+	"os"
 	"regexp"
 	"strings"
 
@@ -58,13 +58,13 @@ var filterCmd = &cobra.Command{
 	Long: `Filters tests from a YAML configuration based on name, workflow, test type, and test IDs.
 Example usage:
 ./e2e_tests_tool filter --file .github/e2e-tests.yml --workflow "Run Nightly E2E Tests" --test-type "docker" --test-ids "test1,test2"`,
-	Run: func(cmd *cobra.Command, args []string) {
+	Run: func(cmd *cobra.Command, _ []string) {
 		yamlFile, _ := cmd.Flags().GetString("file")
 		workflow, _ := cmd.Flags().GetString("workflow")
 		testType, _ := cmd.Flags().GetString("test-type")
 		testIDs, _ := cmd.Flags().GetString("test-ids")
 
-		data, err := ioutil.ReadFile(yamlFile)
+		data, err := os.ReadFile(yamlFile)
 		if err != nil {
 			log.Fatalf("Error reading YAML file: %v", err)
 		}
@@ -91,5 +91,9 @@ func init() {
 	filterCmd.Flags().StringP("workflow", "t", "", "Workflow filter")
 	filterCmd.Flags().StringP("test-type", "y", "", "Type of test to filter by")
 	filterCmd.Flags().StringP("test-ids", "i", "*", "Comma-separated list of test IDs to filter by")
-	filterCmd.MarkFlagRequired("file")
+	err := filterCmd.MarkFlagRequired("file")
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "Error marking flag as required: %v\n", err)
+		os.Exit(1)
+	}
 }
diff --git a/integration-tests/e2e_tests_ci_tool/cmd/mask_test_config_cmd.go b/integration-tests/e2e_tests_ci_tool/cmd/mask_test_config_cmd.go
index 886ba32f7b5..c5fe984f3e5 100644
--- a/integration-tests/e2e_tests_ci_tool/cmd/mask_test_config_cmd.go
+++ b/integration-tests/e2e_tests_ci_tool/cmd/mask_test_config_cmd.go
@@ -8,16 +8,17 @@ import (
 
 	"github.com/olekukonko/tablewriter"
 	"github.com/pelletier/go-toml/v2"
-	"github.com/smartcontractkit/chainlink-testing-framework/blockchain"
-	ctf_config "github.com/smartcontractkit/chainlink-testing-framework/config"
 	"github.com/smartcontractkit/seth"
 	"github.com/spf13/cobra"
+
+	"github.com/smartcontractkit/chainlink-testing-framework/blockchain"
+	ctf_config "github.com/smartcontractkit/chainlink-testing-framework/config"
 )
 
 var maskTestConfigCmd = &cobra.Command{
 	Use:   "mask-secrets",
 	Short: "Run 'echo ::add-mask::${secret}' for all secrets in the test config",
-	Run: func(cmd *cobra.Command, args []string) {
+	Run: func(_ *cobra.Command, _ []string) {
 		decoded, err := base64.StdEncoding.DecodeString(fromBase64TOML)
 		if err != nil {
 			fmt.Fprintf(os.Stderr, "Error decoding base64 TOML: %v\n", err)
@@ -33,41 +34,41 @@ var maskTestConfigCmd = &cobra.Command{
 		showCmdInfo()
 
 		if config.ChainlinkImage != nil {
-			mustMaskSecret("chainlink image", config.ChainlinkImage.Image)
-			mustMaskSecret("chainlink version", config.ChainlinkImage.Version)
+			mustMaskSecret("chainlink image", safeDeref(config.ChainlinkImage.Image))
+			mustMaskSecret("chainlink version", safeDeref(config.ChainlinkImage.Version))
 		}
 		if config.Pyroscope != nil {
-			mustMaskSecret("pyroscope server url", config.Pyroscope.ServerUrl)
-			mustMaskSecret("pyroscope key", config.Pyroscope.Key)
+			mustMaskSecret("pyroscope server url", safeDeref(config.Pyroscope.ServerUrl))
+			mustMaskSecret("pyroscope key", safeDeref(config.Pyroscope.Key))
 		}
 		if config.Logging != nil && config.Logging.Loki != nil {
 			// mustMaskSecret("loki tenant id", config.Logging.RunId)
-			mustMaskSecret("loki endpoint", config.Logging.Loki.Endpoint)
-			mustMaskSecret("loki tenant id", config.Logging.Loki.TenantId)
-			mustMaskSecret("loki basic auth", config.Logging.Loki.BasicAuth)
-			mustMaskSecret("loki bearer token", config.Logging.Loki.BearerToken)
+			mustMaskSecret("loki endpoint", safeDeref(config.Logging.Loki.Endpoint))
+			mustMaskSecret("loki tenant id", safeDeref(config.Logging.Loki.TenantId))
+			mustMaskSecret("loki basic auth", safeDeref(config.Logging.Loki.BasicAuth))
+			mustMaskSecret("loki bearer token", safeDeref(config.Logging.Loki.BearerToken))
 		}
 		if config.Logging != nil && config.Logging.Grafana != nil {
-			mustMaskSecret("loki grafana token", config.Logging.Grafana.BearerToken)
+			mustMaskSecret("loki grafana token", safeDeref(config.Logging.Grafana.BearerToken))
 		}
 		if config.Network != nil && config.Network.RpcHttpUrls != nil {
 			for _, urls := range config.Network.RpcHttpUrls {
 				for _, url := range urls {
-					mustMaskSecret("rpc http url", &url)
+					mustMaskSecret("rpc http url", url)
 				}
 			}
 		}
 		if config.Network != nil && config.Network.RpcWsUrls != nil {
 			for _, urls := range config.Network.RpcWsUrls {
 				for _, url := range urls {
-					mustMaskSecret("rpc ws url", &url)
+					mustMaskSecret("rpc ws url", url)
 				}
 			}
 		}
 		if config.Network != nil && config.Network.WalletKeys != nil {
 			for _, keys := range config.Network.WalletKeys {
 				for _, key := range keys {
-					mustMaskSecret("wallet key", &key)
+					mustMaskSecret("wallet key", key)
 				}
 			}
 		}
@@ -78,7 +79,7 @@ var maskTestConfigCmd = &cobra.Command{
 			}
 		}
 		if config.WaspConfig != nil {
-			mustMaskSecret("wasp repo image version uri", config.WaspConfig.RepoImageVersionURI)
+			mustMaskSecret("wasp repo image version uri", safeDeref(config.WaspConfig.RepoImageVersionURI))
 		}
 		// Mask Seth config
 		if config.Seth != nil {
@@ -93,34 +94,34 @@ var maskTestConfigCmd = &cobra.Command{
 func mustMaskEVMNetworkConfig(network *blockchain.EVMNetwork) {
 	if network != nil {
 		for _, url := range network.HTTPURLs {
-			mustMaskSecret("network rpc url", &url)
+			mustMaskSecret("network rpc url", url)
 		}
 		for _, url := range network.URLs {
-			mustMaskSecret("network ws url", &url)
+			mustMaskSecret("network ws url", url)
 		}
 		for _, key := range network.PrivateKeys {
-			mustMaskSecret("network private key", &key)
+			mustMaskSecret("network private key", key)
 		}
-		mustMaskSecret("network url", &network.URL)
+		mustMaskSecret("network url", network.URL)
 	}
 }
 
 func mustMaskSethNetworkConfig(network *seth.Network) {
 	if network != nil {
 		for _, url := range network.URLs {
-			mustMaskSecret("network url", &url)
+			mustMaskSecret("network url", url)
 		}
 		for _, key := range network.PrivateKeys {
-			mustMaskSecret("network private key", &key)
+			mustMaskSecret("network private key", key)
 		}
 	}
 }
 
-func mustMaskSecret(description string, secret *string) {
-	if secret != nil && *secret != "" {
+func mustMaskSecret(description string, secret string) {
+	if secret != "" {
 		fmt.Printf("Mask '%s'\n", description)
-		fmt.Printf("::add-mask::%s\n", *secret)
-		cmd := exec.Command("bash", "-c", fmt.Sprintf("echo ::add-mask::%s", *secret))
+		fmt.Printf("::add-mask::%s\n", secret)
+		cmd := exec.Command("bash", "-c", "echo ::add-mask::'$0'", "_", secret)
 		err := cmd.Run()
 		if err != nil {
 			fmt.Fprintf(os.Stderr, "Failed to mask secret '%s'", description)
@@ -129,6 +130,13 @@ func mustMaskSecret(description string, secret *string) {
 	}
 }
 
+func safeDeref(s *string) string {
+	if s != nil {
+		return *s
+	}
+	return ""
+}
+
 func showCmdInfo() {
 	data := [][]string{
 		{"This command masks ONLY SELECTED secrets in the test config!"},
@@ -156,5 +164,9 @@ func showCmdInfo() {
 
 func init() {
 	maskTestConfigCmd.Flags().StringVar(&fromBase64TOML, FromBase64ConfigFlag, "", "Base64 encoded TOML config to override")
-	maskTestConfigCmd.MarkFlagRequired(FromBase64ConfigFlag)
+	err := maskTestConfigCmd.MarkFlagRequired(FromBase64ConfigFlag)
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "Error marking flag as required: %v\n", err)
+		os.Exit(1)
+	}
 }
diff --git a/integration-tests/e2e_tests_ci_tool/cmd/override_test_config_cmd.go b/integration-tests/e2e_tests_ci_tool/cmd/override_test_config_cmd.go
index a0fbb133e00..5a48e8d0aae 100644
--- a/integration-tests/e2e_tests_ci_tool/cmd/override_test_config_cmd.go
+++ b/integration-tests/e2e_tests_ci_tool/cmd/override_test_config_cmd.go
@@ -6,9 +6,10 @@ import (
 	"os"
 
 	"github.com/pelletier/go-toml/v2"
-	ctf_config "github.com/smartcontractkit/chainlink-testing-framework/config"
 	"github.com/spf13/cobra"
 	"github.com/spf13/pflag"
+
+	ctf_config "github.com/smartcontractkit/chainlink-testing-framework/config"
 )
 
 const DryRunFlag = "dry-run"
@@ -19,7 +20,7 @@ var fromBase64TOML string
 var overrideTestConfigCmd = &cobra.Command{
 	Use:   "override",
 	Short: "Override base64 encoded TOML config with provided flags. Overrides only existing fields in the base config.",
-	Run: func(cmd *cobra.Command, args []string) {
+	Run: func(cmd *cobra.Command, _ []string) {
 		dryRun, _ := cmd.Flags().GetBool(DryRunFlag)
 
 		decoded, err := base64.StdEncoding.DecodeString(fromBase64TOML)
@@ -191,8 +192,11 @@ var overrideTestConfigCmd = &cobra.Command{
 
 func init() {
 	overrideTestConfigCmd.Flags().StringVar(&fromBase64TOML, FromBase64ConfigFlag, "", "Base64 encoded TOML config to override")
-	overrideTestConfigCmd.MarkFlagRequired(FromBase64ConfigFlag)
-
+	err := overrideTestConfigCmd.MarkFlagRequired(FromBase64ConfigFlag)
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "Error marking flag as required: %v\n", err)
+		os.Exit(1)
+	}
 	overrideTestConfigCmd.Flags().Bool(DryRunFlag, false, "Dry run mode")
 }
 
diff --git a/integration-tests/e2e_tests_ci_tool/cmd/test_config_cmd.go b/integration-tests/e2e_tests_ci_tool/cmd/test_config_cmd.go
index 7f10f1cab4e..be77e861ff5 100644
--- a/integration-tests/e2e_tests_ci_tool/cmd/test_config_cmd.go
+++ b/integration-tests/e2e_tests_ci_tool/cmd/test_config_cmd.go
@@ -88,7 +88,7 @@ func init() {
 		c.Flags().StringVar(&oc.PrivateEthereumNetworkEthereumVersion, PrivateEthereumNetworkEthereumVersionFlag, "", "")
 		c.Flags().StringVar(&oc.PrivateEthereumNetworkCustomDockerImages, PrivateEthereumNetworkCustomDockerImageFlag, "", "")
 
-		c.PreRun = func(cmd *cobra.Command, args []string) {
+		c.PreRun = func(_ *cobra.Command, _ []string) {
 			// Resolve selected networks environment variable if set
 			if len(oc.SelectedNetworks) > 0 {
 				_, hasEnvVar := lookupEnvVarName(oc.SelectedNetworks[0])
diff --git a/integration-tests/e2e_tests_ci_tool/cmd/test_config_cmd_test.go b/integration-tests/e2e_tests_ci_tool/cmd/test_config_cmd_test.go
index 34cf3c0ac5b..fb1ef5332bd 100644
--- a/integration-tests/e2e_tests_ci_tool/cmd/test_config_cmd_test.go
+++ b/integration-tests/e2e_tests_ci_tool/cmd/test_config_cmd_test.go
@@ -5,9 +5,10 @@ import (
 	"testing"
 
 	"github.com/pelletier/go-toml/v2"
-	ctf_config "github.com/smartcontractkit/chainlink-testing-framework/config"
 	"github.com/spf13/cobra"
 	"github.com/stretchr/testify/assert"
+
+	ctf_config "github.com/smartcontractkit/chainlink-testing-framework/config"
 )
 
 func TestCreateTestConfigCmd(t *testing.T) {

From 2de3c68eb3ac5af368e763b8c1a8d7779a9a9e7f Mon Sep 17 00:00:00 2001
From: lukaszcl <120112546+lukaszcl@users.noreply.github.com>
Date: Fri, 28 Jun 2024 15:19:12 +0200
Subject: [PATCH 100/137] Remove unused action

---
 .../actions/run-e2e-docker-test/action.yml    | 111 ------------------
 1 file changed, 111 deletions(-)
 delete mode 100644 .github/actions/run-e2e-docker-test/action.yml

diff --git a/.github/actions/run-e2e-docker-test/action.yml b/.github/actions/run-e2e-docker-test/action.yml
deleted file mode 100644
index a30bfb73297..00000000000
--- a/.github/actions/run-e2e-docker-test/action.yml
+++ /dev/null
@@ -1,111 +0,0 @@
-name: 'Run E2E Docker Test'
-inputs:
-  test:
-    required: true
-    description: 'Test to run'
-  test_config_base64_override:
-    description: 'This is an unsafe way to pass custom base64 test config. It cannot contain any secrets as this value is exposed in Github logs (Set up job -> Inputs) when the workflow is called! The caller workflow must make sure to remove secrets from this input.'
-    required: false
-    type: string        
-
-runs:
-  using: 'composite'
-  steps:
-    - name: Install jq
-      run: sudo apt-get install -y jq
-    - name: Show test configuration
-      run: echo '${{ toJson(inputs.test) }}' | jq .
-    - name: Set up Go
-      uses: actions/setup-go@v2
-      with:
-        go-version: '1.21.7'
-
-    - name: Set default test config override
-      if: ${{ inputs.test_config_base64_override == '' }}
-      run: |      
-        # Mask secret args before using them
-        # For more, see https://docs.github.com/en/actions/using-workflows/workflow-commands-for-github-actions#masking-a-value-in-a-log
-        echo ::add-mask::${{ env.CHAINLINK_IMAGE }}
-        echo "cl image: ${{ env.CHAINLINK_IMAGE }}"
-
-        # Use overrides from e2e-tests.yml or defaults 
-        chainlink_version="${{ inputs.test.testConfigOverride.chainlinkVersion || github.sha }}"
-        chainlink_image="${{ inputs.test.testConfigOverride.chainlinkImage || env.CHAINLINK_IMAGE }}"
-        chainlink_postgres_version="${{ inputs.test.testConfigOverride.chainlinkPostgresVersion }}"
-        chainlink_upgrade_version="${{ inputs.test.testConfigOverride.chainlinkUpgradeVersion }}"
-        selected_networks="${{ inputs.test.testConfigOverride.selectedNetworks || env.SELECTED_NETWORKS}}"
-
-        # Build the command line arguments
-        # Need to check if the variable is set before adding it to the command
-        # Otherwise "" will be passed as an argument which is not what we want
-        cmd_args=""
-        if [ -n "$chainlink_image" ]; then
-          cmd_args+="--chainlink-image=$chainlink_image "
-        fi
-        if [ -n "$chainlink_version" ]; then
-          cmd_args+="--chainlink_version=$chainlink_version "
-        fi
-        if [ -n "$chainlink_postgres_version" ]; then
-          cmd_args+="--chainlink-postgres-version=$chainlink_postgres_version "
-        fi
-        if [ -n "$chainlink_upgrade_version" ]; then
-          cmd_args+="--chainlink-upgrade-version=$chainlink_upgrade_version "
-        fi
-        if [ -n "$selected_networks" ]; then
-          cmd_args+="--selected-networks=$selected_networks "
-        fi    
-        
-        cd integration-tests/e2e_tests_ci_tool/
-
-        # Mask secrets inside the test config override (like resolved env vars)
-        go run main.go test-config mask-secrets $cmd_args
-
-        echo $chainlink_version
-
-        # Create a base64 encoded string of the test config override
-        config_override=$(go run main.go test-config create $cmd_args)
-
-        BASE64_CONFIG_OVERRIDE=$(echo "$config_override" | base64 -w 0)
-        echo ::add-mask::$BASE64_CONFIG_OVERRIDE
-        echo "BASE64_CONFIG_OVERRIDE=$BASE64_CONFIG_OVERRIDE" >> $GITHUB_ENV
-
-    - name: Mask secrets in test config base64 override
-      id: mask-secrets-in-test_config_base64_override
-      if: ${{ inputs.test_config_base64_override != '' }}
-      run: |
-        # Mask secrets inside the test config override
-        cd integration-tests/e2e_tests_ci_tool/
-        go run main.go test-config mask-secrets --from-base64-config ${{ inputs.test_config_base64_override }}
-
-        exit 1 # Fail the job
-
-    - name: Setup GAP for Grafana
-      uses: smartcontractkit/.github/actions/setup-gap@6c9d62fdad050cfb8b59376ded291f1350705944 # setup-gap@0.2.2
-      with:
-        aws-region: ${{ secrets.AWS_REGION }}
-        aws-role-arn: ${{ secrets.AWS_OIDC_IAM_ROLE_VALIDATION_PROD_ARN }}
-        api-gateway-host: ${{ secrets.AWS_API_GW_HOST_GRAFANA }}
-        duplicate-authorization-header: "true"
-    - name: Run Tests
-      uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/run-tests@af92c5fae8dcf1659201e907db82d221fc304b94 # v2.3.21
-      with:
-        test_command_to_run: ${{ inputs.test.testCmd }} 2>&1 | tee /tmp/gotest.log | gotestloghelper -ci -singlepackage -hidepassingtests=false -hidepassinglogs
-        test_download_vendor_packages_command: cd ./integration-tests && go mod download
-        cl_repo: ${{ env.CHAINLINK_IMAGE }}
-        cl_image_tag: ${{ inputs.chainlink_version }}
-        aws_registries: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}
-        artifacts_name: ${{ inputs.test.id }}-test-logs
-        artifacts_location: |
-          ./integration-tests/smoke/logs/
-          /tmp/gotest.log
-        publish_check_name: ${{ inputs.test.id }}
-        token: ${{ secrets.GITHUB_TOKEN }}
-        go_mod_path: ./integration-tests/go.mod
-        cache_key_id: core-e2e-${{ env.MOD_CACHE_VERSION }}
-        cache_restore_only: "true"
-        QA_AWS_REGION: ${{ secrets.QA_AWS_REGION }}
-        QA_AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }}
-        QA_KUBECONFIG: ""
-        should_tidy: "false"
-        go_coverage_src_dir: /var/tmp/go-coverage
-        go_coverage_dest_dir: ${{ github.workspace }}/.covdata
\ No newline at end of file

From 36921121bde3d2dfc2d8bad3877984ee13b22295 Mon Sep 17 00:00:00 2001
From: lukaszcl <120112546+lukaszcl@users.noreply.github.com>
Date: Fri, 28 Jun 2024 15:40:17 +0200
Subject: [PATCH 101/137] Fix handling selected_networks input

---
 .../action.yml                                  | 17 +++++++++++------
 .github/workflows/integration-tests.yml         |  2 +-
 2 files changed, 12 insertions(+), 7 deletions(-)

diff --git a/.github/actions/create-default-e2e-config-override/action.yml b/.github/actions/create-default-e2e-config-override/action.yml
index b210374540a..78b15a26ab2 100644
--- a/.github/actions/create-default-e2e-config-override/action.yml
+++ b/.github/actions/create-default-e2e-config-override/action.yml
@@ -14,7 +14,7 @@ inputs:
     description: 'Chainlink upgrade version to use for the tests'
   selected_networks:
     required: false
-    description: 'Selected networks for the tests'
+    description: 'Selected networks for the tests (comma separated)'
   pyroscope_server:
     required: false
     description: 'Pyroscope server for the tests'
@@ -77,9 +77,6 @@ runs:
         if [ -n "${{ inputs.chainlink_upgrade_version }}" ]; then
           cmd_args+="--chainlink-upgrade-version=\"${{ inputs.chainlink_upgrade_version }}\" "
         fi
-        if [ -n "${{ inputs.selected_networks }}" ]; then
-          cmd_args+="--selected-networks=\"${{ inputs.selected_networks }}\" "
-        fi
         if [ -n "${{ inputs.pyroscope_server }}" ]; then
           cmd_args+="--pyroscope-server-url=\"${{ inputs.pyroscope_server }}\" "
         fi
@@ -110,11 +107,19 @@ runs:
         if [ -n "${{ inputs.private_ethereum_network_custom_docker_image }}" ]; then
           cmd_args+="--private-ethereum-network-custom-docker-image=\"${{ inputs.private_ethereum_network_custom_docker_image }}\" "
         fi
-        # Split the log targets input by comma and add them to the command line arguments
+        
+        # Split the log targets input by comma and add to the command line arguments
         IFS=',' read -ra ADDR <<< "${{ inputs.logstream_log_targets }}"
         for i in "${ADDR[@]}"; do
           cmd_args+="--logging-log-targets=\"$i\" "
-        done        
+        done
+
+        # Split selected_networks input by comma and add to the command line arguments
+        IFS=',' read -ra ADDR <<< "${{ inputs.selected_networks }}"
+        for i in "${ADDR[@]}"; do
+          cmd_args+="--selected-networks=\"$i\" "
+        done
+
         echo "Command line arguments: $cmd_args" 
         echo "CMD_ARGS=$cmd_args" >> $GITHUB_ENV
 
diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml
index 86e7602bdd0..f204ee896d2 100644
--- a/.github/workflows/integration-tests.yml
+++ b/.github/workflows/integration-tests.yml
@@ -335,7 +335,7 @@ jobs:
         with:
           chainlink_version: ${{ inputs.evm-ref || github.sha }}
           chainlink_image: ${{ env.CHAINLINK_IMAGE }}
-          selected_networks: ${{ env.SELECTED_NETWORKS}}
+          selected_networks: ${{ env.SELECTED_NETWORKS }}
           pyroscope_server: ${{ matrix.product.pyroscope_env == '' && '' || !startsWith(github.ref, 'refs/tags/') && '' || secrets.QA_PYROSCOPE_INSTANCE }} # Avoid sending blank envs https://github.com/orgs/community/discussions/25725
           pyroscope_environment: ${{ matrix.product.pyroscope_env }}
           pyroscope_key: ${{ secrets.QA_PYROSCOPE_KEY }}

From 907cba8a25228202bd7d4feea0438786e884a49b Mon Sep 17 00:00:00 2001
From: lukaszcl <120112546+lukaszcl@users.noreply.github.com>
Date: Fri, 28 Jun 2024 15:44:30 +0200
Subject: [PATCH 102/137] Fix cache

---
 .github/workflows/run-e2e-tests-reusable-workflow.yml | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/run-e2e-tests-reusable-workflow.yml b/.github/workflows/run-e2e-tests-reusable-workflow.yml
index b4e03dc6d89..f7942c4dc41 100644
--- a/.github/workflows/run-e2e-tests-reusable-workflow.yml
+++ b/.github/workflows/run-e2e-tests-reusable-workflow.yml
@@ -61,6 +61,7 @@ on:
                
 env: 
   CHAINLINK_IMAGE: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com/chainlink
+  MOD_CACHE_VERSION: 1
 
 jobs:
   validate-inputs:
@@ -277,8 +278,8 @@ jobs:
             /tmp/gotest.log
           publish_check_name: ${{ matrix.tests.idSanitized }}
           token: ${{ secrets.GH_TOKEN }}
+          cache_key_id: run-docker-e2e-tests-${{ env.MOD_CACHE_VERSION }}
           go_mod_path: ./integration-tests/go.mod
-          cache_key_id: core-e2e-${{ env.MOD_CACHE_VERSION }}
           cache_restore_only: "true"
           QA_AWS_REGION: ${{ secrets.QA_AWS_REGION }}
           QA_AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }}
@@ -399,6 +400,7 @@ jobs:
           cl_image_tag: ${{ env.CHAINLINK_VERSION }}
           token: ${{ secrets.GH_TOKEN }}
           should_cleanup: false
+          cache_key_id: run-k8s-e2e-tests-${{ env.MOD_CACHE_VERSION }}
           go_mod_path: ./integration-tests/go.mod
           QA_AWS_REGION: ${{ secrets.QA_AWS_REGION }}
           QA_AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }}

From cfe7de5b6593a4366747e07b0ac9696158852706 Mon Sep 17 00:00:00 2001
From: lukaszcl <120112546+lukaszcl@users.noreply.github.com>
Date: Fri, 28 Jun 2024 15:53:17 +0200
Subject: [PATCH 103/137] Do not run soak/ocr_test.go:^TestOCRSoak$ nightly

---
 .github/e2e-tests.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/e2e-tests.yml b/.github/e2e-tests.yml
index a3e3e11b030..1d7dbea8a0a 100644
--- a/.github/e2e-tests.yml
+++ b/.github/e2e-tests.yml
@@ -100,8 +100,8 @@ runner-test-matrix:
     test-type: k8s-remote-runner
     remote-runner-test-suite: soak
     runs-on: ubuntu-latest
-    workflows:
-      - Run Nightly E2E Tests
+    # workflows:
+    #   - Run Nightly E2E Tests
     test-cmd: cd integration-tests/ && go test soak/ocr_test.go -test.run ^TestOCRSoak$ -test.parallel=1 -timeout 30m -count=1
     pyroscope-env: ci-smoke-ocr2-evm-simulated-nightly
 

From c3b1d4bf55773de267e8d33b15661f3af173ea0d Mon Sep 17 00:00:00 2001
From: lukaszcl <120112546+lukaszcl@users.noreply.github.com>
Date: Fri, 28 Jun 2024 15:53:52 +0200
Subject: [PATCH 104/137] Enable nightly workflow to run some tests

---
 .github/workflows/run-nightly-e2e-tests.yml | 6 ++----
 1 file changed, 2 insertions(+), 4 deletions(-)

diff --git a/.github/workflows/run-nightly-e2e-tests.yml b/.github/workflows/run-nightly-e2e-tests.yml
index cf5205208ee..0afa59418bc 100644
--- a/.github/workflows/run-nightly-e2e-tests.yml
+++ b/.github/workflows/run-nightly-e2e-tests.yml
@@ -1,11 +1,9 @@
 name: Run Nightly E2E Tests
 
 on:
-  # TODO: uncomment when the workflow is ready
-  # schedule:
+  schedule:
     # Run every night at midnight UTC (0:00 AM) TODO: create separate workflow that uses this one 
-    # - cron: '0 0 * * *'
-    workflow_dispatch:
+    - cron: '0 0 * * *'
 
 jobs:
   call-run-e2e-tests-workflow:

From 2fccfbd11e06edafd809b6ea104084caf10c7266 Mon Sep 17 00:00:00 2001
From: lukaszcl <120112546+lukaszcl@users.noreply.github.com>
Date: Fri, 28 Jun 2024 16:48:18 +0200
Subject: [PATCH 105/137] Remove config override input

---
 .github/workflows/run-automation-benchmark-e2e-tests.yml | 5 -----
 .github/workflows/run-e2e-tests-reusable-workflow.yml    | 3 ++-
 .github/workflows/run-selected-e2e-tests.yml             | 5 -----
 3 files changed, 2 insertions(+), 11 deletions(-)

diff --git a/.github/workflows/run-automation-benchmark-e2e-tests.yml b/.github/workflows/run-automation-benchmark-e2e-tests.yml
index 7bcc6839d0c..2e09df5e33a 100644
--- a/.github/workflows/run-automation-benchmark-e2e-tests.yml
+++ b/.github/workflows/run-automation-benchmark-e2e-tests.yml
@@ -11,10 +11,6 @@ on:
         options:
           - TestAutomationBenchmark
           - TestAutomationBenchmark_1000Upkeeps_2_1
-      # test_config_base64_override:
-      #   description: Custom base64 test config
-      #   required: false
-      #   type: string
 
 # TODO:
 # 1. list of tests to select 
@@ -37,7 +33,6 @@ jobs:
       test_ids: ${{ github.event.inputs.test-id }}
       # Select tests to run from .github/e2e_tests.yml based on workflow name 
       test_workflow: Run Automation Product Nightly E2E Tests
-      # test_config_base64_override: ${{ github.event.inputs.test_config_base64_override }}
     secrets:
       QA_AWS_REGION: ${{ secrets.QA_AWS_REGION }}
       QA_AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }}
diff --git a/.github/workflows/run-e2e-tests-reusable-workflow.yml b/.github/workflows/run-e2e-tests-reusable-workflow.yml
index f7942c4dc41..b2a084d14f0 100644
--- a/.github/workflows/run-e2e-tests-reusable-workflow.yml
+++ b/.github/workflows/run-e2e-tests-reusable-workflow.yml
@@ -15,7 +15,8 @@ on:
       test_workflow:
         description: 'Run tests by workflow name. Example: "Run Nightly E2E Tests"'
         required: false
-        type: string        
+        type: string
+      # TODO: Enable once Test Config does not have secrets. Related ticket https://smartcontract-it.atlassian.net/browse/TT-1283
       # test_config_base64_override:
       #   description: 'This is an unsafe way to pass custom base64 test config. It cannot contain any secrets as this value is exposed in Github logs (Set up job -> Inputs) when the workflow is called! The caller workflow must make sure to remove secrets from this input.'
       #   required: false
diff --git a/.github/workflows/run-selected-e2e-tests.yml b/.github/workflows/run-selected-e2e-tests.yml
index cb04fafddd8..ef23b37d989 100644
--- a/.github/workflows/run-selected-e2e-tests.yml
+++ b/.github/workflows/run-selected-e2e-tests.yml
@@ -13,10 +13,6 @@ on:
         default: "*"
         required: true
         type: string
-      # test_config_base64_override:
-      #   description: 'Custom base64 test config'
-      #   required: false
-      #   type: string          
       enable_check_test_configurations:
         description: 'Set to "true" to enable check-test-configurations job'
         required: false
@@ -34,7 +30,6 @@ jobs:
     with:
       chainlink_version: ${{ github.event.inputs.chainlink_version }}
       test_ids: ${{ github.event.inputs.test_ids }}
-      # test_config_base64_override: ${{ github.event.inputs.test_config_base64_override }}
       with_existing_remote_runner_version: ${{ github.event.inputs.with_existing_remote_runner_version }}
       # Use fromJSON to convert string to boolean. More info: https://github.com/actions/runner/issues/2206#issuecomment-1532246677
       enable_check_test_configurations: ${{ fromJSON(github.event.inputs.enable_check_test_configurations) }}

From a3eafebc2c61dd0e9ddb3282ff65b7810ee8bf45 Mon Sep 17 00:00:00 2001
From: lukaszcl <120112546+lukaszcl@users.noreply.github.com>
Date: Fri, 28 Jun 2024 16:49:15 +0200
Subject: [PATCH 106/137] Remove temp test

---
 .github/e2e-tests.yml                          |  9 ---------
 integration-tests/smoke/quick_ci_check_test.go | 13 -------------
 2 files changed, 22 deletions(-)
 delete mode 100644 integration-tests/smoke/quick_ci_check_test.go

diff --git a/.github/e2e-tests.yml b/.github/e2e-tests.yml
index 1d7dbea8a0a..feef287f84b 100644
--- a/.github/e2e-tests.yml
+++ b/.github/e2e-tests.yml
@@ -7,15 +7,6 @@
 #
 runner-test-matrix:
 
-  # TODO: REMOVE WHEN THE CI WORKFLOWS TESTING IS DONE
-  - id: integration-tests/smoke/quick_ci_check_test.go
-    path: integration-tests/smoke/quick_ci_check_test.go
-    test-type: docker
-    runs-on: ubuntu-latest
-    workflows:
-      - Run Nightly E2E Tests
-    test-cmd: cd integration-tests/ && go test smoke/quick_ci_check_test.go -timeout 1m -count=1 -json
-
   # Example of 1 runner for all tests in integration-tests/smoke/ocr_test.go
   - id: smoke/ocr_test.go:* 
     path: integration-tests/smoke/ocr_test.go
diff --git a/integration-tests/smoke/quick_ci_check_test.go b/integration-tests/smoke/quick_ci_check_test.go
deleted file mode 100644
index b784e1f5d27..00000000000
--- a/integration-tests/smoke/quick_ci_check_test.go
+++ /dev/null
@@ -1,13 +0,0 @@
-package smoke
-
-import (
-	"testing"
-
-	"github.com/smartcontractkit/chainlink-testing-framework/logging"
-)
-
-// TODO: REMOVE WHEN THE CI WORKFLOWS TESTING IS DONE
-func TestCICheck_Passing(t *testing.T) {
-	l := logging.GetTestLogger(t)
-	l.Info().Msg("Quick CI check. Always passing")
-}

From 314f7e7c0a0e6f06b37c62e95983259094ba827d Mon Sep 17 00:00:00 2001
From: lukaszcl <120112546+lukaszcl@users.noreply.github.com>
Date: Fri, 28 Jun 2024 17:24:26 +0200
Subject: [PATCH 107/137] Fix setup go

---
 .../run-e2e-tests-reusable-workflow.yml        | 18 ++++++------------
 1 file changed, 6 insertions(+), 12 deletions(-)

diff --git a/.github/workflows/run-e2e-tests-reusable-workflow.yml b/.github/workflows/run-e2e-tests-reusable-workflow.yml
index b2a084d14f0..e608589c7e5 100644
--- a/.github/workflows/run-e2e-tests-reusable-workflow.yml
+++ b/.github/workflows/run-e2e-tests-reusable-workflow.yml
@@ -84,10 +84,8 @@ jobs:
     steps:
       - name: Checkout code
         uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2
-      - name: Set up Go
-        uses: actions/setup-go@v2
-        with:
-          go-version: '1.21.7'
+      - name: Setup Go
+        uses: ./.github/actions/setup-go
       - name: Run Check Tests Command
         run: |
           if ! go run integration-tests/e2e_tests_ci_tool/main.go check-tests ./integration-tests .github/e2e-tests.yml; then
@@ -106,10 +104,8 @@ jobs:
     steps:
       - name: Checkout code
         uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2
-      - name: Set up Go
-        uses: actions/setup-go@v2
-        with:
-          go-version: '1.21.7'
+      - name: Setup Go
+        uses: ./.github/actions/setup-go
       - name: Install jq
         run: sudo apt-get install jq
       - name: Generate Docker Tests Matrix
@@ -186,10 +182,8 @@ jobs:
         run: sudo apt-get install -y jq
       - name: Show test configuration
         run: echo '${{ toJson(matrix.tests) }}' | jq .
-      - name: Set up Go
-        uses: actions/setup-go@v2
-        with:
-          go-version: '1.21.7'
+      - name: Setup Go
+        uses: ./.github/actions/setup-go
 
       - name: Create default test config override (use overrides from e2e-tests.yml or defaults)
         # if: ${{ inputs.test_config_base64_override == '' }}

From 69bee7c53aaad664d6dd42b5585711fc9055235d Mon Sep 17 00:00:00 2001
From: lukaszcl <120112546+lukaszcl@users.noreply.github.com>
Date: Fri, 28 Jun 2024 17:31:52 +0200
Subject: [PATCH 108/137] go mod tidy in core/scripts

---
 core/scripts/go.mod | 14 +++++++++-----
 core/scripts/go.sum | 32 ++++++++++++++++++++------------
 2 files changed, 29 insertions(+), 17 deletions(-)

diff --git a/core/scripts/go.mod b/core/scripts/go.mod
index 3adf2183999..f75332faa2b 100644
--- a/core/scripts/go.mod
+++ b/core/scripts/go.mod
@@ -8,8 +8,8 @@ toolchain go1.22.3
 replace github.com/smartcontractkit/chainlink/v2 => ../../
 
 require (
-	github.com/docker/docker v24.0.7+incompatible
-	github.com/docker/go-connections v0.4.0
+	github.com/docker/docker v25.0.2+incompatible
+	github.com/docker/go-connections v0.5.0
 	github.com/ethereum/go-ethereum v1.13.8
 	github.com/gkampitakis/go-snaps v0.5.4
 	github.com/google/go-cmp v0.6.0
@@ -20,7 +20,7 @@ require (
 	github.com/manyminds/api2go v0.0.0-20171030193247-e7b693844a6f
 	github.com/montanaflynn/stats v0.7.1
 	github.com/olekukonko/tablewriter v0.0.5
-	github.com/pelletier/go-toml/v2 v2.1.1
+	github.com/pelletier/go-toml/v2 v2.2.2
 	github.com/prometheus/client_golang v1.17.0
 	github.com/shopspring/decimal v1.3.1
 	github.com/smartcontractkit/chainlink-automation v1.0.4
@@ -28,7 +28,7 @@ require (
 	github.com/smartcontractkit/chainlink-vrf v0.0.0-20240222010609-cd67d123c772
 	github.com/smartcontractkit/chainlink/v2 v2.0.0-00010101000000-000000000000
 	github.com/smartcontractkit/libocr v0.0.0-20240419185742-fd3cab206b2c
-	github.com/spf13/cobra v1.8.0
+	github.com/spf13/cobra v1.8.1
 	github.com/spf13/viper v1.15.0
 	github.com/stretchr/testify v1.9.0
 	github.com/umbracle/ethgo v0.1.3
@@ -91,6 +91,7 @@ require (
 	github.com/confio/ics23/go v0.9.0 // indirect
 	github.com/consensys/bavard v0.1.13 // indirect
 	github.com/consensys/gnark-crypto v0.12.1 // indirect
+	github.com/containerd/log v0.1.0 // indirect
 	github.com/cosmos/btcutil v1.0.5 // indirect
 	github.com/cosmos/cosmos-proto v1.0.0-beta.2 // indirect
 	github.com/cosmos/cosmos-sdk v0.47.4 // indirect
@@ -100,7 +101,7 @@ require (
 	github.com/cosmos/ibc-go/v7 v7.0.1 // indirect
 	github.com/cosmos/ics23/go v0.9.1-0.20221207100636-b1abd8678aab // indirect
 	github.com/cosmos/ledger-cosmos-go v0.12.1 // indirect
-	github.com/cpuguy83/go-md2man/v2 v2.0.3 // indirect
+	github.com/cpuguy83/go-md2man/v2 v2.0.4 // indirect
 	github.com/crate-crypto/go-ipa v0.0.0-20231025140028-3c0104f4b233 // indirect
 	github.com/crate-crypto/go-kzg-4844 v0.7.0 // indirect
 	github.com/danieljoos/wincred v1.1.2 // indirect
@@ -112,6 +113,7 @@ require (
 	github.com/dgraph-io/badger/v2 v2.2007.4 // indirect
 	github.com/dgraph-io/ristretto v0.1.1 // indirect
 	github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13 // indirect
+	github.com/distribution/reference v0.6.0 // indirect
 	github.com/docker/distribution v2.8.2+incompatible // indirect
 	github.com/docker/go-units v0.5.0 // indirect
 	github.com/dominikbraun/graph v0.23.0 // indirect
@@ -121,6 +123,7 @@ require (
 	github.com/esote/minmaxheap v1.0.0 // indirect
 	github.com/ethereum/c-kzg-4844 v0.4.0 // indirect
 	github.com/fatih/color v1.16.0 // indirect
+	github.com/felixge/httpsnoop v1.0.3 // indirect
 	github.com/fsnotify/fsnotify v1.6.0 // indirect
 	github.com/fxamacker/cbor/v2 v2.6.0 // indirect
 	github.com/gabriel-vasile/mimetype v1.4.2 // indirect
@@ -320,6 +323,7 @@ require (
 	go.opencensus.io v0.24.0 // indirect
 	go.opentelemetry.io/contrib/instrumentation/github.com/gin-gonic/gin/otelgin v0.49.0 // indirect
 	go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.46.1 // indirect
+	go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.45.0 // indirect
 	go.opentelemetry.io/otel v1.24.0 // indirect
 	go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.21.0 // indirect
 	go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.21.0 // indirect
diff --git a/core/scripts/go.sum b/core/scripts/go.sum
index d15cf62a754..c8201ea440f 100644
--- a/core/scripts/go.sum
+++ b/core/scripts/go.sum
@@ -275,6 +275,8 @@ github.com/consensys/gnark-crypto v0.12.1 h1:lHH39WuuFgVHONRl3J0LRBtuYdQTumFSDtJ
 github.com/consensys/gnark-crypto v0.12.1/go.mod h1:v2Gy7L/4ZRosZ7Ivs+9SfUDr0f5UlG+EM5t7MPHiLuY=
 github.com/containerd/continuity v0.4.3 h1:6HVkalIp+2u1ZLH1J/pYX2oBVXlJZvh1X1A7bEZ9Su8=
 github.com/containerd/continuity v0.4.3/go.mod h1:F6PTNCKepoxEaXLQp3wDAjygEnImnZ/7o4JzpodfroQ=
+github.com/containerd/log v0.1.0 h1:TCJt7ioM2cr/tfR8GPbGf9/VRAX8D2B4PjzCpfX540I=
+github.com/containerd/log v0.1.0/go.mod h1:VRRf09a7mHDIRezVKTRCrOq78v577GXq3bSa3EhrzVo=
 github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk=
 github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE=
 github.com/coreos/etcd v3.3.13+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE=
@@ -310,8 +312,8 @@ github.com/cosmos/rosetta-sdk-go v0.10.0/go.mod h1:SImAZkb96YbwvoRkzSMQB6noNJXFg
 github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE=
 github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU=
 github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
-github.com/cpuguy83/go-md2man/v2 v2.0.3 h1:qMCsGGgs+MAzDFyp9LpAe1Lqy/fY/qCovCm0qnXZOBM=
-github.com/cpuguy83/go-md2man/v2 v2.0.3/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
+github.com/cpuguy83/go-md2man/v2 v2.0.4 h1:wfIWP927BUkWJb2NmU/kNDYIBTh/ziUX91+lVfRxZq4=
+github.com/cpuguy83/go-md2man/v2 v2.0.4/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
 github.com/crate-crypto/go-ipa v0.0.0-20231025140028-3c0104f4b233 h1:d28BXYi+wUpz1KBmiF9bWrjEMacUEREV6MBi2ODnrfQ=
 github.com/crate-crypto/go-ipa v0.0.0-20231025140028-3c0104f4b233/go.mod h1:geZJZH3SzKCqnz5VT0q/DyIG/tvu/dZk+VIfXicupJs=
 github.com/crate-crypto/go-kzg-4844 v0.7.0 h1:C0vgZRk4q4EZ/JgPfzuSoxdCq3C3mOZMBShovmncxvA=
@@ -359,14 +361,16 @@ github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUn
 github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13 h1:fAjc9m62+UWV/WAFKLNi6ZS0675eEUC9y3AlwSbQu1Y=
 github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw=
 github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no=
+github.com/distribution/reference v0.6.0 h1:0IXCQ5g4/QMHHkarYzh5l+u8T3t73zM5QvfrDyIgxBk=
+github.com/distribution/reference v0.6.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E=
 github.com/docker/cli v24.0.7+incompatible h1:wa/nIwYFW7BVTGa7SWPVyyXU9lgORqUb1xfI36MSkFg=
 github.com/docker/cli v24.0.7+incompatible/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8=
 github.com/docker/distribution v2.8.2+incompatible h1:T3de5rq0dB1j30rp0sA2rER+m322EBzniBPB6ZIzuh8=
 github.com/docker/distribution v2.8.2+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w=
-github.com/docker/docker v24.0.7+incompatible h1:Wo6l37AuwP3JaMnZa226lzVXGA3F9Ig1seQen0cKYlM=
-github.com/docker/docker v24.0.7+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk=
-github.com/docker/go-connections v0.4.0 h1:El9xVISelRB7BuFusrZozjnkIM5YnzCViNKohAFqRJQ=
-github.com/docker/go-connections v0.4.0/go.mod h1:Gbd7IOopHjR8Iph03tsViu4nIes5XhDvyHbTtUxmeec=
+github.com/docker/docker v25.0.2+incompatible h1:/OaKeauroa10K4Nqavw4zlhcDq/WBcPMc5DbjOGgozY=
+github.com/docker/docker v25.0.2+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk=
+github.com/docker/go-connections v0.5.0 h1:USnMq7hx7gwdVZq1L49hLXaFtUdTADjXGp+uj1Br63c=
+github.com/docker/go-connections v0.5.0/go.mod h1:ov60Kzw0kKElRwhNs9UlUHAE/F9Fe6GLaXnqyDdmEXc=
 github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4=
 github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk=
 github.com/dominikbraun/graph v0.23.0 h1:TdZB4pPqCLFxYhdyMFb1TBdFxp8XLcJfTTBQucVPgCo=
@@ -1091,8 +1095,8 @@ github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/9
 github.com/pelletier/go-toml v1.9.5 h1:4yBQzkHv+7BHq2PQUZF3Mx0IYxG7LsP222s7Agd3ve8=
 github.com/pelletier/go-toml v1.9.5/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c=
 github.com/pelletier/go-toml/v2 v2.0.1/go.mod h1:r9LEWfGN8R5k0VXJ+0BkIe7MYkRdwZOjgMj2KwnJFUo=
-github.com/pelletier/go-toml/v2 v2.1.1 h1:LWAJwfNvjQZCFIDKWYQaM62NcYeYViCmWIwmOStowAI=
-github.com/pelletier/go-toml/v2 v2.1.1/go.mod h1:tJU2Z3ZkXwnxa4DPO899bsyIoywizdUvyaeZurnPPDc=
+github.com/pelletier/go-toml/v2 v2.2.2 h1:aYUidT7k73Pcl9nb2gScu7NSrKCSHIDE89b3+6Wq+LM=
+github.com/pelletier/go-toml/v2 v2.2.2/go.mod h1:1t835xjRzz80PqgE6HHgN2JOsmgYu/h4qDAS4n929Rs=
 github.com/petermattis/goid v0.0.0-20180202154549-b0b1615b78e5/go.mod h1:jvVRKCrJTQWu0XVbaOlby/2lO20uSCHEMzzplHXte1o=
 github.com/petermattis/goid v0.0.0-20230317030725-371a4b8eda08 h1:hDSdbBuw3Lefr6R18ax0tZ2BJeNB3NehB3trOwYBsdU=
 github.com/petermattis/goid v0.0.0-20230317030725-371a4b8eda08/go.mod h1:pxMtw7cyUw6B2bRH0ZBANSPg+AoSud1I1iyJHI69jH4=
@@ -1165,8 +1169,8 @@ github.com/rogpeppe/go-internal v1.8.1/go.mod h1:JeRgkft04UBgHMgCIwADu4Pn6Mtm5d4
 github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs=
 github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8=
 github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4=
-github.com/rs/cors v1.8.3 h1:O+qNyWn7Z+F9M0ILBHgMVPuB1xTOucVd5gtaYyXBpRo=
-github.com/rs/cors v1.8.3/go.mod h1:XyqrcTp5zjWr1wsJ8PIRZssZ8b/WMcMf71DJnit4EMU=
+github.com/rs/cors v1.9.0 h1:l9HGsTsHJcvW14Nk7J9KFz8bzeAWXn3CG6bgt7LsrAE=
+github.com/rs/cors v1.9.0/go.mod h1:XyqrcTp5zjWr1wsJ8PIRZssZ8b/WMcMf71DJnit4EMU=
 github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ=
 github.com/rs/zerolog v1.13.0/go.mod h1:YbFCdg8HfsridGWAh22vktObvhZbQsZXe4/zB0OKkWU=
 github.com/rs/zerolog v1.15.0/go.mod h1:xYTKnLHcpfU2225ny5qZjxnj9NvkumZYjJHlAThCjNc=
@@ -1252,8 +1256,8 @@ github.com/spf13/cast v1.6.0 h1:GEiTHELF+vaR5dhz3VqZfFSzZjYbgeKDpBxQVS4GYJ0=
 github.com/spf13/cast v1.6.0/go.mod h1:ancEpBxwJDODSW/UG4rDrAqiKolqNNh2DX3mk86cAdo=
 github.com/spf13/cobra v0.0.5/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tLCHU=
 github.com/spf13/cobra v1.1.1/go.mod h1:WnodtKOvamDL/PwE2M4iKs8aMDBZ5Q5klgD3qfVJQMI=
-github.com/spf13/cobra v1.8.0 h1:7aJaZx1B85qltLMc546zn58BxxfZdR/W22ej9CFoEf0=
-github.com/spf13/cobra v1.8.0/go.mod h1:WXLWApfZ71AjXPya3WOlMsY9yMs7YeiHhFVlvLyhcho=
+github.com/spf13/cobra v1.8.1 h1:e5/vxKd/rZsfSJMUX1agtjeTDf+qv1/JdBF8gg5k9ZM=
+github.com/spf13/cobra v1.8.1/go.mod h1:wHxEcudfqmLYa8iTfL+OuZPbBZkmvliBWKIezN3kD9Y=
 github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo=
 github.com/spf13/jwalterweatherman v1.1.0 h1:ue6voC5bR5F8YxI5S67j9i582FU4Qvo2bmqnqMYADFk=
 github.com/spf13/jwalterweatherman v1.1.0/go.mod h1:aNWZUN0dPAAO/Ljvb5BEdw96iTZ0EXowPYD95IqWIGo=
@@ -1434,6 +1438,8 @@ go.opentelemetry.io/contrib/instrumentation/github.com/gin-gonic/gin/otelgin v0.
 go.opentelemetry.io/contrib/instrumentation/github.com/gin-gonic/gin/otelgin v0.49.0/go.mod h1:1P/02zM3OwkX9uki+Wmxw3a5GVb6KUXRsa7m7bOC9Fg=
 go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.46.1 h1:SpGay3w+nEwMpfVnbqOLH5gY52/foP8RE8UzTZ1pdSE=
 go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.46.1/go.mod h1:4UoMYEZOC0yN/sPGH76KPkkU7zgiEWYWL9vwmbnTJPE=
+go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.45.0 h1:x8Z78aZx8cOF0+Kkazoc7lwUNMGy0LrzEMxTm4BbTxg=
+go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.45.0/go.mod h1:62CPTSry9QZtOaSsE3tOzhx6LzDhHnXJ6xHeMNNiM6Q=
 go.opentelemetry.io/contrib/propagators/b3 v1.24.0 h1:n4xwCdTx3pZqZs2CjS/CUZAs03y3dZcGhC/FepKtEUY=
 go.opentelemetry.io/contrib/propagators/b3 v1.24.0/go.mod h1:k5wRxKRU2uXx2F8uNJ4TaonuEO/V7/5xoz7kdsDACT8=
 go.opentelemetry.io/otel v1.24.0 h1:0LAOdjNmQeSTzGBzduGe/rU4tZhMwL5rWgtp9Ku5Jfo=
@@ -1442,6 +1448,8 @@ go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.21.0 h1:cl5P5/GIfFh4t6xyruO
 go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.21.0/go.mod h1:zgBdWWAu7oEEMC06MMKc5NLbA/1YDXV1sMpSqEeLQLg=
 go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.21.0 h1:tIqheXEFWAZ7O8A7m+J0aPTmpJN3YQ7qetUAdkkkKpk=
 go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.21.0/go.mod h1:nUeKExfxAQVbiVFn32YXpXZZHZ61Cc3s3Rn1pDBGAb0=
+go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.19.0 h1:IeMeyr1aBvBiPVYihXIaeIZba6b8E1bYp7lbdxK8CQg=
+go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.19.0/go.mod h1:oVdCUtjq9MK9BlS7TtucsQwUcXcymNiEDjgDD2jMtZU=
 go.opentelemetry.io/otel/metric v1.24.0 h1:6EhoGWWK28x1fbpA4tYTOWBkPefTDQnb8WSGXlc88kI=
 go.opentelemetry.io/otel/metric v1.24.0/go.mod h1:VYhLe1rFfxuTXLgj4CBiyz+9WYBA8pNGJgDcSFRKBco=
 go.opentelemetry.io/otel/sdk v1.21.0 h1:FTt8qirL1EysG6sTQRZ5TokkU8d0ugCj8htOgThZXQ8=

From ffb8bddf79082cf20f8c0a0e92a34e2cab4554af Mon Sep 17 00:00:00 2001
From: lukaszcl <120112546+lukaszcl@users.noreply.github.com>
Date: Fri, 28 Jun 2024 17:45:25 +0200
Subject: [PATCH 109/137] Fix

---
 .github/workflows/run-automation-benchmark-e2e-tests.yml | 8 +-------
 .github/workflows/run-automation-nightly-e2e-tests.yml   | 6 ------
 .github/workflows/run-nightly-e2e-tests.yml              | 2 +-
 3 files changed, 2 insertions(+), 14 deletions(-)

diff --git a/.github/workflows/run-automation-benchmark-e2e-tests.yml b/.github/workflows/run-automation-benchmark-e2e-tests.yml
index 2e09df5e33a..920ef4642ba 100644
--- a/.github/workflows/run-automation-benchmark-e2e-tests.yml
+++ b/.github/workflows/run-automation-benchmark-e2e-tests.yml
@@ -21,13 +21,7 @@ on:
 jobs:
   call-run-e2e-tests-workflow:
     name: Run E2E Tests
-    uses: ./.github/workflows/run-e2e-tests-reusable-workflow.yml
-    permissions:
-      actions: read
-      checks: write
-      pull-requests: write
-      id-token: write
-      contents: read    
+    uses: ./.github/workflows/run-e2e-tests-reusable-workflow.yml    
     with:
       chainlink_version: develop
       test_ids: ${{ github.event.inputs.test-id }}
diff --git a/.github/workflows/run-automation-nightly-e2e-tests.yml b/.github/workflows/run-automation-nightly-e2e-tests.yml
index 859012016ff..058127926b0 100644
--- a/.github/workflows/run-automation-nightly-e2e-tests.yml
+++ b/.github/workflows/run-automation-nightly-e2e-tests.yml
@@ -13,12 +13,6 @@ jobs:
   call-run-e2e-tests-workflow:
     name: Run E2E Tests
     uses: ./.github/workflows/run-e2e-tests-reusable-workflow.yml
-    permissions:
-      actions: read
-      checks: write
-      pull-requests: write
-      id-token: write
-      contents: read    
     with:
       chainlink_version: develop
       # Select tests to run from .github/e2e_tests.yml based on workflow name 
diff --git a/.github/workflows/run-nightly-e2e-tests.yml b/.github/workflows/run-nightly-e2e-tests.yml
index 0afa59418bc..db1275e41f7 100644
--- a/.github/workflows/run-nightly-e2e-tests.yml
+++ b/.github/workflows/run-nightly-e2e-tests.yml
@@ -2,7 +2,7 @@ name: Run Nightly E2E Tests
 
 on:
   schedule:
-    # Run every night at midnight UTC (0:00 AM) TODO: create separate workflow that uses this one 
+    # Run every night at midnight UTC (0:00 AM)
     - cron: '0 0 * * *'
 
 jobs:

From 6587671864018e4b9b12d5d028e4cfcbc2cf6b27 Mon Sep 17 00:00:00 2001
From: lukaszcl <120112546+lukaszcl@users.noreply.github.com>
Date: Fri, 28 Jun 2024 17:50:50 +0200
Subject: [PATCH 110/137] Remove unready workflows

---
 .../run-automation-benchmark-e2e-tests.yml    | 45 -------------------
 .../run-automation-nightly-e2e-tests.yml      | 35 ---------------
 2 files changed, 80 deletions(-)
 delete mode 100644 .github/workflows/run-automation-benchmark-e2e-tests.yml
 delete mode 100644 .github/workflows/run-automation-nightly-e2e-tests.yml

diff --git a/.github/workflows/run-automation-benchmark-e2e-tests.yml b/.github/workflows/run-automation-benchmark-e2e-tests.yml
deleted file mode 100644
index 920ef4642ba..00000000000
--- a/.github/workflows/run-automation-benchmark-e2e-tests.yml
+++ /dev/null
@@ -1,45 +0,0 @@
-name: Run Automation Benchmark E2E Tests
-
-on:
-  workflow_dispatch:
-    inputs:
-      test-id:
-        description: Select a test to run
-        required: true
-        default: TestAutomationBenchmark
-        type: choice
-        options:
-          - TestAutomationBenchmark
-          - TestAutomationBenchmark_1000Upkeeps_2_1
-
-# TODO:
-# 1. list of tests to select 
-# 2. prepare toml config with grafana url for automation benchmark, take it from github secrets
-# 3. Still support customBase64Config if provided 
-# 4. Add slack notification, have slackMemberID para as input 
-
-jobs:
-  call-run-e2e-tests-workflow:
-    name: Run E2E Tests
-    uses: ./.github/workflows/run-e2e-tests-reusable-workflow.yml    
-    with:
-      chainlink_version: develop
-      test_ids: ${{ github.event.inputs.test-id }}
-      # Select tests to run from .github/e2e_tests.yml based on workflow name 
-      test_workflow: Run Automation Product Nightly E2E Tests
-    secrets:
-      QA_AWS_REGION: ${{ secrets.QA_AWS_REGION }}
-      QA_AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }}
-      QA_AWS_ACCOUNT_NUMBER: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}
-      QA_PYROSCOPE_INSTANCE: ${{ secrets.QA_PYROSCOPE_INSTANCE }}
-      QA_PYROSCOPE_KEY: ${{ secrets.QA_PYROSCOPE_KEY }}
-      QA_KUBECONFIG: ${{ secrets.QA_KUBECONFIG }}
-      GRAFANA_INTERNAL_TENANT_ID: ${{ secrets.GRAFANA_INTERNAL_TENANT_ID }}
-      GRAFANA_INTERNAL_BASIC_AUTH: ${{ secrets.GRAFANA_INTERNAL_BASIC_AUTH }}
-      GRAFANA_INTERNAL_HOST: ${{ secrets.GRAFANA_INTERNAL_HOST }}
-      GRAFANA_INTERNAL_URL_SHORTENER_TOKEN: ${{ secrets.GRAFANA_INTERNAL_URL_SHORTENER_TOKEN }}
-      GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}  
-      AWS_REGION: ${{ secrets.QA_AWS_REGION }}
-      AWS_OIDC_IAM_ROLE_VALIDATION_PROD_ARN: ${{ secrets.AWS_OIDC_IAM_ROLE_VALIDATION_PROD_ARN }}
-      AWS_API_GW_HOST_GRAFANA: ${{ secrets.AWS_API_GW_HOST_GRAFANA }}        
-
diff --git a/.github/workflows/run-automation-nightly-e2e-tests.yml b/.github/workflows/run-automation-nightly-e2e-tests.yml
deleted file mode 100644
index 058127926b0..00000000000
--- a/.github/workflows/run-automation-nightly-e2e-tests.yml
+++ /dev/null
@@ -1,35 +0,0 @@
-name: Run Automation Product Nightly E2E Tests
-
-on:
-  # TODO: uncomment when the workflow is ready
-  # schedule:
-  #   - cron: "0 0 * * *" # Run nightly
-  # push:
-  #   tags:
-  #     - "*"
-  workflow_dispatch:
-
-jobs:
-  call-run-e2e-tests-workflow:
-    name: Run E2E Tests
-    uses: ./.github/workflows/run-e2e-tests-reusable-workflow.yml
-    with:
-      chainlink_version: develop
-      # Select tests to run from .github/e2e_tests.yml based on workflow name 
-      test_workflow: Run Automation Product Nightly E2E Tests
-    secrets:
-      QA_AWS_REGION: ${{ secrets.QA_AWS_REGION }}
-      QA_AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }}
-      QA_AWS_ACCOUNT_NUMBER: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}
-      QA_PYROSCOPE_INSTANCE: ${{ secrets.QA_PYROSCOPE_INSTANCE }}
-      QA_PYROSCOPE_KEY: ${{ secrets.QA_PYROSCOPE_KEY }}
-      QA_KUBECONFIG: ${{ secrets.QA_KUBECONFIG }}
-      GRAFANA_INTERNAL_TENANT_ID: ${{ secrets.GRAFANA_INTERNAL_TENANT_ID }}
-      GRAFANA_INTERNAL_BASIC_AUTH: ${{ secrets.GRAFANA_INTERNAL_BASIC_AUTH }}
-      GRAFANA_INTERNAL_HOST: ${{ secrets.GRAFANA_INTERNAL_HOST }}
-      GRAFANA_INTERNAL_URL_SHORTENER_TOKEN: ${{ secrets.GRAFANA_INTERNAL_URL_SHORTENER_TOKEN }}
-      GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}  
-      AWS_REGION: ${{ secrets.QA_AWS_REGION }}
-      AWS_OIDC_IAM_ROLE_VALIDATION_PROD_ARN: ${{ secrets.AWS_OIDC_IAM_ROLE_VALIDATION_PROD_ARN }}
-      AWS_API_GW_HOST_GRAFANA: ${{ secrets.AWS_API_GW_HOST_GRAFANA }}        
-

From 464c21d10fcec999e0c11a0ed3b258726eaaeb03 Mon Sep 17 00:00:00 2001
From: lukaszcl <120112546+lukaszcl@users.noreply.github.com>
Date: Fri, 28 Jun 2024 17:52:21 +0200
Subject: [PATCH 111/137] Revert updates to core/scripts

---
 core/scripts/go.mod | 14 +++++---------
 core/scripts/go.sum | 32 ++++++++++++--------------------
 2 files changed, 17 insertions(+), 29 deletions(-)

diff --git a/core/scripts/go.mod b/core/scripts/go.mod
index f75332faa2b..3adf2183999 100644
--- a/core/scripts/go.mod
+++ b/core/scripts/go.mod
@@ -8,8 +8,8 @@ toolchain go1.22.3
 replace github.com/smartcontractkit/chainlink/v2 => ../../
 
 require (
-	github.com/docker/docker v25.0.2+incompatible
-	github.com/docker/go-connections v0.5.0
+	github.com/docker/docker v24.0.7+incompatible
+	github.com/docker/go-connections v0.4.0
 	github.com/ethereum/go-ethereum v1.13.8
 	github.com/gkampitakis/go-snaps v0.5.4
 	github.com/google/go-cmp v0.6.0
@@ -20,7 +20,7 @@ require (
 	github.com/manyminds/api2go v0.0.0-20171030193247-e7b693844a6f
 	github.com/montanaflynn/stats v0.7.1
 	github.com/olekukonko/tablewriter v0.0.5
-	github.com/pelletier/go-toml/v2 v2.2.2
+	github.com/pelletier/go-toml/v2 v2.1.1
 	github.com/prometheus/client_golang v1.17.0
 	github.com/shopspring/decimal v1.3.1
 	github.com/smartcontractkit/chainlink-automation v1.0.4
@@ -28,7 +28,7 @@ require (
 	github.com/smartcontractkit/chainlink-vrf v0.0.0-20240222010609-cd67d123c772
 	github.com/smartcontractkit/chainlink/v2 v2.0.0-00010101000000-000000000000
 	github.com/smartcontractkit/libocr v0.0.0-20240419185742-fd3cab206b2c
-	github.com/spf13/cobra v1.8.1
+	github.com/spf13/cobra v1.8.0
 	github.com/spf13/viper v1.15.0
 	github.com/stretchr/testify v1.9.0
 	github.com/umbracle/ethgo v0.1.3
@@ -91,7 +91,6 @@ require (
 	github.com/confio/ics23/go v0.9.0 // indirect
 	github.com/consensys/bavard v0.1.13 // indirect
 	github.com/consensys/gnark-crypto v0.12.1 // indirect
-	github.com/containerd/log v0.1.0 // indirect
 	github.com/cosmos/btcutil v1.0.5 // indirect
 	github.com/cosmos/cosmos-proto v1.0.0-beta.2 // indirect
 	github.com/cosmos/cosmos-sdk v0.47.4 // indirect
@@ -101,7 +100,7 @@ require (
 	github.com/cosmos/ibc-go/v7 v7.0.1 // indirect
 	github.com/cosmos/ics23/go v0.9.1-0.20221207100636-b1abd8678aab // indirect
 	github.com/cosmos/ledger-cosmos-go v0.12.1 // indirect
-	github.com/cpuguy83/go-md2man/v2 v2.0.4 // indirect
+	github.com/cpuguy83/go-md2man/v2 v2.0.3 // indirect
 	github.com/crate-crypto/go-ipa v0.0.0-20231025140028-3c0104f4b233 // indirect
 	github.com/crate-crypto/go-kzg-4844 v0.7.0 // indirect
 	github.com/danieljoos/wincred v1.1.2 // indirect
@@ -113,7 +112,6 @@ require (
 	github.com/dgraph-io/badger/v2 v2.2007.4 // indirect
 	github.com/dgraph-io/ristretto v0.1.1 // indirect
 	github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13 // indirect
-	github.com/distribution/reference v0.6.0 // indirect
 	github.com/docker/distribution v2.8.2+incompatible // indirect
 	github.com/docker/go-units v0.5.0 // indirect
 	github.com/dominikbraun/graph v0.23.0 // indirect
@@ -123,7 +121,6 @@ require (
 	github.com/esote/minmaxheap v1.0.0 // indirect
 	github.com/ethereum/c-kzg-4844 v0.4.0 // indirect
 	github.com/fatih/color v1.16.0 // indirect
-	github.com/felixge/httpsnoop v1.0.3 // indirect
 	github.com/fsnotify/fsnotify v1.6.0 // indirect
 	github.com/fxamacker/cbor/v2 v2.6.0 // indirect
 	github.com/gabriel-vasile/mimetype v1.4.2 // indirect
@@ -323,7 +320,6 @@ require (
 	go.opencensus.io v0.24.0 // indirect
 	go.opentelemetry.io/contrib/instrumentation/github.com/gin-gonic/gin/otelgin v0.49.0 // indirect
 	go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.46.1 // indirect
-	go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.45.0 // indirect
 	go.opentelemetry.io/otel v1.24.0 // indirect
 	go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.21.0 // indirect
 	go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.21.0 // indirect
diff --git a/core/scripts/go.sum b/core/scripts/go.sum
index c8201ea440f..d15cf62a754 100644
--- a/core/scripts/go.sum
+++ b/core/scripts/go.sum
@@ -275,8 +275,6 @@ github.com/consensys/gnark-crypto v0.12.1 h1:lHH39WuuFgVHONRl3J0LRBtuYdQTumFSDtJ
 github.com/consensys/gnark-crypto v0.12.1/go.mod h1:v2Gy7L/4ZRosZ7Ivs+9SfUDr0f5UlG+EM5t7MPHiLuY=
 github.com/containerd/continuity v0.4.3 h1:6HVkalIp+2u1ZLH1J/pYX2oBVXlJZvh1X1A7bEZ9Su8=
 github.com/containerd/continuity v0.4.3/go.mod h1:F6PTNCKepoxEaXLQp3wDAjygEnImnZ/7o4JzpodfroQ=
-github.com/containerd/log v0.1.0 h1:TCJt7ioM2cr/tfR8GPbGf9/VRAX8D2B4PjzCpfX540I=
-github.com/containerd/log v0.1.0/go.mod h1:VRRf09a7mHDIRezVKTRCrOq78v577GXq3bSa3EhrzVo=
 github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk=
 github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE=
 github.com/coreos/etcd v3.3.13+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE=
@@ -312,8 +310,8 @@ github.com/cosmos/rosetta-sdk-go v0.10.0/go.mod h1:SImAZkb96YbwvoRkzSMQB6noNJXFg
 github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE=
 github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU=
 github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
-github.com/cpuguy83/go-md2man/v2 v2.0.4 h1:wfIWP927BUkWJb2NmU/kNDYIBTh/ziUX91+lVfRxZq4=
-github.com/cpuguy83/go-md2man/v2 v2.0.4/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
+github.com/cpuguy83/go-md2man/v2 v2.0.3 h1:qMCsGGgs+MAzDFyp9LpAe1Lqy/fY/qCovCm0qnXZOBM=
+github.com/cpuguy83/go-md2man/v2 v2.0.3/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
 github.com/crate-crypto/go-ipa v0.0.0-20231025140028-3c0104f4b233 h1:d28BXYi+wUpz1KBmiF9bWrjEMacUEREV6MBi2ODnrfQ=
 github.com/crate-crypto/go-ipa v0.0.0-20231025140028-3c0104f4b233/go.mod h1:geZJZH3SzKCqnz5VT0q/DyIG/tvu/dZk+VIfXicupJs=
 github.com/crate-crypto/go-kzg-4844 v0.7.0 h1:C0vgZRk4q4EZ/JgPfzuSoxdCq3C3mOZMBShovmncxvA=
@@ -361,16 +359,14 @@ github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUn
 github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13 h1:fAjc9m62+UWV/WAFKLNi6ZS0675eEUC9y3AlwSbQu1Y=
 github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw=
 github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no=
-github.com/distribution/reference v0.6.0 h1:0IXCQ5g4/QMHHkarYzh5l+u8T3t73zM5QvfrDyIgxBk=
-github.com/distribution/reference v0.6.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E=
 github.com/docker/cli v24.0.7+incompatible h1:wa/nIwYFW7BVTGa7SWPVyyXU9lgORqUb1xfI36MSkFg=
 github.com/docker/cli v24.0.7+incompatible/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8=
 github.com/docker/distribution v2.8.2+incompatible h1:T3de5rq0dB1j30rp0sA2rER+m322EBzniBPB6ZIzuh8=
 github.com/docker/distribution v2.8.2+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w=
-github.com/docker/docker v25.0.2+incompatible h1:/OaKeauroa10K4Nqavw4zlhcDq/WBcPMc5DbjOGgozY=
-github.com/docker/docker v25.0.2+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk=
-github.com/docker/go-connections v0.5.0 h1:USnMq7hx7gwdVZq1L49hLXaFtUdTADjXGp+uj1Br63c=
-github.com/docker/go-connections v0.5.0/go.mod h1:ov60Kzw0kKElRwhNs9UlUHAE/F9Fe6GLaXnqyDdmEXc=
+github.com/docker/docker v24.0.7+incompatible h1:Wo6l37AuwP3JaMnZa226lzVXGA3F9Ig1seQen0cKYlM=
+github.com/docker/docker v24.0.7+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk=
+github.com/docker/go-connections v0.4.0 h1:El9xVISelRB7BuFusrZozjnkIM5YnzCViNKohAFqRJQ=
+github.com/docker/go-connections v0.4.0/go.mod h1:Gbd7IOopHjR8Iph03tsViu4nIes5XhDvyHbTtUxmeec=
 github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4=
 github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk=
 github.com/dominikbraun/graph v0.23.0 h1:TdZB4pPqCLFxYhdyMFb1TBdFxp8XLcJfTTBQucVPgCo=
@@ -1095,8 +1091,8 @@ github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/9
 github.com/pelletier/go-toml v1.9.5 h1:4yBQzkHv+7BHq2PQUZF3Mx0IYxG7LsP222s7Agd3ve8=
 github.com/pelletier/go-toml v1.9.5/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c=
 github.com/pelletier/go-toml/v2 v2.0.1/go.mod h1:r9LEWfGN8R5k0VXJ+0BkIe7MYkRdwZOjgMj2KwnJFUo=
-github.com/pelletier/go-toml/v2 v2.2.2 h1:aYUidT7k73Pcl9nb2gScu7NSrKCSHIDE89b3+6Wq+LM=
-github.com/pelletier/go-toml/v2 v2.2.2/go.mod h1:1t835xjRzz80PqgE6HHgN2JOsmgYu/h4qDAS4n929Rs=
+github.com/pelletier/go-toml/v2 v2.1.1 h1:LWAJwfNvjQZCFIDKWYQaM62NcYeYViCmWIwmOStowAI=
+github.com/pelletier/go-toml/v2 v2.1.1/go.mod h1:tJU2Z3ZkXwnxa4DPO899bsyIoywizdUvyaeZurnPPDc=
 github.com/petermattis/goid v0.0.0-20180202154549-b0b1615b78e5/go.mod h1:jvVRKCrJTQWu0XVbaOlby/2lO20uSCHEMzzplHXte1o=
 github.com/petermattis/goid v0.0.0-20230317030725-371a4b8eda08 h1:hDSdbBuw3Lefr6R18ax0tZ2BJeNB3NehB3trOwYBsdU=
 github.com/petermattis/goid v0.0.0-20230317030725-371a4b8eda08/go.mod h1:pxMtw7cyUw6B2bRH0ZBANSPg+AoSud1I1iyJHI69jH4=
@@ -1169,8 +1165,8 @@ github.com/rogpeppe/go-internal v1.8.1/go.mod h1:JeRgkft04UBgHMgCIwADu4Pn6Mtm5d4
 github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs=
 github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8=
 github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4=
-github.com/rs/cors v1.9.0 h1:l9HGsTsHJcvW14Nk7J9KFz8bzeAWXn3CG6bgt7LsrAE=
-github.com/rs/cors v1.9.0/go.mod h1:XyqrcTp5zjWr1wsJ8PIRZssZ8b/WMcMf71DJnit4EMU=
+github.com/rs/cors v1.8.3 h1:O+qNyWn7Z+F9M0ILBHgMVPuB1xTOucVd5gtaYyXBpRo=
+github.com/rs/cors v1.8.3/go.mod h1:XyqrcTp5zjWr1wsJ8PIRZssZ8b/WMcMf71DJnit4EMU=
 github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ=
 github.com/rs/zerolog v1.13.0/go.mod h1:YbFCdg8HfsridGWAh22vktObvhZbQsZXe4/zB0OKkWU=
 github.com/rs/zerolog v1.15.0/go.mod h1:xYTKnLHcpfU2225ny5qZjxnj9NvkumZYjJHlAThCjNc=
@@ -1256,8 +1252,8 @@ github.com/spf13/cast v1.6.0 h1:GEiTHELF+vaR5dhz3VqZfFSzZjYbgeKDpBxQVS4GYJ0=
 github.com/spf13/cast v1.6.0/go.mod h1:ancEpBxwJDODSW/UG4rDrAqiKolqNNh2DX3mk86cAdo=
 github.com/spf13/cobra v0.0.5/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tLCHU=
 github.com/spf13/cobra v1.1.1/go.mod h1:WnodtKOvamDL/PwE2M4iKs8aMDBZ5Q5klgD3qfVJQMI=
-github.com/spf13/cobra v1.8.1 h1:e5/vxKd/rZsfSJMUX1agtjeTDf+qv1/JdBF8gg5k9ZM=
-github.com/spf13/cobra v1.8.1/go.mod h1:wHxEcudfqmLYa8iTfL+OuZPbBZkmvliBWKIezN3kD9Y=
+github.com/spf13/cobra v1.8.0 h1:7aJaZx1B85qltLMc546zn58BxxfZdR/W22ej9CFoEf0=
+github.com/spf13/cobra v1.8.0/go.mod h1:WXLWApfZ71AjXPya3WOlMsY9yMs7YeiHhFVlvLyhcho=
 github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo=
 github.com/spf13/jwalterweatherman v1.1.0 h1:ue6voC5bR5F8YxI5S67j9i582FU4Qvo2bmqnqMYADFk=
 github.com/spf13/jwalterweatherman v1.1.0/go.mod h1:aNWZUN0dPAAO/Ljvb5BEdw96iTZ0EXowPYD95IqWIGo=
@@ -1438,8 +1434,6 @@ go.opentelemetry.io/contrib/instrumentation/github.com/gin-gonic/gin/otelgin v0.
 go.opentelemetry.io/contrib/instrumentation/github.com/gin-gonic/gin/otelgin v0.49.0/go.mod h1:1P/02zM3OwkX9uki+Wmxw3a5GVb6KUXRsa7m7bOC9Fg=
 go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.46.1 h1:SpGay3w+nEwMpfVnbqOLH5gY52/foP8RE8UzTZ1pdSE=
 go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.46.1/go.mod h1:4UoMYEZOC0yN/sPGH76KPkkU7zgiEWYWL9vwmbnTJPE=
-go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.45.0 h1:x8Z78aZx8cOF0+Kkazoc7lwUNMGy0LrzEMxTm4BbTxg=
-go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.45.0/go.mod h1:62CPTSry9QZtOaSsE3tOzhx6LzDhHnXJ6xHeMNNiM6Q=
 go.opentelemetry.io/contrib/propagators/b3 v1.24.0 h1:n4xwCdTx3pZqZs2CjS/CUZAs03y3dZcGhC/FepKtEUY=
 go.opentelemetry.io/contrib/propagators/b3 v1.24.0/go.mod h1:k5wRxKRU2uXx2F8uNJ4TaonuEO/V7/5xoz7kdsDACT8=
 go.opentelemetry.io/otel v1.24.0 h1:0LAOdjNmQeSTzGBzduGe/rU4tZhMwL5rWgtp9Ku5Jfo=
@@ -1448,8 +1442,6 @@ go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.21.0 h1:cl5P5/GIfFh4t6xyruO
 go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.21.0/go.mod h1:zgBdWWAu7oEEMC06MMKc5NLbA/1YDXV1sMpSqEeLQLg=
 go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.21.0 h1:tIqheXEFWAZ7O8A7m+J0aPTmpJN3YQ7qetUAdkkkKpk=
 go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.21.0/go.mod h1:nUeKExfxAQVbiVFn32YXpXZZHZ61Cc3s3Rn1pDBGAb0=
-go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.19.0 h1:IeMeyr1aBvBiPVYihXIaeIZba6b8E1bYp7lbdxK8CQg=
-go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.19.0/go.mod h1:oVdCUtjq9MK9BlS7TtucsQwUcXcymNiEDjgDD2jMtZU=
 go.opentelemetry.io/otel/metric v1.24.0 h1:6EhoGWWK28x1fbpA4tYTOWBkPefTDQnb8WSGXlc88kI=
 go.opentelemetry.io/otel/metric v1.24.0/go.mod h1:VYhLe1rFfxuTXLgj4CBiyz+9WYBA8pNGJgDcSFRKBco=
 go.opentelemetry.io/otel/sdk v1.21.0 h1:FTt8qirL1EysG6sTQRZ5TokkU8d0ugCj8htOgThZXQ8=

From a0470ba6fc87a0761fce4d953573ea8015537558 Mon Sep 17 00:00:00 2001
From: lukaszcl <120112546+lukaszcl@users.noreply.github.com>
Date: Fri, 28 Jun 2024 17:52:46 +0200
Subject: [PATCH 112/137] Revert updates to go.mod

---
 go.mod |  9 +++------
 go.sum | 24 ++++++++++++------------
 2 files changed, 15 insertions(+), 18 deletions(-)

diff --git a/go.mod b/go.mod
index 2773be38964..5f2a2c175db 100644
--- a/go.mod
+++ b/go.mod
@@ -58,7 +58,7 @@ require (
 	github.com/onsi/gomega v1.30.0
 	github.com/patrickmn/go-cache v2.1.0+incompatible
 	github.com/pelletier/go-toml v1.9.5
-	github.com/pelletier/go-toml/v2 v2.2.2
+	github.com/pelletier/go-toml/v2 v2.1.1
 	github.com/pkg/errors v0.9.1
 	github.com/pressly/goose/v3 v3.16.0
 	github.com/prometheus/client_golang v1.17.0
@@ -165,7 +165,7 @@ require (
 	github.com/cosmos/ibc-go/v7 v7.0.1 // indirect
 	github.com/cosmos/ics23/go v0.9.1-0.20221207100636-b1abd8678aab // indirect
 	github.com/cosmos/ledger-cosmos-go v0.12.1 // indirect
-	github.com/cpuguy83/go-md2man/v2 v2.0.4 // indirect
+	github.com/cpuguy83/go-md2man/v2 v2.0.3 // indirect
 	github.com/crate-crypto/go-ipa v0.0.0-20231025140028-3c0104f4b233 // indirect
 	github.com/crate-crypto/go-kzg-4844 v0.7.0 // indirect
 	github.com/danieljoos/wincred v1.1.2 // indirect
@@ -177,8 +177,6 @@ require (
 	github.com/dgraph-io/ristretto v0.1.1 // indirect
 	github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13 // indirect
 	github.com/docker/distribution v2.8.2+incompatible // indirect
-	github.com/docker/docker v25.0.2+incompatible // indirect
-	github.com/docker/go-connections v0.5.0 // indirect
 	github.com/dustin/go-humanize v1.0.1 // indirect
 	github.com/dvsekhvalnov/jose2go v1.7.0 // indirect
 	github.com/ethereum/c-kzg-4844 v0.4.0 // indirect
@@ -280,7 +278,6 @@ require (
 	github.com/prometheus/procfs v0.12.0 // indirect
 	github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 // indirect
 	github.com/rivo/uniseg v0.4.4 // indirect
-	github.com/rs/cors v1.9.0 // indirect
 	github.com/russross/blackfriday/v2 v2.1.0 // indirect
 	github.com/santhosh-tekuri/jsonschema/v5 v5.3.1 // indirect
 	github.com/sasha-s/go-deadlock v0.3.1 // indirect
@@ -288,7 +285,7 @@ require (
 	github.com/shirou/gopsutil v3.21.11+incompatible // indirect
 	github.com/sirupsen/logrus v1.9.3 // indirect
 	github.com/spf13/afero v1.9.5 // indirect
-	github.com/spf13/cobra v1.8.1 // indirect
+	github.com/spf13/cobra v1.8.0 // indirect
 	github.com/spf13/jwalterweatherman v1.1.0 // indirect
 	github.com/spf13/pflag v1.0.5 // indirect
 	github.com/spf13/viper v1.15.0 // indirect
diff --git a/go.sum b/go.sum
index ba921efe531..da0951445f7 100644
--- a/go.sum
+++ b/go.sum
@@ -296,8 +296,8 @@ github.com/cosmos/rosetta-sdk-go v0.10.0/go.mod h1:SImAZkb96YbwvoRkzSMQB6noNJXFg
 github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE=
 github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU=
 github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
-github.com/cpuguy83/go-md2man/v2 v2.0.4 h1:wfIWP927BUkWJb2NmU/kNDYIBTh/ziUX91+lVfRxZq4=
-github.com/cpuguy83/go-md2man/v2 v2.0.4/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
+github.com/cpuguy83/go-md2man/v2 v2.0.3 h1:qMCsGGgs+MAzDFyp9LpAe1Lqy/fY/qCovCm0qnXZOBM=
+github.com/cpuguy83/go-md2man/v2 v2.0.3/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
 github.com/crate-crypto/go-ipa v0.0.0-20231025140028-3c0104f4b233 h1:d28BXYi+wUpz1KBmiF9bWrjEMacUEREV6MBi2ODnrfQ=
 github.com/crate-crypto/go-ipa v0.0.0-20231025140028-3c0104f4b233/go.mod h1:geZJZH3SzKCqnz5VT0q/DyIG/tvu/dZk+VIfXicupJs=
 github.com/crate-crypto/go-kzg-4844 v0.7.0 h1:C0vgZRk4q4EZ/JgPfzuSoxdCq3C3mOZMBShovmncxvA=
@@ -345,10 +345,10 @@ github.com/docker/cli v24.0.7+incompatible h1:wa/nIwYFW7BVTGa7SWPVyyXU9lgORqUb1x
 github.com/docker/cli v24.0.7+incompatible/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8=
 github.com/docker/distribution v2.8.2+incompatible h1:T3de5rq0dB1j30rp0sA2rER+m322EBzniBPB6ZIzuh8=
 github.com/docker/distribution v2.8.2+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w=
-github.com/docker/docker v25.0.2+incompatible h1:/OaKeauroa10K4Nqavw4zlhcDq/WBcPMc5DbjOGgozY=
-github.com/docker/docker v25.0.2+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk=
-github.com/docker/go-connections v0.5.0 h1:USnMq7hx7gwdVZq1L49hLXaFtUdTADjXGp+uj1Br63c=
-github.com/docker/go-connections v0.5.0/go.mod h1:ov60Kzw0kKElRwhNs9UlUHAE/F9Fe6GLaXnqyDdmEXc=
+github.com/docker/docker v24.0.7+incompatible h1:Wo6l37AuwP3JaMnZa226lzVXGA3F9Ig1seQen0cKYlM=
+github.com/docker/docker v24.0.7+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk=
+github.com/docker/go-connections v0.4.0 h1:El9xVISelRB7BuFusrZozjnkIM5YnzCViNKohAFqRJQ=
+github.com/docker/go-connections v0.4.0/go.mod h1:Gbd7IOopHjR8Iph03tsViu4nIes5XhDvyHbTtUxmeec=
 github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4=
 github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk=
 github.com/dominikbraun/graph v0.23.0 h1:TdZB4pPqCLFxYhdyMFb1TBdFxp8XLcJfTTBQucVPgCo=
@@ -1048,8 +1048,8 @@ github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/9
 github.com/pelletier/go-toml v1.9.5 h1:4yBQzkHv+7BHq2PQUZF3Mx0IYxG7LsP222s7Agd3ve8=
 github.com/pelletier/go-toml v1.9.5/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c=
 github.com/pelletier/go-toml/v2 v2.0.1/go.mod h1:r9LEWfGN8R5k0VXJ+0BkIe7MYkRdwZOjgMj2KwnJFUo=
-github.com/pelletier/go-toml/v2 v2.2.2 h1:aYUidT7k73Pcl9nb2gScu7NSrKCSHIDE89b3+6Wq+LM=
-github.com/pelletier/go-toml/v2 v2.2.2/go.mod h1:1t835xjRzz80PqgE6HHgN2JOsmgYu/h4qDAS4n929Rs=
+github.com/pelletier/go-toml/v2 v2.1.1 h1:LWAJwfNvjQZCFIDKWYQaM62NcYeYViCmWIwmOStowAI=
+github.com/pelletier/go-toml/v2 v2.1.1/go.mod h1:tJU2Z3ZkXwnxa4DPO899bsyIoywizdUvyaeZurnPPDc=
 github.com/petermattis/goid v0.0.0-20180202154549-b0b1615b78e5/go.mod h1:jvVRKCrJTQWu0XVbaOlby/2lO20uSCHEMzzplHXte1o=
 github.com/petermattis/goid v0.0.0-20230317030725-371a4b8eda08 h1:hDSdbBuw3Lefr6R18ax0tZ2BJeNB3NehB3trOwYBsdU=
 github.com/petermattis/goid v0.0.0-20230317030725-371a4b8eda08/go.mod h1:pxMtw7cyUw6B2bRH0ZBANSPg+AoSud1I1iyJHI69jH4=
@@ -1122,8 +1122,8 @@ github.com/rogpeppe/go-internal v1.8.1/go.mod h1:JeRgkft04UBgHMgCIwADu4Pn6Mtm5d4
 github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs=
 github.com/rogpeppe/go-internal v1.11.0 h1:cWPaGQEPrBb5/AsnsZesgZZ9yb1OQ+GOISoDNXVBh4M=
 github.com/rogpeppe/go-internal v1.11.0/go.mod h1:ddIwULY96R17DhadqLgMfk9H9tvdUzkipdSkR5nkCZA=
-github.com/rs/cors v1.9.0 h1:l9HGsTsHJcvW14Nk7J9KFz8bzeAWXn3CG6bgt7LsrAE=
-github.com/rs/cors v1.9.0/go.mod h1:XyqrcTp5zjWr1wsJ8PIRZssZ8b/WMcMf71DJnit4EMU=
+github.com/rs/cors v1.8.3 h1:O+qNyWn7Z+F9M0ILBHgMVPuB1xTOucVd5gtaYyXBpRo=
+github.com/rs/cors v1.8.3/go.mod h1:XyqrcTp5zjWr1wsJ8PIRZssZ8b/WMcMf71DJnit4EMU=
 github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ=
 github.com/rs/zerolog v1.13.0/go.mod h1:YbFCdg8HfsridGWAh22vktObvhZbQsZXe4/zB0OKkWU=
 github.com/rs/zerolog v1.15.0/go.mod h1:xYTKnLHcpfU2225ny5qZjxnj9NvkumZYjJHlAThCjNc=
@@ -1210,8 +1210,8 @@ github.com/spf13/cast v1.6.0 h1:GEiTHELF+vaR5dhz3VqZfFSzZjYbgeKDpBxQVS4GYJ0=
 github.com/spf13/cast v1.6.0/go.mod h1:ancEpBxwJDODSW/UG4rDrAqiKolqNNh2DX3mk86cAdo=
 github.com/spf13/cobra v0.0.5/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tLCHU=
 github.com/spf13/cobra v1.1.1/go.mod h1:WnodtKOvamDL/PwE2M4iKs8aMDBZ5Q5klgD3qfVJQMI=
-github.com/spf13/cobra v1.8.1 h1:e5/vxKd/rZsfSJMUX1agtjeTDf+qv1/JdBF8gg5k9ZM=
-github.com/spf13/cobra v1.8.1/go.mod h1:wHxEcudfqmLYa8iTfL+OuZPbBZkmvliBWKIezN3kD9Y=
+github.com/spf13/cobra v1.8.0 h1:7aJaZx1B85qltLMc546zn58BxxfZdR/W22ej9CFoEf0=
+github.com/spf13/cobra v1.8.0/go.mod h1:WXLWApfZ71AjXPya3WOlMsY9yMs7YeiHhFVlvLyhcho=
 github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo=
 github.com/spf13/jwalterweatherman v1.1.0 h1:ue6voC5bR5F8YxI5S67j9i582FU4Qvo2bmqnqMYADFk=
 github.com/spf13/jwalterweatherman v1.1.0/go.mod h1:aNWZUN0dPAAO/Ljvb5BEdw96iTZ0EXowPYD95IqWIGo=

From f23cee3a55700359d81ca93fca473143a785017e Mon Sep 17 00:00:00 2001
From: lukaszcl <120112546+lukaszcl@users.noreply.github.com>
Date: Fri, 28 Jun 2024 17:55:02 +0200
Subject: [PATCH 113/137] Remove unused script

---
 .../check-e2e-tests-in-nightly-workflow.go    | 79 -------------------
 1 file changed, 79 deletions(-)
 delete mode 100644 .github/scripts/check-e2e-tests-in-nightly-workflow.go

diff --git a/.github/scripts/check-e2e-tests-in-nightly-workflow.go b/.github/scripts/check-e2e-tests-in-nightly-workflow.go
deleted file mode 100644
index a629c296cef..00000000000
--- a/.github/scripts/check-e2e-tests-in-nightly-workflow.go
+++ /dev/null
@@ -1,79 +0,0 @@
-package main
-
-import (
-	"fmt"
-	"io/ioutil"
-	"os"
-	"strings"
-
-	"gopkg.in/yaml.v3"
-)
-
-// JobConfig represents the structure of the jobs in the YAML file.
-type JobConfig struct {
-	Jobs map[string]struct {
-		Strategy struct {
-			Matrix struct {
-				Test []struct {
-					Path     string `yaml:"path"`
-					TestOpts string `yaml:"testOpts"`
-				} `yaml:"test"`
-			} `yaml:"matrix"`
-		} `yaml:"strategy"`
-	} `yaml:"jobs"`
-}
-
-func main() {
-	// Check command-line arguments
-	if len(os.Args) != 4 {
-		fmt.Println("Usage: go run check_test_in_pipeline.go <yaml file> <test name> <test file path>")
-		os.Exit(1)
-	}
-	filename := os.Args[1]
-	testName := os.Args[2]
-	testFilePath := os.Args[3]
-
-	// Read the YAML file
-	data, err := ioutil.ReadFile(filename)
-	if err != nil {
-		fmt.Printf("Error reading YAML file: %s\n", err)
-		os.Exit(1)
-	}
-
-	// Parse the YAML
-	var config JobConfig
-	err = yaml.Unmarshal(data, &config)
-	if err != nil {
-		fmt.Printf("Error parsing YAML: %s\n", err)
-		os.Exit(1)
-	}
-
-	// Look for the test in the configuration
-	found := false
-	for _, job := range config.Jobs {
-		for _, test := range job.Strategy.Matrix.Test {
-			if test.Path == testFilePath {
-				// Check if -test.run is specified and if it contains the testName
-				if strings.Contains(test.TestOpts, "-test.run") {
-					if strings.Contains(test.TestOpts, testName) {
-						fmt.Printf("Test '%s' is specifically included in the nightly CI pipeline for file '%s'.\n", testName, testFilePath)
-						found = true
-						break
-					}
-				} else {
-					// If -test.run is not specified, assume all tests in the file are included
-					fmt.Printf("All tests in '%s' are included in the nightly CI pipeline, including '%s'.\n", testFilePath, testName)
-					found = true
-					break
-				}
-			}
-		}
-		if found {
-			break
-		}
-	}
-
-	if !found {
-		fmt.Printf("Test '%s' is NOT included in the nightly CI pipeline for file '%s'.\n", testName, testFilePath)
-	}
-}

From 5b0d11d5507e4bcd62d8c97e85edacd0a9ea1441 Mon Sep 17 00:00:00 2001
From: lukaszcl <120112546+lukaszcl@users.noreply.github.com>
Date: Mon, 1 Jul 2024 09:57:50 +0200
Subject: [PATCH 114/137] Add setup-gap id

---
 .github/workflows/run-e2e-tests-reusable-workflow.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.github/workflows/run-e2e-tests-reusable-workflow.yml b/.github/workflows/run-e2e-tests-reusable-workflow.yml
index e608589c7e5..ca8a58f752e 100644
--- a/.github/workflows/run-e2e-tests-reusable-workflow.yml
+++ b/.github/workflows/run-e2e-tests-reusable-workflow.yml
@@ -254,6 +254,7 @@ jobs:
 
       - name: Setup GAP for Grafana
         uses: smartcontractkit/.github/actions/setup-gap@6c9d62fdad050cfb8b59376ded291f1350705944 # setup-gap@0.2.2
+        id: setup-gap
         with:
           aws-region: ${{ secrets.AWS_REGION }}
           aws-role-arn: ${{ secrets.AWS_OIDC_IAM_ROLE_VALIDATION_PROD_ARN }}

From 74c114388d1d667e471ee0462ba63db1d9995c78 Mon Sep 17 00:00:00 2001
From: lukaszcl <120112546+lukaszcl@users.noreply.github.com>
Date: Mon, 1 Jul 2024 09:59:26 +0200
Subject: [PATCH 115/137] Remove unused steps

---
 .../run-e2e-tests-reusable-workflow.yml       | 47 +------------------
 1 file changed, 1 insertion(+), 46 deletions(-)

diff --git a/.github/workflows/run-e2e-tests-reusable-workflow.yml b/.github/workflows/run-e2e-tests-reusable-workflow.yml
index ca8a58f752e..d7a87e6be78 100644
--- a/.github/workflows/run-e2e-tests-reusable-workflow.yml
+++ b/.github/workflows/run-e2e-tests-reusable-workflow.yml
@@ -206,52 +206,6 @@ jobs:
           grafana_dashboard_url: /d/ddf75041-1e39-42af-aa46-361fe4c36e9e/ci-e2e-tests-logs
           grafana_bearer_token: ${{ secrets.GRAFANA_INTERNAL_URL_SHORTENER_TOKEN }}
 
-      # TODO: Remove this step since we do not have to mask secrets anymore
-      # - name: Mask secrets in test config base64 override
-      #   id: mask-secrets-in-test_config_base64_override
-      #   if: ${{ inputs.test_config_base64_override != '' }}
-      #   run: "to do"
-          # - name: Remove secrets from test config base64 override
-          #   id: mask-in
-          # - name: Set outputs
-          #   id: set-outputs
-          #   run: |
-          #     echo "::set-output name=base64ConfigNoSecrets::$BASE64_CONFIG_OVERRIDE"
-
-      # TODO: Remove this step since we do not have to mask or override secrets in custom test config anymore 
-      # - name: Override secrets in custom test config
-      #   if: ${{ inputs.test_config_base64_override != '' }}          
-      #   run: |
-      #     # Show what parts of the test config are being overridden
-      #     go run main.go test-config override \
-      #       --dry-run \
-      #       --from-base64-config ${{ inputs.test_config_base64_override }} \
-      #       --perycope-server-url ${{ secrets.QA_PYROSCOPE_INSTANCE }} \
-      #       --peryscope-key ${{ secrets.QA_PYROSCOPE_KEY }} \
-      #       --loki-endpoint https://${{ secrets.GRAFANA_INTERNAL_HOST }}/loki/api/v1/push \
-      #       --loki-tenant-id ${{ secrets.GRAFANA_INTERNAL_TENANT_ID }} \
-      #       --loki-basic-auth ${{ secrets.GRAFANA_INTERNAL_BASIC_AUTH }} \
-      #       --grafana-url "http://localhost:8080/primary" \
-      #       --grafana-dashboard-url "/d/ddf75041-1e39-42af-aa46-361fe4c36e9e/ci-e2e-tests-logs" \
-      #       --grafana-bearer-token ${{ secrets.GRAFANA_INTERNAL_URL_SHORTENER_TOKEN }}
-
-      #     # Override custom test config with CI secrets
-      #     config_override=$(go run main.go test-config override \
-      #       --from-base64-config ${{ inputs.test_config_base64_override }} \
-      #       --perycope-server-url ${{ secrets.QA_PYROSCOPE_INSTANCE }} \
-      #       --peryscope-key ${{ secrets.QA_PYROSCOPE_KEY }} \
-      #       --loki-endpoint https://${{ secrets.GRAFANA_INTERNAL_HOST }}/loki/api/v1/push \
-      #       --loki-tenant-id ${{ secrets.GRAFANA_INTERNAL_TENANT_ID }} \
-      #       --loki-basic-auth ${{ secrets.GRAFANA_INTERNAL_BASIC_AUTH }} \
-      #       --grafana-url "http://localhost:8080/primary" \
-      #       --grafana-dashboard-url "/d/ddf75041-1e39-42af-aa46-361fe4c36e9e/ci-e2e-tests-logs" \
-      #       --grafana-bearer-token ${{ secrets.GRAFANA_INTERNAL_URL_SHORTENER_TOKEN }}
-      #     )
-
-      #     BASE64_CONFIG_OVERRIDE=$(echo "$config_override" | base64 -w 0)
-      #     echo ::add-mask::$BASE64_CONFIG_OVERRIDE
-      #     echo "BASE64_CONFIG_OVERRIDE=$BASE64_CONFIG_OVERRIDE" >> $GITHUB_ENV
-
       - name: Setup GAP for Grafana
         uses: smartcontractkit/.github/actions/setup-gap@6c9d62fdad050cfb8b59376ded291f1350705944 # setup-gap@0.2.2
         id: setup-gap
@@ -260,6 +214,7 @@ jobs:
           aws-role-arn: ${{ secrets.AWS_OIDC_IAM_ROLE_VALIDATION_PROD_ARN }}
           api-gateway-host: ${{ secrets.AWS_API_GW_HOST_GRAFANA }}
           duplicate-authorization-header: "true"
+          
       - name: Run tests
         uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/run-tests@af92c5fae8dcf1659201e907db82d221fc304b94 # v2.3.21
         with:

From 0e848dbc7f942f8ee1cd037f93eeb76e2d8451ac Mon Sep 17 00:00:00 2001
From: lukaszcl <120112546+lukaszcl@users.noreply.github.com>
Date: Mon, 1 Jul 2024 09:59:58 +0200
Subject: [PATCH 116/137] test nightly workflow

---
 .github/workflows/run-nightly-e2e-tests.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.github/workflows/run-nightly-e2e-tests.yml b/.github/workflows/run-nightly-e2e-tests.yml
index db1275e41f7..93af23d9502 100644
--- a/.github/workflows/run-nightly-e2e-tests.yml
+++ b/.github/workflows/run-nightly-e2e-tests.yml
@@ -4,6 +4,7 @@ on:
   schedule:
     # Run every night at midnight UTC (0:00 AM)
     - cron: '0 0 * * *'
+  workflow_dispatch:
 
 jobs:
   call-run-e2e-tests-workflow:

From 63e2b2cb5a873f85ebcfeccf1afdfdcf73cae350 Mon Sep 17 00:00:00 2001
From: lukaszcl <120112546+lukaszcl@users.noreply.github.com>
Date: Mon, 1 Jul 2024 10:11:53 +0200
Subject: [PATCH 117/137] Add missing env vars in remote runner and fix lack of
 logs

---
 .github/workflows/run-e2e-tests-reusable-workflow.yml | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/run-e2e-tests-reusable-workflow.yml b/.github/workflows/run-e2e-tests-reusable-workflow.yml
index d7a87e6be78..bd5bff3f68e 100644
--- a/.github/workflows/run-e2e-tests-reusable-workflow.yml
+++ b/.github/workflows/run-e2e-tests-reusable-workflow.yml
@@ -214,7 +214,7 @@ jobs:
           aws-role-arn: ${{ secrets.AWS_OIDC_IAM_ROLE_VALIDATION_PROD_ARN }}
           api-gateway-host: ${{ secrets.AWS_API_GW_HOST_GRAFANA }}
           duplicate-authorization-header: "true"
-          
+
       - name: Run tests
         uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/run-tests@af92c5fae8dcf1659201e907db82d221fc304b94 # v2.3.21
         with:
@@ -344,6 +344,8 @@ jobs:
           # We can comment these out when we have a stable soak test and aren't worried about resource consumption
           TEST_UPLOAD_CPU_PROFILE: true
           TEST_UPLOAD_MEM_PROFILE: true
+          TEST_LOG_LEVEL: debug
+          REF_NAME: ${{ github.head_ref || github.ref_name }}
         with:
           test_command_to_run: ${{ matrix.tests.testCmd }}
           test_download_vendor_packages_command: make gomod

From d4a0a240755a06d0321268699905db0bcdf0d2eb Mon Sep 17 00:00:00 2001
From: lukaszcl <120112546+lukaszcl@users.noreply.github.com>
Date: Mon, 1 Jul 2024 10:16:07 +0200
Subject: [PATCH 118/137] bump setup gap

---
 .github/workflows/run-e2e-tests-reusable-workflow.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/run-e2e-tests-reusable-workflow.yml b/.github/workflows/run-e2e-tests-reusable-workflow.yml
index bd5bff3f68e..c22d70a723f 100644
--- a/.github/workflows/run-e2e-tests-reusable-workflow.yml
+++ b/.github/workflows/run-e2e-tests-reusable-workflow.yml
@@ -207,7 +207,7 @@ jobs:
           grafana_bearer_token: ${{ secrets.GRAFANA_INTERNAL_URL_SHORTENER_TOKEN }}
 
       - name: Setup GAP for Grafana
-        uses: smartcontractkit/.github/actions/setup-gap@6c9d62fdad050cfb8b59376ded291f1350705944 # setup-gap@0.2.2
+        uses: smartcontractkit/.github/actions/setup-gap@d316f66b2990ea4daa479daa3de6fc92b00f863e # setup-gap@0.3.2
         id: setup-gap
         with:
           aws-region: ${{ secrets.AWS_REGION }}

From f351eeb7f2e8e77d23e6574f03581af38cd5a936 Mon Sep 17 00:00:00 2001
From: lukaszcl <120112546+lukaszcl@users.noreply.github.com>
Date: Mon, 1 Jul 2024 10:20:21 +0200
Subject: [PATCH 119/137] Fix set-output

---
 .../run-e2e-tests-reusable-workflow.yml          | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/.github/workflows/run-e2e-tests-reusable-workflow.yml b/.github/workflows/run-e2e-tests-reusable-workflow.yml
index c22d70a723f..7db990a9675 100644
--- a/.github/workflows/run-e2e-tests-reusable-workflow.yml
+++ b/.github/workflows/run-e2e-tests-reusable-workflow.yml
@@ -115,7 +115,7 @@ jobs:
           MATRIX_JSON=$(go run main.go filter --file ../../.github/e2e-tests.yml --test-type 'docker' --test-ids '${{ inputs.test_ids }}' --workflow '${{ inputs.test_workflow }}')
           echo "Docker tests:"
           echo "$MATRIX_JSON" | jq
-          echo "::set-output name=matrix::$MATRIX_JSON"
+          echo "matrix=$MATRIX_JSON" >> $GITHUB_OUTPUT
       - name: Generate K8s Tests Matrix
         id: set-k8s-runner-matrix
         run: |
@@ -123,7 +123,7 @@ jobs:
           MATRIX_JSON=$(go run main.go filter --file ../../.github/e2e-tests.yml --test-type 'k8s-remote-runner' --test-ids '${{ inputs.test_ids }}' --workflow '${{ inputs.test_workflow }}')
           echo "K8s tests:"
           echo "$MATRIX_JSON" | jq
-          echo "::set-output name=matrix::$MATRIX_JSON"
+          echo "matrix=$MATRIX_JSON" >> $GITHUB_OUTPUT
       - name: Check Test Matrices
         id: check-matrices
         run: |
@@ -137,14 +137,14 @@ jobs:
           fi
 
           if [[ "$DOCKER_MATRIX_EMPTY" == "true" ]]; then
-            echo "::set-output name=run-docker-tests::false"
+            echo "run-docker-tests=false" >> $GITHUB_OUTPUT
           else
-            echo "::set-output name=run-docker-tests::true"
+            echo "run-docker-tests=true" >> $GITHUB_OUTPUT
           fi
           if [[ "$K8S_MATRIX_EMPTY" == "true" ]]; then
-            echo "::set-output name=run-k8s-tests::false"
+            echo "run-k8s-tests=false" >> $GITHUB_OUTPUT
           else
-            echo "::set-output name=run-k8s-tests::true"
+            echo "run-k8s-tests=true" >> $GITHUB_OUTPUT
           fi
 
           # Check if both matrices are empty
@@ -271,9 +271,9 @@ jobs:
         id: set-remote-runner-version
         run: |
           if [[ -z "${{ inputs.with_existing_remote_runner_version }}" ]]; then
-            echo "::set-output name=remote-runner-version::${{ github.sha }}"
+            echo "remote-runner-version=${{ github.sha }}" >> $GITHUB_OUTPUT
           else
-            echo "::set-output name=remote-runner-version::${{ inputs.with_existing_remote_runner_version }}"
+            echo "remote-runner-version=${{ inputs.with_existing_remote_runner_version }}" >> $GITHUB_OUTPUT
           fi
 
   run-k8s-runner-tests:

From 4f61a7a3cea1b6a2fb2cd42da2319dacdcff75c0 Mon Sep 17 00:00:00 2001
From: lukaszcl <120112546+lukaszcl@users.noreply.github.com>
Date: Mon, 1 Jul 2024 10:40:18 +0200
Subject: [PATCH 120/137] update comment

---
 .github/workflows/run-e2e-tests-reusable-workflow.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/run-e2e-tests-reusable-workflow.yml b/.github/workflows/run-e2e-tests-reusable-workflow.yml
index 7db990a9675..21ef5614d81 100644
--- a/.github/workflows/run-e2e-tests-reusable-workflow.yml
+++ b/.github/workflows/run-e2e-tests-reusable-workflow.yml
@@ -16,7 +16,7 @@ on:
         description: 'Run tests by workflow name. Example: "Run Nightly E2E Tests"'
         required: false
         type: string
-      # TODO: Enable once Test Config does not have secrets. Related ticket https://smartcontract-it.atlassian.net/browse/TT-1283
+      # Enable once Test Config does not have any secrets. Related ticket https://smartcontract-it.atlassian.net/browse/TT-1283
       # test_config_base64_override:
       #   description: 'This is an unsafe way to pass custom base64 test config. It cannot contain any secrets as this value is exposed in Github logs (Set up job -> Inputs) when the workflow is called! The caller workflow must make sure to remove secrets from this input.'
       #   required: false

From 5ceb64b28cac6bdd28558a0088128f2183db9f06 Mon Sep 17 00:00:00 2001
From: lukaszcl <120112546+lukaszcl@users.noreply.github.com>
Date: Mon, 1 Jul 2024 10:40:54 +0200
Subject: [PATCH 121/137] update test name

---
 .github/e2e-tests.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/e2e-tests.yml b/.github/e2e-tests.yml
index feef287f84b..ddb81956536 100644
--- a/.github/e2e-tests.yml
+++ b/.github/e2e-tests.yml
@@ -86,14 +86,14 @@ runner-test-matrix:
   # END Automation upgrade tests
 
   # Example of a configuration for running a single soak test in Kubernetes Remote Runner
-  - id: soak/ocr_test.go:^TestOCRSoak$
+  - id: soak/ocr_test.go:^TestOCRv1Soak$
     path: integration-tests/soak/ocr_test.go
     test-type: k8s-remote-runner
     remote-runner-test-suite: soak
     runs-on: ubuntu-latest
     # workflows:
     #   - Run Nightly E2E Tests
-    test-cmd: cd integration-tests/ && go test soak/ocr_test.go -test.run ^TestOCRSoak$ -test.parallel=1 -timeout 30m -count=1
+    test-cmd: cd integration-tests/ && go test soak/ocr_test.go -test.run ^TestOCRv1Soak$ -test.parallel=1 -timeout 30m -count=1
     pyroscope-env: ci-smoke-ocr2-evm-simulated-nightly
 
   - id: integration-tests/benchmark/keeper_test.go:^TestAutomationBenchmark$

From c3a87641620fb16176794c175f00551671df2ecc Mon Sep 17 00:00:00 2001
From: lukaszcl <120112546+lukaszcl@users.noreply.github.com>
Date: Mon, 1 Jul 2024 11:09:34 +0200
Subject: [PATCH 122/137] update cache

---
 .github/workflows/run-e2e-tests-reusable-workflow.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/run-e2e-tests-reusable-workflow.yml b/.github/workflows/run-e2e-tests-reusable-workflow.yml
index 21ef5614d81..dce0ff19e77 100644
--- a/.github/workflows/run-e2e-tests-reusable-workflow.yml
+++ b/.github/workflows/run-e2e-tests-reusable-workflow.yml
@@ -62,7 +62,7 @@ on:
                
 env: 
   CHAINLINK_IMAGE: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com/chainlink
-  MOD_CACHE_VERSION: 1
+  MOD_CACHE_VERSION: 2
 
 jobs:
   validate-inputs:

From 7a0466b4b56e2bbcbbb83017bcd7c1eefcb9ea2d Mon Sep 17 00:00:00 2001
From: lukaszcl <120112546+lukaszcl@users.noreply.github.com>
Date: Mon, 1 Jul 2024 11:48:46 +0200
Subject: [PATCH 123/137] Fix empty output for k8s tests

---
 .github/e2e-tests.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/e2e-tests.yml b/.github/e2e-tests.yml
index ddb81956536..b0f75fbffa1 100644
--- a/.github/e2e-tests.yml
+++ b/.github/e2e-tests.yml
@@ -93,7 +93,7 @@ runner-test-matrix:
     runs-on: ubuntu-latest
     # workflows:
     #   - Run Nightly E2E Tests
-    test-cmd: cd integration-tests/ && go test soak/ocr_test.go -test.run ^TestOCRv1Soak$ -test.parallel=1 -timeout 30m -count=1
+    test-cmd: cd integration-tests/ && go test soak/ocr_test.go -v -test.run ^TestOCRv1Soak$ -test.parallel=1 -timeout 30m -count=1
     pyroscope-env: ci-smoke-ocr2-evm-simulated-nightly
 
   - id: integration-tests/benchmark/keeper_test.go:^TestAutomationBenchmark$
@@ -104,5 +104,5 @@ runner-test-matrix:
     runs-on: ubuntu-latest
     # workflows:
     #   - Run Nightly E2E Tests
-    test-cmd: cd integration-tests/benchmark && go test -test.run ^TestAutomationBenchmark$ -test.parallel=1 -timeout 30m -count=1
+    test-cmd: cd integration-tests/benchmark && go test -v -test.run ^TestAutomationBenchmark$ -test.parallel=1 -timeout 30m -count=1
     pyroscope-env: ci-benchmark-automation-nightly
\ No newline at end of file

From 2e7d11a6273428c7c1ce2b7f919e5f4f5101d998 Mon Sep 17 00:00:00 2001
From: lukaszcl <120112546+lukaszcl@users.noreply.github.com>
Date: Mon, 1 Jul 2024 13:11:43 +0200
Subject: [PATCH 124/137] Add a test workflow with overrides

---
 .../action.yml                                |   6 +
 .github/e2e-tests.yml                         |  50 ++++--
 .../run-automation-ondemand-e2e-tests.yml     | 149 ++++++++++++++++++
 .../run-e2e-tests-reusable-workflow.yml       |  41 +++--
 .../cmd/create_test_config_cmd.go             |  14 ++
 .../e2e_tests_ci_tool/cmd/envreplace_cmd.go   |  63 --------
 .../e2e_tests_ci_tool/cmd/filter_cmd.go       |  58 ++++++-
 .../e2e_tests_ci_tool/cmd/root_cmd.go         |   1 -
 .../e2e_tests_ci_tool/cmd/test_config_cmd.go  |  10 +-
 .../e2e_tests_ci_tool/cmd/types.go            |  29 ++--
 10 files changed, 301 insertions(+), 120 deletions(-)
 create mode 100644 .github/workflows/run-automation-ondemand-e2e-tests.yml
 delete mode 100644 integration-tests/e2e_tests_ci_tool/cmd/envreplace_cmd.go

diff --git a/.github/actions/create-default-e2e-config-override/action.yml b/.github/actions/create-default-e2e-config-override/action.yml
index 78b15a26ab2..b2fdbe7d6a0 100644
--- a/.github/actions/create-default-e2e-config-override/action.yml
+++ b/.github/actions/create-default-e2e-config-override/action.yml
@@ -9,6 +9,9 @@ inputs:
   chainlink_postgres_version:
     required: false
     description: 'Chainlink Postgres version to use for the tests'
+  chainlink_upgrade_image:
+    required: false
+    description: 'Chainlink upgrade image to use for the tests'
   chainlink_upgrade_version:
     required: false
     description: 'Chainlink upgrade version to use for the tests'
@@ -74,6 +77,9 @@ runs:
         if [ -n "${{ inputs.chainlink_postgres_version }}" ]; then
           cmd_args+="--chainlink-postgres-version=\"${{ inputs.chainlink_postgres_version }}\" "
         fi
+        if [ -n "${{ inputs.chainlink_upgrade_image }}" ]; then
+          cmd_args+="--chainlink-upgrade-image=\"${{ inputs.chainlink_upgrade_image }}\" "
+        fi
         if [ -n "${{ inputs.chainlink_upgrade_version }}" ]; then
           cmd_args+="--chainlink-upgrade-version=\"${{ inputs.chainlink_upgrade_version }}\" "
         fi
diff --git a/.github/e2e-tests.yml b/.github/e2e-tests.yml
index b0f75fbffa1..3aa23adb760 100644
--- a/.github/e2e-tests.yml
+++ b/.github/e2e-tests.yml
@@ -48,11 +48,11 @@ runner-test-matrix:
     workflows:
       - Run Automation Product Nightly E2E Tests
     test-cmd: cd integration-tests/smoke && go test -test.run ^TestAutomationNodeUpgrade/registry_2_0 -test.parallel=1 -timeout 60m -count=1 -json
-    test-config-override:
-      chainlink-upgrade-image: ${{ env.CHAINLINK_IMAGE }}
-      chainlink-upgrade-version: develop
-      chainlink-image: public.ecr.aws/chainlink/chainlink
-      chainlink-version: latest
+    test-config-overrides:
+      chainlinkUpgradeImage: public.ecr.aws/chainlink/chainlink 
+      chainlinkUpgradeVersion: latest
+      chainlinkImage: '{{ env.QA_CHAINLINK_IMAGE }}'
+      chainlinkVersion: develop
     pyroscope-env: ci-smoke-automation-upgrade-tests
 
   - id: smoke/automation_upgrade_test.go:^TestAutomationNodeUpgrade/registry_2_1
@@ -62,11 +62,11 @@ runner-test-matrix:
     workflows:
       - Run Automation Product Nightly E2E Tests    
     test-cmd: cd integration-tests/smoke && go test -test.run ^TestAutomationNodeUpgrade/registry_2_1 -test.parallel=5 -timeout 60m -count=1 -json
-    test-config-override:
-      chainlink-upgrade-image: ${{ env.CHAINLINK_IMAGE }}
-      chainlink-upgrade-version: develop
-      chainlink-image: public.ecr.aws/chainlink/chainlink
-      chainlink-version: latest
+    test-config-overrides:
+      chainlinkUpgradeImage: public.ecr.aws/chainlink/chainlink 
+      chainlinkUpgradeVersion: latest
+      chainlinkImage: '{{ env.QA_CHAINLINK_IMAGE }}'
+      chainlinkVersion: develop
     pyroscope-env: ci-smoke-automation-upgrade-tests
 
   - id: smoke/automation_upgrade_test.go:^TestAutomationNodeUpgrade/registry_2_2
@@ -76,15 +76,35 @@ runner-test-matrix:
     workflows:
       - Run Automation Product Nightly E2E Tests
     test-cmd: cd integration-tests/smoke && go test -test.run ^TestAutomationNodeUpgrade/registry_2_2 -test.parallel=5 -timeout 60m -count=1 -json
-    test-config-override:
-      chainlink-upgrade-image: ${{ env.CHAINLINK_IMAGE }}
-      chainlink-upgrade-version: develop
-      chainlink-image: public.ecr.aws/chainlink/chainlink
-      chainlink-version: latest
+    test-config-overrides:
+      chainlinkUpgradeImage: public.ecr.aws/chainlink/chainlink 
+      chainlinkUpgradeVersion: latest
+      chainlinkImage: '{{ env.QA_CHAINLINK_IMAGE }}'
+      chainlinkVersion: develop
     pyroscope-env: ci-smoke-automation-upgrade-tests
 
   # END Automation upgrade tests
 
+  - id: integration-tests/reorg/automation_reorg_test.go
+    path: integration-tests/reorg/automation_reorg_test.go
+    test-type: k8s-remote-runner
+    remote-runner-test-suite: reorg
+    runs-on: ubuntu-latest
+    workflows:
+      # - Run Automation Product Nightly E2E Tests
+    test-cmd: cd integration-tests/reorg && go test -v -test.run ^TestAutomationReorg$ -test.parallel=5 -timeout 60m -count=1
+    pyroscope-env: ci-automation-on-demand-reorg
+
+  - id: integration-tests/chaos/automation_chaos_test.go
+    path: integration-tests/chaos/automation_chaos_test.go
+    test-type: k8s-remote-runner
+    remote-runner-test-suite: chaos
+    runs-on: ubuntu-latest
+    workflows:
+      # - Run Automation Product Nightly E2E Tests
+    test-cmd: cd integration-tests/chaos && go test -v -test.run ^TestAutomationChaos$ -test.parallel=15 -timeout 60m -count=1
+    pyroscope-env: ci-automation-on-demand-chaos
+
   # Example of a configuration for running a single soak test in Kubernetes Remote Runner
   - id: soak/ocr_test.go:^TestOCRv1Soak$
     path: integration-tests/soak/ocr_test.go
diff --git a/.github/workflows/run-automation-ondemand-e2e-tests.yml b/.github/workflows/run-automation-ondemand-e2e-tests.yml
new file mode 100644
index 00000000000..92a36fdb202
--- /dev/null
+++ b/.github/workflows/run-automation-ondemand-e2e-tests.yml
@@ -0,0 +1,149 @@
+name: Run Automation On Demand Tests (TEST WORKFLOW)
+
+on:
+  workflow_dispatch:
+    inputs:
+      chainlinkImage:
+        description: Chainlink image repo to use (Leave empty to build from head/ref)
+        options:
+          - QA_ECR
+          - public.ecr.aws/chainlink/chainlink
+        default: QA_ECR
+        type: choice
+      chainlinkVersion:
+        description: Chainlink image version to use
+        required: false
+        default: develop
+        type: string        
+      chainlinkImageUpdate:
+        description: Chainlink image repo to use initially for upgrade test
+        required: true
+        options:
+          - QA_ECR
+          - public.ecr.aws/chainlink/chainlink
+        default: QA_ECR
+        type: choice
+      chainlinkVersionUpdate:
+        description: Chainlink image version to use initially for upgrade test
+        default: develop
+        required: true
+        type: string
+      enableChaos:
+        description: Check to enable chaos tests
+        type: boolean
+        default: false
+        required: true
+      enableReorg:
+        description: Check to enable reorg tests
+        type: boolean
+        default: false
+        required: true
+      with_existing_remote_runner_version:
+        description: 'Use the existing remote runner version for k8s tests. Example: "d3bf5044af33e08be788a2df31c4a745cf69d787"'
+        required: false
+        type: string                 
+
+jobs:
+  # Set tests to run based on the workflow inputs
+  set-tests-to-run:
+    name: Set tests to run
+    runs-on: ubuntu-latest
+    outputs:
+      test_map: ${{ steps.set-tests.outputs.test_map }}
+    steps:
+      - name: Set tests to run
+        id: set-tests
+        run: |
+          if [[ "${{ github.event.inputs.chainlinkImage }}" == 'QA_ECR' ]]; then
+            # Instead of passing chainlink image as plain text input, {{ env.QA_CHAINLINK_IMAGE }} will be securely resolved in the E2E tests reusable workflow
+            CHAINLINK_IMAGE="'{{ env.QA_CHAINLINK_IMAGE }}'"
+          elif [[ "${{ github.event.inputs.chainlinkImage }}" == 'public.ecr.aws/chainlink/chainlink' ]]; then
+            CHAINLINK_IMAGE=public.ecr.aws/chainlink/chainlink
+          else
+            echo "Error: Not supported image source '${{ github.event.inputs.chainlinkImage }}'"
+            exit 1
+          fi  
+          
+          if [[ "${{ github.event.inputs.chainlinkImageUpdate }}" == 'QA_ECR' ]]; then
+            CHAINLINK_UPGRADE_IMAGE="'{{ env.QA_CHAINLINK_IMAGE }}'"
+          elif [[ "${{ github.event.inputs.chainlinkImageUpdate }}" == 'public.ecr.aws/chainlink/chainlink' ]]; then
+            CHAINLINK_UPGRADE_IMAGE=public.ecr.aws/chainlink/chainlink
+          else
+            echo "Error: Not supported image source '${{ github.event.inputs.chainlinkImageUpdate }}'"
+            exit 1
+          fi            
+
+          CHAINLINK_VERSION="${{ inputs.chainlinkVersion || github.sha }}"
+          CHAINLINK_UPGRADE_VERSION="${{ inputs.chainlinkVersionUpdate }}"
+
+          # Always run upgrade tests 
+          cat > test_map.yaml <<EOF
+          - id: smoke/automation_upgrade_test.go:^TestAutomationNodeUpgrade/registry_2_0
+            test-config-overrides:
+              chainlinkImage: $CHAINLINK_IMAGE
+              chainlinkVersion: $CHAINLINK_VERSION
+              chainlinkUpgradeImage: $CHAINLINK_UPGRADE_IMAGE
+              chainlinkUpgradeVersion: $CHAINLINK_UPGRADE_VERSION
+
+          - id: smoke/automation_upgrade_test.go:^TestAutomationNodeUpgrade/registry_2_1
+            test-config-overrides:
+              chainlinkImage: $CHAINLINK_IMAGE
+              chainlinkVersion: $CHAINLINK_VERSION
+              chainlinkUpgradeImage: $CHAINLINK_UPGRADE_IMAGE
+              chainlinkUpgradeVersion: $CHAINLINK_UPGRADE_VERSION
+
+          - id: smoke/automation_upgrade_test.go:^TestAutomationNodeUpgrade/registry_2_2
+            test-config-overrides:
+              chainlinkImage: $CHAINLINK_IMAGE
+              chainlinkVersion: $CHAINLINK_VERSION
+              chainlinkUpgradeImage: $CHAINLINK_UPGRADE_IMAGE
+              chainlinkUpgradeVersion: $CHAINLINK_UPGRADE_VERSION              
+          EOF
+
+          # Run reorg tests if enabled
+          if [[ "${{ github.event.inputs.enableReorg }}" == 'true' ]]; then
+          cat >> test_map.yaml <<EOF
+          - id: integration-tests/reorg/automation_reorg_test.go
+            test-config-overrides:
+              chainlinkImage: $CHAINLINK_IMAGE
+              chainlinkVersion: $CHAINLINK_VERSION
+          EOF
+          fi
+
+          # Run chaos tests if enabled
+          if [[ "${{ github.event.inputs.enableChaos }}" == 'true' ]]; then
+          cat >> test_map.yaml <<EOF
+          - id: integration-tests/chaos/automation_chaos_test.go
+            test-config-overrides:
+              chainlinkImage: $CHAINLINK_IMAGE
+              chainlinkVersion: $CHAINLINK_VERSION
+          EOF
+          fi
+
+          echo "chainlink_version=$CHAINLINK_VERSION" >> $GITHUB_OUTPUT
+          echo "test_map=$(cat test_map.yaml | base64 -w 0)" >> $GITHUB_OUTPUT
+
+  call-run-e2e-tests-workflow:
+    name: Run E2E Tests
+    needs: set-tests-to-run
+    uses: ./.github/workflows/run-e2e-tests-reusable-workflow.yml
+    with:
+      chainlink_version: ${{ needs.set-tests-to-run.outputs.chainlink_version }}
+      test_map: ${{ needs.set-tests-to-run.outputs.test_map }}
+      with_existing_remote_runner_version: ${{ github.event.inputs.with_existing_remote_runner_version }}
+    secrets:
+      QA_AWS_REGION: ${{ secrets.QA_AWS_REGION }}
+      QA_AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }}
+      QA_AWS_ACCOUNT_NUMBER: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}
+      QA_PYROSCOPE_INSTANCE: ${{ secrets.QA_PYROSCOPE_INSTANCE }}
+      QA_PYROSCOPE_KEY: ${{ secrets.QA_PYROSCOPE_KEY }}
+      QA_KUBECONFIG: ${{ secrets.QA_KUBECONFIG }}
+      GRAFANA_INTERNAL_TENANT_ID: ${{ secrets.GRAFANA_INTERNAL_TENANT_ID }}
+      GRAFANA_INTERNAL_BASIC_AUTH: ${{ secrets.GRAFANA_INTERNAL_BASIC_AUTH }}
+      GRAFANA_INTERNAL_HOST: ${{ secrets.GRAFANA_INTERNAL_HOST }}
+      GRAFANA_INTERNAL_URL_SHORTENER_TOKEN: ${{ secrets.GRAFANA_INTERNAL_URL_SHORTENER_TOKEN }}
+      GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}  
+      AWS_REGION: ${{ secrets.QA_AWS_REGION }}
+      AWS_OIDC_IAM_ROLE_VALIDATION_PROD_ARN: ${{ secrets.AWS_OIDC_IAM_ROLE_VALIDATION_PROD_ARN }}
+      AWS_API_GW_HOST_GRAFANA: ${{ secrets.AWS_API_GW_HOST_GRAFANA }}        
+
diff --git a/.github/workflows/run-e2e-tests-reusable-workflow.yml b/.github/workflows/run-e2e-tests-reusable-workflow.yml
index dce0ff19e77..57b60c86aa7 100644
--- a/.github/workflows/run-e2e-tests-reusable-workflow.yml
+++ b/.github/workflows/run-e2e-tests-reusable-workflow.yml
@@ -6,12 +6,16 @@ on:
     inputs:
       chainlink_version:
         description: 'Enter Chainlink version to use for the tests. Example: "v2.10.0" or sha'
-        required: true
-        type: string
+        required: false
+        type: string    
       test_ids:
         description: 'Run tests by test ids separated by commas. Example: "run_all_in_ocr_tests_go,run_TestOCRv2Request_in_ocr2_test_go". Check all test IDs in .github/e2e-tests.yml'
         required: false
         type: string
+      test_map:
+        description: 'Run tests by test map.'
+        required: true
+        type: string            
       test_workflow:
         description: 'Run tests by workflow name. Example: "Run Nightly E2E Tests"'
         required: false
@@ -112,7 +116,7 @@ jobs:
         id: set-docker-matrix
         run: |
           cd integration-tests/e2e_tests_ci_tool/
-          MATRIX_JSON=$(go run main.go filter --file ../../.github/e2e-tests.yml --test-type 'docker' --test-ids '${{ inputs.test_ids }}' --workflow '${{ inputs.test_workflow }}')
+          MATRIX_JSON=$(go run main.go filter --file ../../.github/e2e-tests.yml --test-type 'docker' --test-map '${{ inputs.test_map }}' --test-ids '${{ inputs.test_ids }}' --workflow '${{ inputs.test_workflow }}')
           echo "Docker tests:"
           echo "$MATRIX_JSON" | jq
           echo "matrix=$MATRIX_JSON" >> $GITHUB_OUTPUT
@@ -120,7 +124,7 @@ jobs:
         id: set-k8s-runner-matrix
         run: |
           cd integration-tests/e2e_tests_ci_tool/
-          MATRIX_JSON=$(go run main.go filter --file ../../.github/e2e-tests.yml --test-type 'k8s-remote-runner' --test-ids '${{ inputs.test_ids }}' --workflow '${{ inputs.test_workflow }}')
+          MATRIX_JSON=$(go run main.go filter --file ../../.github/e2e-tests.yml --test-type 'k8s-remote-runner' --test-map '${{ inputs.test_map }}' --test-ids '${{ inputs.test_ids }}' --workflow '${{ inputs.test_workflow }}')
           echo "K8s tests:"
           echo "$MATRIX_JSON" | jq
           echo "matrix=$MATRIX_JSON" >> $GITHUB_OUTPUT
@@ -174,7 +178,7 @@ jobs:
       SELECTED_NETWORKS: SIMULATED
       CHAINLINK_ENV_USER: ${{ github.actor }}
       TEST_LOG_LEVEL: debug
-
+      QA_CHAINLINK_IMAGE: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com/chainlink
     steps:
       - name: Checkout repository
         uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2
@@ -189,11 +193,12 @@ jobs:
         # if: ${{ inputs.test_config_base64_override == '' }}
         uses: ./.github/actions/create-default-e2e-config-override
         with:
-          chainlink_version: ${{ inputs.chainlink_version || matrix.tests.testConfigOverride.chainlinkVersion || github.sha }}
-          chainlink_image: ${{ matrix.tests.testConfigOverride.chainlinkImage || env.CHAINLINK_IMAGE }}
-          chainlink_postgres_version: ${{ matrix.tests.testConfigOverride.chainlinkPostgresVersion }}
-          chainlink_upgrade_version: ${{ matrix.tests.testConfigOverride.chainlinkUpgradeVersion }}
-          selected_networks: ${{ matrix.tests.testConfigOverride.selectedNetworks || env.SELECTED_NETWORKS}}
+          chainlink_image: ${{ matrix.tests.testConfigOverrides.chainlinkImage || env.CHAINLINK_IMAGE }}
+          chainlink_version: ${{ inputs.chainlink_version || matrix.tests.testConfigOverrides.chainlinkVersion || github.sha }}
+          chainlink_postgres_version: ${{ matrix.tests.testConfigOverrides.chainlinkPostgresVersion }}
+          chainlink_upgrade_image: ${{ matrix.tests.testConfigOverrides.chainlinkUpgradeImage }}
+          chainlink_upgrade_version: ${{ matrix.tests.testConfigOverrides.chainlinkUpgradeVersion }}
+          selected_networks: ${{ matrix.tests.testConfigOverrides.selectedNetworks || env.SELECTED_NETWORKS}}
           pyroscope_server: ${{ secrets.QA_PYROSCOPE_INSTANCE }}
           pyroscope_environment: ${{ matrix.tests.pyroscopeEnv }}
           pyroscope_key: ${{ secrets.QA_PYROSCOPE_KEY }}
@@ -296,6 +301,7 @@ jobs:
       CHAINLINK_COMMIT_SHA: ${{ inputs.evm-ref || github.sha }}
       CHAINLINK_ENV_USER: ${{ github.actor }}
       TEST_LOG_LEVEL: debug
+      QA_CHAINLINK_IMAGE: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com/chainlink
 
     steps:
       - name: Checkout repository
@@ -312,11 +318,12 @@ jobs:
         # if: ${{ inputs.test_config_base64_override == '' }}
         uses: ./.github/actions/create-default-e2e-config-override
         with:
-          chainlink_version: ${{ inputs.chainlink_version || matrix.tests.testConfigOverride.chainlinkVersion || github.sha }}
-          chainlink_image: ${{ matrix.tests.testConfigOverride.chainlinkImage || env.CHAINLINK_IMAGE }}
-          chainlink_postgres_version: ${{ matrix.tests.testConfigOverride.chainlinkPostgresVersion }}
-          chainlink_upgrade_version: ${{ matrix.tests.testConfigOverride.chainlinkUpgradeVersion }}
-          selected_networks: ${{ matrix.tests.testConfigOverride.selectedNetworks || env.SELECTED_NETWORKS}}
+          chainlink_version: ${{ inputs.chainlink_version || matrix.tests.testConfigOverrides.chainlinkVersion || github.sha }}
+          chainlink_image: ${{ matrix.tests.testConfigOverrides.chainlinkImage || env.CHAINLINK_IMAGE }}
+          chainlink_postgres_version: ${{ matrix.tests.testConfigOverrides.chainlinkPostgresVersion }}
+          chainlink_upgrade_image: ${{ matrix.tests.testConfigOverrides.chainlinkUpgradeImage }}
+          chainlink_upgrade_version: ${{ matrix.tests.testConfigOverrides.chainlinkUpgradeVersion }}
+          selected_networks: ${{ matrix.tests.testConfigOverrides.selectedNetworks || env.SELECTED_NETWORKS}}
           pyroscope_server: ${{ secrets.QA_PYROSCOPE_INSTANCE }}
           pyroscope_environment: ${{ matrix.tests.pyroscopeEnv }}
           pyroscope_key: ${{ secrets.QA_PYROSCOPE_KEY }}
@@ -349,8 +356,8 @@ jobs:
         with:
           test_command_to_run: ${{ matrix.tests.testCmd }}
           test_download_vendor_packages_command: make gomod
-          cl_repo: ${{ env.CHAINLINK_IMAGE }}
-          cl_image_tag: ${{ env.CHAINLINK_VERSION }}
+          cl_repo: ${{ matrix.tests.testConfigOverrides.chainlinkImage || env.CHAINLINK_IMAGE }}
+          cl_image_tag: ${{ inputs.chainlink_version || matrix.tests.testConfigOverrides.chainlinkVersion || github.sha }}
           token: ${{ secrets.GH_TOKEN }}
           should_cleanup: false
           cache_key_id: run-k8s-e2e-tests-${{ env.MOD_CACHE_VERSION }}
diff --git a/integration-tests/e2e_tests_ci_tool/cmd/create_test_config_cmd.go b/integration-tests/e2e_tests_ci_tool/cmd/create_test_config_cmd.go
index f422f03e80b..45c4ee90ab3 100644
--- a/integration-tests/e2e_tests_ci_tool/cmd/create_test_config_cmd.go
+++ b/integration-tests/e2e_tests_ci_tool/cmd/create_test_config_cmd.go
@@ -38,6 +38,20 @@ var createTestConfigCmd = &cobra.Command{
 			}
 		}
 
+		var upgradeImage, upgradeVersion *string
+		if cmd.Flags().Changed(ChainlinkUpgradeImageFlag) {
+			upgradeImage = &oc.ChainlinkUpgradeImage
+		}
+		if cmd.Flags().Changed(ChainlinkUpgradeVersionFlag) {
+			upgradeVersion = &oc.ChainlinkUpgradeVersion
+		}
+		if upgradeImage != nil || upgradeVersion == nil {
+			tc.ChainlinkUpgradeImage = &ctf_config.ChainlinkImageConfig{
+				Image:   upgradeImage,
+				Version: upgradeVersion,
+			}
+		}
+
 		var selectedNetworks *[]string
 		if cmd.Flags().Changed(SelectedNetworksFlag) {
 			selectedNetworks = &oc.SelectedNetworks
diff --git a/integration-tests/e2e_tests_ci_tool/cmd/envreplace_cmd.go b/integration-tests/e2e_tests_ci_tool/cmd/envreplace_cmd.go
deleted file mode 100644
index 1c2caf6c565..00000000000
--- a/integration-tests/e2e_tests_ci_tool/cmd/envreplace_cmd.go
+++ /dev/null
@@ -1,63 +0,0 @@
-package cmd
-
-import (
-	"fmt"
-	"os"
-	"regexp"
-	"strings"
-
-	"github.com/spf13/cobra"
-)
-
-// envreplaceCmd represents the environment replace command
-var envreplaceCmd = &cobra.Command{
-	Use:   "envreplace",
-	Short: "Replace environment variable placeholders in a string",
-	Long: `Replaces placeholders of the form ${{ env.VAR_NAME }} with the actual environment variable values.
-Example usage:
-./e2e_tests_tool envreplace --input 'Example with ${{ env.PATH }} and ${{ env.HOME }}.'`,
-	Run: func(cmd *cobra.Command, _ []string) {
-		input, _ := cmd.Flags().GetString("input")
-
-		output, err := replaceEnvPlaceholders(input)
-		if err != nil {
-			fmt.Printf("Error replacing placeholders: %v\n", err)
-			return
-		}
-
-		fmt.Println(output)
-	},
-}
-
-func replaceEnvPlaceholders(input string) (string, error) {
-	// Regular expression to match ${env.VAR_NAME}
-	r := regexp.MustCompile(`\$\{\{\s*env\.(\w+)\s*\}\}`)
-	var errors []string // Slice to accumulate error messages
-
-	// Replace each match in the input string
-	replaced := r.ReplaceAllStringFunc(input, func(m string) string {
-		varName := r.FindStringSubmatch(m)[1]
-		value := os.Getenv(varName)
-		if value == "" {
-			// If the environment variable is not set or empty, accumulate an error message
-			errors = append(errors, fmt.Sprintf("environment variable '%s' not set or is empty", varName))
-			return m // Return the original placeholder if error occurs
-		}
-		return value
-	})
-
-	// Check if there were any errors
-	if len(errors) > 0 {
-		return replaced, fmt.Errorf("multiple errors occurred: %s", strings.Join(errors, ", "))
-	}
-	return replaced, nil // No error occurred, return the replaced string
-}
-
-func init() {
-	envreplaceCmd.Flags().StringP("input", "i", "", "Input string with placeholders")
-	err := envreplaceCmd.MarkFlagRequired("input")
-	if err != nil {
-		fmt.Fprintf(os.Stderr, "Error marking flag as required: %v\n", err)
-		os.Exit(1)
-	}
-}
diff --git a/integration-tests/e2e_tests_ci_tool/cmd/filter_cmd.go b/integration-tests/e2e_tests_ci_tool/cmd/filter_cmd.go
index 56a88702738..4bafeb9af34 100644
--- a/integration-tests/e2e_tests_ci_tool/cmd/filter_cmd.go
+++ b/integration-tests/e2e_tests_ci_tool/cmd/filter_cmd.go
@@ -1,6 +1,7 @@
 package cmd
 
 import (
+	"encoding/base64"
 	"encoding/json"
 	"fmt"
 	"log"
@@ -13,14 +14,14 @@ import (
 )
 
 // Filter tests based on workflow, test type, and test IDs.
-func filterTests(tests []CITestConf, workflow, testType, ids string) []CITestConf {
+func filterTests(allTests []CITestConf, workflow, testType, ids string) []CITestConf {
 	workflowFilter := workflow
 	typeFilter := testType
 	idFilter := strings.Split(ids, ",")
 
 	var filteredTests []CITestConf
 
-	for _, test := range tests {
+	for _, test := range allTests {
 		workflowMatch := workflow == "" || contains(test.Workflows, workflowFilter)
 		typeMatch := testType == "" || test.TestType == typeFilter
 		idMatch := ids == "*" || ids == "" || contains(idFilter, test.ID)
@@ -34,6 +35,43 @@ func filterTests(tests []CITestConf, workflow, testType, ids string) []CITestCon
 	return filteredTests
 }
 
+func filterAndMergeTests(allTests []CITestConf, workflow, testType, base64Tests string) ([]CITestConf, error) {
+	decodedBytes, err := base64.StdEncoding.DecodeString(base64Tests)
+	if err != nil {
+		return nil, err
+	}
+	var decodedTests []CITestConf
+	err = yaml.Unmarshal(decodedBytes, &decodedTests)
+	if err != nil {
+		return nil, err
+	}
+
+	idFilter := make(map[string]CITestConf)
+	for _, dt := range decodedTests {
+		idFilter[dt.ID] = dt
+	}
+
+	var filteredTests []CITestConf
+	for _, test := range allTests {
+		workflowMatch := workflow == "" || contains(test.Workflows, workflow)
+		typeMatch := testType == "" || test.TestType == testType
+
+		if decodedTest, exists := idFilter[test.ID]; exists && workflowMatch && typeMatch {
+			// Apply config overrides
+			for k, v := range decodedTest.TestConfigOverrides {
+				if test.TestConfigOverrides == nil {
+					test.TestConfigOverrides = make(map[string]string)
+				}
+				test.TestConfigOverrides[k] = v
+			}
+			test.IDSanitized = sanitizeTestID(test.ID)
+			filteredTests = append(filteredTests, test)
+		}
+	}
+
+	return filteredTests, nil
+}
+
 func sanitizeTestID(id string) string {
 	// Define a regular expression that matches any character not a letter, digit, hyphen
 	re := regexp.MustCompile(`[^a-zA-Z0-9-_]+`)
@@ -63,6 +101,7 @@ Example usage:
 		workflow, _ := cmd.Flags().GetString("workflow")
 		testType, _ := cmd.Flags().GetString("test-type")
 		testIDs, _ := cmd.Flags().GetString("test-ids")
+		testMap, _ := cmd.Flags().GetString("test-map")
 
 		data, err := os.ReadFile(yamlFile)
 		if err != nil {
@@ -75,7 +114,15 @@ Example usage:
 			log.Fatalf("Error parsing YAML data: %v", err)
 		}
 
-		filteredTests := filterTests(config.Tests, workflow, testType, testIDs)
+		var filteredTests []CITestConf
+		if testMap == "" {
+			filteredTests = filterTests(config.Tests, workflow, testType, testIDs)
+		} else {
+			filteredTests, err = filterAndMergeTests(config.Tests, workflow, testType, testMap)
+			if err != nil {
+				log.Fatalf("Error filtering and merging tests: %v", err)
+			}
+		}
 		matrix := map[string][]CITestConf{"tests": filteredTests}
 		matrixJSON, err := json.Marshal(matrix)
 		if err != nil {
@@ -88,9 +135,10 @@ Example usage:
 
 func init() {
 	filterCmd.Flags().StringP("file", "f", "", "Path to the YAML file")
-	filterCmd.Flags().StringP("workflow", "t", "", "Workflow filter")
-	filterCmd.Flags().StringP("test-type", "y", "", "Type of test to filter by")
+	filterCmd.Flags().String("test-map", "", "Base64 encoded list of tests (YML objects) to filter by. Can include test-config-overrides for each test.")
 	filterCmd.Flags().StringP("test-ids", "i", "*", "Comma-separated list of test IDs to filter by")
+	filterCmd.Flags().StringP("test-type", "y", "", "Type of test to filter by")
+	filterCmd.Flags().StringP("workflow", "t", "", "Workflow filter")
 	err := filterCmd.MarkFlagRequired("file")
 	if err != nil {
 		fmt.Fprintf(os.Stderr, "Error marking flag as required: %v\n", err)
diff --git a/integration-tests/e2e_tests_ci_tool/cmd/root_cmd.go b/integration-tests/e2e_tests_ci_tool/cmd/root_cmd.go
index 789d67f28df..17c527168ba 100644
--- a/integration-tests/e2e_tests_ci_tool/cmd/root_cmd.go
+++ b/integration-tests/e2e_tests_ci_tool/cmd/root_cmd.go
@@ -26,7 +26,6 @@ func init() {
 
 	rootCmd.AddCommand(checkTestsCmd)
 	rootCmd.AddCommand(filterCmd)
-	rootCmd.AddCommand(envreplaceCmd)
 	rootCmd.AddCommand(testConfigCmd)
 	testConfigCmd.AddCommand(maskTestConfigCmd)
 	testConfigCmd.AddCommand(createTestConfigCmd)
diff --git a/integration-tests/e2e_tests_ci_tool/cmd/test_config_cmd.go b/integration-tests/e2e_tests_ci_tool/cmd/test_config_cmd.go
index be77e861ff5..e783ea349e6 100644
--- a/integration-tests/e2e_tests_ci_tool/cmd/test_config_cmd.go
+++ b/integration-tests/e2e_tests_ci_tool/cmd/test_config_cmd.go
@@ -18,6 +18,8 @@ var testConfigCmd = &cobra.Command{
 type OverrideConfig struct {
 	ChainlinkImage                           string
 	ChainlinkVersion                         string
+	ChainlinkUpgradeImage                    string
+	ChainlinkUpgradeVersion                  string
 	ChainlinkPostgresVersion                 string
 	SelectedNetworks                         []string
 	PyroscopeEnabled                         bool
@@ -41,6 +43,8 @@ type OverrideConfig struct {
 const (
 	ChainlinkImageFlag                          = "chainlink-image"
 	ChainlinkVersionFlag                        = "chainlink-version"
+	ChainlinkUpgradeImageFlag                   = "chainlink-upgrade-image"
+	ChainlinkUpgradeVersionFlag                 = "chainlink-upgrade-version"
 	ChainlinkPostgresVersionFlag                = "chainlink-postgres-version"
 	SelectedNetworksFlag                        = "selected-networks"
 	FromBase64ConfigFlag                        = "from-base64-config"
@@ -70,6 +74,8 @@ func init() {
 		c.Flags().StringArrayVar(&oc.SelectedNetworks, SelectedNetworksFlag, nil, "Selected networks")
 		c.Flags().StringVar(&oc.ChainlinkImage, ChainlinkImageFlag, "", "Chainlink image")
 		c.Flags().StringVar(&oc.ChainlinkVersion, ChainlinkVersionFlag, "", "Chainlink version")
+		c.Flags().StringVar(&oc.ChainlinkUpgradeImage, ChainlinkUpgradeImageFlag, "", "Chainlink upgrade image")
+		c.Flags().StringVar(&oc.ChainlinkUpgradeVersion, ChainlinkUpgradeVersionFlag, "", "Chainlink upgrade version")
 		c.Flags().StringVar(&oc.ChainlinkPostgresVersion, ChainlinkPostgresVersionFlag, "", "Chainlink Postgres version")
 		c.Flags().BoolVar(&oc.PyroscopeEnabled, PyroscopeEnabledFlag, false, "Pyroscope enabled")
 		c.Flags().StringVar(&oc.PyroscopeServerURL, PyroscopeServerURLFlag, "", "Pyroscope server URL")
@@ -101,6 +107,8 @@ func init() {
 			// Resolve all other environment variables
 			oc.ChainlinkImage = mustResolveEnvPlaceholder(oc.ChainlinkImage)
 			oc.ChainlinkVersion = mustResolveEnvPlaceholder(oc.ChainlinkVersion)
+			oc.ChainlinkUpgradeImage = mustResolveEnvPlaceholder(oc.ChainlinkUpgradeImage)
+			oc.ChainlinkUpgradeVersion = mustResolveEnvPlaceholder(oc.ChainlinkUpgradeVersion)
 			oc.ChainlinkPostgresVersion = mustResolveEnvPlaceholder(oc.ChainlinkPostgresVersion)
 			oc.PyroscopeServerURL = mustResolveEnvPlaceholder(oc.PyroscopeServerURL)
 			oc.PyroscopeKey = mustResolveEnvPlaceholder(oc.PyroscopeKey)
@@ -134,7 +142,7 @@ func mustResolveEnvPlaceholder(input string) string {
 }
 
 func lookupEnvVarName(input string) (string, bool) {
-	re := regexp.MustCompile(`^\${{ env\.([a-zA-Z_]+) }}$`)
+	re := regexp.MustCompile(`^{{ env\.([a-zA-Z_]+) }}$`)
 	matches := re.FindStringSubmatch(input)
 	if len(matches) > 1 {
 		return matches[1], true
diff --git a/integration-tests/e2e_tests_ci_tool/cmd/types.go b/integration-tests/e2e_tests_ci_tool/cmd/types.go
index c41012a70ec..a85815835db 100644
--- a/integration-tests/e2e_tests_ci_tool/cmd/types.go
+++ b/integration-tests/e2e_tests_ci_tool/cmd/types.go
@@ -7,24 +7,17 @@ type Test struct {
 
 // CITestConf defines the configuration for running a test in a CI environment, specifying details like test ID, path, type, runner settings, command, and associated workflows.
 type CITestConf struct {
-	ID                    string             `yaml:"id" json:"id"`
-	IDSanitized           string             `json:"idSanitized"`
-	Path                  string             `yaml:"path" json:"path"`
-	TestType              string             `yaml:"test-type" json:"testType"`
-	RunsOn                string             `yaml:"runs-on" json:"runsOn"`
-	TestCmd               string             `yaml:"test-cmd" json:"testCmd"`
-	TestConfigOverride    TestConfigOverride `yaml:"test-config-override" json:"testConfigOverride"`
-	RemoteRunnerTestSuite string             `yaml:"remote-runner-test-suite" json:"remoteRunnerTestSuite"`
-	RemoteRunnerMemory    string             `yaml:"remote-runner-memory" json:"remoteRunnerMemory"`
-	PyroscopeEnv          string             `yaml:"pyroscope-env" json:"pyroscopeEnv"`
-	Workflows             []string           `yaml:"workflows" json:"workflows"`
-}
-
-type TestConfigOverride struct {
-	ChainlinkUpgradeImage   string `yaml:"chainlink-upgrade-image" json:"chainlinkUpgradeImage"`
-	ChainlinkUpgradeVersion string `yaml:"chainlink-upgrade-version" json:"chainlinkUpgradeVersion"`
-	ChainlinkImage          string `yaml:"chainlink-image" json:"chainlinkImage"`
-	ChainlinkVersion        string `yaml:"chainlink-version" json:"chainlinkVersion"`
+	ID                    string            `yaml:"id" json:"id"`
+	IDSanitized           string            `json:"idSanitized"`
+	Path                  string            `yaml:"path" json:"path"`
+	TestType              string            `yaml:"test-type" json:"testType"`
+	RunsOn                string            `yaml:"runs-on" json:"runsOn"`
+	TestCmd               string            `yaml:"test-cmd" json:"testCmd"`
+	TestConfigOverrides   map[string]string `yaml:"test-config-overrides" json:"testConfigOverrides"`
+	RemoteRunnerTestSuite string            `yaml:"remote-runner-test-suite" json:"remoteRunnerTestSuite"`
+	RemoteRunnerMemory    string            `yaml:"remote-runner-memory" json:"remoteRunnerMemory"`
+	PyroscopeEnv          string            `yaml:"pyroscope-env" json:"pyroscopeEnv"`
+	Workflows             []string          `yaml:"workflows" json:"workflows"`
 }
 
 type Config struct {

From 05efabcd3f704a1b7a2b99d530cc270ed69a5c33 Mon Sep 17 00:00:00 2001
From: lukaszcl <120112546+lukaszcl@users.noreply.github.com>
Date: Mon, 1 Jul 2024 18:22:01 +0200
Subject: [PATCH 125/137] Fix cache

---
 .github/workflows/run-e2e-tests-reusable-workflow.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/run-e2e-tests-reusable-workflow.yml b/.github/workflows/run-e2e-tests-reusable-workflow.yml
index 57b60c86aa7..62f42e5fab4 100644
--- a/.github/workflows/run-e2e-tests-reusable-workflow.yml
+++ b/.github/workflows/run-e2e-tests-reusable-workflow.yml
@@ -234,7 +234,7 @@ jobs:
             /tmp/gotest.log
           publish_check_name: ${{ matrix.tests.idSanitized }}
           token: ${{ secrets.GH_TOKEN }}
-          cache_key_id: run-docker-e2e-tests-${{ env.MOD_CACHE_VERSION }}
+          cache_key_id: run_docker_e2e_tests_${{ matrix.tests.idSanitized }}_${{ env.MOD_CACHE_VERSION }}
           go_mod_path: ./integration-tests/go.mod
           cache_restore_only: "true"
           QA_AWS_REGION: ${{ secrets.QA_AWS_REGION }}
@@ -360,7 +360,7 @@ jobs:
           cl_image_tag: ${{ inputs.chainlink_version || matrix.tests.testConfigOverrides.chainlinkVersion || github.sha }}
           token: ${{ secrets.GH_TOKEN }}
           should_cleanup: false
-          cache_key_id: run-k8s-e2e-tests-${{ env.MOD_CACHE_VERSION }}
+          cache_key_id: run_k8s_e2e_tests_${{ matrix.tests.idSanitized }}_${{ env.MOD_CACHE_VERSION }}
           go_mod_path: ./integration-tests/go.mod
           QA_AWS_REGION: ${{ secrets.QA_AWS_REGION }}
           QA_AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }}

From ca08ff7f9ae833e411e27d3b61d86ecc6c7b658a Mon Sep 17 00:00:00 2001
From: lukaszcl <120112546+lukaszcl@users.noreply.github.com>
Date: Mon, 1 Jul 2024 18:42:46 +0200
Subject: [PATCH 126/137] Resolve env vars in filter cmd

---
 .../action.yml                                | 66 +++++++++++--------
 .../run-e2e-tests-reusable-workflow.yml       | 10 +--
 .../cmd/create_test_config_cmd.go             |  2 +-
 .../cmd/envreplace_cmd_test.go                | 64 ------------------
 .../e2e_tests_ci_tool/cmd/envresolve_cmd.go   | 21 ++++++
 .../e2e_tests_ci_tool/cmd/root_cmd.go         |  1 +
 6 files changed, 65 insertions(+), 99 deletions(-)
 delete mode 100644 integration-tests/e2e_tests_ci_tool/cmd/envreplace_cmd_test.go
 create mode 100644 integration-tests/e2e_tests_ci_tool/cmd/envresolve_cmd.go

diff --git a/.github/actions/create-default-e2e-config-override/action.yml b/.github/actions/create-default-e2e-config-override/action.yml
index b2fdbe7d6a0..2f425d8f9a7 100644
--- a/.github/actions/create-default-e2e-config-override/action.yml
+++ b/.github/actions/create-default-e2e-config-override/action.yml
@@ -60,82 +60,90 @@ inputs:
   private_ethereum_network_custom_docker_image:
     required: false
     description: 'Private Ethereum network custom Docker image for the tests'
+outputs:
+  chainlink_image:
+    value: ${{ steps.build-args.outputs.chainlink_image }}
+    description: 'Chainlink image resolved from the input'
+  chainlink_version:
+    value: ${{ steps.build-args.outputs.chainlink_version }}
+    description: 'Chainlink version resolved from the input'
 
 runs:
   using: 'composite'
   steps:
-    - name: Build the command line arguments
+    - name: Create default E2E test config override
+      id: build-args
       shell: bash
       run: |
-        cmd_args=""
+        cd integration-tests/e2e_tests_ci_tool/
+
+        create_args=()
         if [ -n "${{ inputs.chainlink_image }}" ]; then
-          cmd_args+="--chainlink-image=\"${{ inputs.chainlink_image }}\" "
+          create_args+=(--chainlink-image="${{ inputs.chainlink_image }}")
+          echo "chainlink_image=$(go run main.go envresolve "${{ inputs.chainlink_image }}")" >> $GITHUB_OUTPUT
         fi
         if [ -n "${{ inputs.chainlink_version }}" ]; then
-          cmd_args+="--chainlink-version=\"${{ inputs.chainlink_version }}\" "
+          create_args+=(--chainlink-version="${{ inputs.chainlink_version }}")
+          echo "chainlink_version=$(go run main.go envresolve "${{ inputs.chainlink_version }}")" >> $GITHUB_OUTPUT
         fi
         if [ -n "${{ inputs.chainlink_postgres_version }}" ]; then
-          cmd_args+="--chainlink-postgres-version=\"${{ inputs.chainlink_postgres_version }}\" "
+          create_args+=(--chainlink-postgres-version="${{ inputs.chainlink_postgres_version }}")
         fi
         if [ -n "${{ inputs.chainlink_upgrade_image }}" ]; then
-          cmd_args+="--chainlink-upgrade-image=\"${{ inputs.chainlink_upgrade_image }}\" "
+          create_args+=(--chainlink-upgrade-image="${{ inputs.chainlink_upgrade_image }}")
         fi
         if [ -n "${{ inputs.chainlink_upgrade_version }}" ]; then
-          cmd_args+="--chainlink-upgrade-version=\"${{ inputs.chainlink_upgrade_version }}\" "
+          create_args+=(--chainlink-upgrade-version="${{ inputs.chainlink_upgrade_version }}")
         fi
         if [ -n "${{ inputs.pyroscope_server }}" ]; then
-          cmd_args+="--pyroscope-server-url=\"${{ inputs.pyroscope_server }}\" "
+          create_args+=(--pyroscope-server-url="${{ inputs.pyroscope_server }}")
         fi
         if [ -n "${{ inputs.pyroscope_environment }}" ]; then
-          cmd_args+="--pyroscope-environment=\"${{ inputs.pyroscope_environment }}\" "
+          create_args+=(--pyroscope-environment="${{ inputs.pyroscope_environment }}")
         fi
         if [ -n "${{ inputs.pyroscope_key }}" ]; then
-          cmd_args+="--pyroscope-key=\"${{ inputs.pyroscope_key }}\" "
+          create_args+=(--pyroscope-key="${{ inputs.pyroscope_key }}")
         fi
         if [ -n "${{ inputs.loki_endpoint }}" ]; then
-          cmd_args+="--logging-loki-endpoint=\"${{ inputs.loki_endpoint }}\" "
+          create_args+=(--logging-loki-endpoint="${{ inputs.loki_endpoint }}")
         fi
         if [ -n "${{ inputs.loki_tenant_id }}" ]; then
-          cmd_args+="--logging-loki-tenant-id=\"${{ inputs.loki_tenant_id }}\" "
+          create_args+=(--logging-loki-tenant-id="${{ inputs.loki_tenant_id }}")
         fi
         if [ -n "${{ inputs.loki_basic_auth }}" ]; then
-          cmd_args+="--logging-loki-basic-auth=\"${{ inputs.loki_basic_auth }}\" "
+          create_args+=(--logging-loki-basic-auth="${{ inputs.loki_basic_auth }}")
         fi
         if [ -n "${{ inputs.loki_run_id }}" ]; then
-          cmd_args+="--logging-run-id=\"${{ inputs.loki_run_id }}\" "
+          create_args+=(--logging-run-id="${{ inputs.loki_run_id }}")
         fi
         if [ -n "${{ inputs.test_log_collect }}" ]; then
-          cmd_args+="--logging-test-log-collect=\"${{ inputs.test_log_collect }}\" "
+          create_args+=(--logging-test-log-collect="${{ inputs.test_log_collect }}")
         fi
         if [ -n "${{ inputs.private_ethereum_network_execution_layer }}" ]; then
-          cmd_args+="--private-ethereum-network-execution-layer=\"${{ inputs.private_ethereum_network_execution_layer }}\" "
+          create_args+=(--private-ethereum-network-execution-layer="${{ inputs.private_ethereum_network_execution_layer }}")
         fi
         if [ -n "${{ inputs.private_ethereum_network_custom_docker_image }}" ]; then
-          cmd_args+="--private-ethereum-network-custom-docker-image=\"${{ inputs.private_ethereum_network_custom_docker_image }}\" "
+          create_args+=(--private-ethereum-network-custom-docker-image="${{ inputs.private_ethereum_network_custom_docker_image }}")
         fi
         
-        # Split the log targets input by comma and add to the command line arguments
+        # # Split the log targets input by comma and add to the command line arguments
         IFS=',' read -ra ADDR <<< "${{ inputs.logstream_log_targets }}"
         for i in "${ADDR[@]}"; do
-          cmd_args+="--logging-log-targets=\"$i\" "
+          create_args+=(--logging-log-targets="$i")
         done
 
         # Split selected_networks input by comma and add to the command line arguments
         IFS=',' read -ra ADDR <<< "${{ inputs.selected_networks }}"
         for i in "${ADDR[@]}"; do
-          cmd_args+="--selected-networks=\"$i\" "
+          create_args+=(--selected-networks="$i")
         done
 
-        echo "Command line arguments: $cmd_args" 
-        echo "CMD_ARGS=$cmd_args" >> $GITHUB_ENV
-
-    - name: Run the command to create the test config and set it as BASE64_CONFIG_OVERRIDE env variable
-      shell: bash
-      run: |      
-        cd integration-tests/e2e_tests_ci_tool/
-        config_override=$(go run main.go test-config create ${{ env.CMD_ARGS }})
+        config_override=$(go run main.go test-config create "${create_args[@]}")
+        # echo "go run main.go test-config create ${create_args[*]}"
+        # echo "Output:"
+        # echo "$config_override"
 
         BASE64_CONFIG_OVERRIDE=$(echo "$config_override" | base64 -w 0)
         echo ::add-mask::$BASE64_CONFIG_OVERRIDE
-        echo "BASE64_CONFIG_OVERRIDE=$BASE64_CONFIG_OVERRIDE" >> $GITHUB_ENV
+        echo "BASE64_CONFIG_OVERRIDE=$BASE64_CONFIG_OVERRIDE" >> $GITHUB_ENV        
     
\ No newline at end of file
diff --git a/.github/workflows/run-e2e-tests-reusable-workflow.yml b/.github/workflows/run-e2e-tests-reusable-workflow.yml
index 62f42e5fab4..ae7ae8e0c3c 100644
--- a/.github/workflows/run-e2e-tests-reusable-workflow.yml
+++ b/.github/workflows/run-e2e-tests-reusable-workflow.yml
@@ -66,7 +66,8 @@ on:
                
 env: 
   CHAINLINK_IMAGE: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com/chainlink
-  MOD_CACHE_VERSION: 2
+  QA_CHAINLINK_IMAGE: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com/chainlink
+  MOD_CACHE_VERSION: 1
 
 jobs:
   validate-inputs:
@@ -301,7 +302,6 @@ jobs:
       CHAINLINK_COMMIT_SHA: ${{ inputs.evm-ref || github.sha }}
       CHAINLINK_ENV_USER: ${{ github.actor }}
       TEST_LOG_LEVEL: debug
-      QA_CHAINLINK_IMAGE: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com/chainlink
 
     steps:
       - name: Checkout repository
@@ -315,7 +315,7 @@ jobs:
           echo "Remote Runner Version: ${{ needs.prepare-remote-runner-test-image.outputs.remote-runner-version }}"
 
       - name: Create default test config override (use overrides from e2e-tests.yml or defaults)
-        # if: ${{ inputs.test_config_base64_override == '' }}
+        id: create-default-test-config-override
         uses: ./.github/actions/create-default-e2e-config-override
         with:
           chainlink_version: ${{ inputs.chainlink_version || matrix.tests.testConfigOverrides.chainlinkVersion || github.sha }}
@@ -356,8 +356,8 @@ jobs:
         with:
           test_command_to_run: ${{ matrix.tests.testCmd }}
           test_download_vendor_packages_command: make gomod
-          cl_repo: ${{ matrix.tests.testConfigOverrides.chainlinkImage || env.CHAINLINK_IMAGE }}
-          cl_image_tag: ${{ inputs.chainlink_version || matrix.tests.testConfigOverrides.chainlinkVersion || github.sha }}
+          cl_repo: ${{ steps.create-default-test-config-override.outputs.chainlink_image }}
+          cl_image_tag: ${{ steps.create-default-test-config-override.outputs.chainlink_version }}
           token: ${{ secrets.GH_TOKEN }}
           should_cleanup: false
           cache_key_id: run_k8s_e2e_tests_${{ matrix.tests.idSanitized }}_${{ env.MOD_CACHE_VERSION }}
diff --git a/integration-tests/e2e_tests_ci_tool/cmd/create_test_config_cmd.go b/integration-tests/e2e_tests_ci_tool/cmd/create_test_config_cmd.go
index 45c4ee90ab3..384848e4ea5 100644
--- a/integration-tests/e2e_tests_ci_tool/cmd/create_test_config_cmd.go
+++ b/integration-tests/e2e_tests_ci_tool/cmd/create_test_config_cmd.go
@@ -45,7 +45,7 @@ var createTestConfigCmd = &cobra.Command{
 		if cmd.Flags().Changed(ChainlinkUpgradeVersionFlag) {
 			upgradeVersion = &oc.ChainlinkUpgradeVersion
 		}
-		if upgradeImage != nil || upgradeVersion == nil {
+		if upgradeImage != nil || upgradeVersion != nil {
 			tc.ChainlinkUpgradeImage = &ctf_config.ChainlinkImageConfig{
 				Image:   upgradeImage,
 				Version: upgradeVersion,
diff --git a/integration-tests/e2e_tests_ci_tool/cmd/envreplace_cmd_test.go b/integration-tests/e2e_tests_ci_tool/cmd/envreplace_cmd_test.go
deleted file mode 100644
index a20a6fba5d2..00000000000
--- a/integration-tests/e2e_tests_ci_tool/cmd/envreplace_cmd_test.go
+++ /dev/null
@@ -1,64 +0,0 @@
-package cmd
-
-import (
-	"os"
-	"strings"
-	"testing"
-)
-
-func TestReplaceEnvPlaceholders(t *testing.T) {
-	// Define test cases
-	tests := []struct {
-		name       string
-		input      string
-		want       string
-		wantErr    bool
-		wantErrMsg string
-	}{
-		{
-			name:    "All variables set",
-			input:   "Path: ${{ env.PATH }}, Home: ${{ env.HOME }}",
-			want:    "Path: /usr/bin, Home: /home/user",
-			wantErr: false,
-		},
-		{
-			name:       "One variable unset",
-			input:      "Path: ${{ env.PATH }}, Unset: ${{ env.UNSET_VAR }}",
-			want:       "Path: /usr/bin, Unset: ${{ env.UNSET_VAR }}",
-			wantErr:    true,
-			wantErrMsg: "environment variable 'UNSET_VAR' not set or is empty",
-		},
-		{
-			name:       "Multiple variables unset",
-			input:      "Unset1: ${{ env.UNSET_VAR1 }}, Unset2: ${{ env.UNSET_VAR2 }}",
-			want:       "Unset1: ${{ env.UNSET_VAR1 }}, Unset2: ${{ env.UNSET_VAR2 }}",
-			wantErr:    true,
-			wantErrMsg: "environment variable 'UNSET_VAR1' not set or is empty, environment variable 'UNSET_VAR2' not set or is empty",
-		},
-	}
-
-	// Set environment variables for the test
-	os.Setenv("PATH", "/usr/bin")
-	os.Setenv("HOME", "/home/user")
-	defer os.Unsetenv("PATH")
-	defer os.Unsetenv("HOME")
-
-	for _, tt := range tests {
-		t.Run(tt.name, func(t *testing.T) {
-			got, err := replaceEnvPlaceholders(tt.input)
-			if (err != nil) != tt.wantErr {
-				t.Fatalf("replaceEnvPlaceholders() error = %v, wantErr %v", err, tt.wantErr)
-			}
-			if err != nil && !strings.Contains(err.Error(), tt.wantErrMsg) {
-				t.Errorf("replaceEnvPlaceholders() error = %v, wantErrMsg %v", err, tt.wantErrMsg)
-			}
-			if got != tt.want {
-				t.Errorf("replaceEnvPlaceholders() got = %v, want %v", got, tt.want)
-			}
-		})
-	}
-
-	// Unset the variables to clean up after tests
-	os.Unsetenv("UNSET_VAR1")
-	os.Unsetenv("UNSET_VAR2")
-}
diff --git a/integration-tests/e2e_tests_ci_tool/cmd/envresolve_cmd.go b/integration-tests/e2e_tests_ci_tool/cmd/envresolve_cmd.go
new file mode 100644
index 00000000000..977ce66e536
--- /dev/null
+++ b/integration-tests/e2e_tests_ci_tool/cmd/envresolve_cmd.go
@@ -0,0 +1,21 @@
+package cmd
+
+import (
+	"fmt"
+
+	"github.com/spf13/cobra"
+)
+
+var envresolveCmd = &cobra.Command{
+	Use:   "envresolve",
+	Short: "Resolve environment variables in a string",
+	Run: func(cmd *cobra.Command, args []string) {
+		if len(args) < 1 {
+			fmt.Fprintln(cmd.OutOrStdout(), "Error: No input provided")
+			return
+		}
+		input := args[0]
+
+		fmt.Fprintln(cmd.OutOrStdout(), mustResolveEnvPlaceholder(input))
+	},
+}
diff --git a/integration-tests/e2e_tests_ci_tool/cmd/root_cmd.go b/integration-tests/e2e_tests_ci_tool/cmd/root_cmd.go
index 17c527168ba..3be05dbc730 100644
--- a/integration-tests/e2e_tests_ci_tool/cmd/root_cmd.go
+++ b/integration-tests/e2e_tests_ci_tool/cmd/root_cmd.go
@@ -24,6 +24,7 @@ func Execute() {
 func init() {
 	rootCmd.Flags().BoolP("toggle", "t", false, "Help message for toggle")
 
+	rootCmd.AddCommand(envresolveCmd)
 	rootCmd.AddCommand(checkTestsCmd)
 	rootCmd.AddCommand(filterCmd)
 	rootCmd.AddCommand(testConfigCmd)

From 2728c0b3155c8a0ef7ba89ac70101d69afb7942d Mon Sep 17 00:00:00 2001
From: lukaszcl <120112546+lukaszcl@users.noreply.github.com>
Date: Mon, 1 Jul 2024 23:29:15 +0200
Subject: [PATCH 127/137] Fix

---
 .github/workflows/run-automation-ondemand-e2e-tests.yml | 2 --
 .github/workflows/run-e2e-tests-reusable-workflow.yml   | 8 ++++----
 2 files changed, 4 insertions(+), 6 deletions(-)

diff --git a/.github/workflows/run-automation-ondemand-e2e-tests.yml b/.github/workflows/run-automation-ondemand-e2e-tests.yml
index 92a36fdb202..6aef49979b7 100644
--- a/.github/workflows/run-automation-ondemand-e2e-tests.yml
+++ b/.github/workflows/run-automation-ondemand-e2e-tests.yml
@@ -120,7 +120,6 @@ jobs:
           EOF
           fi
 
-          echo "chainlink_version=$CHAINLINK_VERSION" >> $GITHUB_OUTPUT
           echo "test_map=$(cat test_map.yaml | base64 -w 0)" >> $GITHUB_OUTPUT
 
   call-run-e2e-tests-workflow:
@@ -128,7 +127,6 @@ jobs:
     needs: set-tests-to-run
     uses: ./.github/workflows/run-e2e-tests-reusable-workflow.yml
     with:
-      chainlink_version: ${{ needs.set-tests-to-run.outputs.chainlink_version }}
       test_map: ${{ needs.set-tests-to-run.outputs.test_map }}
       with_existing_remote_runner_version: ${{ github.event.inputs.with_existing_remote_runner_version }}
     secrets:
diff --git a/.github/workflows/run-e2e-tests-reusable-workflow.yml b/.github/workflows/run-e2e-tests-reusable-workflow.yml
index ae7ae8e0c3c..1f0777da63c 100644
--- a/.github/workflows/run-e2e-tests-reusable-workflow.yml
+++ b/.github/workflows/run-e2e-tests-reusable-workflow.yml
@@ -191,7 +191,7 @@ jobs:
         uses: ./.github/actions/setup-go
 
       - name: Create default test config override (use overrides from e2e-tests.yml or defaults)
-        # if: ${{ inputs.test_config_base64_override == '' }}
+        id: create-default-test-config-override
         uses: ./.github/actions/create-default-e2e-config-override
         with:
           chainlink_image: ${{ matrix.tests.testConfigOverrides.chainlinkImage || env.CHAINLINK_IMAGE }}
@@ -226,8 +226,8 @@ jobs:
         with:
           test_command_to_run: ${{ matrix.tests.testCmd }} 2>&1 | tee /tmp/gotest.log | gotestloghelper -ci -singlepackage -hidepassingtests=false -hidepassinglogs
           test_download_vendor_packages_command: cd ./integration-tests && go mod download
-          cl_repo: ${{ env.CHAINLINK_IMAGE }}
-          cl_image_tag: ${{ inputs.chainlink_version }}
+          cl_repo: ${{ steps.create-default-test-config-override.outputs.chainlink_image }}
+          cl_image_tag: ${{ steps.create-default-test-config-override.outputs.chainlink_version }}
           aws_registries: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}
           artifacts_name: ${{ matrix.tests.idSanitized }}-test-logs
           artifacts_location: |
@@ -318,8 +318,8 @@ jobs:
         id: create-default-test-config-override
         uses: ./.github/actions/create-default-e2e-config-override
         with:
-          chainlink_version: ${{ inputs.chainlink_version || matrix.tests.testConfigOverrides.chainlinkVersion || github.sha }}
           chainlink_image: ${{ matrix.tests.testConfigOverrides.chainlinkImage || env.CHAINLINK_IMAGE }}
+          chainlink_version: ${{ inputs.chainlink_version || matrix.tests.testConfigOverrides.chainlinkVersion || github.sha }}
           chainlink_postgres_version: ${{ matrix.tests.testConfigOverrides.chainlinkPostgresVersion }}
           chainlink_upgrade_image: ${{ matrix.tests.testConfigOverrides.chainlinkUpgradeImage }}
           chainlink_upgrade_version: ${{ matrix.tests.testConfigOverrides.chainlinkUpgradeVersion }}

From fbed49c7782f86d67c309064d6c4dbf94e9419e2 Mon Sep 17 00:00:00 2001
From: lukaszcl <120112546+lukaszcl@users.noreply.github.com>
Date: Mon, 1 Jul 2024 23:53:24 +0200
Subject: [PATCH 128/137] Rename test overrides to test inputs

---
 .github/e2e-tests.yml                         | 30 +++++++++----------
 .../run-automation-ondemand-e2e-tests.yml     |  2 +-
 .../run-e2e-tests-reusable-workflow.yml       | 28 ++++++++---------
 .../e2e_tests_ci_tool/cmd/filter_cmd.go       | 10 +++----
 .../e2e_tests_ci_tool/cmd/types.go            |  2 +-
 5 files changed, 36 insertions(+), 36 deletions(-)

diff --git a/.github/e2e-tests.yml b/.github/e2e-tests.yml
index 3aa23adb760..3f966d81f52 100644
--- a/.github/e2e-tests.yml
+++ b/.github/e2e-tests.yml
@@ -48,11 +48,11 @@ runner-test-matrix:
     workflows:
       - Run Automation Product Nightly E2E Tests
     test-cmd: cd integration-tests/smoke && go test -test.run ^TestAutomationNodeUpgrade/registry_2_0 -test.parallel=1 -timeout 60m -count=1 -json
-    test-config-overrides:
-      chainlinkUpgradeImage: public.ecr.aws/chainlink/chainlink 
-      chainlinkUpgradeVersion: latest
-      chainlinkImage: '{{ env.QA_CHAINLINK_IMAGE }}'
-      chainlinkVersion: develop
+    test-inputs:
+      chainlinkImage: public.ecr.aws/chainlink/chainlink
+      chainlinkVersion: latest
+      chainlinkUpgradeImage: {{ env.QA_CHAINLINK_IMAGE }}
+      chainlinkUpgradeVersion: develop
     pyroscope-env: ci-smoke-automation-upgrade-tests
 
   - id: smoke/automation_upgrade_test.go:^TestAutomationNodeUpgrade/registry_2_1
@@ -62,11 +62,11 @@ runner-test-matrix:
     workflows:
       - Run Automation Product Nightly E2E Tests    
     test-cmd: cd integration-tests/smoke && go test -test.run ^TestAutomationNodeUpgrade/registry_2_1 -test.parallel=5 -timeout 60m -count=1 -json
-    test-config-overrides:
-      chainlinkUpgradeImage: public.ecr.aws/chainlink/chainlink 
-      chainlinkUpgradeVersion: latest
-      chainlinkImage: '{{ env.QA_CHAINLINK_IMAGE }}'
-      chainlinkVersion: develop
+    test-inputs:
+      chainlinkImage: public.ecr.aws/chainlink/chainlink
+      chainlinkVersion: latest
+      chainlinkUpgradeImage: {{ env.QA_CHAINLINK_IMAGE }}
+      chainlinkUpgradeVersion: develop
     pyroscope-env: ci-smoke-automation-upgrade-tests
 
   - id: smoke/automation_upgrade_test.go:^TestAutomationNodeUpgrade/registry_2_2
@@ -76,11 +76,11 @@ runner-test-matrix:
     workflows:
       - Run Automation Product Nightly E2E Tests
     test-cmd: cd integration-tests/smoke && go test -test.run ^TestAutomationNodeUpgrade/registry_2_2 -test.parallel=5 -timeout 60m -count=1 -json
-    test-config-overrides:
-      chainlinkUpgradeImage: public.ecr.aws/chainlink/chainlink 
-      chainlinkUpgradeVersion: latest
-      chainlinkImage: '{{ env.QA_CHAINLINK_IMAGE }}'
-      chainlinkVersion: develop
+    test-inputs:
+      chainlinkImage: public.ecr.aws/chainlink/chainlink
+      chainlinkVersion: latest
+      chainlinkUpgradeImage: {{ env.QA_CHAINLINK_IMAGE }}
+      chainlinkUpgradeVersion: develop
     pyroscope-env: ci-smoke-automation-upgrade-tests
 
   # END Automation upgrade tests
diff --git a/.github/workflows/run-automation-ondemand-e2e-tests.yml b/.github/workflows/run-automation-ondemand-e2e-tests.yml
index 6aef49979b7..3259f7d8085 100644
--- a/.github/workflows/run-automation-ondemand-e2e-tests.yml
+++ b/.github/workflows/run-automation-ondemand-e2e-tests.yml
@@ -79,7 +79,7 @@ jobs:
           # Always run upgrade tests 
           cat > test_map.yaml <<EOF
           - id: smoke/automation_upgrade_test.go:^TestAutomationNodeUpgrade/registry_2_0
-            test-config-overrides:
+            test-inputs:
               chainlinkImage: $CHAINLINK_IMAGE
               chainlinkVersion: $CHAINLINK_VERSION
               chainlinkUpgradeImage: $CHAINLINK_UPGRADE_IMAGE
diff --git a/.github/workflows/run-e2e-tests-reusable-workflow.yml b/.github/workflows/run-e2e-tests-reusable-workflow.yml
index 1f0777da63c..ec832d7afa9 100644
--- a/.github/workflows/run-e2e-tests-reusable-workflow.yml
+++ b/.github/workflows/run-e2e-tests-reusable-workflow.yml
@@ -190,16 +190,16 @@ jobs:
       - name: Setup Go
         uses: ./.github/actions/setup-go
 
-      - name: Create default test config override (use overrides from e2e-tests.yml or defaults)
+      - name: Create default test config override (use test inputs from e2e-tests.yml or defaults)
         id: create-default-test-config-override
         uses: ./.github/actions/create-default-e2e-config-override
         with:
-          chainlink_image: ${{ matrix.tests.testConfigOverrides.chainlinkImage || env.CHAINLINK_IMAGE }}
-          chainlink_version: ${{ inputs.chainlink_version || matrix.tests.testConfigOverrides.chainlinkVersion || github.sha }}
-          chainlink_postgres_version: ${{ matrix.tests.testConfigOverrides.chainlinkPostgresVersion }}
-          chainlink_upgrade_image: ${{ matrix.tests.testConfigOverrides.chainlinkUpgradeImage }}
-          chainlink_upgrade_version: ${{ matrix.tests.testConfigOverrides.chainlinkUpgradeVersion }}
-          selected_networks: ${{ matrix.tests.testConfigOverrides.selectedNetworks || env.SELECTED_NETWORKS}}
+          chainlink_image: ${{ matrix.tests.testInputs.chainlinkImage || env.CHAINLINK_IMAGE }}
+          chainlink_version: ${{ inputs.chainlink_version || matrix.tests.testInputs.chainlinkVersion || github.sha }}
+          chainlink_postgres_version: ${{ matrix.tests.testInputs.chainlinkPostgresVersion }}
+          chainlink_upgrade_image: ${{ matrix.tests.testInputs.chainlinkUpgradeImage }}
+          chainlink_upgrade_version: ${{ matrix.tests.testInputs.chainlinkUpgradeVersion }}
+          selected_networks: ${{ matrix.tests.testInputs.selectedNetworks || env.SELECTED_NETWORKS}}
           pyroscope_server: ${{ secrets.QA_PYROSCOPE_INSTANCE }}
           pyroscope_environment: ${{ matrix.tests.pyroscopeEnv }}
           pyroscope_key: ${{ secrets.QA_PYROSCOPE_KEY }}
@@ -314,16 +314,16 @@ jobs:
         run: |
           echo "Remote Runner Version: ${{ needs.prepare-remote-runner-test-image.outputs.remote-runner-version }}"
 
-      - name: Create default test config override (use overrides from e2e-tests.yml or defaults)
+      - name: Create default test config override (use test inputs from e2e-tests.yml or defaults)
         id: create-default-test-config-override
         uses: ./.github/actions/create-default-e2e-config-override
         with:
-          chainlink_image: ${{ matrix.tests.testConfigOverrides.chainlinkImage || env.CHAINLINK_IMAGE }}
-          chainlink_version: ${{ inputs.chainlink_version || matrix.tests.testConfigOverrides.chainlinkVersion || github.sha }}
-          chainlink_postgres_version: ${{ matrix.tests.testConfigOverrides.chainlinkPostgresVersion }}
-          chainlink_upgrade_image: ${{ matrix.tests.testConfigOverrides.chainlinkUpgradeImage }}
-          chainlink_upgrade_version: ${{ matrix.tests.testConfigOverrides.chainlinkUpgradeVersion }}
-          selected_networks: ${{ matrix.tests.testConfigOverrides.selectedNetworks || env.SELECTED_NETWORKS}}
+          chainlink_image: ${{ matrix.tests.testInputs.chainlinkImage || env.CHAINLINK_IMAGE }}
+          chainlink_version: ${{ inputs.chainlink_version || matrix.tests.testInputs.chainlinkVersion || github.sha }}
+          chainlink_postgres_version: ${{ matrix.tests.testInputs.chainlinkPostgresVersion }}
+          chainlink_upgrade_image: ${{ matrix.tests.testInputs.chainlinkUpgradeImage }}
+          chainlink_upgrade_version: ${{ matrix.tests.testInputs.chainlinkUpgradeVersion }}
+          selected_networks: ${{ matrix.tests.testInputs.selectedNetworks || env.SELECTED_NETWORKS}}
           pyroscope_server: ${{ secrets.QA_PYROSCOPE_INSTANCE }}
           pyroscope_environment: ${{ matrix.tests.pyroscopeEnv }}
           pyroscope_key: ${{ secrets.QA_PYROSCOPE_KEY }}
diff --git a/integration-tests/e2e_tests_ci_tool/cmd/filter_cmd.go b/integration-tests/e2e_tests_ci_tool/cmd/filter_cmd.go
index 4bafeb9af34..96c89ee512b 100644
--- a/integration-tests/e2e_tests_ci_tool/cmd/filter_cmd.go
+++ b/integration-tests/e2e_tests_ci_tool/cmd/filter_cmd.go
@@ -57,12 +57,12 @@ func filterAndMergeTests(allTests []CITestConf, workflow, testType, base64Tests
 		typeMatch := testType == "" || test.TestType == testType
 
 		if decodedTest, exists := idFilter[test.ID]; exists && workflowMatch && typeMatch {
-			// Apply config overrides
-			for k, v := range decodedTest.TestConfigOverrides {
-				if test.TestConfigOverrides == nil {
-					test.TestConfigOverrides = make(map[string]string)
+			// Override test inputs from the base64 encoded tests
+			for k, v := range decodedTest.TestInputs {
+				if test.TestInputs == nil {
+					test.TestInputs = make(map[string]string)
 				}
-				test.TestConfigOverrides[k] = v
+				test.TestInputs[k] = v
 			}
 			test.IDSanitized = sanitizeTestID(test.ID)
 			filteredTests = append(filteredTests, test)
diff --git a/integration-tests/e2e_tests_ci_tool/cmd/types.go b/integration-tests/e2e_tests_ci_tool/cmd/types.go
index a85815835db..3479c8b9f13 100644
--- a/integration-tests/e2e_tests_ci_tool/cmd/types.go
+++ b/integration-tests/e2e_tests_ci_tool/cmd/types.go
@@ -13,7 +13,7 @@ type CITestConf struct {
 	TestType              string            `yaml:"test-type" json:"testType"`
 	RunsOn                string            `yaml:"runs-on" json:"runsOn"`
 	TestCmd               string            `yaml:"test-cmd" json:"testCmd"`
-	TestConfigOverrides   map[string]string `yaml:"test-config-overrides" json:"testConfigOverrides"`
+	TestInputs            map[string]string `yaml:"test-inputs" json:"testInputs"`
 	RemoteRunnerTestSuite string            `yaml:"remote-runner-test-suite" json:"remoteRunnerTestSuite"`
 	RemoteRunnerMemory    string            `yaml:"remote-runner-memory" json:"remoteRunnerMemory"`
 	PyroscopeEnv          string            `yaml:"pyroscope-env" json:"pyroscopeEnv"`

From 594e948794bcb632f0e849a3fb28cbca221ed90c Mon Sep 17 00:00:00 2001
From: lukaszcl <120112546+lukaszcl@users.noreply.github.com>
Date: Mon, 1 Jul 2024 23:55:29 +0200
Subject: [PATCH 129/137] Update input descriptions

---
 .../run-automation-ondemand-e2e-tests.yml     | 28 +++++++++----------
 .../run-e2e-tests-reusable-workflow.yml       |  8 +++---
 .../e2e_tests_ci_tool/cmd/filter_cmd.go       |  4 +--
 3 files changed, 20 insertions(+), 20 deletions(-)

diff --git a/.github/workflows/run-automation-ondemand-e2e-tests.yml b/.github/workflows/run-automation-ondemand-e2e-tests.yml
index 3259f7d8085..2cc76d180ec 100644
--- a/.github/workflows/run-automation-ondemand-e2e-tests.yml
+++ b/.github/workflows/run-automation-ondemand-e2e-tests.yml
@@ -4,19 +4,19 @@ on:
   workflow_dispatch:
     inputs:
       chainlinkImage:
-        description: Chainlink image repo to use (Leave empty to build from head/ref)
+        description: Chainlink image repo to use to upgrade Chainlink Nodes from
         options:
           - QA_ECR
           - public.ecr.aws/chainlink/chainlink
         default: QA_ECR
         type: choice
       chainlinkVersion:
-        description: Chainlink image version to use
+        description: Chainlink image version to use to upgrade Chainlink Nodes from
         required: false
         default: develop
         type: string        
       chainlinkImageUpdate:
-        description: Chainlink image repo to use initially for upgrade test
+        description: Chainlink image repo to use to upgrade Chainlink Nodes to
         required: true
         options:
           - QA_ECR
@@ -24,7 +24,7 @@ on:
         default: QA_ECR
         type: choice
       chainlinkVersionUpdate:
-        description: Chainlink image version to use initially for upgrade test
+        description: Chainlink image version to use to upgrade Chainlink Nodes to
         default: develop
         required: true
         type: string
@@ -49,7 +49,7 @@ jobs:
     name: Set tests to run
     runs-on: ubuntu-latest
     outputs:
-      test_map: ${{ steps.set-tests.outputs.test_map }}
+      test_list: ${{ steps.set-tests.outputs.test_list }}
     steps:
       - name: Set tests to run
         id: set-tests
@@ -77,7 +77,7 @@ jobs:
           CHAINLINK_UPGRADE_VERSION="${{ inputs.chainlinkVersionUpdate }}"
 
           # Always run upgrade tests 
-          cat > test_map.yaml <<EOF
+          cat > test_list.yaml <<EOF
           - id: smoke/automation_upgrade_test.go:^TestAutomationNodeUpgrade/registry_2_0
             test-inputs:
               chainlinkImage: $CHAINLINK_IMAGE
@@ -86,14 +86,14 @@ jobs:
               chainlinkUpgradeVersion: $CHAINLINK_UPGRADE_VERSION
 
           - id: smoke/automation_upgrade_test.go:^TestAutomationNodeUpgrade/registry_2_1
-            test-config-overrides:
+            test-inputs:
               chainlinkImage: $CHAINLINK_IMAGE
               chainlinkVersion: $CHAINLINK_VERSION
               chainlinkUpgradeImage: $CHAINLINK_UPGRADE_IMAGE
               chainlinkUpgradeVersion: $CHAINLINK_UPGRADE_VERSION
 
           - id: smoke/automation_upgrade_test.go:^TestAutomationNodeUpgrade/registry_2_2
-            test-config-overrides:
+            test-inputs:
               chainlinkImage: $CHAINLINK_IMAGE
               chainlinkVersion: $CHAINLINK_VERSION
               chainlinkUpgradeImage: $CHAINLINK_UPGRADE_IMAGE
@@ -102,9 +102,9 @@ jobs:
 
           # Run reorg tests if enabled
           if [[ "${{ github.event.inputs.enableReorg }}" == 'true' ]]; then
-          cat >> test_map.yaml <<EOF
+          cat >> test_list.yaml <<EOF
           - id: integration-tests/reorg/automation_reorg_test.go
-            test-config-overrides:
+            test-inputs:
               chainlinkImage: $CHAINLINK_IMAGE
               chainlinkVersion: $CHAINLINK_VERSION
           EOF
@@ -112,22 +112,22 @@ jobs:
 
           # Run chaos tests if enabled
           if [[ "${{ github.event.inputs.enableChaos }}" == 'true' ]]; then
-          cat >> test_map.yaml <<EOF
+          cat >> test_list.yaml <<EOF
           - id: integration-tests/chaos/automation_chaos_test.go
-            test-config-overrides:
+            test-inputs:
               chainlinkImage: $CHAINLINK_IMAGE
               chainlinkVersion: $CHAINLINK_VERSION
           EOF
           fi
 
-          echo "test_map=$(cat test_map.yaml | base64 -w 0)" >> $GITHUB_OUTPUT
+          echo "test_list=$(cat test_list.yaml | base64 -w 0)" >> $GITHUB_OUTPUT
 
   call-run-e2e-tests-workflow:
     name: Run E2E Tests
     needs: set-tests-to-run
     uses: ./.github/workflows/run-e2e-tests-reusable-workflow.yml
     with:
-      test_map: ${{ needs.set-tests-to-run.outputs.test_map }}
+      test_list: ${{ needs.set-tests-to-run.outputs.test_list }}
       with_existing_remote_runner_version: ${{ github.event.inputs.with_existing_remote_runner_version }}
     secrets:
       QA_AWS_REGION: ${{ secrets.QA_AWS_REGION }}
diff --git a/.github/workflows/run-e2e-tests-reusable-workflow.yml b/.github/workflows/run-e2e-tests-reusable-workflow.yml
index ec832d7afa9..583c2e9f14d 100644
--- a/.github/workflows/run-e2e-tests-reusable-workflow.yml
+++ b/.github/workflows/run-e2e-tests-reusable-workflow.yml
@@ -12,8 +12,8 @@ on:
         description: 'Run tests by test ids separated by commas. Example: "run_all_in_ocr_tests_go,run_TestOCRv2Request_in_ocr2_test_go". Check all test IDs in .github/e2e-tests.yml'
         required: false
         type: string
-      test_map:
-        description: 'Run tests by test map.'
+      test_list:
+        description: 'Base64 encoded list of tests (YML objects) to run. Example in run-automation-ondemand-e2e-tests.yml'
         required: true
         type: string            
       test_workflow:
@@ -117,7 +117,7 @@ jobs:
         id: set-docker-matrix
         run: |
           cd integration-tests/e2e_tests_ci_tool/
-          MATRIX_JSON=$(go run main.go filter --file ../../.github/e2e-tests.yml --test-type 'docker' --test-map '${{ inputs.test_map }}' --test-ids '${{ inputs.test_ids }}' --workflow '${{ inputs.test_workflow }}')
+          MATRIX_JSON=$(go run main.go filter --file ../../.github/e2e-tests.yml --test-type 'docker' --test-list '${{ inputs.test_list }}' --test-ids '${{ inputs.test_ids }}' --workflow '${{ inputs.test_workflow }}')
           echo "Docker tests:"
           echo "$MATRIX_JSON" | jq
           echo "matrix=$MATRIX_JSON" >> $GITHUB_OUTPUT
@@ -125,7 +125,7 @@ jobs:
         id: set-k8s-runner-matrix
         run: |
           cd integration-tests/e2e_tests_ci_tool/
-          MATRIX_JSON=$(go run main.go filter --file ../../.github/e2e-tests.yml --test-type 'k8s-remote-runner' --test-map '${{ inputs.test_map }}' --test-ids '${{ inputs.test_ids }}' --workflow '${{ inputs.test_workflow }}')
+          MATRIX_JSON=$(go run main.go filter --file ../../.github/e2e-tests.yml --test-type 'k8s-remote-runner' --test-list '${{ inputs.test_list }}' --test-ids '${{ inputs.test_ids }}' --workflow '${{ inputs.test_workflow }}')
           echo "K8s tests:"
           echo "$MATRIX_JSON" | jq
           echo "matrix=$MATRIX_JSON" >> $GITHUB_OUTPUT
diff --git a/integration-tests/e2e_tests_ci_tool/cmd/filter_cmd.go b/integration-tests/e2e_tests_ci_tool/cmd/filter_cmd.go
index 96c89ee512b..a2d3758166b 100644
--- a/integration-tests/e2e_tests_ci_tool/cmd/filter_cmd.go
+++ b/integration-tests/e2e_tests_ci_tool/cmd/filter_cmd.go
@@ -101,7 +101,7 @@ Example usage:
 		workflow, _ := cmd.Flags().GetString("workflow")
 		testType, _ := cmd.Flags().GetString("test-type")
 		testIDs, _ := cmd.Flags().GetString("test-ids")
-		testMap, _ := cmd.Flags().GetString("test-map")
+		testMap, _ := cmd.Flags().GetString("test-list")
 
 		data, err := os.ReadFile(yamlFile)
 		if err != nil {
@@ -135,7 +135,7 @@ Example usage:
 
 func init() {
 	filterCmd.Flags().StringP("file", "f", "", "Path to the YAML file")
-	filterCmd.Flags().String("test-map", "", "Base64 encoded list of tests (YML objects) to filter by. Can include test-config-overrides for each test.")
+	filterCmd.Flags().String("test-list", "", "Base64 encoded list of tests (YML objects) to filter by. Can include test-inputs for each test.")
 	filterCmd.Flags().StringP("test-ids", "i", "*", "Comma-separated list of test IDs to filter by")
 	filterCmd.Flags().StringP("test-type", "y", "", "Type of test to filter by")
 	filterCmd.Flags().StringP("workflow", "t", "", "Workflow filter")

From 3fb53ba5497a4cb3342347cb10270cc96c6b6bf8 Mon Sep 17 00:00:00 2001
From: lukaszcl <120112546+lukaszcl@users.noreply.github.com>
Date: Tue, 2 Jul 2024 10:24:41 +0200
Subject: [PATCH 130/137] Fix

---
 .github/e2e-tests.yml                                 | 6 +++---
 integration-tests/e2e_tests_ci_tool/cmd/filter_cmd.go | 9 ++++++---
 2 files changed, 9 insertions(+), 6 deletions(-)

diff --git a/.github/e2e-tests.yml b/.github/e2e-tests.yml
index 3f966d81f52..a979619ac5c 100644
--- a/.github/e2e-tests.yml
+++ b/.github/e2e-tests.yml
@@ -51,7 +51,7 @@ runner-test-matrix:
     test-inputs:
       chainlinkImage: public.ecr.aws/chainlink/chainlink
       chainlinkVersion: latest
-      chainlinkUpgradeImage: {{ env.QA_CHAINLINK_IMAGE }}
+      chainlinkUpgradeImage: '{{ env.QA_CHAINLINK_IMAGE }}'
       chainlinkUpgradeVersion: develop
     pyroscope-env: ci-smoke-automation-upgrade-tests
 
@@ -65,7 +65,7 @@ runner-test-matrix:
     test-inputs:
       chainlinkImage: public.ecr.aws/chainlink/chainlink
       chainlinkVersion: latest
-      chainlinkUpgradeImage: {{ env.QA_CHAINLINK_IMAGE }}
+      chainlinkUpgradeImage: '{{ env.QA_CHAINLINK_IMAGE }}'
       chainlinkUpgradeVersion: develop
     pyroscope-env: ci-smoke-automation-upgrade-tests
 
@@ -79,7 +79,7 @@ runner-test-matrix:
     test-inputs:
       chainlinkImage: public.ecr.aws/chainlink/chainlink
       chainlinkVersion: latest
-      chainlinkUpgradeImage: {{ env.QA_CHAINLINK_IMAGE }}
+      chainlinkUpgradeImage: '{{ env.QA_CHAINLINK_IMAGE }}'
       chainlinkUpgradeVersion: develop
     pyroscope-env: ci-smoke-automation-upgrade-tests
 
diff --git a/integration-tests/e2e_tests_ci_tool/cmd/filter_cmd.go b/integration-tests/e2e_tests_ci_tool/cmd/filter_cmd.go
index a2d3758166b..01ff6f7a7d7 100644
--- a/integration-tests/e2e_tests_ci_tool/cmd/filter_cmd.go
+++ b/integration-tests/e2e_tests_ci_tool/cmd/filter_cmd.go
@@ -105,13 +105,15 @@ Example usage:
 
 		data, err := os.ReadFile(yamlFile)
 		if err != nil {
-			log.Fatalf("Error reading YAML file: %v", err)
+			fmt.Fprintf(os.Stderr, "Error reading YAML file: %v\n", err)
+			os.Exit(1)
 		}
 
 		var config Config
 		err = yaml.Unmarshal(data, &config)
 		if err != nil {
-			log.Fatalf("Error parsing YAML data: %v", err)
+			fmt.Fprintf(os.Stderr, "Error parsing YAML file %s data: %v\n", yamlFile, err)
+			os.Exit(1)
 		}
 
 		var filteredTests []CITestConf
@@ -126,7 +128,8 @@ Example usage:
 		matrix := map[string][]CITestConf{"tests": filteredTests}
 		matrixJSON, err := json.Marshal(matrix)
 		if err != nil {
-			log.Fatalf("Error marshaling matrix to JSON: %v", err)
+			fmt.Fprintf(os.Stderr, "Error marshaling matrix to JSON: %v\n", err)
+			os.Exit(1)
 		}
 
 		fmt.Printf("%s", matrixJSON)

From 611f77483cac17c2e28727de37c1af0f2b25977b Mon Sep 17 00:00:00 2001
From: lukaszcl <120112546+lukaszcl@users.noreply.github.com>
Date: Tue, 2 Jul 2024 10:52:58 +0200
Subject: [PATCH 131/137] Check chainlink image exist in automation ondemand
 workflow

---
 .../run-automation-ondemand-e2e-tests.yml     | 75 ++++++++++++++++++-
 1 file changed, 72 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/run-automation-ondemand-e2e-tests.yml b/.github/workflows/run-automation-ondemand-e2e-tests.yml
index 2cc76d180ec..fbc560e3a5c 100644
--- a/.github/workflows/run-automation-ondemand-e2e-tests.yml
+++ b/.github/workflows/run-automation-ondemand-e2e-tests.yml
@@ -43,6 +43,10 @@ on:
         required: false
         type: string                 
 
+env:
+  CHAINLINK_VERSION: ${{ inputs.chainlinkVersion || github.sha }}
+  CHAINLINK_UPGRADE_VERSION: ${{ inputs.chainlinkVersionUpdate }}
+
 jobs:
   # Set tests to run based on the workflow inputs
   set-tests-to-run:
@@ -73,8 +77,8 @@ jobs:
             exit 1
           fi            
 
-          CHAINLINK_VERSION="${{ inputs.chainlinkVersion || github.sha }}"
-          CHAINLINK_UPGRADE_VERSION="${{ inputs.chainlinkVersionUpdate }}"
+          CHAINLINK_VERSION="${{ env.CHAINLINK_VERSION }}"
+          CHAINLINK_UPGRADE_VERSION="${{ env.CHAINLINK_UPGRADE_VERSION }}"
 
           # Always run upgrade tests 
           cat > test_list.yaml <<EOF
@@ -122,9 +126,74 @@ jobs:
 
           echo "test_list=$(cat test_list.yaml | base64 -w 0)" >> $GITHUB_OUTPUT
 
+  build-chainlink-image:
+    name: Build required Chainlink images 
+    if: ${{ github.event.inputs.chainlinkImage == 'QA_ECR' ||  github.event.inputs.chainlinkImageUpdate == 'QA_ECR' }}
+    runs-on: ubuntu-latest
+    environment: integration
+    permissions:
+      id-token: write
+      contents: read
+    env:
+      CHAINLINK_IMAGE: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com/chainlink
+    strategy:
+      matrix:
+        image:
+          - name: ""
+            dockerfile: core/chainlink.Dockerfile
+            tag-suffix: ""
+          - name: (plugins)
+            dockerfile: plugins/chainlink.Dockerfile
+            tag-suffix: -plugins
+    steps:
+      - name: Checkout the repo
+        if: ${{ github.event.inputs.chainlinkImage == 'QA_ECR' }}
+        uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2
+
+      - name: Build Chainlink Image (from image upgrade) 
+        if: ${{ github.event.inputs.chainlinkImage == 'QA_ECR' }}
+        uses: ./.github/actions/build-chainlink-image
+        with:
+          dockerfile: core/chainlink.Dockerfile
+          git_commit_sha: ${{ env.CHAINLINK_VERSION }}
+          tag_suffix: ${{ matrix.image.tag-suffix }}
+          check_image_exists: 'true'
+          AWS_REGION: ${{ secrets.QA_AWS_REGION }}
+          AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }}
+
+      - name: Build Chainlink Image (to image upgrade) 
+        if: ${{ github.event.inputs.chainlinkImageUpdate == 'QA_ECR' }}
+        uses: ./.github/actions/build-chainlink-image
+        with:
+          dockerfile: core/chainlink.Dockerfile
+          git_commit_sha: ${{ env.CHAINLINK_UPGRADE_VERSION }}
+          tag_suffix: ${{ matrix.image.tag-suffix }}
+          check_image_exists: 'true'
+          AWS_REGION: ${{ secrets.QA_AWS_REGION }}
+          AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }}          
+
+      # - name: Check if image exists ${{ env.CHAINLINK_VERSION }}${{ matrix.image.tag-suffix }}
+      #   if: ${{ github.event.inputs.chainlinkImage == 'QA_ECR' }}
+      #   id: check-from-image
+      #   uses: smartcontractkit/chainlink-github-actions/docker/image-exists@fc3e0df622521019f50d772726d6bf8dc919dd38 # v2.3.19
+      #   with:
+      #     repository: chainlink
+      #     tag: ${{ env.CHAINLINK_VERSION }}${{ matrix.image.tag-suffix }}
+      #     AWS_REGION: ${{ secrets.QA_AWS_REGION }}
+      #     AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }}      
+      # - name: Check if image exists ${{ env.CHAINLINK_UPGRADE_VERSION }}${{ matrix.image.tag-suffix }}
+      #   if: ${{ github.event.inputs.chainlinkImageUpdate == 'QA_ECR' }}
+      #   id: check-to-image
+      #   uses: smartcontractkit/chainlink-github-actions/docker/image-exists@fc3e0df622521019f50d772726d6bf8dc919dd38 # v2.3.19
+      #   with:
+      #     repository: chainlink
+      #     tag: ${{ env.CHAINLINK_UPGRADE_VERSION }}${{ matrix.image.tag-suffix }}
+      #     AWS_REGION: ${{ secrets.QA_AWS_REGION }}
+      #     AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }}
+
   call-run-e2e-tests-workflow:
     name: Run E2E Tests
-    needs: set-tests-to-run
+    needs: [set-tests-to-run, build-chainlink-image]
     uses: ./.github/workflows/run-e2e-tests-reusable-workflow.yml
     with:
       test_list: ${{ needs.set-tests-to-run.outputs.test_list }}

From 5127068251185d8abd96668f0927899cfcacd273 Mon Sep 17 00:00:00 2001
From: lukaszcl <120112546+lukaszcl@users.noreply.github.com>
Date: Tue, 2 Jul 2024 13:17:03 +0200
Subject: [PATCH 132/137] Add require-chainlink-image-versions-in-qa-ecr job to
 reusable workflow

---
 .../run-automation-ondemand-e2e-tests.yml     |  78 +++-----------
 .../run-e2e-tests-reusable-workflow.yml       | 100 ++++++++++++++++--
 2 files changed, 106 insertions(+), 72 deletions(-)

diff --git a/.github/workflows/run-automation-ondemand-e2e-tests.yml b/.github/workflows/run-automation-ondemand-e2e-tests.yml
index fbc560e3a5c..823b1b95387 100644
--- a/.github/workflows/run-automation-ondemand-e2e-tests.yml
+++ b/.github/workflows/run-automation-ondemand-e2e-tests.yml
@@ -54,6 +54,8 @@ jobs:
     runs-on: ubuntu-latest
     outputs:
       test_list: ${{ steps.set-tests.outputs.test_list }}
+      require_chainlink_image_versions_in_qa_ecr: ${{ steps.set-tests.outputs.require_chainlink_image_versions_in_qa_ecr }}
+      require_chainlink_plugin_versions_in_qa_ecr: ${{ steps.set-tests.outputs.require_chainlink_plugin_versions_in_qa_ecr }}
     steps:
       - name: Set tests to run
         id: set-tests
@@ -126,77 +128,23 @@ jobs:
 
           echo "test_list=$(cat test_list.yaml | base64 -w 0)" >> $GITHUB_OUTPUT
 
-  build-chainlink-image:
-    name: Build required Chainlink images 
-    if: ${{ github.event.inputs.chainlinkImage == 'QA_ECR' ||  github.event.inputs.chainlinkImageUpdate == 'QA_ECR' }}
-    runs-on: ubuntu-latest
-    environment: integration
-    permissions:
-      id-token: write
-      contents: read
-    env:
-      CHAINLINK_IMAGE: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com/chainlink
-    strategy:
-      matrix:
-        image:
-          - name: ""
-            dockerfile: core/chainlink.Dockerfile
-            tag-suffix: ""
-          - name: (plugins)
-            dockerfile: plugins/chainlink.Dockerfile
-            tag-suffix: -plugins
-    steps:
-      - name: Checkout the repo
-        if: ${{ github.event.inputs.chainlinkImage == 'QA_ECR' }}
-        uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2
-
-      - name: Build Chainlink Image (from image upgrade) 
-        if: ${{ github.event.inputs.chainlinkImage == 'QA_ECR' }}
-        uses: ./.github/actions/build-chainlink-image
-        with:
-          dockerfile: core/chainlink.Dockerfile
-          git_commit_sha: ${{ env.CHAINLINK_VERSION }}
-          tag_suffix: ${{ matrix.image.tag-suffix }}
-          check_image_exists: 'true'
-          AWS_REGION: ${{ secrets.QA_AWS_REGION }}
-          AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }}
-
-      - name: Build Chainlink Image (to image upgrade) 
-        if: ${{ github.event.inputs.chainlinkImageUpdate == 'QA_ECR' }}
-        uses: ./.github/actions/build-chainlink-image
-        with:
-          dockerfile: core/chainlink.Dockerfile
-          git_commit_sha: ${{ env.CHAINLINK_UPGRADE_VERSION }}
-          tag_suffix: ${{ matrix.image.tag-suffix }}
-          check_image_exists: 'true'
-          AWS_REGION: ${{ secrets.QA_AWS_REGION }}
-          AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }}          
-
-      # - name: Check if image exists ${{ env.CHAINLINK_VERSION }}${{ matrix.image.tag-suffix }}
-      #   if: ${{ github.event.inputs.chainlinkImage == 'QA_ECR' }}
-      #   id: check-from-image
-      #   uses: smartcontractkit/chainlink-github-actions/docker/image-exists@fc3e0df622521019f50d772726d6bf8dc919dd38 # v2.3.19
-      #   with:
-      #     repository: chainlink
-      #     tag: ${{ env.CHAINLINK_VERSION }}${{ matrix.image.tag-suffix }}
-      #     AWS_REGION: ${{ secrets.QA_AWS_REGION }}
-      #     AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }}      
-      # - name: Check if image exists ${{ env.CHAINLINK_UPGRADE_VERSION }}${{ matrix.image.tag-suffix }}
-      #   if: ${{ github.event.inputs.chainlinkImageUpdate == 'QA_ECR' }}
-      #   id: check-to-image
-      #   uses: smartcontractkit/chainlink-github-actions/docker/image-exists@fc3e0df622521019f50d772726d6bf8dc919dd38 # v2.3.19
-      #   with:
-      #     repository: chainlink
-      #     tag: ${{ env.CHAINLINK_UPGRADE_VERSION }}${{ matrix.image.tag-suffix }}
-      #     AWS_REGION: ${{ secrets.QA_AWS_REGION }}
-      #     AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }}
+          chainlink_image_versions=""
+          if [ "${{ github.event.inputs.chainlinkImage }}" = "QA_ECR" ]; then
+            chainlink_image_versions+="${{ env.CHAINLINK_VERSION }},"
+          fi
+          if [ "${{ github.event.inputs.chainlinkImageUpdate }}" = "QA_ECR" ]; then
+            chainlink_image_versions+="${{ env.CHAINLINK_UPGRADE_VERSION }}"
+          fi
+          echo "require_chainlink_image_versions_in_qa_ecr=$chainlink_image_versions" >> $GITHUB_OUTPUT
 
   call-run-e2e-tests-workflow:
     name: Run E2E Tests
-    needs: [set-tests-to-run, build-chainlink-image]
+    needs: set-tests-to-run
     uses: ./.github/workflows/run-e2e-tests-reusable-workflow.yml
     with:
       test_list: ${{ needs.set-tests-to-run.outputs.test_list }}
+      require_chainlink_image_versions_in_qa_ecr: ${{ needs.set-tests-to-run.outputs.require_chainlink_image_versions_in_qa_ecr }}
+      require_chainlink_plugin_versions_in_qa_ecr: ${{ needs.set-tests-to-run.outputs.require_chainlink_image_versions_in_qa_ecr }}
       with_existing_remote_runner_version: ${{ github.event.inputs.with_existing_remote_runner_version }}
     secrets:
       QA_AWS_REGION: ${{ secrets.QA_AWS_REGION }}
diff --git a/.github/workflows/run-e2e-tests-reusable-workflow.yml b/.github/workflows/run-e2e-tests-reusable-workflow.yml
index 583c2e9f14d..35c462f4970 100644
--- a/.github/workflows/run-e2e-tests-reusable-workflow.yml
+++ b/.github/workflows/run-e2e-tests-reusable-workflow.yml
@@ -14,7 +14,7 @@ on:
         type: string
       test_list:
         description: 'Base64 encoded list of tests (YML objects) to run. Example in run-automation-ondemand-e2e-tests.yml'
-        required: true
+        required: false
         type: string            
       test_workflow:
         description: 'Run tests by workflow name. Example: "Run Nightly E2E Tests"'
@@ -34,6 +34,14 @@ on:
         description: 'Use the existing remote runner version for k8s tests. Example: "d3bf5044af33e08be788a2df31c4a745cf69d787"'
         required: false
         type: string
+      require_chainlink_image_versions_in_qa_ecr:
+        description: 'Check Chainlink image versions to be present in QA ECR. If not, build and push the image to QA ECR. Takes comma separated list of Chainlink image versions. Example: "5733cdcda9a9fc6da6343798b119b2ae136146cd,0b7d2c497a508efa5a827714780d908b7b8eda19"'
+        required: false
+        type: string
+      require_chainlink_plugin_versions_in_qa_ecr:
+        description: 'Check Chainlink plugins versions to be present in QA ECR. If not, build and push the image to QA ECR. Takes comma separated list of Chainlink image versions. Example: "5733cdcda9a9fc6da6343798b119b2ae136146cd,0b7d2c497a508efa5a827714780d908b7b8eda19"'
+        required: false
+        type: string
     secrets:
       QA_AWS_REGION:
         required: true
@@ -73,6 +81,9 @@ jobs:
   validate-inputs:
     name: Validate workflow inputs
     runs-on: ubuntu-latest
+    outputs:
+      require_chainlink_image_versions_in_qa_ecr_matrix: ${{ steps.set-required-chainlink-image-versions-matrix.outputs.versions }}
+      require_chainlink_plugin_versions_in_qa_ecr_matrix: ${{ steps.set-required-chainlink-plugin-versions-matrix.outputs.versions }}
     steps:
       - name: Check input conditions
         run: |
@@ -80,6 +91,22 @@ jobs:
             echo "::error::Error: Both 'test_ids' and 'test_workflow' are provided. Please specify only one."
             exit 1
           fi
+      - name: Install jq
+        run: sudo apt-get install jq          
+      - name: Create matrix for required Chainlink image versions
+        id: set-required-chainlink-image-versions-matrix
+        run: |
+          if [[ "${{ inputs.require_chainlink_image_versions_in_qa_ecr }}" != '' ]]; then
+            image_versions=$(echo "${{ inputs.require_chainlink_image_versions_in_qa_ecr }}" | jq -Rc 'split(",") | if . == [""] then [] else . end')
+            echo "versions=$image_versions" >> $GITHUB_OUTPUT
+          fi
+      - name: Create matrix for required Chainlink plugin versions
+        id: set-required-chainlink-plugin-versions-matrix
+        run: |
+          if [[ "${{ inputs.require_chainlink_plugin_versions_in_qa_ecr }}" != '' ]]; then
+            image_versions=$(echo "${{ inputs.require_chainlink_plugin_versions_in_qa_ecr }}" | jq -Rc 'split(",") | if . == [""] then [] else . end')
+            echo "versions=$image_versions" >> $GITHUB_OUTPUT
+          fi          
 
   check-test-configurations:
     name: Check test configurations
@@ -159,11 +186,70 @@ jobs:
           fi
         shell: bash   
 
+  # Build Chainlink images required for the tests
+  require-chainlink-image-versions-in-qa-ecr:
+    name: Build Chainlink image
+    needs: [validate-inputs, load-test-configurations]
+    if: ${{ needs.validate-inputs.outputs.require_chainlink_image_versions_in_qa_ecr_matrix != '' }}
+    runs-on: ubuntu-latest
+    environment: integration
+    permissions:
+      id-token: write
+      contents: read
+    env:
+      CHAINLINK_IMAGE: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com/chainlink
+    strategy:
+      matrix:
+        version: ${{ fromJson(needs.validate-inputs.outputs.require_chainlink_image_versions_in_qa_ecr_matrix) }}
+    steps:
+      - name: Checkout the repo
+        uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2
+
+      - name: Build Chainlink image for ${{ matrix.version }} and push it to QA ECR
+        uses: ./.github/actions/build-chainlink-image
+        with:
+          dockerfile: core/chainlink.Dockerfile
+          git_commit_sha: ${{ matrix.version }}
+          tag_suffix: ''
+          check_image_exists: 'true'
+          AWS_REGION: ${{ secrets.QA_AWS_REGION }}
+          AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }}        
+
+  # Build Chainlink plugins required for the tests
+  require-chainlink-plugin-versions-in-qa-ecr:
+    name: Build Chainlink plugins 
+    needs: [validate-inputs, load-test-configurations]
+    if: ${{ needs.validate-inputs.outputs.require_chainlink_plugin_versions_in_qa_ecr_matrix != '' }}
+    runs-on: ubuntu-latest
+    environment: integration
+    permissions:
+      id-token: write
+      contents: read
+    env:
+      CHAINLINK_IMAGE: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com/chainlink
+    strategy:
+      matrix:
+        version: ${{ fromJson(needs.validate-inputs.outputs.require_chainlink_plugin_versions_in_qa_ecr_matrix) }}
+    steps:
+      - name: Checkout the repo
+        uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2
+
+      - name: Build Chainlink plugins image for ${{ matrix.version }}
+        uses: ./.github/actions/build-chainlink-image
+        with:
+          dockerfile: plugins/chainlink.Dockerfile
+          git_commit_sha: ${{ matrix.version }}
+          tag_suffix: '-plugins'
+          check_image_exists: 'true'
+          AWS_REGION: ${{ secrets.QA_AWS_REGION }}
+          AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }}          
+
   # Run Docker tests
   docker-tests:
     name: Run docker tests (${{ matrix.tests.id }})
-    needs: load-test-configurations
-    if: ${{ needs.load-test-configurations.outputs.run-docker-tests == 'true' }}
+    needs: [load-test-configurations, require-chainlink-image-versions-in-qa-ecr, require-chainlink-plugin-versions-in-qa-ecr]
+    # Run when none of the needed jobs fail or are cancelled (skipped or successful jobs are ok) 
+    if: ${{ needs.load-test-configurations.outputs.run-docker-tests == 'true' && always() && !failure() && !cancelled() }} 
     runs-on: ${{ matrix.tests.runsOn }}
     strategy:
       fail-fast: false
@@ -248,8 +334,8 @@ jobs:
   # Run K8s tests using old remote runner
 
   prepare-remote-runner-test-image:
-    needs: load-test-configurations
-    if: ${{ needs.load-test-configurations.outputs.run-k8s-tests == 'true' }}
+    needs: [load-test-configurations, require-chainlink-image-versions-in-qa-ecr, require-chainlink-plugin-versions-in-qa-ecr]
+    if: ${{ needs.load-test-configurations.outputs.run-k8s-tests == 'true' && always() && !failure() && !cancelled() }} 
     name: Prepare remote runner test image
     runs-on: ubuntu-latest
     environment: integration
@@ -283,8 +369,8 @@ jobs:
           fi
 
   run-k8s-runner-tests:
-    needs: [load-test-configurations, prepare-remote-runner-test-image]
-    if: ${{ needs.load-test-configurations.outputs.run-k8s-tests == 'true' }}
+    needs: [load-test-configurations, prepare-remote-runner-test-image, require-chainlink-image-versions-in-qa-ecr, require-chainlink-plugin-versions-in-qa-ecr]
+    if: ${{ needs.load-test-configurations.outputs.run-k8s-tests == 'true' && always() && !failure() && !cancelled() }} 
     name: Run k8s tests (${{ matrix.tests.id }})
     runs-on: ${{ matrix.tests.runsOn }}
     strategy:

From b670bbb40ddc6cd114040aa549c1499d68e80638 Mon Sep 17 00:00:00 2001
From: lukaszcl <120112546+lukaszcl@users.noreply.github.com>
Date: Tue, 2 Jul 2024 14:52:29 +0200
Subject: [PATCH 133/137] Set no_cache for run-tests action

---
 .github/workflows/run-e2e-tests-reusable-workflow.yml | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/run-e2e-tests-reusable-workflow.yml b/.github/workflows/run-e2e-tests-reusable-workflow.yml
index 35c462f4970..79791a3b3f2 100644
--- a/.github/workflows/run-e2e-tests-reusable-workflow.yml
+++ b/.github/workflows/run-e2e-tests-reusable-workflow.yml
@@ -321,9 +321,8 @@ jobs:
             /tmp/gotest.log
           publish_check_name: ${{ matrix.tests.idSanitized }}
           token: ${{ secrets.GH_TOKEN }}
-          cache_key_id: run_docker_e2e_tests_${{ matrix.tests.idSanitized }}_${{ env.MOD_CACHE_VERSION }}
+          no_cache: true # Do not restore cache since go was already configured in the previous step
           go_mod_path: ./integration-tests/go.mod
-          cache_restore_only: "true"
           QA_AWS_REGION: ${{ secrets.QA_AWS_REGION }}
           QA_AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }}
           QA_KUBECONFIG: ""
@@ -446,7 +445,7 @@ jobs:
           cl_image_tag: ${{ steps.create-default-test-config-override.outputs.chainlink_version }}
           token: ${{ secrets.GH_TOKEN }}
           should_cleanup: false
-          cache_key_id: run_k8s_e2e_tests_${{ matrix.tests.idSanitized }}_${{ env.MOD_CACHE_VERSION }}
+          no_cache: true # Do not restore cache since go was already configured in the previous step
           go_mod_path: ./integration-tests/go.mod
           QA_AWS_REGION: ${{ secrets.QA_AWS_REGION }}
           QA_AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }}

From bd774110d9771b9f44c1967e17fed887ca155710 Mon Sep 17 00:00:00 2001
From: lukaszcl <120112546+lukaszcl@users.noreply.github.com>
Date: Tue, 9 Jul 2024 18:19:33 +0200
Subject: [PATCH 134/137] Fix

---
 .github/workflows/integration-tests.yml       | 10 +---
 .../run-e2e-tests-reusable-workflow.yml       | 52 ++++++++++---------
 2 files changed, 29 insertions(+), 33 deletions(-)

diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml
index 1d0b488609e..74fff0d71c8 100644
--- a/.github/workflows/integration-tests.yml
+++ b/.github/workflows/integration-tests.yml
@@ -946,20 +946,13 @@ jobs:
         uses: ./.github/actions/setup-create-base64-upgrade-config
         with:
           selectedNetworks: ${{ env.SELECTED_NETWORKS }}
-          chainlinkImage: ${{ env.CHAINLINK_IMAGE }}
           chainlinkVersion: ${{ steps.get_latest_version.outputs.latest_version }}
-          upgradeImage: ${{ env.UPGRADE_IMAGE }}
           upgradeVersion: ${{ env.UPGRADE_VERSION }}
           runId: ${{ github.run_id }}
           testLogCollect: ${{ vars.TEST_LOG_COLLECT }}
-          lokiEndpoint: https://${{ secrets.GRAFANA_INTERNAL_HOST }}/loki/api/v1/push
-          lokiTenantId: ${{ secrets.GRAFANA_INTERNAL_TENANT_ID }}
-          lokiBasicAuth: ${{ secrets.GRAFANA_INTERNAL_BASIC_AUTH }}
           logstreamLogTargets: ${{ vars.LOGSTREAM_LOG_TARGETS }}
-          grafanaUrl: ${{ vars.GRAFANA_URL }}
-          grafanaDashboardUrl: "/d/ddf75041-1e39-42af-aa46-361fe4c36e9e/ci-e2e-tests-logs"
       - name: Run Migration Tests
-        uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/run-tests@75a9005952a9e905649cfb5a6971fd9429436acd # v2.3.24
+        uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/run-tests@351858d53187513fd47f1d2fdce71f6a8c246828 # v2.3.24
         with:
           test_command_to_run: cd ./integration-tests && go test -timeout 20m -count=1 -json ./migration 2>&1 | tee /tmp/gotest.log | gotestloghelper -ci -singlepackage -hidepassingtests=false -hidepassinglogs
           test_download_vendor_packages_command: cd ./integration-tests && go mod download
@@ -983,6 +976,7 @@ jobs:
           go_coverage_dest_dir: ${{ github.workspace }}/.covdata
           should_tidy: "false"
           DEFAULT_CHAINLINK_IMAGE: ${{ env.CHAINLINK_IMAGE }}
+          DEFAULT_CHAINLINK_UPGRADE_IMAGE: ${{ env.UPGRADE_IMAGE }}
           DEFAULT_LOKI_TENANT_ID: ${{ secrets.GRAFANA_INTERNAL_TENANT_ID }}
           DEFAULT_LOKI_ENDPOINT: https://${{ secrets.GRAFANA_INTERNAL_HOST }}/loki/api/v1/push
           DEFAULT_LOKI_BASIC_AUTH: ${{ secrets.GRAFANA_INTERNAL_BASIC_AUTH }}
diff --git a/.github/workflows/run-e2e-tests-reusable-workflow.yml b/.github/workflows/run-e2e-tests-reusable-workflow.yml
index 1021ac8d867..42fb824f016 100644
--- a/.github/workflows/run-e2e-tests-reusable-workflow.yml
+++ b/.github/workflows/run-e2e-tests-reusable-workflow.yml
@@ -280,23 +280,12 @@ jobs:
         id: create-default-test-config-override
         uses: ./.github/actions/create-default-e2e-config-override
         with:
-          chainlink_image: ${{ matrix.tests.testInputs.chainlinkImage || env.CHAINLINK_IMAGE }}
           chainlink_version: ${{ inputs.chainlink_version || matrix.tests.testInputs.chainlinkVersion || github.sha }}
           chainlink_postgres_version: ${{ matrix.tests.testInputs.chainlinkPostgresVersion }}
-          chainlink_upgrade_image: ${{ matrix.tests.testInputs.chainlinkUpgradeImage }}
-          chainlink_upgrade_version: ${{ matrix.tests.testInputs.chainlinkUpgradeVersion }}
+          # chainlink_upgrade_image: ${{ matrix.tests.testInputs.chainlinkUpgradeImage }}
           selected_networks: ${{ matrix.tests.testInputs.selectedNetworks || env.SELECTED_NETWORKS}}
-          pyroscope_server: ${{ secrets.QA_PYROSCOPE_INSTANCE }}
-          pyroscope_environment: ${{ matrix.tests.pyroscopeEnv }}
-          pyroscope_key: ${{ secrets.QA_PYROSCOPE_KEY }}
           logging_run_id: ${{ github.run_id }}
-          loki_endpoint: https://${{ secrets.GRAFANA_INTERNAL_HOST }}/loki/api/v1/push
-          loki_tenant_id: ${{ secrets.GRAFANA_INTERNAL_TENANT_ID }}
-          loki_basic_auth: ${{ secrets.GRAFANA_INTERNAL_BASIC_AUTH }}
           logstream_log_targets: ${{ vars.LOGSTREAM_LOG_TARGETS }}
-          grafana_url: http://localhost:8080/primary # This is Grafana GAP's address
-          grafana_dashboard_url: /d/ddf75041-1e39-42af-aa46-361fe4c36e9e/ci-e2e-tests-logs
-          grafana_bearer_token: ${{ secrets.GRAFANA_INTERNAL_URL_SHORTENER_TOKEN }}
 
       - name: Setup GAP for Grafana
         uses: smartcontractkit/.github/actions/setup-gap@d316f66b2990ea4daa479daa3de6fc92b00f863e # setup-gap@0.3.2
@@ -308,7 +297,7 @@ jobs:
           duplicate-authorization-header: "true"
 
       - name: Run tests
-        uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/run-tests@af92c5fae8dcf1659201e907db82d221fc304b94 # v2.3.21
+        uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/run-tests@75a9005952a9e905649cfb5a6971fd9429436acd # v2.3.24
         with:
           test_command_to_run: ${{ matrix.tests.testCmd }} 2>&1 | tee /tmp/gotest.log | gotestloghelper -ci -singlepackage -hidepassingtests=false -hidepassinglogs
           test_download_vendor_packages_command: cd ./integration-tests && go mod download
@@ -330,6 +319,18 @@ jobs:
           go_coverage_src_dir: /var/tmp/go-coverage
           go_coverage_dest_dir: ${{ github.workspace }}/.covdata
 
+          DEFAULT_CHAINLINK_IMAGE: ${{ matrix.tests.testInputs.chainlinkImage || env.CHAINLINK_IMAGE }}
+          DEFAULT_LOKI_TENANT_ID: ${{ secrets.GRAFANA_INTERNAL_TENANT_ID }}
+          DEFAULT_LOKI_ENDPOINT: https://${{ secrets.GRAFANA_INTERNAL_HOST }}/loki/api/v1/push
+          DEFAULT_LOKI_BASIC_AUTH: ${{ secrets.GRAFANA_INTERNAL_BASIC_AUTH }}
+          DEFAULT_GRAFANA_BASE_URL: "http://localhost:8080/primary"
+          DEFAULT_GRAFANA_DASHBOARD_URL: "/d/ddf75041-1e39-42af-aa46-361fe4c36e9e/ci-e2e-tests-logs"
+          DEFAULT_GRAFANA_BEARER_TOKEN: ${{ secrets.GRAFANA_INTERNAL_URL_SHORTENER_TOKEN }}
+          DEFAULT_PYROSCOPE_SERVER_URL: ${{ secrets.QA_PYROSCOPE_INSTANCE }}
+          DEFAULT_PYROSCOPE_KEY: ${{ secrets.QA_PYROSCOPE_KEY }}
+          DEFAULT_PYROSCOPE_ENVIRONMENT: ${{ matrix.tests.pyroscopeEnv }}
+          DEFAULT_PYROSCOPE_ENABLED: "true"
+
   # Run K8s tests using old remote runner
 
   prepare-remote-runner-test-image:
@@ -403,27 +404,16 @@ jobs:
         id: create-default-test-config-override
         uses: ./.github/actions/create-default-e2e-config-override
         with:
-          chainlink_image: ${{ matrix.tests.testInputs.chainlinkImage || env.CHAINLINK_IMAGE }}
           chainlink_version: ${{ inputs.chainlink_version || matrix.tests.testInputs.chainlinkVersion || github.sha }}
           chainlink_postgres_version: ${{ matrix.tests.testInputs.chainlinkPostgresVersion }}
-          chainlink_upgrade_image: ${{ matrix.tests.testInputs.chainlinkUpgradeImage }}
           chainlink_upgrade_version: ${{ matrix.tests.testInputs.chainlinkUpgradeVersion }}
           selected_networks: ${{ matrix.tests.testInputs.selectedNetworks || env.SELECTED_NETWORKS}}
-          pyroscope_server: ${{ secrets.QA_PYROSCOPE_INSTANCE }}
-          pyroscope_environment: ${{ matrix.tests.pyroscopeEnv }}
-          pyroscope_key: ${{ secrets.QA_PYROSCOPE_KEY }}
           logging_run_id: ${{ github.run_id }}
-          loki_endpoint: https://${{ secrets.GRAFANA_INTERNAL_HOST }}/loki/api/v1/push
-          loki_tenant_id: ${{ secrets.GRAFANA_INTERNAL_TENANT_ID }}
-          loki_basic_auth: ${{ secrets.GRAFANA_INTERNAL_BASIC_AUTH }}
           logstream_log_targets: ${{ vars.LOGSTREAM_LOG_TARGETS }}
-          grafana_url: http://localhost:8080/primary # This is Grafana GAP's address
-          grafana_dashboard_url: /d/ddf75041-1e39-42af-aa46-361fe4c36e9e/ci-e2e-tests-logs
-          grafana_bearer_token: ${{ secrets.GRAFANA_INTERNAL_URL_SHORTENER_TOKEN }}          
           test_log_collect: ${{ vars.TEST_LOG_COLLECT }}
 
       - name: Run tests
-        uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/run-tests@af92c5fae8dcf1659201e907db82d221fc304b94 # v2.3.21
+        uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/run-tests@75a9005952a9e905649cfb5a6971fd9429436acd # v2.3.24
         env:
           DETACH_RUNNER: true
           TEST_SUITE: ${{ matrix.tests.remoteRunnerTestSuite }}
@@ -450,6 +440,18 @@ jobs:
           QA_AWS_REGION: ${{ secrets.QA_AWS_REGION }}
           QA_AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }}
           QA_KUBECONFIG: ${{ secrets.QA_KUBECONFIG }} 
+          DEFAULT_CHAINLINK_IMAGE: ${{ matrix.tests.testInputs.chainlinkImage || env.CHAINLINK_IMAGE }}
+          DEFAULT_LOKI_TENANT_ID: ${{ secrets.GRAFANA_INTERNAL_TENANT_ID }}
+          DEFAULT_LOKI_ENDPOINT: https://${{ secrets.GRAFANA_INTERNAL_HOST }}/loki/api/v1/push
+          DEFAULT_LOKI_BASIC_AUTH: ${{ secrets.GRAFANA_INTERNAL_BASIC_AUTH }}
+          DEFAULT_GRAFANA_BASE_URL: "http://localhost:8080/primary"
+          DEFAULT_GRAFANA_DASHBOARD_URL: "/d/ddf75041-1e39-42af-aa46-361fe4c36e9e/ci-e2e-tests-logs"
+          DEFAULT_GRAFANA_BEARER_TOKEN: ${{ secrets.GRAFANA_INTERNAL_URL_SHORTENER_TOKEN }}
+          DEFAULT_PYROSCOPE_SERVER_URL: ${{ secrets.QA_PYROSCOPE_INSTANCE }}
+          DEFAULT_PYROSCOPE_KEY: ${{ secrets.QA_PYROSCOPE_KEY }}
+          DEFAULT_PYROSCOPE_ENVIRONMENT: ${{ matrix.tests.pyroscopeEnv }}
+          DEFAULT_PYROSCOPE_ENABLED: "true"
+
 
   # Run K8s tests using new remote runner
   # remote-runner-k8s-tests:

From eea4c5c54d2d026be0b3f787aa2b48c94f246e45 Mon Sep 17 00:00:00 2001
From: lukaszcl <120112546+lukaszcl@users.noreply.github.com>
Date: Tue, 9 Jul 2024 18:30:01 +0200
Subject: [PATCH 135/137] Add test_secrets_override_key to run-selected-tests
 workflow

---
 .../run-e2e-tests-reusable-workflow.yml       | 20 ++++++++++++-------
 .github/workflows/run-selected-e2e-tests.yml  |  8 +++++++-
 2 files changed, 20 insertions(+), 8 deletions(-)

diff --git a/.github/workflows/run-e2e-tests-reusable-workflow.yml b/.github/workflows/run-e2e-tests-reusable-workflow.yml
index 42fb824f016..0f105431d36 100644
--- a/.github/workflows/run-e2e-tests-reusable-workflow.yml
+++ b/.github/workflows/run-e2e-tests-reusable-workflow.yml
@@ -21,10 +21,10 @@ on:
         required: false
         type: string
       # Enable once Test Config does not have any secrets. Related ticket https://smartcontract-it.atlassian.net/browse/TT-1283
-      # test_config_base64_override:
-      #   description: 'This is an unsafe way to pass custom base64 test config. It cannot contain any secrets as this value is exposed in Github logs (Set up job -> Inputs) when the workflow is called! The caller workflow must make sure to remove secrets from this input.'
-      #   required: false
-      #   type: string        
+      test_config_override_base64:
+        required: false
+        description: The base64-encoded test config override
+        type: string
       enable_check_test_configurations:
         description: 'Set to "true" to enable check-test-configurations job'
         required: false
@@ -43,6 +43,8 @@ on:
         required: false
         type: string
     secrets:
+      TEST_SECRETS_OVERRIDE_BASE64:
+        required: false
       QA_AWS_REGION:
         required: true
       QA_AWS_ROLE_TO_ASSUME:
@@ -91,6 +93,9 @@ jobs:
             echo "::error::Error: Both 'test_ids' and 'test_workflow' are provided. Please specify only one."
             exit 1
           fi
+          if [[ "${{ secrets.TEST_SECRETS_OVERRIDE_BASE64 }}" != "" ]]; then
+            echo "TEST_SECRETS_OVERRIDE_BASE64 is not empty"
+          fi
       - name: Install jq
         run: sudo apt-get install jq          
       - name: Create matrix for required Chainlink image versions
@@ -297,7 +302,7 @@ jobs:
           duplicate-authorization-header: "true"
 
       - name: Run tests
-        uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/run-tests@75a9005952a9e905649cfb5a6971fd9429436acd # v2.3.24
+        uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/run-tests@351858d53187513fd47f1d2fdce71f6a8c246828 # v2.3.24
         with:
           test_command_to_run: ${{ matrix.tests.testCmd }} 2>&1 | tee /tmp/gotest.log | gotestloghelper -ci -singlepackage -hidepassingtests=false -hidepassinglogs
           test_download_vendor_packages_command: cd ./integration-tests && go mod download
@@ -318,7 +323,8 @@ jobs:
           should_tidy: "false"
           go_coverage_src_dir: /var/tmp/go-coverage
           go_coverage_dest_dir: ${{ github.workspace }}/.covdata
-
+          test_config_override_base64: ${{ inputs.test_config_override_base64 }}
+          test_secrets_override_base64: ${{ secrets.TEST_SECRETS_OVERRIDE_BASE64 }}
           DEFAULT_CHAINLINK_IMAGE: ${{ matrix.tests.testInputs.chainlinkImage || env.CHAINLINK_IMAGE }}
           DEFAULT_LOKI_TENANT_ID: ${{ secrets.GRAFANA_INTERNAL_TENANT_ID }}
           DEFAULT_LOKI_ENDPOINT: https://${{ secrets.GRAFANA_INTERNAL_HOST }}/loki/api/v1/push
@@ -413,7 +419,7 @@ jobs:
           test_log_collect: ${{ vars.TEST_LOG_COLLECT }}
 
       - name: Run tests
-        uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/run-tests@75a9005952a9e905649cfb5a6971fd9429436acd # v2.3.24
+        uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/run-tests@351858d53187513fd47f1d2fdce71f6a8c246828 # v2.3.24
         env:
           DETACH_RUNNER: true
           TEST_SUITE: ${{ matrix.tests.remoteRunnerTestSuite }}
diff --git a/.github/workflows/run-selected-e2e-tests.yml b/.github/workflows/run-selected-e2e-tests.yml
index ef23b37d989..37754b88cf9 100644
--- a/.github/workflows/run-selected-e2e-tests.yml
+++ b/.github/workflows/run-selected-e2e-tests.yml
@@ -13,6 +13,10 @@ on:
         default: "*"
         required: true
         type: string
+      test_secrets_override_key:
+        description: 'Enter the secret key to override test secrets'
+        required: false
+        type: string  
       enable_check_test_configurations:
         description: 'Set to "true" to enable check-test-configurations job'
         required: false
@@ -47,5 +51,7 @@ jobs:
       GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}  
       AWS_REGION: ${{ secrets.QA_AWS_REGION }}
       AWS_OIDC_IAM_ROLE_VALIDATION_PROD_ARN: ${{ secrets.AWS_OIDC_IAM_ROLE_VALIDATION_PROD_ARN }}
-      AWS_API_GW_HOST_GRAFANA: ${{ secrets.AWS_API_GW_HOST_GRAFANA }}        
+      AWS_API_GW_HOST_GRAFANA: ${{ secrets.AWS_API_GW_HOST_GRAFANA }}
+      TEST_SECRETS_OVERRIDE_BASE64: ${{ secrets[inputs.test_secrets_override_key] }}
+              
 

From 614cf9247d036f06aaf415569ff756a8c37a8504 Mon Sep 17 00:00:00 2001
From: lukaszcl <120112546+lukaszcl@users.noreply.github.com>
Date: Tue, 9 Jul 2024 18:46:04 +0200
Subject: [PATCH 136/137] Remove chainlink image and version from create test
 config cmd

---
 .../e2e_tests_ci_tool/cmd/create_test_config_cmd.go           | 4 ----
 1 file changed, 4 deletions(-)

diff --git a/integration-tests/e2e_tests_ci_tool/cmd/create_test_config_cmd.go b/integration-tests/e2e_tests_ci_tool/cmd/create_test_config_cmd.go
index 384848e4ea5..bda4f1ca239 100644
--- a/integration-tests/e2e_tests_ci_tool/cmd/create_test_config_cmd.go
+++ b/integration-tests/e2e_tests_ci_tool/cmd/create_test_config_cmd.go
@@ -26,10 +26,6 @@ var createTestConfigCmd = &cobra.Command{
 		if cmd.Flags().Changed(ChainlinkPostgresVersionFlag) {
 			version = &oc.ChainlinkPostgresVersion
 		}
-		if image != nil && version == nil || image == nil && version != nil {
-			fmt.Fprintf(os.Stderr, "Error: both chainlink-image and chainlink-version must be set\n")
-			os.Exit(1)
-		}
 		if image != nil && version != nil {
 			tc.ChainlinkImage = &ctf_config.ChainlinkImageConfig{
 				Image:           image,

From 28b0e4aa81cbe48fccea64d8fae4d5f5169224e3 Mon Sep 17 00:00:00 2001
From: lukaszcl <120112546+lukaszcl@users.noreply.github.com>
Date: Tue, 9 Jul 2024 19:33:50 +0200
Subject: [PATCH 137/137] Update log

---
 .github/workflows/run-e2e-tests-reusable-workflow.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/run-e2e-tests-reusable-workflow.yml b/.github/workflows/run-e2e-tests-reusable-workflow.yml
index 0f105431d36..06b2d421ddb 100644
--- a/.github/workflows/run-e2e-tests-reusable-workflow.yml
+++ b/.github/workflows/run-e2e-tests-reusable-workflow.yml
@@ -94,7 +94,7 @@ jobs:
             exit 1
           fi
           if [[ "${{ secrets.TEST_SECRETS_OVERRIDE_BASE64 }}" != "" ]]; then
-            echo "TEST_SECRETS_OVERRIDE_BASE64 is not empty"
+            echo "Will run tests with custom test secrets"
           fi
       - name: Install jq
         run: sudo apt-get install jq