[TT-877] Add Live VRF Tests Workflow (#12263)

* Make Live Testnet Tests More Configurable

* Cleaner if

* Fix typo

* Whater, let's try

* Scale Back Ambitions

* Add new VRF Action

.github/workflows/live-testnet-tests.yml

@@ -9,12 +9,34 @@
 
 name: Live Testnet Tests
 on:
-  schedule:
-    - cron: "0 5 * * *" # Run every night at midnight EST
-  push:
-    tags:
-      - "*"
+  # Disable refular runs for now until we can fix some test client flakiness and improve stability
+  # schedule:
+  #   - cron: "0 5 * * *" # Run every night at midnight EST
+  # push:
+  #   tags:
+  #     - "*"
   workflow_dispatch:
+    inputs:
+      slack_user_id:
+        description: "The Slack member ID to notify"
+        required: true
+        type: string
+      network:
+        description: "The network to run tests on"
+        required: true
+        type: choice
+        options:
+          - "All"
+          - "Sepolia"
+          - "Optimism Sepolia"
+          - "Arbitrum Sepolia"
+          - "Base Sepolia"
+          - "Polygon Mumbai"
+          - "Avalanche Fuji"
+          - "Fantom Testnet"
+          - "Celo Alfajores"
+          - "Linea Goerli"
+          - "BSC Testnet"
 
 env:
   CHAINLINK_IMAGE: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com/chainlink
@@ -144,7 +166,7 @@ jobs:
                       "type": "section",
                       "text": {
                         "type": "mrkdwn",
-                        "text": "${{ contains(join(needs.*.result, ','), 'failure') && 'Some tests failed, notifying <@U01Q4N37KFG>' || 'All Good!' }}"
+                        "text": "Notifying <@${{ inputs.slack_user_id }}>"
                       }
                     },
                     {
@@ -200,6 +222,7 @@ jobs:
 
   sepolia-smoke-tests:
     environment: integration
+    if: ${{ (github.event.inputs.network == 'All' || github.event.inputs.network == 'Sepolia') }}
     permissions:
       checks: write
       pull-requests: write
@@ -272,6 +295,7 @@ jobs:
 
   bsc-testnet-smoke-tests:
     environment: integration
+    if: ${{ github.event.inputs.network == 'All' || github.event.inputs.network == 'BSC Testnet' }}
     permissions:
       checks: write
       pull-requests: write
@@ -344,6 +368,7 @@ jobs:
 
   optimism-sepolia-smoke-tests:
     environment: integration
+    if: ${{ github.event.inputs.network == 'All' || github.event.inputs.network == 'Optimism Sepolia' }}
     permissions:
       checks: write
       pull-requests: write
@@ -416,6 +441,7 @@ jobs:
 
   arbitrum-sepolia-smoke-tests:
     environment: integration
+    if: ${{ github.event.inputs.network == 'All' || github.event.inputs.network == 'Arbitrum Sepolia' }}
     permissions:
       checks: write
       pull-requests: write
@@ -488,6 +514,7 @@ jobs:
 
   base-sepolia-smoke-tests:
     environment: integration
+    if: ${{ github.event.inputs.network == 'All' || github.event.inputs.network == 'Base Sepolia' }}
     permissions:
       checks: write
       pull-requests: write
@@ -556,6 +583,7 @@ jobs:
 
   polygon-mumbai-smoke-tests:
     environment: integration
+    if: ${{ github.event.inputs.network == 'All' || github.event.inputs.network == 'Polygon Mumbai' }}
     permissions:
       checks: write
       pull-requests: write
@@ -628,6 +656,7 @@ jobs:
 
   avalanche-fuji-smoke-tests:
     environment: integration
+    if: ${{ github.event.inputs.network == 'All' || github.event.inputs.network == 'Avalanche Fuji' }}
     permissions:
       checks: write
       pull-requests: write
@@ -700,6 +729,7 @@ jobs:
 
   fantom-testnet-smoke-tests:
     environment: integration
+    if: ${{ github.event.inputs.network == 'All' || github.event.inputs.network == 'Fantom Testnet' }}
     permissions:
       checks: write
       pull-requests: write
@@ -772,6 +802,7 @@ jobs:
 
   celo-alfajores-smoke-tests:
     environment: integration
+    if: ${{ github.event.inputs.network == 'All' || github.event.inputs.network == 'Celo Alfajores' }}
     permissions:
       checks: write
       pull-requests: write
@@ -909,6 +940,7 @@ jobs:
 
   linea-goerli-smoke-tests:
     environment: integration
+    if: ${{ github.event.inputs.network == 'All' || github.event.inputs.network == 'Linea Goerli' }}
     permissions:
       checks: write
       pull-requests: write

.github/workflows/live-vrf-tests.yml

@@ -0,0 +1,191 @@
+name: Live VRF Tests
+on:
+  workflow_dispatch:
+    inputs:
+      slack_user_id:
+        description: "The Slack member ID to notify"
+        required: true
+        type: string
+      networks:
+        description: "Remove any networks you don't want to run tests on"
+        required: true
+        default: "SEPOLIA,OPTIMISM_SEPOLIA,ARBITRUM_SEPOLIA"
+      test:
+        description: "Choose test you want to run"
+        required: true
+        type: choice
+        options:
+          - "TestVRFBasic"
+          - "TestVRFv2Basic"
+          - "TestVRFv2Plus"
+
+env:
+  CHAINLINK_IMAGE: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com/chainlink
+  INTERNAL_DOCKER_REPO: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com
+  MOD_CACHE_VERSION: 2
+  CHAINLINK_NODE_FUNDING: .5
+  PYROSCOPE_KEY: ${{ secrets.QA_PYROSCOPE_KEY }}
+  LOKI_TENANT_ID: ${{ vars.LOKI_TENANT_ID }}
+  LOKI_URL: ${{ secrets.LOKI_URL }}
+  LOKI_BASIC_AUTH: ${{ secrets.LOKI_BASIC_AUTH }}
+  LOGSTREAM_LOG_TARGETS: loki
+  GRAFANA_URL: ${{ vars.GRAFANA_URL }}
+  RUN_ID: ${{ github.run_id }}
+
+  CHAINLINK_COMMIT_SHA: ${{ github.sha }}
+  CHAINLINK_ENV_USER: ${{ github.actor }}
+  TEST_LOG_LEVEL: debug
+
+jobs:
+
+    # Build Test Dependencies
+
+  build-chainlink:
+    environment: integration
+    permissions:
+      id-token: write
+      contents: read
+    name: Build Chainlink Image
+    runs-on: ubuntu-latest
+    steps:
+      - name: Collect Metrics
+        id: collect-gha-metrics
+        uses: smartcontractkit/push-gha-metrics-action@0281b09807758be1dcc41651e44e62b353808c47 # v2.1.0
+        with:
+          org-id: ${{ secrets.GRAFANA_INTERNAL_TENANT_ID }}
+          basic-auth: ${{ secrets.GRAFANA_INTERNAL_BASIC_AUTH }}
+          hostname: ${{ secrets.GRAFANA_INTERNAL_HOST }}
+          this-job-name: Build Chainlink Image
+        continue-on-error: true
+      - name: Checkout the repo
+        uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
+        with:
+          ref: ${{ github.event.pull_request.head.sha || github.event.merge_group.head_sha }}
+      - name: Build Chainlink Image
+        uses: ./.github/actions/build-chainlink-image
+        with:
+          tag_suffix: ""
+          dockerfile: core/chainlink.Dockerfile
+          git_commit_sha: ${{ github.sha }}
+          GRAFANA_CLOUD_BASIC_AUTH: ${{ secrets.GRAFANA_CLOUD_BASIC_AUTH }}
+          GRAFANA_CLOUD_HOST: ${{ secrets.GRAFANA_CLOUD_HOST }}
+          AWS_REGION: ${{ secrets.QA_AWS_REGION }}
+          AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }}
+
+  build-tests:
+    environment: integration
+    permissions:
+      id-token: write
+      contents: read
+    name: Build Tests Binary
+    runs-on: ubuntu-latest
+    steps:
+      - name: Collect Metrics
+        id: collect-gha-metrics
+        uses: smartcontractkit/push-gha-metrics-action@0281b09807758be1dcc41651e44e62b353808c47 # v2.1.0
+        with:
+          org-id: ${{ secrets.GRAFANA_INTERNAL_TENANT_ID }}
+          basic-auth: ${{ secrets.GRAFANA_INTERNAL_BASIC_AUTH }}
+          hostname: ${{ secrets.GRAFANA_INTERNAL_HOST }}
+          this-job-name: Build Tests Binary
+        continue-on-error: true
+      - name: Checkout the repo
+        uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
+        with:
+          ref: ${{ github.event.pull_request.head.sha || github.event.merge_group.head_sha }}
+      - name: Build Tests
+        uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/build-tests@c67a09566412d153ff7640d99f96b43aa03abc04 # v2.3.6
+        with:
+          test_download_vendor_packages_command: cd ./integration-tests && go mod download
+          token: ${{ secrets.GITHUB_TOKEN }}
+          go_mod_path: ./integration-tests/go.mod
+          go_tags: embed
+          cache_key_id: core-e2e-${{ env.MOD_CACHE_VERSION }}
+          cache_restore_only: "true"
+          binary_name: tests
+
+  build-matrix:
+    environment: integration
+    permissions:
+      id-token: write
+      contents: read
+    name: Build Matrix
+    runs-on: ubuntu-latest
+    outputs:
+      matrix: ${{ steps.build-matrix.outputs.matrix }}
+    steps:
+      - id: build-matrix
+        run: |
+          NETWORKS="[\"${{ github.event.inputs.networks }}\"]"
+          NETWORKS="${NETWORKS//,/\",\"}"
+          echo "matrix=${NETWORKS}"
+
+  # End Build Test Dependencies
+
+  live-vrf-tests:
+    environment: integration
+    permissions:
+      checks: write
+      pull-requests: write
+      id-token: write
+      contents: read
+    needs: [build-chainlink, build-tests, build-matrix]
+    strategy:
+      max-parallel: 1
+      fail-fast: false
+      matrix: 
+        network: ${{fromJson(needs.build-matrix.outputs.matrix)}}
+    name: VRF Tests on ${{ matrix.network }}
+    runs-on: ubuntu-latest
+    steps:
+      - name: Build Secrets Names
+        id: build-secrets-names
+        run: |
+          echo "HTTP_URLS_SECRET_NAME=QA_${{ matrix }}_HTTP_URLS"
+          echo "URLS_SECRET_NAME=QA_${{ matrix }}_URLS"
+      - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
+        with:
+          fetch-depth: 0
+      - name: Prepare Base64 TOML override
+        uses: ./.github/actions/setup-create-base64-config-live-testnets
+        with:
+          runId: ${{ github.run_id }}
+          testLogCollect: ${{ vars.TEST_LOG_COLLECT }}
+          chainlinkImage: ${{ env.CHAINLINK_IMAGE }}
+          chainlinkVersion: ${{ github.sha }}
+          lokiEndpoint: ${{ secrets.LOKI_URL }}
+          lokiTenantId: ${{ vars.LOKI_TENANT_ID }}
+          lokiBasicAuth: ${{ secrets.LOKI_BASIC_AUTH }}
+          logstreamLogTargets: ${{ vars.LOGSTREAM_LOG_TARGETS }}
+          grafanaUrl: ${{ vars.GRAFANA_URL }}
+          grafanaDashboardUrl: "/d/ddf75041-1e39-42af-aa46-361fe4c36e9e/ci-e2e-tests-logs"
+          network: ${{ matrix.network }}
+          httpEndpoints: ${{ secrets[env.HTTP_URLS_SECRET_NAME] }}
+          wsEndpoints: ${{ secrets[env.URLS_SECRET_NAME] }}
+          fundingKeys: ${{ secrets.QA_EVM_KEYS }}
+      - name: Download Tests Binary
+        uses: actions/download-artifact@9bc31d5ccc31df68ecc42ccf4149144866c47d8a # v3.0.2
+        with:
+          name: tests
+      - name: Run Tests
+        uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/run-tests-binary@c67a09566412d153ff7640d99f96b43aa03abc04 # v2.3.6
+        with:
+          test_command_to_run: ./tests -test.timeout 1h -test.count=1 -test.parallel=1 -test.run ${{ inputs.test }}
+          binary_name: tests
+          cl_repo: ${{ env.CHAINLINK_IMAGE }}
+          cl_image_tag: ${{ github.sha }}
+          aws_registries: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}
+          dockerhub_username: ${{ secrets.DOCKERHUB_READONLY_USERNAME }}
+          dockerhub_password: ${{ secrets.DOCKERHUB_READONLY_PASSWORD }}
+          artifacts_location: ./logs
+          token: ${{ secrets.GITHUB_TOKEN }}
+          cache_key_id: core-e2e-${{ env.MOD_CACHE_VERSION }}
+          cache_restore_only: "true"
+          QA_AWS_REGION: ${{ secrets.QA_AWS_REGION }}
+          QA_AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }}
+          QA_KUBECONFIG: ${{ secrets.QA_KUBECONFIG }}
+      - name: Print failed test summary
+        if: always()
+        uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/show-test-summary@c67a09566412d153ff7640d99f96b43aa03abc04 # v2.3.6
+        with:
+          test_directory: "./"