diff --git a/.changeset/pretty-dancers-pull.md b/.changeset/pretty-dancers-pull.md new file mode 100644 index 00000000000..af8ec9db0c8 --- /dev/null +++ b/.changeset/pretty-dancers-pull.md @@ -0,0 +1,5 @@ +--- +"chainlink": patch +--- + +#changed Expand EVM implementation compatibility pipeline diff --git a/.github/actions/build-chainlink-image/action.yml b/.github/actions/build-chainlink-image/action.yml index 26d55ed7f5e..bee48af3989 100644 --- a/.github/actions/build-chainlink-image/action.yml +++ b/.github/actions/build-chainlink-image/action.yml @@ -18,12 +18,16 @@ inputs: dep_evm_sha: description: The chainlink-evm commit sha to use in go deps required: false + check_image_exists: + description: "Check if the image exists in ECR before building" + required: false + default: 'false' runs: using: composite steps: - name: Check if image exists - if: ${{ inputs.dep_evm_sha != '' }} + if: ${{ inputs.dep_evm_sha != '' || inputs.check_image_exists == 'true'}} id: check-image uses: smartcontractkit/chainlink-github-actions/docker/image-exists@fc3e0df622521019f50d772726d6bf8dc919dd38 # v2.3.19 with: diff --git a/.github/actions/notify-slack-jobs-result/action.yml b/.github/actions/notify-slack-jobs-result/action.yml index f5df87bb909..da7daa5b621 100644 --- a/.github/actions/notify-slack-jobs-result/action.yml +++ b/.github/actions/notify-slack-jobs-result/action.yml @@ -25,6 +25,9 @@ inputs: slack_thread_ts: description: "The Slack thread timestamp to post the message to, handy for keeping multiple related results in a single thread" required: false + base64_parsed_results: + description: "Base64 encoded parsed results to use" + required: false runs: using: composite @@ -33,43 +36,54 @@ runs: shell: bash id: test-results run: | - # I feel like there's some clever, fully jq way to do this, but I ain't got the motivation to figure it out - echo "Querying test results at https://api.github.com/repos/${{inputs.github_repository}}/actions/runs/${{ inputs.workflow_run_id }}/jobs" - - # we can get a maximum of 100 jobs per page, after that we need to start using pagination - PARSED_RESULTS=$(curl \ - -H "Authorization: Bearer ${{ inputs.github_token }}" \ - 'https://api.github.com/repos/${{inputs.github_repository}}/actions/runs/${{ inputs.workflow_run_id }}/jobs?per_page=100' \ - | jq -r --arg pattern "${{ inputs.github_job_name_regex }}" '.jobs[] - | select(.name | test($pattern)) as $job - | $job.steps[] - | select(.name == "Run Tests") - | { conclusion: (if .conclusion == "success" then ":white_check_mark:" else ":x:" end), cap: ("*" + ($job.name | capture($pattern).cap) + "*"), html_url: $job.html_url }') - + if [ -n "${{ inputs.base64_parsed_results }}" ]; then + echo "Using base64 parsed results" + PARSED_RESULTS=$(echo "${{ inputs.base64_parsed_results }}" | base64 -d) + else + go install github.com/smartcontractkit/chainlink-testing-framework/tools/workflowresultparser@v1.0.0 + PATH=$PATH:$(go env GOPATH)/bin + export PATH + + workflowresultparser -workflowRunID ${{ inputs.workflow_run_id }} -githubToken ${{ inputs.github_token }} -githubRepo "${{ inputs.github_repository }}" -jobNameRegex "${{ inputs.github_job_name_regex }}" -outputFile=output.json + + if [ ! -f output.json ]; then + PARSED_RESULTS='""' + else + PARSED_RESULTS=$(cat output.json | jq -c "select(has(\"results\")) | .results[]") + fi + + fi + echo "Parsed Results:" echo $PARSED_RESULTS - + ALL_SUCCESS=true - echo "Checking for failures" - echo "$PARSED_RESULTS" | jq -s | jq -r '.[] | select(.conclusion != ":white_check_mark:")' - for row in $(echo "$PARSED_RESULTS" | jq -s | jq -r '.[] | select(.conclusion != ":white_check_mark:")'); do - ALL_SUCCESS=false - break - done - echo "Success: $ALL_SUCCESS" - - echo all_success=$ALL_SUCCESS >> $GITHUB_OUTPUT - - FORMATTED_RESULTS=$(echo $PARSED_RESULTS | jq -s '[.[] - | { - conclusion: .conclusion, - cap: .cap, - html_url: .html_url - } - ] - | map("{\"type\": \"section\", \"text\": {\"type\": \"mrkdwn\", \"text\": \"<\(.html_url)|\(.cap)>: \(.conclusion)\"}}") - | join(",")') - + + if [ "$PARSED_RESULTS" != '""' ]; then + echo "Checking for failures" + echo "$PARSED_RESULTS" | jq -s | jq -r '.[] | select(.conclusion != ":white_check_mark:")' + for row in $(echo "$PARSED_RESULTS" | jq -s | jq -r '.[] | select(.conclusion != ":white_check_mark:")'); do + ALL_SUCCESS=false + break + done + echo "Success: $ALL_SUCCESS" + + echo all_success=$ALL_SUCCESS >> $GITHUB_OUTPUT + + FORMATTED_RESULTS=$(echo $PARSED_RESULTS | jq -s '[.[] + | { + conclusion: .conclusion, + cap: .cap, + html_url: .html_url + } + ] + | map("{\"type\": \"section\", \"text\": {\"type\": \"mrkdwn\", \"text\": \"<\(.html_url)|\(.cap)>: \(.conclusion)\"}}") + | join(",")') + else + echo "Nothing to post, no results found" + exit 0 + fi + echo "Formatted Results:" echo $FORMATTED_RESULTS @@ -82,6 +96,7 @@ runs: echo results=$CLEAN_RESULTS >> $GITHUB_OUTPUT - name: Post Results uses: slackapi/slack-github-action@6c661ce58804a1a20f6dc5fbee7f0381b469e001 # v1.25.0 + if: steps.test-results.outputs.results != '' env: SLACK_BOT_TOKEN: ${{ inputs.slack_bot_token }} with: @@ -108,4 +123,4 @@ runs: ] } ] - } + } diff --git a/.github/workflows/client-compatibility-tests.yml b/.github/workflows/client-compatibility-tests.yml index d953a087d43..d8a4709fb61 100644 --- a/.github/workflows/client-compatibility-tests.yml +++ b/.github/workflows/client-compatibility-tests.yml @@ -1,26 +1,240 @@ name: Client Compatibility Tests on: schedule: - - cron: "30 5 * * *" # Run every night at midnight + 30min EST + - cron: "30 5 * * TUE,FRI" # Run every Tuesday and Friday at midnight + 30min EST push: tags: - "*" + merge_group: + pull_request: workflow_dispatch: + inputs: + chainlinkVersion: + description: commit SHA or tag of the Chainlink version to test + required: false + type: string + evmImplementations: + description: comma separated list of EVM implementations to test (ignored if base64TestList is used) + required: true + type: string + default: "geth,besu,nethermind,erigon" + latestVersionsNumber: + description: how many of latest images of EVM implementations to test with (ignored if base64TestList is used) + required: true + type: number + default: 3 + base64TestList: + description: base64 encoded list of tests to run (same as base64-ed output of testlistgenerator tool) + required: false + type: string env: CHAINLINK_IMAGE: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com/chainlink INTERNAL_DOCKER_REPO: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com MOD_CACHE_VERSION: 2 +concurrency: + group: ${{ github.ref }}-${{ github.repository }}-${{ github.event_name }}--evm-compatibility-tests + cancel-in-progress: true + jobs: # Build Test Dependencies + check-dependency-bump: + name: Check for go-ethereum dependency bump + if: github.event_name == 'pull_request' || github.event_name == 'merge_queue' + runs-on: ubuntu-latest + outputs: + dependency_changed: ${{ steps.changes.outputs.dependency_changed }} + steps: + - name: Checkout code + uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2 + with: + fetch-depth: 0 + - name: Check for go.mod changes + id: changes + run: | + git fetch origin ${{ github.base_ref }} + # if no match is found then grep exits with code 1, but if there is a match it exits with code 0 + # this will return a match if there are any changes on that corresponding line, for example if spacing was changed + DEPENDENCY_CHANGED=$(git diff -U0 origin/${{ github.base_ref }}...HEAD -- go.mod | grep -q 'github.com/ethereum/go-ethereum'; echo $?) + PR_VERSION=$(grep 'github.com/ethereum/go-ethereum' go.mod | awk '{print $2}') + + # here 0 means a match was found, 1 means no match was found + if [ "$DEPENDENCY_CHANGED" -eq 0 ]; then + # Dependency was changed in the PR, now compare with the base branch + git fetch origin ${{ github.base_ref }} + BASE_VERSION=$(git show origin/${{ github.base_ref }}:go.mod | grep 'github.com/ethereum/go-ethereum' | awk '{print $2}') + + echo "Base branch version: $BASE_VERSION" + echo "PR branch version: $PR_VERSION" + + echo "Dependency version changed in the PR compared to the base branch." + echo "dependency_changed=true" >> $GITHUB_OUTPUT + else + echo "No changes to ethereum/go-ethereum dependency in the PR." + echo "PR branch version: $PR_VERSION" + echo "dependency_changed=false" >> $GITHUB_OUTPUT + fi + + should-run: + if: always() + name: Check if the job should run + needs: check-dependency-bump + runs-on: ubuntu-latest + outputs: + should_run: ${{ steps.should-run.outputs.should_run }} + eth_implementations : ${{ steps.should-run.outputs.eth_implementations }} + env: + GITHUB_REF_TYPE: ${{ github.ref_type }} + steps: + - name: Check if the job should run + id: should-run + run: | + if [ "${{ needs.check-dependency-bump.outputs.dependency_changed }}" == "true" ]; then + echo "Will run tests, because go-ethereum dependency was bumped" + echo "should_run=true" >> $GITHUB_OUTPUT + elif [ "$GITHUB_EVENT_NAME" = "schedule" ]; then + echo "Will run tests, because trigger event was $GITHUB_EVENT_NAME" + echo "should_run=true" >> $GITHUB_OUTPUT + elif [ "$GITHUB_EVENT_NAME" = "workflow_dispatch" ]; then + echo "Will run tests, because trigger event was $GITHUB_EVENT_NAME" + echo "should_run=true" >> $GITHUB_OUTPUT + elif [ "$GITHUB_REF_TYPE" = "tag" ]; then + echo "Will run tests, because new tag was created" + echo "should_run=true" >> $GITHUB_OUTPUT + else + echo "Will not run tests" + echo "should_run=false" >> $GITHUB_OUTPUT + fi + + select-versions: + if: always() && needs.should-run.outputs.should_run == 'true' + name: Select Versions + needs: should-run + runs-on: ubuntu-latest + env: + RELEASED_DAYS_AGO: 4 + GITHUB_REF_TYPE: ${{ github.ref_type }} + outputs: + evm_implementations : ${{ steps.select-implementations.outputs.evm_implementations }} + chainlink_version: ${{ steps.select-chainlink-version.outputs.chainlink_version }} + latest_image_count: ${{ steps.get-image-count.outputs.image_count }} + steps: + # ghlatestreleasechecker is a tool to check if new release is available for a given repo + - name: Set Up ghlatestreleasechecker + shell: bash + run: | + go install github.com/smartcontractkit/chainlink-testing-framework/tools/ghlatestreleasechecker@v1.0.0 + - name: Select EVM implementations to test + id: select-implementations + run: | + PATH=$PATH:$(go env GOPATH)/bin + export PATH + + if [ "$GITHUB_EVENT_NAME" = "schedule" ]; then + echo "Checking for new releases" + implementations_arr=() + new_geth=$(ghlatestreleasechecker "ethereum/go-ethereum" $RELEASED_DAYS_AGO) + if [ "$new_geth" != "none" ]; then + echo "New geth release found: $new_geth" + implementations_arr+=("geth") + fi + new_besu=$(ghlatestreleasechecker "hyperledger/besu" $RELEASED_DAYS_AGO) + if [ "new_besu" != "none" ]; then + echo "New besu release found: $new_besu" + implementations_arr+=("besu") + fi + new_erigon=$(ghlatestreleasechecker "ledgerwatch/erigon" $RELEASED_DAYS_AGO) + if [ "new_erigon" != "none" ]; then + echo "New erigon release found: $new_erigon" + implementations_arr+=("erigon") + fi + new_nethermind=$(ghlatestreleasechecker "nethermindEth/nethermind" $RELEASED_DAYS_AGO) + if [ "new_nethermind" != "none" ]; then + echo "New nethermind release found: $new_nethermind" + implementations_arr+=("nethermind") + fi + IFS=',' + eth_implementations="${implementations_arr[*]}" + echo "Found new releases for: $eth_implementations" + echo "evm_implementations=$eth_implementations" >> $GITHUB_OUTPUT + elif [ "$GITHUB_EVENT_NAME" = "workflow_dispatch" ]; then + if [ -n "${{ github.event.inputs.base64TestList }}" ]; then + echo "Base64-ed Test Input provided, ignoring EVM implementations" + else + echo "Will test following EVM implementations: ${{ github.event.inputs.evmImplementations }}" + echo "evm_implementations=${{ github.event.inputs.evmImplementations }}" >> $GITHUB_OUTPUT + fi + else + echo "Will test all EVM implementations" + echo "evm_implementations=geth,besu,nethermind,erigon" >> $GITHUB_OUTPUT + fi + - name: Select Chainlink version + id: select-chainlink-version + run: | + PATH=$PATH:$(go env GOPATH)/bin + export PATH + + if [ "$GITHUB_EVENT_NAME" = "schedule" ]; then + echo "Fetching latest Chainlink stable version" + implementations_arr=() + # we use 100 days since we really want the latest one, and it's highly improbable there won't be a release in last 100 days + chainlink_version=$(ghlatestreleasechecker "smartcontractkit/chainlink" 100) + echo "chainlink_version=$chainlink_version" >> $GITHUB_OUTPUT + elif [ "$GITHUB_EVENT_NAME" = "workflow_dispatch" ]; then + echo "Fetching Chainlink version from input" + if [ -n "${{ github.event.inputs.chainlinkVersion }}" ]; then + echo "Chainlink version provided in input" + chainlink_version="${{ github.event.inputs.chainlinkVersion }}" + else + echo "Chainlink version not provided in input. Using latest commit SHA." + chainlink_version=${{ github.sha }} + fi + echo "chainlink_version=$chainlink_version" >> $GITHUB_OUTPUT + elif [ "$GITHUB_EVENT_NAME" = "pull_request" ]; then + echo "Fetching Chainlink version from PR's head commit" + chainlink_version="${{ github.event.pull_request.head.sha }}" + echo "chainlink_version=$chainlink_version" >> $GITHUB_OUTPUT + elif [ "$GITHUB_EVENT_NAME" = "merge_queue" ]; then + echo "Fetching Chainlink version from merge queue's head commit" + chainlink_version="${{ github.event.merge_group.head_sha }}" + echo "chainlink_version=$chainlink_version" >> $GITHUB_OUTPUT + elif [ "$GITHUB_REF_TYPE" = "tag" ]; then + echo "Fetching Chainlink version from tag" + chainlink_version="${{ github.ref }}" + echo "chainlink_version=$chainlink_version" >> $GITHUB_OUTPUT + else + echo "Unsupported trigger event. It's probably an issue with the pipeline definition. Please reach out to the Test Tooling team." + exit 1 + fi + echo "Will use following Chainlink version: $chainlink_version" + - name: Get image count + id: get-image-count + run: | + if [ "$GITHUB_EVENT_NAME" = "workflow_dispatch" ]; then + echo "Fetching latest image count from input" + if [ -n "${{ github.event.inputs.base64TestList }}" ]; then + echo "Base64-ed Test Input provided, ignoring latest image count" + else + image_count="${{ github.event.inputs.latestVersionsNumber }}" + echo "image_count=$image_count" >> $GITHUB_OUTPUT + fi + else + echo "Fetching default latest image count" + image_count=3 + echo "image_count=$image_count" >> $GITHUB_OUTPUT + fi + echo "Will use following latest image count: $image_count" + check-ecr-images-exist: + name: Check images used as test dependencies exist in ECR + if: always() && needs.should-run.outputs.should_run == 'true' environment: integration permissions: id-token: write contents: read - name: Check images used as test dependencies exist in ECR + needs: [should-run] runs-on: ubuntu-latest strategy: fail-fast: false @@ -36,7 +250,7 @@ jobs: - name: nethermind/nethermind expression: '^[0-9]+\.[0-9]+\.[0-9]+$' - name: tofelb/ethereum-genesis-generator - expression: '^[0-9]+\.[0-9]+\.[0-9]+(\-slots\-per\-epoch)?' + expression: '^[0-9]+\.[0-9]+\.[0-9]+(\-slots\-per\-epoch)?' steps: - name: Update internal ECR if the latest Ethereum client image does not exist uses: smartcontractkit/chainlink-testing-framework/.github/actions/update-internal-mirrors@5eea86ee4f7742b4e944561a570a6b268e712d9e # v1.30.3 @@ -47,14 +261,16 @@ jobs: image_name: ${{matrix.mirror.name}} expression: ${{matrix.mirror.expression}} page_size: ${{matrix.mirror.page_size}} - + build-chainlink: + if: always() && needs.should-run.outputs.should_run == 'true' environment: integration permissions: id-token: write contents: read name: Build Chainlink Image runs-on: ubuntu-latest + needs: [should-run, select-versions] steps: - name: Collect Metrics id: collect-gha-metrics @@ -69,202 +285,310 @@ jobs: - name: Checkout the repo uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2 with: - ref: ${{ github.event.pull_request.head.sha || github.event.merge_group.head_sha }} + ref: ${{ needs.select-versions.outputs.chainlink_version }} - name: Build Chainlink Image uses: ./.github/actions/build-chainlink-image with: tag_suffix: "" dockerfile: core/chainlink.Dockerfile - git_commit_sha: ${{ github.sha }} + git_commit_sha: ${{ needs.select-versions.outputs.chainlink_version }} + check_image_exists: 'true' AWS_REGION: ${{ secrets.QA_AWS_REGION }} AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }} + get-latest-available-images: + name: Get Latest EVM Implementation's Images + if: always() && needs.should-run.outputs.should_run == 'true' + environment: integration + runs-on: ubuntu-latest + needs: [check-ecr-images-exist, should-run, select-versions] + permissions: + id-token: write + contents: read + env: + LATEST_IMAGE_COUNT: ${{ needs.select-versions.outputs.latest_image_count }} + outputs: + geth_images: ${{ env.GETH_IMAGES }} + nethermind_images: ${{ env.NETHERMIND_IMAGES }} + besu_images: ${{ env.BESU_IMAGES }} + erigon_images: ${{ env.ERIGON_IMAGES }} + steps: + # Setup AWS creds + - name: Configure AWS Credentials + uses: aws-actions/configure-aws-credentials@e3dd6a429d7300a6a4c196c26e071d42e0343502 # v4.0.2 + with: + aws-region: ${{ secrets.QA_AWS_REGION }} + role-to-assume: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }} + role-duration-seconds: 3600 + # Login to ECR + - name: Login to Amazon ECR + id: login-ecr + uses: aws-actions/amazon-ecr-login@062b18b96a7aff071d4dc91bc00c4c1a7945b076 # v2.0.1 + with: + mask-password: "true" + env: + AWS_REGION: ${{ secrets.QA_AWS_REGION }} + # ecrimagefetcher is a tool to get latest images from ECR + - name: Set Up ecrimagefetcher + shell: bash + run: | + go install github.com/smartcontractkit/chainlink-testing-framework/tools/ecrimagefetcher@v1.0.1 + - name: Get latest docker images from ECR + if: ${{ github.event.inputs.base64TestList == '' }} + env: + AWS_REGION: ${{ secrets.QA_AWS_REGION }} + ETH_IMPLEMENTATIONS: ${{ needs.select-versions.outputs.evm_implementations }} + run: | + PATH=$PATH:$(go env GOPATH)/bin + export PATH + if [[ "$ETH_IMPLEMENTATIONS" == *"geth"* ]]; then + geth_images=$(ecrimagefetcher 'ethereum/client-go' '^v[0-9]+\.[0-9]+\.[0-9]+$' ${{ env.LATEST_IMAGE_COUNT }}) + echo "GETH_IMAGES=$geth_images" >> $GITHUB_ENV + echo "Geth latest images: $geth_images" + fi + + if [[ "$ETH_IMPLEMENTATIONS" == *"nethermind"* ]]; then + nethermind_images=$(ecrimagefetcher 'nethermind/nethermind' '^[0-9]+\.[0-9]+\.[0-9]+$' ${{ env.LATEST_IMAGE_COUNT }}) + echo "NETHERMIND_IMAGES=$nethermind_images" >> $GITHUB_ENV + echo "Nethermind latest images: $nethermind_images" + fi + + if [[ "$ETH_IMPLEMENTATIONS" == *"besu"* ]]; then + # 24.3.3 is ignored as it doesn't support data & input fields in eth_call + besu_images=$(ecrimagefetcher 'hyperledger/besu' '^[0-9]+\.[0-9]+(\.[0-9]+)?$' ${{ env.LATEST_IMAGE_COUNT }} ">=24.5.1") + echo "BESU_IMAGES=$besu_images" >> $GITHUB_ENV + echo "Besu latest images: $besu_images" + fi + + if [[ "$ETH_IMPLEMENTATIONS" == *"erigon"* ]]; then + # 2.60.0 and 2.60.1 are ignored as they stopped working with CL node + erigon_images=$(ecrimagefetcher 'thorax/erigon' '^v[0-9]+\.[0-9]+\.[0-9]+$' ${{ env.LATEST_IMAGE_COUNT }} "> $GITHUB_ENV + echo "Erigon latest images: $erigon_images" + fi + # End Build Test Dependencies - client-compatibility-matrix: + prepare-compatibility-matrix: + name: Prepare Compatibility Matrix + if: always() && needs.should-run.outputs.should_run == 'true' environment: integration permissions: checks: write pull-requests: write id-token: write contents: read - needs: [build-chainlink, check-ecr-images-exist] + needs: [get-latest-available-images,should-run,select-versions] + runs-on: ubuntu-latest + env: + ETH_IMPLEMENTATIONS: ${{ needs.select-versions.outputs.evm_implementations }} + BASE64_TEST_LIST: ${{ github.event.inputs.base64TestList }} + outputs: + matrix: ${{ env.JOB_MATRIX_JSON }} + steps: + - name: Decode Base64 Test List Input if Set + if: env.BASE64_TEST_LIST != '' + run: | + echo "Decoding base64 tests list from the input" + DECODED_BASE64_TEST_LIST=$(echo $BASE64_TEST_LIST | base64 -d) + echo "Decoded input:" + echo "$DECODED_BASE64_TEST_LIST" + is_valid=$(echo "$DECODED_BASE64_TEST_LIST" | jq . > /dev/null 2>&1; echo $?) + if [ "$is_valid" -ne 0 ]; then + echo "Invalid base64 input. Please provide a valid base64 encoded JSON list of tests." + echo "Here is an example of valid JSON:" + cat <> $GITHUB_ENV + # testlistgenerator is a tool that builds a matrix of tests to run + - name: Set Up testlistgenerator + if: env.BASE64_TEST_LIST == '' + shell: bash + run: | + go install github.com/smartcontractkit/chainlink-testing-framework/tools/testlistgenerator@v1.1.0 + - name: Prepare matrix input + if: env.BASE64_TEST_LIST == '' + run: | + PATH=$PATH:$(go env GOPATH)/bin + export PATH + + if [[ "$ETH_IMPLEMENTATIONS" == *"geth"* ]]; then + echo "Will test compatibility with geth" + testlistgenerator -o compatibility_test_list.json -p cron -r TestCronBasic -f './smoke/cron_test.go' -e geth -d "${{ needs.get-latest-available-images.outputs.geth_images }}" -t "evm-implementation-compatibility-test" -n "ubuntu-latest" + testlistgenerator -o compatibility_test_list.json -p flux -r TestFluxBasic -f './smoke/flux_test.go' -e geth -d "${{ needs.get-latest-available-images.outputs.geth_images }}" -t "evm-implementation-compatibility-test" -n "ubuntu-latest" + testlistgenerator -o compatibility_test_list.json -p runlog -r TestRunLogBasic -f './smoke/runlog_test.go' -e geth -d "${{ needs.get-latest-available-images.outputs.geth_images }}" -t "evm-implementation-compatibility-test" -n "ubuntu-latest" + testlistgenerator -o compatibility_test_list.json -p log_poller -r TestLogPollerFewFiltersFixedDepth -f './smoke/log_poller_test.go' -e geth -d "${{ needs.get-latest-available-images.outputs.geth_images }}" -t "evm-implementation-compatibility-test" -n "ubuntu-latest" + testlistgenerator -o compatibility_test_list.json -p ocr -r TestOCRBasic -f './smoke/ocr_test.go' -e geth -d "${{ needs.get-latest-available-images.outputs.geth_images }}" -t "evm-implementation-compatibility-test" -n "ubuntu-latest" + testlistgenerator -o compatibility_test_list.json -p ocr2 -r '^TestOCRv2Basic/plugins$' -f './smoke/ocr2_test.go' -e geth -d "${{ needs.get-latest-available-images.outputs.geth_images }}" -t "evm-implementation-compatibility-test" -n "ubuntu-latest" + testlistgenerator -o compatibility_test_list.json -p automation -r 'TestAutomationBasic/registry_2_1_logtrigger' -f './smoke/automation_test.go' -e geth -d "${{ needs.get-latest-available-images.outputs.geth_images }}" -t "evm-implementation-compatibility-test" -n "ubuntu-latest" + testlistgenerator -o compatibility_test_list.json -p keeper -r 'TestKeeperBasicSmoke/registry_1_3' -f './smoke/keeper_test.go' -e geth -d "${{ needs.get-latest-available-images.outputs.geth_images }}" -t "evm-implementation-compatibility-test" -n "ubuntu-latest" + testlistgenerator -o compatibility_test_list.json -p vrf -r '^TestVRFBasic/Request_Randomness$' -f './smoke/vrf_test.go' -e geth -d "${{ needs.get-latest-available-images.outputs.geth_images }}" -t "evm-implementation-compatibility-test" -n "ubuntu-latest" + testlistgenerator -o compatibility_test_list.json -p vrfv2 -r '^TestVRFv2Basic/Request_Randomness$' -f './smoke/vrfv2_test.go' -e geth -d "${{ needs.get-latest-available-images.outputs.geth_images }}" -t "evm-implementation-compatibility-test" -n "ubuntu-latest" + testlistgenerator -o compatibility_test_list.json -p vrfv2plus -r '^TestVRFv2Plus$/^Link_Billing$' -f './smoke/vrfv2plus_test.go' -e geth -d "${{ needs.get-latest-available-images.outputs.geth_images }}" -t "evm-implementation-compatibility-test" -n "ubuntu-latest" + else + echo "Will not test compatibility with geth" + fi + + if [[ "$ETH_IMPLEMENTATIONS" == *"besu"* ]]; then + echo "Will test compatibility with besu" + testlistgenerator -o compatibility_test_list.json -p cron -r TestCronBasic -f './smoke/cron_test.go' -e besu -d "${{ needs.get-latest-available-images.outputs.besu_images }}" -t "evm-implementation-compatibility-test" -n "ubuntu-latest" + testlistgenerator -o compatibility_test_list.json -p flux -r TestFluxBasic -f './smoke/flux_test.go' -e besu -d "${{ needs.get-latest-available-images.outputs.besu_images }}" -t "evm-implementation-compatibility-test" -n "ubuntu-latest" + testlistgenerator -o compatibility_test_list.json -p runlog -r TestRunLogBasic -f './smoke/runlog_test.go' -e besu -d "${{ needs.get-latest-available-images.outputs.besu_images }}" -t "evm-implementation-compatibility-test" -n "ubuntu-latest" + testlistgenerator -o compatibility_test_list.json -p log_poller -r TestLogPollerFewFiltersFixedDepth -f './smoke/log_poller_test.go' -e besu -d "${{ needs.get-latest-available-images.outputs.besu_images }}" -t "evm-implementation-compatibility-test" -n "ubuntu-latest" + testlistgenerator -o compatibility_test_list.json -p ocr -r TestOCRBasic -f './smoke/ocr_test.go' -e besu -d "${{ needs.get-latest-available-images.outputs.besu_images }}" -t "evm-implementation-compatibility-test" -n "ubuntu-latest" + testlistgenerator -o compatibility_test_list.json -p ocr2 -r '^TestOCRv2Basic/plugins$' -f './smoke/ocr2_test.go' -e besu -d "${{ needs.get-latest-available-images.outputs.besu_images }}" -t "evm-implementation-compatibility-test" -n "ubuntu-latest" + testlistgenerator -o compatibility_test_list.json -p automation -r 'TestAutomationBasic/registry_2_1_logtrigger' -f './smoke/automation_test.go' -e besu -d "${{ needs.get-latest-available-images.outputs.besu_images }}" -t "evm-implementation-compatibility-test" -n "ubuntu-latest" + testlistgenerator -o compatibility_test_list.json -p keeper -r 'TestKeeperBasicSmoke/registry_1_3' -f './smoke/keeper_test.go' -e besu -d "${{ needs.get-latest-available-images.outputs.besu_images }}" -t "evm-implementation-compatibility-test" -n "ubuntu-latest" + testlistgenerator -o compatibility_test_list.json -p vrf -r '^TestVRFBasic/Request_Randomness$' -f './smoke/vrf_test.go' -e besu -d "${{ needs.get-latest-available-images.outputs.besu_images }}" -t "evm-implementation-compatibility-test" -n "ubuntu-latest" + # VRFv2 and VRFV2Plus tests are disabled for besu until the functionalities they rely on are supported + # testlistgenerator -o compatibility_test_list.json -p vrfv2 -r '^TestVRFv2Basic/Request_Randomness$' -f './smoke/vrfv2_test.go' -e besu -d "${{ needs.get-latest-available-images.outputs.besu_images }}" -t "evm-implementation-compatibility-test" -n "ubuntu-latest" + # testlistgenerator -o compatibility_test_list.json -p vrfv2plus -r '^TestVRFv2Plus$/^Link_Billing$' -f './smoke/vrfv2plus_test.go' -e besu -d "${{ needs.get-latest-available-images.outputs.besu_images }}" -t "evm-implementation-compatibility-test" -n "ubuntu-latest" + else + echo "Will not test compatibility with besu" + fi + + if [[ "$ETH_IMPLEMENTATIONS" == *"erigon"* ]]; then + echo "Will test compatibility with erigon" + testlistgenerator -o compatibility_test_list.json -p cron -r TestCronBasic -f './smoke/cron_test.go' -e erigon -d "${{ needs.get-latest-available-images.outputs.erigon_images }}" -t "evm-implementation-compatibility-test" -n "ubuntu-latest" + testlistgenerator -o compatibility_test_list.json -p flux -r TestFluxBasic -f './smoke/flux_test.go' -e erigon -d "${{ needs.get-latest-available-images.outputs.erigon_images }}" -t "evm-implementation-compatibility-test" -n "ubuntu-latest" + testlistgenerator -o compatibility_test_list.json -p runlog -r TestRunLogBasic -f './smoke/runlog_test.go' -e erigon -d "${{ needs.get-latest-available-images.outputs.erigon_images }}" -t "evm-implementation-compatibility-test" -n "ubuntu-latest" + testlistgenerator -o compatibility_test_list.json -p log_poller -r TestLogPollerFewFiltersFixedDepth -f './smoke/log_poller_test.go' -e erigon -d "${{ needs.get-latest-available-images.outputs.erigon_images }}" -t "evm-implementation-compatibility-test" -n "ubuntu-latest" + testlistgenerator -o compatibility_test_list.json -p ocr -r TestOCRBasic -f './smoke/ocr_test.go' -e erigon -d "${{ needs.get-latest-available-images.outputs.erigon_images }}" -t "evm-implementation-compatibility-test" -n "ubuntu-latest" + testlistgenerator -o compatibility_test_list.json -p ocr2 -r '^TestOCRv2Basic/plugins$' -f './smoke/ocr2_test.go' -e erigon -d "${{ needs.get-latest-available-images.outputs.erigon_images }}" -t "evm-implementation-compatibility-test" -n "ubuntu-latest" + testlistgenerator -o compatibility_test_list.json -p automation -r 'TestAutomationBasic/registry_2_1_logtrigger' -f './smoke/automation_test.go' -e erigon -d "${{ needs.get-latest-available-images.outputs.erigon_images }}" -t "evm-implementation-compatibility-test" -n "ubuntu-latest" + testlistgenerator -o compatibility_test_list.json -p keeper -r 'TestKeeperBasicSmoke/registry_1_3' -f './smoke/keeper_test.go' -e erigon -d "${{ needs.get-latest-available-images.outputs.erigon_images }}" -t "evm-implementation-compatibility-test" -n "ubuntu-latest" + testlistgenerator -o compatibility_test_list.json -p vrf -r '^TestVRFBasic/Request_Randomness$' -f './smoke/vrf_test.go' -e erigon -d "${{ needs.get-latest-available-images.outputs.erigon_images }}" -t "evm-implementation-compatibility-test" -n "ubuntu-latest" + testlistgenerator -o compatibility_test_list.json -p vrfv2 -r '^TestVRFv2Basic/Request_Randomness$' -f './smoke/vrfv2_test.go' -e erigon -d "${{ needs.get-latest-available-images.outputs.erigon_images }}" -t "evm-implementation-compatibility-test" -n "ubuntu-latest" + testlistgenerator -o compatibility_test_list.json -p vrfv2plus -r '^TestVRFv2Plus$/^Link_Billing$' -f './smoke/vrfv2plus_test.go' -e erigon -d "${{ needs.get-latest-available-images.outputs.erigon_images }}" -t "evm-implementation-compatibility-test" -n "ubuntu-latest" + else + echo "Will not test compatibility with erigon" + fi + + if [[ "$ETH_IMPLEMENTATIONS" == *"nethermind"* ]]; then + echo "Will test compatibility with nethermind" + testlistgenerator -o compatibility_test_list.json -p cron -r TestCronBasic -f './smoke/cron_test.go' -e nethermind -d "${{ needs.get-latest-available-images.outputs.nethermind_images }}" -t "evm-implementation-compatibility-test" -n "ubuntu-latest" + testlistgenerator -o compatibility_test_list.json -p flux -r TestFluxBasic -f './smoke/flux_test.go' -e nethermind -d "${{ needs.get-latest-available-images.outputs.nethermind_images }}" -t "evm-implementation-compatibility-test" -n "ubuntu-latest" + testlistgenerator -o compatibility_test_list.json -p runlog -r TestRunLogBasic -f './smoke/runlog_test.go' -e nethermind -d "${{ needs.get-latest-available-images.outputs.nethermind_images }}" -t "evm-implementation-compatibility-test" -n "ubuntu-latest" + testlistgenerator -o compatibility_test_list.json -p log_poller -r TestLogPollerFewFiltersFixedDepth -f './smoke/log_poller_test.go' -e nethermind -d "${{ needs.get-latest-available-images.outputs.nethermind_images }}" -t "evm-implementation-compatibility-test" -n "ubuntu-latest" + testlistgenerator -o compatibility_test_list.json -p ocr -r TestOCRBasic -f './smoke/ocr_test.go' -e nethermind -d "${{ needs.get-latest-available-images.outputs.nethermind_images }}" -t "evm-implementation-compatibility-test" -n "ubuntu-latest" + testlistgenerator -o compatibility_test_list.json -p ocr2 -r '^TestOCRv2Basic/plugins$' -f './smoke/ocr2_test.go' -e nethermind -d "${{ needs.get-latest-available-images.outputs.nethermind_images }}" -t "evm-implementation-compatibility-test" -n "ubuntu-latest" + testlistgenerator -o compatibility_test_list.json -p automation -r 'TestAutomationBasic/registry_2_1_logtrigger' -f './smoke/automation_test.go' -e nethermind -d "${{ needs.get-latest-available-images.outputs.nethermind_images }}" -t "evm-implementation-compatibility-test" -n "ubuntu-latest" + testlistgenerator -o compatibility_test_list.json -p keeper -r 'TestKeeperBasicSmoke/registry_1_3' -f './smoke/keeper_test.go' -e nethermind -d "${{ needs.get-latest-available-images.outputs.nethermind_images }}" -t "evm-implementation-compatibility-test" -n "ubuntu-latest" + testlistgenerator -o compatibility_test_list.json -p vrf -r '^TestVRFBasic/Request_Randomness$' -f './smoke/vrf_test.go' -e nethermind -d "${{ needs.get-latest-available-images.outputs.nethermind_images }}" -t "evm-implementation-compatibility-test" -n "ubuntu-latest" + # VRFv2 and VRFV2Plus tests are disabled for nethermind until the functionalities they rely on are supported + # testlistgenerator -o compatibility_test_list.json -p vrfv2 -r '^TestVRFv2Basic/Request_Randomness$' -f './smoke/vrfv2_test.go' -e nethermind -d "${{ needs.get-latest-available-images.outputs.nethermind_images }}" -t "evm-implementation-compatibility-test" -n "ubuntu-latest" + # testlistgenerator -o compatibility_test_list.json -p vrfv2plus -r '^TestVRFv2Plus$/^Link_Billing$' -f './smoke/vrfv2plus_test.go' -e nethermind -d "${{ needs.get-latest-available-images.outputs.nethermind_images }}" -t "evm-implementation-compatibility-test" -n "ubuntu-latest" + else + echo "Will not test compatibility with nethermind" + fi + + jq . compatibility_test_list.json + JOB_MATRIX_JSON=$(jq -c . compatibility_test_list.json) + echo "JOB_MATRIX_JSON=${JOB_MATRIX_JSON}" >> $GITHUB_ENV + + run-client-compatibility-matrix: + name: ${{ matrix.evm_node.product }} compatibility with ${{ matrix.evm_node.docker_image }} + if: always() && needs.should-run.outputs.should_run == 'true' && needs.build-chainlink.result == 'success' + environment: integration + permissions: + checks: write + pull-requests: write + id-token: write + contents: read + needs: [build-chainlink, prepare-compatibility-matrix, should-run, select-versions] env: SELECTED_NETWORKS: SIMULATED,SIMULATED_1,SIMULATED_2 - CHAINLINK_COMMIT_SHA: ${{ github.sha }} + CHAINLINK_COMMIT_SHA: ${{ needs.select-versions.outputs.chainlink_version }} CHAINLINK_ENV_USER: ${{ github.actor }} TEST_LOG_LEVEL: debug strategy: fail-fast: false matrix: - include: - - name: ocr-geth - os: ubuntu-latest - test: TestOCRBasic - file: ocr - client: geth - timeout: 30m - pyroscope_env: ci-smoke-ocr-geth-simulated - - name: ocr-nethermind - test: TestOCRBasic - file: ocr - client: nethermind - timeout: 30m - pyroscope_env: ci-smoke-ocr-nethermind-simulated - - name: ocr-besu - test: TestOCRBasic - file: ocr - client: besu - timeout: 30m - pyroscope_env: ci-smoke-ocr-besu-simulated - - name: ocr-erigon - test: TestOCRBasic - file: ocr - client: erigon - timeout: 30m - pyroscope_env: ci-smoke-ocr-erigon-simulated - - name: ocr2-geth - test: "^TestOCRv2Basic/plugins$" - file: ocr2 - client: geth - timeout: 30m - pyroscope_env: ci-smoke-ocr2-geth-simulated - - name: ocr2-nethermind - test: "^TestOCRv2Basic/plugins$" - file: ocr2 - client: nethermind - timeout: 30m - pyroscope_env: ci-smoke-nethermind-evm-simulated - - name: ocr2-besu - test: "^TestOCRv2Basic/plugins$" - file: ocr2 - client: besu - timeout: 30m - pyroscope_env: ci-smoke-ocr2-besu-simulated - - name: ocr2-erigon - test: "^TestOCRv2Basic/plugins$" - file: ocr2 - client: erigon - timeout: 60m - pyroscope_env: ci-smoke-ocr2-erigon-simulated + evm_node: ${{fromJson(needs.prepare-compatibility-matrix.outputs.matrix)}} runs-on: ubuntu-latest - name: Client Compatibility Test ${{ matrix.name }} steps: - name: Checkout the repo uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2 with: repository: smartcontractkit/chainlink - ref: ${{ github.sha }} - - name: Prepare Base64 TOML config - env: - RUN_ID: ${{ github.run_id }} - SELECTED_NETWORKS: SIMULATED,SIMULATED_1,SIMULATED_2 - PYROSCOPE_SERVER: ${{ secrets.QA_PYROSCOPE_INSTANCE }} - PYROSCOPE_ENVIRONMENT: ci-client-compatability-${{ matrix.client }}-testnet - PYROSCOPE_KEY: ${{ secrets.QA_PYROSCOPE_KEY }} - ETH2_EL_CLIENT: ${{matrix.client}} - CHAINLINK_VERSION: ${{ github.sha }} - GRAFANA_URL: "http://localhost:8080/primary" - GRAFANA_DASHBOARD_URL: "/d/ddf75041-1e39-42af-aa46-361fe4c36e9e/ci-e2e-tests-logs" - GRAFANA_BEARER_TOKEN: ${{ secrets.GRAFANA_INTERNAL_URL_SHORTENER_TOKEN }} - LOKI_ENDPOINT: https://${{ secrets.GRAFANA_INTERNAL_HOST }}/loki/api/v1/push - LOKI_TENANT_ID: ${{ secrets.GRAFANA_INTERNAL_TENANT_ID }} - LOKI_BASIC_AUTH: ${{ secrets.GRAFANA_INTERNAL_BASIC_AUTH }} + ref: ${{ needs.select-versions.outputs.chainlink_version }} + - name: Setup GAP for Grafana + uses: smartcontractkit/.github/actions/setup-gap@d316f66b2990ea4daa479daa3de6fc92b00f863e # setup-gap@0.3.2 + with: + # aws inputs + aws-region: ${{ secrets.AWS_REGION }} + aws-role-arn: ${{ secrets.AWS_OIDC_IAM_ROLE_VALIDATION_PROD_ARN }} + api-gateway-host: ${{ secrets.AWS_API_GW_HOST_GRAFANA }} + # other inputs + duplicate-authorization-header: "true" + - name: Prepare Base64 TOML override + uses: ./.github/actions/setup-create-base64-config + with: + runId: ${{ github.run_id }} + testLogCollect: ${{ vars.TEST_LOG_COLLECT }} + selectedNetworks: ${{ env.SELECTED_NETWORKS }} + chainlinkImage: ${{ env.CHAINLINK_IMAGE }} + chainlinkVersion: ${{ needs.select-versions.outputs.chainlink_version }} + lokiEndpoint: https://${{ secrets.GRAFANA_INTERNAL_HOST }}/loki/api/v1/push + lokiTenantId: ${{ secrets.GRAFANA_INTERNAL_TENANT_ID }} + lokiBasicAuth: ${{ secrets.GRAFANA_INTERNAL_BASIC_AUTH }} + logstreamLogTargets: ${{ vars.LOGSTREAM_LOG_TARGETS }} + grafanaUrl: "http://localhost:8080/primary" + grafanaDashboardUrl: "/d/ddf75041-1e39-42af-aa46-361fe4c36e9e/ci-e2e-tests-logs" + grafanaBearerToken: ${{ secrets.GRAFANA_INTERNAL_URL_SHORTENER_TOKEN }} + ethExecutionClient: ${{ matrix.evm_node.eth_implementation }} + customEthClientDockerImage: ${{ matrix.evm_node.docker_image }} + pyroscopeServer: ${{ secrets.QA_PYROSCOPE_INSTANCE }} + pyroscopeEnvironment: ci-client-compatability-${{ matrix.eth_client }}-testnet + pyroscopeKey: ${{ secrets.QA_PYROSCOPE_KEY }} + - name: Prepare test log name run: | - convert_to_toml_array() { - local IFS=',' - local input_array=($1) - local toml_array_format="[" - - for element in "${input_array[@]}"; do - toml_array_format+="\"$element\"," - done - - toml_array_format="${toml_array_format%,}]" - echo "$toml_array_format" - } - - selected_networks=$(convert_to_toml_array "$SELECTED_NETWORKS") - - if [ -n "$ETH2_EL_CLIENT" ]; then - execution_layer="$ETH2_EL_CLIENT" - else - execution_layer="geth" - fi - - if [ -n "$PYROSCOPE_SERVER" ]; then - pyroscope_enabled=true - else - pyroscope_enabled=false - fi + replace_special_chars() { + if [ -z "$1" ]; then + echo "Please provide a string as an argument." + return 1 + fi - grafana_bearer_token="" - if [ -n "$GRAFANA_BEARER_TOKEN" ]; then - grafana_bearer_token="bearer_token_secret=\"$GRAFANA_BEARER_TOKEN\"" - fi - - cat << EOF > config.toml - [Network] - selected_networks=$selected_networks + local input_string="$1" - [ChainlinkImage] - image="$CHAINLINK_IMAGE" - version="$CHAINLINK_VERSION" + # Replace '/' with '-' + local modified_string="${input_string//\//-}" - [Pyroscope] - enabled=$pyroscope_enabled - server_url="$PYROSCOPE_SERVER" - environment="$PYROSCOPE_ENVIRONMENT" - key_secret="$PYROSCOPE_KEY" - - [Logging] - run_id="$RUN_ID" - - [Logging.LogStream] - log_targets=["file","loki"] - - [Logging.Loki] - tenant_id="$LOKI_TENANT_ID" - endpoint="$LOKI_ENDPOINT" - basic_auth_secret="$LOKI_BASIC_AUTH" - - [Logging.Grafana] - base_url="$GRAFANA_URL" - dashboard_url="$GRAFANA_DASHBOARD_URL" - $grafana_bearer_token + # Replace ':' with '-' + modified_string="${modified_string//:/-}" - [PrivateEthereumNetwork] - ethereum_version="eth2" - consensus_layer="prysm" - execution_layer="$execution_layer" - wait_for_finalization=false - - [PrivateEthereumNetwork.EthereumChainConfig] - chain_id=1337 - genesis_delay=15 - seconds_per_slot=3 - validator_count=8 - slots_per_epoch=2 - addresses_to_fund=["0xf39Fd6e51aad88F6F4ce6aB8827279cffFb92266"] - EOF + # Replace '.' with '-' + modified_string="${modified_string//./-}" - BASE64_CONFIG_OVERRIDE=$(cat config.toml | base64 -w 0) - echo ::add-mask::$BASE64_CONFIG_OVERRIDE - echo "BASE64_CONFIG_OVERRIDE=$BASE64_CONFIG_OVERRIDE" >> $GITHUB_ENV - touch .root_dir + echo "$modified_string" + } + echo "TEST_LOG_NAME=$(replace_special_chars "${{ matrix.evm_node.product }}-${{ matrix.evm_node.docker_image }}-test-logs")" >> $GITHUB_ENV - name: Run Tests uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/run-tests@af92c5fae8dcf1659201e907db82d221fc304b94 # v2.3.21 with: - test_command_to_run: cd ./integration-tests/smoke && go test -timeout ${{ matrix.timeout }} -count=1 -json -test.run ${{ matrix.test }} 2>&1 | tee /tmp/gotest.log | gotestloghelper -ci -singlepackage -hidepassingtests=false -hidepassinglogs + test_command_to_run: cd ./integration-tests && touch .root_dir && go test -timeout 30m -count=1 -json ${{ matrix.evm_node.run }} 2>&1 | tee /tmp/gotest.log | gotestloghelper -ci -singlepackage -hidepassingtests=false -hidepassinglogs test_download_vendor_packages_command: cd ./integration-tests && go mod download cl_repo: ${{ env.CHAINLINK_IMAGE }} - cl_image_tag: ${{ github.sha }} + cl_image_tag: ${{ needs.select-versions.outputs.chainlink_version }} aws_registries: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }} - artifacts_name: ${{ matrix.name }}-test-logs + artifacts_name: ${{ env.TEST_LOG_NAME }} artifacts_location: | ./integration-tests/smoke/logs/ /tmp/gotest.log - publish_check_name: ${{ matrix.name }} + publish_check_name: ${{ matrix.evm_node.product }}-${{ matrix.evm_node.eth_implementation }} token: ${{ secrets.GITHUB_TOKEN }} go_mod_path: ./integration-tests/go.mod cache_key_id: core-e2e-${{ env.MOD_CACHE_VERSION }} @@ -281,7 +605,7 @@ jobs: start-slack-thread: name: Start Slack Thread - if: ${{ always() && needs.*.result != 'skipped' && needs.*.result != 'cancelled' }} + if: always() && needs.*.result != 'skipped' && needs.*.result != 'cancelled' && needs.should-run.outputs.should_run == 'true' environment: integration outputs: thread_ts: ${{ steps.slack.outputs.thread_ts }} @@ -291,7 +615,7 @@ jobs: id-token: write contents: read runs-on: ubuntu-latest - needs: [client-compatibility-matrix] + needs: [run-client-compatibility-matrix, should-run, select-versions] steps: - name: Debug Result run: echo ${{ join(needs.*.result, ',') }} @@ -310,7 +634,7 @@ jobs: "type": "header", "text": { "type": "plain_text", - "text": "Client Compatability Test Results ${{ contains(join(needs.*.result, ','), 'failure') && ':x:' || ':white_check_mark:'}}", + "text": "EVM Implementation Compatibility Test Results ${{ contains(join(needs.*.result, ','), 'failure') && ':x:' || ':white_check_mark:'}}", "emoji": true } }, @@ -328,7 +652,7 @@ jobs: "type": "section", "text": { "type": "mrkdwn", - "text": "<${{ github.server_url }}/${{ github.repository }}/releases/tag/${{ github.ref_name }}|${{ github.ref_name }}> | <${{ github.server_url }}/${{ github.repository }}/commit/${{ github.sha }}|${{ github.sha }}> | <${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}|Run>" + "text": "<${{ github.server_url }}/${{ github.repository }}/releases/tag/${{ github.ref_name }}|${{ github.ref_name }}> | <${{ github.server_url }}/${{ github.repository }}/commit/${{ needs.select-versions.outputs.chainlink_version }}|${{ needs.select-versions.outputs.chainlink_version }}> | <${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}|Run>" } } ] @@ -338,9 +662,95 @@ jobs: env: SLACK_BOT_TOKEN: ${{ secrets.QA_SLACK_API_KEY }} + parse-test-results: + name: Parse Test Results + if: always() && needs.*.result != 'skipped' && needs.*.result != 'cancelled' && needs.should-run.outputs.should_run == 'true' + environment: integration + permissions: + checks: write + pull-requests: write + id-token: write + contents: read + runs-on: ubuntu-latest + needs: [run-client-compatibility-matrix,should-run] + outputs: + base64_parsed_results: ${{ steps.get-test-results.outputs.base64_parsed_results }} + steps: + # workflowresultparser is a tool to get job results from a workflow run + - name: Set Up workflowresultparser + shell: bash + run: | + go install github.com/smartcontractkit/chainlink-testing-framework/tools/workflowresultparser@v1.0.0 + - name: Get and parse Test Results + shell: bash + id: get-test-results + run: | + PATH=$PATH:$(go env GOPATH)/bin + export PATH + + workflowresultparser -workflowRunID ${{ github.run_id }} -githubToken ${{ github.token }} -githubRepo "${{ github.repository }}" -jobNameRegex "^automation compatibility with (.*?)$" -namedKey="automation" -outputFile=output.json + workflowresultparser -workflowRunID ${{ github.run_id }} -githubToken ${{ github.token }} -githubRepo "${{ github.repository }}" -jobNameRegex "^keeper compatibility with (.*?)$" -namedKey="keeper" -outputFile=output.json + workflowresultparser -workflowRunID ${{ github.run_id }} -githubToken ${{ github.token }} -githubRepo "${{ github.repository }}" -jobNameRegex "^log_poller compatibility with (.*?)$" -namedKey="log_poller" -outputFile=output.json + workflowresultparser -workflowRunID ${{ github.run_id }} -githubToken ${{ github.token }} -githubRepo "${{ github.repository }}" -jobNameRegex "^ocr compatibility with (.*?)$" -namedKey="ocr" -outputFile=output.json + workflowresultparser -workflowRunID ${{ github.run_id }} -githubToken ${{ github.token }} -githubRepo "${{ github.repository }}" -jobNameRegex "^ocr2 compatibility with (.*?)$" -namedKey="ocr2" -outputFile=output.json + workflowresultparser -workflowRunID ${{ github.run_id }} -githubToken ${{ github.token }} -githubRepo "${{ github.repository }}" -jobNameRegex "^vrf compatibility with (.*?)$" -namedKey="vrf" -outputFile=output.json + workflowresultparser -workflowRunID ${{ github.run_id }} -githubToken ${{ github.token }} -githubRepo "${{ github.repository }}" -jobNameRegex "^vrfv2 compatibility with (.*?)$" -namedKey="vrfv2" -outputFile=output.json + workflowresultparser -workflowRunID ${{ github.run_id }} -githubToken ${{ github.token }} -githubRepo "${{ github.repository }}" -jobNameRegex "^vrfv2plus compatibility with (.*?)$" -namedKey="vrfv2plus" -outputFile=output.json + workflowresultparser -workflowRunID ${{ github.run_id }} -githubToken ${{ github.token }} -githubRepo "${{ github.repository }}" -jobNameRegex "^flux compatibility with (.*?)$" -namedKey="flux" -outputFile=output.json + workflowresultparser -workflowRunID ${{ github.run_id }} -githubToken ${{ github.token }} -githubRepo "${{ github.repository }}" -jobNameRegex "^runlog compatibility with (.*?)$" -namedKey="runlog" -outputFile=output.json + workflowresultparser -workflowRunID ${{ github.run_id }} -githubToken ${{ github.token }} -githubRepo "${{ github.repository }}" -jobNameRegex "^cron compatibility with (.*?)$" -namedKey="cron" -outputFile=output.json + + echo "base64_parsed_results=$(base64 -w 0 output.json)" >> $GITHUB_OUTPUT + + display-test-results: + name: Aggregated test results + if: always() && needs.*.result != 'skipped' && needs.*.result != 'cancelled' && needs.should-run.outputs.should_run == 'true' && needs.parse-test-results.result == 'success' + environment: integration + permissions: + checks: write + pull-requests: write + id-token: write + contents: read + runs-on: ubuntu-latest + needs: [start-slack-thread, should-run, select-versions, parse-test-results] + steps: + # asciitable is a tool that prints results in a nice ASCII table + - name: Set Up asciitable + shell: bash + run: | + go install github.com/smartcontractkit/chainlink-testing-framework/tools/asciitable@v1.0.2 + - name: Print aggregated test results + shell: bash + run: | + PATH=$PATH:$(go env GOPATH)/bin + export PATH + + raw_results="$(echo ${{ needs.parse-test-results.outputs.base64_parsed_results }} | base64 -d)" + echo $raw_results > input.json + asciitable --firstColumn "EVM Implementation" --secondColumn Result --jsonfile input.json --outputFile output.txt --section "automation" --namedKey "automation" + asciitable --firstColumn "EVM Implementation" --secondColumn Result --jsonfile input.json --outputFile output.txt --section "keeper" --namedKey "keeper" + asciitable --firstColumn "EVM Implementation" --secondColumn Result --jsonfile input.json --outputFile output.txt --section "log_poller" --namedKey "log_poller" + asciitable --firstColumn "EVM Implementation" --secondColumn Result --jsonfile input.json --outputFile output.txt --section "ocr" --namedKey "ocr" + asciitable --firstColumn "EVM Implementation" --secondColumn Result --jsonfile input.json --outputFile output.txt --section "ocr2" --namedKey "ocr2" + asciitable --firstColumn "EVM Implementation" --secondColumn Result --jsonfile input.json --outputFile output.txt --section "vrf" --namedKey "vrf" + asciitable --firstColumn "EVM Implementation" --secondColumn Result --jsonfile input.json --outputFile output.txt --section "vrfv2" --namedKey "vrfv2" + asciitable --firstColumn "EVM Implementation" --secondColumn Result --jsonfile input.json --outputFile output.txt --section "vrfv2plus" --namedKey "vrfv2plus" + asciitable --firstColumn "EVM Implementation" --secondColumn Result --jsonfile input.json --outputFile output.txt --section "flux" --namedKey "flux" + asciitable --firstColumn "EVM Implementation" --secondColumn Result --jsonfile input.json --outputFile output.txt --section "cron" --namedKey "cron" + asciitable --firstColumn "EVM Implementation" --secondColumn Result --jsonfile input.json --outputFile output.txt --section "runlog" --namedKey "runlog" + + echo + echo "AGGREGATED RESULTS" + cat output.txt + + echo "## Aggregated EVM Implementations compatibility results summary" >> $GITHUB_STEP_SUMMARY + echo '```' >> $GITHUB_STEP_SUMMARY + cat output.txt >> $GITHUB_STEP_SUMMARY + echo '```' >> $GITHUB_STEP_SUMMARY + post-test-results-to-slack: name: Post Test Results for ${{matrix.product}} - if: ${{ always() && needs.*.result != 'skipped' && needs.*.result != 'cancelled' }} + if: ${{ always() && needs.*.result != 'skipped' && needs.*.result != 'cancelled' && needs.should-run.outputs.should_run == 'true' }} environment: integration permissions: checks: write @@ -348,24 +758,38 @@ jobs: id-token: write contents: read runs-on: ubuntu-latest - needs: start-slack-thread + needs: [start-slack-thread, should-run, parse-test-results] strategy: fail-fast: false matrix: - product: [ocr, ocr2] + product: [automation, keeper, log_poller, ocr, ocr2, vrf, vrfv2, vrfv2plus, cron, flux, runlog] steps: - name: Checkout the repo uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2 with: - ref: ${{ github.event.pull_request.head.sha || github.event.merge_group.head_sha }} + ref: ${{ needs.select-versions.outputs.chainlink_version }} + - name: Get test results for ${{ matrix.product }} + id: get-product-results + shell: bash + run: | + raw_results="$(echo ${{ needs.parse-test-results.outputs.base64_parsed_results }} | base64 -d)" + product_result=$(echo "$raw_results" | jq -c "select(has(\"${{ matrix.product }}\")) | .${{ matrix.product }}[]") + if [ -n "$product_result" ]; then + base64_result=$(echo $product_result | base64 -w 0) + echo "base64_result=$base64_result" >> $GITHUB_OUTPUT + else + echo "No results found for ${{ matrix.product }}" + echo "base64_result=" >> $GITHUB_OUTPUT + fi - name: Post Test Results to Slack uses: ./.github/actions/notify-slack-jobs-result with: github_token: ${{ github.token }} github_repository: ${{ github.repository }} workflow_run_id: ${{ github.run_id }} - github_job_name_regex: ^Client Compatibility Test ${{ matrix.product }}-(?.*?)$ + github_job_name_regex: ^${{ matrix.product }} compatibility with (.*?)$ message_title: ${{ matrix.product }} slack_channel_id: ${{ secrets.QA_SLACK_CHANNEL }} slack_bot_token: ${{ secrets.QA_SLACK_API_KEY }} - slack_thread_ts: ${{ needs.start-slack-thread.outputs.thread_ts }} \ No newline at end of file + slack_thread_ts: ${{ needs.start-slack-thread.outputs.thread_ts }} + base64_parsed_results: ${{ steps.get-product-results.outputs.base64_result }} diff --git a/.github/workflows/evm-version-compatibility-tests.yml b/.github/workflows/evm-version-compatibility-tests.yml deleted file mode 100644 index 6705ec79e7d..00000000000 --- a/.github/workflows/evm-version-compatibility-tests.yml +++ /dev/null @@ -1,370 +0,0 @@ -name: EVM Node Version Compatibility Tests -on: - merge_group: - pull_request: - push: - tags: - - "*" - workflow_dispatch: - inputs: - base64_test_list: - description: Base64 encoded test list (same format as ./integration-tests/evm_node_compatibility_test_list.json) - required: false - type: string - -env: - CHAINLINK_IMAGE: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com/chainlink - INTERNAL_DOCKER_REPO: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com - MOD_CACHE_VERSION: 2 - -jobs: - # Check if go.mod has changed - check-dependency-bump: - runs-on: ubuntu-latest - outputs: - dependency_changed: ${{ steps.changes.outputs.dependency_changed }} - steps: - - name: Checkout code - uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2 - with: - fetch-depth: 0 - - name: Check for go.mod changes - id: changes - run: | - git fetch origin ${{ github.base_ref }} - # if no match is found then grep exits with code 1, but if there is a match it exits with code 0 - # this will return a match if there are any changes on that corresponding line, for example if spacing was changed - DEPENDENCY_CHANGED=$(git diff -U0 origin/${{ github.base_ref }}...HEAD -- go.mod | grep -q 'github.com/ethereum/go-ethereum'; echo $?) - PR_VERSION=$(grep 'github.com/ethereum/go-ethereum' go.mod | awk '{print $2}') - - # here 0 means a match was found, 1 means no match was found - if [ "$DEPENDENCY_CHANGED" -eq 0 ]; then - # Dependency was changed in the PR, now compare with the base branch - git fetch origin ${{ github.base_ref }} - BASE_VERSION=$(git show origin/${{ github.base_ref }}:go.mod | grep 'github.com/ethereum/go-ethereum' | awk '{print $2}') - - echo "Base branch version: $BASE_VERSION" - echo "PR branch version: $PR_VERSION" - - echo "Dependency version changed in the PR compared to the base branch." - echo "dependency_changed=true" >> $GITHUB_OUTPUT - else - echo "No changes to ethereum/go-ethereum dependency in the PR." - echo "PR branch version: $PR_VERSION" - echo "dependency_changed=false" >> $GITHUB_OUTPUT - fi - - # Check if images used as test dependencies exist in the ECR - check-ecr-images-exist: - if: needs.check-dependency-bump.outputs.dependency_changed == 'true' || github.event_name == 'workflow_dispatch' - needs: [check-dependency-bump] - environment: integration - permissions: - id-token: write - contents: read - name: Check images used as test dependencies exist in ECR - runs-on: ubuntu-latest - steps: - - name: Update internal ECR if the latest Ethereum client image does not exist - uses: smartcontractkit/chainlink-testing-framework/.github/actions/update-internal-mirrors@7eb04a030823b316d8dd5bb555f1e49593a503fc - with: - aws_region: ${{ secrets.QA_AWS_REGION }} - role_to_assume: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }} - aws_account_number: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }} - image_name: 'ethereum/client-go' - expression: '^(alltools-v|v)[0-9]\.[0-9]+\.[0-9]+$' - - # Build Test Dependencies - - build-chainlink: - if: needs.check-dependency-bump.outputs.dependency_changed == 'true' || github.event_name == 'workflow_dispatch' - needs: [check-dependency-bump, check-ecr-images-exist] - environment: integration - permissions: - id-token: write - contents: read - name: Build Chainlink Image - runs-on: ubuntu-latest - steps: - - name: Collect Metrics - id: collect-gha-metrics - uses: smartcontractkit/push-gha-metrics-action@d9da21a2747016b3e13de58c7d4115a3d5c97935 # v3.0.1 - with: - id: evm-build-chainlink - org-id: ${{ secrets.GRAFANA_INTERNAL_TENANT_ID }} - basic-auth: ${{ secrets.GRAFANA_INTERNAL_BASIC_AUTH }} - hostname: ${{ secrets.GRAFANA_INTERNAL_HOST }} - this-job-name: Build Chainlink Image - continue-on-error: true - - name: Checkout the repo - uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2 - with: - ref: ${{ github.event.pull_request.head.sha || github.event.merge_group.head_sha }} - - name: Build Chainlink Image - uses: ./.github/actions/build-chainlink-image - with: - tag_suffix: "" - dockerfile: core/chainlink.Dockerfile - git_commit_sha: ${{ github.sha }} - AWS_REGION: ${{ secrets.QA_AWS_REGION }} - AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }} - - build-tests: - if: needs.check-dependency-bump.outputs.dependency_changed == 'true' || github.event_name == 'workflow_dispatch' - needs: [check-dependency-bump, check-ecr-images-exist] - environment: integration - permissions: - id-token: write - contents: read - name: Build Tests Binary - runs-on: ubuntu-latest - steps: - - name: Collect Metrics - id: collect-gha-metrics - uses: smartcontractkit/push-gha-metrics-action@d9da21a2747016b3e13de58c7d4115a3d5c97935 # v3.0.1 - with: - id: evm-build-tests - org-id: ${{ secrets.GRAFANA_INTERNAL_TENANT_ID }} - basic-auth: ${{ secrets.GRAFANA_INTERNAL_BASIC_AUTH }} - hostname: ${{ secrets.GRAFANA_INTERNAL_HOST }} - this-job-name: Build Tests Binary - continue-on-error: true - - name: Checkout the repo - uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2 - with: - ref: ${{ github.event.pull_request.head.sha || github.event.merge_group.head_sha }} - - name: Build Tests - uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/build-tests@fc3e0df622521019f50d772726d6bf8dc919dd38 # v2.3.19 - with: - test_download_vendor_packages_command: cd ./integration-tests && go mod download - token: ${{ secrets.GITHUB_TOKEN }} - go_mod_path: ./integration-tests/go.mod - go_tags: embed - cache_key_id: core-e2e-${{ env.MOD_CACHE_VERSION }} - cache_restore_only: "true" - binary_name: tests - - build-test-matrix: - if: needs.check-dependency-bump.outputs.dependency_changed == 'true' || github.event_name == 'workflow_dispatch' - needs: [check-dependency-bump, check-ecr-images-exist] - runs-on: ubuntu-latest - name: Build Test Matrix - outputs: - matrix: ${{ env.JOB_MATRIX_JSON }} - steps: - - name: Checkout the repo - uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2 - - name: Setup environment variables - run: | - echo "BASE64_TEST_LIST=${{ github.event.inputs.base64_test_list }}" >> $GITHUB_ENV - - name: Decode Base64 Test List Input if Set - id: decode-base64-test-list - if: env.BASE64_TEST_LIST != '' - run: | - echo "Decoding base64 test list..." - DECODED_BASE64_TEST_LIST=$(echo $BASE64_TEST_LIST | base64 -d) - echo $DECODED_BASE64_TEST_LIST - cd ./integration-tests - echo $DECODED_BASE64_TEST_LIST >> ./evm_node_compatibility_test_list.json - - name: Override Test List If Present - if: env.BASE64_TEST_LIST == '' - id: build-test-matrix-list - run: | - cd ./integration-tests - cp ./smoke/evm_node_compatibility_test_list.json . - - name: Create Test Matrix - id: create-test-matrix-list - run: | - cd ./integration-tests - JOB_MATRIX_JSON=$(./scripts/buildEvmClientTestMatrixList.sh ./evm_node_compatibility_test_list.json ubuntu-latest) - echo "JOB_MATRIX_JSON=${JOB_MATRIX_JSON}" >> $GITHUB_ENV - echo $JOB_MATRIX_JSON | jq . - - # End Build Test Dependencies - - evm-node-compatiblity-matrix: - environment: integration - permissions: - checks: write - pull-requests: write - id-token: write - contents: read - needs: - [check-dependency-bump, build-chainlink, build-tests, build-test-matrix] - env: - SELECTED_NETWORKS: SIMULATED - CHAINLINK_COMMIT_SHA: ${{ github.sha }} - CHAINLINK_ENV_USER: ${{ github.actor }} - TEST_LOG_LEVEL: debug - strategy: - fail-fast: false - matrix: - evm_node: ${{fromJson(needs.build-test-matrix.outputs.matrix)}} - runs-on: ${{ matrix.evm_node.os }} - name: EVM node compatibility of ${{ matrix.evm_node.product }} with ${{ matrix.evm_node.docker_image }} - steps: - - name: Collect Metrics - if: needs.changes.outputs.src == 'true' || github.event_name == 'workflow_dispatch' - id: collect-gha-metrics - uses: smartcontractkit/push-gha-metrics-action@d9da21a2747016b3e13de58c7d4115a3d5c97935 # v3.0.1 - with: - id: evm-e2e-compatability-tests-${{ matrix.evm_node.name }} - basic-auth: ${{ secrets.GRAFANA_INTERNAL_BASIC_AUTH }} - hostname: ${{ secrets.GRAFANA_INTERNAL_HOST }} - org-id: ${{ secrets.GRAFANA_INTERNAL_TENANT_ID }} - this-job-name: EVM node compatibility ${{ matrix.evm_node.name }} ${{ matrix.evm_node.docker_image }} - test-results-file: '{"testType":"go","filePath":"/tmp/gotest.log"}' - continue-on-error: true - - name: Checkout the repo - uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2 - with: - ref: ${{ github.event.pull_request.head.sha || github.event.merge_group.head_sha }} - - name: Build Go Test Command - id: build-go-test-command - run: | - # if the matrix.evm_node.run is set, use it for a different command - if [ "${{ matrix.evm_node.run }}" != "" ]; then - echo "run_command=${{ matrix.evm_node.run }} ./smoke/${{ matrix.evm_node.product }}_test.go" >> "$GITHUB_OUTPUT" - else - echo "run_command=./smoke/${{ matrix.evm_node.product }}_test.go" >> "$GITHUB_OUTPUT" - fi - - name: Setup GAP for Grafana - uses: smartcontractkit/.github/actions/setup-gap@6c9d62fdad050cfb8b59376ded291f1350705944 # setup-gap@0.2.2 - with: - # aws inputs - aws-region: ${{ secrets.AWS_REGION }} - aws-role-arn: ${{ secrets.AWS_OIDC_IAM_ROLE_VALIDATION_PROD_ARN }} - api-gateway-host: ${{ secrets.AWS_API_GW_HOST_GRAFANA }} - # other inputs - duplicate-authorization-header: "true" - - name: Prepare Base64 TOML override - uses: ./.github/actions/setup-create-base64-config - with: - runId: ${{ github.run_id }} - testLogCollect: ${{ vars.TEST_LOG_COLLECT }} - selectedNetworks: ${{ env.SELECTED_NETWORKS }} - chainlinkImage: ${{ env.CHAINLINK_IMAGE }} - chainlinkVersion: ${{ github.sha }} - lokiEndpoint: ${{ secrets.LOKI_URL }} - lokiTenantId: ${{ vars.LOKI_TENANT_ID }} - lokiBasicAuth: ${{ secrets.LOKI_BASIC_AUTH }} - logstreamLogTargets: ${{ vars.LOGSTREAM_LOG_TARGETS }} - grafanaUrl: "http://localhost:8080/primary" - grafanaDashboardUrl: "/d/ddf75041-1e39-42af-aa46-361fe4c36e9e/ci-e2e-tests-logs" - grafanaBearerToken: ${{ secrets.GRAFANA_INTERNAL_URL_SHORTENER_TOKEN }} - ethExecutionClient: ${{ matrix.evm_node.eth_client }} - customEthClientDockerImage: ${{ matrix.evm_node.docker_image }} - - - name: Run Tests - uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/run-tests@af92c5fae8dcf1659201e907db82d221fc304b94 # v2.3.21 - with: - test_command_to_run: cd ./integration-tests && go test -timeout 45m -count=1 -json -test.parallel=2 ${{ steps.build-go-test-command.outputs.run_command }} 2>&1 | tee /tmp/gotest.log | gotestloghelper -ci -singlepackage -hidepassingtests=false -hidepassinglogs - test_download_vendor_packages_command: cd ./integration-tests && go mod download - cl_repo: ${{ env.CHAINLINK_IMAGE }} - cl_image_tag: ${{ github.sha }} - aws_registries: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }} - artifacts_location: ./integration-tests/smoke/logs/ - publish_check_name: ${{ matrix.evm_node.product }}-compatibility-${{ matrix.evm_node.eth_client }}-${{ matrix.evm_node.docker_image }} - token: ${{ secrets.GITHUB_TOKEN }} - go_mod_path: ./integration-tests/go.mod - cache_key_id: core-e2e-${{ env.MOD_CACHE_VERSION }} - cache_restore_only: "true" - QA_AWS_REGION: ${{ secrets.QA_AWS_REGION }} - QA_AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }} - QA_KUBECONFIG: "" - should_tidy: "false" - - name: Print failed test summary - if: always() - uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/show-test-summary@fc3e0df622521019f50d772726d6bf8dc919dd38 # v2.3.19 - - start-slack-thread: - name: Start Slack Thread - if: ${{ always() && needs.check-dependency-bump.outputs.dependency_changed == 'true' && needs.*.result != 'skipped' && needs.*.result != 'cancelled' }} - environment: integration - outputs: - thread_ts: ${{ steps.slack.outputs.thread_ts }} - permissions: - checks: write - pull-requests: write - id-token: write - contents: read - runs-on: ubuntu-latest - needs: [ evm-node-compatiblity-matrix] - steps: - - name: Debug Result - run: echo ${{ join(needs.*.result, ',') }} - - name: Main Slack Notification - uses: slackapi/slack-github-action@6c661ce58804a1a20f6dc5fbee7f0381b469e001 # v1.25.0 - id: slack - with: - channel-id: ${{ secrets.QA_SLACK_CHANNEL }} - payload: | - { - "attachments": [ - { - "color": "${{ contains(join(needs.*.result, ','), 'failure') && '#C62828' || '#2E7D32' }}", - "blocks": [ - { - "type": "header", - "text": { - "type": "plain_text", - "text": "EVM Node Compatability Test Results ${{ contains(join(needs.*.result, ','), 'failure') && ':x:' || ':white_check_mark:'}}", - "emoji": true - } - }, - { - "type": "section", - "text": { - "type": "mrkdwn", - "text": "${{ contains(join(needs.*.result, ','), 'failure') && 'Some tests failed, notifying <@U060CGGPY8H>' || 'All Good!' }}" - } - }, - { - "type": "divider" - }, - { - "type": "section", - "text": { - "type": "mrkdwn", - "text": "<${{ github.server_url }}/${{ github.repository }}/releases/tag/${{ github.ref_name }}|${{ github.ref_name }}> | <${{ github.server_url }}/${{ github.repository }}/commit/${{ github.sha }}|${{ github.sha }}> | <${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}|Run>" - } - } - ] - } - ] - } - env: - SLACK_BOT_TOKEN: ${{ secrets.QA_SLACK_API_KEY }} - - post-test-results-to-slack: - name: Post Test Results for ${{matrix.evm_node.eth_client}} to Slack - if: ${{ always() && needs.check-dependency-bump.outputs.dependency_changed == 'true' && needs.*.result != 'skipped' && needs.*.result != 'cancelled' }} - environment: integration - permissions: - checks: write - pull-requests: write - id-token: write - contents: read - runs-on: ubuntu-latest - needs: [start-slack-thread, build-test-matrix] - strategy: - fail-fast: false - matrix: - # this basically works as group by in SQL; we should update it when we update the test list JSON file - product: [automation,ocr,ocr2,vrf,vrfv2,vrfv2plus] - steps: - - name: Checkout the repo - uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2 - with: - ref: ${{ github.event.pull_request.head.sha || github.event.merge_group.head_sha }} - - name: Post Test Results to Slack - uses: ./.github/actions/notify-slack-jobs-result - with: - github_token: ${{ github.token }} - github_repository: ${{ github.repository }} - workflow_run_id: ${{ github.run_id }} - github_job_name_regex: ^EVM node compatibility of ${{ matrix.product }} with (?.*?)$ - message_title: ${{ matrix.product }} - slack_channel_id: ${{ secrets.QA_SLACK_CHANNEL }} - slack_bot_token: ${{ secrets.QA_SLACK_API_KEY }} - slack_thread_ts: ${{ needs.start-slack-thread.outputs.thread_ts }} diff --git a/.github/workflows/live-testnet-tests.yml b/.github/workflows/live-testnet-tests.yml index b9ecde6116d..6edbf57f253 100644 --- a/.github/workflows/live-testnet-tests.yml +++ b/.github/workflows/live-testnet-tests.yml @@ -213,7 +213,7 @@ jobs: github_token: ${{ github.token }} github_repository: ${{ github.repository }} workflow_run_id: ${{ github.run_id }} - github_job_name_regex: ^${{ matrix.network }} (?.*?) Tests$ + github_job_name_regex: ^${{ matrix.network }} (.*?) Tests$ message_title: ${{ matrix.network }} slack_channel_id: ${{ secrets.QA_SLACK_CHANNEL }} slack_bot_token: ${{ secrets.QA_SLACK_API_KEY }} diff --git a/integration-tests/docker/node_coverage_helper.go b/integration-tests/docker/node_coverage_helper.go index 52531bd35cf..c24804b4612 100644 --- a/integration-tests/docker/node_coverage_helper.go +++ b/integration-tests/docker/node_coverage_helper.go @@ -136,7 +136,7 @@ func (c *NodeCoverageHelper) copyCoverageFromNodes(ctx context.Context) error { errorsChan <- fmt.Errorf("failed to copy folder from container for node %d: %w", id, err) return } - finalDestPath = filepath.Join(finalDestPath, "go-coverage") // Assuming path structure /var/tmp/go-coverage/TestName/node_X/go-coverage + finalDestPath = filepath.Join(finalDestPath, "go-coverage") // Assuming path structure /var/tmp/go-coverage/TestRegex/node_X/go-coverage c.NodeCoverageDirs = append(c.NodeCoverageDirs, finalDestPath) }(node, i) } diff --git a/integration-tests/scripts/buildEvmClientTestMatrixList.sh b/integration-tests/scripts/buildEvmClientTestMatrixList.sh deleted file mode 100755 index 2f0e27b7fb8..00000000000 --- a/integration-tests/scripts/buildEvmClientTestMatrixList.sh +++ /dev/null @@ -1,64 +0,0 @@ -#!/usr/bin/env bash - -# requires a path to a json file with all the tests it should run -# requires a node label to be passed in, for example "ubuntu-latest" - -set -e - -# get this script's directory -SCRIPT_DIR=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" &>/dev/null && pwd) - -cd "$SCRIPT_DIR"/../ || exit 1 - -JSONFILE=$1 -NODE_LABEL=$2 - -COUNTER=1 - -# Build a JSON object in the format expected by our evm-version-compatibility-tests workflow matrix -matrix_output() { - local counter=$1 - local job_name=$2 - local test_name=$3 - local node_label=$4 - local eth_client=$5 - local docker_image=$6 - local product=$7 - local counter_out=$(printf "%02d\n" $counter) - echo -n "{\"name\": \"${job_name}-${counter_out}\", \"os\": \"${node_label}\", \"product\": \"${product}\", \"eth_client\": \"${eth_client}\", \"docker_image\": \"${docker_image}\", \"run\": \"-run '^${test_name}$'\"}" -} - -# Read the JSON file and loop through 'tests' and 'run' -jq -c '.tests[]' ${JSONFILE} | while read -r test; do - testName=$(echo ${test} | jq -r '.name') - label=$(echo ${test} | jq -r '.label // empty') - effective_node_label=${label:-$NODE_LABEL} - eth_client=$(echo ${test} | jq -r '.eth_client') - docker_image=$(echo ${test} | jq -r '.docker_image') - product=$(echo ${test} | jq -r '.product') - subTests=$(echo ${test} | jq -r '.run[]?.name // empty') - output="" - - if [ $COUNTER -ne 1 ]; then - echo -n "," - fi - - # Loop through subtests, if any, and print in the desired format - if [ -n "$subTests" ]; then - subTestString="" - subTestCounter=1 - for subTest in $subTests; do - if [ $subTestCounter -ne 1 ]; then - subTestString+="|" - fi - subTestString+="${testName}\/${subTest}" - ((subTestCounter++)) - done - testName="${subTestString}" - fi - matrix_output $COUNTER "emv-node-version-compatibility-test" "${testName}" ${effective_node_label} "${eth_client}" "${docker_image}" "${product}" - ((COUNTER++)) -done > "./tmpout.json" -OUTPUT=$(cat ./tmpout.json) -echo "[${OUTPUT}]" -rm ./tmpout.json \ No newline at end of file diff --git a/integration-tests/smoke/evm_node_compatibility_test_list.json b/integration-tests/smoke/evm_node_compatibility_test_list.json deleted file mode 100644 index 45b303a0a27..00000000000 --- a/integration-tests/smoke/evm_node_compatibility_test_list.json +++ /dev/null @@ -1,184 +0,0 @@ -{ - "tests": [ - { - "product": "ocr", - "name": "TestOCRBasic", - "eth_client": "geth", - "docker_image": "ethereum/client-go:latest_stable" - }, - { - "product": "ocr", - "name": "TestOCRBasic", - "eth_client": "geth", - "docker_image": "ethereum/client-go:v1.13.0" - }, - { - "product": "ocr", - "name": "TestOCRBasic", - "eth_client": "geth", - "docker_image": "ethereum/client-go:v1.12.0" - }, - { - "product": "ocr", - "name": "TestOCRBasic", - "eth_client": "geth", - "docker_image": "ethereum/client-go:v1.11.0" - }, - { - "product": "ocr", - "name": "TestOCRBasic", - "eth_client": "geth", - "docker_image": "ethereum/client-go:v1.10.0" - }, - { - "product": "ocr2", - "name": "TestOCRv2Request", - "eth_client": "geth", - "docker_image": "ethereum/client-go:latest_stable" - }, - { - "product": "ocr2", - "name": "TestOCRv2Request", - "eth_client": "geth", - "docker_image": "ethereum/client-go:v1.13.0" - }, - { - "product": "ocr2", - "name": "TestOCRv2Request", - "eth_client": "geth", - "docker_image": "ethereum/client-go:v1.12.0" - }, - { - "product": "ocr2", - "name": "TestOCRv2Request", - "eth_client": "geth", - "docker_image": "ethereum/client-go:v1.11.0" - }, - { - "product": "ocr2", - "name": "TestOCRv2Request", - "eth_client": "geth", - "docker_image": "ethereum/client-go:v1.10.0" - }, - { - "product": "vrf", - "name": "TestVRFBasic", - "eth_client": "geth", - "docker_image": "ethereum/client-go:latest_stable" - }, - { - "product": "vrf", - "name": "TestVRFBasic", - "eth_client": "geth", - "docker_image": "ethereum/client-go:v1.13.0" - }, - { - "product": "vrf", - "name": "TestVRFBasic", - "eth_client": "geth", - "docker_image": "ethereum/client-go:v1.12.0" - }, - { - "product": "vrf", - "name": "TestVRFBasic", - "eth_client": "geth", - "docker_image": "ethereum/client-go:v1.11.0" - }, - { - "product": "vrf", - "name": "TestVRFBasic", - "eth_client": "geth", - "docker_image": "ethereum/client-go:v1.10.0" - }, - { - "product": "vrfv2", - "name": "TestVRFv2Basic/Request Randomness", - "eth_client": "geth", - "docker_image": "ethereum/client-go:latest_stable" - }, - { - "product": "vrfv2", - "name": "TestVRFv2Basic/Request Randomness", - "eth_client": "geth", - "docker_image": "ethereum/client-go:v1.13.0" - }, - { - "product": "vrfv2", - "name": "TestVRFv2Basic/Request Randomness", - "eth_client": "geth", - "docker_image": "ethereum/client-go:v1.12.0" - }, - { - "product": "vrfv2", - "name": "TestVRFv2Basic/Request Randomness", - "eth_client": "geth", - "docker_image": "ethereum/client-go:v1.11.0" - }, - { - "product": "vrfv2", - "name": "TestVRFv2Basic/Request Randomness", - "eth_client": "geth", - "docker_image": "ethereum/client-go:v1.10.0" - }, - { - "product": "vrfv2plus", - "name": "TestVRFv2Plus/Link Billing", - "eth_client": "geth", - "docker_image": "ethereum/client-go:latest_stable" - }, - { - "product": "vrfv2plus", - "name": "TestVRFv2Plus/Link Billing", - "eth_client": "geth", - "docker_image": "ethereum/client-go:v1.13.0" - }, - { - "product": "vrfv2plus", - "name": "TestVRFv2Plus/Link Billing", - "eth_client": "geth", - "docker_image": "ethereum/client-go:v1.12.0" - }, - { - "product": "vrfv2plus", - "name": "TestVRFv2Plus/Link Billing", - "eth_client": "geth", - "docker_image": "ethereum/client-go:v1.11.0" - }, - { - "product": "vrfv2plus", - "name": "TestVRFv2Plus/Link Billing", - "eth_client": "geth", - "docker_image": "ethereum/client-go:v1.10.0" - }, - { - "product": "automation", - "name": "TestSetUpkeepTriggerConfig", - "eth_client": "geth", - "docker_image": "ethereum/client-go:latest_stable" - }, - { - "product": "automation", - "name": "TestSetUpkeepTriggerConfig", - "eth_client": "geth", - "docker_image": "ethereum/client-go:v1.13.0" - }, - { - "product": "automation", - "name": "TestSetUpkeepTriggerConfig", - "eth_client": "geth", - "docker_image": "ethereum/client-go:v1.12.0" - }, - { - "product": "automation", - "name": "TestSetUpkeepTriggerConfig", - "eth_client": "geth", - "docker_image": "ethereum/client-go:v1.11.0" - }, - { - "product": "automation", - "name": "TestSetUpkeepTriggerConfig", - "eth_client": "geth", - "docker_image": "ethereum/client-go:v1.10.0" - } - ] -} \ No newline at end of file