diff --git a/.github/actions/setup-create-base64-config-ccip/action.yml b/.github/actions/setup-create-base64-config-ccip/action.yml index 88d9fe8078c..cb20c886e38 100644 --- a/.github/actions/setup-create-base64-config-ccip/action.yml +++ b/.github/actions/setup-create-base64-config-ccip/action.yml @@ -11,62 +11,39 @@ inputs: default: "false" selectedNetworks: description: The networks to run tests against - chainlinkImage: - description: The chainlink image to use - default: "public.ecr.aws/chainlink/chainlink" chainlinkVersion: description: The git commit sha to use for the image tag - upgradeImage: - description: The chainlink image to upgrade to - default: "" upgradeVersion: - description: The git commit sha to use for the image tag - lokiEndpoint: - description: Loki push endpoint - lokiTenantId: - description: Loki tenant id - lokiBasicAuth: - description: Loki basic auth + description: The git commit sha to use for the image tag logstreamLogTargets: description: Where to send logs (e.g. file, loki) - grafanaUrl: - description: Grafana URL - grafanaDashboardUrl: - description: Grafana dashboard URL - grafanaBearerToken: - description: Grafana bearer token customEvmNodes: description: Custom EVM nodes to use in key=value format, where key is chain id and value is docker image to use. If they are provided the number of networksSelected must be equal to the number of customEvmNodes evmNodeLogLevel: description: Log level for the custom EVM nodes default: "info" +outputs: + base64_config: + description: The base64-encoded config + value: ${{ steps.base64_config_override.outputs.base64_config }} runs: using: composite steps: - name: Prepare Base64 TOML override shell: bash - id: base64-config-override + id: base64_config_override env: RUN_ID: ${{ inputs.runId }} SELECTED_NETWORKS: ${{ inputs.selectedNetworks }} EXISTING_NAMESPACE: ${{ inputs.existingNamespace }} TEST_LOG_COLLECT: ${{ inputs.testLogCollect }} - CHAINLINK_IMAGE: ${{ inputs.chainlinkImage }} CHAINLINK_VERSION: ${{ inputs.chainlinkVersion }} - UPGRADE_IMAGE: ${{ inputs.upgradeImage }} UPGRADE_VERSION: ${{ inputs.upgradeVersion }} - LOKI_ENDPOINT: ${{ inputs.lokiEndpoint }} - LOKI_TENANT_ID: ${{ inputs.lokiTenantId }} - LOKI_BASIC_AUTH: ${{ inputs.lokiBasicAuth }} LOGSTREAM_LOG_TARGETS: ${{ inputs.logstreamLogTargets }} - GRAFANA_URL: ${{ inputs.grafanaUrl }} - GRAFANA_DASHBOARD_URL: ${{ inputs.grafanaDashboardUrl }} - GRAFANA_BEARER_TOKEN: ${{ inputs.grafanaBearerToken }} CUSTOM_EVM_NODES: ${{ inputs.customEvmNodes }} EVM_NODE_LOG_LEVEL: ${{ inputs.evmNodeLogLevel }} run: | - echo ::add-mask::$CHAINLINK_IMAGE function convert_to_toml_array() { local IFS=',' local input_array=($1) @@ -133,11 +110,6 @@ runs: fi fi - grafana_bearer_token="" - if [ -n "$GRAFANA_BEARER_TOKEN" ]; then - grafana_bearer_token="bearer_token_secret=\"$GRAFANA_BEARER_TOKEN\"" - fi - cat << EOF > config.toml [CCIP] [CCIP.Env] @@ -147,13 +119,8 @@ runs: [CCIP.Env.NewCLCluster] [CCIP.Env.NewCLCluster.Common] [CCIP.Env.NewCLCluster.Common.ChainlinkImage] - image="$CHAINLINK_IMAGE" version="$CHAINLINK_VERSION" - [CCIP.Env.NewCLCluster.Common.ChainlinkUpgradeImage] - image="$UPGRADE_IMAGE" - version="$UPGRADE_VERSION" - $custom_nodes_toml [CCIP.Env.Logging] @@ -163,16 +130,6 @@ runs: [CCIP.Env.Logging.LogStream] log_targets=$log_targets - [CCIP.Env.Logging.Loki] - tenant_id="$LOKI_TENANT_ID" - endpoint="$LOKI_ENDPOINT" - basic_auth_secret="$LOKI_BASIC_AUTH" - - [CCIP.Env.Logging.Grafana] - base_url="$GRAFANA_URL" - dashboard_url="$GRAFANA_DASHBOARD_URL" - $grafana_bearer_token - [CCIP.Groups.load] TestRunName = '$EXISTING_NAMESPACE' @@ -181,7 +138,14 @@ runs: EOF - BASE64_CCIP_SECRETS_CONFIG=$(cat config.toml | base64 -w 0) - echo ::add-mask::$BASE64_CCIP_SECRETS_CONFIG - echo "BASE64_CCIP_SECRETS_CONFIG=$BASE64_CCIP_SECRETS_CONFIG" >> $GITHUB_ENV - echo "TEST_BASE64_CCIP_SECRETS_CONFIG=$BASE64_CCIP_SECRETS_CONFIG" >> $GITHUB_ENV + # Check if UPGRADE_VERSION is not empty and append to config.toml + if [ -n "$UPGRADE_VERSION" ]; then + cat << EOF >> config.toml + [CCIP.Env.NewCLCluster.Common.ChainlinkUpgradeImage] + version="$UPGRADE_VERSION" + EOF + fi + + BASE64_CONFIG=$(cat config.toml | base64 -w 0) + echo ::add-mask::$BASE64_CONFIG + echo "base64_config=$BASE64_CONFIG" >> $GITHUB_OUTPUT \ No newline at end of file diff --git a/.github/workflows/ccip-chaos-tests.yml b/.github/workflows/ccip-chaos-tests.yml index 493322ae420..03f809b44c9 100644 --- a/.github/workflows/ccip-chaos-tests.yml +++ b/.github/workflows/ccip-chaos-tests.yml @@ -6,8 +6,6 @@ on: branches: [ develop ] workflow_dispatch: - - # Only run 1 of this workflow at a time per PR concurrency: group: chaos-ccip-tests-chainlink-${{ github.ref }} @@ -16,7 +14,7 @@ concurrency: env: # TODO: TT-1470 - Update image names as we solidify new realease strategy CL_ECR: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com/chainlink - ENV_JOB_IMAGE: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com/chainlink-ccip-tests:${{ github.sha }} + ENV_JOB_IMAGE: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com/chainlink-tests:${{ github.sha }} MOD_CACHE_VERSION: 1 jobs: @@ -124,23 +122,20 @@ jobs: uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2 - name: Prepare Base64 TOML override for CCIP secrets uses: ./.github/actions/setup-create-base64-config-ccip + id: setup_create_base64_config_ccip with: runId: ${{ github.run_id }} testLogCollect: ${{ vars.TEST_LOG_COLLECT }} - chainlinkImage: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com/chainlink chainlinkVersion: ${{ github.sha }} - lokiEndpoint: ${{ secrets.LOKI_URL }} - lokiTenantId: ${{ vars.LOKI_TENANT_ID }} logstreamLogTargets: ${{ vars.LOGSTREAM_LOG_TARGETS }} - grafanaUrl: ${{ vars.GRAFANA_URL }} - grafanaDashboardUrl: "/d/ddf75041-1e39-42af-aa46-361fe4c36e9e/ci-e2e-tests-logs" - name: Run Chaos Tests - uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/run-tests@b49a9d04744b0237908831730f8553f26d73a94b # v2.3.17 + uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/run-tests@d38226be720c5ccc1ff4d3cee40608ebf264cd59 # v2.3.26 + env: + BASE64_CCIP_CONFIG_OVERRIDE: ${{ steps.setup_create_base64_config_ccip.outputs.base64_config }} + TEST_BASE64_CCIP_CONFIG_OVERRIDE: ${{ steps.setup_create_base64_config_ccip.outputs.base64_config }} with: test_command_to_run: cd ./integration-tests && go test -timeout 1h -count=1 -json -test.parallel 11 -run 'TestChaosCCIP' ./chaos 2>&1 | tee /tmp/gotest.log | gotestloghelper -ci test_download_vendor_packages_command: make gomod - cl_repo: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com/chainlink - cl_image_tag: ${{ github.sha }} artifacts_location: ./integration-tests/chaos/logs publish_check_name: CCIP Chaos Test Results publish_report_paths: ./tests-chaos-report.xml @@ -154,6 +149,13 @@ jobs: aws_registries: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }} cache_key_id: ccip-load-${{ env.MOD_CACHE_VERSION }} cache_restore_only: "true" + DEFAULT_LOKI_TENANT_ID: ${{ vars.LOKI_TENANT_ID }} + DEFAULT_LOKI_ENDPOINT: ${{ secrets.LOKI_URL }} + DEFAULT_LOKI_BASIC_AUTH: ${{ secrets.LOKI_BASIC_AUTH }} + DEFAULT_CHAINLINK_IMAGE: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com/chainlink + DEFAULT_GRAFANA_BASE_URL: ${{ vars.GRAFANA_URL }} + DEFAULT_GRAFANA_DASHBOARD_URL: "/d/ddf75041-1e39-42af-aa46-361fe4c36e9e/ci-e2e-tests-logs" + ## Notify in slack if the job fails - name: Notify Slack if: failure() && github.event_name != 'workflow_dispatch' @@ -206,23 +208,20 @@ jobs: uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2 - name: Prepare Base64 TOML override for CCIP secrests uses: ./.github/actions/setup-create-base64-config-ccip + id: setup_create_base64_config_ccip with: runId: ${{ github.run_id }} testLogCollect: ${{ vars.TEST_LOG_COLLECT }} - chainlinkImage: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com/chainlink chainlinkVersion: ${{ github.sha }} - lokiEndpoint: ${{ secrets.LOKI_URL }} - lokiTenantId: ${{ vars.LOKI_TENANT_ID }} logstreamLogTargets: ${{ vars.LOGSTREAM_LOG_TARGETS }} - grafanaUrl: ${{ vars.GRAFANA_URL }} - grafanaDashboardUrl: "/d/6vjVx-1V8/ccip-long-running-tests" - name: Run Load With Chaos Tests - uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/run-tests@b49a9d04744b0237908831730f8553f26d73a94b # v2.3.17 + uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/run-tests@d38226be720c5ccc1ff4d3cee40608ebf264cd59 # v2.3.26 + env: + BASE64_CCIP_CONFIG_OVERRIDE: ${{ steps.setup_create_base64_config_ccip.outputs.base64_config }} + TEST_BASE64_CCIP_CONFIG_OVERRIDE: ${{ steps.setup_create_base64_config_ccip.outputs.base64_config }} with: test_command_to_run: cd ./integration-tests/ccip-tests && go test -timeout 2h -count=1 -json -test.parallel 4 -run '^TestLoadCCIPStableWithPodChaosDiffCommitAndExec' ./load 2>&1 | tee /tmp/gotest.log | gotestfmt test_download_vendor_packages_command: make gomod - cl_repo: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com/chainlink - cl_image_tag: ${{ github.sha }} artifacts_location: ./integration-tests/load/logs publish_check_name: CCIP Chaos With Load Test Results publish_report_paths: ./tests-chaos-with-load-report.xml @@ -236,6 +235,12 @@ jobs: aws_registries: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }} cache_key_id: ccip-load-${{ env.MOD_CACHE_VERSION }} cache_restore_only: "true" + DEFAULT_LOKI_TENANT_ID: ${{ vars.LOKI_TENANT_ID }} + DEFAULT_LOKI_ENDPOINT: ${{ secrets.LOKI_URL }} + DEFAULT_LOKI_BASIC_AUTH: ${{ secrets.LOKI_BASIC_AUTH }} + DEFAULT_CHAINLINK_IMAGE: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com/chainlink + DEFAULT_GRAFANA_BASE_URL: ${{ vars.GRAFANA_URL }} + DEFAULT_GRAFANA_DASHBOARD_URL: "/d/6vjVx-1V8/ccip-long-running-tests" ## Notify in slack if the job fails - name: Notify Slack if: failure() && github.event_name != 'workflow_dispatch' @@ -250,4 +255,4 @@ jobs: if: always() uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/cleanup@b49a9d04744b0237908831730f8553f26d73a94b # v2.3.17 with: - triggered_by: ${{ env.TEST_TRIGGERED_BY }} + triggered_by: ${{ env.TEST_TRIGGERED_BY }} \ No newline at end of file diff --git a/.github/workflows/ccip-client-compatibility-tests.yml b/.github/workflows/ccip-client-compatibility-tests.yml deleted file mode 100644 index ff0e4be25c6..00000000000 --- a/.github/workflows/ccip-client-compatibility-tests.yml +++ /dev/null @@ -1,743 +0,0 @@ -name: CCIP Client Compatibility Tests -on: - schedule: - - cron: "30 5 * * TUE,FRI" # Run every Tuesday and Friday at midnight + 30min EST - push: - tags: - - "*" - merge_group: - pull_request: - workflow_dispatch: - inputs: - chainlinkVersion: - description: commit SHA or tag of the Chainlink version to test - required: true - type: string - evmImplementations: - description: comma separated list of EVM implementations to test (ignored if base64TestList is used) - required: true - type: string - default: "geth,besu,nethermind,erigon" - latestVersionsNumber: - description: how many of latest images of EVM implementations to test with (ignored if base64TestList is used) - required: true - type: number - default: 3 - base64TestList: - description: base64 encoded list of tests to run (same as base64-ed output of testlistgenerator tool) - required: false - type: string - -env: - # TODO: TT-1470 - Update image names as we solidify new realease strategy - CHAINLINK_IMAGE: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com/ccip - INTERNAL_DOCKER_REPO: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com - MOD_CACHE_VERSION: 2 - -jobs: - # Build Test Dependencies - - check-dependency-bump: - name: Check for go-ethereum dependency bump - if: github.event_name == 'pull_request' || github.event_name == 'merge_queue' - runs-on: ubuntu-latest - outputs: - dependency_changed: ${{ steps.changes.outputs.dependency_changed }} - steps: - - name: Checkout code - uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2 - with: - repository: smartcontractkit/chainlink - fetch-depth: 0 - - name: Check for go.mod changes - id: changes - run: | - git fetch origin ${{ github.base_ref }} - # if no match is found then grep exits with code 1, but if there is a match it exits with code 0 - # this will return a match if there are any changes on that corresponding line, for example if spacing was changed - DEPENDENCY_CHANGED=$(git diff -U0 origin/${{ github.base_ref }}...HEAD -- go.mod | grep -q 'github.com/ethereum/go-ethereum'; echo $?) - PR_VERSION=$(grep 'github.com/ethereum/go-ethereum' go.mod | awk '{print $2}') - - # here 0 means a match was found, 1 means no match was found - if [ "$DEPENDENCY_CHANGED" -eq 0 ]; then - # Dependency was changed in the PR, now compare with the base branch - git fetch origin ${{ github.base_ref }} - BASE_VERSION=$(git show origin/${{ github.base_ref }}:go.mod | grep 'github.com/ethereum/go-ethereum' | awk '{print $2}') - - echo "Base branch version: $BASE_VERSION" - echo "PR branch version: $PR_VERSION" - - echo "Dependency version changed in the PR compared to the base branch." - echo "dependency_changed=true" >> $GITHUB_OUTPUT - else - echo "No changes to ethereum/go-ethereum dependency in the PR." - echo "PR branch version: $PR_VERSION" - echo "dependency_changed=false" >> $GITHUB_OUTPUT - fi - - should-run: - if: always() - name: Check if the job should run - needs: check-dependency-bump - runs-on: ubuntu-latest - outputs: - should_run: ${{ steps.should-run.outputs.should_run }} - eth_implementations : ${{ steps.should-run.outputs.eth_implementations }} - env: - GITHUB_REF_TYPE: ${{ github.ref_type }} - steps: - - name: Check if the job should run - id: should-run - run: | - if [ "${{ needs.check-dependency-bump.outputs.dependency_changed }}" == "true" ]; then - echo "Will run tests, because go-ethereum dependency was bumped" - echo "should_run=true" >> $GITHUB_OUTPUT - elif [ "$GITHUB_EVENT_NAME" = "schedule" ]; then - echo "Will run tests, because trigger event was $GITHUB_EVENT_NAME" - echo "should_run=true" >> $GITHUB_OUTPUT - elif [ "$GITHUB_EVENT_NAME" = "workflow_dispatch" ]; then - echo "Will run tests, because trigger event was $GITHUB_EVENT_NAME" - echo "should_run=true" >> $GITHUB_OUTPUT - elif [ "$GITHUB_REF_TYPE" = "tag" ]; then - echo "Will run tests, because new tag was created" - echo "should_run=true" >> $GITHUB_OUTPUT - else - echo "Will not run tests" - echo "should_run=false" >> $GITHUB_OUTPUT - fi - - select-versions: - if: always() && needs.should-run.outputs.should_run == 'true' - name: Select Versions - needs: should-run - runs-on: ubuntu-latest - env: - RELEASED_DAYS_AGO: 4 - GITHUB_REF_TYPE: ${{ github.ref_type }} - outputs: - evm_implementations : ${{ steps.select-implementations.outputs.evm_implementations }} - chainlink_version: ${{ steps.select-chainlink-version.outputs.chainlink_version }} - latest_image_count: ${{ steps.get-image-count.outputs.image_count }} - steps: - # ghlatestreleasechecker is a tool to check if new release is available for a given repo - - name: Set Up ghlatestreleasechecker - shell: bash - run: | - go install github.com/smartcontractkit/chainlink-testing-framework/tools/ghlatestreleasechecker@v1.0.0 - - name: Select EVM implementations to test - id: select-implementations - run: | - PATH=$PATH:$(go env GOPATH)/bin - export PATH - - if [ "$GITHUB_EVENT_NAME" = "schedule" ]; then - echo "Checking for new releases" - implementations_arr=() - new_geth=$(ghlatestreleasechecker "ethereum/go-ethereum" $RELEASED_DAYS_AGO) - if [ "$new_geth" != "none" ]; then - echo "New geth release found: $new_geth" - implementations_arr+=("geth") - fi - new_besu=$(ghlatestreleasechecker "hyperledger/besu" $RELEASED_DAYS_AGO) - if [ "new_besu" != "none" ]; then - echo "New besu release found: $new_besu" - implementations_arr+=("besu") - fi - new_erigon=$(ghlatestreleasechecker "ledgerwatch/erigon" $RELEASED_DAYS_AGO) - if [ "new_erigon" != "none" ]; then - echo "New erigon release found: $new_erigon" - implementations_arr+=("erigon") - fi - new_nethermind=$(ghlatestreleasechecker "nethermindEth/nethermind" $RELEASED_DAYS_AGO) - if [ "new_nethermind" != "none" ]; then - echo "New nethermind release found: $new_nethermind" - implementations_arr+=("nethermind") - fi - IFS=',' - eth_implementations="${implementations_arr[*]}" - echo "Found new releases for: $eth_implementations" - echo "evm_implementations=$eth_implementations" >> $GITHUB_OUTPUT - elif [ "$GITHUB_EVENT_NAME" = "workflow_dispatch" ]; then - if [ -n "${{ github.event.inputs.base64TestList }}" ]; then - echo "Base64-ed Test Input provided, ignoring EVM implementations" - else - echo "Will test following EVM implementations: ${{ github.event.inputs.evmImplementations }}" - echo "evm_implementations=${{ github.event.inputs.evmImplementations }}" >> $GITHUB_OUTPUT - fi - else - echo "Will test all EVM implementations" - echo "evm_implementations=geth,besu,nethermind,erigon" >> $GITHUB_OUTPUT - fi - - name: Select Chainlink CCIP version - id: select-chainlink-version - run: | - PATH=$PATH:$(go env GOPATH)/bin - export PATH - - if [ "$GITHUB_EVENT_NAME" = "schedule" ]; then - echo "Fetching latest Chainlink CCIP stable version" - implementations_arr=() - # we use 100 days since we really want the latest one, and it's highly improbable there won't be a release in last 100 days - chainlink_version=$(ghlatestreleasechecker "smartcontractkit/ccip" 100) - echo "chainlink_version=$chainlink_version" >> $GITHUB_OUTPUT - elif [ "$GITHUB_EVENT_NAME" = "workflow_dispatch" ]; then - echo "Fetching Chainlink version from input" - if [ -n "${{ github.event.inputs.chainlinkVersion }}" ]; then - echo "Chainlink version provided in input" - chainlink_version="${{ github.event.inputs.chainlinkVersion }}" - else - echo "Chainlink version not provided in input. Using latest commit SHA." - chainlink_version=${{ github.sha }} - fi - echo "chainlink_version=$chainlink_version" >> $GITHUB_OUTPUT - elif [ "$GITHUB_EVENT_NAME" = "pull_request" ]; then - echo "Fetching Chainlink version from PR's head commit" - chainlink_version="${{ github.event.pull_request.head.sha }}" - echo "chainlink_version=$chainlink_version" >> $GITHUB_OUTPUT - elif [ "$GITHUB_EVENT_NAME" = "merge_queue" ]; then - echo "Fetching Chainlink version from merge queue's head commit" - chainlink_version="${{ github.event.merge_group.head_sha }}" - echo "chainlink_version=$chainlink_version" >> $GITHUB_OUTPUT - elif [ "$GITHUB_REF_TYPE" = "tag" ]; then - echo "Fetching Chainlink version from tag" - chainlink_version="${{ github.ref_name }}" - echo "chainlink_version=$chainlink_version" >> $GITHUB_OUTPUT - else - echo "Unsupported trigger event. It's probably an issue with the pipeline definition. Please reach out to the Test Tooling team." - exit 1 - fi - echo "Will use following Chainlink version: $chainlink_version" - - name: Get image count - id: get-image-count - run: | - if [ "$GITHUB_EVENT_NAME" = "workflow_dispatch" ]; then - echo "Fetching latest image count from input" - if [ -n "${{ github.event.inputs.base64TestList }}" ]; then - echo "Base64-ed Test Input provided, ignoring latest image count" - else - image_count="${{ github.event.inputs.latestVersionsNumber }}" - echo "image_count=$image_count" >> $GITHUB_OUTPUT - fi - else - echo "Fetching default latest image count" - image_count=3 - echo "image_count=$image_count" >> $GITHUB_OUTPUT - fi - echo "Will use following latest image count: $image_count" - - check-ecr-images-exist: - name: Check images used as test dependencies exist in ECR - if: always() && needs.should-run.outputs.should_run == 'true' - environment: integration - permissions: - id-token: write - contents: read - needs: [should-run] - runs-on: ubuntu-latest - strategy: - fail-fast: false - matrix: - mirror: - - name: ethereum/client-go - expression: '^(alltools-v|v)[0-9]\.[0-9]+\.[0-9]+$' - - name: hyperledger/besu - expression: '^[0-9]+\.[0-9]+(\.[0-9]+)?$' - page_size: 300 - - name: thorax/erigon - expression: '^v[0-9]+\.[0-9]+\.[0-9]+$' - - name: nethermind/nethermind - expression: '^[0-9]+\.[0-9]+\.[0-9]+$' - - name: tofelb/ethereum-genesis-generator - expression: '^[0-9]+\.[0-9]+\.[0-9]+(\-slots\-per\-epoch)?' - steps: - - name: Update internal ECR if the latest Ethereum client image does not exist - uses: smartcontractkit/chainlink-testing-framework/.github/actions/update-internal-mirrors@5eea86ee4f7742b4e944561a570a6b268e712d9e # v1.30.3 - with: - aws_region: ${{ secrets.QA_AWS_REGION }} - role_to_assume: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }} - aws_account_number: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }} - image_name: ${{matrix.mirror.name}} - expression: ${{matrix.mirror.expression}} - page_size: ${{matrix.mirror.page_size}} - - build-chainlink: - if: always() && needs.should-run.outputs.should_run == 'true' - environment: integration - permissions: - id-token: write - contents: read - name: Build Chainlink Image - needs: [should-run, select-versions] - runs-on: ubuntu-latest - steps: - - name: Collect Metrics - id: collect-gha-metrics - uses: smartcontractkit/push-gha-metrics-action@dea9b546553cb4ca936607c2267a09c004e4ab3f # v3.0.0 - with: - id: client-compatablility-build-chainlink - org-id: ${{ secrets.GRAFANA_INTERNAL_TENANT_ID }} - basic-auth: ${{ secrets.GRAFANA_INTERNAL_BASIC_AUTH }} - hostname: ${{ secrets.GRAFANA_INTERNAL_HOST }} - this-job-name: Build Chainlink Image - continue-on-error: true - - name: Checkout the repo - uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2 - with: - repository: smartcontractkit/chainlink - ref: ${{ needs.select-versions.outputs.chainlink_version }} - - name: Build Chainlink Image - uses: ./.github/actions/build-chainlink-image - with: - tag_suffix: "" - dockerfile: core/chainlink.Dockerfile - git_commit_sha: ${{ needs.select-versions.outputs.chainlink_version }} - check_image_exists: 'true' - AWS_REGION: ${{ secrets.QA_AWS_REGION }} - AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }} - - get-latest-available-images: - name: Get Latest EVM Implementation's Images - if: always() && needs.should-run.outputs.should_run == 'true' - environment: integration - runs-on: ubuntu-latest - needs: [check-ecr-images-exist, should-run, select-versions] - permissions: - id-token: write - contents: read - env: - LATEST_IMAGE_COUNT: ${{ needs.select-versions.outputs.latest_image_count }} - outputs: - geth_images: ${{ env.GETH_IMAGES }} - nethermind_images: ${{ env.NETHERMIND_IMAGES }} - besu_images: ${{ env.BESU_IMAGES }} - erigon_images: ${{ env.ERIGON_IMAGES }} - steps: - # Setup AWS creds - - name: Configure AWS Credentials - uses: aws-actions/configure-aws-credentials@e3dd6a429d7300a6a4c196c26e071d42e0343502 # v4.0.2 - with: - aws-region: ${{ secrets.QA_AWS_REGION }} - role-to-assume: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }} - role-duration-seconds: 3600 - # Login to ECR - - name: Login to Amazon ECR - id: login-ecr - uses: aws-actions/amazon-ecr-login@062b18b96a7aff071d4dc91bc00c4c1a7945b076 # v2.0.1 - with: - mask-password: "true" - env: - AWS_REGION: ${{ secrets.QA_AWS_REGION }} - # ecrimagefetcher is a tool to get latest images from ECR - - name: Set Up ecrimagefetcher - shell: bash - run: | - go install github.com/smartcontractkit/chainlink-testing-framework/tools/ecrimagefetcher@v1.0.1 - - name: Get latest docker images from ECR - if: ${{ github.event.inputs.base64TestList == '' }} - env: - AWS_REGION: ${{ secrets.QA_AWS_REGION }} - ETH_IMPLEMENTATIONS: ${{ needs.select-versions.outputs.evm_implementations }} - run: | - PATH=$PATH:$(go env GOPATH)/bin - export PATH - if [[ "$ETH_IMPLEMENTATIONS" == *"geth"* ]]; then - geth_images=$(ecrimagefetcher 'ethereum/client-go' '^v[0-9]+\.[0-9]+\.[0-9]+$' ${{ env.LATEST_IMAGE_COUNT }}) - echo "GETH_IMAGES=$geth_images" >> $GITHUB_ENV - echo "Geth latest images: $geth_images" - fi - - if [[ "$ETH_IMPLEMENTATIONS" == *"nethermind"* ]]; then - nethermind_images=$(ecrimagefetcher 'nethermind/nethermind' '^[0-9]+\.[0-9]+\.[0-9]+$' ${{ env.LATEST_IMAGE_COUNT }}) - echo "NETHERMIND_IMAGES=$nethermind_images" >> $GITHUB_ENV - echo "Nethermind latest images: $nethermind_images" - fi - - if [[ "$ETH_IMPLEMENTATIONS" == *"besu"* ]]; then - # 24.3.3 is ignored as it doesn't support data & input fields in eth_call - besu_images=$(ecrimagefetcher 'hyperledger/besu' '^[0-9]+\.[0-9]+(\.[0-9]+)?$' ${{ env.LATEST_IMAGE_COUNT }} ">=24.5.1") - echo "BESU_IMAGES=$besu_images" >> $GITHUB_ENV - echo "Besu latest images: $besu_images" - fi - - if [[ "$ETH_IMPLEMENTATIONS" == *"erigon"* ]]; then - # 2.60.0 and 2.60.1 are ignored as they stopped working with CL node - erigon_images=$(ecrimagefetcher 'thorax/erigon' '^v[0-9]+\.[0-9]+\.[0-9]+$' ${{ env.LATEST_IMAGE_COUNT }} "> $GITHUB_ENV - echo "Erigon latest images: $erigon_images" - fi - - # End Build Test Dependencies - - prepare-compatibility-matrix: - name: Prepare Compatibility Matrix - if: always() && needs.should-run.outputs.should_run == 'true' - environment: integration - permissions: - checks: write - pull-requests: write - id-token: write - contents: read - needs: [get-latest-available-images,should-run,select-versions] - runs-on: ubuntu-latest - env: - ETH_IMPLEMENTATIONS: ${{ needs.select-versions.outputs.evm_implementations }} - BASE64_TEST_LIST: ${{ github.event.inputs.base64TestList }} - outputs: - matrix: ${{ env.JOB_MATRIX_JSON }} - steps: - - name: Decode Base64 Test List Input if Set - if: env.BASE64_TEST_LIST != '' - run: | - echo "Decoding base64 tests list from the input" - DECODED_BASE64_TEST_LIST=$(echo $BASE64_TEST_LIST | base64 -d) - echo "Decoded input:" - echo "$DECODED_BASE64_TEST_LIST" - is_valid=$(echo "$DECODED_BASE64_TEST_LIST" | jq . > /dev/null 2>&1; echo $?) - if [ "$is_valid" -ne 0 ]; then - echo "Invalid base64 input. Please provide a valid base64 encoded JSON list of tests." - echo "Here is an example of valid JSON:" - cat <> $GITHUB_ENV - # testlistgenerator is a tool that builds a matrix of tests to run - - name: Set Up testlistgenerator - if: env.BASE64_TEST_LIST == '' - shell: bash - run: | - go install github.com/smartcontractkit/chainlink-testing-framework/tools/testlistgenerator@v1.1.0 - - name: Prepare matrix input - if: env.BASE64_TEST_LIST == '' - run: | - PATH=$PATH:$(go env GOPATH)/bin - export PATH - - if [[ "$ETH_IMPLEMENTATIONS" == *"geth"* ]]; then - echo "Will test compatibility with geth" - testlistgenerator -o compatibility_test_list.json -p ccip -r TestSmokeCCIPForBidirectionalLane -f './ccip-tests/smoke/ccip_test.go' -e geth -d "${{ needs.get-latest-available-images.outputs.geth_images }}" -t "ccip-geth-compatibility-test" -w "SIMULATED_1,SIMULATED_2" -c 1337,2337 -n ubuntu-latest - else - echo "Will not test compatibility with geth" - fi - - if [[ "$ETH_IMPLEMENTATIONS" == *"besu"* ]]; then - echo "Will test compatibility with besu" - testlistgenerator -o compatibility_test_list.json -p ccip -r TestSmokeCCIPForBidirectionalLane -f './ccip-tests/smoke/ccip_test.go' -e besu -d "${{ needs.get-latest-available-images.outputs.besu_images }}" -t "ccip-besu-compatibility-test" -w "SIMULATED_BESU_NONDEV_1,SIMULATED_BESU_NONDEV_2" -c 1337,2337 -n ubuntu-latest - else - echo "Will not test compatibility with besu" - fi - - # TODO: Waiting for CCIP-2255 to be resolved - if [[ "$ETH_IMPLEMENTATIONS" == *"erigon"* ]]; then - echo "Will test compatibility with erigon" - testlistgenerator -o compatibility_test_list.json -p ccip -r TestSmokeCCIPForBidirectionalLane -f './ccip-tests/smoke/ccip_test.go' -e erigon -d "${{ needs.get-latest-available-images.outputs.erigon_images }}" -t "ccip-erigon-compatibility-test" -w "SIMULATED_1,SIMULATED_2" -c 1337,2337 -n ubuntu-latest - else - echo "Will not test compatibility with erigon" - fi - - # TODO: uncomment when nethermind flake reason is addressed - if [[ "$ETH_IMPLEMENTATIONS" == *"nethermind"* ]]; then - echo "Will not test compatibility with nethermind due to flakiness" - # echo "Will test compatibility with nethermind" - # testlistgenerator -o compatibility_test_list.json -p ccip -r TestSmokeCCIPForBidirectionalLane -f './ccip-tests/smoke/ccip_test.go' -e nethermind -d "${{ needs.get-latest-available-images.outputs.nethermind_images }}" -t "ccip-nethermind-compatibility-test" -w "SIMULATED_1,SIMULATED_2" -c 1337,2337 -n ubuntu-latest - else - echo "Will not test compatibility with nethermind" - fi - - jq . compatibility_test_list.json - echo "Adding human-readable name" - jq 'map(. + {visible_name: (.docker_image | split(",")[0] | split("=")[1])})' compatibility_test_list.json > compatibility_test_list_modified.json - jq . compatibility_test_list_modified.json - JOB_MATRIX_JSON=$(jq -c . compatibility_test_list_modified.json) - echo "JOB_MATRIX_JSON=${JOB_MATRIX_JSON}" >> $GITHUB_ENV - - run-client-compatibility-matrix: - name: CCIP Compatibility with ${{ matrix.evm_node.visible_name }} - if: always() && needs.should-run.outputs.should_run == 'true' - environment: integration - permissions: - checks: write - pull-requests: write - id-token: write - contents: read - needs: [build-chainlink, prepare-compatibility-matrix, should-run, select-versions] - env: - CHAINLINK_COMMIT_SHA: ${{ needs.select-versions.outputs.chainlink_version }} - CHAINLINK_ENV_USER: ${{ github.actor }} - TEST_LOG_LEVEL: debug - strategy: - fail-fast: false - matrix: - evm_node: ${{fromJson(needs.prepare-compatibility-matrix.outputs.matrix)}} - runs-on: ubuntu-latest - steps: - - name: Checkout the repo - uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2 - with: - repository: smartcontractkit/chainlink - ref: ${{ needs.select-versions.outputs.chainlink_version }} - - name: Prepare Base64 TOML override - uses: ./.github/actions/setup-create-base64-config - with: - runId: ${{ github.run_id }} - testLogCollect: ${{ vars.TEST_LOG_COLLECT }} - selectedNetworks: ${{ matrix.evm_node.networks }} - chainlinkImage: ${{ env.CHAINLINK_IMAGE }} - chainlinkVersion: ${{ needs.select-versions.outputs.chainlink_version }} - pyroscopeServer: ${{ !startsWith(github.ref, 'refs/tags/') && '' || secrets.QA_PYROSCOPE_INSTANCE }} # Avoid sending blank envs https://github.com/orgs/community/discussions/25725 - pyroscopeEnvironment: ci-ccip-bidirectional-lane-${{ matrix.evm_node.name }} - pyroscopeKey: ${{ secrets.QA_PYROSCOPE_KEY }} - lokiEndpoint: ${{ secrets.LOKI_URL_CI }} - lokiTenantId: ${{ vars.LOKI_TENANT_ID }} - lokiBasicAuth: ${{ secrets.LOKI_BASIC_AUTH }} - logstreamLogTargets: ${{ vars.LOGSTREAM_LOG_TARGETS }} - grafanaUrl: ${{ vars.GRAFANA_URL }} - grafanaDashboardUrl: "/d/ddf75041-1e39-42af-aa46-361fe4c36e9e/ci-e2e-tests-logs" - - name: Prepare Base64 TOML override for CCIP secrets - uses: ./.github/actions/setup-create-base64-config-ccip - with: - runId: ${{ github.run_id }} - testLogCollect: ${{ vars.TEST_LOG_COLLECT }} - selectedNetworks: ${{ matrix.evm_node.networks }} - chainlinkImage: ${{ env.CHAINLINK_IMAGE }} - chainlinkVersion: ${{ needs.select-versions.outputs.chainlink_version }} - lokiEndpoint: ${{ secrets.LOKI_URL_CI }} - lokiTenantId: ${{ vars.LOKI_TENANT_ID }} - lokiBasicAuth: ${{ secrets.LOKI_BASIC_AUTH }} - logstreamLogTargets: ${{ vars.LOGSTREAM_LOG_TARGETS }} - grafanaUrl: ${{ vars.GRAFANA_URL }} - grafanaDashboardUrl: "/d/ddf75041-1e39-42af-aa46-361fe4c36e9e/ci-e2e-tests-logs" - customEvmNodes: ${{ matrix.evm_node.docker_image }} - evmNodeLogLevel: "trace" - - name: Prepare test log name - run: | - replace_special_chars() { - if [ -z "$1" ]; then - echo "Please provide a string as an argument." - return 1 - fi - - local input_string="$1" - - # Replace '/' with '-' - local modified_string="${input_string//\//-}" - - # Replace ':' with '-' - modified_string="${modified_string//:/-}" - - # Replace '.' with '-' - modified_string="${modified_string//./-}" - - echo "$modified_string" - } - echo "TEST_LOG_NAME=$(replace_special_chars "ccip-${{ matrix.evm_node.name }}-test-logs")" >> $GITHUB_ENV - - name: Print Test details - ${{ matrix.evm_node.docker_image }} - run: | - echo "EVM Implementation Docker Image: ${{ matrix.evm_node.docker_image }}" - echo "EVM Implementation Networks: ${{ matrix.evm_node.networks }}" - echo "Test identifier: ${{ matrix.evm_node.name }}" - - name: Run Tests - uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/run-tests@fc3e0df622521019f50d772726d6bf8dc919dd38 # v2.3.19 - with: - test_command_to_run: cd ./integration-tests && go test -timeout 30m -count=1 -json -test.parallel=2 ${{ matrix.evm_node.run }} 2>&1 | tee /tmp/gotest.log | gotestloghelper -ci - test_download_vendor_packages_command: cd ./integration-tests && go mod download - cl_repo: ${{ env.CHAINLINK_IMAGE }} - cl_image_tag: ${{ needs.select-versions.outputs.chainlink_version }} - aws_registries: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }} - artifacts_name: ${{ env.TEST_LOG_NAME }} - artifacts_location: | - ./integration-tests/smoke/logs/ - ./integration-tests/ccip-tests/smoke/logs/* - /tmp/gotest.log - publish_check_name: ${{ matrix.evm_node.name }} - token: ${{ secrets.GITHUB_TOKEN }} - go_mod_path: ./integration-tests/go.mod - cache_key_id: core-e2e-${{ env.MOD_CACHE_VERSION }} - cache_restore_only: "true" - QA_AWS_REGION: ${{ secrets.QA_AWS_REGION }} - QA_AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }} - QA_KUBECONFIG: "" - should_tidy: "false" - - name: Print failed test summary - if: always() - uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/show-test-summary@1587f59bfd626b668d303abbc90fee41b12397e6 # v2.3.23 - with: - test_directories: ./integration-tests/smoke/,./integration-tests/ccip-tests/smoke/ - - start-slack-thread: - name: Start Slack Thread - if: ${{ always() && needs.*.result != 'skipped' && needs.*.result != 'cancelled' && needs.should-run.outputs.should_run == 'true' }} - environment: integration - outputs: - thread_ts: ${{ steps.slack.outputs.thread_ts }} - permissions: - checks: write - pull-requests: write - id-token: write - contents: read - runs-on: ubuntu-latest - needs: [run-client-compatibility-matrix,should-run,select-versions] - steps: - - name: Debug Result - run: echo ${{ join(needs.*.result, ',') }} - - name: Main Slack Notification - uses: slackapi/slack-github-action@6c661ce58804a1a20f6dc5fbee7f0381b469e001 # v1.25.0 - id: slack - with: - channel-id: ${{ secrets.QA_CCIP_SLACK_CHANNEL }} - payload: | - { - "attachments": [ - { - "color": "${{ contains(join(needs.*.result, ','), 'failure') && '#C62828' || '#2E7D32' }}", - "blocks": [ - { - "type": "header", - "text": { - "type": "plain_text", - "text": "CCIP Compatibility Test Results ${{ contains(join(needs.*.result, ','), 'failure') && ':x:' || ':white_check_mark:'}}", - "emoji": true - } - }, - { - "type": "section", - "text": { - "type": "mrkdwn", - "text": "${{ contains(join(needs.*.result, ','), 'failure') && 'Some tests failed! Notifying ' || 'All Good!' }}" - } - }, - { - "type": "divider" - }, - { - "type": "section", - "text": { - "type": "mrkdwn", - "text": "<${{ github.server_url }}/${{ github.repository }}/releases/tag/${{ github.ref_name }}|${{ github.ref_name }}> | <${{ github.server_url }}/${{ github.repository }}/commit/${{ needs.select-versions.outputs.chainlink_version }}|${{ needs.select-versions.outputs.chainlink_version }}> | <${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}|Run>" - } - } - ] - } - ] - } - env: - SLACK_BOT_TOKEN: ${{ secrets.QA_SLACK_API_KEY }} - - parse-test-results: - name: Parse Test Results - if: always() && needs.*.result != 'skipped' && needs.*.result != 'cancelled' && needs.should-run.outputs.should_run == 'true' - environment: integration - permissions: - checks: write - pull-requests: write - id-token: write - contents: read - runs-on: ubuntu-latest - needs: [run-client-compatibility-matrix,should-run] - outputs: - base64_parsed_results: ${{ steps.get-test-results.outputs.base64_parsed_results }} - steps: - # workflowresultparser is a tool to get job results from a workflow run - - name: Set Up workflowresultparser - shell: bash - run: | - go install github.com/smartcontractkit/chainlink-testing-framework/tools/workflowresultparser@v1.0.0 - - name: Get and parse Test Results - shell: bash - id: get-test-results - run: | - PATH=$PATH:$(go env GOPATH)/bin - export PATH - - workflowresultparser -workflowRunID ${{ github.run_id }} -githubToken ${{ github.token }} -githubRepo "${{ github.repository }}" -jobNameRegex "^CCIP Compatibility with (.*)$" -namedKey="CCIP" -outputFile=output.json - - echo "base64_parsed_results=$(base64 -w 0 output.json)" >> $GITHUB_OUTPUT - - display-test-results: - name: Aggregated test results - if: always() && needs.*.result != 'skipped' && needs.*.result != 'cancelled' && needs.should-run.outputs.should_run == 'true' && needs.parse-test-results.result == 'success' - environment: integration - permissions: - checks: write - pull-requests: write - id-token: write - contents: read - runs-on: ubuntu-latest - needs: [start-slack-thread, should-run, select-versions, parse-test-results] - steps: - # asciitable is a tool that prints results in a nice ASCII table - - name: Set Up asciitable - shell: bash - run: | - go install github.com/smartcontractkit/chainlink-testing-framework/tools/asciitable@v1.0.2 - - name: Print aggregated test results - shell: bash - run: | - PATH=$PATH:$(go env GOPATH)/bin - export PATH - - raw_results="$(echo ${{ needs.parse-test-results.outputs.base64_parsed_results }} | base64 -d)" - echo $raw_results > input.json - asciitable --firstColumn "EVM Implementation" --secondColumn Result --jsonfile input.json --outputFile output.txt --section "CCIP" --namedKey "CCIP" - - echo - echo "AGGREGATED RESULTS" - cat output.txt - - echo "## Aggregated EVM Implementations compatibility results summary" >> $GITHUB_STEP_SUMMARY - echo '```' >> $GITHUB_STEP_SUMMARY - cat output.txt >> $GITHUB_STEP_SUMMARY - echo '```' >> $GITHUB_STEP_SUMMARY - - post-test-results-to-slack: - name: Post Test Results - if: ${{ always() && needs.*.result != 'skipped' && needs.*.result != 'cancelled' && needs.should-run.outputs.should_run == 'true' }} - environment: integration - permissions: - checks: write - pull-requests: write - id-token: write - contents: read - runs-on: ubuntu-latest - needs: [start-slack-thread,should-run,select-versions] - steps: - - name: Checkout the repo - uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2 - with: - ref: ${{ needs.select-versions.outputs.chainlink_version }} - - name: Get test results for CCIP - id: get-product-results - shell: bash - run: | - raw_results="$(echo ${{ needs.parse-test-results.outputs.base64_parsed_results }} | base64 -d)" - product_result=$(echo "$raw_results" | jq -c "select(has(\"CCIP\")) | .CCIP[]") - if [ -n "$product_result" ]; then - base64_result=$(echo $product_result | base64 -w 0) - echo "base64_result=$base64_result" >> $GITHUB_OUTPUT - else - echo "No results found for CCIP" - echo "base64_result=" >> $GITHUB_OUTPUT - fi - - name: Post Test Results to Slack - uses: ./.github/actions/notify-slack-jobs-result - with: - github_token: ${{ github.token }} - github_repository: ${{ github.repository }} - workflow_run_id: ${{ github.run_id }} - github_job_name_regex: ^CCIP Compatibility with (.*?)$ - message_title: CCIP Compatibility Test Results - slack_channel_id: ${{ secrets.QA_CCIP_SLACK_CHANNEL }} - slack_bot_token: ${{ secrets.QA_SLACK_API_KEY }} - slack_thread_ts: ${{ needs.start-slack-thread.outputs.thread_ts }} - base64_parsed_results: ${{ steps.get-product-results.outputs.base64_result }} diff --git a/.github/workflows/ccip-live-network-tests.yml b/.github/workflows/ccip-live-network-tests.yml deleted file mode 100644 index 6649a881b97..00000000000 --- a/.github/workflows/ccip-live-network-tests.yml +++ /dev/null @@ -1,313 +0,0 @@ -name: CCIP Live Network Tests -on: - # TODO: TT-1470 - Turn back on as we solidify new realease and image strategy - # schedule: - # - cron: '0 */6 * * *' - workflow_dispatch: - inputs: - base64_test_input : # base64 encoded toml for test input - description: 'Base64 encoded toml test input' - required: false - slackMemberID: - description: 'Slack member ID to notify' - required: false - test_type: - description: 'Type of test to run' - required: false - type: choice - options: - - 'load' - - 'smoke' - -# Only run 1 of this workflow at a time per PR -concurrency: - group: live-testnet-tests - cancel-in-progress: true - -env: - # TODO: TT-1470 - Update image names as we solidify new realease strategy - CHAINLINK_IMAGE: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com/chainlink - CHAINLINK_VERSION: ${{ github.sha}} - CHAINLINK_TEST_VERSION: ${{ github.sha}} - ENV_JOB_IMAGE: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com/chainlink-ccip-tests:${{ github.sha }} - INTERNAL_DOCKER_REPO: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com - AWS_ECR_REPO_PUBLIC_REGISTRY: public.ecr.aws - -jobs: - build-chainlink: - environment: integration - permissions: - id-token: write - contents: read - name: Build Chainlink Image - runs-on: ubuntu20.04-16cores-64GB - steps: - - name: Checkout the repo - uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2 - - name: Check if image exists - id: check-image - uses: smartcontractkit/chainlink-github-actions/docker/image-exists@5dd916d08c03cb5f9a97304f4f174820421bb946 # v2.3.11 - with: - repository: chainlink - tag: ${{ env.CHAINLINK_VERSION }} - AWS_REGION: ${{ secrets.QA_AWS_REGION }} - AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }} - - name: Build Image - if: steps.check-image.outputs.exists == 'false' - uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/build-image@5dd916d08c03cb5f9a97304f4f174820421bb946 # v2.3.11 - env: - GH_TOKEN: ${{ github.token }} - with: - cl_repo: smartcontractkit/chainlink-ccip - cl_ref: ${{ env.CHAINLINK_VERSION }} - push_tag: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com/chainlink:${{ env.CHAINLINK_VERSION }} - QA_AWS_REGION: ${{ secrets.QA_AWS_REGION }} - QA_AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }} - - name: Collect Metrics - if: always() - id: collect-gha-metrics - uses: smartcontractkit/push-gha-metrics-action@dea9b546553cb4ca936607c2267a09c004e4ab3f # v3.0.0 - with: - id: ccip-on-demand-live-testnet-tests-build-chainlink-image - org-id: ${{ secrets.GRAFANA_INTERNAL_TENANT_ID }} - basic-auth: ${{ secrets.GRAFANA_INTERNAL_BASIC_AUTH }} - hostname: ${{ secrets.GRAFANA_INTERNAL_HOST }} - this-job-name: Build Chainlink Image - continue-on-error: true - - build-test-image: - environment: integration - permissions: - id-token: write - contents: read - name: Build Test Image - runs-on: ubuntu20.04-16cores-64GB - steps: - - name: Collect Metrics - id: collect-gha-metrics - uses: smartcontractkit/push-gha-metrics-action@dea9b546553cb4ca936607c2267a09c004e4ab3f # v3.0.0 - with: - id: ccip-on-demand-live-testnet-tests-build-test-image - org-id: ${{ secrets.GRAFANA_INTERNAL_TENANT_ID }} - basic-auth: ${{ secrets.GRAFANA_INTERNAL_BASIC_AUTH }} - hostname: ${{ secrets.GRAFANA_INTERNAL_HOST }} - this-job-name: Build Test Image - continue-on-error: true - - name: Checkout the repo - uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2 - - name: Build Test Image - uses: ./.github/actions/build-test-image - with: - tag: ${{ env.CHAINLINK_TEST_VERSION }} - QA_AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }} - QA_AWS_REGION: ${{ secrets.QA_AWS_REGION }} - QA_AWS_ACCOUNT_NUMBER: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }} - - ccip-load-test: - name: CCIP Load Test - environment: integration - runs-on: ubuntu-latest - strategy: - matrix: - config: [mainnet.toml] - needs: [ build-chainlink, build-test-image ] - # if the event is a scheduled event or the test type is load and no previous job failed - if: ${{ (github.event_name == 'schedule' || inputs.test_type == 'load') && !contains(needs.*.result, 'failure') }} - permissions: - issues: read - checks: write - pull-requests: write - id-token: write - contents: read - env: - CHAINLINK_ENV_USER: ${{ github.actor }} - SLACK_API_KEY: ${{ secrets.QA_SLACK_API_KEY }} - SLACK_CHANNEL: ${{ secrets.QA_SLACK_CHANNEL }} - TEST_LOG_LEVEL: info - REF_NAME: ${{ github.head_ref || github.ref_name }} - ENV_JOB_IMAGE_BASE: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com/chainlink-ccip-tests - BASE64_NETWORK_CONFIG: ${{ secrets.BASE64_NETWORK_CONFIG }} - - steps: - - name: Collect Metrics - id: collect-gha-metrics - uses: smartcontractkit/push-gha-metrics-action@dea9b546553cb4ca936607c2267a09c004e4ab3f # v3.0.0 - with: - id: ccip-on-demand-live-testnet-tests-load-tests - org-id: ${{ secrets.GRAFANA_INTERNAL_TENANT_ID }} - basic-auth: ${{ secrets.GRAFANA_INTERNAL_BASIC_AUTH }} - hostname: ${{ secrets.GRAFANA_INTERNAL_HOST }} - this-job-name: CCIP Load Test - continue-on-error: true - - name: Checkout the repo - uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2 - with: - ref: ${{ env.REF_NAME }} - - name: Prepare Base64 TOML override - shell: bash - run: | - # this key secrets.QA_SHARED_803C_KEY has a story behind it. To know more, see CCIP-2875 and SECHD-16575 tickets. - BASE64_NETWORK_CONFIG=$(echo $BASE64_NETWORK_CONFIG | base64 -w 0 -d | sed -e 's/evm_key/${{ secrets.QA_SHARED_803C_KEY }}/g' | base64 -w 0) - echo ::add-mask::$BASE64_NETWORK_CONFIG - echo "BASE64_NETWORK_CONFIG=$BASE64_NETWORK_CONFIG" >> "$GITHUB_ENV" - SLACK_USER=$(jq -r '.inputs.slackMemberID' $GITHUB_EVENT_PATH) - echo ::add-mask::$SLACK_USER - echo "SLACK_USER=$SLACK_USER" >> "$GITHUB_ENV" - if [[ "${{ github.event_name }}" == "workflow_dispatch" ]]; then - BASE64_CCIP_CONFIG_OVERRIDE=$(jq -r '.inputs.base64_test_input' $GITHUB_EVENT_PATH) - echo ::add-mask::$BASE64_CCIP_CONFIG_OVERRIDE - echo "BASE64_CCIP_CONFIG_OVERRIDE=$BASE64_CCIP_CONFIG_OVERRIDE" >> $GITHUB_ENV - echo "TEST_BASE64_CCIP_CONFIG_OVERRIDE=$BASE64_CCIP_CONFIG_OVERRIDE" >> $GITHUB_ENV - fi - if [[ "${{ github.event_name }}" == "schedule" ]]; then - BASE64_CCIP_CONFIG_OVERRIDE=$(base64 -w 0 -i ./integration-tests/ccip-tests/testconfig/override/${{ matrix.config }}) - echo ::add-mask::$BASE64_CCIP_CONFIG_OVERRIDE - echo "BASE64_CCIP_CONFIG_OVERRIDE=$BASE64_CCIP_CONFIG_OVERRIDE" >> $GITHUB_ENV - echo "TEST_BASE64_CCIP_CONFIG_OVERRIDE=$BASE64_CCIP_CONFIG_OVERRIDE" >> $GITHUB_ENV - echo "SLACK_USER=${{ secrets.QA_SLACK_USER }}" >> $GITHUB_ENV - fi - - name: step summary - shell: bash - run: | - echo "### chainlink image used for this test run :link:" >>$GITHUB_STEP_SUMMARY - echo "\`${{ env.CHAINLINK_VERSION }}\`" >> $GITHUB_STEP_SUMMARY - echo "### chainlink-tests image tag for this test run :ship:" >>$GITHUB_STEP_SUMMARY - echo "\`${{ env.CHAINLINK_TEST_VERSION }}\`" >> $GITHUB_STEP_SUMMARY - - name: Prepare Base64 TOML override for CCIP secrets - uses: ./.github/actions/setup-create-base64-config-ccip - with: - runId: ${{ github.run_id }} - testLogCollect: ${{ vars.TEST_LOG_COLLECT }} - chainlinkImage: ${{ env.CHAINLINK_IMAGE }} - chainlinkVersion: ${{ github.sha }} - lokiEndpoint: ${{ secrets.LOKI_URL }} - lokiTenantId: ${{ vars.LOKI_TENANT_ID }} - lokiBasicAuth: ${{ secrets.LOKI_BASIC_AUTH }} - logstreamLogTargets: ${{ vars.LOGSTREAM_LOG_TARGETS }} - grafanaUrl: ${{ vars.GRAFANA_URL }} - grafanaDashboardUrl: "/d/6vjVx-1V8/ccip-long-running-tests" - - name: Run Tests - uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/run-tests@5dd916d08c03cb5f9a97304f4f174820421bb946 # v2.3.11 - env: - TEST_SUITE: load - TEST_ARGS: -test.timeout 900h - DATABASE_URL: postgresql://postgres:node@localhost:5432/chainlink_test?sslmode=disable - RR_MEM: 8Gi - RR_CPU: 4 - DETACH_RUNNER: true - TEST_TRIGGERED_BY: ccip-load-test-ci - with: - test_command_to_run: cd ./integration-tests/ccip-tests && go test -v -timeout 70m -count=1 -json -run ^TestLoadCCIPStableRPS$ ./load 2>&1 | tee /tmp/gotest.log | gotestfmt - test_download_vendor_packages_command: cd ./integration-tests && go mod download - cl_repo: ${{ env.CHAINLINK_IMAGE }} - cl_image_tag: ${{ env.CHAINLINK_VERSION }} - token: ${{ secrets.GITHUB_TOKEN }} - go_mod_path: ./integration-tests/go.mod - QA_AWS_REGION: ${{ secrets.QA_AWS_REGION }} - QA_AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }} - QA_KUBECONFIG: ${{ secrets.QA_KUBECONFIG }} - triggered_by: ${{ env.TEST_TRIGGERED_BY }} - artifacts_location: ./integration-tests/load/logs/payload_ccip.json - aws_registries: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }} - cache_key_id: ccip-load-${{ env.MOD_CACHE_VERSION }} - cache_restore_only: "true" - should_cleanup: false - - ccip-smoke-test: - name: CCIP smoke Test - environment: integration - runs-on: ubuntu-latest - needs: [ build-chainlink, build-test-image ] - # if the event is a scheduled event or the test type is load and no previous job failed - if: ${{ github.event_name == 'workflow_dispatch' && inputs.test_type == 'smoke' && !contains(needs.*.result, 'failure') }} - permissions: - issues: read - checks: write - pull-requests: write - id-token: write - contents: read - env: - CHAINLINK_ENV_USER: ${{ github.actor }} - SLACK_API_KEY: ${{ secrets.QA_SLACK_API_KEY }} - SLACK_CHANNEL: ${{ secrets.QA_SLACK_CHANNEL }} - TEST_LOG_LEVEL: info - REF_NAME: ${{ github.head_ref || github.ref_name }} - ENV_JOB_IMAGE_BASE: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com/chainlink-ccip-tests - BASE64_NETWORK_CONFIG: ${{ secrets.BASE64_NETWORK_CONFIG }} - - steps: - - name: Collect Metrics - id: collect-gha-metrics - uses: smartcontractkit/push-gha-metrics-action@dea9b546553cb4ca936607c2267a09c004e4ab3f # v3.0.0 - with: - id: ccip-on-demand-live-testnet-tests-smoke-test - org-id: ${{ secrets.GRAFANA_INTERNAL_TENANT_ID }} - basic-auth: ${{ secrets.GRAFANA_INTERNAL_BASIC_AUTH }} - hostname: ${{ secrets.GRAFANA_INTERNAL_HOST }} - this-job-name: CCIP Smoke Test - continue-on-error: true - - name: Checkout the repo - uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2 - with: - ref: ${{ env.REF_NAME }} - - name: Prepare Base64 TOML override - shell: bash - run: | - BASE64_NETWORK_CONFIG=$(echo $BASE64_NETWORK_CONFIG | base64 -w 0 -d | sed -e 's/evm_key/${{ secrets.QA_SHARED_803C_KEY }}/g' | base64 -w 0) - echo ::add-mask::$BASE64_NETWORK_CONFIG - echo "BASE64_NETWORK_CONFIG=$BASE64_NETWORK_CONFIG" >> "$GITHUB_ENV" - SLACK_USER=$(jq -r '.inputs.slackMemberID' $GITHUB_EVENT_PATH) - echo ::add-mask::$SLACK_USER - echo "SLACK_USER=$SLACK_USER" >> "$GITHUB_ENV" - if [[ "${{ github.event_name }}" == "workflow_dispatch" ]]; then - BASE64_CCIP_CONFIG_OVERRIDE=$(jq -r '.inputs.base64_test_input' $GITHUB_EVENT_PATH) - echo ::add-mask::$BASE64_CCIP_CONFIG_OVERRIDE - echo "BASE64_CCIP_CONFIG_OVERRIDE=$BASE64_CCIP_CONFIG_OVERRIDE" >> $GITHUB_ENV - echo "TEST_BASE64_CCIP_CONFIG_OVERRIDE=$BASE64_CCIP_CONFIG_OVERRIDE" >> $GITHUB_ENV - fi - - name: step summary - shell: bash - run: | - echo "### chainlink image used for this test run :link:" >>$GITHUB_STEP_SUMMARY - echo "\`${{ env.CHAINLINK_VERSION }}\`" >> $GITHUB_STEP_SUMMARY - echo "### chainlink-tests image tag for this test run :ship:" >>$GITHUB_STEP_SUMMARY - echo "\`${{ env.CHAINLINK_TEST_VERSION }}\`" >> $GITHUB_STEP_SUMMARY - - name: Prepare Base64 TOML override for CCIP secrets - uses: ./.github/actions/setup-create-base64-config-ccip - with: - runId: ${{ github.run_id }} - testLogCollect: ${{ vars.TEST_LOG_COLLECT }} - chainlinkImage: ${{ env.CHAINLINK_IMAGE }} - chainlinkVersion: ${{ github.sha }} - lokiEndpoint: ${{ secrets.LOKI_URL }} - lokiTenantId: ${{ vars.LOKI_TENANT_ID }} - lokiBasicAuth: ${{ secrets.LOKI_BASIC_AUTH }} - logstreamLogTargets: ${{ vars.LOGSTREAM_LOG_TARGETS }} - grafanaUrl: ${{ vars.GRAFANA_URL }} - grafanaDashboardUrl: "/d/ddf75041-1e39-42af-aa46-361fe4c36e9e/ci-e2e-tests-logs" - - name: Run Tests - uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/run-tests@5dd916d08c03cb5f9a97304f4f174820421bb946 # v2.3.11 - env: - TEST_SUITE: smoke - TEST_ARGS: -test.timeout 900h - DETACH_RUNNER: true - DATABASE_URL: postgresql://postgres:node@localhost:5432/chainlink_test?sslmode=disable - RR_MEM: 8Gi - RR_CPU: 4 - TEST_TRIGGERED_BY: ccip-smoke-test-ci - with: - test_command_to_run: cd ./integration-tests/ccip-tests && go test -v -timeout 70m -count=1 -p 30 -json -run ^TestSmokeCCIPForBidirectionalLane$ ./smoke 2>&1 | tee /tmp/gotest.log | gotestfmt - test_download_vendor_packages_command: cd ./integration-tests && go mod download - cl_repo: ${{ env.CHAINLINK_IMAGE }} - cl_image_tag: ${{ env.CHAINLINK_VERSION }} - token: ${{ secrets.GITHUB_TOKEN }} - go_mod_path: ./integration-tests/go.mod - QA_AWS_REGION: ${{ secrets.QA_AWS_REGION }} - QA_AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }} - QA_KUBECONFIG: ${{ secrets.QA_KUBECONFIG }} - triggered_by: ${{ env.TEST_TRIGGERED_BY }} - artifacts_location: ./integration-tests/smoke/logs/payload_ccip.json - aws_registries: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }} - cache_key_id: ccip-smoke-${{ env.MOD_CACHE_VERSION }} - cache_restore_only: "true" - should_cleanup: false \ No newline at end of file diff --git a/.github/workflows/ccip-load-tests.yml b/.github/workflows/ccip-load-tests.yml index 0d53d803d1c..a6d073715d1 100644 --- a/.github/workflows/ccip-load-tests.yml +++ b/.github/workflows/ccip-load-tests.yml @@ -13,6 +13,10 @@ on: base64_test_input: # base64 encoded toml for test input description: 'Base64 encoded toml test input' required: false + test_secrets_override_key: + description: 'Key to run tests with custom test secrets' + required: false + type: string # Only run 1 of this workflow at a time per PR concurrency: @@ -24,7 +28,7 @@ env: CHAINLINK_IMAGE: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com/chainlink CHAINLINK_VERSION: ${{ github.sha}} INPUT_CHAINLINK_TEST_VERSION: ${{ github.sha}} - ENV_JOB_IMAGE: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com/chainlink-ccip-tests:${{ github.sha }} + ENV_JOB_IMAGE: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com/chainlink-tests:${{ github.sha }} INTERNAL_DOCKER_REPO: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com AWS_ECR_REPO_PUBLIC_REGISTRY: public.ecr.aws MOD_CACHE_VERSION: 1 @@ -56,7 +60,7 @@ jobs: with: cl_repo: smartcontractkit/chainlink cl_ref: ${{ env.CHAINLINK_VERSION }} - push_tag: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com/chainlink:${{ env.CHAINLINK_VERSION }} + push_tag: ${{ env.CHAINLINK_IMAGE }}:${{ env.CHAINLINK_VERSION }} QA_AWS_REGION: ${{ secrets.QA_AWS_REGION }} QA_AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }} - name: Collect Metrics @@ -111,7 +115,6 @@ jobs: contents: read env: CHAINLINK_ENV_USER: ${{ github.actor }} - SLACK_USER: ${{ inputs.slackMemberID }} SLACK_API_KEY: ${{ secrets.QA_SLACK_API_KEY }} SLACK_CHANNEL: ${{ secrets.QA_SLACK_CHANNEL }} TEST_LOG_LEVEL: info @@ -147,20 +150,19 @@ jobs: with: ref: ${{ env.REF_NAME }} - name: Sets env vars + id: set_override_config shell: bash run: | # if the matrix.type.config_path is set, use it as the override config if [ -n "${{ matrix.type.config_path }}" ]; then - echo "BASE64_CCIP_CONFIG_OVERRIDE=$(base64 -w 0 -i ${{ matrix.type.config_path }})" >> $GITHUB_ENV - echo "TEST_BASE64_CCIP_CONFIG_OVERRIDE=$(base64 -w 0 -i ${{ matrix.type.config_path }})" >> $GITHUB_ENV + BASE64_CCIP_CONFIG_OVERRIDE=$(base64 -w 0 -i ${{ matrix.type.config_path }}) + echo ::add-mask::$BASE64_CCIP_CONFIG_OVERRIDE + echo "base_64_override=$BASE64_CCIP_CONFIG_OVERRIDE" >> $GITHUB_OUTPUT fi if [[ "${{ github.event_name }}" == "workflow_dispatch" ]]; then BASE64_CCIP_CONFIG_OVERRIDE=$(jq -r '.inputs.base64_test_input' $GITHUB_EVENT_PATH) echo ::add-mask::$BASE64_CCIP_CONFIG_OVERRIDE - if [ -n "${BASE64_CCIP_CONFIG_OVERRIDE}" && "$BASE64_CCIP_CONFIG_OVERRIDE" != "null"]; then - echo "BASE64_CCIP_CONFIG_OVERRIDE=$BASE64_CCIP_CONFIG_OVERRIDE" >> $GITHUB_ENV - echo "TEST_BASE64_CCIP_CONFIG_OVERRIDE=$BASE64_CCIP_CONFIG_OVERRIDE" >> $GITHUB_ENV - fi + echo "base_64_override=$BASE64_CCIP_CONFIG_OVERRIDE" >> $GITHUB_OUTPUT fi - name: step summary shell: bash @@ -171,18 +173,14 @@ jobs: echo "\`${{ env.INPUT_CHAINLINK_TEST_VERSION }}\`" >> $GITHUB_STEP_SUMMARY - name: Prepare Base64 TOML override for CCIP secrets uses: ./.github/actions/setup-create-base64-config-ccip + id: setup_create_base64_config_ccip with: runId: ${{ github.run_id }} testLogCollect: ${{ vars.TEST_LOG_COLLECT }} - chainlinkImage: ${{ env.CHAINLINK_IMAGE }} chainlinkVersion: ${{ github.sha }} - lokiEndpoint: ${{ secrets.LOKI_URL }} - lokiTenantId: ${{ vars.LOKI_TENANT_ID }} logstreamLogTargets: ${{ vars.LOGSTREAM_LOG_TARGETS }} - grafanaUrl: ${{ vars.GRAFANA_URL }} - grafanaDashboardUrl: "/d/6vjVx-1V8/ccip-long-running-tests" - name: Run Tests - uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/run-tests@b49a9d04744b0237908831730f8553f26d73a94b # v2.3.17 + uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/run-tests@d38226be720c5ccc1ff4d3cee40608ebf264cd59 # v2.3.26 env: TEST_SUITE: load TEST_ARGS: -test.timeout 900h @@ -190,11 +188,12 @@ jobs: RR_MEM: 8Gi RR_CPU: 4 TEST_TRIGGERED_BY: ccip-load-test-ci-${{ matrix.type.name }} + BASE64_CCIP_CONFIG_OVERRIDE: ${{ steps.set_override_config.outputs.base_64_override }},${{ steps.setup_create_base64_config_ccip.outputs.base64_config }} + TEST_BASE64_CCIP_CONFIG_OVERRIDE: ${{ steps.set_override_config.outputs.base_64_override }},${{ steps.setup_create_base64_config_ccip.outputs.base64_config }} with: test_command_to_run: cd ./integration-tests/ccip-tests && go test -v -timeout 70m -count=1 -json -run ${{ matrix.type.run }} ./load 2>&1 | tee /tmp/gotest.log | gotestloghelper -ci test_download_vendor_packages_command: cd ./integration-tests && go mod download - cl_repo: ${{ env.CHAINLINK_IMAGE }} - cl_image_tag: ${{ env.CHAINLINK_VERSION }} + test_secrets_override_base64: ${{ secrets[inputs.test_secrets_override_key] }} token: ${{ secrets.GITHUB_TOKEN }} go_mod_path: ./integration-tests/go.mod QA_AWS_REGION: ${{ secrets.QA_AWS_REGION }} @@ -207,6 +206,12 @@ jobs: cache_key_id: ccip-load-${{ env.MOD_CACHE_VERSION }} cache_restore_only: "true" should_cleanup: "true" + DEFAULT_CHAINLINK_IMAGE: ${{ env.CHAINLINK_IMAGE }} + DEFAULT_LOKI_TENANT_ID: ${{ vars.LOKI_TENANT_ID }} + DEFAULT_LOKI_ENDPOINT: ${{ secrets.LOKI_URL }} + DEFAULT_LOKI_BASIC_AUTH: ${{ secrets.LOKI_BASIC_AUTH }} + DEFAULT_GRAFANA_BASE_URL: ${{ vars.GRAFANA_URL }} + DEFAULT_GRAFANA_DASHBOARD_URL: "/d/6vjVx-1V8/ccip-long-running-tests" # Reporting Jobs start-slack-thread: @@ -289,4 +294,4 @@ jobs: slack_bot_token: ${{ secrets.QA_SLACK_API_KEY }} slack_thread_ts: ${{ needs.start-slack-thread.outputs.thread_ts }} - # End Reporting Jobs + # End Reporting Jobs \ No newline at end of file diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml index 2b7a3b728d1..46aa53e9c16 100644 --- a/.github/workflows/integration-tests.yml +++ b/.github/workflows/integration-tests.yml @@ -725,26 +725,20 @@ jobs: - name: Prepare Base64 CCIP TOML secrets uses: ./.github/actions/setup-create-base64-config-ccip + id: setup_create_base64_config_ccip with: runId: ${{ github.run_id }} - pyroscopeServer: ${{ matrix.product.pyroscope_env == '' && '' || !startsWith(github.ref, 'refs/tags/') && '' || secrets.QA_PYROSCOPE_INSTANCE }} # Avoid sending blank envs https://github.com/orgs/community/discussions/25725 - pyroscopeEnvironment: ${{ matrix.product.pyroscope_env }} - pyroscopeKey: ${{ secrets.QA_PYROSCOPE_KEY }} testLogCollect: ${{ vars.TEST_LOG_COLLECT }} selectedNetworks: SIMULATED_1,SIMULATED_2 - chainlinkImage: ${{ env.CHAINLINK_IMAGE }} - chainlinkVersion: ${{ github.sha }} - lokiEndpoint: https://${{ secrets.GRAFANA_INTERNAL_HOST }}/loki/api/v1/push - lokiTenantId: ${{ vars.LOKI_TENANT_ID }} - lokiBasicAuth: ${{ secrets.LOKI_BASIC_AUTH }} + chainlinkVersion: ${{ inputs.evm-ref || github.sha }} logstreamLogTargets: ${{ vars.LOGSTREAM_LOG_TARGETS }} - grafanaUrl: "http://localhost:8080/primary" - grafanaDashboardUrl: "/d/ddf75041-1e39-42af-aa46-361fe4c36e9e/ci-e2e-tests-logs" - grafanaBearerToken: ${{ secrets.GRAFANA_INTERNAL_URL_SHORTENER_TOKEN }} ## Run this step when changes that require tests to be run are made - name: Run Tests if: needs.changes.outputs.src == 'true' || github.event_name == 'workflow_dispatch' + env: + BASE64_CCIP_CONFIG_OVERRIDE: ${{ steps.set_override_config.outputs.base_64_override }},${{ steps.setup_create_base64_config_ccip.outputs.base64_config }} + TEST_BASE64_CCIP_CONFIG_OVERRIDE: ${{ steps.set_override_config.outputs.base_64_override }},${{ steps.setup_create_base64_config_ccip.outputs.base64_config }} uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/run-tests@aa8eea635029ab8d95abd3c206f56dae1e22e623 # v2.3.28 with: test_command_to_run: cd ./integration-tests && go test -timeout 30m -count=1 -json -test.parallel=${{ matrix.product.nodes }} ${{ steps.build-go-test-command.outputs.run_command }} 2>&1 | tee /tmp/gotest.log | gotestloghelper -ci -singlepackage -hidepassingtests=false -hidepassinglogs @@ -1027,25 +1021,6 @@ jobs: # other inputs duplicate-authorization-header: "true" - - name: Prepare Base64 CCIP TOML secrets - uses: ./.github/actions/setup-create-base64-config-ccip - with: - runId: ${{ github.run_id }} - pyroscopeServer: ${{ matrix.product.pyroscope_env == '' && '' || !startsWith(github.ref, 'refs/tags/') && '' || secrets.QA_PYROSCOPE_INSTANCE }} # Avoid sending blank envs https://github.com/orgs/community/discussions/25725 - pyroscopeEnvironment: ${{ matrix.product.pyroscope_env }} - pyroscopeKey: ${{ secrets.QA_PYROSCOPE_KEY }} - testLogCollect: ${{ vars.TEST_LOG_COLLECT }} - selectedNetworks: SIMULATED_1,SIMULATED_2 - chainlinkImage: ${{ env.CHAINLINK_IMAGE }} - chainlinkVersion: ${{ github.sha }} - lokiEndpoint: https://${{ secrets.GRAFANA_INTERNAL_HOST }}/loki/api/v1/push - lokiTenantId: ${{ vars.LOKI_TENANT_ID }} - lokiBasicAuth: ${{ secrets.LOKI_BASIC_AUTH }} - logstreamLogTargets: ${{ vars.LOGSTREAM_LOG_TARGETS }} - grafanaUrl: "http://localhost:8080/primary" - grafanaDashboardUrl: "/d/ddf75041-1e39-42af-aa46-361fe4c36e9e/ci-e2e-tests-logs" - grafanaBearerToken: ${{ secrets.GRAFANA_INTERNAL_URL_SHORTENER_TOKEN }} - ## Run this step when changes that require tests to be run are made - name: Run Tests if: needs.changes.outputs.src == 'true' || github.event_name == 'workflow_dispatch' diff --git a/core/scripts/go.mod b/core/scripts/go.mod index 9f1844693c9..0a9b4f7a34f 100644 --- a/core/scripts/go.mod +++ b/core/scripts/go.mod @@ -269,7 +269,7 @@ require ( github.com/sethvargo/go-retry v0.2.4 // indirect github.com/shirou/gopsutil v3.21.11+incompatible // indirect github.com/shirou/gopsutil/v3 v3.24.3 // indirect - github.com/smartcontractkit/chain-selectors v1.0.10 // indirect + github.com/smartcontractkit/chain-selectors v1.0.21 // indirect github.com/smartcontractkit/chainlink-ccip v0.0.0-20240806144315-04ac101e9c95 // indirect github.com/smartcontractkit/chainlink-cosmos v0.4.1-0.20240710121324-3ed288aa9b45 // indirect github.com/smartcontractkit/chainlink-data-streams v0.0.0-20240801131703-fd75761c982f // indirect diff --git a/core/scripts/go.sum b/core/scripts/go.sum index 35226b0de3d..c228e654155 100644 --- a/core/scripts/go.sum +++ b/core/scripts/go.sum @@ -1180,8 +1180,8 @@ github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMB github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= -github.com/smartcontractkit/chain-selectors v1.0.10 h1:t9kJeE6B6G+hKD0GYR4kGJSCqR1LNS7aI3jT0V+xGrg= -github.com/smartcontractkit/chain-selectors v1.0.10/go.mod h1:d4Hi+E1zqjy9HqMkjBE5q1vcG9VGgxf5VxiRHfzi2kE= +github.com/smartcontractkit/chain-selectors v1.0.21 h1:KCR9SA7PhOexaBzFieHoLv1WonwhVOPtOStpqTmLC4E= +github.com/smartcontractkit/chain-selectors v1.0.21/go.mod h1:d4Hi+E1zqjy9HqMkjBE5q1vcG9VGgxf5VxiRHfzi2kE= github.com/smartcontractkit/chainlink-automation v1.0.4 h1:iyW181JjKHLNMnDleI8umfIfVVlwC7+n5izbLSFgjw8= github.com/smartcontractkit/chainlink-automation v1.0.4/go.mod h1:u4NbPZKJ5XiayfKHD/v3z3iflQWqvtdhj13jVZXj/cM= github.com/smartcontractkit/chainlink-ccip v0.0.0-20240806144315-04ac101e9c95 h1:LAgJTg9Yr/uCo2g7Krp88Dco2U45Y6sbJVl8uKoLkys= diff --git a/go.mod b/go.mod index 7976cbcd440..e6fd4f9e232 100644 --- a/go.mod +++ b/go.mod @@ -72,7 +72,7 @@ require ( github.com/scylladb/go-reflectx v1.0.1 github.com/shirou/gopsutil/v3 v3.24.3 github.com/shopspring/decimal v1.4.0 - github.com/smartcontractkit/chain-selectors v1.0.10 + github.com/smartcontractkit/chain-selectors v1.0.21 github.com/smartcontractkit/chainlink-automation v1.0.4 github.com/smartcontractkit/chainlink-ccip v0.0.0-20240806144315-04ac101e9c95 github.com/smartcontractkit/chainlink-common v0.2.2-0.20240816202716-6930d109fd82 diff --git a/go.sum b/go.sum index 704b9507c69..e4843248667 100644 --- a/go.sum +++ b/go.sum @@ -1135,8 +1135,8 @@ github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMB github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= -github.com/smartcontractkit/chain-selectors v1.0.10 h1:t9kJeE6B6G+hKD0GYR4kGJSCqR1LNS7aI3jT0V+xGrg= -github.com/smartcontractkit/chain-selectors v1.0.10/go.mod h1:d4Hi+E1zqjy9HqMkjBE5q1vcG9VGgxf5VxiRHfzi2kE= +github.com/smartcontractkit/chain-selectors v1.0.21 h1:KCR9SA7PhOexaBzFieHoLv1WonwhVOPtOStpqTmLC4E= +github.com/smartcontractkit/chain-selectors v1.0.21/go.mod h1:d4Hi+E1zqjy9HqMkjBE5q1vcG9VGgxf5VxiRHfzi2kE= github.com/smartcontractkit/chainlink-automation v1.0.4 h1:iyW181JjKHLNMnDleI8umfIfVVlwC7+n5izbLSFgjw8= github.com/smartcontractkit/chainlink-automation v1.0.4/go.mod h1:u4NbPZKJ5XiayfKHD/v3z3iflQWqvtdhj13jVZXj/cM= github.com/smartcontractkit/chainlink-ccip v0.0.0-20240806144315-04ac101e9c95 h1:LAgJTg9Yr/uCo2g7Krp88Dco2U45Y6sbJVl8uKoLkys= diff --git a/integration-tests/ccip-tests/Makefile b/integration-tests/ccip-tests/Makefile index 5a40f7ca0f6..d33d9569153 100644 --- a/integration-tests/ccip-tests/Makefile +++ b/integration-tests/ccip-tests/Makefile @@ -1,11 +1,13 @@ -## To Override the default config, and secret config: -# example usage: make set_config override_toml=../config/config.toml secret_toml=../config/secret.toml network_config_toml=../config/network.toml +## To Override the default config: +# example usage: make set_config override_toml=../config/config.toml network_config_toml=../config/network.toml .PHONY: set_config set_config: if [ -s "$(override_toml)" ]; then \ echo "Overriding config with $(override_toml)"; \ echo "export BASE64_CCIP_CONFIG_OVERRIDE=$$(base64 -i $(override_toml))" > ./testconfig/override/.env; \ echo "export TEST_BASE64_CCIP_CONFIG_OVERRIDE=$$(base64 -i $(override_toml))" >> ./testconfig/override/.env; \ + echo "BASE64_CCIP_CONFIG_OVERRIDE=$$(base64 -i $(override_toml))" > ./testconfig/override/debug.env; \ + echo "TEST_BASE64_CCIP_CONFIG_OVERRIDE=$$(base64 -i $(override_toml))" >> ./testconfig/override/debug.env; \ else \ echo "No override config found, using default config"; \ echo > ./testconfig/override/.env; \ @@ -15,14 +17,15 @@ set_config: echo "export BASE64_NETWORK_CONFIG=$$(base64 -i $(network_config_toml))" >> ./testconfig/override/.env; \ fi - @echo "setting secret config with $(secret_toml)" - @echo "export BASE64_CCIP_SECRETS_CONFIG=$$(base64 -i $(secret_toml))" >> ./testconfig/override/.env - @echo "export TEST_BASE64_CCIP_SECRETS_CONFIG=$$(base64 -i $(secret_toml))" >> ./testconfig/override/.env - @echo "BASE64_CCIP_SECRETS_CONFIG=$$(base64 -i $(secret_toml))" > ./testconfig/override/debug.env - @echo "TEST_BASE64_CCIP_SECRETS_CONFIG=$$(base64 -i $(secret_toml))" >> ./testconfig/override/debug.env + @echo "Checking for test secrets file in ~/.testsecrets..."; + @if [ ! -f ~/.testsecrets ]; then \ + echo "WARNING: ~/.testsecrets file not found. No test secrets will be set. To set test secrets, refer to ./testconfig/examples/.testsecrets.example for the list of secrets and instruction how to set them up."; \ + else \ + echo "Test secrets will be loaded from ~/.testsecrets file."; \ + fi -# example usage: make test_load_ccip testimage=chainlink-ccip-tests:latest testname=TestLoadCCIPStableRPS override_toml=./testconfig/override/config.toml secret_toml=./testconfig/tomls/secrets.toml network_config_toml=../config/network.toml +# example usage: make test_load_ccip testimage=chainlink-ccip-tests:latest testname=TestLoadCCIPStableRPS override_toml=./testconfig/override/config.toml network_config_toml=../config/network.toml .PHONY: test_load_ccip test_load_ccip: set_config source ./testconfig/override/.env && \ @@ -36,7 +39,7 @@ test_load_ccip: set_config go test -timeout 24h -count=1 -v -run ^$(testname)$$ ./load -# example usage: make test_smoke_ccip testimage=chainlink-ccip-tests:latest testname=TestSmokeCCIPForBidirectionalLane override_toml=../testconfig/override/config.toml secret_toml=./testconfig/tomls/secrets.toml network_config_toml=../config/network.toml +# example usage: make test_smoke_ccip testimage=chainlink-ccip-tests:latest testname=TestSmokeCCIPForBidirectionalLane override_toml=../testconfig/override/config.toml network_config_toml=../config/network.toml .PHONY: test_smoke_ccip test_smoke_ccip: set_config source ./testconfig/override/.env && \ @@ -48,7 +51,7 @@ test_smoke_ccip: set_config go test -timeout 24h -count=1 -v -run ^$(testname)$$ ./smoke # run ccip smoke tests with default config; explicitly sets the override config to empty -# example usage: make test_smoke_ccip_default testname=TestSmokeCCIPForBidirectionalLane secret_toml=./testconfig/tomls/secrets.toml +# example usage: make test_smoke_ccip_default testname=TestSmokeCCIPForBidirectionalLane .PHONY: test_smoke_ccip_default test_smoke_ccip_default: set_config source ./testconfig/override/.env && \ diff --git a/integration-tests/ccip-tests/README.md b/integration-tests/ccip-tests/README.md index 0e000561fa4..f7182338f7f 100644 --- a/integration-tests/ccip-tests/README.md +++ b/integration-tests/ccip-tests/README.md @@ -34,34 +34,17 @@ For example, if you want to override the `Network` input in test and want to run export BASE64_CCIP_CONFIG_OVERRIDE=$(base64 -i ./testconfig/override/mainnet.toml) ``` -3. Secrets - You also need to set some secrets. This is a mandatory step needed to run the tests. Please refer to [sample-secrets.toml](./testconfig/examples/secrets.toml.example) for the list of secrets that are mandatory to run the tests. +3. Secrets - You also need to set some secrets. This is a mandatory step needed to run the tests. Please refer to [.testsecrets.example](./examples/.testsecrets.example) for the list of secrets and instruction how to set them up. - The chainlink image and tag are required secrets for all the tests. - If you are running tests in live networks like testnet and mainnet, you need to set the secrets (rpc urls and private keys) for the respective networks. - If you are running tests in simulated networks no network specific secrets are required. here is a sample secrets.toml file, for running the tests in simulated networks, with the chainlink image and tag set as secrets: - ```toml - [CCIP] - [CCIP.Env] - # ChainlinkImage is mandatory for all tests. - [CCIP.Env.NewCLCluster] - [CCIP.Env.NewCLCluster.Common] - [CCIP.Env.NewCLCluster.Common.ChainlinkImage] - image = "chainlink-ccip" - version = "latest" - ``` - - We consider secrets similar to test input overrides and encode them using `base64` command. - Once you have the secrets.toml file, you can encode it using `base64` command (similar to step 2) and set the env var `BASE64_CCIP_SECRETS_CONFIG` with the encoded content. - - ```bash - export BASE64_CCIP_SECRETS_CONFIG=$(base64 -i ./testconfig/tomls/secrets.toml) - ``` - **Please note that the secrets should NOT be checked in to the repo and should be kept locally.** -We recommend against changing the content of [sample-secrets.toml](./testconfig/examples/secrets.toml.example). Please create a new file and set it as the secrets file. -You can run the command to ignore the changes to the file. +We recommend against changing the content of [secrets.toml.example](./testconfig/examples/secrets.toml.example). Please create a new file and set it as the secrets file. + +You can run this command to ignore any changes to the file. ```bash git update-index --skip-worktree diff --git a/integration-tests/ccip-tests/testconfig/README.md b/integration-tests/ccip-tests/testconfig/README.md index c32aee3d913..51009e49a20 100644 --- a/integration-tests/ccip-tests/testconfig/README.md +++ b/integration-tests/ccip-tests/testconfig/README.md @@ -1,21 +1,23 @@ # CCIP Configuration -The CCIP configuration is used to specify the test configuration for running the CCIP integration tests. +The CCIP configuration is used to specify the test configuration for running the CCIP integration tests. The configuration is specified in a TOML file. The configuration is used to specify the test environment, test type, test parameters, and other necessary details for running the tests. The test config is read in following order: -- The test reads the default configuration from [ccip-default.toml](./tomls/ccip-default.toml). -- The default can be overridden by specifying the test config in a separate file. - - The file content needs to be encoded in base64 format and set in `BASE64_CCIP_CONFIG_OVERRIDE` environment variable. + +- The test reads the default configuration from [ccip-default.toml](./tomls/ccip-default.toml). +- The default can be overridden by specifying the test config in a separate file. + - The file content needs to be encoded in base64 format and set in `BASE64_CCIP_CONFIG_OVERRIDE` environment variable. - The config mentioned in this file will override the default config. - Example override file - [override.toml.example](./examples/override.toml.example) -- If there are sensitive details like private keys, credentials in test config, they can be specified in a separate secret file. - - The file content needs to be encoded in base64 format and set in `BASE64_CCIP_SECRETS_CONFIG` environment variable. - - The config mentioned in this file will override the default and override config. - - Example secret file - [secrets.toml.example](./examples/secrets.toml.example) +- If there are sensitive details like private keys, credentials in test config, they can be specified in a separate dotenv file as env vars + - The `~/.testsecrets` file in home directory is automatically loaded and should have all test secrets as env vars + - Example secret file - [.testsecrets.example](./examples/.testsecrets.example) ## CCIP.ContractVersions + Specifies contract versions of different contracts to be referred by test. Supported versions are: + - **PriceRegistry**: '1.2.0', 'Latest' - **OffRamp**: '1.2.0', 'Latest' - **OnRamp**: '1.2.0', 'Latest' @@ -23,6 +25,7 @@ Supported versions are: - **CommitStore**: '1.2.0', 'Latest' Example Usage: + ```toml [CCIP.ContractVersions] PriceRegistry = "1.2.0" @@ -33,14 +36,17 @@ CommitStore = "1.2.0" ``` ## CCIP.Deployments -CCIP Deployment contains all necessary contract addresses for various networks. This is mandatory if the test are to be run for [existing deployments](#ccipgroupstestgroupexistingdeployment) + +CCIP Deployment contains all necessary contract addresses for various networks. This is mandatory if the test are to be run for [existing deployments](#ccipgroupstestgroupexistingdeployment) The deployment data can be specified - - - Under `CCIP.Deployments.Data` field with value as stringify format of json. - - Under `CCIP.Deployments.DataFile` field with value as the path of the file containing the deployment data in json format. + +- Under `CCIP.Deployments.Data` field with value as stringify format of json. +- Under `CCIP.Deployments.DataFile` field with value as the path of the file containing the deployment data in json format. The json schema is specified in https://github.com/smartcontractkit/ccip/blob/ccip-develop/integration-tests/ccip-tests/contracts/laneconfig/parse_contracts.go#L96 Example Usage: + ```toml [CCIP.Deployments] Data = """ @@ -96,32 +102,39 @@ Data = """ } """ ``` -Or + +Or, + ```toml [CCIP.Deployments] DataFile = '' ``` -## CCIP.Env +## CCIP.Env + Specifies the environment details for the test to be run on. Mandatory fields are: + - **Networks**: [CCIP.Env.Networks](#ccipenvnetworks) - **NewCLCluster**: [CCIP.Env.NewCLCluster](#ccipenvnewclcluster) - This is mandatory if the test needs to deploy Chainlink nodes. - **ExistingCLCluster**: [CCIP.Env.ExistingCLCluster](#ccipenvexistingclcluster) - This is mandatory if the test needs to run on existing Chainlink nodes to deploy ccip jobs. Test needs network/chain details to be set through configuration. This configuration is mandatory for running the tests. you have option to set the network details in two ways: -1. Using [CCIP.Env.Networks](#ccipenvnetworks) + +1. Using [CCIP.Env.Networks](#ccipenvnetworks) 2. Using a separate network config file - - * refer to the example - [network_config.toml.example](./examples/network_config.toml.example) - * once all necessary values are set, encode the toml file content in base64 format, - * set the base64'ed string content in `BASE64_NETWORK_CONFIG` environment variable. + - refer to the example - [network_config.toml.example](./examples/network_config.toml.example) + - once all necessary values are set, encode the toml file content in base64 format, + - set the base64'ed string content in `BASE64_NETWORK_CONFIG` environment variable. ### CCIP.Env.Networks + Specifies the network details for the test to be run. The NetworkConfig is imported from https://github.com/smartcontractkit/chainlink-testing-framework/blob/main/config/network.go#L39 #### CCIP.Env.Networks.selected_networks + It denotes the network names in which tests will be run. These networks are used to deploy ccip contracts and set up lanes between them. If more than 2 networks are specified, then lanes will be set up between all possible pairs of networks. @@ -132,28 +145,32 @@ The name of the networks are taken from [known_networks](https://github.com/smar If the network is not present in known_networks, then the network details can be specified in the config file itself under the following `EVMNetworks` key. #### CCIP.Env.Network.EVMNetworks -Specifies the network config to be used while creating blockchain EVMClient for test. + +Specifies the network config to be used while creating blockchain EVMClient for test. It is a map of network name to EVMNetworks where key is network name specified under `CCIP.Env.Networks.selected_networks` and value is `EVMNetwork`. The EVMNetwork is imported from [EVMNetwork](https://github.com/smartcontractkit/chainlink-testing-framework/blob/main/blockchain/config.go#L43) in chainlink-testing-framework. If `CCIP.Env.Network.EVMNetworks` config is not set for a network name specified under `CCIP.Env.Networks.selected_networks`, test will try to find the corresponding network config from defined networks in `MappedNetworks` under [known_networks.go](https://github.com/smartcontractkit/chainlink-testing-framework/blob/main/networks/known_networks.go) #### CCIP.Env.Network.AnvilConfigs -If the test needs to run on chains created using Anvil, then the AnvilConfigs can be specified. -It is a map of network name to `AnvilConfig` where key is network name specified under `CCIP.Env.Networks.selected_networks` and value is `AnvilConfig`. -The AnvilConfig is imported from [AnvilConfig](https://github.com/smartcontractkit/chainlink-testing-framework/blob/main/config/network.go#L20) in chainlink-testing-framework. +If the test needs to run on chains created using Anvil, then the AnvilConfigs can be specified. +It is a map of network name to `AnvilConfig` where key is network name specified under `CCIP.Env.Networks.selected_networks` and value is `AnvilConfig`. +The AnvilConfig is imported from [AnvilConfig](https://github.com/smartcontractkit/chainlink-testing-framework/blob/main/config/network.go#L20) in chainlink-testing-framework. **The following network configs are required for tests running on live networks. It can be ignored if the tests are running on simulated networks.** Refer to [secrets.toml.example](./examples/secrets.toml.example) for details. #### CCIP.ENV.Network.RpcHttpUrls + RpcHttpUrls is the RPC HTTP endpoints for each network, key is the network name as declared in selected_networks slice. #### CCIP.ENV.Network.RpcWsUrls + RpcWsUrls is the RPC WS endpoints for each network, key is the network name as declared in selected_networks slice. #### CCIP.ENV.Network.WalletKeys + WalletKeys is the private keys for each network, key is the network name as declared in selected_networks slice. Example Usage of Network Config: @@ -202,69 +219,95 @@ block_time = 1 ``` ### CCIP.Env.NewCLCluster -The NewCLCluster config holds the overall deployment configuration for Chainlink nodes. + +The NewCLCluster config holds the overall deployment configuration for Chainlink nodes. #### CCIP.Env.NewCLCluster.NoOfNodes + Specifies the number of Chainlink nodes to be deployed. #### CCIP.Env.NewCLCluster.Common + Specifies the common configuration for all Chainlink nodes if they share the same configuration. -##### Name: + +##### Name + Name of the node. -##### NeedsUpgrade: + +##### NeedsUpgrade + Indicates if the node needs an upgrade during test. -##### ChainlinkImage: + +##### ChainlinkImage + Configuration for the Chainlink image. -##### ChainlinkUpgradeImage: +##### ChainlinkUpgradeImage + Configuration for the Chainlink upgrade image. It is used when the node needs an upgrade. -##### BaseConfigTOML: +##### BaseConfigTOML + String containing the base configuration toml content for the Chainlink node config. -##### CommonChainConfigTOML: +##### CommonChainConfigTOML + String containing the common chain configuration toml content for all EVMNodes in chainlink node config. -##### ChainConfigTOMLByChain: +##### ChainConfigTOMLByChain + String containing the chain-specific configuration toml content for individual EVMNodes in chainlink node config. This is keyed by chain ID. -##### DBImage: +##### DBImage + Database image for the Chainlink node. -##### DBTag: +##### DBTag + Database tag/version for the Chainlink node. #### CCIP.Env.NewCLCluster.Nodes + Specifies the configuration for individual nodes if they differ from the common configuration. The fields are the same as the common configuration. #### CCIP.Env.NewCLCluster.NodeMemory + Specifies the memory to be allocated for each Chainlink node. This is valid only if the deployment is on Kubernetes. #### CCIP.Env.NewCLCluster.NodeCPU + Specifies the CPU to be allocated for each Chainlink node. This is valid only if the deployment is on Kubernetes. #### CCIP.Env.NewCLCluster.DBMemory + Specifies the memory to be allocated for the database. This is valid only if the deployment is on Kubernetes. #### CCIP.Env.NewCLCluster.DBCPU + Specifies the CPU to be allocated for the database. This is valid only if the deployment is on Kubernetes. #### CCIP.Env.NewCLCluster.IsStateful + Specifies whether the deployment is StatefulSet on Kubernetes. #### CCIP.Env.NewCLCluster.DBStorageClass + Specifies the storage class for the database. This is valid only if the deployment is StatefulSet on Kubernetes. #### CCIP.Env.NewCLCluster.DBCapacity + Specifies the capacity of the database. This is valid only if the deployment is StatefulSet on Kubernetes. #### CCIP.Env.NewCLCluster.PromPgExporter + Specifies whether to enable Prometheus PostgreSQL exporter. This is valid only if the deployment is on Kubernetes. #### CCIP.Env.NewCLCluster.DBArgs + Specifies the arguments to be passed to the database. This is valid only if the deployment is on Kubernetes. Example Usage: + ```toml [CCIP.Env.NewCLCluster] NoOfNodes = 17 @@ -294,29 +337,42 @@ FeeCapDefault = '200 gwei' ``` ### CCIP.Env.ExistingCLCluster -The ExistingCLCluster config holds the overall connection configuration for existing Chainlink nodes. -It is needed when the tests are to be run on Chainlink nodes already deployed on some environment. -If this is specified, test will not need to connect to k8 namespace using [CCIP.Env.EnvToConnect](#ccipenvenvtoconnect) . + +The ExistingCLCluster config holds the overall connection configuration for existing Chainlink nodes. +It is needed when the tests are to be run on Chainlink nodes already deployed on some environment. +If this is specified, test will not need to connect to k8 namespace using [CCIP.Env.EnvToConnect](#ccipenvenvtoconnect). Test can directly connect to the existing Chainlink nodes using node credentials without knowing the k8 namespace details. #### CCIP.Env.ExistingCLCluster.Name + Specifies the name of the existing Chainlink cluster. This is used to identify the cluster in the test. #### CCIP.Env.ExistingCLCluster.NoOfNodes + Specifies the number of Chainlink nodes in the existing cluster. #### CCIP.Env.ExistingCLCluster.NodeConfigs + Specifies the configuration for individual nodes in the existing cluster. Each node config contains the following fields to connect to the Chainlink node: + ##### CCIP.Env.ExistingCLCluster.NodeConfigs.URL + The URL of the Chainlink node. -##### CCIP.Env.ExistingCLCluster.NodeConfigs.Email + +##### CCIP.Env.ExistingCLCluster.NodeConfigs.Email + The username/email of the Chainlink node credential. + ##### CCIP.Env.ExistingCLCluster.NodeConfigs.Password + The password of the Chainlink node credential. + ##### CCIP.Env.ExistingCLCluster.NodeConfigs.InternalIP + The internal IP of the Chainlink node. Example Usage: + ```toml [CCIP.Env.ExistingCLCluster] Name = 'crib-sample' @@ -355,23 +411,30 @@ InternalIP = 'app-node-5' ``` ### CCIP.Env.EnvToConnect + This is specified when the test needs to connect to already existing k8s namespace. User needs to have access to the k8 namespace to run the tests through specific kubeconfig file. Example usage: + ```toml [CCIP.Env] EnvToConnect="load-ccip-c8972" ``` + ### CCIP.ENV.TTL + Specifies the time to live for the k8 namespace. This is used to terminate the namespace after the tests are run. This is only valid if the tests are run on k8s. Example usage: + ```toml [CCIP.Env] TTL = "11h" ``` ### CCIP.Env.Logging + Specifies the logging configuration for the test. Imported from [LoggingConfig](https://github.com/smartcontractkit/chainlink-testing-framework/blob/main/config/logging.go#L11) in chainlink-testing-framework. Example usage: + ```toml [CCIP.Env.Logging] test_log_collect = false # if set to true will save logs even if test did not fail @@ -394,53 +457,67 @@ dashboard_url = "/d/6vjVx-1V8/ccip-long-running-tests" ``` ### CCIP.Env.Lane.LeaderLaneEnabled + Specifies whether to enable the leader lane feature. This setting is only applicable for new deployments. ## CCIP.Groups + Specifies the test config specific to each test type. Available test types are: + - **CCIP.Groups.load** - **CCIP.Groups.smoke** - **CCIP.Groups.chaos** ### CCIP.Groups.[testgroup].KeepEnvAlive + Specifies whether to keep the k8 namespace alive after the test is run. This is only valid if the tests are run on k8s. ### CCIP.Groups.[testgroup].BiDirectionalLane -Specifies whether to set up bi-directional lanes between networks. + +Specifies whether to set up bi-directional lanes between networks. ### CCIP.Groups.[testgroup].CommitAndExecuteOnSameDON + Specifies whether commit and execution jobs are to be run on the same Chainlink node. ### CCIP.Groups.[testgroup].NoOfCommitNodes + Specifies the number of nodes on which commit jobs are to be run. This needs to be lesser than the total number of nodes mentioned in [CCIP.Env.NewCLCluster.NoOfNodes](#ccipenvnewclclusternoofnodes) or [CCIP.Env.ExistingCLCluster.NoOfNodes](#ccipenvexistingclclusternoofnodes). If the value of total nodes is `n`, then the max value of NoOfCommitNodes should be less than `n-1`. As the first node is used for bootstrap job. If the NoOfCommitNodes is lesser than `n-1`, then the remaining nodes are used for execution jobs if `CCIP.Groups.[testgroup].CommitAndExecuteOnSameDON` is set to false. ### CCIP.Groups.[testgroup].TokenConfig + Specifies the token configuration for the test. The token configuration is used to set up tokens and token pools for all chains. #### CCIP.Groups.[testgroup].TokenConfig.NoOfTokensPerChain + Specifies the number of tokens to be set up for each chain. #### CCIP.Groups.[testgroup].TokenConfig.WithPipeline -Specifies whether to set up token pipelines in commit jobspec. If set to false, the token prices will be set with DynamicPriceGetterConfig. + +Specifies whether to set up token pipelines in commit jobspec. If set to false, the token prices will be set with DynamicPriceGetterConfig. #### CCIP.Groups.[testgroup].TokenConfig.TimeoutForPriceUpdate -Specifies the timeout to wait for token and gas price updates to be available in price registry for each chain. + +Specifies the timeout to wait for token and gas price updates to be available in price registry for each chain. #### CCIP.Groups.[testgroup].TokenConfig.NoOfTokensWithDynamicPrice + Specifies the number of tokens to be set up with dynamic price update. The rest of the tokens will be set up with static price. This is only valid if [WithPipeline](#ccipgroupstestgrouptokenconfigwithpipeline) is set to false. #### CCIP.Groups.[testgroup].TokenConfig.DynamicPriceUpdateInterval + Specifies the interval for dynamic price update for tokens. This is only valid if [NoOfTokensWithDynamicPrice](#ccipgroupstestgrouptokenconfignooftokenswithdynamicprice) is set to value greater tha zero. #### CCIP.Groups.[testgroup].TokenConfig.CCIPOwnerTokens + Specifies the tokens to be owned by the CCIP owner. If this is false, the tokens and pools will be owned by an address other than rest of CCIP contract admin addresses. This is applicable only if the contract versions are '1.5' or higher. Example Usage: -```toml +```toml [CCIP.Groups.load.TokenConfig] TimeoutForPriceUpdate = '15m' NoOfTokensPerChain = 60 @@ -450,27 +527,36 @@ CCIPOwnerTokens = true ``` ### CCIP.Groups.[testgroup].MsgDetails -Specifies the ccip message details to be sent by the test. + +Specifies the ccip message details to be sent by the test. + #### CCIP.Groups.[testgroup].MsgDetails.MsgType + Specifies the type of message to be sent. The supported message types are: + - **Token** - **Data** - **DataWithToken** #### CCIP.Groups.[testgroup].MsgDetails.DestGasLimit + Specifies the gas limit for the destination chain. This is used to in `ExtraArgs` field of CCIPMessage. Change this to 0, if you are doing ccip-send to an EOA in the destination chain. #### CCIP.Groups.[testgroup].MsgDetails.DataLength + Specifies the length of data to be sent in the message. This is only valid if [MsgType](#ccipgroupstestgroupmsgdetailsmsgtype) is set to 'Data' or 'DataWithToken'. #### CCIP.Groups.[testgroup].MsgDetails.NoOfTokens + Specifies the number of tokens to be sent in the message. This is only valid if [MsgType](#ccipgroupstestgroupmsgdetailsmsgtype) is set to 'Token' or 'DataWithToken'. It needs to be less than or equal to [NoOfTokensPerChain](#ccipgroupstestgrouptokenconfignooftokensperchain) specified in the test config. #### CCIP.Groups.[testgroup].MsgDetails.TokenAmount + Specifies the amount for each token to be sent in the message. This is only valid if [MsgType](#ccipgroupstestgroupmsgdetailsmsgtype) is set to 'Token' or 'DataWithToken'. Example Usage: + ```toml [CCIP.Groups.smoke.MsgDetails] MsgType = 'DataWithToken' @@ -481,15 +567,19 @@ AmountPerToken = 1 ``` ### CCIP.Groups.[testgroup].MulticallInOneTx + Specifies whether to send multiple ccip messages in a single transaction. ### CCIP.Groups.[testgroup].NoOfSendsInMulticall + Specifies the number of ccip messages to be sent in a single transaction. This is only valid if [MulticallInOneTx](#ccipgroupstestgroupmulticallinonetx) is set to true. ### CCIP.Groups.[testgroup].PhaseTimeout + The test validates various events in a ccip request lifecycle, like commit, execute, etc. This field specifies the timeout for each phase in the lifecycle. The timeout is calculated from the time the previous phase event is received. The following contract events are validated: + - **CCIPSendRequested on OnRamp** - **CCIPSendRequested event log to be Finalized** - **ReportAccepted on CommitStore** @@ -497,13 +587,16 @@ The following contract events are validated: - **ExecutionStateChanged on OffRamp** ### CCIP.Groups.[testgroup].LocalCluster + Specifies whether the test is to be run on a local docker. If set to true, the test environment will be set up on a local docker. ### CCIP.Groups.[testgroup].ExistingDeployment + Specifies whether the test is to be run on existing deployments. If set to true, the test will use the deployment data specified in [CCIP.Deployments](#ccipdeployments) for interacting with the ccip contracts. If the deployment data does not contain the required contract addresses, the test will fail. ### CCIP.Groups.[testgroup].ReuseContracts + Test loads contract/lane config from [contracts.json](../contracts/laneconfig/contracts.json) if no lane config is specified in [CCIP.Deployments](#ccipdeployments) If a certain contract is present in the contracts.json, the test will use the contract address from the contracts.json. This field specifies whether to reuse the contracts from [contracts.json](../contracts/laneconfig/contracts.json) @@ -511,58 +604,74 @@ For example if the contracts.json contains the contract address for PriceRegistr If `ReuseContracts` is set to false, the test will redeploy the contract instead of using the contract address from contracts.json. ### CCIP.Groups.[testgroup].NodeFunding + Specified the native token funding for each Chainlink node. It assumes that the native token decimals is 18. The funding is done by the private key specified in [CCIP.Env.Networks](#ccipenvnetworks) for each network. The funding is done only if the test is run on local docker or k8s. This is not applicable for [existing deployments](#ccipgroupstestgroupexistingdeployment) is set to true. ### CCIP.Groups.[testgroup].NetworkPairs -Specifies the network pairs for which the test is to be run. The test will set up lanes only between the specified network pairs. + +Specifies the network pairs for which the test is to be run. The test will set up lanes only between the specified network pairs. If the network pairs are not specified, the test will set up lanes between all possible pairs of networks mentioned in selected_networks in [CCIP.Env.Networks](#ccipenvnetworksselectednetworks) ### CCIP.Groups.[testgroup].NoOfNetworks -Specifies the number of networks to be used for the test. -If the number of networks is greater than the total number of networks specified in [CCIP.Env.Networks.selected_networks](#ccipenvnetworksselectednetworks) : + +Specifies the number of networks to be used for the test. +If the number of networks is greater than the total number of networks specified in [CCIP.Env.Networks.selected_networks](#ccipenvnetworksselectednetworks): + - the test will fail if the networks are live networks. - the test will create equal number of replicas of the first network with a new chain id if the networks are simulated networks. For example, if the `selected_networks` is ['SIMULATED_1','SIMULATED_2'] and `NoOfNetworks` is 3, the test will create 1 more network config by copying the network config of `SIMULATED_1` with a different chain id and use that as 3rd network. ### CCIP.Groups.[testgroup].NoOfRoutersPerPair + Specifies the number of routers to be set up for each network. ### CCIP.Groups.[testgroup].MaxNoOfLanes + Specifies the maximum number of lanes to be set up between networks. If this values is not set, the test will set up lanes between all possible pairs of networks mentioned in `selected_networks` in [CCIP.Env.Networks](#ccipenvnetworksselectednetworks). For example, if `selected_networks = ['SIMULATED_1', 'SIMULATED_2', 'SIMULATED_3']`, and `MaxNoOfLanes` is set to 2, it denotes that the test will randomly select the 2 lanes between all possible pairs `SIMULATED_1`, `SIMULATED_2`, and `SIMULATED_3` for the test run. ### CCIP.Groups.[testgroup].DenselyConnectedNetworkChainIds -This is applicable only if [MaxNoOfLanes](#ccipgroupstestgroupmaxnooflanes) is specified. + +This is applicable only if [MaxNoOfLanes](#ccipgroupstestgroupmaxnooflanes) is specified. Specifies the chain ids for networks to be densely connected. If this is provided the test will include all possible pairs of networks mentioned in `DenselyConnectedNetworkChainIds`. The rest of the networks will be connected randomly based on the value of `MaxNoOfLanes`. ### CCIP.Groups.[testgroup].ChaosDuration + Specifies the duration for which the chaos experiment is to be run. This is only valid if the test type is 'chaos'. ### CCIP.Groups.[testgroup].USDCMockDeployment + Specifies whether to deploy USDC mock contract for the test. This is only valid if the test is not run on [existing deployments](#ccipgroupstestgroupexistingdeployment). The following fields are used for various parameters in OCR2 commit and execution jobspecs. All of these are only valid if the test is not run on [existing deployments](#ccipgroupstestgroupexistingdeployment). + ### CCIP.Groups.[testgroup].CommitOCRParams + Specifies the OCR parameters for the commit job. This is only valid if the test is not run on [existing deployments](#ccipgroupstestgroupexistingdeployment). ### CCIP.Groups.[testgroup].ExecuteOCRParams + Specifies the OCR parameters for the execute job. This is only valid if the test is not run on [existing deployments](#ccipgroupstestgroupexistingdeployment). ### CCIP.Groups.[testgroup].CommitInflightExpiry + Specifies the value for the `InflightExpiry` in commit job's offchain config. This is only valid if the test is not run on [existing deployments](#ccipgroupstestgroupexistingdeployment). ### CCIP.Groups.[testgroup].OffRampConfig + Specifies the offramp configuration for the execution job. This is only valid if the test is not run on [existing deployments](#ccipgroupstestgroupexistingdeployment). This is used to set values for following fields in execution jobspec's offchain and onchain config: + - **OffRampConfig.MaxDataBytes** - **OffRampConfig.BatchGasLimit** - **OffRampConfig.InflightExpiry** - **OffRampConfig.RootSnooze** Example Usage: + ```toml [CCIP.Groups.load] CommitInflightExpiry = '5m' @@ -594,43 +703,53 @@ BatchGasLimit = 11000000 MaxDataBytes = 1000 InflightExpiry = '5m' RootSnooze = '5m' - ``` ### CCIP.Groups.[testgroup].StoreLaneConfig + This is only valid if the tests are run on remote runners in k8s. If set to true, the test will store the lane config in the remote runner. ### CCIP.Groups.[testgroup].LoadProfile -Specifies the load profile for the test. Only valid if the testgroup is 'load'. + +Specifies the load profile for the test. Only valid if the testgroup is 'load'. ### CCIP.Groups.[testgroup].LoadProfile.LoadFrequency.[DestNetworkName] #### CCIP.Groups.[testgroup].LoadProfile.RequestPerUnitTime + Specifies the number of requests to be sent per unit time. This is applicable to all networks if [LoadFrequency](#ccipgroupstestgrouploadprofileloadfrequencydestnetworkname) is not specified for a destination network. #### CCIP.Groups.[testgroup].LoadProfile.TimeUnit + Specifies the unit of time for the load profile. This is applicable to all networks if [LoadFrequency](#ccipgroupstestgrouploadprofileloadfrequencydestnetworkname) is not specified for a destination network. #### CCIP.Groups.[testgroup].LoadProfile.StepDuration + Specifies the duration for each step in the load profile. This is applicable to all networks if [LoadFrequency](#ccipgroupstestgrouploadprofileloadfrequencydestnetworkname) is not specified for a destination network. #### CCIP.Groups.[testgroup].LoadProfile.TestDuration + Specifies the total duration for the load test. #### CCIP.Groups.[testgroup].LoadProfile.NetworkChaosDelay + Specifies the duration network delay used for `NetworkChaos` experiment. This is only valid if the test is run on k8s and not on [existing deployments](#ccipgroupstestgroupexistingdeployment). #### CCIP.Groups.[testgroup].LoadProfile.WaitBetweenChaosDuringLoad + If there are multiple chaos experiments, this specifies the duration to wait between each chaos experiment. This is only valid if the test is run on k8s and not on [existing deployments](#ccipgroupstestgroupexistingdeployment). #### CCIP.Groups.[testgroup].LoadProfile.SkipRequestIfAnotherRequestTriggeredWithin + If a request is triggered within this duration, the test will skip sending another request during load run. For Example, if `SkipRequestIfAnotherRequestTriggeredWithin` is set to `40m`, and a request is triggered at 0th second, the test will skip sending another request for another 40m. This particular field is used to avoid sending multiple requests in a short duration during load run. #### CCIP.Groups.[testgroup].LoadProfile.OptimizeSpace -This is used internally to optimize memory usage during load run. If set to true, after the initial lane set up is over the test will discard the lane config to save memory. -The test will only store contract addresses strictly necessary to trigger/validate ccip-send requests. + +This is used internally to optimize memory usage during load run. If set to true, after the initial lane set up is over the test will discard the lane config to save memory. +The test will only store contract addresses strictly necessary to trigger/validate ccip-send requests. Except for following contracts all other contract addresses will be discarded after the initial lane set up - + - Router - ARM - CommitStore @@ -638,37 +757,45 @@ Except for following contracts all other contract addresses will be discarded af - OnRamp #### CCIP.Groups.[testgroup].LoadProfile.FailOnFirstErrorInLoad + If set to true, the test will fail on the first error encountered during load run. If set to false, the test will continue to run even if there are errors during load run. #### CCIP.Groups.[testgroup].LoadProfile.SendMaxDataInEveryMsgCount -Specifies the number of requests to send with maximum data in every mentioned count iteration. + +Specifies the number of requests to send with maximum data in every mentioned count iteration. For example, if `SendMaxDataInEveryMsgCount` is set to 5, the test will send ccip message with max allowable data length(as set in onRamp config) in every 5th request. #### CCIP.Groups.[testgroup].LoadProfile.TestRunName + Specifies the name of the test run. This is used to identify the test run in CCIP test dashboard or logs. If multiple tests are run with same `TestRunName`, the test results will be aggregated under the same test run in grafana dashboard. This is used when multiple iterations of tests are run against same release version to aggregate the results under same dashboard view. #### CCIP.Groups.[testgroup].LoadProfile.MsgProfile + Specifies the message profile for the test. The message profile is used to set up multiple ccip message details during load test. ##### CCIP.Groups.[testgroup].LoadProfile.MsgProfile.Frequencies -Specifies the frequency of each message profile. + +Specifies the frequency of each message profile. For example, if `Frequencies` is set to [1, 2, 3], the test will send 1st message profile 1 time, 2nd message profile 2 times, and 3rd message profile 3 times in each iteration. Each iteration will be defined by (1+2+3) = 6 requests. Example Breakdown: + - Frequencies = [4, 12, 3, 1] - Total Sum of Frequencies = 4 + 12 + 3 + 1 = 20 - Percentages: - - Message Type 1: (4 / 20) * 100% = 20% - - Message Type 2: (12 / 20) * 100% = 60% - - Message Type 3: (3 / 20) * 100% = 15% - - Message Type 4: (1 / 20) * 100% = 5% + - Message Type 1: (4 / 20) * 100% = 20% + - Message Type 2: (12 / 20) * 100% = 60% + - Message Type 3: (3 / 20) * 100% = 15% + - Message Type 4: (1 / 20) * 100% = 5% These percentages reflect how often each message type should appear in the total set of messages. Please note - if the total set of messages is not equal to the multiple of sum of frequencies, the percentages will not be accurate. ##### CCIP.Groups.[testgroup].LoadProfile.MsgProfile.MsgDetails + Specifies the message details for each message profile. The fields are the same as [CCIP.Groups.[testgroup].MsgDetails](#ccipgroupstestgroupmsgdetails). example usage: + ```toml # to represent 20%, 60%, 15%, 5% of the total messages [CCIP.Groups.load.LoadProfile.MsgProfile] diff --git a/integration-tests/ccip-tests/testconfig/ccip.go b/integration-tests/ccip-tests/testconfig/ccip.go index 60d7055cb31..7d9419828e9 100644 --- a/integration-tests/ccip-tests/testconfig/ccip.go +++ b/integration-tests/ccip-tests/testconfig/ccip.go @@ -387,6 +387,18 @@ type CCIP struct { Groups map[string]*CCIPTestGroupConfig `toml:",omitempty"` } +// LoadFromEnv loads selected env vars into the CCIP config +func (c *CCIP) LoadFromEnv() error { + if c.Env == nil { + c.Env = &Common{} + } + err := c.Env.ReadFromEnvVar() + if err != nil { + return err + } + return nil +} + func (c *CCIP) Validate() error { if c.Env != nil { err := c.Env.Validate() diff --git a/integration-tests/ccip-tests/testconfig/examples/.testsecrets.example b/integration-tests/ccip-tests/testconfig/examples/.testsecrets.example new file mode 100644 index 00000000000..52afa77015e --- /dev/null +++ b/integration-tests/ccip-tests/testconfig/examples/.testsecrets.example @@ -0,0 +1,39 @@ +# DO NOT UPDATE THIS FILE WITH ANY SECRET VALUES. +# This file serves as a template for the actual ~/.testsecrets file. Follow these steps to use it: +# 1. Create a copy of this template in your home directory under ~/.testsecrets +# 2. Update ~/.testsecrets with actual secret values required for your tests. The file will be automatically loaded by the test framework +# 3. Only include secrets necessary for the tests you are running. For example, if you are only running tests on Ethereum Mainnet, you do not need secrets for Base Mainnet. Comment other env vars. +# DO NOT COMMIT THE ACTUAL SECRETS FILE TO THE REPOSITORY. + +# Chainlink image used for NewCLCluster +E2E_TEST_CHAINLINK_IMAGE="***.dkr.ecr.***.amazonaws.com/chainlink-ccip" + +# Chainlink upgrade image for NewCLCluster. Used only for upgrade tests +E2E_TEST_CHAINLINK_UPGRADE_IMAGE="***.dkr.ecr.***.amazonaws.com/chainlink-ccip" + +# Ethereum network secrets +E2E_TEST_ETHEREUM_MAINNET_WALLET_KEY="" +E2E_TEST_ETHEREUM_MAINNET_RPC_HTTP_URL="" +E2E_TEST_ETHEREUM_MAINNET_RPC_HTTP_URL_1="" +E2E_TEST_ETHEREUM_MAINNET_RPC_WS_URL="" +E2E_TEST_ETHEREUM_MAINNET_RPC_WS_URL_1="" + +# Base network secrets +E2E_TEST_BASE_MAINNET_WALLET_KEY="" +E2E_TEST_BASE_MAINNET_RPC_HTTP_URL="" +E2E_TEST_BASE_MAINNET_RPC_HTTP_URL_1="" +E2E_TEST_BASE_MAINNET_RPC_WS_URL="" +E2E_TEST_BASE_MAINNET_RPC_WS_URL_1="" + +# Other networks secrets (pattern for envs) +# E2E_TEST_(.+)_WALLET_KEY_(\d+)="" +# E2E_TEST_(.+)_RPC_HTTP_URL_(\d+)="" +# E2E_TEST_(.+)_RPC_WS_URL_(\d+)="" + +# Loki secrets +E2E_TEST_LOKI_TENANT_ID="" +E2E_TEST_LOKI_ENDPOINT="" + +# Grafana secrets +E2E_TEST_GRAFANA_BASE_URL="" +E2E_TEST_GRAFANA_DASHBOARD_URL="/d/6vjVx-1V8/ccip-long-running-tests" \ No newline at end of file diff --git a/integration-tests/ccip-tests/testconfig/examples/secrets.toml.example b/integration-tests/ccip-tests/testconfig/examples/secrets.toml.example deleted file mode 100644 index 3045f51759d..00000000000 --- a/integration-tests/ccip-tests/testconfig/examples/secrets.toml.example +++ /dev/null @@ -1,52 +0,0 @@ -# This file contains all secret parameters for ccip tests. -# DO NOT UPDATE THIS FILE WITH ANY SECRET VALUES. -# Use this file as a template for the actual secret file and update all the parameter values accordingly. -# DO NOT COMMIT THE ACTUAL SECRET FILE TO THE REPOSITORY. -[CCIP] -[CCIP.Env] - -# ChainlinkImage is mandatory for all tests. -[CCIP.Env.NewCLCluster] -[CCIP.Env.NewCLCluster.Common] -[CCIP.Env.NewCLCluster.Common.ChainlinkImage] -image = "chainlink-ccip" -version = "latest" - -# Chainlink upgrade image is used only for upgrade tests -#[CCIP.Env.NewCLCluster.Common.ChainlinkUpgradeImage] -#image = "***.dkr.ecr.***.amazonaws.com/chainlink-ccip" -#version = "****" - - -# Networks configuration with rpc urls and wallet keys are mandatory only for tests running on live networks -# The following example is for 3 networks: Ethereum, Base and Arbitrum -# Network configuration can be ignored for tests running on simulated/private networks -[CCIP.Env.Network] -selected_networks= [ - 'ETHEREUM_MAINNET','BASE_MAINNET', 'ARBITRUM_MAINNET', -] - -[CCIP.Env.Network.RpcHttpUrls] -ETHEREUM_MAINNET = ['', ''] -BASE_MAINNET = ['', ''] -ARBITRUM_MAINNET = ['', ''] - -[CCIP.Env.Network.RpcWsUrls] -ETHEREUM_MAINNET = ['', ''] -BASE_MAINNET = ['', ''] -ARBITRUM_MAINNET = ['', ''] - -[CCIP.Env.Network.WalletKeys] -ETHEREUM_MAINNET = [''] -BASE_MAINNET = [''] -ARBITRUM_MAINNET = [''] - -# Used for tests using 1. loki logging for test results. -# Mandatory for load tests -[CCIP.Env.Logging.Loki] -tenant_id="" -endpoint="" - -[CCIP.Env.Logging.Grafana] -base_url="" -dashboard_url="/d/6vjVx-1V8/ccip-long-running-tests" diff --git a/integration-tests/ccip-tests/testconfig/global.go b/integration-tests/ccip-tests/testconfig/global.go index 331737c5fbf..a1658a48416 100644 --- a/integration-tests/ccip-tests/testconfig/global.go +++ b/integration-tests/ccip-tests/testconfig/global.go @@ -15,6 +15,7 @@ import ( "github.com/smartcontractkit/seth" "github.com/smartcontractkit/chainlink-testing-framework/docker/test_env" + "github.com/smartcontractkit/chainlink-testing-framework/logging" "github.com/smartcontractkit/chainlink-testing-framework/networks" "github.com/smartcontractkit/chainlink-testing-framework/blockchain" @@ -29,7 +30,6 @@ import ( const ( OVERIDECONFIG = "BASE64_CCIP_CONFIG_OVERRIDE" - SECRETSCONFIG = "BASE64_CCIP_SECRETS_CONFIG" ErrReadConfig = "failed to read TOML config" ErrUnmarshalConfig = "failed to unmarshal TOML config" Load string = "load" @@ -105,52 +105,46 @@ func EncodeConfigAndSetEnv(c any, envVar string) (string, error) { func NewConfig() (*Config, error) { cfg := &Config{} var override *Config - var secrets *Config + // var secrets *Config // load config from default file err := config.DecodeTOML(bytes.NewReader(DefaultConfig), cfg) if err != nil { return nil, errors.Wrap(err, ErrReadConfig) } - // load config from env var if specified - rawConfig, _ := osutil.GetEnv(OVERIDECONFIG) - if rawConfig != "" { - err = DecodeConfig(rawConfig, &override) - if err != nil { - return nil, fmt.Errorf("failed to decode override config: %w", err) - } - } - if override != nil { - // apply overrides for all products - if override.CCIP != nil { - if cfg.CCIP == nil { - cfg.CCIP = override.CCIP - } else { - err = cfg.CCIP.ApplyOverrides(override.CCIP) - if err != nil { - return nil, err + // load config overrides from env var if specified + // there can be multiple overrides separated by comma + rawConfigs, _ := osutil.GetEnv(OVERIDECONFIG) + if rawConfigs != "" { + for _, rawConfig := range strings.Split(rawConfigs, ",") { + err = DecodeConfig(rawConfig, &override) + if err != nil { + return nil, fmt.Errorf("failed to decode override config: %w", err) + } + if override != nil { + // apply overrides for all products + if override.CCIP != nil { + if cfg.CCIP == nil { + cfg.CCIP = override.CCIP + } else { + err = cfg.CCIP.ApplyOverrides(override.CCIP) + if err != nil { + return nil, err + } + } } } } } // read secrets for all products if cfg.CCIP != nil { - // load config from env var if specified for secrets - secretRawConfig, _ := osutil.GetEnv(SECRETSCONFIG) - if secretRawConfig != "" { - err = DecodeConfig(secretRawConfig, &secrets) - if err != nil { - return nil, fmt.Errorf("failed to decode secrets config: %w", err) - } - if secrets != nil { - // apply secrets for all products - if secrets.CCIP != nil { - err = cfg.CCIP.ApplyOverrides(secrets.CCIP) - if err != nil { - return nil, fmt.Errorf("failed to apply secrets: %w", err) - } - } - } + err := ctfconfig.LoadSecretEnvsFromFiles() + if err != nil { + return nil, errors.Wrap(err, "error loading testsecrets files") + } + err = cfg.CCIP.LoadFromEnv() + if err != nil { + return nil, errors.Wrap(err, "error loading env vars into CCIP config") } // validate all products err = cfg.CCIP.Validate() @@ -176,6 +170,156 @@ type Common struct { Logging *ctfconfig.LoggingConfig `toml:",omitempty"` } +// ReadFromEnvVar loads selected env vars into the config +func (p *Common) ReadFromEnvVar() error { + logger := logging.GetTestLogger(nil) + + lokiTenantID := ctfconfig.MustReadEnvVar_String(ctfconfig.E2E_TEST_LOKI_TENANT_ID_ENV) + if lokiTenantID != "" { + if p.Logging == nil { + p.Logging = &ctfconfig.LoggingConfig{} + } + if p.Logging.Loki == nil { + p.Logging.Loki = &ctfconfig.LokiConfig{} + } + logger.Debug().Msgf("Using %s env var to override Logging.Loki.TenantId", ctfconfig.E2E_TEST_LOKI_TENANT_ID_ENV) + p.Logging.Loki.TenantId = &lokiTenantID + } + + lokiEndpoint := ctfconfig.MustReadEnvVar_String(ctfconfig.E2E_TEST_LOKI_ENDPOINT_ENV) + if lokiEndpoint != "" { + if p.Logging == nil { + p.Logging = &ctfconfig.LoggingConfig{} + } + if p.Logging.Loki == nil { + p.Logging.Loki = &ctfconfig.LokiConfig{} + } + logger.Debug().Msgf("Using %s env var to override Logging.Loki.Endpoint", ctfconfig.E2E_TEST_LOKI_ENDPOINT_ENV) + p.Logging.Loki.Endpoint = &lokiEndpoint + } + + lokiBasicAuth := ctfconfig.MustReadEnvVar_String(ctfconfig.E2E_TEST_LOKI_BASIC_AUTH_ENV) + if lokiBasicAuth != "" { + if p.Logging == nil { + p.Logging = &ctfconfig.LoggingConfig{} + } + if p.Logging.Loki == nil { + p.Logging.Loki = &ctfconfig.LokiConfig{} + } + logger.Debug().Msgf("Using %s env var to override Logging.Loki.BasicAuth", ctfconfig.E2E_TEST_LOKI_BASIC_AUTH_ENV) + p.Logging.Loki.BasicAuth = &lokiBasicAuth + } + + lokiBearerToken := ctfconfig.MustReadEnvVar_String(ctfconfig.E2E_TEST_LOKI_BEARER_TOKEN_ENV) + if lokiBearerToken != "" { + if p.Logging == nil { + p.Logging = &ctfconfig.LoggingConfig{} + } + if p.Logging.Loki == nil { + p.Logging.Loki = &ctfconfig.LokiConfig{} + } + logger.Debug().Msgf("Using %s env var to override Logging.Loki.BearerToken", ctfconfig.E2E_TEST_LOKI_BEARER_TOKEN_ENV) + p.Logging.Loki.BearerToken = &lokiBearerToken + } + + grafanaBaseUrl := ctfconfig.MustReadEnvVar_String(ctfconfig.E2E_TEST_GRAFANA_BASE_URL_ENV) + if grafanaBaseUrl != "" { + if p.Logging == nil { + p.Logging = &ctfconfig.LoggingConfig{} + } + if p.Logging.Grafana == nil { + p.Logging.Grafana = &ctfconfig.GrafanaConfig{} + } + logger.Debug().Msgf("Using %s env var to override Logging.Grafana.BaseUrl", ctfconfig.E2E_TEST_GRAFANA_BASE_URL_ENV) + p.Logging.Grafana.BaseUrl = &grafanaBaseUrl + } + + grafanaDashboardUrl := ctfconfig.MustReadEnvVar_String(ctfconfig.E2E_TEST_GRAFANA_DASHBOARD_URL_ENV) + if grafanaDashboardUrl != "" { + if p.Logging == nil { + p.Logging = &ctfconfig.LoggingConfig{} + } + if p.Logging.Grafana == nil { + p.Logging.Grafana = &ctfconfig.GrafanaConfig{} + } + logger.Debug().Msgf("Using %s env var to override Logging.Grafana.DashboardUrl", ctfconfig.E2E_TEST_GRAFANA_DASHBOARD_URL_ENV) + p.Logging.Grafana.DashboardUrl = &grafanaDashboardUrl + } + + grafanaBearerToken := ctfconfig.MustReadEnvVar_String(ctfconfig.E2E_TEST_GRAFANA_BEARER_TOKEN_ENV) + if grafanaBearerToken != "" { + if p.Logging == nil { + p.Logging = &ctfconfig.LoggingConfig{} + } + if p.Logging.Grafana == nil { + p.Logging.Grafana = &ctfconfig.GrafanaConfig{} + } + logger.Debug().Msgf("Using %s env var to override Logging.Grafana.BearerToken", ctfconfig.E2E_TEST_GRAFANA_BEARER_TOKEN_ENV) + p.Logging.Grafana.BearerToken = &grafanaBearerToken + } + + walletKeys := ctfconfig.ReadEnvVarGroupedMap(ctfconfig.E2E_TEST_WALLET_KEY_ENV, ctfconfig.E2E_TEST_WALLET_KEYS_ENV) + if len(walletKeys) > 0 { + if p.Network == nil { + p.Network = &ctfconfig.NetworkConfig{} + } + logger.Debug().Msgf("Using %s and/or %s env vars to override Network.WalletKeys", ctfconfig.E2E_TEST_WALLET_KEY_ENV, ctfconfig.E2E_TEST_WALLET_KEYS_ENV) + p.Network.WalletKeys = walletKeys + } + + rpcHttpUrls := ctfconfig.ReadEnvVarGroupedMap(ctfconfig.E2E_TEST_RPC_HTTP_URL_ENV, ctfconfig.E2E_TEST_RPC_HTTP_URLS_ENV) + if len(rpcHttpUrls) > 0 { + if p.Network == nil { + p.Network = &ctfconfig.NetworkConfig{} + } + logger.Debug().Msgf("Using %s and/or %s env vars to override Network.RpcHttpUrls", ctfconfig.E2E_TEST_RPC_HTTP_URL_ENV, ctfconfig.E2E_TEST_RPC_HTTP_URLS_ENV) + p.Network.RpcHttpUrls = rpcHttpUrls + } + + rpcWsUrls := ctfconfig.ReadEnvVarGroupedMap(ctfconfig.E2E_TEST_RPC_WS_URL_ENV, ctfconfig.E2E_TEST_RPC_WS_URLS_ENV) + if len(rpcWsUrls) > 0 { + if p.Network == nil { + p.Network = &ctfconfig.NetworkConfig{} + } + logger.Debug().Msgf("Using %s and/or %s env vars to override Network.RpcWsUrls", ctfconfig.E2E_TEST_RPC_WS_URL_ENV, ctfconfig.E2E_TEST_RPC_WS_URLS_ENV) + p.Network.RpcWsUrls = rpcWsUrls + } + + chainlinkImage := ctfconfig.MustReadEnvVar_String(ctfconfig.E2E_TEST_CHAINLINK_IMAGE_ENV) + if chainlinkImage != "" { + if p.NewCLCluster == nil { + p.NewCLCluster = &ChainlinkDeployment{} + } + if p.NewCLCluster.Common == nil { + p.NewCLCluster.Common = &Node{} + } + if p.NewCLCluster.Common.ChainlinkImage == nil { + p.NewCLCluster.Common.ChainlinkImage = &ctfconfig.ChainlinkImageConfig{} + } + + logger.Debug().Msgf("Using %s env var to override NewCLCluster.Common.ChainlinkImage.Image", ctfconfig.E2E_TEST_CHAINLINK_IMAGE_ENV) + p.NewCLCluster.Common.ChainlinkImage.Image = &chainlinkImage + } + + chainlinkUpgradeImage := ctfconfig.MustReadEnvVar_String(ctfconfig.E2E_TEST_CHAINLINK_UPGRADE_IMAGE_ENV) + if chainlinkUpgradeImage != "" { + if p.NewCLCluster == nil { + p.NewCLCluster = &ChainlinkDeployment{} + } + if p.NewCLCluster.Common == nil { + p.NewCLCluster.Common = &Node{} + } + if p.NewCLCluster.Common.ChainlinkUpgradeImage == nil { + p.NewCLCluster.Common.ChainlinkUpgradeImage = &ctfconfig.ChainlinkImageConfig{} + } + + logger.Debug().Msgf("Using %s env var to override NewCLCluster.Common.ChainlinkUpgradeImage.Image", ctfconfig.E2E_TEST_CHAINLINK_UPGRADE_IMAGE_ENV) + p.NewCLCluster.Common.ChainlinkUpgradeImage.Image = &chainlinkUpgradeImage + } + + return nil +} + func (p *Common) GetNodeConfig() *ctfconfig.NodeConfig { return &ctfconfig.NodeConfig{ BaseConfigTOML: p.NewCLCluster.Common.BaseConfigTOML,