From cf949615865bfaa3c99734d3c250a4058ff7b0b3 Mon Sep 17 00:00:00 2001 From: Lukasz <120112546+lukaszcl@users.noreply.github.com> Date: Tue, 9 Jul 2024 14:08:51 +0200 Subject: [PATCH] Load E2E test secrets from env vars (#13762) * Load test config secrets from env vars * Remove secrets from test config in automation-ondemand-tests.yml * Pass config override * Update on-demand-vrfv2plus-eth2-clients-test.yml to accept test_secrets_override_key * Bump gha * bump ctf * Use debug in automation workflow test log * Update input name * Fix integration-tests.yml * Override with env vars after overriding with test config * Clean up setup-create-base64-config action * bump ctf * bump ctf and gha * Update env var names to make it easier to set by QA * Add .secrets to gitignore * Load secrets from testconfig/.secrets * Fix MissingImageInfoAsError * Load secrets from env file in home dir * Add checkSecretsInToml * Update checkSecretsInToml * Update checkSecretsInToml * Revert on-demand-vrfv2-eth2-clients-test.yml * Revert on-demand-vrfv2plus-eth2-clients-test.yml * Fix go mod * Fix * Bump run-tests gha * bump ctf * test * bump gha and ctf * Fix workflows that use new GHA * Fix * Fix lint * Fix log level * Use E2E_TEST_ prefix to forward env vars to remote runner * Add docs * Fix and bump * bump ctf and gha --- .../setup-create-base64-config/action.yml | 55 ------------ .../workflows/automation-ondemand-tests.yml | 79 +++++++--------- .../workflows/client-compatibility-tests.yml | 24 ++--- .github/workflows/integration-tests.yml | 90 +++++++++++-------- .../on-demand-keeper-smoke-tests.yml | 22 ++--- .../on-demand-vrfv2-eth2-clients-test.yml | 9 +- .../on-demand-vrfv2plus-eth2-clients-test.yml | 9 +- integration-tests/go.mod | 3 +- integration-tests/go.sum | 8 +- integration-tests/load/go.mod | 3 +- integration-tests/load/go.sum | 8 +- integration-tests/testconfig/README.md | 22 ++--- integration-tests/testconfig/testconfig.go | 68 +++++++++++++- .../testconfig/testconfig_utils.go | 31 +++---- 14 files changed, 219 insertions(+), 212 deletions(-) diff --git a/.github/actions/setup-create-base64-config/action.yml b/.github/actions/setup-create-base64-config/action.yml index d54554278d1..a5e7c8f0b4d 100644 --- a/.github/actions/setup-create-base64-config/action.yml +++ b/.github/actions/setup-create-base64-config/action.yml @@ -9,34 +9,13 @@ inputs: default: "false" selectedNetworks: description: The networks to run tests against - chainlinkImage: - description: The chainlink image to use - default: "public.ecr.aws/chainlink/chainlink" chainlinkPostgresVersion: description: The postgres version to use with the chainlink node default: "15.6" chainlinkVersion: description: The git commit sha to use for the image tag - pyroscopeServer: - description: URL of Pyroscope server - pyroscopeEnvironment: - description: Name of Pyroscope environment - pyroscopeKey: - description: Pyroscope server key - lokiEndpoint: - description: Loki push endpoint - lokiTenantId: - description: Loki tenant id - lokiBasicAuth: - description: Loki basic auth logstreamLogTargets: description: Where to send logs (e.g. file, loki) - grafanaUrl: - description: Grafana URL - grafanaDashboardUrl: - description: Grafana dashboard URL - grafanaBearerToken: - description: Grafana bearer token ethExecutionClient: description: Ethereum execution client to use (geth, besu, nethermind or erigon) customEthClientDockerImage: @@ -52,23 +31,12 @@ runs: RUN_ID: ${{ inputs.runId }} TEST_LOG_COLLECT: ${{ inputs.testLogCollect }} SELECTED_NETWORKS: ${{ inputs.selectedNetworks }} - PYROSCOPE_SERVER: ${{ inputs.pyroscopeServer }} - PYROSCOPE_ENVIRONMENT: ${{ inputs.pyroscopeEnvironment }} - PYROSCOPE_KEY: ${{ inputs.pyroscopeKey }} - CHAINLINK_IMAGE: ${{ inputs.chainlinkImage }} CHAINLINK_VERSION: ${{ inputs.chainlinkVersion }} CHAINLINK_POSTGRES_VERSION: ${{ inputs.chainlinkPostgresVersion }} - LOKI_ENDPOINT: ${{ inputs.lokiEndpoint }} - LOKI_TENANT_ID: ${{ inputs.lokiTenantId }} - LOKI_BASIC_AUTH: ${{ inputs.lokiBasicAuth }} LOGSTREAM_LOG_TARGETS: ${{ inputs.logstreamLogTargets }} - GRAFANA_URL: ${{ inputs.grafanaUrl }} - GRAFANA_DASHBOARD_URL: ${{ inputs.grafanaDashboardUrl }} - GRAFANA_BEARER_TOKEN: ${{ inputs.grafanaBearerToken }} ETH_EXECUTION_CLIENT: ${{ inputs.ethExecutionClient }} CUSTOM_ETH_CLIENT_DOCKER_IMAGE: ${{ inputs.customEthClientDockerImage }} run: | - echo ::add-mask::$CHAINLINK_IMAGE function convert_to_toml_array() { local IFS=',' local input_array=($1) @@ -85,12 +53,6 @@ runs: selected_networks=$(convert_to_toml_array "$SELECTED_NETWORKS") log_targets=$(convert_to_toml_array "$LOGSTREAM_LOG_TARGETS") - if [ -n "$PYROSCOPE_SERVER" ]; then - pyroscope_enabled=true - else - pyroscope_enabled=false - fi - if [ -n "$TEST_LOG_COLLECT" ]; then test_log_collect=true else @@ -122,16 +84,9 @@ runs: selected_networks=$selected_networks [ChainlinkImage] - image="$CHAINLINK_IMAGE" version="$CHAINLINK_VERSION" postgres_version="$CHAINLINK_POSTGRES_VERSION" - [Pyroscope] - enabled=$pyroscope_enabled - server_url="$PYROSCOPE_SERVER" - environment="$PYROSCOPE_ENVIRONMENT" - key_secret="$PYROSCOPE_KEY" - [Logging] test_log_collect=$test_log_collect run_id="$RUN_ID" @@ -139,16 +94,6 @@ runs: [Logging.LogStream] log_targets=$log_targets - [Logging.Loki] - tenant_id="$LOKI_TENANT_ID" - endpoint="$LOKI_ENDPOINT" - basic_auth_secret="$LOKI_BASIC_AUTH" - - [Logging.Grafana] - base_url="$GRAFANA_URL" - dashboard_url="$GRAFANA_DASHBOARD_URL" - $grafana_bearer_token - [PrivateEthereumNetwork] execution_layer="$execution_layer" $ethereum_version diff --git a/.github/workflows/automation-ondemand-tests.yml b/.github/workflows/automation-ondemand-tests.yml index 741603eddbe..4f147f7f4ed 100644 --- a/.github/workflows/automation-ondemand-tests.yml +++ b/.github/workflows/automation-ondemand-tests.yml @@ -8,8 +8,10 @@ on: type: string chainlinkImage: description: Chainlink image repo to use (Leave empty to build from head/ref) - required: false - type: string + options: + - public.ecr.aws/chainlink/chainlink + - QA_ECR + type: choice chainlinkVersionUpdate: description: Chainlink image version to use initially for upgrade test default: latest @@ -18,8 +20,10 @@ on: chainlinkImageUpdate: description: Chainlink image repo to use initially for upgrade test required: true - default: public.ecr.aws/chainlink/chainlink - type: string + options: + - public.ecr.aws/chainlink/chainlink + - QA_ECR + type: choice enableChaos: description: Check to enable chaos tests type: boolean @@ -198,23 +202,24 @@ jobs: id: determine-build shell: bash run: | - if [[ "${{ inputs.chainlinkImage }}" == "" ]]; then + if [[ "${{ inputs.chainlinkImage }}" == "QA_ECR" ]]; then echo "image=${{ env.CHAINLINK_IMAGE }}" >>$GITHUB_OUTPUT echo "version=${{ github.sha }}" >>$GITHUB_OUTPUT echo "upgrade_version=${{ github.sha }}" >>$GITHUB_OUTPUT echo "upgrade_image=${{ env.CHAINLINK_IMAGE }}" >>$GITHUB_OUTPUT - else - READ_CL_IMAGE=$(jq -r '.inputs.chainlinkImage' $GITHUB_EVENT_PATH) - echo ::add-mask::$READ_CL_IMAGE + elif [[ "${{ inputs.chainlinkImage }}" == "public.ecr.aws/chainlink/chainlink" ]]; then + READ_CL_IMAGE="public.ecr.aws/chainlink/chainlink" echo "image=$READ_CL_IMAGE" >>$GITHUB_OUTPUT echo "version=${{ inputs.chainlinkVersion }}" >>$GITHUB_OUTPUT echo "upgrade_version=${{ inputs.chainlinkVersion }}" >>$GITHUB_OUTPUT echo "upgrade_image=$READ_CL_IMAGE" >>$GITHUB_OUTPUT fi if [[ "${{ matrix.tests.type }}" == "upgrade" ]]; then - READ_CL_UPGR_IMAGE=$(jq -r '.inputs.chainlinkImageUpdate' $GITHUB_EVENT_PATH) - echo ::add-mask::$READ_CL_UPGR_IMAGE - echo "image=$READ_CL_UPGR_IMAGE" >>$GITHUB_OUTPUT + if [[ "${{ inputs.chainlinkImageUpdate }}" == "QA_ECR" ]]; then + echo "image=${{ env.CHAINLINK_IMAGE }}" >>$GITHUB_OUTPUT + elif [[ "${{ inputs.chainlinkImageUpdate }}" == "public.ecr.aws/chainlink/chainlink" ]]; then + echo "image=public.ecr.aws/chainlink/chainlink" >>$GITHUB_OUTPUT + fi echo "version=${{ inputs.chainlinkVersionUpdate }}" >>$GITHUB_OUTPUT fi - name: Setup GAP for Grafana @@ -233,18 +238,9 @@ jobs: OLD_VERSION: ${{ steps.determine-build.outputs.version }} UPGRADE_VERSION: ${{ steps.determine-build.outputs.upgrade_version }} UPGRADE_IMAGE: ${{ steps.determine-build.outputs.upgrade_image }} - PYROSCOPE_SERVER: ${{ matrix.tests.pyroscope_env == '' && '' || !startsWith(github.ref, 'refs/tags/') && '' || secrets.QA_PYROSCOPE_INSTANCE }} # Avoid sending blank envs https://github.com/orgs/community/discussions/25725 - PYROSCOPE_ENVIRONMENT: ${{ matrix.tests.pyroscope_env }} - PYROSCOPE_KEY: ${{ secrets.QA_PYROSCOPE_KEY }} RUN_ID: ${{ github.run_id }} TEST_LOG_COLLECT: "true" - LOKI_ENDPOINT: https://${{ secrets.GRAFANA_INTERNAL_HOST }}/loki/api/v1/push - LOKI_TENANT_ID: ${{ secrets.GRAFANA_INTERNAL_TENANT_ID }} - LOKI_BASIC_AUTH: ${{ secrets.GRAFANA_INTERNAL_BASIC_AUTH }} LOGSTREAM_LOG_TARGETS: ${{ vars.LOGSTREAM_LOG_TARGETS }} - GRAFANA_URL: "http://localhost:8080/primary" - GRAFANA_DASHBOARD_URL: "/d/ddf75041-1e39-42af-aa46-361fe4c36e9e/ci-e2e-tests-logs" - $GRAFANA_BEARER_TOKEN: ${{ secrets.GRAFANA_INTERNAL_URL_SHORTENER_TOKEN }} run: | echo ::add-mask::$UPGRADE_IMAGE echo ::add-mask::$OLD_IMAGE @@ -254,12 +250,6 @@ jobs: selected_networks=$(convert_to_toml_array "$SELECTED_NETWORKS") - if [ -n "$PYROSCOPE_SERVER" ]; then - pyroscope_enabled=true - else - pyroscope_enabled=false - fi - if [ -n "$TEST_LOG_COLLECT" ]; then test_log_collect=true else @@ -268,11 +258,6 @@ jobs: log_targets=$(convert_to_toml_array "$LOGSTREAM_LOG_TARGETS") - grafana_bearer_token="" - if [ -n "$GRAFANA_BEARER_TOKEN" ]; then - grafana_bearer_token="bearer_token_secret=\"$GRAFANA_BEARER_TOKEN\"" - fi - cat << EOF > config.toml [Network] selected_networks=$selected_networks @@ -291,33 +276,18 @@ jobs: [Logging.LogStream] log_targets=$log_targets - - [Logging.Loki] - tenant_id="$LOKI_TENANT_ID" - endpoint="$LOKI_ENDPOINT" - basic_auth_secret="$LOKI_BASIC_AUTH" - - [Logging.Grafana] - base_url="$GRAFANA_URL" - dashboard_url="$GRAFANA_DASHBOARD_URL" - $grafana_bearer_token - - [Pyroscope] - enabled=$pyroscope_enabled - server_url="$PYROSCOPE_SERVER" - environment="$PYROSCOPE_ENVIRONMENT" - key_secret="$PYROSCOPE_KEY" EOF - + BASE64_CONFIG_OVERRIDE=$(cat config.toml | base64 -w 0) echo ::add-mask::$BASE64_CONFIG_OVERRIDE echo "BASE64_CONFIG_OVERRIDE=$BASE64_CONFIG_OVERRIDE" >> $GITHUB_ENV - name: Run Tests - uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/run-tests@af92c5fae8dcf1659201e907db82d221fc304b94 # v2.3.21 + uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/run-tests@75a9005952a9e905649cfb5a6971fd9429436acd # v2.3.24 if: ${{ matrix.tests.enabled == true }} env: TEST_SUITE: ${{ matrix.tests.suite }} with: + test_config_override_base64: ${{ env.BASE64_CONFIG_OVERRIDE }} test_command_to_run: cd ./integration-tests && go test -timeout 60m -count=1 -json -test.parallel=${{ matrix.tests.nodes }} ${{ matrix.tests.command }} 2>&1 | tee /tmp/gotest.log | gotestloghelper -ci -singlepackage -hidepassingtests=false -hidepassinglogs test_download_vendor_packages_command: cd ./integration-tests && go mod download cl_repo: ${{ steps.determine-build.outputs.image }} @@ -330,6 +300,17 @@ jobs: QA_AWS_REGION: ${{ secrets.QA_AWS_REGION }} QA_AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }} QA_KUBECONFIG: ${{ secrets.QA_KUBECONFIG }} + DEFAULT_LOKI_TENANT_ID: ${{ secrets.GRAFANA_INTERNAL_TENANT_ID }} + DEFAULT_LOKI_ENDPOINT: https://${{ secrets.GRAFANA_INTERNAL_HOST }}/loki/api/v1/push + DEFAULT_LOKI_BASIC_AUTH: ${{ secrets.GRAFANA_INTERNAL_BASIC_AUTH }} + DEFAULT_GRAFANA_BASE_URL: "http://localhost:8080/primary" + DEFAULT_GRAFANA_DASHBOARD_URL: "/d/ddf75041-1e39-42af-aa46-361fe4c36e9e/ci-e2e-tests-logs" + DEFAULT_GRAFANA_BEARER_TOKEN: ${{ secrets.GRAFANA_INTERNAL_URL_SHORTENER_TOKEN }} + DEFAULT_PYROSCOPE_SERVER_URL: ${{ matrix.tests.pyroscope_env == '' && '' || !startsWith(github.ref, 'refs/tags/') && '' || secrets.QA_PYROSCOPE_INSTANCE }} # Avoid sending blank envs https://github.com/orgs/community/discussions/25725 + DEFAULT_PYROSCOPE_KEY: ${{ secrets.QA_PYROSCOPE_KEY }} + DEFAULT_PYROSCOPE_ENVIRONMENT: ${{ matrix.tests.pyroscope_env }} + DEFAULT_PYROSCOPE_ENABLED: ${{ matrix.tests.pyroscope_env == '' || !startsWith(github.ref, 'refs/tags/') && 'false' || 'true' }} + - name: Upload test log uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # v4.3.1 if: failure() diff --git a/.github/workflows/client-compatibility-tests.yml b/.github/workflows/client-compatibility-tests.yml index b3bbc115de5..b1b9c669cf6 100644 --- a/.github/workflows/client-compatibility-tests.yml +++ b/.github/workflows/client-compatibility-tests.yml @@ -544,20 +544,10 @@ jobs: runId: ${{ github.run_id }} testLogCollect: ${{ vars.TEST_LOG_COLLECT }} selectedNetworks: ${{ env.SELECTED_NETWORKS }} - chainlinkImage: ${{ env.CHAINLINK_IMAGE }} chainlinkVersion: ${{ needs.select-versions.outputs.chainlink_version }} - lokiEndpoint: https://${{ secrets.GRAFANA_INTERNAL_HOST }}/loki/api/v1/push - lokiTenantId: ${{ secrets.GRAFANA_INTERNAL_TENANT_ID }} - lokiBasicAuth: ${{ secrets.GRAFANA_INTERNAL_BASIC_AUTH }} logstreamLogTargets: ${{ vars.LOGSTREAM_LOG_TARGETS }} - grafanaUrl: "http://localhost:8080/primary" - grafanaDashboardUrl: "/d/ddf75041-1e39-42af-aa46-361fe4c36e9e/ci-e2e-tests-logs" - grafanaBearerToken: ${{ secrets.GRAFANA_INTERNAL_URL_SHORTENER_TOKEN }} ethExecutionClient: ${{ matrix.evm_node.eth_implementation }} customEthClientDockerImage: ${{ matrix.evm_node.docker_image }} - pyroscopeServer: ${{ secrets.QA_PYROSCOPE_INSTANCE }} - pyroscopeEnvironment: ci-client-compatability-${{ matrix.eth_client }}-testnet - pyroscopeKey: ${{ secrets.QA_PYROSCOPE_KEY }} - name: Prepare test log name run: | replace_special_chars() { @@ -581,7 +571,7 @@ jobs: } echo "TEST_LOG_NAME=$(replace_special_chars "${{ matrix.evm_node.product }}-${{ matrix.evm_node.docker_image }}-test-logs")" >> $GITHUB_ENV - name: Run Tests - uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/run-tests@af92c5fae8dcf1659201e907db82d221fc304b94 # v2.3.21 + uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/run-tests@75a9005952a9e905649cfb5a6971fd9429436acd # v2.3.24 with: test_command_to_run: cd ./integration-tests && touch .root_dir && go test -timeout 30m -count=1 -json ${{ matrix.evm_node.run }} 2>&1 | tee /tmp/gotest.log | gotestloghelper -ci -singlepackage -hidepassingtests=false -hidepassinglogs test_download_vendor_packages_command: cd ./integration-tests && go mod download @@ -603,6 +593,18 @@ jobs: should_tidy: "false" go_coverage_src_dir: /var/tmp/go-coverage go_coverage_dest_dir: ${{ github.workspace }}/.covdata + DEFAULT_CHAINLINK_IMAGE: ${{ env.CHAINLINK_IMAGE }} + DEFAULT_LOKI_TENANT_ID: ${{ secrets.GRAFANA_INTERNAL_TENANT_ID }} + DEFAULT_LOKI_ENDPOINT: https://${{ secrets.GRAFANA_INTERNAL_HOST }}/loki/api/v1/push + DEFAULT_LOKI_BASIC_AUTH: ${{ secrets.GRAFANA_INTERNAL_BASIC_AUTH }} + DEFAULT_GRAFANA_BASE_URL: "http://localhost:8080/primary" + DEFAULT_GRAFANA_DASHBOARD_URL: "/d/ddf75041-1e39-42af-aa46-361fe4c36e9e/ci-e2e-tests-logs" + DEFAULT_GRAFANA_BEARER_TOKEN: ${{ secrets.GRAFANA_INTERNAL_URL_SHORTENER_TOKEN }} + DEFAULT_PYROSCOPE_SERVER_URL: ${{ secrets.QA_PYROSCOPE_INSTANCE }} + DEFAULT_PYROSCOPE_KEY: ${{ secrets.QA_PYROSCOPE_KEY }} + DEFAULT_PYROSCOPE_ENVIRONMENT: ci-client-compatability-${{ matrix.eth_client }}-testnet + DEFAULT_PYROSCOPE_ENABLED: "true" + - name: Print failed test summary if: always() uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/show-test-summary@fc3e0df622521019f50d772726d6bf8dc919dd38 # v2.3.19 diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml index 82eca4294a0..21a094043e1 100644 --- a/.github/workflows/integration-tests.yml +++ b/.github/workflows/integration-tests.yml @@ -337,26 +337,17 @@ jobs: runId: ${{ github.run_id }} testLogCollect: ${{ vars.TEST_LOG_COLLECT }} selectedNetworks: ${{ env.SELECTED_NETWORKS }} - chainlinkImage: ${{ env.CHAINLINK_IMAGE }} chainlinkVersion: ${{ inputs.evm-ref || github.sha }} - pyroscopeServer: ${{ matrix.product.pyroscope_env == '' && '' || !startsWith(github.ref, 'refs/tags/') && '' || secrets.QA_PYROSCOPE_INSTANCE }} # Avoid sending blank envs https://github.com/orgs/community/discussions/25725 - pyroscopeEnvironment: ${{ matrix.product.pyroscope_env }} - pyroscopeKey: ${{ secrets.QA_PYROSCOPE_KEY }} - lokiEndpoint: https://${{ secrets.GRAFANA_INTERNAL_HOST }}/loki/api/v1/push - lokiTenantId: ${{ secrets.GRAFANA_INTERNAL_TENANT_ID }} - lokiBasicAuth: ${{ secrets.GRAFANA_INTERNAL_BASIC_AUTH }} logstreamLogTargets: ${{ vars.LOGSTREAM_LOG_TARGETS }} - grafanaUrl: "http://localhost:8080/primary" # This is GAP's address - grafanaDashboardUrl: "/d/ddf75041-1e39-42af-aa46-361fe4c36e9e/ci-e2e-tests-logs" - grafanaBearerToken: ${{ secrets.GRAFANA_INTERNAL_URL_SHORTENER_TOKEN }} ## Run this step when changes that require tests to be run are made - name: Run Tests if: needs.changes.outputs.src == 'true' || github.event_name == 'workflow_dispatch' - uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/run-tests@af92c5fae8dcf1659201e907db82d221fc304b94 # v2.3.21 + uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/run-tests@75a9005952a9e905649cfb5a6971fd9429436acd # v2.3.24 with: test_command_to_run: cd ./integration-tests && go test -timeout 30m -count=1 -json -test.parallel=${{ matrix.product.nodes }} ${{ steps.build-go-test-command.outputs.run_command }} 2>&1 | tee /tmp/gotest.log | gotestloghelper -ci -singlepackage -hidepassingtests=false -hidepassinglogs test_download_vendor_packages_command: cd ./integration-tests && go mod download + test_config_override_base64: ${{ env.BASE64_CONFIG_OVERRIDE }} cl_repo: ${{ env.CHAINLINK_IMAGE }} cl_image_tag: ${{ inputs.evm-ref || github.sha }} aws_registries: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }} @@ -375,6 +366,17 @@ jobs: should_tidy: "false" go_coverage_src_dir: /var/tmp/go-coverage go_coverage_dest_dir: ${{ github.workspace }}/.covdata + DEFAULT_CHAINLINK_IMAGE: ${{ env.CHAINLINK_IMAGE }} + DEFAULT_LOKI_TENANT_ID: ${{ secrets.GRAFANA_INTERNAL_TENANT_ID }} + DEFAULT_LOKI_ENDPOINT: https://${{ secrets.GRAFANA_INTERNAL_HOST }}/loki/api/v1/push + DEFAULT_LOKI_BASIC_AUTH: ${{ secrets.GRAFANA_INTERNAL_BASIC_AUTH }} + DEFAULT_GRAFANA_BASE_URL: "http://localhost:8080/primary" + DEFAULT_GRAFANA_DASHBOARD_URL: "/d/ddf75041-1e39-42af-aa46-361fe4c36e9e/ci-e2e-tests-logs" + DEFAULT_GRAFANA_BEARER_TOKEN: ${{ secrets.GRAFANA_INTERNAL_URL_SHORTENER_TOKEN }} + DEFAULT_PYROSCOPE_SERVER_URL: ${{ matrix.product.pyroscope_env == '' && '' || !startsWith(github.ref, 'refs/tags/') && '' || secrets.QA_PYROSCOPE_INSTANCE }} # Avoid sending blank envs https://github.com/orgs/community/discussions/25725 + DEFAULT_PYROSCOPE_KEY: ${{ secrets.QA_PYROSCOPE_KEY }} + DEFAULT_PYROSCOPE_ENVIRONMENT: ${{ matrix.product.pyroscope_env }} + DEFAULT_PYROSCOPE_ENABLED: ${{ matrix.product.pyroscope_env == '' || !startsWith(github.ref, 'refs/tags/') && 'false' || 'true' }} - name: Upload Coverage Data uses: actions/upload-artifact@65462800fd760344b1a7b4382951275a0abb4808 # v4.3.3 @@ -453,25 +455,17 @@ jobs: runId: ${{ github.run_id }} testLogCollect: ${{ vars.TEST_LOG_COLLECT }} selectedNetworks: ${{ env.SELECTED_NETWORKS }} - chainlinkImage: ${{ env.CHAINLINK_IMAGE }} chainlinkVersion: ${{ inputs.evm-ref || github.sha }} - pyroscopeServer: ${{ matrix.product.pyroscope_env == '' && '' || !startsWith(github.ref, 'refs/tags/') && '' || secrets.QA_PYROSCOPE_INSTANCE }} # Avoid sending blank envs https://github.com/orgs/community/discussions/25725 - pyroscopeEnvironment: ${{ matrix.product.pyroscope_env }} - pyroscopeKey: ${{ secrets.QA_PYROSCOPE_KEY }} - lokiEndpoint: https://${{ secrets.GRAFANA_INTERNAL_HOST }}/loki/api/v1/push - lokiTenantId: ${{ secrets.GRAFANA_INTERNAL_TENANT_ID }} - lokiBasicAuth: ${{ secrets.GRAFANA_INTERNAL_BASIC_AUTH }} logstreamLogTargets: ${{ vars.LOGSTREAM_LOG_TARGETS }} - grafanaUrl: "http://localhost:8080/primary" # This is GAP's address - grafanaDashboardUrl: "/d/ddf75041-1e39-42af-aa46-361fe4c36e9e/ci-e2e-tests-logs" - grafanaBearerToken: ${{ secrets.GRAFANA_INTERNAL_URL_SHORTENER_TOKEN }} + ## Run this step when changes that require tests to be run are made - name: Run Tests if: needs.changes.outputs.src == 'true' - uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/run-tests@af92c5fae8dcf1659201e907db82d221fc304b94 # v2.3.21 + uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/run-tests@75a9005952a9e905649cfb5a6971fd9429436acd # v2.3.24 with: test_command_to_run: cd ./integration-tests && go test -timeout 30m -count=1 -json -test.parallel=${{ matrix.product.nodes }} ${{ steps.build-go-test-command.outputs.run_command }} 2>&1 | tee /tmp/gotest.log | gotestloghelper -ci -singlepackage -hidepassingtests=false -hidepassinglogs test_download_vendor_packages_command: cd ./integration-tests && go mod download + test_config_override_base64: ${{ env.BASE64_CONFIG_OVERRIDE }} cl_repo: ${{ env.CHAINLINK_IMAGE }} cl_image_tag: ${{ inputs.evm-ref || github.sha }} aws_registries: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }} @@ -490,6 +484,17 @@ jobs: should_tidy: "false" go_coverage_src_dir: /var/tmp/go-coverage go_coverage_dest_dir: ${{ github.workspace }}/.covdata + DEFAULT_CHAINLINK_IMAGE: ${{ env.CHAINLINK_IMAGE }} + DEFAULT_LOKI_TENANT_ID: ${{ secrets.GRAFANA_INTERNAL_TENANT_ID }} + DEFAULT_LOKI_ENDPOINT: https://${{ secrets.GRAFANA_INTERNAL_HOST }}/loki/api/v1/push + DEFAULT_LOKI_BASIC_AUTH: ${{ secrets.GRAFANA_INTERNAL_BASIC_AUTH }} + DEFAULT_GRAFANA_BASE_URL: "http://localhost:8080/primary" + DEFAULT_GRAFANA_DASHBOARD_URL: "/d/ddf75041-1e39-42af-aa46-361fe4c36e9e/ci-e2e-tests-logs" + DEFAULT_GRAFANA_BEARER_TOKEN: ${{ secrets.GRAFANA_INTERNAL_URL_SHORTENER_TOKEN }} + DEFAULT_PYROSCOPE_SERVER_URL: ${{ matrix.product.pyroscope_env == '' && '' || !startsWith(github.ref, 'refs/tags/') && '' || secrets.QA_PYROSCOPE_INSTANCE }} # Avoid sending blank envs https://github.com/orgs/community/discussions/25725 + DEFAULT_PYROSCOPE_KEY: ${{ secrets.QA_PYROSCOPE_KEY }} + DEFAULT_PYROSCOPE_ENVIRONMENT: ${{ matrix.product.pyroscope_env }} + DEFAULT_PYROSCOPE_ENABLED: ${{ matrix.product.pyroscope_env == '' || !startsWith(github.ref, 'refs/tags/') && 'false' || 'true' }} - name: Upload Coverage Data uses: actions/upload-artifact@65462800fd760344b1a7b4382951275a0abb4808 # v4.3.3 @@ -692,26 +697,17 @@ jobs: runId: ${{ github.run_id }} testLogCollect: ${{ vars.TEST_LOG_COLLECT }} selectedNetworks: ${{ env.SELECTED_NETWORKS }} - chainlinkImage: ${{ env.CHAINLINK_IMAGE }} - chainlinkVersion: ${{ inputs.evm-ref || github.sha }} - pyroscopeServer: ${{ matrix.product.pyroscope_env == '' && '' || !startsWith(github.ref, 'refs/tags/') && '' || secrets.QA_PYROSCOPE_INSTANCE }} # Avoid sending blank envs https://github.com/orgs/community/discussions/25725 - pyroscopeEnvironment: ${{ matrix.product.pyroscope_env }} - pyroscopeKey: ${{ secrets.QA_PYROSCOPE_KEY }} - lokiEndpoint: https://${{ secrets.GRAFANA_INTERNAL_HOST }}/loki/api/v1/push - lokiTenantId: ${{ secrets.GRAFANA_INTERNAL_TENANT_ID }} - lokiBasicAuth: ${{ secrets.GRAFANA_INTERNAL_BASIC_AUTH }} + chainlinkVersion: ${{ inputs.evm-ref || github.sha }} logstreamLogTargets: ${{ vars.LOGSTREAM_LOG_TARGETS }} - grafanaUrl: "http://localhost:8080/primary" - grafanaDashboardUrl: "/d/ddf75041-1e39-42af-aa46-361fe4c36e9e/ci-e2e-tests-logs" - grafanaBearerToken: ${{ secrets.GRAFANA_INTERNAL_URL_SHORTENER_TOKEN }} ## Run this step when changes that require tests to be run are made - name: Run Tests if: needs.changes.outputs.src == 'true' || github.event_name == 'workflow_dispatch' - uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/run-tests@af92c5fae8dcf1659201e907db82d221fc304b94 # v2.3.21 + uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/run-tests@75a9005952a9e905649cfb5a6971fd9429436acd # v2.3.24 with: test_command_to_run: cd ./integration-tests && go test -timeout 30m -count=1 -json -test.parallel=${{ matrix.product.nodes }} ${{ steps.build-go-test-command.outputs.run_command }} 2>&1 | tee /tmp/gotest.log | gotestloghelper -ci -singlepackage -hidepassingtests=false -hidepassinglogs test_download_vendor_packages_command: cd ./integration-tests && go mod download + test_config_override_base64: ${{ env.BASE64_CONFIG_OVERRIDE }} cl_repo: ${{ env.CHAINLINK_IMAGE }} cl_image_tag: ${{ inputs.evm-ref || github.sha }}${{ matrix.product.tag_suffix }} aws_registries: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }} @@ -730,6 +726,17 @@ jobs: should_tidy: "false" go_coverage_src_dir: /var/tmp/go-coverage go_coverage_dest_dir: ${{ github.workspace }}/.covdata + DEFAULT_CHAINLINK_IMAGE: ${{ env.CHAINLINK_IMAGE }} + DEFAULT_LOKI_TENANT_ID: ${{ secrets.GRAFANA_INTERNAL_TENANT_ID }} + DEFAULT_LOKI_ENDPOINT: https://${{ secrets.GRAFANA_INTERNAL_HOST }}/loki/api/v1/push + DEFAULT_LOKI_BASIC_AUTH: ${{ secrets.GRAFANA_INTERNAL_BASIC_AUTH }} + DEFAULT_GRAFANA_BASE_URL: "http://localhost:8080/primary" + DEFAULT_GRAFANA_DASHBOARD_URL: "/d/ddf75041-1e39-42af-aa46-361fe4c36e9e/ci-e2e-tests-logs" + DEFAULT_GRAFANA_BEARER_TOKEN: ${{ secrets.GRAFANA_INTERNAL_URL_SHORTENER_TOKEN }} + DEFAULT_PYROSCOPE_SERVER_URL: ${{ matrix.product.pyroscope_env == '' && '' || !startsWith(github.ref, 'refs/tags/') && '' || secrets.QA_PYROSCOPE_INSTANCE }} # Avoid sending blank envs https://github.com/orgs/community/discussions/25725 + DEFAULT_PYROSCOPE_KEY: ${{ secrets.QA_PYROSCOPE_KEY }} + DEFAULT_PYROSCOPE_ENVIRONMENT: ${{ matrix.product.pyroscope_env }} + DEFAULT_PYROSCOPE_ENABLED: ${{ matrix.product.pyroscope_env == '' || !startsWith(github.ref, 'refs/tags/') && 'false' || 'true' }} - name: Upload Coverage Data uses: actions/upload-artifact@65462800fd760344b1a7b4382951275a0abb4808 # v4.3.3 @@ -952,10 +959,11 @@ jobs: grafanaUrl: ${{ vars.GRAFANA_URL }} grafanaDashboardUrl: "/d/ddf75041-1e39-42af-aa46-361fe4c36e9e/ci-e2e-tests-logs" - name: Run Migration Tests - uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/run-tests@af92c5fae8dcf1659201e907db82d221fc304b94 # v2.3.21 + uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/run-tests@75a9005952a9e905649cfb5a6971fd9429436acd # v2.3.24 with: test_command_to_run: cd ./integration-tests && go test -timeout 20m -count=1 -json ./migration 2>&1 | tee /tmp/gotest.log | gotestloghelper -ci -singlepackage -hidepassingtests=false -hidepassinglogs test_download_vendor_packages_command: cd ./integration-tests && go mod download + test_config_override_base64: ${{ env.BASE64_CONFIG_OVERRIDE }} cl_repo: ${{ env.CHAINLINK_IMAGE }} cl_image_tag: ${{ steps.get_latest_version.outputs.latest_version }} aws_registries: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }} @@ -974,6 +982,18 @@ jobs: go_coverage_src_dir: /var/tmp/go-coverage go_coverage_dest_dir: ${{ github.workspace }}/.covdata should_tidy: "false" + DEFAULT_CHAINLINK_IMAGE: ${{ env.CHAINLINK_IMAGE }} + DEFAULT_LOKI_TENANT_ID: ${{ secrets.GRAFANA_INTERNAL_TENANT_ID }} + DEFAULT_LOKI_ENDPOINT: https://${{ secrets.GRAFANA_INTERNAL_HOST }}/loki/api/v1/push + DEFAULT_LOKI_BASIC_AUTH: ${{ secrets.GRAFANA_INTERNAL_BASIC_AUTH }} + DEFAULT_GRAFANA_BASE_URL: "http://localhost:8080/primary" + DEFAULT_GRAFANA_DASHBOARD_URL: "/d/ddf75041-1e39-42af-aa46-361fe4c36e9e/ci-e2e-tests-logs" + DEFAULT_GRAFANA_BEARER_TOKEN: ${{ secrets.GRAFANA_INTERNAL_URL_SHORTENER_TOKEN }} + DEFAULT_PYROSCOPE_SERVER_URL: ${{ matrix.product.pyroscope_env == '' && '' || !startsWith(github.ref, 'refs/tags/') && '' || secrets.QA_PYROSCOPE_INSTANCE }} # Avoid sending blank envs https://github.com/orgs/community/discussions/25725 + DEFAULT_PYROSCOPE_KEY: ${{ secrets.QA_PYROSCOPE_KEY }} + DEFAULT_PYROSCOPE_ENVIRONMENT: ${{ matrix.product.pyroscope_env }} + DEFAULT_PYROSCOPE_ENABLED: ${{ matrix.product.pyroscope_env == '' || !startsWith(github.ref, 'refs/tags/') && 'false' || 'true' }} + - name: Upload Coverage Data uses: actions/upload-artifact@65462800fd760344b1a7b4382951275a0abb4808 # v4.3.3 timeout-minutes: 2 diff --git a/.github/workflows/on-demand-keeper-smoke-tests.yml b/.github/workflows/on-demand-keeper-smoke-tests.yml index f6fa8f4467a..9ac39e37f3b 100644 --- a/.github/workflows/on-demand-keeper-smoke-tests.yml +++ b/.github/workflows/on-demand-keeper-smoke-tests.yml @@ -137,21 +137,12 @@ jobs: runId: ${{ github.run_id }} testLogCollect: ${{ vars.TEST_LOG_COLLECT }} selectedNetworks: ${{ env.SELECTED_NETWORKS }} - chainlinkImage: ${{ env.CHAINLINK_IMAGE }} chainlinkVersion: ${{ inputs.evm-ref || github.sha }} - pyroscopeServer: ${{ matrix.product.pyroscope_env == '' && '' || !startsWith(github.ref, 'refs/tags/') && '' || secrets.QA_PYROSCOPE_INSTANCE }} # Avoid sending blank envs https://github.com/orgs/community/discussions/25725 - pyroscopeEnvironment: ${{ matrix.product.pyroscope_env }} - pyroscopeKey: ${{ secrets.QA_PYROSCOPE_KEY }} - lokiEndpoint: https://${{ secrets.GRAFANA_INTERNAL_HOST }}/loki/api/v1/push - lokiTenantId: ${{ secrets.GRAFANA_INTERNAL_TENANT_ID }} - lokiBasicAuth: ${{ secrets.GRAFANA_INTERNAL_BASIC_AUTH }} logstreamLogTargets: ${{ vars.LOGSTREAM_LOG_TARGETS }} - grafanaUrl: ${{ vars.GRAFANA_URL }} - grafanaDashboardUrl: "/d/ddf75041-1e39-42af-aa46-361fe4c36e9e/ci-e2e-tests-logs" ## Run this step when changes that require tests to be run are made - name: Run Tests - uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/run-tests@af92c5fae8dcf1659201e907db82d221fc304b94 # v2.3.21 + uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/run-tests@75a9005952a9e905649cfb5a6971fd9429436acd # v2.3.24 with: test_command_to_run: cd ./integration-tests && go test -timeout 30m -count=1 -json -test.parallel=${{ matrix.product.nodes }} ${{ steps.build-go-test-command.outputs.run_command }} 2>&1 | tee /tmp/gotest.log | gotestloghelper -ci -singlepackage -hidepassingtests=false -hidepassinglogs test_download_vendor_packages_command: cd ./integration-tests && go mod download @@ -173,6 +164,17 @@ jobs: should_tidy: "false" go_coverage_src_dir: /var/tmp/go-coverage go_coverage_dest_dir: ${{ github.workspace }}/.covdata + DEFAULT_CHAINLINK_IMAGE: ${{ env.CHAINLINK_IMAGE }} + DEFAULT_LOKI_TENANT_ID: ${{ secrets.GRAFANA_INTERNAL_TENANT_ID }} + DEFAULT_LOKI_ENDPOINT: https://${{ secrets.GRAFANA_INTERNAL_HOST }}/loki/api/v1/push + DEFAULT_LOKI_BASIC_AUTH: ${{ secrets.GRAFANA_INTERNAL_BASIC_AUTH }} + DEFAULT_GRAFANA_BASE_URL: ${{ vars.GRAFANA_URL }} + DEFAULT_GRAFANA_DASHBOARD_URL: "/d/ddf75041-1e39-42af-aa46-361fe4c36e9e/ci-e2e-tests-logs" + DEFAULT_GRAFANA_BEARER_TOKEN: ${{ secrets.GRAFANA_INTERNAL_URL_SHORTENER_TOKEN }} + DEFAULT_PYROSCOPE_SERVER_URL: ${{ matrix.product.pyroscope_env == '' && '' || !startsWith(github.ref, 'refs/tags/') && '' || secrets.QA_PYROSCOPE_INSTANCE }} # Avoid sending blank envs https://github.com/orgs/community/discussions/25725 + DEFAULT_PYROSCOPE_KEY: ${{ secrets.QA_PYROSCOPE_KEY }} + DEFAULT_PYROSCOPE_ENVIRONMENT: ${{ matrix.product.pyroscope_env }} + DEFAULT_PYROSCOPE_ENABLED: ${{ matrix.product.pyroscope_env == '' || !startsWith(github.ref, 'refs/tags/') && 'false' || 'true' }} - name: Upload Coverage Data uses: actions/upload-artifact@65462800fd760344b1a7b4382951275a0abb4808 # v4.3.3 diff --git a/.github/workflows/on-demand-vrfv2-eth2-clients-test.yml b/.github/workflows/on-demand-vrfv2-eth2-clients-test.yml index 03b07ab1b1b..3f9d9855071 100644 --- a/.github/workflows/on-demand-vrfv2-eth2-clients-test.yml +++ b/.github/workflows/on-demand-vrfv2-eth2-clients-test.yml @@ -46,7 +46,7 @@ jobs: echo "### Execution client used" >>$GITHUB_STEP_SUMMARY echo "\`${{ env.ETH2_EL_CLIENT }}\`" >>$GITHUB_STEP_SUMMARY - name: Run Tests - uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/run-tests@af92c5fae8dcf1659201e907db82d221fc304b94 # v2.3.21 + uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/run-tests@75a9005952a9e905649cfb5a6971fd9429436acd # v2.3.24 with: test_command_to_run: cd ./integration-tests && go test -timeout 30m -count=1 -json -run TestVRFv2Basic ./smoke/vrfv2_test.go 2>&1 | tee /tmp/gotest.log | gotestloghelper -ci -singlepackage -hidepassingtests=false -hidepassinglogs test_download_vendor_packages_command: cd ./integration-tests && go mod download @@ -61,3 +61,10 @@ jobs: QA_AWS_REGION: ${{ secrets.QA_AWS_REGION }} QA_AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }} QA_KUBECONFIG: "" + DEFAULT_CHAINLINK_IMAGE: ${{ env.CHAINLINK_IMAGE }} + DEFAULT_LOKI_TENANT_ID: ${{ secrets.GRAFANA_INTERNAL_TENANT_ID }} + DEFAULT_LOKI_ENDPOINT: https://${{ secrets.GRAFANA_INTERNAL_HOST }}/loki/api/v1/push + DEFAULT_LOKI_BASIC_AUTH: ${{ secrets.GRAFANA_INTERNAL_BASIC_AUTH }} + DEFAULT_GRAFANA_BASE_URL: "http://localhost:8080/primary" + DEFAULT_GRAFANA_DASHBOARD_URL: "/d/ddf75041-1e39-42af-aa46-361fe4c36e9e/ci-e2e-tests-logs" + DEFAULT_GRAFANA_BEARER_TOKEN: ${{ secrets.GRAFANA_INTERNAL_URL_SHORTENER_TOKEN }} \ No newline at end of file diff --git a/.github/workflows/on-demand-vrfv2plus-eth2-clients-test.yml b/.github/workflows/on-demand-vrfv2plus-eth2-clients-test.yml index 242eed23640..64c047ffbb5 100644 --- a/.github/workflows/on-demand-vrfv2plus-eth2-clients-test.yml +++ b/.github/workflows/on-demand-vrfv2plus-eth2-clients-test.yml @@ -46,7 +46,7 @@ jobs: echo "### Execution client used" >>$GITHUB_STEP_SUMMARY echo "\`${{ env.ETH2_EL_CLIENT }}\`" >>$GITHUB_STEP_SUMMARY - name: Run Tests - uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/run-tests@af92c5fae8dcf1659201e907db82d221fc304b94 # v2.3.21 + uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/run-tests@75a9005952a9e905649cfb5a6971fd9429436acd # v2.3.24 with: test_command_to_run: cd ./integration-tests && go test -timeout 30m -count=1 -json -run ^TestVRFv2Plus$/^Link_Billing$ ./smoke/vrfv2plus_test.go 2>&1 | tee /tmp/gotest.log | gotestloghelper -ci -singlepackage -hidepassingtests=false -hidepassinglogs test_download_vendor_packages_command: cd ./integration-tests && go mod download @@ -61,3 +61,10 @@ jobs: QA_AWS_REGION: ${{ secrets.QA_AWS_REGION }} QA_AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }} QA_KUBECONFIG: "" + DEFAULT_CHAINLINK_IMAGE: ${{ env.CHAINLINK_IMAGE }} + DEFAULT_LOKI_TENANT_ID: ${{ secrets.GRAFANA_INTERNAL_TENANT_ID }} + DEFAULT_LOKI_ENDPOINT: https://${{ secrets.GRAFANA_INTERNAL_HOST }}/loki/api/v1/push + DEFAULT_LOKI_BASIC_AUTH: ${{ secrets.GRAFANA_INTERNAL_BASIC_AUTH }} + DEFAULT_GRAFANA_BASE_URL: "http://localhost:8080/primary" + DEFAULT_GRAFANA_DASHBOARD_URL: "/d/ddf75041-1e39-42af-aa46-361fe4c36e9e/ci-e2e-tests-logs" + DEFAULT_GRAFANA_BEARER_TOKEN: ${{ secrets.GRAFANA_INTERNAL_URL_SHORTENER_TOKEN }} \ No newline at end of file diff --git a/integration-tests/go.mod b/integration-tests/go.mod index dd6f2be123d..2e5a400632b 100644 --- a/integration-tests/go.mod +++ b/integration-tests/go.mod @@ -29,7 +29,7 @@ require ( github.com/slack-go/slack v0.12.2 github.com/smartcontractkit/chainlink-automation v1.0.4 github.com/smartcontractkit/chainlink-common v0.1.7-0.20240703234618-dc1fbe45acc0 - github.com/smartcontractkit/chainlink-testing-framework v1.31.7 + github.com/smartcontractkit/chainlink-testing-framework v1.31.9 github.com/smartcontractkit/chainlink-testing-framework/grafana v0.0.0-20240405215812-5a72bc9af239 github.com/smartcontractkit/chainlink/v2 v2.0.0-00010101000000-000000000000 github.com/smartcontractkit/havoc/k8schaos v0.0.0-20240409145249-e78d20847e37 @@ -289,6 +289,7 @@ require ( github.com/jackpal/go-nat-pmp v1.0.2 // indirect github.com/jmespath/go-jmespath v0.4.0 // indirect github.com/jmhodges/levigo v1.0.0 // indirect + github.com/joho/godotenv v1.5.1 // indirect github.com/jonboulle/clockwork v0.4.0 // indirect github.com/josharian/intern v1.0.0 // indirect github.com/jpillora/backoff v1.0.0 // indirect diff --git a/integration-tests/go.sum b/integration-tests/go.sum index 6b29839d5c4..ad9c49fc92c 100644 --- a/integration-tests/go.sum +++ b/integration-tests/go.sum @@ -1069,8 +1069,8 @@ github.com/jmhodges/levigo v1.0.0/go.mod h1:Q6Qx+uH3RAqyK4rFQroq9RL7mdkABMcfhEI+ github.com/jmoiron/sqlx v1.4.0 h1:1PLqN7S1UYp5t4SrVVnt4nUVNemrDAtxlulVe+Qgm3o= github.com/jmoiron/sqlx v1.4.0/go.mod h1:ZrZ7UsYB/weZdl2Bxg6jCRO9c3YHl8r3ahlKmRT4JLY= github.com/joho/godotenv v1.3.0/go.mod h1:7hK45KPybAkOC6peb+G5yklZfMxEjkZhHbwpqxOKXbg= -github.com/joho/godotenv v1.4.0 h1:3l4+N6zfMWnkbPEXKng2o2/MR5mSwTrBih4ZEkkz1lg= -github.com/joho/godotenv v1.4.0/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4= +github.com/joho/godotenv v1.5.1 h1:7eLL/+HRGLY0ldzfGMeQkb7vMd0as4CfYvUVzLqw0N0= +github.com/joho/godotenv v1.5.1/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4= github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= github.com/jonboulle/clockwork v0.4.0 h1:p4Cf1aMWXnXAUh8lVfewRBx1zaTSYKrKMF2g3ST4RZ4= github.com/jonboulle/clockwork v0.4.0/go.mod h1:xgRqUGwRcjKCO1vbZUEtSLrqKoPSsUpK7fnezOII0kc= @@ -1524,8 +1524,8 @@ github.com/smartcontractkit/chainlink-solana v1.0.3-0.20240708130426-294b81e4afe github.com/smartcontractkit/chainlink-solana v1.0.3-0.20240708130426-294b81e4afe7/go.mod h1:NbXXQaNFskVMYRut0MvBlcHu/vDgipGMwYjamvjVB9Y= github.com/smartcontractkit/chainlink-starknet/relayer v0.0.1-beta-test.0.20240625074951-06ab5e670dba h1:YNSlhK5mobyAaw02LPGgIEuS3lXyCTXcc6oaV2L6uUI= github.com/smartcontractkit/chainlink-starknet/relayer v0.0.1-beta-test.0.20240625074951-06ab5e670dba/go.mod h1:UVFRacRkP7O7TQAzFmR52v5mUlxf+G1ovMlCQAB/cHU= -github.com/smartcontractkit/chainlink-testing-framework v1.31.7 h1:Vy4ah8VAfj+Y7vVmhjvwyAO6wG+Fp2vzdkSJwJPMQO4= -github.com/smartcontractkit/chainlink-testing-framework v1.31.7/go.mod h1:E6uNEZhZZid9PHv6/Kq5Vn63GlO61ZcKB+/f0DKo3Q4= +github.com/smartcontractkit/chainlink-testing-framework v1.31.9 h1:NiPGdXYSjVyAxCZRZ3yBMqnInQKU7FyVZ24wt8RWuX0= +github.com/smartcontractkit/chainlink-testing-framework v1.31.9/go.mod h1:wSRZGoukZliwfTkghdF4cI1RLHz5k3aDnL+rXhJ6f7k= github.com/smartcontractkit/chainlink-testing-framework/grafana v0.0.0-20240405215812-5a72bc9af239 h1:Kk5OVlx/5g9q3Z3lhxytZS4/f8ds1MiNM8yaHgK3Oe8= github.com/smartcontractkit/chainlink-testing-framework/grafana v0.0.0-20240405215812-5a72bc9af239/go.mod h1:DC8sQMyTlI/44UCTL8QWFwb0bYNoXCfjwCv2hMivYZU= github.com/smartcontractkit/go-plugin v0.0.0-20240208201424-b3b91517de16 h1:TFe+FvzxClblt6qRfqEhUfa4kFQx5UobuoFGO2W4mMo= diff --git a/integration-tests/load/go.mod b/integration-tests/load/go.mod index 9cb8968a7e2..863c06c1977 100644 --- a/integration-tests/load/go.mod +++ b/integration-tests/load/go.mod @@ -17,7 +17,7 @@ require ( github.com/slack-go/slack v0.12.2 github.com/smartcontractkit/chainlink-automation v1.0.4 github.com/smartcontractkit/chainlink-common v0.1.7-0.20240703234618-dc1fbe45acc0 - github.com/smartcontractkit/chainlink-testing-framework v1.31.7 + github.com/smartcontractkit/chainlink-testing-framework v1.31.9 github.com/smartcontractkit/chainlink/integration-tests v0.0.0-20240214231432-4ad5eb95178c github.com/smartcontractkit/chainlink/v2 v2.9.0-beta0.0.20240216210048-da02459ddad8 github.com/smartcontractkit/libocr v0.0.0-20240702141926-063ceef8c42e @@ -271,6 +271,7 @@ require ( github.com/jmespath/go-jmespath v0.4.0 // indirect github.com/jmhodges/levigo v1.0.0 // indirect github.com/jmoiron/sqlx v1.4.0 // indirect + github.com/joho/godotenv v1.5.1 // indirect github.com/jonboulle/clockwork v0.4.0 // indirect github.com/josharian/intern v1.0.0 // indirect github.com/jpillora/backoff v1.0.0 // indirect diff --git a/integration-tests/load/go.sum b/integration-tests/load/go.sum index 44b7798a447..029bd2bb7c9 100644 --- a/integration-tests/load/go.sum +++ b/integration-tests/load/go.sum @@ -1065,8 +1065,8 @@ github.com/jmhodges/levigo v1.0.0/go.mod h1:Q6Qx+uH3RAqyK4rFQroq9RL7mdkABMcfhEI+ github.com/jmoiron/sqlx v1.4.0 h1:1PLqN7S1UYp5t4SrVVnt4nUVNemrDAtxlulVe+Qgm3o= github.com/jmoiron/sqlx v1.4.0/go.mod h1:ZrZ7UsYB/weZdl2Bxg6jCRO9c3YHl8r3ahlKmRT4JLY= github.com/joho/godotenv v1.3.0/go.mod h1:7hK45KPybAkOC6peb+G5yklZfMxEjkZhHbwpqxOKXbg= -github.com/joho/godotenv v1.4.0 h1:3l4+N6zfMWnkbPEXKng2o2/MR5mSwTrBih4ZEkkz1lg= -github.com/joho/godotenv v1.4.0/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4= +github.com/joho/godotenv v1.5.1 h1:7eLL/+HRGLY0ldzfGMeQkb7vMd0as4CfYvUVzLqw0N0= +github.com/joho/godotenv v1.5.1/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4= github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= github.com/jonboulle/clockwork v0.4.0 h1:p4Cf1aMWXnXAUh8lVfewRBx1zaTSYKrKMF2g3ST4RZ4= github.com/jonboulle/clockwork v0.4.0/go.mod h1:xgRqUGwRcjKCO1vbZUEtSLrqKoPSsUpK7fnezOII0kc= @@ -1514,8 +1514,8 @@ github.com/smartcontractkit/chainlink-solana v1.0.3-0.20240708130426-294b81e4afe github.com/smartcontractkit/chainlink-solana v1.0.3-0.20240708130426-294b81e4afe7/go.mod h1:NbXXQaNFskVMYRut0MvBlcHu/vDgipGMwYjamvjVB9Y= github.com/smartcontractkit/chainlink-starknet/relayer v0.0.1-beta-test.0.20240625074951-06ab5e670dba h1:YNSlhK5mobyAaw02LPGgIEuS3lXyCTXcc6oaV2L6uUI= github.com/smartcontractkit/chainlink-starknet/relayer v0.0.1-beta-test.0.20240625074951-06ab5e670dba/go.mod h1:UVFRacRkP7O7TQAzFmR52v5mUlxf+G1ovMlCQAB/cHU= -github.com/smartcontractkit/chainlink-testing-framework v1.31.7 h1:Vy4ah8VAfj+Y7vVmhjvwyAO6wG+Fp2vzdkSJwJPMQO4= -github.com/smartcontractkit/chainlink-testing-framework v1.31.7/go.mod h1:E6uNEZhZZid9PHv6/Kq5Vn63GlO61ZcKB+/f0DKo3Q4= +github.com/smartcontractkit/chainlink-testing-framework v1.31.9 h1:NiPGdXYSjVyAxCZRZ3yBMqnInQKU7FyVZ24wt8RWuX0= +github.com/smartcontractkit/chainlink-testing-framework v1.31.9/go.mod h1:wSRZGoukZliwfTkghdF4cI1RLHz5k3aDnL+rXhJ6f7k= github.com/smartcontractkit/chainlink-testing-framework/grafana v0.0.0-20240405215812-5a72bc9af239 h1:Kk5OVlx/5g9q3Z3lhxytZS4/f8ds1MiNM8yaHgK3Oe8= github.com/smartcontractkit/chainlink-testing-framework/grafana v0.0.0-20240405215812-5a72bc9af239/go.mod h1:DC8sQMyTlI/44UCTL8QWFwb0bYNoXCfjwCv2hMivYZU= github.com/smartcontractkit/go-plugin v0.0.0-20240208201424-b3b91517de16 h1:TFe+FvzxClblt6qRfqEhUfa4kFQx5UobuoFGO2W4mMo= diff --git a/integration-tests/testconfig/README.md b/integration-tests/testconfig/README.md index bb3d7c000f7..7ff6cedd24c 100644 --- a/integration-tests/testconfig/README.md +++ b/integration-tests/testconfig/README.md @@ -64,31 +64,16 @@ cat << EOF > config.toml selected_networks=["$SELECTED_NETWORKS"] [ChainlinkImage] -image="$CHAINLINK_IMAGE" +image="" version="$CHAINLINK_VERSION" postgres_version="$CHAINLINK_POSTGRES_VERSION" -[Pyroscope] -enabled=$pyroscope_enabled -server_url="$PYROSCOPE_SERVER" -environment="$PYROSCOPE_ENVIRONMENT" -key_secret="$PYROSCOPE_KEY" - [Logging] test_log_collect=false run_id="$RUN_ID" [Logging.LogStream] log_targets=["$LOG_TARGETS"] - -[Logging.Loki] -tenant_id="$LOKI_TENANT_ID" -endpoint="$LOKI_ENDPOINT" -basic_auth_secret="$LOKI_BASIC_AUTH" - -[Logging.Grafana] -base_url="$GRAFANA_URL" -dashboard_url="$GRAFANA_DASHBOARD_URL" EOF BASE64_CONFIG_OVERRIDE=$(cat config.toml | base64 -w 0) @@ -98,6 +83,11 @@ echo "BASE64_CONFIG_OVERRIDE=$BASE64_CONFIG_OVERRIDE" >> $GITHUB_ENV **It is highly recommended to use reusable GHA actions present in [.actions](../../../.github/.actions) to generate and apply the base64-encoded configuration.** Own implementation of `BASE64_CONFIG_OVERRIDE` generation is discouraged and should be used only if existing actions do not cover the use case. But even in that case it might be a better idea to extend existing actions. This variable is automatically relayed to Kubernetes-based tests, eliminating the need for manual intervention in test scripts. +## Test Secrets + +Test secrets are not stored directly within the `TestConfig` TOML for security reasons. Instead, they are passed into `TestConfig` via environment variables. This ensures sensitive data is handled securely throughout our testing processes. + +For detailed instructions on how to set test secrets both locally and within CI environments, please visit: [Test Secrets Guide in CTF](https://github.com/smartcontractkit/chainlink-testing-framework/blob/main/config/README.md#test-secrets) ## Named Configurations diff --git a/integration-tests/testconfig/testconfig.go b/integration-tests/testconfig/testconfig.go index 34832625154..80f44356c3e 100644 --- a/integration-tests/testconfig/testconfig.go +++ b/integration-tests/testconfig/testconfig.go @@ -343,6 +343,8 @@ func GetConfig(configurationNames []string, product Product) (TestConfig, error) return TestConfig{}, errors.Wrapf(err, "error reading file %s", filePath) } + _ = checkSecretsInToml(content) + for _, configurationName := range configurationNames { err = ctf_config.BytesToAnyTomlStruct(logger, fileName, configurationName, &testConfig, content) if err != nil { @@ -352,6 +354,12 @@ func GetConfig(configurationNames []string, product Product) (TestConfig, error) } } + // it needs some custom logic, so we do it separately + err := testConfig.readNetworkConfiguration() + if err != nil { + return TestConfig{}, errors.Wrapf(err, "error reading network config") + } + logger.Info().Msg("Reading configs from Base64 override env var") configEncoded, isSet := os.LookupEnv(Base64OverrideEnvVarName) if isSet && configEncoded != "" { @@ -361,6 +369,8 @@ func GetConfig(configurationNames []string, product Product) (TestConfig, error) return TestConfig{}, err } + _ = checkSecretsInToml(decoded) + for _, configurationName := range configurationNames { err = ctf_config.BytesToAnyTomlStruct(logger, Base64OverrideEnvVarName, configurationName, &testConfig, decoded) if err != nil { @@ -371,10 +381,16 @@ func GetConfig(configurationNames []string, product Product) (TestConfig, error) logger.Debug().Msg("Base64 config override from environment variable not found") } - // it needs some custom logic, so we do it separately - err := testConfig.readNetworkConfiguration() + logger.Info().Msg("Loading config values from default ~/.testsecrets env file") + err = ctf_config.LoadSecretEnvsFromFile() if err != nil { - return TestConfig{}, errors.Wrapf(err, "error reading network config") + return TestConfig{}, errors.Wrapf(err, "error reading test config values from ~/.testsecrets file") + } + + logger.Info().Msg("Reading values from environment variables") + err = testConfig.ReadConfigValuesFromEnvVars() + if err != nil { + return TestConfig{}, errors.Wrapf(err, "error reading test config values from env vars") } logger.Debug().Msg("Validating test config") @@ -459,6 +475,52 @@ root_key_funds_buffer = 1_000 } +// checkSecretsInToml checks if the TOML file contains secrets and shows error logs if it does +// This is a temporary and will be removed after migration to test secrets from env vars +func checkSecretsInToml(content []byte) error { + logger := logging.GetTestLogger(nil) + data := make(map[string]interface{}) + + // Decode the TOML data + err := toml.Unmarshal(content, &data) + if err != nil { + return errors.Wrapf(err, "error decoding TOML file") + } + + logError := func(key, envVar string) { + logger.Error().Msgf("Error in TOML test config!! TOML cannot have '%s' key. Remove it and set %s env in ~/.testsecrets instead", key, envVar) + } + + if data["ChainlinkImage"] != nil { + chainlinkImage := data["ChainlinkImage"].(map[string]interface{}) + if chainlinkImage["image"] != nil { + logError("ChainlinkImage.image", "E2E_TEST_CHAINLINK_IMAGE") + } + } + + if data["ChainlinkUpgradeImage"] != nil { + chainlinkUpgradeImage := data["ChainlinkUpgradeImage"].(map[string]interface{}) + if chainlinkUpgradeImage["image"] != nil { + logError("ChainlinkUpgradeImage.image", "E2E_TEST_CHAINLINK_UPGRADE_IMAGE") + } + } + + if data["Network"] != nil { + network := data["Network"].(map[string]interface{}) + if network["RpcHttpUrls"] != nil { + logError("Network.RpcHttpUrls", "`E2E_TEST_(.+)_RPC_HTTP_URL$` like E2E_TEST_ARBITRUM_SEPOLIA_RPC_HTTP_URL") + } + if network["RpcWsUrls"] != nil { + logError("Network.RpcWsUrls", "`E2E_TEST_(.+)_RPC_WS_URL$` like E2E_TEST_ARBITRUM_SEPOLIA_RPC_WS_URL") + } + if network["WalletKeys"] != nil { + logError("Network.wallet_keys", "`E2E_TEST_(.+)_WALLET_KEY$` E2E_TEST_ARBITRUM_SEPOLIA_WALLET_KEY") + } + } + + return nil +} + func (c *TestConfig) readNetworkConfiguration() error { // currently we need to read that kind of secrets only for network configuration if c.Network == nil { diff --git a/integration-tests/testconfig/testconfig_utils.go b/integration-tests/testconfig/testconfig_utils.go index 6529771f491..e7b38ea3e4e 100644 --- a/integration-tests/testconfig/testconfig_utils.go +++ b/integration-tests/testconfig/testconfig_utils.go @@ -9,32 +9,21 @@ import ( // MissingImageInfoAsError return a helfpul error message when the no Chainlink image info is found in TOML config. // If legacy env vars are found it prints ready to use TOML configuration func MissingImageInfoAsError(errStr string) error { - intro := ` -You might have used old configuration approach. If so, use TOML instead of env vars. + missingImage := ` +Chainlink image is a secret and must be set as env var in ~/.testsecrets file or passed as env var (either E2E_TEST_CHAINLINK_IMAGE or E2E_TEST_CHAINLINK_UPGRADE_IMAGE). You might have used old configuration approach. Please refer to integration-tests/testconfig/README.md for more information. +` + missingVersion := ` +Chainlink version must be set in toml config. ` - var imgStr, versionStr string - - if img := os.Getenv("CHAINLINK_IMAGE"); img != "" { - imgStr = fmt.Sprintf("image = \"%s\"\n", img) - } - - if version := os.Getenv("CHAINLINK_VERSION"); version != "" { - versionStr = fmt.Sprintf("version = \"%s\"\n", version) + if os.Getenv("E2E_TEST_CHAINLINK_IMAGE") == "" || os.Getenv("E2E_TEST_CHAINLINK_UPGRADE_IMAGE") == "" { + return fmt.Errorf(fmt.Sprintf("%s\n%s", errStr, missingImage)) } - - finalErrStr := fmt.Sprintf("%s\n%s", errStr, intro) - - if imgStr != "" && versionStr != "" { - extraInfo := ` -Or if you want to run your tests right now add following content to integration-tests/testconfig/overrides.toml: -[ChainlinkImage] -` - finalErrStr = fmt.Sprintf("%s\n%s%s%s%s", errStr, intro, extraInfo, imgStr, versionStr) + if os.Getenv("CHAINLINK_VERSION") == "" || os.Getenv("CHAINLINK_UPGRADE_VERSION") == "" { + return fmt.Errorf(fmt.Sprintf("%s\n%s", errStr, missingVersion)) } - - return fmt.Errorf(finalErrStr) + return fmt.Errorf(errStr) } // NoSelectedNetworkInfoAsError return a helfpul error message when the no selected network info is found in TOML config.