Load E2E test secrets from env vars (#13762)

* Load test config secrets from env vars * Remove secrets from test config in automation-ondemand-tests.yml * Pass config override * Update on-demand-vrfv2plus-eth2-clients-test.yml to accept test_secrets_override_key * Bump gha * bump ctf * Use debug in automation workflow test log * Update input name * Fix integration-tests.yml * Override with env vars after overriding with test config * Clean up setup-create-base64-config action * bump ctf * bump ctf and gha * Update env var names to make it easier to set by QA * Add .secrets to gitignore * Load secrets from testconfig/.secrets * Fix MissingImageInfoAsError * Load secrets from env file in home dir * Add checkSecretsInToml * Update checkSecretsInToml * Update checkSecretsInToml * Revert on-demand-vrfv2-eth2-clients-test.yml * Revert on-demand-vrfv2plus-eth2-clients-test.yml * Fix go mod * Fix * Bump run-tests gha * bump ctf * test * bump gha and ctf * Fix workflows that use new GHA * Fix * Fix lint * Fix log level * Use E2E_TEST_ prefix to forward env vars to remote runner * Add docs * Fix and bump * bump ctf and gha
smartcontractkit · Jul 9, 2024 · cf94961 · cf94961
1 parent 697e469
commit cf94961
Show file tree

Hide file tree

Showing 14 changed files with 219 additions and 212 deletions.
diff --git a/.github/actions/setup-create-base64-config/action.yml b/.github/actions/setup-create-base64-config/action.yml
@@ -9,34 +9,13 @@ inputs:
     default: "false"
   selectedNetworks:
     description: The networks to run tests against
-  chainlinkImage:
-    description: The chainlink image to use
-    default: "public.ecr.aws/chainlink/chainlink"
   chainlinkPostgresVersion:
     description: The postgres version to use with the chainlink node
     default: "15.6"
   chainlinkVersion:
     description: The git commit sha to use for the image tag
-  pyroscopeServer:
-    description: URL of Pyroscope server
-  pyroscopeEnvironment:
-    description: Name of Pyroscope environment
-  pyroscopeKey:
-    description: Pyroscope server key
-  lokiEndpoint:
-    description: Loki push endpoint
-  lokiTenantId:
-    description: Loki tenant id
-  lokiBasicAuth:
-    description: Loki basic auth
   logstreamLogTargets:
     description: Where to send logs (e.g. file, loki)
-  grafanaUrl:
-    description: Grafana URL
-  grafanaDashboardUrl:
-    description: Grafana dashboard URL
-  grafanaBearerToken:
-    description: Grafana bearer token
   ethExecutionClient:
     description: Ethereum execution client to use (geth, besu, nethermind or erigon)
   customEthClientDockerImage:
@@ -52,23 +31,12 @@ runs:
         RUN_ID: ${{ inputs.runId }}
         TEST_LOG_COLLECT: ${{ inputs.testLogCollect }}
         SELECTED_NETWORKS: ${{ inputs.selectedNetworks }}
-        PYROSCOPE_SERVER: ${{ inputs.pyroscopeServer }}
-        PYROSCOPE_ENVIRONMENT: ${{ inputs.pyroscopeEnvironment }}
-        PYROSCOPE_KEY: ${{ inputs.pyroscopeKey }}
-        CHAINLINK_IMAGE: ${{ inputs.chainlinkImage }}
         CHAINLINK_VERSION: ${{ inputs.chainlinkVersion }}
         CHAINLINK_POSTGRES_VERSION: ${{ inputs.chainlinkPostgresVersion }}
-        LOKI_ENDPOINT: ${{ inputs.lokiEndpoint }}
-        LOKI_TENANT_ID: ${{ inputs.lokiTenantId }}
-        LOKI_BASIC_AUTH: ${{ inputs.lokiBasicAuth }}
         LOGSTREAM_LOG_TARGETS: ${{ inputs.logstreamLogTargets }}
-        GRAFANA_URL: ${{ inputs.grafanaUrl }}
-        GRAFANA_DASHBOARD_URL: ${{ inputs.grafanaDashboardUrl }}
-        GRAFANA_BEARER_TOKEN: ${{ inputs.grafanaBearerToken }}
         ETH_EXECUTION_CLIENT: ${{ inputs.ethExecutionClient }}
         CUSTOM_ETH_CLIENT_DOCKER_IMAGE: ${{ inputs.customEthClientDockerImage }}
       run: |
-        echo ::add-mask::$CHAINLINK_IMAGE
         function convert_to_toml_array() {
           local IFS=','
           local input_array=($1)
@@ -85,12 +53,6 @@ runs:
         selected_networks=$(convert_to_toml_array "$SELECTED_NETWORKS")
         log_targets=$(convert_to_toml_array "$LOGSTREAM_LOG_TARGETS")
 
-        if [ -n "$PYROSCOPE_SERVER" ]; then
-          pyroscope_enabled=true
-        else
-          pyroscope_enabled=false
-        fi
-
         if [ -n "$TEST_LOG_COLLECT" ]; then
           test_log_collect=true
         else
@@ -122,33 +84,16 @@ runs:
         selected_networks=$selected_networks
 
         [ChainlinkImage]
-        image="$CHAINLINK_IMAGE"
         version="$CHAINLINK_VERSION"
         postgres_version="$CHAINLINK_POSTGRES_VERSION"
 
-        [Pyroscope]
-        enabled=$pyroscope_enabled
-        server_url="$PYROSCOPE_SERVER"
-        environment="$PYROSCOPE_ENVIRONMENT"
-        key_secret="$PYROSCOPE_KEY"
-
         [Logging]
         test_log_collect=$test_log_collect
         run_id="$RUN_ID"
 
         [Logging.LogStream]
         log_targets=$log_targets
 
-        [Logging.Loki]
-        tenant_id="$LOKI_TENANT_ID"
-        endpoint="$LOKI_ENDPOINT"
-        basic_auth_secret="$LOKI_BASIC_AUTH"
-
-        [Logging.Grafana]
-        base_url="$GRAFANA_URL"
-        dashboard_url="$GRAFANA_DASHBOARD_URL"
-        $grafana_bearer_token
-
         [PrivateEthereumNetwork]
         execution_layer="$execution_layer"
         $ethereum_version

diff --git a/.github/workflows/automation-ondemand-tests.yml b/.github/workflows/automation-ondemand-tests.yml
@@ -8,8 +8,10 @@ on:
         type: string
       chainlinkImage:
         description: Chainlink image repo to use (Leave empty to build from head/ref)
-        required: false
-        type: string
+        options:
+          - public.ecr.aws/chainlink/chainlink
+          - QA_ECR
+        type: choice
       chainlinkVersionUpdate:
         description: Chainlink image version to use initially for upgrade test
         default: latest
@@ -18,8 +20,10 @@ on:
       chainlinkImageUpdate:
         description: Chainlink image repo to use initially for upgrade test
         required: true
-        default: public.ecr.aws/chainlink/chainlink
-        type: string
+        options:
+          - public.ecr.aws/chainlink/chainlink
+          - QA_ECR
+        type: choice
       enableChaos:
         description: Check to enable chaos tests
         type: boolean
@@ -198,23 +202,24 @@ jobs:
         id: determine-build
         shell: bash
         run: |
-          if [[ "${{ inputs.chainlinkImage }}" == "" ]]; then
+          if [[ "${{ inputs.chainlinkImage }}" == "QA_ECR" ]]; then
             echo "image=${{ env.CHAINLINK_IMAGE }}" >>$GITHUB_OUTPUT
             echo "version=${{ github.sha }}" >>$GITHUB_OUTPUT
             echo "upgrade_version=${{ github.sha }}" >>$GITHUB_OUTPUT
             echo "upgrade_image=${{ env.CHAINLINK_IMAGE }}" >>$GITHUB_OUTPUT
-          else
-            READ_CL_IMAGE=$(jq -r '.inputs.chainlinkImage' $GITHUB_EVENT_PATH)
-            echo ::add-mask::$READ_CL_IMAGE
+          elif [[ "${{ inputs.chainlinkImage }}" == "public.ecr.aws/chainlink/chainlink" ]]; then
+            READ_CL_IMAGE="public.ecr.aws/chainlink/chainlink"
             echo "image=$READ_CL_IMAGE" >>$GITHUB_OUTPUT
             echo "version=${{ inputs.chainlinkVersion }}" >>$GITHUB_OUTPUT
             echo "upgrade_version=${{ inputs.chainlinkVersion }}" >>$GITHUB_OUTPUT
             echo "upgrade_image=$READ_CL_IMAGE" >>$GITHUB_OUTPUT
           fi
           if [[ "${{ matrix.tests.type }}" == "upgrade" ]]; then
-            READ_CL_UPGR_IMAGE=$(jq -r '.inputs.chainlinkImageUpdate' $GITHUB_EVENT_PATH)
-            echo ::add-mask::$READ_CL_UPGR_IMAGE
-            echo "image=$READ_CL_UPGR_IMAGE" >>$GITHUB_OUTPUT
+            if [[ "${{ inputs.chainlinkImageUpdate }}" == "QA_ECR" ]]; then
+              echo "image=${{ env.CHAINLINK_IMAGE }}" >>$GITHUB_OUTPUT
+            elif [[ "${{ inputs.chainlinkImageUpdate }}" == "public.ecr.aws/chainlink/chainlink" ]]; then
+              echo "image=public.ecr.aws/chainlink/chainlink" >>$GITHUB_OUTPUT
+            fi
             echo "version=${{ inputs.chainlinkVersionUpdate }}" >>$GITHUB_OUTPUT
           fi
       - name: Setup GAP for Grafana
@@ -233,18 +238,9 @@ jobs:
           OLD_VERSION: ${{ steps.determine-build.outputs.version }}          
           UPGRADE_VERSION: ${{ steps.determine-build.outputs.upgrade_version }}
           UPGRADE_IMAGE: ${{ steps.determine-build.outputs.upgrade_image }}
-          PYROSCOPE_SERVER: ${{ matrix.tests.pyroscope_env == '' && '' || !startsWith(github.ref, 'refs/tags/') && '' || secrets.QA_PYROSCOPE_INSTANCE }} # Avoid sending blank envs https://github.com/orgs/community/discussions/25725
-          PYROSCOPE_ENVIRONMENT: ${{ matrix.tests.pyroscope_env }}
-          PYROSCOPE_KEY: ${{ secrets.QA_PYROSCOPE_KEY }}
           RUN_ID: ${{ github.run_id }}
           TEST_LOG_COLLECT: "true"
-          LOKI_ENDPOINT: https://${{ secrets.GRAFANA_INTERNAL_HOST }}/loki/api/v1/push
-          LOKI_TENANT_ID: ${{ secrets.GRAFANA_INTERNAL_TENANT_ID }}
-          LOKI_BASIC_AUTH:  ${{ secrets.GRAFANA_INTERNAL_BASIC_AUTH }}
           LOGSTREAM_LOG_TARGETS: ${{ vars.LOGSTREAM_LOG_TARGETS }}
-          GRAFANA_URL: "http://localhost:8080/primary"
-          GRAFANA_DASHBOARD_URL: "/d/ddf75041-1e39-42af-aa46-361fe4c36e9e/ci-e2e-tests-logs"
-          $GRAFANA_BEARER_TOKEN: ${{ secrets.GRAFANA_INTERNAL_URL_SHORTENER_TOKEN }}
         run: |
           echo ::add-mask::$UPGRADE_IMAGE
           echo ::add-mask::$OLD_IMAGE
@@ -254,12 +250,6 @@ jobs:
           
           selected_networks=$(convert_to_toml_array "$SELECTED_NETWORKS")          
 
-          if [ -n "$PYROSCOPE_SERVER" ]; then
-            pyroscope_enabled=true
-          else
-            pyroscope_enabled=false
-          fi          
-
           if [ -n "$TEST_LOG_COLLECT" ]; then
           test_log_collect=true
           else
@@ -268,11 +258,6 @@ jobs:
 
           log_targets=$(convert_to_toml_array "$LOGSTREAM_LOG_TARGETS")
 
-          grafana_bearer_token=""
-          if [ -n "$GRAFANA_BEARER_TOKEN" ]; then
-            grafana_bearer_token="bearer_token_secret=\"$GRAFANA_BEARER_TOKEN\""
-          fi
-
           cat << EOF > config.toml
           [Network]
           selected_networks=$selected_networks
@@ -291,33 +276,18 @@ jobs:
 
           [Logging.LogStream]
           log_targets=$log_targets
-
-          [Logging.Loki]
-          tenant_id="$LOKI_TENANT_ID"
-          endpoint="$LOKI_ENDPOINT"
-          basic_auth_secret="$LOKI_BASIC_AUTH"
-
-          [Logging.Grafana]
-          base_url="$GRAFANA_URL"
-          dashboard_url="$GRAFANA_DASHBOARD_URL"
-          $grafana_bearer_token
-
-          [Pyroscope]
-          enabled=$pyroscope_enabled
-          server_url="$PYROSCOPE_SERVER"
-          environment="$PYROSCOPE_ENVIRONMENT"
-          key_secret="$PYROSCOPE_KEY"          
           EOF
-          
+
           BASE64_CONFIG_OVERRIDE=$(cat config.toml | base64 -w 0)
           echo ::add-mask::$BASE64_CONFIG_OVERRIDE
           echo "BASE64_CONFIG_OVERRIDE=$BASE64_CONFIG_OVERRIDE" >> $GITHUB_ENV               
       - name: Run Tests
-        uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/run-tests@af92c5fae8dcf1659201e907db82d221fc304b94 # v2.3.21
+        uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/run-tests@75a9005952a9e905649cfb5a6971fd9429436acd # v2.3.24
         if: ${{ matrix.tests.enabled == true }}
         env:
           TEST_SUITE: ${{ matrix.tests.suite }}
         with:
+          test_config_override_base64: ${{ env.BASE64_CONFIG_OVERRIDE }}
           test_command_to_run: cd ./integration-tests && go test -timeout 60m -count=1 -json -test.parallel=${{ matrix.tests.nodes }} ${{ matrix.tests.command }} 2>&1 | tee /tmp/gotest.log | gotestloghelper -ci -singlepackage -hidepassingtests=false -hidepassinglogs
           test_download_vendor_packages_command: cd ./integration-tests && go mod download
           cl_repo: ${{ steps.determine-build.outputs.image }}
@@ -330,6 +300,17 @@ jobs:
           QA_AWS_REGION: ${{ secrets.QA_AWS_REGION }}
           QA_AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }}
           QA_KUBECONFIG: ${{ secrets.QA_KUBECONFIG }}
+          DEFAULT_LOKI_TENANT_ID: ${{ secrets.GRAFANA_INTERNAL_TENANT_ID }}
+          DEFAULT_LOKI_ENDPOINT: https://${{ secrets.GRAFANA_INTERNAL_HOST }}/loki/api/v1/push
+          DEFAULT_LOKI_BASIC_AUTH: ${{ secrets.GRAFANA_INTERNAL_BASIC_AUTH }}
+          DEFAULT_GRAFANA_BASE_URL: "http://localhost:8080/primary"
+          DEFAULT_GRAFANA_DASHBOARD_URL: "/d/ddf75041-1e39-42af-aa46-361fe4c36e9e/ci-e2e-tests-logs"
+          DEFAULT_GRAFANA_BEARER_TOKEN: ${{ secrets.GRAFANA_INTERNAL_URL_SHORTENER_TOKEN }}
+          DEFAULT_PYROSCOPE_SERVER_URL: ${{ matrix.tests.pyroscope_env == '' && '' || !startsWith(github.ref, 'refs/tags/') && '' || secrets.QA_PYROSCOPE_INSTANCE }} # Avoid sending blank envs https://github.com/orgs/community/discussions/25725
+          DEFAULT_PYROSCOPE_KEY: ${{ secrets.QA_PYROSCOPE_KEY }}
+          DEFAULT_PYROSCOPE_ENVIRONMENT: ${{ matrix.tests.pyroscope_env }}
+          DEFAULT_PYROSCOPE_ENABLED: ${{ matrix.tests.pyroscope_env == '' || !startsWith(github.ref, 'refs/tags/') && 'false' || 'true' }}
+
       - name: Upload test log
         uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # v4.3.1
         if: failure()

diff --git a/.github/workflows/client-compatibility-tests.yml b/.github/workflows/client-compatibility-tests.yml
@@ -544,20 +544,10 @@ jobs:
           runId: ${{ github.run_id }}
           testLogCollect: ${{ vars.TEST_LOG_COLLECT }}
           selectedNetworks: ${{ env.SELECTED_NETWORKS }}
-          chainlinkImage: ${{ env.CHAINLINK_IMAGE }}
           chainlinkVersion: ${{ needs.select-versions.outputs.chainlink_version }}
-          lokiEndpoint: https://${{ secrets.GRAFANA_INTERNAL_HOST }}/loki/api/v1/push
-          lokiTenantId: ${{ secrets.GRAFANA_INTERNAL_TENANT_ID }}
-          lokiBasicAuth:  ${{ secrets.GRAFANA_INTERNAL_BASIC_AUTH }}
           logstreamLogTargets: ${{ vars.LOGSTREAM_LOG_TARGETS }}
-          grafanaUrl: "http://localhost:8080/primary"
-          grafanaDashboardUrl: "/d/ddf75041-1e39-42af-aa46-361fe4c36e9e/ci-e2e-tests-logs"
-          grafanaBearerToken: ${{ secrets.GRAFANA_INTERNAL_URL_SHORTENER_TOKEN }}
           ethExecutionClient: ${{ matrix.evm_node.eth_implementation }}
           customEthClientDockerImage: ${{ matrix.evm_node.docker_image }}
-          pyroscopeServer: ${{ secrets.QA_PYROSCOPE_INSTANCE }}
-          pyroscopeEnvironment: ci-client-compatability-${{ matrix.eth_client }}-testnet
-          pyroscopeKey: ${{ secrets.QA_PYROSCOPE_KEY }}
       - name: Prepare test log name
         run: |
           replace_special_chars() {
@@ -581,7 +571,7 @@ jobs:
           }
           echo "TEST_LOG_NAME=$(replace_special_chars "${{ matrix.evm_node.product }}-${{ matrix.evm_node.docker_image }}-test-logs")" >> $GITHUB_ENV
       - name: Run Tests
-        uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/run-tests@af92c5fae8dcf1659201e907db82d221fc304b94 # v2.3.21
+        uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/run-tests@75a9005952a9e905649cfb5a6971fd9429436acd # v2.3.24
         with:
           test_command_to_run: cd ./integration-tests && touch .root_dir && go test -timeout 30m -count=1 -json ${{ matrix.evm_node.run }} 2>&1 | tee /tmp/gotest.log | gotestloghelper -ci -singlepackage -hidepassingtests=false -hidepassinglogs
           test_download_vendor_packages_command: cd ./integration-tests && go mod download
@@ -603,6 +593,18 @@ jobs:
           should_tidy: "false"
           go_coverage_src_dir: /var/tmp/go-coverage
           go_coverage_dest_dir: ${{ github.workspace }}/.covdata
+          DEFAULT_CHAINLINK_IMAGE: ${{ env.CHAINLINK_IMAGE }}
+          DEFAULT_LOKI_TENANT_ID: ${{ secrets.GRAFANA_INTERNAL_TENANT_ID }}
+          DEFAULT_LOKI_ENDPOINT: https://${{ secrets.GRAFANA_INTERNAL_HOST }}/loki/api/v1/push
+          DEFAULT_LOKI_BASIC_AUTH: ${{ secrets.GRAFANA_INTERNAL_BASIC_AUTH }}
+          DEFAULT_GRAFANA_BASE_URL: "http://localhost:8080/primary"
+          DEFAULT_GRAFANA_DASHBOARD_URL: "/d/ddf75041-1e39-42af-aa46-361fe4c36e9e/ci-e2e-tests-logs"
+          DEFAULT_GRAFANA_BEARER_TOKEN: ${{ secrets.GRAFANA_INTERNAL_URL_SHORTENER_TOKEN }}
+          DEFAULT_PYROSCOPE_SERVER_URL: ${{ secrets.QA_PYROSCOPE_INSTANCE }}
+          DEFAULT_PYROSCOPE_KEY: ${{ secrets.QA_PYROSCOPE_KEY }}
+          DEFAULT_PYROSCOPE_ENVIRONMENT: ci-client-compatability-${{ matrix.eth_client }}-testnet
+          DEFAULT_PYROSCOPE_ENABLED: "true"
+
       - name: Print failed test summary
         if: always()
         uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/show-test-summary@fc3e0df622521019f50d772726d6bf8dc919dd38 # v2.3.19