Update CI workflows for E2E tests #13740
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,149 @@ | ||
| name: 'Create Default E2E Test Config Override From Inputs' | ||
| inputs: | ||
| chainlink_image: | ||
| required: false | ||
| description: 'Chainlink image to use for the tests. Example: "chainlink/chainlink:develop"' | ||
| chainlink_version: | ||
| required: false | ||
| description: 'Chainlink version to use for the tests. Example: "v2.10.0" or sha' | ||
| chainlink_postgres_version: | ||
| required: false | ||
| description: 'Chainlink Postgres version to use for the tests' | ||
| chainlink_upgrade_image: | ||
| required: false | ||
| description: 'Chainlink upgrade image to use for the tests' | ||
| chainlink_upgrade_version: | ||
| required: false | ||
| description: 'Chainlink upgrade version to use for the tests' | ||
| selected_networks: | ||
| required: false | ||
| description: 'Selected networks for the tests (comma separated)' | ||
| pyroscope_server: | ||
| required: false | ||
| description: 'Pyroscope server for the tests' | ||
| pyroscope_environment: | ||
| required: false | ||
| description: 'Pyroscope environment for the tests' | ||
| pyroscope_key: | ||
| required: false | ||
| description: 'Pyroscope key for the tests' | ||
| test_log_collect: | ||
| required: false | ||
| description: 'Whether to always collect test logs' | ||
| logging_run_id: | ||
| required: false | ||
| description: 'Logging run ID for the tests' | ||
| loki_endpoint: | ||
| required: false | ||
| description: 'Loki endpoint for the tests' | ||
| loki_tenant_id: | ||
| required: false | ||
| description: 'Loki tenant ID for the tests' | ||
| loki_basic_auth: | ||
| required: false | ||
| description: 'Loki basic authentication for the tests' | ||
| logstream_log_targets: | ||
| required: false | ||
| description: 'Logstream log targets for the tests (comma separated)' | ||
| grafana_url: | ||
| required: false | ||
| description: 'Grafana URL for the tests' | ||
| grafana_dashboard_url: | ||
| required: false | ||
| description: 'Grafana dashboard URL for the tests' | ||
| grafana_bearer_token: | ||
| required: false | ||
| description: 'Grafana bearer token for the tests' | ||
| private_ethereum_network_execution_layer: | ||
| required: false | ||
| description: 'Private Ethereum network execution layer for the tests' | ||
| private_ethereum_network_custom_docker_image: | ||
| required: false | ||
| description: 'Private Ethereum network custom Docker image for the tests' | ||
| outputs: | ||
| chainlink_image: | ||
| value: ${{ steps.build-args.outputs.chainlink_image }} | ||
| description: 'Chainlink image resolved from the input' | ||
| chainlink_version: | ||
| value: ${{ steps.build-args.outputs.chainlink_version }} | ||
| description: 'Chainlink version resolved from the input' | ||
|
|
||
| runs: | ||
| using: 'composite' | ||
| steps: | ||
| - name: Create default E2E test config override | ||
| id: build-args | ||
| shell: bash | ||
| run: | | ||
| cd integration-tests/e2e_tests_ci_tool/ | ||
|
|
||
| create_args=() | ||
| if [ -n "${{ inputs.chainlink_image }}" ]; then | ||
| create_args+=(--chainlink-image="${{ inputs.chainlink_image }}") | ||
| echo "chainlink_image=$(go run main.go envresolve "${{ inputs.chainlink_image }}")" >> $GITHUB_OUTPUT | ||
| fi | ||
| if [ -n "${{ inputs.chainlink_version }}" ]; then | ||
| create_args+=(--chainlink-version="${{ inputs.chainlink_version }}") | ||
| echo "chainlink_version=$(go run main.go envresolve "${{ inputs.chainlink_version }}")" >> $GITHUB_OUTPUT | ||
| fi | ||
| if [ -n "${{ inputs.chainlink_postgres_version }}" ]; then | ||
| create_args+=(--chainlink-postgres-version="${{ inputs.chainlink_postgres_version }}") | ||
| fi | ||
| if [ -n "${{ inputs.chainlink_upgrade_image }}" ]; then | ||
| create_args+=(--chainlink-upgrade-image="${{ inputs.chainlink_upgrade_image }}") | ||
| fi | ||
| if [ -n "${{ inputs.chainlink_upgrade_version }}" ]; then | ||
| create_args+=(--chainlink-upgrade-version="${{ inputs.chainlink_upgrade_version }}") | ||
| fi | ||
| if [ -n "${{ inputs.pyroscope_server }}" ]; then | ||
| create_args+=(--pyroscope-server-url="${{ inputs.pyroscope_server }}") | ||
| fi | ||
| if [ -n "${{ inputs.pyroscope_environment }}" ]; then | ||
| create_args+=(--pyroscope-environment="${{ inputs.pyroscope_environment }}") | ||
| fi | ||
| if [ -n "${{ inputs.pyroscope_key }}" ]; then | ||
| create_args+=(--pyroscope-key="${{ inputs.pyroscope_key }}") | ||
| fi | ||
| if [ -n "${{ inputs.loki_endpoint }}" ]; then | ||
| create_args+=(--logging-loki-endpoint="${{ inputs.loki_endpoint }}") | ||
| fi | ||
| if [ -n "${{ inputs.loki_tenant_id }}" ]; then | ||
| create_args+=(--logging-loki-tenant-id="${{ inputs.loki_tenant_id }}") | ||
| fi | ||
| if [ -n "${{ inputs.loki_basic_auth }}" ]; then | ||
| create_args+=(--logging-loki-basic-auth="${{ inputs.loki_basic_auth }}") | ||
| fi | ||
| if [ -n "${{ inputs.logging_run_id }}" ]; then | ||
| create_args+=(--logging-run-id="${{ inputs.logging_run_id }}") | ||
| fi | ||
| if [ -n "${{ inputs.test_log_collect }}" ]; then | ||
| create_args+=(--logging-test-log-collect="${{ inputs.test_log_collect }}") | ||
| fi | ||
| if [ -n "${{ inputs.private_ethereum_network_execution_layer }}" ]; then | ||
| create_args+=(--private-ethereum-network-execution-layer="${{ inputs.private_ethereum_network_execution_layer }}") | ||
| fi | ||
| if [ -n "${{ inputs.private_ethereum_network_custom_docker_image }}" ]; then | ||
| create_args+=(--private-ethereum-network-custom-docker-image="${{ inputs.private_ethereum_network_custom_docker_image }}") | ||
| fi | ||
|
There was a problem hiding this comment. this is scary... can we get rid of it somehow? or if see if it's possible to loop over all input fields? |
||
|
|
||
| # # Split the log targets input by comma and add to the command line arguments | ||
| IFS=',' read -ra ADDR <<< "${{ inputs.logstream_log_targets }}" | ||
| for i in "${ADDR[@]}"; do | ||
| create_args+=(--logging-log-targets="$i") | ||
| done | ||
|
|
||
| # Split selected_networks input by comma and add to the command line arguments | ||
| IFS=',' read -ra ADDR <<< "${{ inputs.selected_networks }}" | ||
| for i in "${ADDR[@]}"; do | ||
| create_args+=(--selected-networks="$i") | ||
|
There was a problem hiding this comment. when it comes to these two, maybe we could modify the tool so that it understands various formats and we don't have do that split in bash and instead we'd do that in go? |
||
| done | ||
|
|
||
| config_override=$(go run main.go test-config create "${create_args[@]}") | ||
| # echo "go run main.go test-config create ${create_args[*]}" | ||
| # echo "Output:" | ||
| # echo "$config_override" | ||
|
|
||
|
|
||
| BASE64_CONFIG_OVERRIDE=$(echo "$config_override" | base64 -w 0) | ||
| echo ::add-mask::$BASE64_CONFIG_OVERRIDE | ||
| echo "BASE64_CONFIG_OVERRIDE=$BASE64_CONFIG_OVERRIDE" >> $GITHUB_ENV | ||
|
There was a problem hiding this comment. can we do all/some of these operations inside the tool? pretty sure we could add a |
||
|
|
||
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,128 @@ | ||
| # This file specifies the GitHub runner for each E2E test and is utilized by all E2E CI workflows. | ||
| # | ||
| # Each entry in this file includes the following: | ||
| # - The GitHub runner (runs-on field) that will execute tests. | ||
| # - The tests that will be run by the runner. | ||
| # - The workflows (e.g., Run PR E2E Tests, Run Nightly E2E Tests) that should trigger these tests. | ||
| # | ||
| runner-test-matrix: | ||
|
|
||
| # Example of 1 runner for all tests in integration-tests/smoke/ocr_test.go | ||
| - id: smoke/ocr_test.go:* | ||
| path: integration-tests/smoke/ocr_test.go | ||
| test-type: docker | ||
| runs-on: ubuntu-latest | ||
| workflows: | ||
| - Run PR E2E Tests | ||
| - Run Nightly E2E Tests | ||
| test-cmd: cd integration-tests/ && go test smoke/ocr_test.go -timeout 30m -count=1 -json | ||
| pyroscope-env: ci-smoke-ocr-evm-simulated-nightly | ||
|
|
||
| # Example of 2 separate runners for the same test file but different tests. Can be used if tests if are too heavy to run on the same runner | ||
| - id: smoke/ocr2_test.go:^TestOCRv2Request$ | ||
| path: integration-tests/smoke/ocr2_test.go | ||
| test-type: docker | ||
| runs-on: ubuntu-latest | ||
| workflows: | ||
| - Run PR E2E Tests | ||
| - Run Nightly E2E Tests | ||
| test-cmd: cd integration-tests/ && go test smoke/ocr2_test.go -test.run ^TestOCRv2Request$ -test.parallel=1 -timeout 30m -count=1 -json | ||
|
There was a problem hiding this comment. wonder if that's explicitly stated somewhere (where?) that it's up to the person defining a workflow to make sure that |
||
| pyroscope-env: ci-smoke-ocr2-evm-simulated-nightly | ||
|
|
||
| - id: smoke/ocr2_test.go:^TestOCRv2Basic$ | ||
| path: integration-tests/smoke/ocr2_test.go | ||
| test-type: docker | ||
| runs-on: ubuntu-latest | ||
| workflows: | ||
| - Run PR E2E Tests | ||
| - Run Nightly E2E Tests | ||
| test-cmd: cd integration-tests/ && go test smoke/ocr2_test.go -test.run ^TestOCRv2Basic$ -test.parallel=1 -timeout 30m -count=1 -json | ||
| pyroscope-env: ci-smoke-ocr2-evm-simulated-nightly | ||
|
|
||
| # Automation upgrade tests | ||
|
There was a problem hiding this comment. Are we only running automation and OCR tests as a part of this? Are others being added on later? There was a problem hiding this comment. @kalverra we will add all of the E2E tests in the next PRs There was a problem hiding this comment. @lukaszcl I understand we will also remove all the other actions we have currently that deal with base64 config once we have migrated all workflows to the new way of life? do we have a ticket remind us to do so? |
||
|
|
||
| - id: smoke/automation_upgrade_test.go:^TestAutomationNodeUpgrade/registry_2_0 | ||
| path: integration-tests/smoke/automation_upgrade_test.go | ||
| test-type: docker | ||
| runs-on: ubuntu22.04-8cores-32GB | ||
| workflows: | ||
| - Run Automation Product Nightly E2E Tests | ||
| test-cmd: cd integration-tests/smoke && go test -test.run ^TestAutomationNodeUpgrade/registry_2_0 -test.parallel=1 -timeout 60m -count=1 -json | ||
skudasov marked this conversation as resolved.
Show resolved
Hide resolved
|
||
| test-inputs: | ||
| chainlinkImage: public.ecr.aws/chainlink/chainlink | ||
| chainlinkVersion: latest | ||
| chainlinkUpgradeImage: '{{ env.QA_CHAINLINK_IMAGE }}' | ||
| chainlinkUpgradeVersion: develop | ||
| pyroscope-env: ci-smoke-automation-upgrade-tests | ||
|
|
||
| - id: smoke/automation_upgrade_test.go:^TestAutomationNodeUpgrade/registry_2_1 | ||
| path: integration-tests/smoke/automation_upgrade_test.go | ||
| test-type: docker | ||
| runs-on: ubuntu22.04-8cores-32GB | ||
| workflows: | ||
| - Run Automation Product Nightly E2E Tests | ||
| test-cmd: cd integration-tests/smoke && go test -test.run ^TestAutomationNodeUpgrade/registry_2_1 -test.parallel=5 -timeout 60m -count=1 -json | ||
| test-inputs: | ||
| chainlinkImage: public.ecr.aws/chainlink/chainlink | ||
| chainlinkVersion: latest | ||
| chainlinkUpgradeImage: '{{ env.QA_CHAINLINK_IMAGE }}' | ||
| chainlinkUpgradeVersion: develop | ||
| pyroscope-env: ci-smoke-automation-upgrade-tests | ||
|
|
||
| - id: smoke/automation_upgrade_test.go:^TestAutomationNodeUpgrade/registry_2_2 | ||
| path: integration-tests/smoke/automation_upgrade_test.go | ||
| test-type: docker | ||
| runs-on: ubuntu22.04-8cores-32GB | ||
| workflows: | ||
| - Run Automation Product Nightly E2E Tests | ||
| test-cmd: cd integration-tests/smoke && go test -test.run ^TestAutomationNodeUpgrade/registry_2_2 -test.parallel=5 -timeout 60m -count=1 -json | ||
| test-inputs: | ||
| chainlinkImage: public.ecr.aws/chainlink/chainlink | ||
| chainlinkVersion: latest | ||
| chainlinkUpgradeImage: '{{ env.QA_CHAINLINK_IMAGE }}' | ||
| chainlinkUpgradeVersion: develop | ||
| pyroscope-env: ci-smoke-automation-upgrade-tests | ||
|
|
||
| # END Automation upgrade tests | ||
|
|
||
| - id: integration-tests/reorg/automation_reorg_test.go | ||
| path: integration-tests/reorg/automation_reorg_test.go | ||
| test-type: k8s-remote-runner | ||
| remote-runner-test-suite: reorg | ||
| runs-on: ubuntu-latest | ||
| workflows: | ||
| # - Run Automation Product Nightly E2E Tests | ||
| test-cmd: cd integration-tests/reorg && go test -v -test.run ^TestAutomationReorg$ -test.parallel=5 -timeout 60m -count=1 | ||
| pyroscope-env: ci-automation-on-demand-reorg | ||
|
|
||
| - id: integration-tests/chaos/automation_chaos_test.go | ||
| path: integration-tests/chaos/automation_chaos_test.go | ||
| test-type: k8s-remote-runner | ||
| remote-runner-test-suite: chaos | ||
| runs-on: ubuntu-latest | ||
| workflows: | ||
| # - Run Automation Product Nightly E2E Tests | ||
| test-cmd: cd integration-tests/chaos && go test -v -test.run ^TestAutomationChaos$ -test.parallel=15 -timeout 60m -count=1 | ||
| pyroscope-env: ci-automation-on-demand-chaos | ||
|
|
||
| # Example of a configuration for running a single soak test in Kubernetes Remote Runner | ||
| - id: soak/ocr_test.go:^TestOCRv1Soak$ | ||
| path: integration-tests/soak/ocr_test.go | ||
| test-type: k8s-remote-runner | ||
| remote-runner-test-suite: soak | ||
| runs-on: ubuntu-latest | ||
| # workflows: | ||
| # - Run Nightly E2E Tests | ||
| test-cmd: cd integration-tests/ && go test soak/ocr_test.go -v -test.run ^TestOCRv1Soak$ -test.parallel=1 -timeout 30m -count=1 | ||
| pyroscope-env: ci-smoke-ocr2-evm-simulated-nightly | ||
|
|
||
| - id: integration-tests/benchmark/keeper_test.go:^TestAutomationBenchmark$ | ||
| path: integration-tests/benchmark/keeper_test.go | ||
| test-type: k8s-remote-runner | ||
| remote-runner-test-suite: benchmark | ||
| remote-runner-memory: 4Gi | ||
| runs-on: ubuntu-latest | ||
| # workflows: | ||
| # - Run Nightly E2E Tests | ||
| test-cmd: cd integration-tests/benchmark && go test -v -test.run ^TestAutomationBenchmark$ -test.parallel=1 -timeout 30m -count=1 | ||
| pyroscope-env: ci-benchmark-automation-nightly | ||
Oops, something went wrong.
Oops, something went wrong.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
these two worry me in the context of CCIP, which might be passing here multiple (comma-separated?) values? have you checked whether this structure fits into what CCIP needs?