From e0850a6a31843606015d1c49d52b5a6ad8727378 Mon Sep 17 00:00:00 2001 From: Domino Valdano Date: Thu, 8 Aug 2024 10:53:43 -0700 Subject: [PATCH 1/3] BCI-3492 [LogPoller]: Allow withObservedExecAndRowsAffected to report non-zero rows affected (#14057) * Fix withObservedExecAndRowsAffected Also: - Change behavior of DeleteExpiredLogs to delete logs which don't match any filter - Add a test case to ensure the dataset size is published properly during pruning * pnpm changeset * changeset #fix -> #bugfix --- .changeset/sweet-pumas-refuse.md | 5 ++++ core/chains/evm/logpoller/observability.go | 2 +- .../evm/logpoller/observability_test.go | 11 +++++++ core/chains/evm/logpoller/orm.go | 30 ++++++++----------- core/chains/evm/logpoller/orm_test.go | 13 ++++---- 5 files changed, 37 insertions(+), 24 deletions(-) create mode 100644 .changeset/sweet-pumas-refuse.md diff --git a/.changeset/sweet-pumas-refuse.md b/.changeset/sweet-pumas-refuse.md new file mode 100644 index 00000000000..fd642a9c94c --- /dev/null +++ b/.changeset/sweet-pumas-refuse.md @@ -0,0 +1,5 @@ +--- +"chainlink": minor +--- + +#bugfix Addresses 2 minor issues with the pruning of LogPoller's db tables: logs not matching any filter will now be pruned, and rows deleted are now properly reported for observability diff --git a/core/chains/evm/logpoller/observability.go b/core/chains/evm/logpoller/observability.go index 7842a060eca..782307e7d06 100644 --- a/core/chains/evm/logpoller/observability.go +++ b/core/chains/evm/logpoller/observability.go @@ -285,7 +285,7 @@ func withObservedExecAndRowsAffected(o *ObservedORM, queryName string, queryType WithLabelValues(o.chainId, queryName, string(queryType)). Observe(float64(time.Since(queryStarted))) - if err != nil { + if err == nil { o.datasetSize. WithLabelValues(o.chainId, queryName, string(queryType)). Set(float64(rowsAffected)) diff --git a/core/chains/evm/logpoller/observability_test.go b/core/chains/evm/logpoller/observability_test.go index 78c27b4b8f7..4ea7adceab0 100644 --- a/core/chains/evm/logpoller/observability_test.go +++ b/core/chains/evm/logpoller/observability_test.go @@ -16,6 +16,7 @@ import ( "github.com/prometheus/client_golang/prometheus/testutil" "github.com/smartcontractkit/chainlink-common/pkg/logger" + "github.com/smartcontractkit/chainlink/v2/core/chains/evm/utils" ubig "github.com/smartcontractkit/chainlink/v2/core/chains/evm/utils/big" "github.com/smartcontractkit/chainlink/v2/core/internal/testutils" @@ -117,6 +118,16 @@ func TestCountersAreProperlyPopulatedForWrites(t *testing.T) { assert.Equal(t, float64(20), testutil.ToFloat64(orm.logsInserted.WithLabelValues("420"))) assert.Equal(t, float64(2), testutil.ToFloat64(orm.blocksInserted.WithLabelValues("420"))) + rowsAffected, err := orm.DeleteExpiredLogs(ctx, 3) + require.NoError(t, err) + require.Equal(t, int64(3), rowsAffected) + assert.Equal(t, 3, counterFromGaugeByLabels(orm.datasetSize, "420", "DeleteExpiredLogs", "delete")) + + rowsAffected, err = orm.DeleteBlocksBefore(ctx, 30, 0) + require.NoError(t, err) + require.Equal(t, int64(2), rowsAffected) + assert.Equal(t, 2, counterFromGaugeByLabels(orm.datasetSize, "420", "DeleteBlocksBefore", "delete")) + // Don't update counters in case of an error require.Error(t, orm.InsertLogsWithBlock(ctx, logs, NewLogPollerBlock(utils.RandomBytes32(), 0, time.Now(), 0))) assert.Equal(t, float64(20), testutil.ToFloat64(orm.logsInserted.WithLabelValues("420"))) diff --git a/core/chains/evm/logpoller/orm.go b/core/chains/evm/logpoller/orm.go index 1d249760736..22870efccf3 100644 --- a/core/chains/evm/logpoller/orm.go +++ b/core/chains/evm/logpoller/orm.go @@ -15,6 +15,7 @@ import ( "github.com/smartcontractkit/chainlink-common/pkg/types/query" "github.com/smartcontractkit/chainlink-common/pkg/sqlutil" + evmtypes "github.com/smartcontractkit/chainlink/v2/core/chains/evm/types" "github.com/smartcontractkit/chainlink/v2/core/chains/evm/types" @@ -313,34 +314,29 @@ type Exp struct { ShouldDelete bool } +// DeleteExpiredLogs removes any logs which either: +// - don't match any currently registered filters, or +// - have a timestamp older than any matching filter's retention, UNLESS there is at +// least one matching filter with retention=0 func (o *DSORM) DeleteExpiredLogs(ctx context.Context, limit int64) (int64, error) { var err error var result sql.Result - if limit > 0 { - result, err = o.ds.ExecContext(ctx, ` - DELETE FROM evm.logs + query := `DELETE FROM evm.logs WHERE (evm_chain_id, address, event_sig, block_number) IN ( SELECT l.evm_chain_id, l.address, l.event_sig, l.block_number FROM evm.logs l - INNER JOIN ( - SELECT address, event, MAX(retention) AS retention + LEFT JOIN ( + SELECT address, event, CASE WHEN MIN(retention) = 0 THEN 0 ELSE MAX(retention) END AS retention FROM evm.log_poller_filters WHERE evm_chain_id = $1 GROUP BY evm_chain_id, address, event - HAVING NOT 0 = ANY(ARRAY_AGG(retention)) ) r ON l.evm_chain_id = $1 AND l.address = r.address AND l.event_sig = r.event - AND l.block_timestamp <= STATEMENT_TIMESTAMP() - (r.retention / 10^9 * interval '1 second') - LIMIT $2 - )`, ubig.New(o.chainID), limit) + WHERE r.retention IS NULL OR (r.retention != 0 AND l.block_timestamp <= STATEMENT_TIMESTAMP() - (r.retention / 10^9 * interval '1 second')) %s)` + + if limit > 0 { + result, err = o.ds.ExecContext(ctx, fmt.Sprintf(query, "LIMIT $2"), ubig.New(o.chainID), limit) } else { - result, err = o.ds.ExecContext(ctx, `WITH r AS - ( SELECT address, event, MAX(retention) AS retention - FROM evm.log_poller_filters WHERE evm_chain_id=$1 - GROUP BY evm_chain_id,address, event HAVING NOT 0 = ANY(ARRAY_AGG(retention)) - ) DELETE FROM evm.logs l USING r - WHERE l.evm_chain_id = $1 AND l.address=r.address AND l.event_sig=r.event - AND l.block_timestamp <= STATEMENT_TIMESTAMP() - (r.retention / 10^9 * interval '1 second')`, // retention is in nanoseconds (time.Duration aka BIGINT) - ubig.New(o.chainID)) + result, err = o.ds.ExecContext(ctx, fmt.Sprintf(query, ""), ubig.New(o.chainID)) } if err != nil { diff --git a/core/chains/evm/logpoller/orm_test.go b/core/chains/evm/logpoller/orm_test.go index 6f431b6db92..0df34196ff9 100644 --- a/core/chains/evm/logpoller/orm_test.go +++ b/core/chains/evm/logpoller/orm_test.go @@ -458,20 +458,21 @@ func TestORM(t *testing.T) { time.Sleep(2 * time.Millisecond) // just in case we haven't reached the end of the 1ms retention period deleted, err := o1.DeleteExpiredLogs(ctx, 0) require.NoError(t, err) - assert.Equal(t, int64(1), deleted) + assert.Equal(t, int64(4), deleted) + logs, err = o1.SelectLogsByBlockRange(ctx, 1, latest.BlockNumber) require.NoError(t, err) - // The only log which should be deleted is the one which matches filter1 (ret=1ms) but not filter12 (ret=1 hour) - // Importantly, it shouldn't delete any logs matching only filter0 (ret=0 meaning permanent retention). Anything - // matching filter12 should be kept regardless of what other filters it matches. - assert.Len(t, logs, 7) + // It should have retained the log matching filter0 (due to ret=0 meaning permanent retention) as well as all + // 3 logs matching filter12 (ret=1 hour). It should have deleted 3 logs not matching any filter, as well as 1 + // of the 2 logs matching filter1 (ret=1ms)--the one that doesn't also match filter12. + assert.Len(t, logs, 4) // Delete logs after should delete all logs. err = o1.DeleteLogsAndBlocksAfter(ctx, 1) require.NoError(t, err) logs, err = o1.SelectLogsByBlockRange(ctx, 1, latest.BlockNumber) require.NoError(t, err) - require.Zero(t, len(logs)) + assert.Zero(t, len(logs)) } type PgxLogger struct { From 84630b846894bc3a770e5f8ebf58a6ae7f143669 Mon Sep 17 00:00:00 2001 From: Erik Burton Date: Thu, 8 Aug 2024 11:05:17 -0700 Subject: [PATCH 2/3] fix: refactor sonarqube scan args (#13875) --- .github/workflows/ci-core.yml | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/.github/workflows/ci-core.yml b/.github/workflows/ci-core.yml index aac8e578d13..fc7e6e372d3 100644 --- a/.github/workflows/ci-core.yml +++ b/.github/workflows/ci-core.yml @@ -341,37 +341,37 @@ jobs: - name: Download all workflow run artifacts uses: actions/download-artifact@c850b930e6ba138125429b7e5c93fc707a7f8427 # v4.1.4 - - name: Set SonarQube Report Paths - id: sonarqube_report_paths + - name: Check and Set SonarQube Report Paths shell: bash run: | - echo "sonarqube_tests_report_paths=$(find go_core_tests_logs -name output.txt | paste -sd "," -)" >> $GITHUB_OUTPUT - echo "sonarqube_coverage_report_paths=$(find go_core_tests_logs -name coverage.txt | paste -sd "," -)" >> $GITHUB_OUTPUT - echo "sonarqube_lint_report_paths=$(find golangci-lint-report -name golangci-lint-report.xml | paste -sd "," -)" >> $GITHUB_OUTPUT + sonarqube_tests_report_paths=$(find go_core_tests_logs -name output.txt | paste -sd "," -) + sonarqube_coverage_report_paths=$(find go_core_tests_logs -name coverage.txt | paste -sd "," -) + sonarqube_lint_report_paths=$(find golangci-lint-report -name golangci-lint-report.xml | paste -sd "," -) - - name: Check SonarQube Report Paths - id: check_sonarqube_paths - run: | ARGS="" - if [[ -z "${{ steps.sonarqube_report_paths.outputs.sonarqube_tests_report_paths }}" ]]; then + if [[ -z "$sonarqube_tests_report_paths" ]]; then echo "::warning::No test report paths found, will not pass to sonarqube" else - ARGS="$ARGS -Dsonar.go.tests.reportPaths=${{ steps.sonarqube_report_paths.outputs.sonarqube_tests_report_paths }}" + echo "Found test report paths: $sonarqube_tests_report_paths" + ARGS="$ARGS -Dsonar.go.tests.reportPaths=$sonarqube_tests_report_paths" fi - if [[ -z "${{ steps.sonarqube_report_paths.outputs.sonarqube_coverage_report_paths }}" ]]; then + if [[ -z "$sonarqube_coverage_report_paths" ]]; then echo "::warning::No coverage report paths found, will not pass to sonarqube" else - ARGS="$ARGS -Dsonar.go.coverage.reportPaths=${{ steps.sonarqube_report_paths.outputs.sonarqube_coverage_report_paths }}" + echo "Found coverage report paths: $sonarqube_coverage_report_paths" + ARGS="$ARGS -Dsonar.go.coverage.reportPaths=$sonarqube_coverage_report_paths" fi - if [[ -z "${{ steps.sonarqube_report_paths.outputs.sonarqube_lint_report_paths }}" ]]; then + if [[ -z "$sonarqube_lint_report_paths" ]]; then echo "::warning::No lint report paths found, will not pass to sonarqube" else - ARGS="$ARGS -Dsonar.go.golangci-lint.reportPaths=${{ steps.sonarqube_report_paths.outputs.sonarqube_lint_report_paths }}" + echo "Found lint report paths: $sonarqube_lint_report_paths" + ARGS="$ARGS -Dsonar.go.golangci-lint.reportPaths=$sonarqube_lint_report_paths" fi + echo "Final SONARQUBE_ARGS: $ARGS" echo "SONARQUBE_ARGS=$ARGS" >> $GITHUB_ENV - name: SonarQube Scan From 08638ff2b963b1afe731de9910f73654914a45ff Mon Sep 17 00:00:00 2001 From: Sergey Kudasov Date: Thu, 8 Aug 2024 20:13:25 +0200 Subject: [PATCH 3/3] CRIB CI integration (#13924) * decouple CRIB vars * move CRIB vars to CTF, connect with GAP * bump deps * finalize deps * update go.mod * Spin up a separate GAP for crib and k8s * Change up the ports since 8080 is expected for CRIB connections * Use released version of setup-gap action * increase timeout * less logs * increase timeout * increase timeout even more * try creds for one hour * run without chaos * again * try to spin up CRIB * use GATI * update GATI secrets * use different port * fix working dir * update ref * try fixing working dir * another try * another try * another try * another try * nix develop * Fix nix develop * turn debug logs on * use local-dev-simulated-core-ocr1 profile * add teardown step * uppdate crib actions refs * add ref comments * reduce logging * add a confluence link * pin CI versions * temporary enable a nightly run --------- Co-authored-by: chainchad <96362174+chainchad@users.noreply.github.com> Co-authored-by: Radek Scheibinger --- .github/workflows/crib-integration-test.yml | 183 +++++++++++------- integration-tests/README.md | 6 + .../actions/vrf/common/actions.go | 2 +- integration-tests/client/chainlink.go | 16 +- integration-tests/client/chainlink_k8s.go | 7 +- integration-tests/client/chainlink_models.go | 11 +- integration-tests/crib/README.md | 17 +- integration-tests/crib/connect.go | 87 +++------ integration-tests/crib/ocr_test.go | 39 ++-- integration-tests/load/go.mod | 2 +- integration-tests/load/go.sum | 4 +- integration-tests/smoke/README.md | 6 - integration-tests/testsetups/ocr.go | 1 - 13 files changed, 199 insertions(+), 182 deletions(-) diff --git a/.github/workflows/crib-integration-test.yml b/.github/workflows/crib-integration-test.yml index 75b2215d2fc..248004636bc 100644 --- a/.github/workflows/crib-integration-test.yml +++ b/.github/workflows/crib-integration-test.yml @@ -1,74 +1,111 @@ -# this is disabled because of GAP limitations, should be re-enabled when github-actions-controller will be installed +name: CRIB Integration Tests +on: + schedule: + - cron: "0 1 * * *" + workflow_call: +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true +jobs: + test: + runs-on: ubuntu-latest + environment: integration + permissions: + id-token: write + contents: read + actions: read + steps: + - name: Checkout repository + uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2 -#name: CRIB Integration Tests -#on: -# push: -# workflow_call: -#concurrency: -# group: ${{ github.workflow }}-${{ github.ref }} -# cancel-in-progress: true -#jobs: -# test: -# runs-on: ubuntu-latest -# environment: integration -# permissions: -# id-token: write -# contents: read -# actions: read -# steps: -# - name: Checkout repository -# uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2 -# -# - name: Setup Nix + GATI environment -# uses: smartcontractkit/.github/actions/setup-nix-gati@514fe346780e2eddf7ea8b9f48120c2fba120d94 -# with: -# aws-role-arn: ${{ secrets.AWS_OIDC_CHAINLINK_AUTO_PR_TOKEN_ISSUER_ROLE_ARN }} -# aws-lambda-url: ${{ secrets.AWS_CORE_TOKEN_ISSUER_LAMBDA_URL }} # see https://github.com/smartcontractkit/ infra/blob/a79bcfb48315c4411023c182e98eb80ff9e9cda6/accounts/production/us-west-2/lambda/ github-app-token-issuer-production/teams/releng/config.json#L9 -# aws-region: ${{ secrets.AWS_REGION }} -# aws-role-duration-seconds: ${{ secrets.AWS_ROLE_DURATION_SECONDS }} -# enable-magic-cache: true -# -# - name: Nix Develop Action -# uses: nicknovitski/nix-develop@v1 -# with: -# arguments: "--accept-flake-config" -# - name: setup-gap -# uses: smartcontractkit/.github/actions/setup-gap@d316f66b2990ea4daa479daa3de6fc92b00f863e # setup-gap@0.3.2 -# with: -# aws-role-arn: ${{ secrets.AWS_OIDC_CRIB_ROLE_ARN_STAGE }} -# api-gateway-host: ${{ secrets.AWS_API_GW_HOST_K8S_STAGE }} -# aws-region: ${{ secrets.AWS_REGION }} -# ecr-private-registry: ${{ secrets.AWS_ACCOUNT_ID_PROD }} -# k8s-cluster-name: ${{ secrets.AWS_K8S_CLUSTER_NAME_STAGE }} -# use-private-ecr-registry: true -# use-k8s: true -# metrics-job-name: "k8s" -# gc-basic-auth: ${{ secrets.GRAFANA_INTERNAL_BASIC_AUTH }} -# gc-host: ${{ secrets.GRAFANA_INTERNAL_HOST }} -# gc-org-id: ${{ secrets.GRAFANA_INTERNAL_TENANT_ID }} -# - uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2 -# name: Checkout CRIB repository -# with: -# repository: 'smartcontractkit/crib' -# ref: 'main' -# - name: Generate Short UUID -# id: uuid -# run: echo "CRIB_NAMESPACE=$(uuidgen | cut -c1-5)" >> $GITHUB_ENV -# - name: Create a new CRIB environment -# run: |- -# devspace use namespace $CRIB_NAMESPACE -# devspace deploy --profile local-dev-simulated-core-ocr1 -# - uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2 -# - name: Setup go -# uses: actions/setup-go@0c52d547c9bc32b1aa3301fd7a9cb496313a4491 # v5.0.0 -# with: -# go-version-file: "go.mod" -# - name: Run CRIB integration test -# working-directory: integration-tests/crib -# env: -# K8S_STAGING_INGRESS_SUFFIX: ${{ secrets.K8S_STAGING_INGRESS_SUFFIX }} -# CRIB_NAMESPACE: ${{ env.CRIB_NAMESPACE }} -# CRIB_NETWORK: geth -# CRIB_NODES: 5 -# run: |- -# go test -v -run TestCRIB \ No newline at end of file + - uses: cachix/install-nix-action@ba0dd844c9180cbf77aa72a116d6fbc515d0e87b # v27 + with: + nix_path: nixpkgs=channel:nixos-unstable + + - name: setup-gap crib + uses: smartcontractkit/.github/actions/setup-gap@00b58566e0ee2761e56d9db0ea72b783fdb89b8d # setup-gap@0.4.0 + with: + aws-role-duration-seconds: 3600 # 1 hour + aws-role-arn: ${{ secrets.AWS_OIDC_CRIB_ROLE_ARN_STAGE }} + api-gateway-host: ${{ secrets.AWS_API_GW_HOST_CRIB_STAGE }} + aws-region: ${{ secrets.AWS_REGION }} + ecr-private-registry: ${{ secrets.AWS_ACCOUNT_ID_PROD }} + k8s-cluster-name: ${{ secrets.AWS_K8S_CLUSTER_NAME_STAGE }} + gap-name: crib + use-private-ecr-registry: true + use-tls: true + proxy-port: 8080 + metrics-job-name: "test" + gc-basic-auth: ${{ secrets.GRAFANA_INTERNAL_BASIC_AUTH }} + gc-host: ${{ secrets.GRAFANA_INTERNAL_HOST }} + gc-org-id: ${{ secrets.GRAFANA_INTERNAL_TENANT_ID }} + + - name: setup-gap k8s + uses: smartcontractkit/.github/actions/setup-gap@00b58566e0ee2761e56d9db0ea72b783fdb89b8d # setup-gap@0.4.0 + with: + aws-role-duration-seconds: 3600 # 1 hour + aws-role-arn: ${{ secrets.AWS_OIDC_CRIB_ROLE_ARN_STAGE }} + api-gateway-host: ${{ secrets.AWS_API_GW_HOST_K8S_STAGE }} + aws-region: ${{ secrets.AWS_REGION }} + ecr-private-registry: ${{ secrets.AWS_ACCOUNT_ID_PROD }} + k8s-cluster-name: ${{ secrets.AWS_K8S_CLUSTER_NAME_STAGE }} + gap-name: k8s + use-private-ecr-registry: true + use-k8s: true + proxy-port: 8443 + metrics-job-name: "test" + gc-basic-auth: ${{ secrets.GRAFANA_INTERNAL_BASIC_AUTH }} + gc-host: ${{ secrets.GRAFANA_INTERNAL_HOST }} + gc-org-id: ${{ secrets.GRAFANA_INTERNAL_TENANT_ID }} + + - name: Setup GitHub token using GATI + id: token + uses: smartcontractkit/.github/actions/setup-github-token@c0b38e6c40d72d01b8d2f24f92623a2538b3dedb # main + with: + aws-role-arn: ${{ secrets.AWS_OIDC_GLOBAL_READ_ONLY_TOKEN_ISSUER_ROLE_ARN }} + aws-lambda-url: ${{ secrets.AWS_INFRA_RELENG_TOKEN_ISSUER_LAMBDA_URL }} + aws-region: ${{ secrets.AWS_REGION }} + aws-role-duration-seconds: "1800" + - name: Debug workspace dir + shell: bash + run: | + echo ${{ github.workspace }} + echo $GITHUB_WORKSPACE + + - name: Deploy and validate CRIB Environment for Core + uses: smartcontractkit/.github/actions/crib-deploy-environment@c0b38e6c40d72d01b8d2f24f92623a2538b3dedb # crib-deploy-environment@0.5.0 + id: deploy-crib + with: + github-token: ${{ steps.token.outputs.access-token }} + api-gateway-host: ${{ secrets.AWS_API_GW_HOST_K8S_STAGE }} + aws-region: ${{ secrets.AWS_REGION }} + aws-role-arn: ${{ secrets.AWS_OIDC_CRIB_ROLE_ARN_STAGE }} + ecr-private-registry-stage: ${{ secrets.AWS_ACCOUNT_ID_STAGE }} + ecr-private-registry: ${{ secrets.AWS_ACCOUNT_ID_PROD }} + ingress-base-domain: ${{ secrets.INGRESS_BASE_DOMAIN_STAGE }} + k8s-cluster-name: ${{ secrets.AWS_K8S_CLUSTER_NAME_STAGE }} + devspace-profiles: "local-dev-simulated-core-ocr1" + - uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2 + - name: Setup go + uses: actions/setup-go@0c52d547c9bc32b1aa3301fd7a9cb496313a4491 # v5.0.0 + with: + go-version-file: "go.mod" + - name: Run CRIB integration test + working-directory: integration-tests/crib + env: + K8S_STAGING_INGRESS_SUFFIX: ${{ secrets.K8S_STAGING_INGRESS_SUFFIX }} + CRIB_NAMESPACE: ${{ steps.deploy-crib.outputs.devspace-namespace }} + CRIB_NETWORK: geth + CRIB_NODES: 5 + GAP_URL: ${{ secrets.GAP_URL }} +# SETH_LOG_LEVEL: debug +# RESTY_DEBUG: true +# TEST_PERSISTENCE: true + run: |- + go test -v -run TestCRIB + - name: Destroy CRIB Environment + id: destroy + if: always() && steps.deploy-crib.outputs.devspace-namespace != '' + uses: smartcontractkit/.github/actions/crib-purge-environment@c0b38e6c40d72d01b8d2f24f92623a2538b3dedb # crib-purge-environment@0.1.0 + with: + namespace: ${{ steps.deploy-crib.outputs.devspace-namespace }} \ No newline at end of file diff --git a/integration-tests/README.md b/integration-tests/README.md index 180021efeef..1510c8c91b7 100644 --- a/integration-tests/README.md +++ b/integration-tests/README.md @@ -129,3 +129,9 @@ Run soak/ocr_test.go with RPC network chaos by bringing down network to RPC node ```bash make test_soak_ocr_rpc_down_half_cl_nodes ``` + +### Debugging HTTP and RPC clients +```bash +export SETH_LOG_LEVEL=debug +export RESTY_DEBUG=true +``` diff --git a/integration-tests/actions/vrf/common/actions.go b/integration-tests/actions/vrf/common/actions.go index 1300ac8b726..5697a261766 100644 --- a/integration-tests/actions/vrf/common/actions.go +++ b/integration-tests/actions/vrf/common/actions.go @@ -429,7 +429,7 @@ type RPCRawClient struct { } func NewRPCRawClient(url string) *RPCRawClient { - isDebug := os.Getenv("DEBUG_RESTY") == "true" + isDebug := os.Getenv("RESTY_DEBUG") == "true" restyClient := resty.New().SetDebug(isDebug).SetBaseURL(url) return &RPCRawClient{ resty: restyClient, diff --git a/integration-tests/client/chainlink.go b/integration-tests/client/chainlink.go index 08a47101dc1..da17dcf0d75 100644 --- a/integration-tests/client/chainlink.go +++ b/integration-tests/client/chainlink.go @@ -2,6 +2,7 @@ package client import ( + "crypto/tls" "fmt" "math/big" "net/http" @@ -45,14 +46,10 @@ type ChainlinkClient struct { // NewChainlinkClient creates a new Chainlink model using a provided config func NewChainlinkClient(c *ChainlinkConfig, logger zerolog.Logger) (*ChainlinkClient, error) { - rc, err := initRestyClient(c.URL, c.Email, c.Password, c.HTTPTimeout) + rc, err := initRestyClient(c.URL, c.Email, c.Password, c.Headers, c.HTTPTimeout) if err != nil { return nil, err } - _, isSet := os.LookupEnv("CL_CLIENT_DEBUG") - if isSet { - rc.SetDebug(true) - } return &ChainlinkClient{ Config: c, APIClient: rc, @@ -61,8 +58,11 @@ func NewChainlinkClient(c *ChainlinkConfig, logger zerolog.Logger) (*ChainlinkCl }, nil } -func initRestyClient(url string, email string, password string, timeout *time.Duration) (*resty.Client, error) { - rc := resty.New().SetBaseURL(url) +func initRestyClient(url string, email string, password string, headers map[string]string, timeout *time.Duration) (*resty.Client, error) { + isDebug := os.Getenv("RESTY_DEBUG") == "true" + // G402 - TODO: certificates + //nolint + rc := resty.New().SetBaseURL(url).SetHeaders(headers).SetTLSClientConfig(&tls.Config{InsecureSkipVerify: true}).SetDebug(isDebug) if timeout != nil { rc.SetTimeout(*timeout) } @@ -74,7 +74,7 @@ func initRestyClient(url string, email string, password string, timeout *time.Du for i := 0; i < retryCount; i++ { resp, err = rc.R().SetBody(session).Post("/sessions") if err != nil { - log.Debug().Err(err).Str("URL", url).Interface("Session Details", session).Msg("Error connecting to Chainlink node, retrying") + log.Warn().Err(err).Str("URL", url).Interface("Session Details", session).Msg("Error connecting to Chainlink node, retrying") time.Sleep(5 * time.Second) } else { break diff --git a/integration-tests/client/chainlink_k8s.go b/integration-tests/client/chainlink_k8s.go index 794e93f7276..077b8f7ca48 100644 --- a/integration-tests/client/chainlink_k8s.go +++ b/integration-tests/client/chainlink_k8s.go @@ -2,7 +2,6 @@ package client import ( - "os" "regexp" "github.com/rs/zerolog/log" @@ -23,14 +22,10 @@ type ChainlinkK8sClient struct { // NewChainlink creates a new Chainlink model using a provided config func NewChainlinkK8sClient(c *ChainlinkConfig, podName, chartName string) (*ChainlinkK8sClient, error) { - rc, err := initRestyClient(c.URL, c.Email, c.Password, c.HTTPTimeout) + rc, err := initRestyClient(c.URL, c.Email, c.Password, c.Headers, c.HTTPTimeout) if err != nil { return nil, err } - _, isSet := os.LookupEnv("CL_CLIENT_DEBUG") - if isSet { - rc.SetDebug(true) - } return &ChainlinkK8sClient{ ChainlinkClient: &ChainlinkClient{ APIClient: rc, diff --git a/integration-tests/client/chainlink_models.go b/integration-tests/client/chainlink_models.go index a0435d53368..86e9f75902d 100644 --- a/integration-tests/client/chainlink_models.go +++ b/integration-tests/client/chainlink_models.go @@ -20,11 +20,12 @@ type EIServiceConfig struct { // ChainlinkConfig represents the variables needed to connect to a Chainlink node type ChainlinkConfig struct { - URL string `toml:",omitempty"` - Email string `toml:",omitempty"` - Password string `toml:",omitempty"` - InternalIP string `toml:",omitempty"` - HTTPTimeout *time.Duration `toml:"-"` + URL string `toml:",omitempty"` + Email string `toml:",omitempty"` + Password string `toml:",omitempty"` + InternalIP string `toml:",omitempty"` + Headers map[string]string `toml:",omitempty"` + HTTPTimeout *time.Duration `toml:"-"` } // ResponseSlice is the generic model that can be used for all Chainlink API responses that are an slice diff --git a/integration-tests/crib/README.md b/integration-tests/crib/README.md index ecf393f780d..e895cca6763 100644 --- a/integration-tests/crib/README.md +++ b/integration-tests/crib/README.md @@ -1,4 +1,4 @@ -### CRIB Health Check Test +### Example e2e product test using CRIB ## Setup CRIB This is a simple smoke + chaos test for CRIB deployment. @@ -12,8 +12,15 @@ devspace deploy --debug --profile local-dev-simulated-core-ocr1 ## Run the tests ```shell -CRIB_NAMESPACE=crib-oh-my-crib -CRIB_NETWORK=geth # only "geth" is supported for now -CRIB_NODES=5 # min 5 nodes +export CRIB_NAMESPACE=crib-oh-my-crib +export CRIB_NETWORK=geth # only "geth" is supported for now +export CRIB_NODES=5 # min 5 nodes +#export SETH_LOG_LEVEL=debug # these two can be enabled to debug connection issues +#export RESTY_DEBUG=true +#export TEST_PERSISTENCE=true # to run the chaos test +export GAP_URL=https://localhost:8080/primary # only applicable in CI, unset the var to connect locally go test -v -run TestCRIB -``` \ No newline at end of file +``` + +## Configuring CI workflow +We are using GAP and GATI to access the infrastructure, please follow [configuration guide](https://smartcontract-it.atlassian.net/wiki/spaces/CRIB/pages/909967436/CRIB+CI+Integration) \ No newline at end of file diff --git a/integration-tests/crib/connect.go b/integration-tests/crib/connect.go index 91d7d8ee1a0..c180b2ff2ea 100644 --- a/integration-tests/crib/connect.go +++ b/integration-tests/crib/connect.go @@ -1,11 +1,11 @@ package crib import ( - "fmt" - "os" - "strconv" + "net/http" "time" + "github.com/smartcontractkit/chainlink-testing-framework/crib" + "github.com/pkg/errors" "github.com/smartcontractkit/seth" @@ -18,17 +18,7 @@ import ( "github.com/smartcontractkit/chainlink/integration-tests/client" ) -const ( - // these are constants for simulated CRIB that should never change - // CRIB: https://github.com/smartcontractkit/crib/tree/main/core - // Core Chart: https://github.com/smartcontractkit/infra-charts/tree/main/chainlink-cluster - mockserverCRIBTemplate = "https://%s-mockserver%s" - internalNodeDNSTemplate = "app-node%d" - ingressNetworkWSURLTemplate = "wss://%s-geth-1337-ws%s" - ingressNetworkHTTPURLTemplate = "https://%s-geth-1337-http%s" -) - -func setSethConfig(cfg tc.TestConfig, netWSURL string, netHTTPURL string) { +func setSethConfig(cfg tc.TestConfig, netWSURL string, netHTTPURL string, headers http.Header) { netName := "CRIB_SIMULATED" cfg.Network.SelectedNetworks = []string{netName} cfg.Network.RpcHttpUrls = map[string][]string{} @@ -36,6 +26,7 @@ func setSethConfig(cfg tc.TestConfig, netWSURL string, netHTTPURL string) { cfg.Network.RpcWsUrls = map[string][]string{} cfg.Network.RpcWsUrls[netName] = []string{netWSURL} cfg.Seth.EphemeralAddrs = ptr.Ptr(int64(0)) + cfg.Seth.RPCHeaders = headers } // ConnectRemote connects to a local environment, see https://github.com/smartcontractkit/crib/tree/main/core @@ -47,52 +38,33 @@ func ConnectRemote() ( []*client.ChainlinkK8sClient, error, ) { - ingressSuffix := os.Getenv("K8S_STAGING_INGRESS_SUFFIX") - if ingressSuffix == "" { - return nil, nil, nil, nil, errors.New("K8S_STAGING_INGRESS_SUFFIX must be set to connect to k8s ingresses") - } - cribNamespace := os.Getenv("CRIB_NAMESPACE") - if cribNamespace == "" { - return nil, nil, nil, nil, errors.New("CRIB_NAMESPACE must be set to connect") - } - cribNetwork := os.Getenv("CRIB_NETWORK") - if cribNetwork == "" { - return nil, nil, nil, nil, errors.New("CRIB_NETWORK must be set to connect, only 'geth' is supported for now") - } - cribNodes := os.Getenv("CRIB_NODES") - nodes, err := strconv.Atoi(cribNodes) + vars, err := crib.CoreDONSimulatedConnection() if err != nil { - return nil, nil, nil, nil, errors.New("CRIB_NODES must be a number, 5-19 nodes") + return nil, nil, nil, nil, err } + // TODO: move all the parts of ConnectRemote() to CTF when Seth config refactor is finalized config, err := tc.GetConfig([]string{"CRIB"}, tc.OCR) if err != nil { return nil, nil, nil, nil, err } - if nodes < 2 { - return nil, nil, nil, nil, fmt.Errorf("not enough chainlink nodes, need at least 2, TOML key: [CRIB.nodes]") - } - mockserverURL := fmt.Sprintf(mockserverCRIBTemplate, cribNamespace, ingressSuffix) var sethClient *seth.Client - switch cribNetwork { + switch vars.Network { case "geth": - netWSURL := fmt.Sprintf(ingressNetworkWSURLTemplate, cribNamespace, ingressSuffix) - netHTTPURL := fmt.Sprintf(ingressNetworkHTTPURLTemplate, cribNamespace, ingressSuffix) - setSethConfig(config, netWSURL, netHTTPURL) + setSethConfig(config, vars.NetworkWSURL, vars.NetworkHTTPURL, vars.BlockchainNodeHeaders) net := blockchain.EVMNetwork{ - Name: cribNetwork, - Simulated: true, - SupportsEIP1559: true, - ClientImplementation: blockchain.EthereumClientImplementation, - ChainID: 1337, - PrivateKeys: []string{ - "ac0974bec39a17e36ba4a6b4d238ff944bacb478cbed5efcae784d7bf4f2ff80", - }, - URLs: []string{netWSURL}, - HTTPURLs: []string{netHTTPURL}, + Name: vars.Network, + Simulated: true, + SupportsEIP1559: true, + ClientImplementation: blockchain.EthereumClientImplementation, + ChainID: vars.ChainID, + PrivateKeys: vars.PrivateKeys, + URLs: []string{vars.NetworkWSURL}, + HTTPURLs: []string{vars.NetworkHTTPURL}, ChainlinkTransactionLimit: 500000, Timeout: blockchain.StrDuration{Duration: 2 * time.Minute}, MinimumConfirmations: 1, GasEstimationBuffer: 10000, + Headers: vars.BlockchainNodeHeaders, } sethClient, err = seth_utils.GetChainClient(config, net) if err != nil { @@ -104,31 +76,34 @@ func ConnectRemote() ( // bootstrap node clClients := make([]*client.ChainlinkK8sClient, 0) c, err := client.NewChainlinkK8sClient(&client.ChainlinkConfig{ - URL: fmt.Sprintf("https://%s-node%d%s", cribNamespace, 1, ingressSuffix), Email: client.CLNodeTestEmail, - InternalIP: fmt.Sprintf(internalNodeDNSTemplate, 1), Password: client.CLNodeTestPassword, - }, fmt.Sprintf(internalNodeDNSTemplate, 1), cribNamespace) + URL: vars.NodeURLs[0], + InternalIP: vars.NodeInternalDNS[0], + Headers: vars.NodeHeaders[0], + }, vars.NodeInternalDNS[0], vars.Namespace) if err != nil { return nil, nil, nil, nil, err } clClients = append(clClients, c) // all the other nodes, indices of nodes in CRIB starts with 1 - for i := 2; i <= nodes; i++ { + for i := 1; i < vars.Nodes; i++ { cl, err := client.NewChainlinkK8sClient(&client.ChainlinkConfig{ - URL: fmt.Sprintf("https://%s-node%d%s", cribNamespace, i, ingressSuffix), Email: client.CLNodeTestEmail, - InternalIP: fmt.Sprintf(internalNodeDNSTemplate, i), Password: client.CLNodeTestPassword, - }, fmt.Sprintf(internalNodeDNSTemplate, i), cribNamespace) + URL: vars.NodeURLs[i], + InternalIP: vars.NodeInternalDNS[i], + Headers: vars.NodeHeaders[i], + }, vars.NodeInternalDNS[i], vars.Namespace) if err != nil { return nil, nil, nil, nil, err } clClients = append(clClients, cl) } mockServerClient := msClient.NewMockserverClient(&msClient.MockserverConfig{ - LocalURL: mockserverURL, - ClusterURL: mockserverURL, + LocalURL: vars.MockserverURL, + ClusterURL: "http://mockserver:1080", + Headers: vars.MockserverHeaders, }) //nolint:gosec // G602 - false positive https://github.com/securego/gosec/issues/1005 diff --git a/integration-tests/crib/ocr_test.go b/integration-tests/crib/ocr_test.go index b84af02a196..91a7a1d76b8 100644 --- a/integration-tests/crib/ocr_test.go +++ b/integration-tests/crib/ocr_test.go @@ -7,6 +7,7 @@ import ( "time" "github.com/smartcontractkit/havoc/k8schaos" + "github.com/stretchr/testify/require" "github.com/smartcontractkit/chainlink/integration-tests/actions" @@ -33,22 +34,24 @@ func TestCRIB(t *testing.T) { err = actions.WatchNewOCRRound(l, sethClient, 1, contracts.V1OffChainAgrregatorToOffChainAggregatorWithRounds(ocrInstances), 5*time.Minute) require.NoError(t, err, "Error watching for new OCR round") - ch, err := rebootCLNamespace( - 1*time.Second, - os.Getenv("CRIB_NAMESPACE"), - ) - ch.Create(context.Background()) - ch.AddListener(k8schaos.NewChaosLogger(l)) - t.Cleanup(func() { - err := ch.Delete(context.Background()) - require.NoError(t, err) - }) - require.Eventually(t, func() bool { - err = actions.WatchNewOCRRound(l, sethClient, 3, contracts.V1OffChainAgrregatorToOffChainAggregatorWithRounds(ocrInstances), 5*time.Minute) - if err != nil { - l.Info().Err(err).Msg("OCR round is not there yet") - return false - } - return true - }, 3*time.Minute, 5*time.Second) + if os.Getenv("TEST_PERSISTENCE") != "" { + ch, err := rebootCLNamespace( + 1*time.Second, + os.Getenv("CRIB_NAMESPACE"), + ) + ch.Create(context.Background()) + ch.AddListener(k8schaos.NewChaosLogger(l)) + t.Cleanup(func() { + err := ch.Delete(context.Background()) + require.NoError(t, err) + }) + require.Eventually(t, func() bool { + err = actions.WatchNewOCRRound(l, sethClient, 3, contracts.V1OffChainAgrregatorToOffChainAggregatorWithRounds(ocrInstances), 5*time.Minute) + if err != nil { + l.Info().Err(err).Msg("OCR round is not there yet") + return false + } + return true + }, 20*time.Minute, 5*time.Second) + } } diff --git a/integration-tests/load/go.mod b/integration-tests/load/go.mod index 2dec28df99f..bd47a97f48a 100644 --- a/integration-tests/load/go.mod +++ b/integration-tests/load/go.mod @@ -375,7 +375,7 @@ require ( github.com/smartcontractkit/chainlink-feeds v0.0.0-20240710170203-5b41615da827 // indirect github.com/smartcontractkit/chainlink-solana v1.1.1-0.20240806154405-8e5684f98564 // indirect github.com/smartcontractkit/chainlink-starknet/relayer v0.0.1-beta-test.0.20240709043547-03612098f799 // indirect - github.com/smartcontractkit/chainlink-testing-framework/grafana v0.0.0-20240405215812-5a72bc9af239 // indirect + github.com/smartcontractkit/chainlink-testing-framework/grafana v0.0.2-0.20240805111647-acf86c1e347a // indirect github.com/smartcontractkit/havoc/k8schaos v0.0.0-20240409145249-e78d20847e37 // indirect github.com/smartcontractkit/tdh2/go/ocr2/decryptionplugin v0.0.0-20230906073235-9e478e5e19f1 // indirect github.com/smartcontractkit/wsrpc v0.7.3 // indirect diff --git a/integration-tests/load/go.sum b/integration-tests/load/go.sum index 484b5eb2482..7789b949449 100644 --- a/integration-tests/load/go.sum +++ b/integration-tests/load/go.sum @@ -1486,8 +1486,8 @@ github.com/smartcontractkit/chainlink-starknet/relayer v0.0.1-beta-test.0.202407 github.com/smartcontractkit/chainlink-starknet/relayer v0.0.1-beta-test.0.20240709043547-03612098f799/go.mod h1:UVFRacRkP7O7TQAzFmR52v5mUlxf+G1ovMlCQAB/cHU= github.com/smartcontractkit/chainlink-testing-framework v1.34.2 h1:YL3ft7KJB7SAopdmJeyeR4/kv0j4jOdagNihXq8OZ38= github.com/smartcontractkit/chainlink-testing-framework v1.34.2/go.mod h1:hRZEDh2+afO8MSZb9qYNscmWb+3mHZf01J5ACZuIdTQ= -github.com/smartcontractkit/chainlink-testing-framework/grafana v0.0.0-20240405215812-5a72bc9af239 h1:Kk5OVlx/5g9q3Z3lhxytZS4/f8ds1MiNM8yaHgK3Oe8= -github.com/smartcontractkit/chainlink-testing-framework/grafana v0.0.0-20240405215812-5a72bc9af239/go.mod h1:DC8sQMyTlI/44UCTL8QWFwb0bYNoXCfjwCv2hMivYZU= +github.com/smartcontractkit/chainlink-testing-framework/grafana v0.0.2-0.20240805111647-acf86c1e347a h1:8GtvGJaGyKzx/ar1yX74GxrzIYWTZVTyv4pYB/1ln8w= +github.com/smartcontractkit/chainlink-testing-framework/grafana v0.0.2-0.20240805111647-acf86c1e347a/go.mod h1:DC8sQMyTlI/44UCTL8QWFwb0bYNoXCfjwCv2hMivYZU= github.com/smartcontractkit/go-plugin v0.0.0-20240208201424-b3b91517de16 h1:TFe+FvzxClblt6qRfqEhUfa4kFQx5UobuoFGO2W4mMo= github.com/smartcontractkit/go-plugin v0.0.0-20240208201424-b3b91517de16/go.mod h1:lBS5MtSSBZk0SHc66KACcjjlU6WzEVP/8pwz68aMkCI= github.com/smartcontractkit/grpc-proxy v0.0.0-20230731113816-f1be6620749f h1:hgJif132UCdjo8u43i7iPN1/MFnu49hv7lFGFftCHKU= diff --git a/integration-tests/smoke/README.md b/integration-tests/smoke/README.md index 266720c7bc6..c4aa2b91a14 100644 --- a/integration-tests/smoke/README.md +++ b/integration-tests/smoke/README.md @@ -75,9 +75,3 @@ Then execute: go test -v -run ${TestName} ``` - - -### Debugging CL client API calls -```bash -export CL_CLIENT_DEBUG=true -``` \ No newline at end of file diff --git a/integration-tests/testsetups/ocr.go b/integration-tests/testsetups/ocr.go index b38c39eebe4..084ea5eca10 100644 --- a/integration-tests/testsetups/ocr.go +++ b/integration-tests/testsetups/ocr.go @@ -547,7 +547,6 @@ func (o *OCRSoakTest) LoadState() error { } o.mockServer = ctf_client.ConnectMockServerURL(testState.MockServerURL) - return err }