Merge branch 'develop' into task/KS-297/db-syncer

.changeset/sweet-pumas-refuse.md

@@ -0,0 +1,5 @@
+---
+"chainlink": minor
+---
+
+#bugfix Addresses 2 minor issues with the pruning of LogPoller's db tables: logs not matching any filter will now be pruned, and rows deleted are now properly reported for observability

.github/workflows/ci-core.yml

@@ -341,37 +341,37 @@ jobs:
       - name: Download all workflow run artifacts
         uses: actions/download-artifact@c850b930e6ba138125429b7e5c93fc707a7f8427 # v4.1.4
 
-      - name: Set SonarQube Report Paths
-        id: sonarqube_report_paths
+      - name: Check and Set SonarQube Report Paths
         shell: bash
         run: |
-          echo "sonarqube_tests_report_paths=$(find go_core_tests_logs -name output.txt | paste -sd "," -)" >> $GITHUB_OUTPUT
-          echo "sonarqube_coverage_report_paths=$(find go_core_tests_logs -name coverage.txt | paste -sd "," -)" >> $GITHUB_OUTPUT
-          echo "sonarqube_lint_report_paths=$(find golangci-lint-report -name golangci-lint-report.xml | paste -sd "," -)" >> $GITHUB_OUTPUT
+          sonarqube_tests_report_paths=$(find go_core_tests_logs -name output.txt | paste -sd "," -)
+          sonarqube_coverage_report_paths=$(find go_core_tests_logs -name coverage.txt | paste -sd "," -)
+          sonarqube_lint_report_paths=$(find golangci-lint-report -name golangci-lint-report.xml | paste -sd "," -)
 
-      - name: Check SonarQube Report Paths
-        id: check_sonarqube_paths
-        run: |
           ARGS=""
 
-          if [[ -z "${{ steps.sonarqube_report_paths.outputs.sonarqube_tests_report_paths }}" ]]; then
+          if [[ -z "$sonarqube_tests_report_paths" ]]; then
             echo "::warning::No test report paths found, will not pass to sonarqube"
           else
-            ARGS="$ARGS -Dsonar.go.tests.reportPaths=${{ steps.sonarqube_report_paths.outputs.sonarqube_tests_report_paths }}"
+            echo "Found test report paths: $sonarqube_tests_report_paths"
+            ARGS="$ARGS -Dsonar.go.tests.reportPaths=$sonarqube_tests_report_paths"
           fi
 
-          if [[ -z "${{ steps.sonarqube_report_paths.outputs.sonarqube_coverage_report_paths }}" ]]; then
+          if [[ -z "$sonarqube_coverage_report_paths" ]]; then
             echo "::warning::No coverage report paths found, will not pass to sonarqube"
           else
-            ARGS="$ARGS -Dsonar.go.coverage.reportPaths=${{ steps.sonarqube_report_paths.outputs.sonarqube_coverage_report_paths }}"
+            echo "Found coverage report paths: $sonarqube_coverage_report_paths"
+            ARGS="$ARGS -Dsonar.go.coverage.reportPaths=$sonarqube_coverage_report_paths"
           fi
 
-          if [[ -z "${{ steps.sonarqube_report_paths.outputs.sonarqube_lint_report_paths }}" ]]; then
+          if [[ -z "$sonarqube_lint_report_paths" ]]; then
             echo "::warning::No lint report paths found, will not pass to sonarqube"
           else
-            ARGS="$ARGS -Dsonar.go.golangci-lint.reportPaths=${{ steps.sonarqube_report_paths.outputs.sonarqube_lint_report_paths }}"
+            echo "Found lint report paths: $sonarqube_lint_report_paths"
+            ARGS="$ARGS -Dsonar.go.golangci-lint.reportPaths=$sonarqube_lint_report_paths"
           fi
 
+          echo "Final SONARQUBE_ARGS: $ARGS"
           echo "SONARQUBE_ARGS=$ARGS" >> $GITHUB_ENV
 
       - name: SonarQube Scan

.github/workflows/crib-integration-test.yml

@@ -1,74 +1,111 @@
-# this is disabled because of GAP limitations, should be re-enabled when github-actions-controller will be installed
+name: CRIB Integration Tests
+on:
+  schedule:
+    - cron: "0 1 * * *"
+  workflow_call:
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    environment: integration
+    permissions:
+      id-token: write
+      contents: read
+      actions: read
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2
 
-#name: CRIB Integration Tests
-#on:
-#  push:
-#  workflow_call:
-#concurrency:
-#  group: ${{ github.workflow }}-${{ github.ref }}
-#  cancel-in-progress: true
-#jobs:
-#  test:
-#    runs-on: ubuntu-latest
-#    environment: integration
-#    permissions:
-#      id-token: write
-#      contents: read
-#      actions: read
-#    steps:
-#      - name: Checkout repository
-#        uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2
-#
-#      - name: Setup Nix + GATI environment
-#        uses: smartcontractkit/.github/actions/setup-nix-gati@514fe346780e2eddf7ea8b9f48120c2fba120d94
-#        with:
-#          aws-role-arn: ${{ secrets.AWS_OIDC_CHAINLINK_AUTO_PR_TOKEN_ISSUER_ROLE_ARN }}
-#          aws-lambda-url: ${{ secrets.AWS_CORE_TOKEN_ISSUER_LAMBDA_URL }} # see https://github.com/smartcontractkit/  infra/blob/a79bcfb48315c4411023c182e98eb80ff9e9cda6/accounts/production/us-west-2/lambda/  github-app-token-issuer-production/teams/releng/config.json#L9
-#          aws-region: ${{ secrets.AWS_REGION }}
-#          aws-role-duration-seconds: ${{ secrets.AWS_ROLE_DURATION_SECONDS }}
-#          enable-magic-cache: true
-#
-#      - name: Nix Develop Action
-#        uses: nicknovitski/nix-develop@v1
-#        with:
-#          arguments: "--accept-flake-config"
-#      - name: setup-gap
-#        uses: smartcontractkit/.github/actions/setup-gap@d316f66b2990ea4daa479daa3de6fc92b00f863e # [email protected]
-#        with:
-#          aws-role-arn: ${{ secrets.AWS_OIDC_CRIB_ROLE_ARN_STAGE }}
-#          api-gateway-host: ${{ secrets.AWS_API_GW_HOST_K8S_STAGE }}
-#          aws-region: ${{ secrets.AWS_REGION }}
-#          ecr-private-registry: ${{ secrets.AWS_ACCOUNT_ID_PROD }}
-#          k8s-cluster-name: ${{ secrets.AWS_K8S_CLUSTER_NAME_STAGE }}
-#          use-private-ecr-registry: true
-#          use-k8s: true
-#          metrics-job-name: "k8s"
-#          gc-basic-auth: ${{ secrets.GRAFANA_INTERNAL_BASIC_AUTH }}
-#          gc-host: ${{ secrets.GRAFANA_INTERNAL_HOST }}
-#          gc-org-id: ${{ secrets.GRAFANA_INTERNAL_TENANT_ID }}
-#      - uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2
-#        name: Checkout CRIB repository
-#        with:
-#          repository: 'smartcontractkit/crib'
-#          ref: 'main'
-#      - name: Generate Short UUID
-#        id: uuid
-#        run: echo "CRIB_NAMESPACE=$(uuidgen | cut -c1-5)" >> $GITHUB_ENV
-#      - name: Create a new CRIB environment
-#        run: |-
-#          devspace use namespace $CRIB_NAMESPACE
-#          devspace deploy --profile local-dev-simulated-core-ocr1
-#      - uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2
-#      - name: Setup go
-#        uses: actions/setup-go@0c52d547c9bc32b1aa3301fd7a9cb496313a4491 # v5.0.0
-#        with:
-#          go-version-file: "go.mod"
-#      - name: Run CRIB integration test
-#        working-directory: integration-tests/crib
-#        env:
-#          K8S_STAGING_INGRESS_SUFFIX: ${{ secrets.K8S_STAGING_INGRESS_SUFFIX }}
-#          CRIB_NAMESPACE: ${{ env.CRIB_NAMESPACE }}
-#          CRIB_NETWORK: geth
-#          CRIB_NODES: 5
-#        run: |-
-#          go test -v -run TestCRIB
+      - uses: cachix/install-nix-action@ba0dd844c9180cbf77aa72a116d6fbc515d0e87b # v27
+        with:
+          nix_path: nixpkgs=channel:nixos-unstable
+
+      - name: setup-gap crib
+        uses: smartcontractkit/.github/actions/setup-gap@00b58566e0ee2761e56d9db0ea72b783fdb89b8d # [email protected]
+        with:
+          aws-role-duration-seconds: 3600 # 1 hour
+          aws-role-arn: ${{ secrets.AWS_OIDC_CRIB_ROLE_ARN_STAGE }}
+          api-gateway-host: ${{ secrets.AWS_API_GW_HOST_CRIB_STAGE }}
+          aws-region: ${{ secrets.AWS_REGION }}
+          ecr-private-registry: ${{ secrets.AWS_ACCOUNT_ID_PROD }}
+          k8s-cluster-name: ${{ secrets.AWS_K8S_CLUSTER_NAME_STAGE }}
+          gap-name: crib
+          use-private-ecr-registry: true
+          use-tls: true
+          proxy-port: 8080
+          metrics-job-name: "test"
+          gc-basic-auth: ${{ secrets.GRAFANA_INTERNAL_BASIC_AUTH }}
+          gc-host: ${{ secrets.GRAFANA_INTERNAL_HOST }}
+          gc-org-id: ${{ secrets.GRAFANA_INTERNAL_TENANT_ID }}
+
+      - name: setup-gap k8s
+        uses: smartcontractkit/.github/actions/setup-gap@00b58566e0ee2761e56d9db0ea72b783fdb89b8d # [email protected]
+        with:
+          aws-role-duration-seconds: 3600 # 1 hour
+          aws-role-arn: ${{ secrets.AWS_OIDC_CRIB_ROLE_ARN_STAGE }}
+          api-gateway-host: ${{ secrets.AWS_API_GW_HOST_K8S_STAGE }}
+          aws-region: ${{ secrets.AWS_REGION }}
+          ecr-private-registry: ${{ secrets.AWS_ACCOUNT_ID_PROD }}
+          k8s-cluster-name: ${{ secrets.AWS_K8S_CLUSTER_NAME_STAGE }}
+          gap-name: k8s
+          use-private-ecr-registry: true
+          use-k8s: true
+          proxy-port: 8443
+          metrics-job-name: "test"
+          gc-basic-auth: ${{ secrets.GRAFANA_INTERNAL_BASIC_AUTH }}
+          gc-host: ${{ secrets.GRAFANA_INTERNAL_HOST }}
+          gc-org-id: ${{ secrets.GRAFANA_INTERNAL_TENANT_ID }}
+
+      - name: Setup GitHub token using GATI
+        id: token
+        uses: smartcontractkit/.github/actions/setup-github-token@c0b38e6c40d72d01b8d2f24f92623a2538b3dedb # main
+        with:
+          aws-role-arn: ${{ secrets.AWS_OIDC_GLOBAL_READ_ONLY_TOKEN_ISSUER_ROLE_ARN }}
+          aws-lambda-url: ${{ secrets.AWS_INFRA_RELENG_TOKEN_ISSUER_LAMBDA_URL }}
+          aws-region: ${{ secrets.AWS_REGION }}
+          aws-role-duration-seconds: "1800"
+      - name: Debug workspace dir
+        shell: bash
+        run: |
+          echo ${{ github.workspace }}
+          echo $GITHUB_WORKSPACE
+
+      - name: Deploy and validate CRIB Environment for Core
+        uses: smartcontractkit/.github/actions/crib-deploy-environment@c0b38e6c40d72d01b8d2f24f92623a2538b3dedb # [email protected]
+        id: deploy-crib
+        with:
+          github-token: ${{ steps.token.outputs.access-token }}
+          api-gateway-host: ${{ secrets.AWS_API_GW_HOST_K8S_STAGE }}
+          aws-region: ${{ secrets.AWS_REGION }}
+          aws-role-arn: ${{ secrets.AWS_OIDC_CRIB_ROLE_ARN_STAGE }}
+          ecr-private-registry-stage: ${{ secrets.AWS_ACCOUNT_ID_STAGE }}
+          ecr-private-registry: ${{ secrets.AWS_ACCOUNT_ID_PROD }}
+          ingress-base-domain: ${{ secrets.INGRESS_BASE_DOMAIN_STAGE }}
+          k8s-cluster-name: ${{ secrets.AWS_K8S_CLUSTER_NAME_STAGE }}
+          devspace-profiles: "local-dev-simulated-core-ocr1"
+      - uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2
+      - name: Setup go
+        uses: actions/setup-go@0c52d547c9bc32b1aa3301fd7a9cb496313a4491 # v5.0.0
+        with:
+          go-version-file: "go.mod"
+      - name: Run CRIB integration test
+        working-directory: integration-tests/crib
+        env:
+          K8S_STAGING_INGRESS_SUFFIX: ${{ secrets.K8S_STAGING_INGRESS_SUFFIX }}
+          CRIB_NAMESPACE: ${{ steps.deploy-crib.outputs.devspace-namespace }}
+          CRIB_NETWORK: geth
+          CRIB_NODES: 5
+          GAP_URL: ${{ secrets.GAP_URL }}
+#          SETH_LOG_LEVEL: debug
+#          RESTY_DEBUG: true
+#          TEST_PERSISTENCE: true
+        run: |-
+          go test -v -run TestCRIB
+      - name: Destroy CRIB Environment
+        id: destroy
+        if: always() && steps.deploy-crib.outputs.devspace-namespace != ''
+        uses: smartcontractkit/.github/actions/crib-purge-environment@c0b38e6c40d72d01b8d2f24f92623a2538b3dedb # [email protected]
+        with:
+          namespace: ${{ steps.deploy-crib.outputs.devspace-namespace }}

core/chains/evm/logpoller/observability.go

@@ -285,7 +285,7 @@ func withObservedExecAndRowsAffected(o *ObservedORM, queryName string, queryType
 		WithLabelValues(o.chainId, queryName, string(queryType)).
 		Observe(float64(time.Since(queryStarted)))
 
-	if err != nil {
+	if err == nil {
 		o.datasetSize.
 			WithLabelValues(o.chainId, queryName, string(queryType)).
 			Set(float64(rowsAffected))

core/chains/evm/logpoller/observability_test.go

@@ -16,6 +16,7 @@ import (
 	"github.com/prometheus/client_golang/prometheus/testutil"
 
 	"github.com/smartcontractkit/chainlink-common/pkg/logger"
+
 	"github.com/smartcontractkit/chainlink/v2/core/chains/evm/utils"
 	ubig "github.com/smartcontractkit/chainlink/v2/core/chains/evm/utils/big"
 	"github.com/smartcontractkit/chainlink/v2/core/internal/testutils"
@@ -117,6 +118,16 @@ func TestCountersAreProperlyPopulatedForWrites(t *testing.T) {
 	assert.Equal(t, float64(20), testutil.ToFloat64(orm.logsInserted.WithLabelValues("420")))
 	assert.Equal(t, float64(2), testutil.ToFloat64(orm.blocksInserted.WithLabelValues("420")))
 
+	rowsAffected, err := orm.DeleteExpiredLogs(ctx, 3)
+	require.NoError(t, err)
+	require.Equal(t, int64(3), rowsAffected)
+	assert.Equal(t, 3, counterFromGaugeByLabels(orm.datasetSize, "420", "DeleteExpiredLogs", "delete"))
+
+	rowsAffected, err = orm.DeleteBlocksBefore(ctx, 30, 0)
+	require.NoError(t, err)
+	require.Equal(t, int64(2), rowsAffected)
+	assert.Equal(t, 2, counterFromGaugeByLabels(orm.datasetSize, "420", "DeleteBlocksBefore", "delete"))
+
 	// Don't update counters in case of an error
 	require.Error(t, orm.InsertLogsWithBlock(ctx, logs, NewLogPollerBlock(utils.RandomBytes32(), 0, time.Now(), 0)))
 	assert.Equal(t, float64(20), testutil.ToFloat64(orm.logsInserted.WithLabelValues("420")))

core/chains/evm/logpoller/orm.go

@@ -15,6 +15,7 @@ import (
 	"github.com/smartcontractkit/chainlink-common/pkg/types/query"
 
 	"github.com/smartcontractkit/chainlink-common/pkg/sqlutil"
+
 	evmtypes "github.com/smartcontractkit/chainlink/v2/core/chains/evm/types"
 
 	"github.com/smartcontractkit/chainlink/v2/core/chains/evm/types"
@@ -313,34 +314,29 @@ type Exp struct {
 	ShouldDelete bool
 }
 
+// DeleteExpiredLogs removes any logs which either:
+//   - don't match any currently registered filters, or
+//   - have a timestamp older than any matching filter's retention, UNLESS there is at
+//     least one matching filter with retention=0
 func (o *DSORM) DeleteExpiredLogs(ctx context.Context, limit int64) (int64, error) {
 	var err error
 	var result sql.Result
-	if limit > 0 {
-		result, err = o.ds.ExecContext(ctx, `
-		DELETE FROM evm.logs
+	query := `DELETE FROM evm.logs
 		WHERE (evm_chain_id, address, event_sig, block_number) IN (
 			SELECT l.evm_chain_id, l.address, l.event_sig, l.block_number
 			FROM evm.logs l
-			INNER JOIN (
-				SELECT address, event, MAX(retention) AS retention
+			LEFT JOIN (
+				SELECT address, event, CASE WHEN MIN(retention) = 0 THEN 0 ELSE MAX(retention) END AS retention
 				FROM evm.log_poller_filters
 				WHERE evm_chain_id = $1
 				GROUP BY evm_chain_id, address, event
-				HAVING NOT 0 = ANY(ARRAY_AGG(retention))
 			) r ON l.evm_chain_id = $1 AND l.address = r.address AND l.event_sig = r.event
-			AND l.block_timestamp <= STATEMENT_TIMESTAMP() - (r.retention / 10^9 * interval '1 second')
-			LIMIT $2
-		)`, ubig.New(o.chainID), limit)
+			WHERE r.retention IS NULL OR (r.retention != 0 AND l.block_timestamp <= STATEMENT_TIMESTAMP() - (r.retention / 10^9 * interval '1 second')) %s)`
+
+	if limit > 0 {
+		result, err = o.ds.ExecContext(ctx, fmt.Sprintf(query, "LIMIT $2"), ubig.New(o.chainID), limit)
 	} else {
-		result, err = o.ds.ExecContext(ctx, `WITH r AS
-		( SELECT address, event, MAX(retention) AS retention
-			FROM evm.log_poller_filters WHERE evm_chain_id=$1 
-			GROUP BY evm_chain_id,address, event HAVING NOT 0 = ANY(ARRAY_AGG(retention))
-		) DELETE FROM evm.logs l USING r
-			WHERE l.evm_chain_id = $1 AND l.address=r.address AND l.event_sig=r.event
-			AND l.block_timestamp <= STATEMENT_TIMESTAMP() - (r.retention / 10^9 * interval '1 second')`, // retention is in nanoseconds (time.Duration aka BIGINT)
-			ubig.New(o.chainID))
+		result, err = o.ds.ExecContext(ctx, fmt.Sprintf(query, ""), ubig.New(o.chainID))
 	}
 
 	if err != nil {

core/chains/evm/logpoller/orm_test.go

@@ -458,20 +458,21 @@ func TestORM(t *testing.T) {
 	time.Sleep(2 * time.Millisecond) // just in case we haven't reached the end of the 1ms retention period
 	deleted, err := o1.DeleteExpiredLogs(ctx, 0)
 	require.NoError(t, err)
-	assert.Equal(t, int64(1), deleted)
+	assert.Equal(t, int64(4), deleted)
+
 	logs, err = o1.SelectLogsByBlockRange(ctx, 1, latest.BlockNumber)
 	require.NoError(t, err)
-	// The only log which should be deleted is the one which matches filter1 (ret=1ms) but not filter12 (ret=1 hour)
-	// Importantly, it shouldn't delete any logs matching only filter0 (ret=0 meaning permanent retention).  Anything
-	// matching filter12 should be kept regardless of what other filters it matches.
-	assert.Len(t, logs, 7)
+	// It should have retained the log matching filter0 (due to ret=0 meaning permanent retention) as well as all
+	// 3 logs matching filter12 (ret=1 hour). It should have deleted 3 logs not matching any filter, as well as 1
+	// of the 2 logs matching filter1 (ret=1ms)--the one that doesn't also match filter12.
+	assert.Len(t, logs, 4)
 
 	// Delete logs after should delete all logs.
 	err = o1.DeleteLogsAndBlocksAfter(ctx, 1)
 	require.NoError(t, err)
 	logs, err = o1.SelectLogsByBlockRange(ctx, 1, latest.BlockNumber)
 	require.NoError(t, err)
-	require.Zero(t, len(logs))
+	assert.Zero(t, len(logs))
 }
 
 type PgxLogger struct {

integration-tests/README.md

@@ -129,3 +129,9 @@ Run soak/ocr_test.go with RPC network chaos by bringing down network to RPC node
 ```bash
 make test_soak_ocr_rpc_down_half_cl_nodes
 ```
+
+### Debugging HTTP and RPC clients
+```bash
+export SETH_LOG_LEVEL=debug
+export RESTY_DEBUG=true
+```

integration-tests/actions/vrf/common/actions.go

@@ -429,7 +429,7 @@ type RPCRawClient struct {
 }
 
 func NewRPCRawClient(url string) *RPCRawClient {
-	isDebug := os.Getenv("DEBUG_RESTY") == "true"
+	isDebug := os.Getenv("RESTY_DEBUG") == "true"
 	restyClient := resty.New().SetDebug(isDebug).SetBaseURL(url)
 	return &RPCRawClient{
 		resty: restyClient,