Merge branch 'develop' into feature/BCI-2649-latest-finalized-block

# Conflicts:
#	core/chains/evm/headtracker/orm.go

GNUmakefile

@@ -145,7 +145,7 @@ config-docs: ## Generate core node configuration documentation
 .PHONY: golangci-lint
 golangci-lint: ## Run golangci-lint for all issues.
 	[ -d "./golangci-lint" ] || mkdir ./golangci-lint && \
-	docker run --rm -v $(shell pwd):/app -w /app golangci/golangci-lint:v1.55.2 golangci-lint run --max-issues-per-linter 0 --max-same-issues 0 > ./golangci-lint/$(shell date +%Y-%m-%d_%H:%M:%S).txt
+	docker run --rm -v $(shell pwd):/app -w /app golangci/golangci-lint:v1.56.2 golangci-lint run --max-issues-per-linter 0 --max-same-issues 0 > ./golangci-lint/$(shell date +%Y-%m-%d_%H:%M:%S).txt
 
 
 GORELEASER_CONFIG ?= .goreleaser.yaml

charts/chainlink-cluster/Chart.lock

@@ -4,12 +4,12 @@ dependencies:
   version: 5.14.0
 - name: opentelemetry-collector
   repository: https://open-telemetry.github.io/opentelemetry-helm-charts
-  version: 0.81.2
+  version: 0.82.0
 - name: tempo
   repository: https://grafana.github.io/helm-charts
   version: 1.7.2
 - name: grafana
   repository: https://grafana.github.io/helm-charts
   version: 7.3.2
-digest: sha256:fb13a8a7b490fd1e388a6b082e50d65a39f44a28030d66559d4640c687a70e20
-generated: "2024-02-23T15:22:09.284013-06:00"
+digest: sha256:37722063f68689c42ac1d6549ddfae4756370c1659b8ac1c0d7da8916c6fad3d
+generated: "2024-02-27T11:04:29.920915-06:00"

charts/chainlink-cluster/Chart.yaml

@@ -1,15 +1,15 @@
 apiVersion: v2
 name: chainlink-cluster
 description: Chainlink nodes cluster
-version: 0.3.0
+version: 0.4.0
 appVersion: "2.6.0"
 dependencies:
   - name: mockserver
     version: "5.14.0"
     repository: "https://www.mock-server.com"
     condition: mockserver.enabled
   - name: opentelemetry-collector
-    version: "0.81.2"
+    version: "0.82.0"
     repository: "https://open-telemetry.github.io/opentelemetry-helm-charts"
     condition: opentelemetry-collector.enabled
   - name: tempo

charts/chainlink-cluster/README.md

@@ -31,6 +31,13 @@ Build and deploy current commit
 devspace deploy
 ```
 
+Default `ttl` is `72h`, use `ttl` command to update if you need more time
+
+Valid values are `1h`, `2m`, `3s`, etc. Go time format is invalid `1h2m3s`
+```
+devspace run ttl ${namespace} 120h
+```
+
 If you don't need a build use
 ```
 devspace deploy --skip-build

charts/chainlink-cluster/dashboard/dashboard.go

@@ -14,7 +14,7 @@ import (
 	"github.com/K-Phoen/grabana/timeseries"
 	"github.com/K-Phoen/grabana/timeseries/axis"
 	"github.com/K-Phoen/grabana/variable/query"
-	"github.com/pkg/errors"
+	pkgerrors "github.com/pkg/errors"
 )
 
 /*
@@ -954,10 +954,10 @@ func (m *CLClusterDashboard) Deploy(ctx context.Context) error {
 	client := grabana.NewClient(&http.Client{}, m.GrafanaURL, grabana.WithAPIToken(m.GrafanaToken))
 	folder, err := client.FindOrCreateFolder(ctx, m.Folder)
 	if err != nil {
-		return errors.Wrap(err, ErrFailedToCreateFolder)
+		return pkgerrors.Wrap(err, ErrFailedToCreateFolder)
 	}
 	if _, err := client.UpsertDashboard(ctx, folder, m.builder); err != nil {
-		return errors.Wrap(err, ErrFailedToCreateDashboard)
+		return pkgerrors.Wrap(err, ErrFailedToCreateDashboard)
 	}
 	return nil
 }

charts/chainlink-cluster/devspace.yaml

@@ -89,6 +89,8 @@ pipelines:
 commands:
   connect: |-
     sudo kubefwd svc -n $1
+  ttl: |-
+    kubectl annotate namespace $1 janitor/ttl=$2 --overwrite
 
 images:
   app:
@@ -300,12 +302,6 @@ deployments:
         # monitoring.coreos.com/v1 PodMonitor for each node
         prometheusMonitor: true
 
-        networkPolicy:
-          ingress:
-            # Should be a comma separated list of CIDR blocks. To include
-            # AWS ALB private CIDRs and optionally other custom CIDRs.
-            # Example format: 10.0.0.0/16,192.168.0.1/24
-            allowCustomCidrs: ${DEVSPACE_INGRESS_CIDRS}
         # These ingresses create AWS ALB resources and Route 53 Records.
         ingress:
           enabled: true
@@ -411,7 +407,13 @@ deployments:
                         name: mockserver
                         port:
                           number: 1080
-
+        networkPolicyDefault:
+          ingress:
+            allowCustomCidrs: true
+            # Should be a comma separated list of CIDR blocks. To include
+            # AWS ALB private CIDRs and optionally other custom CIDRs.
+            # Example format: 10.0.0.0/16,192.168.0.1/24
+            customCidrs: ${DEVSPACE_INGRESS_CIDRS}
         # deployment placement, standard helm stuff
         podAnnotations:
         nodeSelector:

charts/chainlink-cluster/templates/ingress.yaml

@@ -15,6 +15,7 @@ metadata:
     alb.ingress.kubernetes.io/group.name: {{ $.Values.ingress.annotation_group_name | quote }}
     alb.ingress.kubernetes.io/scheme: internal
     alb.ingress.kubernetes.io/target-type: ip
+    external-dns.alpha.kubernetes.io/ttl: "120"
   {{- if .Values.ingress.extra_annotations }}
   {{- range $key, $value := .Values.ingress.extra_annotations }}
     {{ $key }}: {{ $value | quote }}

charts/chainlink-cluster/values.yaml

@@ -197,7 +197,7 @@ opentelemetry-collector:
   mode: deployment
   image:
     repository: otel/opentelemetry-collector
-    tag: 0.81.2
+    tag: 0.95.0
   command:
     name: otelcol
   extraVolumes:
@@ -272,6 +272,8 @@ grafana:
   enabled: true
   image:
     tag: 7.3.2
+  rbac:
+    namespaced: true
   datasources:
     datasources.yaml:
       apiVersion: 1
@@ -402,6 +404,55 @@ affinity:
 
 networkPolicies:
   enabled: true
+  customPolicies:
+    grafanaToTempoEgress:
+      podSelector:
+        matchLabels:
+          app: grafana
+      egress:
+        - to:
+            - podSelector:
+                matchLabels:
+                  app: tempo
+          ports:
+            - protocol: TCP
+              port: 3100
+    tempoIngressFromGrafana:
+      podSelector:
+        matchLabels:
+          app: tempo
+      ingress:
+        - from:
+            - podSelector:
+                matchLabels:
+                  app: grafana
+          ports:
+            - protocol: TCP
+              port: 3100
+    chainlinkToOtelCollectorEgress:
+      podSelector:
+        matchLabels:
+          app: chainlink
+      egress:
+        - to:
+            - podSelector:
+                matchLabels:
+                  app: otel-collector
+          ports:
+            - protocol: TCP
+              port: 4317
+    otelCollectorToTempoEgress:
+      podSelector:
+        matchLabels:
+          app: otel-collector
+      egress:
+        - to:
+            - podSelector:
+                matchLabels:
+                  app: tempo
+          ports:
+            - protocol: TCP
+              port: 3100
 
 # Configure the default network policy.
 networkPolicyDefault:

common/client/node.go

@@ -42,6 +42,7 @@ type NodeConfig interface {
 	PollInterval() time.Duration
 	SelectionMode() string
 	SyncThreshold() uint32
+	NodeIsSyncingEnabled() bool
 }
 
 //go:generate mockery --quiet --name Node --structname mockNode --filename "mock_node_test.go" --inpackage --case=underscore
@@ -224,53 +225,83 @@ func (n *node[CHAIN_ID, HEAD, RPC]) start(startCtx context.Context) {
 	}
 	n.setState(nodeStateDialed)
 
-	if err := n.verify(startCtx); errors.Is(err, errInvalidChainID) {
-		n.lfcLog.Errorw("Verify failed: Node has the wrong chain ID", "err", err)
-		n.declareInvalidChainID()
-		return
-	} else if err != nil {
-		n.lfcLog.Errorw(fmt.Sprintf("Verify failed: %v", err), "err", err)
-		n.declareUnreachable()
-		return
-	}
-
-	n.declareAlive()
+	state := n.verifyConn(startCtx, n.lfcLog)
+	n.declareState(state)
 }
 
-// verify checks that all connections to eth nodes match the given chain ID
+// verifyChainID checks that connection to the node matches the given chain ID
 // Not thread-safe
-// Pure verify: does not mutate node "state" field.
-func (n *node[CHAIN_ID, HEAD, RPC]) verify(callerCtx context.Context) (err error) {
+// Pure verifyChainID: does not mutate node "state" field.
+func (n *node[CHAIN_ID, HEAD, RPC]) verifyChainID(callerCtx context.Context, lggr logger.Logger) nodeState {
 	promPoolRPCNodeVerifies.WithLabelValues(n.chainFamily, n.chainID.String(), n.name).Inc()
 	promFailed := func() {
 		promPoolRPCNodeVerifiesFailed.WithLabelValues(n.chainFamily, n.chainID.String(), n.name).Inc()
 	}
 
 	st := n.State()
 	switch st {
-	case nodeStateDialed, nodeStateOutOfSync, nodeStateInvalidChainID:
+	case nodeStateDialed, nodeStateOutOfSync, nodeStateInvalidChainID, nodeStateSyncing:
 	default:
 		panic(fmt.Sprintf("cannot verify node in state %v", st))
 	}
 
 	var chainID CHAIN_ID
+	var err error
 	if chainID, err = n.rpc.ChainID(callerCtx); err != nil {
 		promFailed()
-		return fmt.Errorf("failed to verify chain ID for node %s: %w", n.name, err)
+		lggr.Errorw("Failed to verify chain ID for node", "err", err, "nodeState", n.State())
+		return nodeStateUnreachable
 	} else if chainID.String() != n.chainID.String() {
 		promFailed()
-		return fmt.Errorf(
+		err = fmt.Errorf(
 			"rpc ChainID doesn't match local chain ID: RPC ID=%s, local ID=%s, node name=%s: %w",
 			chainID.String(),
 			n.chainID.String(),
 			n.name,
 			errInvalidChainID,
 		)
+		lggr.Errorw("Failed to verify RPC node; remote endpoint returned the wrong chain ID", "err", err, "nodeState", n.State())
+		return nodeStateInvalidChainID
 	}
 
 	promPoolRPCNodeVerifiesSuccess.WithLabelValues(n.chainFamily, n.chainID.String(), n.name).Inc()
 
-	return nil
+	return nodeStateAlive
+}
+
+// createVerifiedConn - establishes new connection with the RPC and verifies that it's valid: chainID matches, and it's not syncing.
+// Returns desired state if one of the verifications fails. Otherwise, returns nodeStateAlive.
+func (n *node[CHAIN_ID, HEAD, RPC]) createVerifiedConn(ctx context.Context, lggr logger.Logger) nodeState {
+	if err := n.rpc.Dial(ctx); err != nil {
+		n.lfcLog.Errorw("Dial failed: Node is unreachable", "err", err, "nodeState", n.State())
+		return nodeStateUnreachable
+	}
+
+	return n.verifyConn(ctx, lggr)
+}
+
+// verifyConn - verifies that current connection is valid: chainID matches, and it's not syncing.
+// Returns desired state if one of the verifications fails. Otherwise, returns nodeStateAlive.
+func (n *node[CHAIN_ID, HEAD, RPC]) verifyConn(ctx context.Context, lggr logger.Logger) nodeState {
+	state := n.verifyChainID(ctx, lggr)
+	if state != nodeStateAlive {
+		return state
+	}
+
+	if n.nodePoolCfg.NodeIsSyncingEnabled() {
+		isSyncing, err := n.rpc.IsSyncing(ctx)
+		if err != nil {
+			lggr.Errorw("Unexpected error while verifying RPC node synchronization status", "err", err, "nodeState", n.State())
+			return nodeStateUnreachable
+		}
+
+		if isSyncing {
+			lggr.Errorw("Verification failed: Node is syncing", "nodeState", n.State())
+			return nodeStateSyncing
+		}
+	}
+
+	return nodeStateAlive
 }
 
 // disconnectAll disconnects all clients connected to the node