OCRv1 CRIB smoke + chaos test

.github/workflows/crib-integration-test.yml

@@ -0,0 +1,74 @@
+# this is disabled because of GAP limitations, should be re-enabled when github-actions-controller will be installed
+
+#name: CRIB Integration Tests
+#on:
+#  push:
+#  workflow_call:
+#concurrency:
+#  group: ${{ github.workflow }}-${{ github.ref }}
+#  cancel-in-progress: true
+#jobs:
+#  test:
+#    runs-on: ubuntu-latest
+#    environment: integration
+#    permissions:
+#      id-token: write
+#      contents: read
+#      actions: read
+#    steps:
+#      - name: Checkout repository
+#        uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2
+#
+#      - name: Setup Nix + GATI environment
+#        uses: smartcontractkit/.github/actions/setup-nix-gati@514fe346780e2eddf7ea8b9f48120c2fba120d94
+#        with:
+#          aws-role-arn: ${{ secrets.AWS_OIDC_CHAINLINK_AUTO_PR_TOKEN_ISSUER_ROLE_ARN }}
+#          aws-lambda-url: ${{ secrets.AWS_CORE_TOKEN_ISSUER_LAMBDA_URL }} # see https://github.com/smartcontractkit/  infra/blob/a79bcfb48315c4411023c182e98eb80ff9e9cda6/accounts/production/us-west-2/lambda/  github-app-token-issuer-production/teams/releng/config.json#L9
+#          aws-region: ${{ secrets.AWS_REGION }}
+#          aws-role-duration-seconds: ${{ secrets.AWS_ROLE_DURATION_SECONDS }}
+#          enable-magic-cache: true
+#
+#      - name: Nix Develop Action
+#        uses: nicknovitski/nix-develop@v1
+#        with:
+#          arguments: "--accept-flake-config"
+#      - name: setup-gap
+#        uses: smartcontractkit/.github/actions/setup-gap@d316f66b2990ea4daa479daa3de6fc92b00f863e # [email protected]
+#        with:
+#          aws-role-arn: ${{ secrets.AWS_OIDC_CRIB_ROLE_ARN_STAGE }}
+#          api-gateway-host: ${{ secrets.AWS_API_GW_HOST_K8S_STAGE }}
+#          aws-region: ${{ secrets.AWS_REGION }}
+#          ecr-private-registry: ${{ secrets.AWS_ACCOUNT_ID_PROD }}
+#          k8s-cluster-name: ${{ secrets.AWS_K8S_CLUSTER_NAME_STAGE }}
+#          use-private-ecr-registry: true
+#          use-k8s: true
+#          metrics-job-name: "k8s"
+#          gc-basic-auth: ${{ secrets.GRAFANA_INTERNAL_BASIC_AUTH }}
+#          gc-host: ${{ secrets.GRAFANA_INTERNAL_HOST }}
+#          gc-org-id: ${{ secrets.GRAFANA_INTERNAL_TENANT_ID }}
+#      - uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2
+#        name: Checkout CRIB repository
+#        with:
+#          repository: 'smartcontractkit/crib'
+#          ref: 'main'
+#      - name: Generate Short UUID
+#        id: uuid
+#        run: echo "CRIB_NAMESPACE=$(uuidgen | cut -c1-5)" >> $GITHUB_ENV
+#      - name: Create a new CRIB environment
+#        run: |-
+#          devspace use namespace $CRIB_NAMESPACE
+#          devspace deploy --profile local-dev-simulated-core-ocr1
+#      - uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2
+#      - name: Setup go
+#        uses: actions/setup-go@0c52d547c9bc32b1aa3301fd7a9cb496313a4491 # v5.0.0
+#        with:
+#          go-version-file: "go.mod"
+#      - name: Run CRIB integration test
+#        working-directory: integration-tests/crib
+#        env:
+#          K8S_STAGING_INGRESS_SUFFIX: ${{ secrets.K8S_STAGING_INGRESS_SUFFIX }}
+#          CRIB_NAMESPACE: ${{ env.CRIB_NAMESPACE }}
+#          CRIB_NETWORK: geth
+#          CRIB_NODES: 5
+#        run: |-
+#          go test -v -run TestCRIB

integration-tests/actions/ocr_helpers.go

@@ -2,8 +2,15 @@ package actions
 
 import (
 	"fmt"
+	"math/big"
+	"math/rand"
 	"strings"
 	"testing"
+	"time"
+
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/rs/zerolog"
+	"github.com/smartcontractkit/seth"
 
 	"github.com/google/uuid"
 	"github.com/stretchr/testify/require"
@@ -217,3 +224,55 @@ func BuildNodeContractPairID(node contracts.ChainlinkNodeWithKeysAndAddress, ocr
 	shortOCRAddr := ocrInstance.Address()[2:12]
 	return strings.ToLower(fmt.Sprintf("node_%s_contract_%s", shortNodeAddr, shortOCRAddr)), nil
 }
+
+func SetupOCRv1Cluster(
+	l zerolog.Logger,
+	seth *seth.Client,
+	workerNodes []*client.ChainlinkK8sClient,
+) (common.Address, error) {
+	err := FundChainlinkNodesFromRootAddress(l, seth, contracts.ChainlinkK8sClientToChainlinkNodeWithKeysAndAddress(workerNodes), big.NewFloat(3))
+	if err != nil {
+		return common.Address{}, err
+	}
+	linkContract, err := contracts.DeployLinkTokenContract(l, seth)
+	if err != nil {
+		return common.Address{}, err
+	}
+	return common.HexToAddress(linkContract.Address()), nil
+}
+
+func SetupOCRv1Feed(
+	l zerolog.Logger,
+	seth *seth.Client,
+	lta common.Address,
+	msClient *ctfClient.MockserverClient,
+	bootstrapNode *client.ChainlinkK8sClient,
+	workerNodes []*client.ChainlinkK8sClient,
+) ([]contracts.OffchainAggregator, error) {
+	ocrInstances, err := DeployOCRv1Contracts(l, seth, 1, lta, contracts.ChainlinkK8sClientToChainlinkNodeWithKeysAndAddress(workerNodes))
+	if err != nil {
+		return nil, err
+	}
+	err = CreateOCRJobs(ocrInstances, bootstrapNode, workerNodes, 5, msClient, fmt.Sprint(seth.ChainID))
+	if err != nil {
+		return nil, err
+	}
+	return ocrInstances, nil
+}
+
+func SimulateOCRv1EAActivity(
+	l zerolog.Logger,
+	eaChangeInterval time.Duration,
+	ocrInstances []contracts.OffchainAggregator,
+	workerNodes []*client.ChainlinkK8sClient,
+	msClient *ctfClient.MockserverClient,
+) {
+	go func() {
+		for {
+			time.Sleep(eaChangeInterval)
+			if err := SetAllAdapterResponsesToTheSameValue(rand.Intn(1000), ocrInstances, workerNodes, msClient); err != nil {
+				l.Error().Err(err).Msg("failed to update mockserver responses")
+			}
+		}
+	}()
+}

integration-tests/crib/README.md

@@ -0,0 +1,19 @@
+### CRIB Health Check Test
+
+## Setup CRIB
+This is a simple smoke + chaos test for CRIB deployment.
+It runs OCRv1 and reboots the environment confirming integration with environment is working and data is properly saved even after reboots.
+Go to the [CRIB](https://github.com/smartcontractkit/crib) repository and spin up a cluster.
+
+```shell
+./scripts/cribbit.sh crib-oh-my-crib
+devspace deploy --debug --profile local-dev-simulated-core-ocr1
+```
+
+## Run the tests
+```shell
+CRIB_NAMESPACE=crib-oh-my-crib
+CRIB_NETWORK=geth # only "geth" is supported for now
+CRIB_NODES=5 # min 5 nodes
+go test -v -run TestCRIB
+```

integration-tests/crib/chaos.go

@@ -0,0 +1,45 @@
+package crib
+
+import (
+	"time"
+
+	"github.com/chaos-mesh/chaos-mesh/api/v1alpha1"
+	"github.com/smartcontractkit/havoc/k8schaos"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+)
+
+func rebootCLNamespace(delay time.Duration, namespace string) (*k8schaos.Chaos, error) {
+	k8sClient, err := k8schaos.NewChaosMeshClient()
+	if err != nil {
+		return nil, err
+	}
+	return k8schaos.NewChaos(k8schaos.ChaosOpts{
+		Description: "Reboot CRIB",
+		DelayCreate: delay,
+		Object: &v1alpha1.PodChaos{
+			TypeMeta: metav1.TypeMeta{
+				Kind:       "PodChaos",
+				APIVersion: "chaos-mesh.org/v1alpha1",
+			},
+			ObjectMeta: metav1.ObjectMeta{
+				Name:      "reboot-crib",
+				Namespace: namespace,
+			},
+			Spec: v1alpha1.PodChaosSpec{
+				ContainerSelector: v1alpha1.ContainerSelector{
+					PodSelector: v1alpha1.PodSelector{
+						Mode: v1alpha1.AllMode,
+						Selector: v1alpha1.PodSelectorSpec{
+							GenericSelectorSpec: v1alpha1.GenericSelectorSpec{
+								Namespaces: []string{namespace},
+							},
+						},
+					},
+				},
+				Action: v1alpha1.PodKillAction,
+			},
+		},
+		Client: k8sClient,
+		Logger: &k8schaos.Logger,
+	})
+}

integration-tests/crib/connect.go

@@ -3,6 +3,7 @@ package crib
 import (
 	"fmt"
 	"os"
+	"strconv"
 	"time"
 
 	"github.com/pkg/errors"
@@ -39,7 +40,7 @@ func setSethConfig(cfg tc.TestConfig, netWSURL string, netHTTPURL string) {
 
 // ConnectRemote connects to a local environment, see https://github.com/smartcontractkit/crib/tree/main/core
 // connects to default CRIB network if simulated = true
-func ConnectRemote(simulated bool) (
+func ConnectRemote() (
 	*seth.Client,
 	*msClient.MockserverClient,
 	*client.ChainlinkK8sClient,
@@ -50,22 +51,35 @@ func ConnectRemote(simulated bool) (
 	if ingressSuffix == "" {
 		return nil, nil, nil, nil, errors.New("K8S_STAGING_INGRESS_SUFFIX must be set to connect to k8s ingresses")
 	}
+	cribNamespace := os.Getenv("CRIB_NAMESPACE")
+	if cribNamespace == "" {
+		return nil, nil, nil, nil, errors.New("CRIB_NAMESPACE must be set to connect")
+	}
+	cribNetwork := os.Getenv("CRIB_NETWORK")
+	if cribNetwork == "" {
+		return nil, nil, nil, nil, errors.New("CRIB_NETWORK must be set to connect, only 'geth' is supported for now")
+	}
+	cribNodes := os.Getenv("CRIB_NODES")
+	nodes, err := strconv.Atoi(cribNodes)
+	if err != nil {
+		return nil, nil, nil, nil, errors.New("CRIB_NODES must be a number, 5-19 nodes")
+	}
 	config, err := tc.GetConfig([]string{"CRIB"}, tc.OCR)
 	if err != nil {
 		return nil, nil, nil, nil, err
 	}
-	if config.CRIB.CLNodesNum < 2 {
+	if nodes < 2 {
 		return nil, nil, nil, nil, fmt.Errorf("not enough chainlink nodes, need at least 2, TOML key: [CRIB.nodes]")
 	}
-	cfg := config.CRIB
-	mockserverURL := fmt.Sprintf(mockserverCRIBTemplate, cfg.Namespace, ingressSuffix)
+	mockserverURL := fmt.Sprintf(mockserverCRIBTemplate, cribNamespace, ingressSuffix)
 	var sethClient *seth.Client
-	if simulated {
-		netWSURL := fmt.Sprintf(ingressNetworkWSURLTemplate, cfg.Namespace, ingressSuffix)
-		netHTTPURL := fmt.Sprintf(ingressNetworkHTTPURLTemplate, cfg.Namespace, ingressSuffix)
+	switch cribNetwork {
+	case "geth":
+		netWSURL := fmt.Sprintf(ingressNetworkWSURLTemplate, cribNamespace, ingressSuffix)
+		netHTTPURL := fmt.Sprintf(ingressNetworkHTTPURLTemplate, cribNamespace, ingressSuffix)
 		setSethConfig(config, netWSURL, netHTTPURL)
 		net := blockchain.EVMNetwork{
-			Name:                 cfg.NetworkName,
+			Name:                 cribNetwork,
 			Simulated:            true,
 			SupportsEIP1559:      true,
 			ClientImplementation: blockchain.EthereumClientImplementation,
@@ -84,27 +98,29 @@ func ConnectRemote(simulated bool) (
 		if err != nil {
 			return nil, nil, nil, nil, err
 		}
+	default:
+		return nil, nil, nil, nil, errors.New("CRIB network is not supported")
 	}
 	// bootstrap node
 	clClients := make([]*client.ChainlinkK8sClient, 0)
 	c, err := client.NewChainlinkK8sClient(&client.ChainlinkConfig{
-		URL:        fmt.Sprintf("https://%s-node%d%s", cfg.Namespace, 1, ingressSuffix),
+		URL:        fmt.Sprintf("https://%s-node%d%s", cribNamespace, 1, ingressSuffix),
 		Email:      client.CLNodeTestEmail,
 		InternalIP: fmt.Sprintf(internalNodeDNSTemplate, 1),
 		Password:   client.CLNodeTestPassword,
-	}, fmt.Sprintf(internalNodeDNSTemplate, 1), cfg.Namespace)
+	}, fmt.Sprintf(internalNodeDNSTemplate, 1), cribNamespace)
 	if err != nil {
 		return nil, nil, nil, nil, err
 	}
 	clClients = append(clClients, c)
 	// all the other nodes, indices of nodes in CRIB starts with 1
-	for i := 2; i <= cfg.CLNodesNum; i++ {
+	for i := 2; i <= nodes; i++ {
 		cl, err := client.NewChainlinkK8sClient(&client.ChainlinkConfig{
-			URL:        fmt.Sprintf("https://%s-node%d%s", cfg.Namespace, i, ingressSuffix),
+			URL:        fmt.Sprintf("https://%s-node%d%s", cribNamespace, i, ingressSuffix),
 			Email:      client.CLNodeTestEmail,
 			InternalIP: fmt.Sprintf(internalNodeDNSTemplate, i),
 			Password:   client.CLNodeTestPassword,
-		}, fmt.Sprintf(internalNodeDNSTemplate, i), cfg.Namespace)
+		}, fmt.Sprintf(internalNodeDNSTemplate, i), cribNamespace)
 		if err != nil {
 			return nil, nil, nil, nil, err
 		}

integration-tests/crib/ocr_test.go

@@ -0,0 +1,54 @@
+package crib
+
+import (
+	"context"
+	"os"
+	"testing"
+	"time"
+
+	"github.com/smartcontractkit/havoc/k8schaos"
+	"github.com/stretchr/testify/require"
+
+	"github.com/smartcontractkit/chainlink/integration-tests/actions"
+	"github.com/smartcontractkit/chainlink/integration-tests/contracts"
+
+	"github.com/smartcontractkit/chainlink-testing-framework/logging"
+)
+
+func TestCRIB(t *testing.T) {
+	l := logging.GetTestLogger(t)
+
+	sethClient, msClient, bootstrapNode, workerNodes, err := ConnectRemote()
+	require.NoError(t, err)
+
+	lta, err := actions.SetupOCRv1Cluster(l, sethClient, workerNodes)
+	require.NoError(t, err)
+	ocrInstances, err := actions.SetupOCRv1Feed(l, sethClient, lta, msClient, bootstrapNode, workerNodes)
+	require.NoError(t, err)
+
+	err = actions.SetAllAdapterResponsesToTheSameValue(10, ocrInstances, workerNodes, msClient)
+	require.NoError(t, err)
+	actions.SimulateOCRv1EAActivity(l, 3*time.Second, ocrInstances, workerNodes, msClient)
+
+	err = actions.WatchNewOCRRound(l, sethClient, 1, contracts.V1OffChainAgrregatorToOffChainAggregatorWithRounds(ocrInstances), 5*time.Minute)
+	require.NoError(t, err, "Error watching for new OCR round")
+
+	ch, err := rebootCLNamespace(
+		1*time.Second,
+		os.Getenv("CRIB_NAMESPACE"),
+	)
+	ch.Create(context.Background())
+	ch.AddListener(k8schaos.NewChaosLogger(l))
+	t.Cleanup(func() {
+		err := ch.Delete(context.Background())
+		require.NoError(t, err)
+	})
+	require.Eventually(t, func() bool {
+		err = actions.WatchNewOCRRound(l, sethClient, 3, contracts.V1OffChainAgrregatorToOffChainAggregatorWithRounds(ocrInstances), 5*time.Minute)
+		if err != nil {
+			l.Info().Err(err).Msg("OCR round is not there yet")
+			return false
+		}
+		return true
+	}, 3*time.Minute, 5*time.Second)
+}

integration-tests/load/go.mod

@@ -374,6 +374,7 @@ require (
 	github.com/smartcontractkit/chainlink-solana v1.0.3-0.20240712132946-267a37c5ac6e // indirect
 	github.com/smartcontractkit/chainlink-starknet/relayer v0.0.1-beta-test.0.20240709043547-03612098f799 // indirect
 	github.com/smartcontractkit/chainlink-testing-framework/grafana v0.0.0-20240405215812-5a72bc9af239 // indirect
+	github.com/smartcontractkit/havoc/k8schaos v0.0.0-20240409145249-e78d20847e37 // indirect
 	github.com/smartcontractkit/tdh2/go/ocr2/decryptionplugin v0.0.0-20230906073235-9e478e5e19f1 // indirect
 	github.com/smartcontractkit/wsrpc v0.7.3 // indirect
 	github.com/soheilhy/cmux v0.1.5 // indirect

integration-tests/load/go.sum

@@ -1483,6 +1483,8 @@ github.com/smartcontractkit/go-plugin v0.0.0-20240208201424-b3b91517de16 h1:TFe+
 github.com/smartcontractkit/go-plugin v0.0.0-20240208201424-b3b91517de16/go.mod h1:lBS5MtSSBZk0SHc66KACcjjlU6WzEVP/8pwz68aMkCI=
 github.com/smartcontractkit/grpc-proxy v0.0.0-20230731113816-f1be6620749f h1:hgJif132UCdjo8u43i7iPN1/MFnu49hv7lFGFftCHKU=
 github.com/smartcontractkit/grpc-proxy v0.0.0-20230731113816-f1be6620749f/go.mod h1:MvMXoufZAtqExNexqi4cjrNYE9MefKddKylxjS+//n0=
+github.com/smartcontractkit/havoc/k8schaos v0.0.0-20240409145249-e78d20847e37 h1:ZEhn2Yo1jY4hqy8nasDL4k4pNtopT3rS3Ap1GDb7ODc=
+github.com/smartcontractkit/havoc/k8schaos v0.0.0-20240409145249-e78d20847e37/go.mod h1:/kFr0D7SI/vueXl1N03uzOun4nViGPFRyA5X6eL3jXw=
 github.com/smartcontractkit/libocr v0.0.0-20240702141926-063ceef8c42e h1:9ypZ/8aW8Vm497i1gXHcT96oNLiu88jbg9QdX+IUE3E=
 github.com/smartcontractkit/libocr v0.0.0-20240702141926-063ceef8c42e/go.mod h1:fb1ZDVXACvu4frX3APHZaEBp0xi1DIm34DcA0CwTsZM=
 github.com/smartcontractkit/seth v1.0.12 h1:iVdgMx42XWanPPnBaM5StR4c1XsTr/0/B/kKRZL5BsY=