From f9fad1f83814d15573952d9298e6649a5f412975 Mon Sep 17 00:00:00 2001 From: skudasov Date: Mon, 8 Jul 2024 16:21:44 +0200 Subject: [PATCH 01/12] quick OCRv1 CRIB test (WIP) --- .github/workflows/crib-integration-test.yml | 44 +++++++++++++ integration-tests/actions/ocr_helpers.go | 58 +++++++++++++++++ integration-tests/crib/ocr_test.go | 36 +++++++++++ integration-tests/load/ocr/helper.go | 70 --------------------- integration-tests/load/ocr/ocr_test.go | 9 +-- 5 files changed, 143 insertions(+), 74 deletions(-) create mode 100644 .github/workflows/crib-integration-test.yml create mode 100644 integration-tests/crib/ocr_test.go delete mode 100644 integration-tests/load/ocr/helper.go diff --git a/.github/workflows/crib-integration-test.yml b/.github/workflows/crib-integration-test.yml new file mode 100644 index 00000000000..175fe221069 --- /dev/null +++ b/.github/workflows/crib-integration-test.yml @@ -0,0 +1,44 @@ +name: CRIB Integration Tests +on: + push: + workflow_call: +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true +jobs: + test: + runs-on: ubuntu-latest + environment: integration + permissions: + id-token: write + contents: read + actions: read + steps: + - name: setup-gap + uses: smartcontractkit/.github/actions/setup-gap@d316f66b2990ea4daa479daa3de6fc92b00f863e # setup-gap@0.3.2 + with: + aws-role-arn: ${{ secrets.AWS_OIDC_CRIB_ROLE_ARN_STAGE }} + api-gateway-host: ${{ secrets.AWS_API_GW_HOST_K8S_STAGE }} + aws-region: ${{ secrets.AWS_REGION }} + ecr-private-registry: ${{ secrets.AWS_ACCOUNT_ID_PROD }} + k8s-cluster-name: ${{ secrets.AWS_K8S_CLUSTER_NAME_STAGE }} + use-private-ecr-registry: true + use-k8s: true + metrics-job-name: "k8s" + gc-basic-auth: ${{ secrets.GRAFANA_INTERNAL_BASIC_AUTH }} + gc-host: ${{ secrets.GRAFANA_INTERNAL_HOST }} + gc-org-id: ${{ secrets.GRAFANA_INTERNAL_TENANT_ID }} + - name: Test + shell: bash + run: kubectl describe pods -n crib-skudasov + - uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2 + - name: Setup go + uses: actions/setup-go@0c52d547c9bc32b1aa3301fd7a9cb496313a4491 # v5.0.0 + with: + go-version-file: "go.mod" + - name: Run tests + working-directory: integration-tests/crib + env: + K8S_STAGING_INGRESS_SUFFIX: ${{ secrets.K8S_STAGING_INGRESS_SUFFIX }} + run: |- + go test -v -run TestCRIB \ No newline at end of file diff --git a/integration-tests/actions/ocr_helpers.go b/integration-tests/actions/ocr_helpers.go index 6e54e81cd84..ec3a910cef0 100644 --- a/integration-tests/actions/ocr_helpers.go +++ b/integration-tests/actions/ocr_helpers.go @@ -2,8 +2,14 @@ package actions import ( "fmt" + "github.com/ethereum/go-ethereum/common" + "github.com/rs/zerolog" + "github.com/smartcontractkit/seth" + "math/big" + "math/rand" "strings" "testing" + "time" "github.com/google/uuid" "github.com/stretchr/testify/require" @@ -217,3 +223,55 @@ func BuildNodeContractPairID(node contracts.ChainlinkNodeWithKeysAndAddress, ocr shortOCRAddr := ocrInstance.Address()[2:12] return strings.ToLower(fmt.Sprintf("node_%s_contract_%s", shortNodeAddr, shortOCRAddr)), nil } + +func SetupOCRv1Cluster( + l zerolog.Logger, + seth *seth.Client, + workerNodes []*client.ChainlinkK8sClient, +) (common.Address, error) { + err := FundChainlinkNodesFromRootAddress(l, seth, contracts.ChainlinkK8sClientToChainlinkNodeWithKeysAndAddress(workerNodes), big.NewFloat(3)) + if err != nil { + return common.Address{}, err + } + linkContract, err := contracts.DeployLinkTokenContract(l, seth) + if err != nil { + return common.Address{}, err + } + return common.HexToAddress(linkContract.Address()), nil +} + +func SetupOCRv1Feed( + l zerolog.Logger, + seth *seth.Client, + lta common.Address, + msClient *ctfClient.MockserverClient, + bootstrapNode *client.ChainlinkK8sClient, + workerNodes []*client.ChainlinkK8sClient, +) ([]contracts.OffchainAggregator, error) { + ocrInstances, err := DeployOCRv1Contracts(l, seth, 1, lta, contracts.ChainlinkK8sClientToChainlinkNodeWithKeysAndAddress(workerNodes)) + if err != nil { + return nil, err + } + err = CreateOCRJobs(ocrInstances, bootstrapNode, workerNodes, 5, msClient, fmt.Sprint(seth.ChainID)) + if err != nil { + return nil, err + } + return ocrInstances, nil +} + +func SimulateOCRv1EAActivity( + l zerolog.Logger, + eaChangeInterval time.Duration, + ocrInstances []contracts.OffchainAggregator, + workerNodes []*client.ChainlinkK8sClient, + msClient *ctfClient.MockserverClient, +) { + go func() { + for { + time.Sleep(eaChangeInterval) + if err := SetAllAdapterResponsesToTheSameValue(rand.Intn(1000), ocrInstances, workerNodes, msClient); err != nil { + l.Error().Err(err).Msg("failed to update mockserver responses") + } + } + }() +} diff --git a/integration-tests/crib/ocr_test.go b/integration-tests/crib/ocr_test.go new file mode 100644 index 00000000000..b733ec70c05 --- /dev/null +++ b/integration-tests/crib/ocr_test.go @@ -0,0 +1,36 @@ +package crib + +import ( + "testing" + "time" + + "github.com/stretchr/testify/require" + + "github.com/smartcontractkit/chainlink-testing-framework/logging" + "github.com/smartcontractkit/chainlink-testing-framework/utils/testcontext" + + "github.com/smartcontractkit/chainlink/integration-tests/actions" + "github.com/smartcontractkit/chainlink/integration-tests/contracts" +) + +func TestCRIB(t *testing.T) { + l := logging.GetTestLogger(t) + + sethClient, msClient, bootstrapNode, workerNodes, err := ConnectRemote(true) + require.NoError(t, err) + + lta, err := actions.SetupOCRv1Cluster(l, sethClient, workerNodes) + require.NoError(t, err) + ocrInstances, err := actions.SetupOCRv1Feed(l, sethClient, lta, msClient, bootstrapNode, workerNodes) + require.NoError(t, err) + + err = actions.SetAllAdapterResponsesToTheSameValue(10, ocrInstances, workerNodes, msClient) + require.NoError(t, err) + + err = actions.WatchNewOCRRound(l, sethClient, 2, contracts.V1OffChainAgrregatorToOffChainAggregatorWithRounds(ocrInstances), time.Duration(3*time.Minute)) + require.NoError(t, err, "Error watching for new OCR round") + + answer, err := ocrInstances[0].GetLatestAnswer(testcontext.Get(t)) + require.NoError(t, err, "Error getting latest OCR answer") + require.Equal(t, int64(10), answer.Int64(), "Expected latest answer from OCR contract to be 10 but got %d", answer.Int64()) +} diff --git a/integration-tests/load/ocr/helper.go b/integration-tests/load/ocr/helper.go deleted file mode 100644 index f95bf143bce..00000000000 --- a/integration-tests/load/ocr/helper.go +++ /dev/null @@ -1,70 +0,0 @@ -package ocr - -import ( - "fmt" - "math/big" - "math/rand" - "time" - - "github.com/ethereum/go-ethereum/common" - "github.com/rs/zerolog" - - "github.com/smartcontractkit/seth" - - client2 "github.com/smartcontractkit/chainlink-testing-framework/client" - "github.com/smartcontractkit/chainlink/integration-tests/actions" - "github.com/smartcontractkit/chainlink/integration-tests/client" - "github.com/smartcontractkit/chainlink/integration-tests/contracts" -) - -func SetupCluster( - l zerolog.Logger, - seth *seth.Client, - workerNodes []*client.ChainlinkK8sClient, -) (common.Address, error) { - err := actions.FundChainlinkNodesFromRootAddress(l, seth, contracts.ChainlinkK8sClientToChainlinkNodeWithKeysAndAddress(workerNodes), big.NewFloat(3)) - if err != nil { - return common.Address{}, err - } - linkContract, err := contracts.DeployLinkTokenContract(l, seth) - if err != nil { - return common.Address{}, err - } - return common.HexToAddress(linkContract.Address()), nil -} - -func SetupFeed( - l zerolog.Logger, - seth *seth.Client, - lta common.Address, - msClient *client2.MockserverClient, - bootstrapNode *client.ChainlinkK8sClient, - workerNodes []*client.ChainlinkK8sClient, -) ([]contracts.OffchainAggregator, error) { - ocrInstances, err := actions.DeployOCRv1Contracts(l, seth, 1, lta, contracts.ChainlinkK8sClientToChainlinkNodeWithKeysAndAddress(workerNodes)) - if err != nil { - return nil, err - } - err = actions.CreateOCRJobs(ocrInstances, bootstrapNode, workerNodes, 5, msClient, fmt.Sprint(seth.ChainID)) - if err != nil { - return nil, err - } - return ocrInstances, nil -} - -func SimulateEAActivity( - l zerolog.Logger, - eaChangeInterval time.Duration, - ocrInstances []contracts.OffchainAggregator, - workerNodes []*client.ChainlinkK8sClient, - msClient *client2.MockserverClient, -) { - go func() { - for { - time.Sleep(eaChangeInterval) - if err := actions.SetAllAdapterResponsesToTheSameValue(rand.Intn(1000), ocrInstances, workerNodes, msClient); err != nil { - l.Error().Err(err).Msg("failed to update mockserver responses") - } - } - }() -} diff --git a/integration-tests/load/ocr/ocr_test.go b/integration-tests/load/ocr/ocr_test.go index 8d4638f9771..042f6867771 100644 --- a/integration-tests/load/ocr/ocr_test.go +++ b/integration-tests/load/ocr/ocr_test.go @@ -1,6 +1,7 @@ package ocr import ( + "github.com/smartcontractkit/chainlink/integration-tests/actions" "testing" "github.com/stretchr/testify/require" @@ -29,14 +30,14 @@ func TestOCRLoad(t *testing.T) { sethClient, msClient, bootstrapNode, workerNodes, err := crib.ConnectRemote(true) require.NoError(t, err) - lta, err := SetupCluster(l, sethClient, workerNodes) + lta, err := actions.SetupOCRv1Cluster(l, sethClient, workerNodes) require.NoError(t, err) - ocrInstances, err := SetupFeed(l, sethClient, lta, msClient, bootstrapNode, workerNodes) + ocrInstances, err := actions.SetupOCRv1Feed(l, sethClient, lta, msClient, bootstrapNode, workerNodes) require.NoError(t, err) cfg := config.OCR cfgl := config.Logging.Loki - SimulateEAActivity(l, cfg.Load.EAChangeInterval.Duration, ocrInstances, workerNodes, msClient) + actions.SimulateOCRv1EAActivity(l, cfg.Load.EAChangeInterval.Duration, ocrInstances, workerNodes, msClient) p := wasp.NewProfile() p.Add(wasp.NewGenerator(&wasp.Config{ @@ -62,7 +63,7 @@ func TestOCRVolume(t *testing.T) { sethClient, msClient, bootstrapNode, workerNodes, err := crib.ConnectRemote(true) require.NoError(t, err) - lta, err := SetupCluster(l, sethClient, workerNodes) + lta, err := actions.SetupOCRv1Cluster(l, sethClient, workerNodes) require.NoError(t, err) cfg := config.OCR From 2449fbd94775c20e42bedccbe74db1b70ec863d7 Mon Sep 17 00:00:00 2001 From: skudasov Date: Mon, 8 Jul 2024 17:47:43 +0200 Subject: [PATCH 02/12] test default static CRIB config --- integration-tests/testconfig/ocr/ocr.toml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/integration-tests/testconfig/ocr/ocr.toml b/integration-tests/testconfig/ocr/ocr.toml index ac0249334da..d65b31f4e7e 100644 --- a/integration-tests/testconfig/ocr/ocr.toml +++ b/integration-tests/testconfig/ocr/ocr.toml @@ -188,3 +188,9 @@ number_of_contracts=2 time_between_rounds="1m" [TestOCRSoak_RPCDownForHalfCLNodes.Network] selected_networks=["simulated"] + +# CRIB defaults +[CRIB] +namespace = "crib-skudasov" +network_name = "geth" +nodes = 5 \ No newline at end of file From 5afb019f88ef79fde7729eafd5615a6c0092e127 Mon Sep 17 00:00:00 2001 From: skudasov Date: Mon, 8 Jul 2024 18:03:49 +0200 Subject: [PATCH 03/12] try to deploy a new CRIB --- .github/workflows/crib-integration-test.yml | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/.github/workflows/crib-integration-test.yml b/.github/workflows/crib-integration-test.yml index 175fe221069..c796b89a75a 100644 --- a/.github/workflows/crib-integration-test.yml +++ b/.github/workflows/crib-integration-test.yml @@ -31,6 +31,17 @@ jobs: - name: Test shell: bash run: kubectl describe pods -n crib-skudasov + - uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2 + with: + repository: 'smartcontractkit/crib' + ref: 'main' + - name: Generate Short UUID + id: uuid + run: echo "NS_NAME=$(uuidgen | cut -c1-5)" >> $GITHUB_ENV + - name: Create a new CRIB environment + run: |- + devspace use namespace $NS_NAME + devspace deploy --profile local-dev-simulated-core-ocr1 - uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2 - name: Setup go uses: actions/setup-go@0c52d547c9bc32b1aa3301fd7a9cb496313a4491 # v5.0.0 From c65ddda5e872923f7028e9363ab342b87c798bcf Mon Sep 17 00:00:00 2001 From: skudasov Date: Tue, 9 Jul 2024 11:36:59 +0200 Subject: [PATCH 04/12] add nix step --- .github/workflows/crib-integration-test.yml | 20 +++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/.github/workflows/crib-integration-test.yml b/.github/workflows/crib-integration-test.yml index c796b89a75a..758b21fa077 100644 --- a/.github/workflows/crib-integration-test.yml +++ b/.github/workflows/crib-integration-test.yml @@ -14,6 +14,22 @@ jobs: contents: read actions: read steps: + - name: Checkout repository + uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2 + + - name: Setup Nix + GATI environment + uses: smartcontractkit/.github/actions/setup-nix-gati@514fe346780e2eddf7ea8b9f48120c2fba120d94 + with: + aws-role-arn: ${{ secrets.AWS_OIDC_CHAINLINK_AUTO_PR_TOKEN_ISSUER_ROLE_ARN }} + aws-lambda-url: ${{ secrets.AWS_CORE_TOKEN_ISSUER_LAMBDA_URL }} # see https://github.com/smartcontractkit/ infra/blob/a79bcfb48315c4411023c182e98eb80ff9e9cda6/accounts/production/us-west-2/lambda/ github-app-token-issuer-production/teams/releng/config.json#L9 + aws-region: ${{ secrets.AWS_REGION }} + aws-role-duration-seconds: ${{ secrets.AWS_ROLE_DURATION_SECONDS }} + enable-magic-cache: true + + - name: Nix Develop Action + uses: nicknovitski/nix-develop@v1 + with: + arguments: "--accept-flake-config" - name: setup-gap uses: smartcontractkit/.github/actions/setup-gap@d316f66b2990ea4daa479daa3de6fc92b00f863e # setup-gap@0.3.2 with: @@ -28,10 +44,8 @@ jobs: gc-basic-auth: ${{ secrets.GRAFANA_INTERNAL_BASIC_AUTH }} gc-host: ${{ secrets.GRAFANA_INTERNAL_HOST }} gc-org-id: ${{ secrets.GRAFANA_INTERNAL_TENANT_ID }} - - name: Test - shell: bash - run: kubectl describe pods -n crib-skudasov - uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2 + name: Checkout CRIB repository with: repository: 'smartcontractkit/crib' ref: 'main' From e619db1a52daa212537ee8e2dbbb20e9564645a5 Mon Sep 17 00:00:00 2001 From: skudasov Date: Tue, 9 Jul 2024 20:43:23 +0200 Subject: [PATCH 05/12] CRIB chaos test --- .github/workflows/crib-integration-test.yml | 9 +++-- integration-tests/crib/README.md | 12 ++++++ integration-tests/crib/chaos.go | 45 +++++++++++++++++++++ integration-tests/crib/connect.go | 45 +++++++++++++++------ integration-tests/crib/ocr_test.go | 40 +++++++++++++----- integration-tests/load/ocr/ocr_test.go | 4 +- integration-tests/testconfig/ocr/ocr.toml | 8 +--- integration-tests/testconfig/testconfig.go | 7 ---- 8 files changed, 127 insertions(+), 43 deletions(-) create mode 100644 integration-tests/crib/README.md create mode 100644 integration-tests/crib/chaos.go diff --git a/.github/workflows/crib-integration-test.yml b/.github/workflows/crib-integration-test.yml index 758b21fa077..87dbc4bda3e 100644 --- a/.github/workflows/crib-integration-test.yml +++ b/.github/workflows/crib-integration-test.yml @@ -51,19 +51,22 @@ jobs: ref: 'main' - name: Generate Short UUID id: uuid - run: echo "NS_NAME=$(uuidgen | cut -c1-5)" >> $GITHUB_ENV + run: echo "CRIB_NAMESPACE=$(uuidgen | cut -c1-5)" >> $GITHUB_ENV - name: Create a new CRIB environment run: |- - devspace use namespace $NS_NAME + devspace use namespace $CRIB_NAMESPACE devspace deploy --profile local-dev-simulated-core-ocr1 - uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2 - name: Setup go uses: actions/setup-go@0c52d547c9bc32b1aa3301fd7a9cb496313a4491 # v5.0.0 with: go-version-file: "go.mod" - - name: Run tests + - name: Run CRIB integration test working-directory: integration-tests/crib env: K8S_STAGING_INGRESS_SUFFIX: ${{ secrets.K8S_STAGING_INGRESS_SUFFIX }} + CRIB_NAMESPACE: ${{ env.CRIB_NAMESPACE }} + CRIB_NETWORK: geth + CRIB_NODES: 5 run: |- go test -v -run TestCRIB \ No newline at end of file diff --git a/integration-tests/crib/README.md b/integration-tests/crib/README.md new file mode 100644 index 00000000000..7abbd093a0f --- /dev/null +++ b/integration-tests/crib/README.md @@ -0,0 +1,12 @@ +### CRIB Health Check Test + +## Setup CRIB +This is a simple smoke + chaos test for CRIB deployment. +It runs OCRv1 and reboots the environment confirming integration with environment is working and data is properly saved even after reboots. + +```shell +CRIB_NAMESPACE=any-crib-namespace +CRIB_NETWORK=geth +CRIB_NODES=5 +go test -v -run TestCRIB +``` \ No newline at end of file diff --git a/integration-tests/crib/chaos.go b/integration-tests/crib/chaos.go new file mode 100644 index 00000000000..2d8ce83f28f --- /dev/null +++ b/integration-tests/crib/chaos.go @@ -0,0 +1,45 @@ +package crib + +import ( + "github.com/chaos-mesh/chaos-mesh/api/v1alpha1" + "github.com/smartcontractkit/havoc/k8schaos" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "time" +) + +func rebootCLNodes(delay time.Duration, namespace string, labels map[string]string) (*k8schaos.Chaos, error) { + k8sClient, err := k8schaos.NewChaosMeshClient() + if err != nil { + return nil, err + } + return k8schaos.NewChaos(k8schaos.ChaosOpts{ + Description: "Reboot CRIB", + DelayCreate: delay, + Object: &v1alpha1.PodChaos{ + TypeMeta: metav1.TypeMeta{ + Kind: "PodChaos", + APIVersion: "chaos-mesh.org/v1alpha1", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "reboot-crib", + Namespace: namespace, + }, + Spec: v1alpha1.PodChaosSpec{ + ContainerSelector: v1alpha1.ContainerSelector{ + PodSelector: v1alpha1.PodSelector{ + Mode: v1alpha1.AllMode, + Selector: v1alpha1.PodSelectorSpec{ + GenericSelectorSpec: v1alpha1.GenericSelectorSpec{ + LabelSelectors: labels, + Namespaces: []string{namespace}, + }, + }, + }, + }, + Action: v1alpha1.PodKillAction, + }, + }, + Client: k8sClient, + Logger: &k8schaos.Logger, + }) +} diff --git a/integration-tests/crib/connect.go b/integration-tests/crib/connect.go index 446c53d8f38..6e3223adb8b 100644 --- a/integration-tests/crib/connect.go +++ b/integration-tests/crib/connect.go @@ -3,6 +3,7 @@ package crib import ( "fmt" "os" + "strconv" "time" "github.com/pkg/errors" @@ -39,7 +40,7 @@ func setSethConfig(cfg tc.TestConfig, netWSURL string, netHTTPURL string) { // ConnectRemote connects to a local environment, see https://github.com/smartcontractkit/crib/tree/main/core // connects to default CRIB network if simulated = true -func ConnectRemote(simulated bool) ( +func ConnectRemote() ( *seth.Client, *msClient.MockserverClient, *client.ChainlinkK8sClient, @@ -50,22 +51,38 @@ func ConnectRemote(simulated bool) ( if ingressSuffix == "" { return nil, nil, nil, nil, errors.New("K8S_STAGING_INGRESS_SUFFIX must be set to connect to k8s ingresses") } + cribNamespace := os.Getenv("CRIB_NAMESPACE") + if cribNamespace == "" { + return nil, nil, nil, nil, errors.New("CRIB_NAMESPACE must be set to connect") + } + cribNetwork := os.Getenv("CRIB_NETWORK") + if cribNetwork == "" { + return nil, nil, nil, nil, errors.New("CRIB_SIMULATED must be set to connect") + } + cribNodes := os.Getenv("CRIB_NODES") + if cribNodes == "" { + return nil, nil, nil, nil, errors.New("CRIB_NODES must be set to connect") + } + nodes, err := strconv.Atoi(cribNodes) + if err != nil { + return nil, nil, nil, nil, errors.New("CRIB_NODES must be a number, 5-19 nodes") + } config, err := tc.GetConfig([]string{"CRIB"}, tc.OCR) if err != nil { return nil, nil, nil, nil, err } - if config.CRIB.CLNodesNum < 2 { + if nodes < 2 { return nil, nil, nil, nil, fmt.Errorf("not enough chainlink nodes, need at least 2, TOML key: [CRIB.nodes]") } - cfg := config.CRIB - mockserverURL := fmt.Sprintf(mockserverCRIBTemplate, cfg.Namespace, ingressSuffix) + mockserverURL := fmt.Sprintf(mockserverCRIBTemplate, cribNamespace, ingressSuffix) var sethClient *seth.Client - if simulated { - netWSURL := fmt.Sprintf(ingressNetworkWSURLTemplate, cfg.Namespace, ingressSuffix) - netHTTPURL := fmt.Sprintf(ingressNetworkHTTPURLTemplate, cfg.Namespace, ingressSuffix) + switch cribNetwork { + case "geth": + netWSURL := fmt.Sprintf(ingressNetworkWSURLTemplate, cribNamespace, ingressSuffix) + netHTTPURL := fmt.Sprintf(ingressNetworkHTTPURLTemplate, cribNamespace, ingressSuffix) setSethConfig(config, netWSURL, netHTTPURL) net := blockchain.EVMNetwork{ - Name: cfg.NetworkName, + Name: cribNetwork, Simulated: true, SupportsEIP1559: true, ClientImplementation: blockchain.EthereumClientImplementation, @@ -84,27 +101,29 @@ func ConnectRemote(simulated bool) ( if err != nil { return nil, nil, nil, nil, err } + default: + return nil, nil, nil, nil, errors.New("CRIB network is not supported") } // bootstrap node clClients := make([]*client.ChainlinkK8sClient, 0) c, err := client.NewChainlinkK8sClient(&client.ChainlinkConfig{ - URL: fmt.Sprintf("https://%s-node%d%s", cfg.Namespace, 1, ingressSuffix), + URL: fmt.Sprintf("https://%s-node%d%s", cribNamespace, 1, ingressSuffix), Email: client.CLNodeTestEmail, InternalIP: fmt.Sprintf(internalNodeDNSTemplate, 1), Password: client.CLNodeTestPassword, - }, fmt.Sprintf(internalNodeDNSTemplate, 1), cfg.Namespace) + }, fmt.Sprintf(internalNodeDNSTemplate, 1), cribNamespace) if err != nil { return nil, nil, nil, nil, err } clClients = append(clClients, c) // all the other nodes, indices of nodes in CRIB starts with 1 - for i := 2; i <= cfg.CLNodesNum; i++ { + for i := 2; i <= nodes; i++ { cl, err := client.NewChainlinkK8sClient(&client.ChainlinkConfig{ - URL: fmt.Sprintf("https://%s-node%d%s", cfg.Namespace, i, ingressSuffix), + URL: fmt.Sprintf("https://%s-node%d%s", cribNamespace, i, ingressSuffix), Email: client.CLNodeTestEmail, InternalIP: fmt.Sprintf(internalNodeDNSTemplate, i), Password: client.CLNodeTestPassword, - }, fmt.Sprintf(internalNodeDNSTemplate, i), cfg.Namespace) + }, fmt.Sprintf(internalNodeDNSTemplate, i), cribNamespace) if err != nil { return nil, nil, nil, nil, err } diff --git a/integration-tests/crib/ocr_test.go b/integration-tests/crib/ocr_test.go index b733ec70c05..91d800b5111 100644 --- a/integration-tests/crib/ocr_test.go +++ b/integration-tests/crib/ocr_test.go @@ -1,22 +1,22 @@ package crib import ( + "context" + "github.com/smartcontractkit/chainlink/integration-tests/actions" + "github.com/smartcontractkit/chainlink/integration-tests/contracts" + "github.com/smartcontractkit/havoc/k8schaos" + "github.com/stretchr/testify/require" + "os" "testing" "time" - "github.com/stretchr/testify/require" - "github.com/smartcontractkit/chainlink-testing-framework/logging" - "github.com/smartcontractkit/chainlink-testing-framework/utils/testcontext" - - "github.com/smartcontractkit/chainlink/integration-tests/actions" - "github.com/smartcontractkit/chainlink/integration-tests/contracts" ) func TestCRIB(t *testing.T) { l := logging.GetTestLogger(t) - sethClient, msClient, bootstrapNode, workerNodes, err := ConnectRemote(true) + sethClient, msClient, bootstrapNode, workerNodes, err := ConnectRemote() require.NoError(t, err) lta, err := actions.SetupOCRv1Cluster(l, sethClient, workerNodes) @@ -26,11 +26,29 @@ func TestCRIB(t *testing.T) { err = actions.SetAllAdapterResponsesToTheSameValue(10, ocrInstances, workerNodes, msClient) require.NoError(t, err) + actions.SimulateOCRv1EAActivity(l, 3*time.Second, ocrInstances, workerNodes, msClient) - err = actions.WatchNewOCRRound(l, sethClient, 2, contracts.V1OffChainAgrregatorToOffChainAggregatorWithRounds(ocrInstances), time.Duration(3*time.Minute)) + err = actions.WatchNewOCRRound(l, sethClient, 1, contracts.V1OffChainAgrregatorToOffChainAggregatorWithRounds(ocrInstances), 5*time.Minute) require.NoError(t, err, "Error watching for new OCR round") - answer, err := ocrInstances[0].GetLatestAnswer(testcontext.Get(t)) - require.NoError(t, err, "Error getting latest OCR answer") - require.Equal(t, int64(10), answer.Int64(), "Expected latest answer from OCR contract to be 10 but got %d", answer.Int64()) + ch, err := rebootCLNodes( + 1*time.Second, + os.Getenv("CRIB_NAMESPACE"), + map[string]string{ + "app": "app", + }) + ch.Create(context.Background()) + ch.AddListener(k8schaos.NewChaosLogger(l)) + t.Cleanup(func() { + err := ch.Delete(context.Background()) + require.NoError(t, err) + }) + require.Eventually(t, func() bool { + err = actions.WatchNewOCRRound(l, sethClient, 3, contracts.V1OffChainAgrregatorToOffChainAggregatorWithRounds(ocrInstances), 5*time.Minute) + if err != nil { + l.Info().Err(err).Msg("OCR round is not there yet") + return false + } + return true + }, 3*time.Minute, 5*time.Second) } diff --git a/integration-tests/load/ocr/ocr_test.go b/integration-tests/load/ocr/ocr_test.go index 042f6867771..e1efef1eb9e 100644 --- a/integration-tests/load/ocr/ocr_test.go +++ b/integration-tests/load/ocr/ocr_test.go @@ -27,7 +27,7 @@ func TestOCRLoad(t *testing.T) { config, err := tc.GetConfig([]string{"Load"}, tc.OCR) require.NoError(t, err) - sethClient, msClient, bootstrapNode, workerNodes, err := crib.ConnectRemote(true) + sethClient, msClient, bootstrapNode, workerNodes, err := crib.ConnectRemote() require.NoError(t, err) lta, err := actions.SetupOCRv1Cluster(l, sethClient, workerNodes) @@ -60,7 +60,7 @@ func TestOCRVolume(t *testing.T) { config, err := tc.GetConfig([]string{"Volume"}, tc.OCR) require.NoError(t, err) - sethClient, msClient, bootstrapNode, workerNodes, err := crib.ConnectRemote(true) + sethClient, msClient, bootstrapNode, workerNodes, err := crib.ConnectRemote() require.NoError(t, err) lta, err := actions.SetupOCRv1Cluster(l, sethClient, workerNodes) diff --git a/integration-tests/testconfig/ocr/ocr.toml b/integration-tests/testconfig/ocr/ocr.toml index d65b31f4e7e..17ee4d7b687 100644 --- a/integration-tests/testconfig/ocr/ocr.toml +++ b/integration-tests/testconfig/ocr/ocr.toml @@ -187,10 +187,4 @@ test_duration="15m" number_of_contracts=2 time_between_rounds="1m" [TestOCRSoak_RPCDownForHalfCLNodes.Network] -selected_networks=["simulated"] - -# CRIB defaults -[CRIB] -namespace = "crib-skudasov" -network_name = "geth" -nodes = 5 \ No newline at end of file +selected_networks=["simulated"] \ No newline at end of file diff --git a/integration-tests/testconfig/testconfig.go b/integration-tests/testconfig/testconfig.go index 3a73d39dfe9..8f06433fb07 100644 --- a/integration-tests/testconfig/testconfig.go +++ b/integration-tests/testconfig/testconfig.go @@ -82,17 +82,10 @@ type TestConfig struct { VRF *vrf_config.Config `toml:"VRF"` VRFv2 *vrfv2_config.Config `toml:"VRFv2"` VRFv2Plus *vrfv2plus_config.Config `toml:"VRFv2Plus"` - CRIB *CRIB `toml:"CRIB"` ConfigurationNames []string `toml:"-"` } -type CRIB struct { - Namespace string `toml:"namespace"` - NetworkName string `toml:"network_name"` - CLNodesNum int `toml:"nodes"` -} - var embeddedConfigs embed.FS var areConfigsEmbedded bool From 904887a15c8fb859491895420a3e0fd7830d130f Mon Sep 17 00:00:00 2001 From: skudasov Date: Tue, 9 Jul 2024 20:43:23 +0200 Subject: [PATCH 06/12] CRIB chaos test --- .github/workflows/crib-integration-test.yml | 9 +++-- integration-tests/crib/README.md | 12 ++++++ integration-tests/crib/chaos.go | 45 +++++++++++++++++++++ integration-tests/crib/connect.go | 45 +++++++++++++++------ integration-tests/crib/ocr_test.go | 40 +++++++++++++----- integration-tests/load/ocr/ocr_test.go | 4 +- integration-tests/testconfig/ocr/ocr.toml | 8 +--- integration-tests/testconfig/testconfig.go | 7 ---- 8 files changed, 127 insertions(+), 43 deletions(-) create mode 100644 integration-tests/crib/README.md create mode 100644 integration-tests/crib/chaos.go diff --git a/.github/workflows/crib-integration-test.yml b/.github/workflows/crib-integration-test.yml index 758b21fa077..87dbc4bda3e 100644 --- a/.github/workflows/crib-integration-test.yml +++ b/.github/workflows/crib-integration-test.yml @@ -51,19 +51,22 @@ jobs: ref: 'main' - name: Generate Short UUID id: uuid - run: echo "NS_NAME=$(uuidgen | cut -c1-5)" >> $GITHUB_ENV + run: echo "CRIB_NAMESPACE=$(uuidgen | cut -c1-5)" >> $GITHUB_ENV - name: Create a new CRIB environment run: |- - devspace use namespace $NS_NAME + devspace use namespace $CRIB_NAMESPACE devspace deploy --profile local-dev-simulated-core-ocr1 - uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2 - name: Setup go uses: actions/setup-go@0c52d547c9bc32b1aa3301fd7a9cb496313a4491 # v5.0.0 with: go-version-file: "go.mod" - - name: Run tests + - name: Run CRIB integration test working-directory: integration-tests/crib env: K8S_STAGING_INGRESS_SUFFIX: ${{ secrets.K8S_STAGING_INGRESS_SUFFIX }} + CRIB_NAMESPACE: ${{ env.CRIB_NAMESPACE }} + CRIB_NETWORK: geth + CRIB_NODES: 5 run: |- go test -v -run TestCRIB \ No newline at end of file diff --git a/integration-tests/crib/README.md b/integration-tests/crib/README.md new file mode 100644 index 00000000000..7abbd093a0f --- /dev/null +++ b/integration-tests/crib/README.md @@ -0,0 +1,12 @@ +### CRIB Health Check Test + +## Setup CRIB +This is a simple smoke + chaos test for CRIB deployment. +It runs OCRv1 and reboots the environment confirming integration with environment is working and data is properly saved even after reboots. + +```shell +CRIB_NAMESPACE=any-crib-namespace +CRIB_NETWORK=geth +CRIB_NODES=5 +go test -v -run TestCRIB +``` \ No newline at end of file diff --git a/integration-tests/crib/chaos.go b/integration-tests/crib/chaos.go new file mode 100644 index 00000000000..2d8ce83f28f --- /dev/null +++ b/integration-tests/crib/chaos.go @@ -0,0 +1,45 @@ +package crib + +import ( + "github.com/chaos-mesh/chaos-mesh/api/v1alpha1" + "github.com/smartcontractkit/havoc/k8schaos" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "time" +) + +func rebootCLNodes(delay time.Duration, namespace string, labels map[string]string) (*k8schaos.Chaos, error) { + k8sClient, err := k8schaos.NewChaosMeshClient() + if err != nil { + return nil, err + } + return k8schaos.NewChaos(k8schaos.ChaosOpts{ + Description: "Reboot CRIB", + DelayCreate: delay, + Object: &v1alpha1.PodChaos{ + TypeMeta: metav1.TypeMeta{ + Kind: "PodChaos", + APIVersion: "chaos-mesh.org/v1alpha1", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "reboot-crib", + Namespace: namespace, + }, + Spec: v1alpha1.PodChaosSpec{ + ContainerSelector: v1alpha1.ContainerSelector{ + PodSelector: v1alpha1.PodSelector{ + Mode: v1alpha1.AllMode, + Selector: v1alpha1.PodSelectorSpec{ + GenericSelectorSpec: v1alpha1.GenericSelectorSpec{ + LabelSelectors: labels, + Namespaces: []string{namespace}, + }, + }, + }, + }, + Action: v1alpha1.PodKillAction, + }, + }, + Client: k8sClient, + Logger: &k8schaos.Logger, + }) +} diff --git a/integration-tests/crib/connect.go b/integration-tests/crib/connect.go index 446c53d8f38..d12e459925a 100644 --- a/integration-tests/crib/connect.go +++ b/integration-tests/crib/connect.go @@ -3,6 +3,7 @@ package crib import ( "fmt" "os" + "strconv" "time" "github.com/pkg/errors" @@ -39,7 +40,7 @@ func setSethConfig(cfg tc.TestConfig, netWSURL string, netHTTPURL string) { // ConnectRemote connects to a local environment, see https://github.com/smartcontractkit/crib/tree/main/core // connects to default CRIB network if simulated = true -func ConnectRemote(simulated bool) ( +func ConnectRemote() ( *seth.Client, *msClient.MockserverClient, *client.ChainlinkK8sClient, @@ -50,22 +51,38 @@ func ConnectRemote(simulated bool) ( if ingressSuffix == "" { return nil, nil, nil, nil, errors.New("K8S_STAGING_INGRESS_SUFFIX must be set to connect to k8s ingresses") } + cribNamespace := os.Getenv("CRIB_NAMESPACE") + if cribNamespace == "" { + return nil, nil, nil, nil, errors.New("CRIB_NAMESPACE must be set to connect") + } + cribNetwork := os.Getenv("CRIB_NETWORK") + if cribNetwork == "" { + return nil, nil, nil, nil, errors.New("CRIB_NETWORK must be set to connect, only 'geth' is supported for now") + } + cribNodes := os.Getenv("CRIB_NODES") + if cribNodes == "" { + return nil, nil, nil, nil, errors.New("CRIB_NODES must be set to connect") + } + nodes, err := strconv.Atoi(cribNodes) + if err != nil { + return nil, nil, nil, nil, errors.New("CRIB_NODES must be a number, 5-19 nodes") + } config, err := tc.GetConfig([]string{"CRIB"}, tc.OCR) if err != nil { return nil, nil, nil, nil, err } - if config.CRIB.CLNodesNum < 2 { + if nodes < 2 { return nil, nil, nil, nil, fmt.Errorf("not enough chainlink nodes, need at least 2, TOML key: [CRIB.nodes]") } - cfg := config.CRIB - mockserverURL := fmt.Sprintf(mockserverCRIBTemplate, cfg.Namespace, ingressSuffix) + mockserverURL := fmt.Sprintf(mockserverCRIBTemplate, cribNamespace, ingressSuffix) var sethClient *seth.Client - if simulated { - netWSURL := fmt.Sprintf(ingressNetworkWSURLTemplate, cfg.Namespace, ingressSuffix) - netHTTPURL := fmt.Sprintf(ingressNetworkHTTPURLTemplate, cfg.Namespace, ingressSuffix) + switch cribNetwork { + case "geth": + netWSURL := fmt.Sprintf(ingressNetworkWSURLTemplate, cribNamespace, ingressSuffix) + netHTTPURL := fmt.Sprintf(ingressNetworkHTTPURLTemplate, cribNamespace, ingressSuffix) setSethConfig(config, netWSURL, netHTTPURL) net := blockchain.EVMNetwork{ - Name: cfg.NetworkName, + Name: cribNetwork, Simulated: true, SupportsEIP1559: true, ClientImplementation: blockchain.EthereumClientImplementation, @@ -84,27 +101,29 @@ func ConnectRemote(simulated bool) ( if err != nil { return nil, nil, nil, nil, err } + default: + return nil, nil, nil, nil, errors.New("CRIB network is not supported") } // bootstrap node clClients := make([]*client.ChainlinkK8sClient, 0) c, err := client.NewChainlinkK8sClient(&client.ChainlinkConfig{ - URL: fmt.Sprintf("https://%s-node%d%s", cfg.Namespace, 1, ingressSuffix), + URL: fmt.Sprintf("https://%s-node%d%s", cribNamespace, 1, ingressSuffix), Email: client.CLNodeTestEmail, InternalIP: fmt.Sprintf(internalNodeDNSTemplate, 1), Password: client.CLNodeTestPassword, - }, fmt.Sprintf(internalNodeDNSTemplate, 1), cfg.Namespace) + }, fmt.Sprintf(internalNodeDNSTemplate, 1), cribNamespace) if err != nil { return nil, nil, nil, nil, err } clClients = append(clClients, c) // all the other nodes, indices of nodes in CRIB starts with 1 - for i := 2; i <= cfg.CLNodesNum; i++ { + for i := 2; i <= nodes; i++ { cl, err := client.NewChainlinkK8sClient(&client.ChainlinkConfig{ - URL: fmt.Sprintf("https://%s-node%d%s", cfg.Namespace, i, ingressSuffix), + URL: fmt.Sprintf("https://%s-node%d%s", cribNamespace, i, ingressSuffix), Email: client.CLNodeTestEmail, InternalIP: fmt.Sprintf(internalNodeDNSTemplate, i), Password: client.CLNodeTestPassword, - }, fmt.Sprintf(internalNodeDNSTemplate, i), cfg.Namespace) + }, fmt.Sprintf(internalNodeDNSTemplate, i), cribNamespace) if err != nil { return nil, nil, nil, nil, err } diff --git a/integration-tests/crib/ocr_test.go b/integration-tests/crib/ocr_test.go index b733ec70c05..91d800b5111 100644 --- a/integration-tests/crib/ocr_test.go +++ b/integration-tests/crib/ocr_test.go @@ -1,22 +1,22 @@ package crib import ( + "context" + "github.com/smartcontractkit/chainlink/integration-tests/actions" + "github.com/smartcontractkit/chainlink/integration-tests/contracts" + "github.com/smartcontractkit/havoc/k8schaos" + "github.com/stretchr/testify/require" + "os" "testing" "time" - "github.com/stretchr/testify/require" - "github.com/smartcontractkit/chainlink-testing-framework/logging" - "github.com/smartcontractkit/chainlink-testing-framework/utils/testcontext" - - "github.com/smartcontractkit/chainlink/integration-tests/actions" - "github.com/smartcontractkit/chainlink/integration-tests/contracts" ) func TestCRIB(t *testing.T) { l := logging.GetTestLogger(t) - sethClient, msClient, bootstrapNode, workerNodes, err := ConnectRemote(true) + sethClient, msClient, bootstrapNode, workerNodes, err := ConnectRemote() require.NoError(t, err) lta, err := actions.SetupOCRv1Cluster(l, sethClient, workerNodes) @@ -26,11 +26,29 @@ func TestCRIB(t *testing.T) { err = actions.SetAllAdapterResponsesToTheSameValue(10, ocrInstances, workerNodes, msClient) require.NoError(t, err) + actions.SimulateOCRv1EAActivity(l, 3*time.Second, ocrInstances, workerNodes, msClient) - err = actions.WatchNewOCRRound(l, sethClient, 2, contracts.V1OffChainAgrregatorToOffChainAggregatorWithRounds(ocrInstances), time.Duration(3*time.Minute)) + err = actions.WatchNewOCRRound(l, sethClient, 1, contracts.V1OffChainAgrregatorToOffChainAggregatorWithRounds(ocrInstances), 5*time.Minute) require.NoError(t, err, "Error watching for new OCR round") - answer, err := ocrInstances[0].GetLatestAnswer(testcontext.Get(t)) - require.NoError(t, err, "Error getting latest OCR answer") - require.Equal(t, int64(10), answer.Int64(), "Expected latest answer from OCR contract to be 10 but got %d", answer.Int64()) + ch, err := rebootCLNodes( + 1*time.Second, + os.Getenv("CRIB_NAMESPACE"), + map[string]string{ + "app": "app", + }) + ch.Create(context.Background()) + ch.AddListener(k8schaos.NewChaosLogger(l)) + t.Cleanup(func() { + err := ch.Delete(context.Background()) + require.NoError(t, err) + }) + require.Eventually(t, func() bool { + err = actions.WatchNewOCRRound(l, sethClient, 3, contracts.V1OffChainAgrregatorToOffChainAggregatorWithRounds(ocrInstances), 5*time.Minute) + if err != nil { + l.Info().Err(err).Msg("OCR round is not there yet") + return false + } + return true + }, 3*time.Minute, 5*time.Second) } diff --git a/integration-tests/load/ocr/ocr_test.go b/integration-tests/load/ocr/ocr_test.go index 042f6867771..e1efef1eb9e 100644 --- a/integration-tests/load/ocr/ocr_test.go +++ b/integration-tests/load/ocr/ocr_test.go @@ -27,7 +27,7 @@ func TestOCRLoad(t *testing.T) { config, err := tc.GetConfig([]string{"Load"}, tc.OCR) require.NoError(t, err) - sethClient, msClient, bootstrapNode, workerNodes, err := crib.ConnectRemote(true) + sethClient, msClient, bootstrapNode, workerNodes, err := crib.ConnectRemote() require.NoError(t, err) lta, err := actions.SetupOCRv1Cluster(l, sethClient, workerNodes) @@ -60,7 +60,7 @@ func TestOCRVolume(t *testing.T) { config, err := tc.GetConfig([]string{"Volume"}, tc.OCR) require.NoError(t, err) - sethClient, msClient, bootstrapNode, workerNodes, err := crib.ConnectRemote(true) + sethClient, msClient, bootstrapNode, workerNodes, err := crib.ConnectRemote() require.NoError(t, err) lta, err := actions.SetupOCRv1Cluster(l, sethClient, workerNodes) diff --git a/integration-tests/testconfig/ocr/ocr.toml b/integration-tests/testconfig/ocr/ocr.toml index d65b31f4e7e..17ee4d7b687 100644 --- a/integration-tests/testconfig/ocr/ocr.toml +++ b/integration-tests/testconfig/ocr/ocr.toml @@ -187,10 +187,4 @@ test_duration="15m" number_of_contracts=2 time_between_rounds="1m" [TestOCRSoak_RPCDownForHalfCLNodes.Network] -selected_networks=["simulated"] - -# CRIB defaults -[CRIB] -namespace = "crib-skudasov" -network_name = "geth" -nodes = 5 \ No newline at end of file +selected_networks=["simulated"] \ No newline at end of file diff --git a/integration-tests/testconfig/testconfig.go b/integration-tests/testconfig/testconfig.go index 3a73d39dfe9..8f06433fb07 100644 --- a/integration-tests/testconfig/testconfig.go +++ b/integration-tests/testconfig/testconfig.go @@ -82,17 +82,10 @@ type TestConfig struct { VRF *vrf_config.Config `toml:"VRF"` VRFv2 *vrfv2_config.Config `toml:"VRFv2"` VRFv2Plus *vrfv2plus_config.Config `toml:"VRFv2Plus"` - CRIB *CRIB `toml:"CRIB"` ConfigurationNames []string `toml:"-"` } -type CRIB struct { - Namespace string `toml:"namespace"` - NetworkName string `toml:"network_name"` - CLNodesNum int `toml:"nodes"` -} - var embeddedConfigs embed.FS var areConfigsEmbedded bool From a87900cada4f5826203d460f000b67b7df9b2cc4 Mon Sep 17 00:00:00 2001 From: skudasov Date: Thu, 11 Jul 2024 15:55:51 +0200 Subject: [PATCH 07/12] use full reboot --- integration-tests/crib/chaos.go | 5 ++--- integration-tests/crib/ocr_test.go | 6 ++---- 2 files changed, 4 insertions(+), 7 deletions(-) diff --git a/integration-tests/crib/chaos.go b/integration-tests/crib/chaos.go index 2d8ce83f28f..1067f51d8f0 100644 --- a/integration-tests/crib/chaos.go +++ b/integration-tests/crib/chaos.go @@ -7,7 +7,7 @@ import ( "time" ) -func rebootCLNodes(delay time.Duration, namespace string, labels map[string]string) (*k8schaos.Chaos, error) { +func rebootCLNamespace(delay time.Duration, namespace string) (*k8schaos.Chaos, error) { k8sClient, err := k8schaos.NewChaosMeshClient() if err != nil { return nil, err @@ -30,8 +30,7 @@ func rebootCLNodes(delay time.Duration, namespace string, labels map[string]stri Mode: v1alpha1.AllMode, Selector: v1alpha1.PodSelectorSpec{ GenericSelectorSpec: v1alpha1.GenericSelectorSpec{ - LabelSelectors: labels, - Namespaces: []string{namespace}, + Namespaces: []string{namespace}, }, }, }, diff --git a/integration-tests/crib/ocr_test.go b/integration-tests/crib/ocr_test.go index 91d800b5111..f8672de9559 100644 --- a/integration-tests/crib/ocr_test.go +++ b/integration-tests/crib/ocr_test.go @@ -31,12 +31,10 @@ func TestCRIB(t *testing.T) { err = actions.WatchNewOCRRound(l, sethClient, 1, contracts.V1OffChainAgrregatorToOffChainAggregatorWithRounds(ocrInstances), 5*time.Minute) require.NoError(t, err, "Error watching for new OCR round") - ch, err := rebootCLNodes( + ch, err := rebootCLNamespace( 1*time.Second, os.Getenv("CRIB_NAMESPACE"), - map[string]string{ - "app": "app", - }) + ) ch.Create(context.Background()) ch.AddListener(k8schaos.NewChaosLogger(l)) t.Cleanup(func() { From 1d2fdd76f3d6fcc6577f0fb42aca85cb83b42115 Mon Sep 17 00:00:00 2001 From: skudasov Date: Fri, 19 Jul 2024 11:06:40 +0200 Subject: [PATCH 08/12] bump deps --- integration-tests/load/go.mod | 1 + integration-tests/load/go.sum | 2 ++ 2 files changed, 3 insertions(+) diff --git a/integration-tests/load/go.mod b/integration-tests/load/go.mod index 862e1ed9949..ebea9d1294f 100644 --- a/integration-tests/load/go.mod +++ b/integration-tests/load/go.mod @@ -371,6 +371,7 @@ require ( github.com/smartcontractkit/chainlink-solana v1.0.3-0.20240708150922-0546185ead68 // indirect github.com/smartcontractkit/chainlink-starknet/relayer v0.0.1-beta-test.0.20240709043547-03612098f799 // indirect github.com/smartcontractkit/chainlink-testing-framework/grafana v0.0.0-20240405215812-5a72bc9af239 // indirect + github.com/smartcontractkit/havoc/k8schaos v0.0.0-20240409145249-e78d20847e37 // indirect github.com/smartcontractkit/tdh2/go/ocr2/decryptionplugin v0.0.0-20230906073235-9e478e5e19f1 // indirect github.com/smartcontractkit/wsrpc v0.7.3 // indirect github.com/soheilhy/cmux v0.1.5 // indirect diff --git a/integration-tests/load/go.sum b/integration-tests/load/go.sum index e9c8d750aa7..a4324316187 100644 --- a/integration-tests/load/go.sum +++ b/integration-tests/load/go.sum @@ -1524,6 +1524,8 @@ github.com/smartcontractkit/go-plugin v0.0.0-20240208201424-b3b91517de16 h1:TFe+ github.com/smartcontractkit/go-plugin v0.0.0-20240208201424-b3b91517de16/go.mod h1:lBS5MtSSBZk0SHc66KACcjjlU6WzEVP/8pwz68aMkCI= github.com/smartcontractkit/grpc-proxy v0.0.0-20230731113816-f1be6620749f h1:hgJif132UCdjo8u43i7iPN1/MFnu49hv7lFGFftCHKU= github.com/smartcontractkit/grpc-proxy v0.0.0-20230731113816-f1be6620749f/go.mod h1:MvMXoufZAtqExNexqi4cjrNYE9MefKddKylxjS+//n0= +github.com/smartcontractkit/havoc/k8schaos v0.0.0-20240409145249-e78d20847e37 h1:ZEhn2Yo1jY4hqy8nasDL4k4pNtopT3rS3Ap1GDb7ODc= +github.com/smartcontractkit/havoc/k8schaos v0.0.0-20240409145249-e78d20847e37/go.mod h1:/kFr0D7SI/vueXl1N03uzOun4nViGPFRyA5X6eL3jXw= github.com/smartcontractkit/libocr v0.0.0-20240702141926-063ceef8c42e h1:9ypZ/8aW8Vm497i1gXHcT96oNLiu88jbg9QdX+IUE3E= github.com/smartcontractkit/libocr v0.0.0-20240702141926-063ceef8c42e/go.mod h1:fb1ZDVXACvu4frX3APHZaEBp0xi1DIm34DcA0CwTsZM= github.com/smartcontractkit/seth v1.0.12 h1:iVdgMx42XWanPPnBaM5StR4c1XsTr/0/B/kKRZL5BsY= From 4e70d6de98969d4b2872b9684c7c7c4f7c7defca Mon Sep 17 00:00:00 2001 From: skudasov Date: Fri, 19 Jul 2024 11:37:37 +0200 Subject: [PATCH 09/12] comment CI --- .github/workflows/crib-integration-test.yml | 144 ++++++++++---------- integration-tests/load/ocr/ocr_test.go | 3 +- 2 files changed, 75 insertions(+), 72 deletions(-) diff --git a/.github/workflows/crib-integration-test.yml b/.github/workflows/crib-integration-test.yml index 87dbc4bda3e..75b2215d2fc 100644 --- a/.github/workflows/crib-integration-test.yml +++ b/.github/workflows/crib-integration-test.yml @@ -1,72 +1,74 @@ -name: CRIB Integration Tests -on: - push: - workflow_call: -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true -jobs: - test: - runs-on: ubuntu-latest - environment: integration - permissions: - id-token: write - contents: read - actions: read - steps: - - name: Checkout repository - uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2 +# this is disabled because of GAP limitations, should be re-enabled when github-actions-controller will be installed - - name: Setup Nix + GATI environment - uses: smartcontractkit/.github/actions/setup-nix-gati@514fe346780e2eddf7ea8b9f48120c2fba120d94 - with: - aws-role-arn: ${{ secrets.AWS_OIDC_CHAINLINK_AUTO_PR_TOKEN_ISSUER_ROLE_ARN }} - aws-lambda-url: ${{ secrets.AWS_CORE_TOKEN_ISSUER_LAMBDA_URL }} # see https://github.com/smartcontractkit/ infra/blob/a79bcfb48315c4411023c182e98eb80ff9e9cda6/accounts/production/us-west-2/lambda/ github-app-token-issuer-production/teams/releng/config.json#L9 - aws-region: ${{ secrets.AWS_REGION }} - aws-role-duration-seconds: ${{ secrets.AWS_ROLE_DURATION_SECONDS }} - enable-magic-cache: true - - - name: Nix Develop Action - uses: nicknovitski/nix-develop@v1 - with: - arguments: "--accept-flake-config" - - name: setup-gap - uses: smartcontractkit/.github/actions/setup-gap@d316f66b2990ea4daa479daa3de6fc92b00f863e # setup-gap@0.3.2 - with: - aws-role-arn: ${{ secrets.AWS_OIDC_CRIB_ROLE_ARN_STAGE }} - api-gateway-host: ${{ secrets.AWS_API_GW_HOST_K8S_STAGE }} - aws-region: ${{ secrets.AWS_REGION }} - ecr-private-registry: ${{ secrets.AWS_ACCOUNT_ID_PROD }} - k8s-cluster-name: ${{ secrets.AWS_K8S_CLUSTER_NAME_STAGE }} - use-private-ecr-registry: true - use-k8s: true - metrics-job-name: "k8s" - gc-basic-auth: ${{ secrets.GRAFANA_INTERNAL_BASIC_AUTH }} - gc-host: ${{ secrets.GRAFANA_INTERNAL_HOST }} - gc-org-id: ${{ secrets.GRAFANA_INTERNAL_TENANT_ID }} - - uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2 - name: Checkout CRIB repository - with: - repository: 'smartcontractkit/crib' - ref: 'main' - - name: Generate Short UUID - id: uuid - run: echo "CRIB_NAMESPACE=$(uuidgen | cut -c1-5)" >> $GITHUB_ENV - - name: Create a new CRIB environment - run: |- - devspace use namespace $CRIB_NAMESPACE - devspace deploy --profile local-dev-simulated-core-ocr1 - - uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2 - - name: Setup go - uses: actions/setup-go@0c52d547c9bc32b1aa3301fd7a9cb496313a4491 # v5.0.0 - with: - go-version-file: "go.mod" - - name: Run CRIB integration test - working-directory: integration-tests/crib - env: - K8S_STAGING_INGRESS_SUFFIX: ${{ secrets.K8S_STAGING_INGRESS_SUFFIX }} - CRIB_NAMESPACE: ${{ env.CRIB_NAMESPACE }} - CRIB_NETWORK: geth - CRIB_NODES: 5 - run: |- - go test -v -run TestCRIB \ No newline at end of file +#name: CRIB Integration Tests +#on: +# push: +# workflow_call: +#concurrency: +# group: ${{ github.workflow }}-${{ github.ref }} +# cancel-in-progress: true +#jobs: +# test: +# runs-on: ubuntu-latest +# environment: integration +# permissions: +# id-token: write +# contents: read +# actions: read +# steps: +# - name: Checkout repository +# uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2 +# +# - name: Setup Nix + GATI environment +# uses: smartcontractkit/.github/actions/setup-nix-gati@514fe346780e2eddf7ea8b9f48120c2fba120d94 +# with: +# aws-role-arn: ${{ secrets.AWS_OIDC_CHAINLINK_AUTO_PR_TOKEN_ISSUER_ROLE_ARN }} +# aws-lambda-url: ${{ secrets.AWS_CORE_TOKEN_ISSUER_LAMBDA_URL }} # see https://github.com/smartcontractkit/ infra/blob/a79bcfb48315c4411023c182e98eb80ff9e9cda6/accounts/production/us-west-2/lambda/ github-app-token-issuer-production/teams/releng/config.json#L9 +# aws-region: ${{ secrets.AWS_REGION }} +# aws-role-duration-seconds: ${{ secrets.AWS_ROLE_DURATION_SECONDS }} +# enable-magic-cache: true +# +# - name: Nix Develop Action +# uses: nicknovitski/nix-develop@v1 +# with: +# arguments: "--accept-flake-config" +# - name: setup-gap +# uses: smartcontractkit/.github/actions/setup-gap@d316f66b2990ea4daa479daa3de6fc92b00f863e # setup-gap@0.3.2 +# with: +# aws-role-arn: ${{ secrets.AWS_OIDC_CRIB_ROLE_ARN_STAGE }} +# api-gateway-host: ${{ secrets.AWS_API_GW_HOST_K8S_STAGE }} +# aws-region: ${{ secrets.AWS_REGION }} +# ecr-private-registry: ${{ secrets.AWS_ACCOUNT_ID_PROD }} +# k8s-cluster-name: ${{ secrets.AWS_K8S_CLUSTER_NAME_STAGE }} +# use-private-ecr-registry: true +# use-k8s: true +# metrics-job-name: "k8s" +# gc-basic-auth: ${{ secrets.GRAFANA_INTERNAL_BASIC_AUTH }} +# gc-host: ${{ secrets.GRAFANA_INTERNAL_HOST }} +# gc-org-id: ${{ secrets.GRAFANA_INTERNAL_TENANT_ID }} +# - uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2 +# name: Checkout CRIB repository +# with: +# repository: 'smartcontractkit/crib' +# ref: 'main' +# - name: Generate Short UUID +# id: uuid +# run: echo "CRIB_NAMESPACE=$(uuidgen | cut -c1-5)" >> $GITHUB_ENV +# - name: Create a new CRIB environment +# run: |- +# devspace use namespace $CRIB_NAMESPACE +# devspace deploy --profile local-dev-simulated-core-ocr1 +# - uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2 +# - name: Setup go +# uses: actions/setup-go@0c52d547c9bc32b1aa3301fd7a9cb496313a4491 # v5.0.0 +# with: +# go-version-file: "go.mod" +# - name: Run CRIB integration test +# working-directory: integration-tests/crib +# env: +# K8S_STAGING_INGRESS_SUFFIX: ${{ secrets.K8S_STAGING_INGRESS_SUFFIX }} +# CRIB_NAMESPACE: ${{ env.CRIB_NAMESPACE }} +# CRIB_NETWORK: geth +# CRIB_NODES: 5 +# run: |- +# go test -v -run TestCRIB \ No newline at end of file diff --git a/integration-tests/load/ocr/ocr_test.go b/integration-tests/load/ocr/ocr_test.go index e1efef1eb9e..49bce3eca58 100644 --- a/integration-tests/load/ocr/ocr_test.go +++ b/integration-tests/load/ocr/ocr_test.go @@ -1,9 +1,10 @@ package ocr import ( - "github.com/smartcontractkit/chainlink/integration-tests/actions" "testing" + "github.com/smartcontractkit/chainlink/integration-tests/actions" + "github.com/stretchr/testify/require" "github.com/smartcontractkit/wasp" From d54238e6fa4b7521c1ebef9292a7037d2f932b5b Mon Sep 17 00:00:00 2001 From: skudasov Date: Fri, 19 Jul 2024 12:29:30 +0200 Subject: [PATCH 10/12] goimports --- integration-tests/actions/ocr_helpers.go | 7 ++++--- integration-tests/crib/chaos.go | 3 ++- integration-tests/crib/ocr_test.go | 10 ++++++---- 3 files changed, 12 insertions(+), 8 deletions(-) diff --git a/integration-tests/actions/ocr_helpers.go b/integration-tests/actions/ocr_helpers.go index ec3a910cef0..0622167c997 100644 --- a/integration-tests/actions/ocr_helpers.go +++ b/integration-tests/actions/ocr_helpers.go @@ -2,15 +2,16 @@ package actions import ( "fmt" - "github.com/ethereum/go-ethereum/common" - "github.com/rs/zerolog" - "github.com/smartcontractkit/seth" "math/big" "math/rand" "strings" "testing" "time" + "github.com/ethereum/go-ethereum/common" + "github.com/rs/zerolog" + "github.com/smartcontractkit/seth" + "github.com/google/uuid" "github.com/stretchr/testify/require" "golang.org/x/sync/errgroup" diff --git a/integration-tests/crib/chaos.go b/integration-tests/crib/chaos.go index 1067f51d8f0..bbf6ca681cd 100644 --- a/integration-tests/crib/chaos.go +++ b/integration-tests/crib/chaos.go @@ -1,10 +1,11 @@ package crib import ( + "time" + "github.com/chaos-mesh/chaos-mesh/api/v1alpha1" "github.com/smartcontractkit/havoc/k8schaos" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "time" ) func rebootCLNamespace(delay time.Duration, namespace string) (*k8schaos.Chaos, error) { diff --git a/integration-tests/crib/ocr_test.go b/integration-tests/crib/ocr_test.go index f8672de9559..b84af02a196 100644 --- a/integration-tests/crib/ocr_test.go +++ b/integration-tests/crib/ocr_test.go @@ -2,14 +2,16 @@ package crib import ( "context" - "github.com/smartcontractkit/chainlink/integration-tests/actions" - "github.com/smartcontractkit/chainlink/integration-tests/contracts" - "github.com/smartcontractkit/havoc/k8schaos" - "github.com/stretchr/testify/require" "os" "testing" "time" + "github.com/smartcontractkit/havoc/k8schaos" + "github.com/stretchr/testify/require" + + "github.com/smartcontractkit/chainlink/integration-tests/actions" + "github.com/smartcontractkit/chainlink/integration-tests/contracts" + "github.com/smartcontractkit/chainlink-testing-framework/logging" ) From 9479cd2bba8c756dda9f4b011fb7579a7e6affc4 Mon Sep 17 00:00:00 2001 From: skudasov Date: Fri, 19 Jul 2024 12:34:07 +0200 Subject: [PATCH 11/12] readme --- integration-tests/crib/README.md | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/integration-tests/crib/README.md b/integration-tests/crib/README.md index 7abbd093a0f..8ab317aaa79 100644 --- a/integration-tests/crib/README.md +++ b/integration-tests/crib/README.md @@ -3,10 +3,17 @@ ## Setup CRIB This is a simple smoke + chaos test for CRIB deployment. It runs OCRv1 and reboots the environment confirming integration with environment is working and data is properly saved even after reboots. +Go to the [CRIB](https://github.com/smartcontractkit/crib) repository and spin up a cluster. +```toml +./scripts/cribbit.sh crib-oh-my-crib +devspace deploy --debug --profile local-dev-simulated-core-ocr1 +``` + +## Run the tests ```shell -CRIB_NAMESPACE=any-crib-namespace -CRIB_NETWORK=geth -CRIB_NODES=5 +CRIB_NAMESPACE=crib-oh-my-crib +CRIB_NETWORK=geth # only "geth" is supported for now +CRIB_NODES=5 # min 5 nodes go test -v -run TestCRIB ``` \ No newline at end of file From 2b82ee62ca69270b68d6139ba8aa5102b62282aa Mon Sep 17 00:00:00 2001 From: skudasov Date: Fri, 19 Jul 2024 12:37:27 +0200 Subject: [PATCH 12/12] simplify nodes check --- integration-tests/crib/README.md | 2 +- integration-tests/crib/connect.go | 3 --- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/integration-tests/crib/README.md b/integration-tests/crib/README.md index 8ab317aaa79..ecf393f780d 100644 --- a/integration-tests/crib/README.md +++ b/integration-tests/crib/README.md @@ -5,7 +5,7 @@ This is a simple smoke + chaos test for CRIB deployment. It runs OCRv1 and reboots the environment confirming integration with environment is working and data is properly saved even after reboots. Go to the [CRIB](https://github.com/smartcontractkit/crib) repository and spin up a cluster. -```toml +```shell ./scripts/cribbit.sh crib-oh-my-crib devspace deploy --debug --profile local-dev-simulated-core-ocr1 ``` diff --git a/integration-tests/crib/connect.go b/integration-tests/crib/connect.go index d12e459925a..91d7d8ee1a0 100644 --- a/integration-tests/crib/connect.go +++ b/integration-tests/crib/connect.go @@ -60,9 +60,6 @@ func ConnectRemote() ( return nil, nil, nil, nil, errors.New("CRIB_NETWORK must be set to connect, only 'geth' is supported for now") } cribNodes := os.Getenv("CRIB_NODES") - if cribNodes == "" { - return nil, nil, nil, nil, errors.New("CRIB_NODES must be set to connect") - } nodes, err := strconv.Atoi(cribNodes) if err != nil { return nil, nil, nil, nil, errors.New("CRIB_NODES must be a number, 5-19 nodes")