Skip to content

Commit

Permalink
Refactor
Browse files Browse the repository at this point in the history
  • Loading branch information
@lukaszcl
lukaszcl committed Jun 19, 2024
1 parent ec8c11b commit 576af9d
Show file tree
Hide file tree
Showing 4 changed files with 217 additions and 130 deletions.
167 changes: 147 additions & 20 deletions integration-tests/soak/ocr_test.go
View file
Original file line number Diff line number Diff line change
@@ -1,15 +1,23 @@
package soak

import (
"fmt"
"testing"

"github.com/stretchr/testify/require"

"github.com/smartcontractkit/chainlink-testing-framework/logging"

"time"

"github.com/chaos-mesh/chaos-mesh/api/v1alpha1"
"github.com/google/uuid"
"github.com/smartcontractkit/chainlink-testing-framework/utils/ptr"
actions_seth "github.com/smartcontractkit/chainlink/integration-tests/actions/seth"
tc "github.com/smartcontractkit/chainlink/integration-tests/testconfig"
"github.com/smartcontractkit/chainlink/integration-tests/testsetups"
"github.com/smartcontractkit/havoc/k8schaos"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)

func TestOCRSoak(t *testing.T) {
Expand All @@ -22,64 +30,183 @@ func TestOCRSoak(t *testing.T) {
// fmt.Println("---------------------")
config, err := tc.GetConfig("Soak", tc.OCR)
require.NoError(t, err, "Error getting config")
runOCRSoakTest(t, config, customNetworkTOML)
ocrSoakTest, err := testsetups.NewOCRSoakTest(t, &config, false)
require.NoError(t, err, "Error creating OCR soak test")
runOCRSoakTest(t, ocrSoakTest, config, customNetworkTOML)
}

func TestOCRSoak_GethReorgBelowFinality_FinalityTagDisabled(t *testing.T) {
config, err := tc.GetConfig(t.Name(), tc.OCR)
require.NoError(t, err, "Error getting config")
runOCRSoakTest(t, config, "")
ocrSoakTest, err := testsetups.NewOCRSoakTest(t, &config, false)
require.NoError(t, err, "Error creating OCR soak test")
runOCRSoakTest(t, ocrSoakTest, config, "")
}

func TestOCRSoak_GethReorgBelowFinality_FinalityTagEnabled(t *testing.T) {
config, err := tc.GetConfig(t.Name(), tc.OCR)
require.NoError(t, err, "Error getting config")
runOCRSoakTest(t, config, "")
ocrSoakTest, err := testsetups.NewOCRSoakTest(t, &config, false)
require.NoError(t, err, "Error creating OCR soak test")
runOCRSoakTest(t, ocrSoakTest, config, "")
}

func TestOCRSoak_GasSpike(t *testing.T) {
config, err := tc.GetConfig(t.Name(), tc.OCR)
require.NoError(t, err, "Error getting config")
runOCRSoakTest(t, config, "")
ocrSoakTest, err := testsetups.NewOCRSoakTest(t, &config, false)
require.NoError(t, err, "Error creating OCR soak test")
runOCRSoakTest(t, ocrSoakTest, config, "")
}

// TestOCRSoak_ChangeBlockGasLimit changes next block gas limit and sets it to percentage of last gasUsed in previous block creating congestion
func TestOCRSoak_ChangeBlockGasLimit(t *testing.T) {
config, err := tc.GetConfig(t.Name(), tc.OCR)
require.NoError(t, err, "Error getting config")
runOCRSoakTest(t, config, "")
ocrSoakTest, err := testsetups.NewOCRSoakTest(t, &config, false)
require.NoError(t, err, "Error creating OCR soak test")
runOCRSoakTest(t, ocrSoakTest, config, "")
}

func TestOCRSoak_RPCDown(t *testing.T) {
// TestOCRSoak_RPCDownForAllCLNodes simulates a network chaos by bringing down network to RPC node for all Chainlink Nodes
func TestOCRSoak_RPCDownForAllCLNodes(t *testing.T) {
config, err := tc.GetConfig(t.Name(), tc.OCR)
require.NoError(t, err, "Error getting config")
runOCRSoakTest(t, config, "")
require.True(t, config.Network.IsSimulatedGethSelected(), "This test requires simulated geth")

namespace := fmt.Sprintf("%s-%s", "soak-ocr-simulated-geth", uuid.NewString()[0:5])
chaos, err := gethNetworkDownChaos(GethNetworkDownChaosOpts{
DelayCreate: time.Minute * 2,
Duration: time.Minute * 5,
Name: "ocr-soak-test-geth-network-down-all-nodes",
Description: "Geth Network Down For All Chainlink Nodes",
Namespace: namespace,
TargetSelector: v1alpha1.PodSelector{
Selector: v1alpha1.PodSelectorSpec{
GenericSelectorSpec: v1alpha1.GenericSelectorSpec{
Namespaces: []string{namespace},
LabelSelectors: map[string]string{"app": "chainlink-0"},
},
},
Mode: v1alpha1.AllMode,
},
})
require.NoError(t, err, "Error creating chaos")
ocrSoakTest, err := testsetups.NewOCRSoakTest(t, &config, false,
testsetups.WithNamespace(namespace),
testsetups.WithChaos([]*k8schaos.Chaos{chaos}),
)
require.NoError(t, err, "Error creating OCR soak test")
runOCRSoakTest(t, ocrSoakTest, config, "")
}

func runOCRSoakTest(t *testing.T, config tc.TestConfig, customNetworkTOML string) {
// TestOCRSoak_RPCDownForAllCLNodes simulates a network chaos by bringing down network to RPC node for 50% of Chainlink Nodes
func TestOCRSoak_RPCDownForHalfCLNodes(t *testing.T) {
config, err := tc.GetConfig(t.Name(), tc.OCR)
require.NoError(t, err, "Error getting config")
require.True(t, config.Network.IsSimulatedGethSelected(), "This test requires simulated geth")

namespace := fmt.Sprintf("%s-%s", "soak-ocr-simulated-geth", uuid.NewString()[0:5])
chaos, err := gethNetworkDownChaos(GethNetworkDownChaosOpts{
DelayCreate: time.Minute * 2,
Duration: time.Minute * 5,
Name: "ocr-soak-test-geth-network-down-half-nodes",
Description: "Geth Network Down For 50 Percent Of Chainlink Nodes",
Namespace: namespace,
TargetSelector: v1alpha1.PodSelector{
Selector: v1alpha1.PodSelectorSpec{
GenericSelectorSpec: v1alpha1.GenericSelectorSpec{
Namespaces: []string{namespace},
LabelSelectors: map[string]string{"app": "chainlink-0"},
},
},
Mode: v1alpha1.FixedPercentMode,
Value: "50",
},
})
require.NoError(t, err, "Error creating chaos")
ocrSoakTest, err := testsetups.NewOCRSoakTest(t, &config, false,
testsetups.WithNamespace(namespace),
testsetups.WithChaos([]*k8schaos.Chaos{chaos}),
)
require.NoError(t, err, "Error creating OCR soak test")
runOCRSoakTest(t, ocrSoakTest, config, "")
}

func runOCRSoakTest(t *testing.T, test *testsetups.OCRSoakTest, config tc.TestConfig, customNetworkTOML string) {
l := logging.GetTestLogger(t)

l.Info().Str("test", t.Name()).Msg("Starting OCR soak test")

ocrSoakTest, err := testsetups.NewOCRSoakTest(t, &config, false)
require.NoError(t, err, "Error creating soak test")
if !ocrSoakTest.Interrupted() {
ocrSoakTest.DeployEnvironment(customNetworkTOML, &config)
if !test.Interrupted() {
test.DeployEnvironment(customNetworkTOML, &config)
}
if ocrSoakTest.Environment().WillUseRemoteRunner() {
if test.Environment().WillUseRemoteRunner() {
return
}
t.Cleanup(func() {
if err := actions_seth.TeardownRemoteSuite(ocrSoakTest.TearDownVals(t)); err != nil {
if err := actions_seth.TeardownRemoteSuite(test.TearDownVals(t)); err != nil {
l.Error().Err(err).Msg("Error tearing down environment")
}
})
if ocrSoakTest.Interrupted() {
err = ocrSoakTest.LoadState()
if test.Interrupted() {
err := test.LoadState()
require.NoError(t, err, "Error loading state")
ocrSoakTest.Resume()
test.Resume()
} else {
ocrSoakTest.Setup(&config)
ocrSoakTest.Run()
test.Setup(&config)
test.Run()
}
}

type GethNetworkDownChaosOpts struct {
Name string
Namespace string
Description string
TargetSelector v1alpha1.PodSelector
DelayCreate time.Duration
Duration time.Duration
}

func gethNetworkDownChaos(opts GethNetworkDownChaosOpts) (*k8schaos.Chaos, error) {
k8sClient, err := k8schaos.NewChaosMeshClient()
if err != nil {
return nil, err
}
return k8schaos.NewChaos(k8schaos.ChaosOpts{
Description: opts.Description,
DelayCreate: opts.DelayCreate,
Object: &v1alpha1.NetworkChaos{
TypeMeta: metav1.TypeMeta{
Kind: "NetworkChaos",
APIVersion: "chaos-mesh.org/v1alpha1",
},
ObjectMeta: metav1.ObjectMeta{
Name: opts.Name,
Namespace: opts.Namespace,
},
Spec: v1alpha1.NetworkChaosSpec{
Action: v1alpha1.LossAction,
PodSelector: v1alpha1.PodSelector{
Mode: v1alpha1.AllMode,
Selector: v1alpha1.PodSelectorSpec{
GenericSelectorSpec: v1alpha1.GenericSelectorSpec{
Namespaces: []string{opts.Namespace},
LabelSelectors: map[string]string{"app": "geth"},
},
},
},
Duration: ptr.Ptr(opts.Duration.String()),
Direction: v1alpha1.Both,
Target: &opts.TargetSelector,
TcParameter: v1alpha1.TcParameter{
Loss: &v1alpha1.LossSpec{
Loss: "100",
},
},
},
},
Client: k8sClient,
Logger: &k8schaos.Logger,
})

}
22 changes: 15 additions & 7 deletions integration-tests/testconfig/ocr/ocr.toml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -173,16 +173,24 @@ next_gas_limit_percentage = 0.5
delay_create = "1m"
duration = "3m"

[TestOCRSoak_RPCDown.Common]
[TestOCRSoak_RPCDownForAllCLNodes.Common]
chainlink_node_funding = 100
[TestOCRSoak_RPCDown.OCR.Common]
[TestOCRSoak_RPCDownForAllCLNodes.OCR.Common]
test_duration="15m"
[TestOCRSoak_RPCDown.OCR.Soak]
[TestOCRSoak_RPCDownForAllCLNodes.OCR.Soak]
ocr_version="1"
number_of_contracts=2
time_between_rounds="1m"
[TestOCRSoak_RPCDown.Network]
[TestOCRSoak_RPCDownForAllCLNodes.Network]
selected_networks=["simulated"]
[TestOCRSoak_RPCDown.ChaosSimulation]
geth_network_delay_create = "2m"
geth_network_down_duration = "3m"

[TestOCRSoak_RPCDownForHalfCLNodes.Common]
chainlink_node_funding = 100
[TestOCRSoak_RPCDownForHalfCLNodes.OCR.Common]
test_duration="15m"
[TestOCRSoak_RPCDownForHalfCLNodes.OCR.Soak]
ocr_version="1"
number_of_contracts=2
time_between_rounds="1m"
[TestOCRSoak_RPCDownForHalfCLNodes.Network]
selected_networks=["simulated"]
31 changes: 10 additions & 21 deletions integration-tests/testconfig/testconfig.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,6 @@ import (

"github.com/smartcontractkit/seth"

"github.com/smartcontractkit/chainlink-testing-framework/blockchain"
ctf_config "github.com/smartcontractkit/chainlink-testing-framework/config"
k8s_config "github.com/smartcontractkit/chainlink-testing-framework/k8s/config"
"github.com/smartcontractkit/chainlink-testing-framework/logging"
Expand Down Expand Up @@ -74,17 +73,16 @@ type Ocr2TestConfig interface {
type TestConfig struct {
ctf_config.TestConfig

Common *Common `toml:"Common"`
ChaosSimulation *ChaosSimulation `toml:"ChaosSimulation"`
Automation *a_config.Config `toml:"Automation"`
Functions *f_config.Config `toml:"Functions"`
Keeper *keeper_config.Config `toml:"Keeper"`
LogPoller *lp_config.Config `toml:"LogPoller"`
OCR *ocr_config.Config `toml:"OCR"`
OCR2 *ocr2_config.Config `toml:"OCR2"`
VRF *vrf_config.Config `toml:"VRF"`
VRFv2 *vrfv2_config.Config `toml:"VRFv2"`
VRFv2Plus *vrfv2plus_config.Config `toml:"VRFv2Plus"`
Common *Common `toml:"Common"`
Automation *a_config.Config `toml:"Automation"`
Functions *f_config.Config `toml:"Functions"`
Keeper *keeper_config.Config `toml:"Keeper"`
LogPoller *lp_config.Config `toml:"LogPoller"`
OCR *ocr_config.Config `toml:"OCR"`
OCR2 *ocr2_config.Config `toml:"OCR2"`
VRF *vrf_config.Config `toml:"VRF"`
VRFv2 *vrfv2_config.Config `toml:"VRFv2"`
VRFv2Plus *vrfv2plus_config.Config `toml:"VRFv2Plus"`

ConfigurationName string `toml:"-"`
}
Expand Down Expand Up @@ -215,15 +213,6 @@ func (c *TestConfig) AsBase64() (string, error) {
return base64.StdEncoding.EncodeToString(content), nil
}

type ChaosSimulation struct {
GethNetworkDownDuration blockchain.StrDuration `toml:"geth_network_down_duration"` // GethNetworkDownDuration is the duration for which the Geth network will be down
GethNetworkDownDelayCreate blockchain.StrDuration `toml:"geth_network_delay_create"` // GethNetworkDownDelayCreate is the delay before the Geth network is down
}

func (c *ChaosSimulation) Validate() error {
return nil
}

type Common struct {
ChainlinkNodeFunding *float64 `toml:"chainlink_node_funding"`
}
Expand Down
Loading

0 comments on commit 576af9d

Please sign in to comment.