Skip to content

Commit

Permalink
update automation tests
Browse files Browse the repository at this point in the history 
load - separate pyroscope profiling per node
benchmark - add pyroscope profiling per node
benchmark - improve slack notification ux
  • Loading branch information
@anirudhwarrier
anirudhwarrier committed Oct 3, 2024
1 parent 659b75a commit 0740012
Show file tree
Hide file tree
Showing 5 changed files with 64 additions and 21 deletions.
18 changes: 14 additions & 4 deletions integration-tests/benchmark/automation_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ package benchmark
import (
"fmt"
"math/big"
"strconv"
"strings"
"testing"
"time"
Expand Down Expand Up @@ -334,6 +335,10 @@ func SetupAutomationBenchmarkEnv(t *testing.T, keeperTestConfig types.Automation
return testEnvironment, testNetwork
}

if *keeperTestConfig.GetPyroscopeConfig().Enabled {
keeperTestConfig.GetPyroscopeConfig().Environment = &testEnvironment.Cfg.Namespace
}

// separate RPC urls per CL node
internalWsURLs := make([]string, 0)
internalHttpURLs := make([]string, 0)
Expand All @@ -360,22 +365,27 @@ func SetupAutomationBenchmarkEnv(t *testing.T, keeperTestConfig types.Automation
l.Debug().Strs("internalWsURLs", internalWsURLs).Strs("internalHttpURLs", internalHttpURLs).Msg("internalURLs")

for i := 0; i < numberOfNodes; i++ {
config := keeperTestConfig
if *config.GetPyroscopeConfig().Enabled {
name := testEnvironment.Cfg.Namespace + "-" + strconv.Itoa(i)
config.GetPyroscopeConfig().Environment = &name
}
testNetwork.HTTPURLs = []string{internalHttpURLs[i]}
testNetwork.URLs = []string{internalWsURLs[i]}

var overrideFn = func(_ interface{}, target interface{}) {
ctfconfig.MustConfigOverrideChainlinkVersion(keeperTestConfig.GetChainlinkImageConfig(), target)
ctfconfig.MightConfigOverridePyroscopeKey(keeperTestConfig.GetPyroscopeConfig(), target)
ctfconfig.MustConfigOverrideChainlinkVersion(config.GetChainlinkImageConfig(), target)
ctfconfig.MightConfigOverridePyroscopeKey(config.GetPyroscopeConfig(), target)
}

tomlConfig, err := actions.BuildTOMLNodeConfigForK8s(keeperTestConfig, testNetwork)
tomlConfig, err := actions.BuildTOMLNodeConfigForK8s(config, testNetwork)
require.NoError(t, err, "Error building TOML config")

cd := chainlink.NewWithOverride(i, map[string]any{
"toml": tomlConfig,
"chainlink": chainlinkResources,
"db": dbResources,
}, keeperTestConfig.GetChainlinkImageConfig(), overrideFn)
}, config.GetChainlinkImageConfig(), overrideFn)

testEnvironment.AddHelm(cd)
}
Expand Down
19 changes: 12 additions & 7 deletions integration-tests/load/automationv2_1/automationv2_1_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -277,24 +277,29 @@ Load Config:
secretsTOML = ""
}

numberOfUpkeeps := *loadedTestConfig.Automation.General.NumberOfNodes
numberOfNodes := *loadedTestConfig.Automation.General.NumberOfNodes

for i := 0; i < numberOfUpkeeps+1; i++ { // +1 for the OCR boot node
for i := 0; i < numberOfNodes+1; i++ { // +1 for the OCR boot node
config := loadedTestConfig
if *config.Pyroscope.Enabled {
name := testEnvironment.Cfg.Namespace + "-" + strconv.Itoa(i)
config.Pyroscope.Environment = &name
}
var overrideFn = func(_ interface{}, target interface{}) {
ctfconfig.MustConfigOverrideChainlinkVersion(loadedTestConfig.GetChainlinkImageConfig(), target)
ctfconfig.MightConfigOverridePyroscopeKey(loadedTestConfig.GetPyroscopeConfig(), target)
ctfconfig.MustConfigOverrideChainlinkVersion(config.GetChainlinkImageConfig(), target)
ctfconfig.MightConfigOverridePyroscopeKey(config.GetPyroscopeConfig(), target)
}

tomlConfig, err := actions.BuildTOMLNodeConfigForK8s(&loadedTestConfig, testNetwork)
tomlConfig, err := actions.BuildTOMLNodeConfigForK8s(&config, testNetwork)
require.NoError(t, err, "Error building TOML config")

cd := chainlink.NewWithOverride(i, map[string]any{
"toml": tomlConfig,
"chainlink": nodeSpec,
"db": dbSpec,
"prometheus": *loadedTestConfig.Automation.General.UsePrometheus,
"prometheus": *config.Automation.General.UsePrometheus,
"secretsToml": secretsTOML,
}, loadedTestConfig.ChainlinkImage, overrideFn)
}, config.ChainlinkImage, overrideFn)
testEnvironment.AddHelm(cd)
}

Expand Down
2 changes: 1 addition & 1 deletion integration-tests/load/automationv2_1/helpers.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -52,7 +52,7 @@ func sendSlackNotification(header string, l zerolog.Logger, config *tc.TestConfi
notificationBlocks = append(notificationBlocks, slack.NewDividerBlock())
if *config.Pyroscope.Enabled {
pyroscopeServer := *config.Pyroscope.ServerUrl
pyroscopeEnvironment := *config.Pyroscope.Environment
pyroscopeEnvironment := *config.Pyroscope.Environment + "-1"

formattedPyroscopeUrl := fmt.Sprintf("%s/?query=chainlink-node.cpu{Environment=\"%s\"}&from=%s&to=%s", pyroscopeServer, pyroscopeEnvironment, startingTime, endingTime)
l.Info().Str("Pyroscope", formattedPyroscopeUrl).Msg("Dashboard URL")
Expand Down
24 changes: 20 additions & 4 deletions integration-tests/testreporters/keeper_benchmark.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,8 @@ import (
"sync"
"testing"

"github.com/smartcontractkit/chainlink/integration-tests/testconfig"

"github.com/rs/zerolog/log"
"github.com/slack-go/slack"

Expand All @@ -26,6 +28,7 @@ type KeeperBenchmarkTestReporter struct {
NumRevertedUpkeeps int64
NumStaleUpkeepReports int64
Summary KeeperBenchmarkTestSummary `json:"summary"`
SlackTs string

namespace string
keeperReportFile string
Expand All @@ -49,8 +52,9 @@ type KeeperBenchmarkTestLoad struct {
}

type KeeperBenchmarkTestConfig struct {
Chainlink map[string]map[string]string `json:"chainlink"`
Geth map[string]map[string]string `json:"geth"`
Chainlink map[string]map[string]string `json:"chainlink"`
Geth map[string]map[string]string `json:"geth"`
TestConfig testconfig.TestConfig
}

type KeeperBenchmarkTestMetrics struct {
Expand Down Expand Up @@ -259,7 +263,7 @@ func (k *KeeperBenchmarkTestReporter) SendSlackNotification(t *testing.T, slackC
messageBlocks := testreporters.CommonSlackNotificationBlocks(
headerText, k.namespace, k.keeperReportFile,
)
ts, err := testreporters.SendSlackMessage(slackClient, slack.MsgOptionBlocks(messageBlocks...))
ts, err := testreporters.SendSlackMessage(slackClient, slack.MsgOptionBlocks(messageBlocks...), slack.MsgOptionTS(k.SlackTs))
if err != nil {
return err
}
Expand All @@ -277,12 +281,24 @@ func (k *KeeperBenchmarkTestReporter) SendSlackNotification(t *testing.T, slackC
formattedDashboardUrl := fmt.Sprintf("%s%s?from=%d&to=%d&var-namespace=%s&var-cl_node=chainlink-0-0", grafanaUrl, dashboardUrl, k.Summary.StartTime, k.Summary.EndTime, k.namespace)
log.Info().Str("Dashboard", formattedDashboardUrl).Msg("Dashboard URL")

notificationText := fmt.Sprintf("Automation Benchmark Test Summary %s.\n<%s|Test Dashboard> ", k.namespace, formattedDashboardUrl)

config := k.Summary.Config.TestConfig
if *config.GetPyroscopeConfig().Enabled {
pyroscopeServer := *config.GetPyroscopeConfig().ServerUrl
pyroscopeEnvironment := *config.GetPyroscopeConfig().Environment + "-1"

formattedPyroscopeUrl := fmt.Sprintf("%s/?query=chainlink-node.cpu{Environment=\"%s\"}&from=%d&to=%d", pyroscopeServer, pyroscopeEnvironment, k.Summary.StartTime, k.Summary.EndTime)
log.Info().Str("Pyroscope", formattedPyroscopeUrl).Msg("Dashboard URL")
notificationText += fmt.Sprintf("\n<%s|Pyroscope>", formattedPyroscopeUrl)
}

if err := testreporters.UploadSlackFile(slackClient, slack.FileUploadParameters{
Title: fmt.Sprintf("Automation Benchmark Test Summary %s", k.namespace),
Filetype: "json",
Filename: fmt.Sprintf("automation_benchmark_summary_%s.json", k.namespace),
File: k.keeperSummaryFile,
InitialComment: fmt.Sprintf("Automation Benchmark Test Summary %s.\n<%s|Test Dashboard> ", k.namespace, formattedDashboardUrl),
InitialComment: notificationText,
Channels: []string{testreporters.SlackChannel},
ThreadTimestamp: ts,
}); err != nil {
Expand Down
22 changes: 17 additions & 5 deletions integration-tests/testsetups/automation_benchmark.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -110,6 +110,7 @@ func NewKeeperBenchmarkTest(t *testing.T, inputs KeeperBenchmarkTestInputs) *Kee
func (k *KeeperBenchmarkTest) Setup(env *environment.Environment, config testconfig.TestConfig) {
startTime := time.Now()
k.TestReporter.Summary.StartTime = startTime.UnixMilli()
k.TestReporter.Summary.Config.TestConfig = config
k.ensureInputValues()
k.env = env
k.namespace = k.env.Cfg.Namespace
Expand Down Expand Up @@ -180,10 +181,11 @@ func (k *KeeperBenchmarkTest) Setup(env *environment.Environment, config testcon
}

k.log.Info().Str("Setup Time", time.Since(startTime).String()).Msg("Finished Keeper Benchmark Test Setup")
err = k.SendSlackNotification(nil, &config)
ts, err := k.SendSlackNotification(nil, &config)
if err != nil {
k.log.Warn().Msg("Sending test start slack notification failed")
}
k.TestReporter.SlackTs = ts
}

// Run runs the keeper benchmark test
Expand Down Expand Up @@ -618,19 +620,19 @@ func (k *KeeperBenchmarkTest) ensureInputValues() {
}
}

func (k *KeeperBenchmarkTest) SendSlackNotification(slackClient *slack.Client, config tt.AutomationBenchmarkTestConfig) error {
func (k *KeeperBenchmarkTest) SendSlackNotification(slackClient *slack.Client, config tt.AutomationBenchmarkTestConfig) (string, error) {
if slackClient == nil {
slackClient = slack.New(reportModel.SlackAPIKey)
}

grafanaUrl, err := config.GetGrafanaBaseURL()
if err != nil {
return err
return "", err
}

dashboardUrl, err := config.GetGrafanaDashboardURL()
if err != nil {
return err
return "", err
}

headerText := ":white_check_mark: Automation Benchmark Test STARTED :white_check_mark:"
Expand All @@ -646,10 +648,20 @@ func (k *KeeperBenchmarkTest) SendSlackNotification(slackClient *slack.Client, c
notificationBlocks = append(notificationBlocks, slack.NewSectionBlock(slack.NewTextBlockObject("mrkdwn",
fmt.Sprintf("<%s|Test Dashboard> \nNotifying <@%s>",
formattedDashboardUrl, reportModel.SlackUserID), false, true), nil, nil))
if *config.GetPyroscopeConfig().Enabled {
pyroscopeServer := *config.GetPyroscopeConfig().ServerUrl
pyroscopeEnvironment := *config.GetPyroscopeConfig().Environment + "-1"

formattedPyroscopeUrl := fmt.Sprintf("%s/?query=chainlink-node.cpu{Environment=\"%s\"}&from=%d&to=%s", pyroscopeServer, pyroscopeEnvironment, k.TestReporter.Summary.StartTime, "now")
log.Info().Str("Pyroscope", formattedPyroscopeUrl).Msg("Dashboard URL")
notificationBlocks = append(notificationBlocks, slack.NewSectionBlock(slack.NewTextBlockObject("mrkdwn",
fmt.Sprintf("<%s|Pyroscope>",
formattedPyroscopeUrl), false, true), nil, nil))
}

ts, err := reportModel.SendSlackMessage(slackClient, slack.MsgOptionBlocks(notificationBlocks...))
log.Debug().Str("ts", ts).Msg("Sent Slack Message")
return err
return ts, err
}

// SetupBenchmarkKeeperContracts deploys a set amount of keeper Benchmark contracts registered to a single registry
Expand Down

0 comments on commit 0740012

Please sign in to comment.