From 074001221475bff679b837f1f16aa5a69eb3334b Mon Sep 17 00:00:00 2001 From: anirudhwarrier <12178754+anirudhwarrier@users.noreply.github.com> Date: Thu, 3 Oct 2024 12:18:29 +0530 Subject: [PATCH] update automation tests load - separate pyroscope profiling per node benchmark - add pyroscope profiling per node benchmark - improve slack notification ux --- .../benchmark/automation_test.go | 18 ++++++++++---- .../automationv2_1/automationv2_1_test.go | 19 +++++++++------ .../load/automationv2_1/helpers.go | 2 +- .../testreporters/keeper_benchmark.go | 24 +++++++++++++++---- .../testsetups/automation_benchmark.go | 22 +++++++++++++---- 5 files changed, 64 insertions(+), 21 deletions(-) diff --git a/integration-tests/benchmark/automation_test.go b/integration-tests/benchmark/automation_test.go index 0a63ff2c27a..b8ec408c300 100644 --- a/integration-tests/benchmark/automation_test.go +++ b/integration-tests/benchmark/automation_test.go @@ -3,6 +3,7 @@ package benchmark import ( "fmt" "math/big" + "strconv" "strings" "testing" "time" @@ -334,6 +335,10 @@ func SetupAutomationBenchmarkEnv(t *testing.T, keeperTestConfig types.Automation return testEnvironment, testNetwork } + if *keeperTestConfig.GetPyroscopeConfig().Enabled { + keeperTestConfig.GetPyroscopeConfig().Environment = &testEnvironment.Cfg.Namespace + } + // separate RPC urls per CL node internalWsURLs := make([]string, 0) internalHttpURLs := make([]string, 0) @@ -360,22 +365,27 @@ func SetupAutomationBenchmarkEnv(t *testing.T, keeperTestConfig types.Automation l.Debug().Strs("internalWsURLs", internalWsURLs).Strs("internalHttpURLs", internalHttpURLs).Msg("internalURLs") for i := 0; i < numberOfNodes; i++ { + config := keeperTestConfig + if *config.GetPyroscopeConfig().Enabled { + name := testEnvironment.Cfg.Namespace + "-" + strconv.Itoa(i) + config.GetPyroscopeConfig().Environment = &name + } testNetwork.HTTPURLs = []string{internalHttpURLs[i]} testNetwork.URLs = []string{internalWsURLs[i]} var overrideFn = func(_ interface{}, target interface{}) { - ctfconfig.MustConfigOverrideChainlinkVersion(keeperTestConfig.GetChainlinkImageConfig(), target) - ctfconfig.MightConfigOverridePyroscopeKey(keeperTestConfig.GetPyroscopeConfig(), target) + ctfconfig.MustConfigOverrideChainlinkVersion(config.GetChainlinkImageConfig(), target) + ctfconfig.MightConfigOverridePyroscopeKey(config.GetPyroscopeConfig(), target) } - tomlConfig, err := actions.BuildTOMLNodeConfigForK8s(keeperTestConfig, testNetwork) + tomlConfig, err := actions.BuildTOMLNodeConfigForK8s(config, testNetwork) require.NoError(t, err, "Error building TOML config") cd := chainlink.NewWithOverride(i, map[string]any{ "toml": tomlConfig, "chainlink": chainlinkResources, "db": dbResources, - }, keeperTestConfig.GetChainlinkImageConfig(), overrideFn) + }, config.GetChainlinkImageConfig(), overrideFn) testEnvironment.AddHelm(cd) } diff --git a/integration-tests/load/automationv2_1/automationv2_1_test.go b/integration-tests/load/automationv2_1/automationv2_1_test.go index 3cd931ecef0..6247552f8c8 100644 --- a/integration-tests/load/automationv2_1/automationv2_1_test.go +++ b/integration-tests/load/automationv2_1/automationv2_1_test.go @@ -277,24 +277,29 @@ Load Config: secretsTOML = "" } - numberOfUpkeeps := *loadedTestConfig.Automation.General.NumberOfNodes + numberOfNodes := *loadedTestConfig.Automation.General.NumberOfNodes - for i := 0; i < numberOfUpkeeps+1; i++ { // +1 for the OCR boot node + for i := 0; i < numberOfNodes+1; i++ { // +1 for the OCR boot node + config := loadedTestConfig + if *config.Pyroscope.Enabled { + name := testEnvironment.Cfg.Namespace + "-" + strconv.Itoa(i) + config.Pyroscope.Environment = &name + } var overrideFn = func(_ interface{}, target interface{}) { - ctfconfig.MustConfigOverrideChainlinkVersion(loadedTestConfig.GetChainlinkImageConfig(), target) - ctfconfig.MightConfigOverridePyroscopeKey(loadedTestConfig.GetPyroscopeConfig(), target) + ctfconfig.MustConfigOverrideChainlinkVersion(config.GetChainlinkImageConfig(), target) + ctfconfig.MightConfigOverridePyroscopeKey(config.GetPyroscopeConfig(), target) } - tomlConfig, err := actions.BuildTOMLNodeConfigForK8s(&loadedTestConfig, testNetwork) + tomlConfig, err := actions.BuildTOMLNodeConfigForK8s(&config, testNetwork) require.NoError(t, err, "Error building TOML config") cd := chainlink.NewWithOverride(i, map[string]any{ "toml": tomlConfig, "chainlink": nodeSpec, "db": dbSpec, - "prometheus": *loadedTestConfig.Automation.General.UsePrometheus, + "prometheus": *config.Automation.General.UsePrometheus, "secretsToml": secretsTOML, - }, loadedTestConfig.ChainlinkImage, overrideFn) + }, config.ChainlinkImage, overrideFn) testEnvironment.AddHelm(cd) } diff --git a/integration-tests/load/automationv2_1/helpers.go b/integration-tests/load/automationv2_1/helpers.go index 41fa9a0ce1e..689dcddc8ec 100644 --- a/integration-tests/load/automationv2_1/helpers.go +++ b/integration-tests/load/automationv2_1/helpers.go @@ -52,7 +52,7 @@ func sendSlackNotification(header string, l zerolog.Logger, config *tc.TestConfi notificationBlocks = append(notificationBlocks, slack.NewDividerBlock()) if *config.Pyroscope.Enabled { pyroscopeServer := *config.Pyroscope.ServerUrl - pyroscopeEnvironment := *config.Pyroscope.Environment + pyroscopeEnvironment := *config.Pyroscope.Environment + "-1" formattedPyroscopeUrl := fmt.Sprintf("%s/?query=chainlink-node.cpu{Environment=\"%s\"}&from=%s&to=%s", pyroscopeServer, pyroscopeEnvironment, startingTime, endingTime) l.Info().Str("Pyroscope", formattedPyroscopeUrl).Msg("Dashboard URL") diff --git a/integration-tests/testreporters/keeper_benchmark.go b/integration-tests/testreporters/keeper_benchmark.go index c85de92493e..f5e81a021e2 100644 --- a/integration-tests/testreporters/keeper_benchmark.go +++ b/integration-tests/testreporters/keeper_benchmark.go @@ -11,6 +11,8 @@ import ( "sync" "testing" + "github.com/smartcontractkit/chainlink/integration-tests/testconfig" + "github.com/rs/zerolog/log" "github.com/slack-go/slack" @@ -26,6 +28,7 @@ type KeeperBenchmarkTestReporter struct { NumRevertedUpkeeps int64 NumStaleUpkeepReports int64 Summary KeeperBenchmarkTestSummary `json:"summary"` + SlackTs string namespace string keeperReportFile string @@ -49,8 +52,9 @@ type KeeperBenchmarkTestLoad struct { } type KeeperBenchmarkTestConfig struct { - Chainlink map[string]map[string]string `json:"chainlink"` - Geth map[string]map[string]string `json:"geth"` + Chainlink map[string]map[string]string `json:"chainlink"` + Geth map[string]map[string]string `json:"geth"` + TestConfig testconfig.TestConfig } type KeeperBenchmarkTestMetrics struct { @@ -259,7 +263,7 @@ func (k *KeeperBenchmarkTestReporter) SendSlackNotification(t *testing.T, slackC messageBlocks := testreporters.CommonSlackNotificationBlocks( headerText, k.namespace, k.keeperReportFile, ) - ts, err := testreporters.SendSlackMessage(slackClient, slack.MsgOptionBlocks(messageBlocks...)) + ts, err := testreporters.SendSlackMessage(slackClient, slack.MsgOptionBlocks(messageBlocks...), slack.MsgOptionTS(k.SlackTs)) if err != nil { return err } @@ -277,12 +281,24 @@ func (k *KeeperBenchmarkTestReporter) SendSlackNotification(t *testing.T, slackC formattedDashboardUrl := fmt.Sprintf("%s%s?from=%d&to=%d&var-namespace=%s&var-cl_node=chainlink-0-0", grafanaUrl, dashboardUrl, k.Summary.StartTime, k.Summary.EndTime, k.namespace) log.Info().Str("Dashboard", formattedDashboardUrl).Msg("Dashboard URL") + notificationText := fmt.Sprintf("Automation Benchmark Test Summary %s.\n<%s|Test Dashboard> ", k.namespace, formattedDashboardUrl) + + config := k.Summary.Config.TestConfig + if *config.GetPyroscopeConfig().Enabled { + pyroscopeServer := *config.GetPyroscopeConfig().ServerUrl + pyroscopeEnvironment := *config.GetPyroscopeConfig().Environment + "-1" + + formattedPyroscopeUrl := fmt.Sprintf("%s/?query=chainlink-node.cpu{Environment=\"%s\"}&from=%d&to=%d", pyroscopeServer, pyroscopeEnvironment, k.Summary.StartTime, k.Summary.EndTime) + log.Info().Str("Pyroscope", formattedPyroscopeUrl).Msg("Dashboard URL") + notificationText += fmt.Sprintf("\n<%s|Pyroscope>", formattedPyroscopeUrl) + } + if err := testreporters.UploadSlackFile(slackClient, slack.FileUploadParameters{ Title: fmt.Sprintf("Automation Benchmark Test Summary %s", k.namespace), Filetype: "json", Filename: fmt.Sprintf("automation_benchmark_summary_%s.json", k.namespace), File: k.keeperSummaryFile, - InitialComment: fmt.Sprintf("Automation Benchmark Test Summary %s.\n<%s|Test Dashboard> ", k.namespace, formattedDashboardUrl), + InitialComment: notificationText, Channels: []string{testreporters.SlackChannel}, ThreadTimestamp: ts, }); err != nil { diff --git a/integration-tests/testsetups/automation_benchmark.go b/integration-tests/testsetups/automation_benchmark.go index ff8100ea437..054765ee057 100644 --- a/integration-tests/testsetups/automation_benchmark.go +++ b/integration-tests/testsetups/automation_benchmark.go @@ -110,6 +110,7 @@ func NewKeeperBenchmarkTest(t *testing.T, inputs KeeperBenchmarkTestInputs) *Kee func (k *KeeperBenchmarkTest) Setup(env *environment.Environment, config testconfig.TestConfig) { startTime := time.Now() k.TestReporter.Summary.StartTime = startTime.UnixMilli() + k.TestReporter.Summary.Config.TestConfig = config k.ensureInputValues() k.env = env k.namespace = k.env.Cfg.Namespace @@ -180,10 +181,11 @@ func (k *KeeperBenchmarkTest) Setup(env *environment.Environment, config testcon } k.log.Info().Str("Setup Time", time.Since(startTime).String()).Msg("Finished Keeper Benchmark Test Setup") - err = k.SendSlackNotification(nil, &config) + ts, err := k.SendSlackNotification(nil, &config) if err != nil { k.log.Warn().Msg("Sending test start slack notification failed") } + k.TestReporter.SlackTs = ts } // Run runs the keeper benchmark test @@ -618,19 +620,19 @@ func (k *KeeperBenchmarkTest) ensureInputValues() { } } -func (k *KeeperBenchmarkTest) SendSlackNotification(slackClient *slack.Client, config tt.AutomationBenchmarkTestConfig) error { +func (k *KeeperBenchmarkTest) SendSlackNotification(slackClient *slack.Client, config tt.AutomationBenchmarkTestConfig) (string, error) { if slackClient == nil { slackClient = slack.New(reportModel.SlackAPIKey) } grafanaUrl, err := config.GetGrafanaBaseURL() if err != nil { - return err + return "", err } dashboardUrl, err := config.GetGrafanaDashboardURL() if err != nil { - return err + return "", err } headerText := ":white_check_mark: Automation Benchmark Test STARTED :white_check_mark:" @@ -646,10 +648,20 @@ func (k *KeeperBenchmarkTest) SendSlackNotification(slackClient *slack.Client, c notificationBlocks = append(notificationBlocks, slack.NewSectionBlock(slack.NewTextBlockObject("mrkdwn", fmt.Sprintf("<%s|Test Dashboard> \nNotifying <@%s>", formattedDashboardUrl, reportModel.SlackUserID), false, true), nil, nil)) + if *config.GetPyroscopeConfig().Enabled { + pyroscopeServer := *config.GetPyroscopeConfig().ServerUrl + pyroscopeEnvironment := *config.GetPyroscopeConfig().Environment + "-1" + + formattedPyroscopeUrl := fmt.Sprintf("%s/?query=chainlink-node.cpu{Environment=\"%s\"}&from=%d&to=%s", pyroscopeServer, pyroscopeEnvironment, k.TestReporter.Summary.StartTime, "now") + log.Info().Str("Pyroscope", formattedPyroscopeUrl).Msg("Dashboard URL") + notificationBlocks = append(notificationBlocks, slack.NewSectionBlock(slack.NewTextBlockObject("mrkdwn", + fmt.Sprintf("<%s|Pyroscope>", + formattedPyroscopeUrl), false, true), nil, nil)) + } ts, err := reportModel.SendSlackMessage(slackClient, slack.MsgOptionBlocks(notificationBlocks...)) log.Debug().Str("ts", ts).Msg("Sent Slack Message") - return err + return ts, err } // SetupBenchmarkKeeperContracts deploys a set amount of keeper Benchmark contracts registered to a single registry