diff --git a/.github/workflows/automation-benchmark-tests.yml b/.github/workflows/automation-benchmark-tests.yml index c251737e50f..0a43aa97031 100644 --- a/.github/workflows/automation-benchmark-tests.yml +++ b/.github/workflows/automation-benchmark-tests.yml @@ -112,6 +112,8 @@ jobs: - name: Run Tests uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/run-tests@7d541cbbca52d45b8a718257af86d9cf49774d1f # v2.2.15 env: + RR_CPU: 2000m + RR_MEM: 2Gi DETACH_RUNNER: false TEST_SUITE: automationv2_1 TEST_ARGS: -test.timeout 720h diff --git a/integration-tests/load/automationv2_1/automationv2_1_test.go b/integration-tests/load/automationv2_1/automationv2_1_test.go index 232ef67bf18..7cf44d2508c 100644 --- a/integration-tests/load/automationv2_1/automationv2_1_test.go +++ b/integration-tests/load/automationv2_1/automationv2_1_test.go @@ -2,6 +2,7 @@ package automationv2_1 import ( "context" + "encoding/json" "fmt" geth "github.com/ethereum/go-ethereum" "github.com/ethereum/go-ethereum/common" @@ -16,11 +17,15 @@ import ( "github.com/smartcontractkit/chainlink/integration-tests/client" "github.com/smartcontractkit/chainlink/integration-tests/contracts" contractseth "github.com/smartcontractkit/chainlink/integration-tests/contracts/ethereum" + "github.com/smartcontractkit/chainlink/integration-tests/testreporters" registrar21 "github.com/smartcontractkit/chainlink/v2/core/gethwrappers/generated/automation_registrar_wrapper2_1" "github.com/smartcontractkit/chainlink/v2/core/gethwrappers/generated/automation_utils_2_1" "github.com/smartcontractkit/chainlink/v2/core/gethwrappers/generated/log_emitter" "github.com/smartcontractkit/chainlink/v2/core/gethwrappers/generated/simple_log_upkeep_counter_wrapper" + ocr3 "github.com/smartcontractkit/libocr/offchainreporting2plus/ocr3confighelper" + ocr2keepers30config "github.com/smartcontractkit/ocr2keepers/pkg/v3/config" "github.com/smartcontractkit/wasp" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "math/big" "os" @@ -58,9 +63,11 @@ func getEnv(key, fallback string) string { var ( numberofNodes, _ = strconv.Atoi(getEnv("NUMBEROFNODES", "6")) - numberOfUpkeeps, _ = strconv.Atoi(getEnv("NUMBEROFUPKEEPS", "500")) - duration, _ = strconv.Atoi(getEnv("DURATION", "300")) + numberOfUpkeeps, _ = strconv.Atoi(getEnv("NUMBEROFUPKEEPS", "100")) + duration, _ = strconv.Atoi(getEnv("DURATION", "900")) blockTime, _ = strconv.Atoi(getEnv("BLOCKTIME", "1")) + numberOfEvents, _ = strconv.Atoi(getEnv("NUMBEROFEVENTS", "1")) + debug, _ = strconv.ParseBool(getEnv("DEBUG", "false")) ) func TestLogTrigger(t *testing.T) { @@ -184,8 +191,63 @@ func TestLogTrigger(t *testing.T) { contractseth.RegistryVersion_2_1, ) - ocrConfig, err := actions.BuildAutoOCR2ConfigVars(t, chainlinkNodes[1:], *registrySettings, registrar.Address(), time.Second*15) - require.NoError(t, err, "Error building OCR config vars") + S, oracleIdentities, err := actions.GetOracleIdentities(chainlinkNodes) + require.NoError(t, err, "Error getting oracle identities") + offC, err := json.Marshal(ocr2keepers30config.OffchainConfig{ + TargetProbability: "0.999", + TargetInRounds: 1, + PerformLockoutWindow: 3600000, // Intentionally set to be higher than in prod for testing purpose + GasLimitPerReport: 5_300_000, + GasOverheadPerUpkeep: 300_000, + MinConfirmations: 0, + MaxUpkeepBatchSize: 10, + }) + require.NoError(t, err, "Error marshalling offchain config") + + signerOnchainPublicKeys, transmitterAccounts, f, _, offchainConfigVersion, offchainConfig, err := ocr3.ContractSetConfigArgsForTests( + 10*time.Second, // deltaProgress time.Duration, + 15*time.Second, // deltaResend time.Duration, + 500*time.Millisecond, // deltaInitial time.Duration, + 1000*time.Millisecond, // deltaRound time.Duration, + 200*time.Millisecond, // deltaGrace time.Duration, + 300*time.Millisecond, // deltaCertifiedCommitRequest time.Duration + 15*time.Second, // deltaStage time.Duration, + 24, // rMax uint64, + S, // s []int, + oracleIdentities, // oracles []OracleIdentityExtra, + offC, // reportingPluginConfig []byte, + 20*time.Millisecond, // maxDurationQuery time.Duration, + 20*time.Millisecond, // maxDurationObservation time.Duration, // good to here + 1200*time.Millisecond, // maxDurationShouldAcceptAttestedReport time.Duration, + 20*time.Millisecond, // maxDurationShouldTransmitAcceptedReport time.Duration, + 1, // f int, + nil, // onchainConfig []byte, + ) + require.NoError(t, err, "Error setting OCR config vars") + + var signers []common.Address + for _, signer := range signerOnchainPublicKeys { + require.Equal(t, 20, len(signer), "OnChainPublicKey '%v' has wrong length for address", signer) + signers = append(signers, common.BytesToAddress(signer)) + } + + var transmitters []common.Address + for _, transmitter := range transmitterAccounts { + require.True(t, common.IsHexAddress(string(transmitter)), "TransmitAccount '%s' is not a valid Ethereum address", string(transmitter)) + transmitters = append(transmitters, common.HexToAddress(string(transmitter))) + } + + onchainConfig, err := registrySettings.EncodeOnChainConfig(registrar.Address(), common.HexToAddress(chainClient.GetDefaultWallet().Address())) + require.NoError(t, err, "Error encoding onchain config") + l.Info().Msg("Done building OCR config") + ocrConfig := contracts.OCRv2Config{ + Signers: signers, + Transmitters: transmitters, + F: f, + OnchainConfig: onchainConfig, + OffchainConfigVersion: offchainConfigVersion, + OffchainConfig: offchainConfig, + } err = registry.SetConfig(*registrySettings, ocrConfig) require.NoError(t, err, "Error setting registry config") @@ -287,6 +349,7 @@ func TestLogTrigger(t *testing.T) { } l.Info().Msg("Successfully registered all Automation Consumer Contracts") l.Info().Interface("Upkeep IDs", upkeepIds).Msg("Upkeep IDs") + l.Info().Msg("Waiting 30s for plugin to start") time.Sleep(time.Second * 30) startingBlock, err := chainClient.LatestBlockNumber(context.Background()) @@ -299,7 +362,7 @@ func TestLogTrigger(t *testing.T) { T: t, LoadType: wasp.RPS, GenName: fmt.Sprintf("log_trigger_gen_%s", triggerContract.Address().String()), - CallTimeout: time.Minute * 3, + CallTimeout: time.Second * 10, Schedule: wasp.Plain( 1, loadDuration, @@ -308,36 +371,50 @@ func TestLogTrigger(t *testing.T) { triggerContract, consumerContracts[i], l, + numberOfEvents, ), }) p.Add(g, err) } l.Info().Msg("Starting load generators") + startTime := time.Now() _, err = p.Run(true) require.NoError(t, err, "Error running load generators") l.Info().Msg("Finished load generators") l.Info().Msg("Waiting for upkeeps to be performed") time.Sleep(time.Second * 60) - l.Info().Msg("Finished waiting for upkeeps to be performed") + l.Info().Msg("Finished waiting 60s for upkeeps to be performed") + endTime := time.Now() + testDuration := endTime.Sub(startTime) + l.Info().Dur("Duration", testDuration).Msg("Test Duration") upkeepCounters := make([]int64, 0) upkeepDelays := make([][]int64, 0) - for i, consumerContract := range consumerContracts { - count, err := consumerContract.Counter(nil) - require.NoError(t, err, "Error getting counter value") - upkeepCounters = append(upkeepCounters, count.Int64()) - l.Debug(). - Int("Count", int(count.Int64())). - Int("Number", i+1). - Int("Out Of", numberOfUpkeeps). - Msg("Counter Value") - //assert.GreaterOrEqual(t, count.Int64(), int64(durationInSeconds+1), "Counter should be greater than 2") + if debug { + for i, consumerContract := range consumerContracts { + count, err := consumerContract.Counter(nil) + require.NoError(t, err, "Error getting counter value") + upkeepCounters = append(upkeepCounters, count.Int64()) + l.Debug(). + Int("Count", int(count.Int64())). + Int("Number", i+1). + Int("Out Of", numberOfUpkeeps). + Msg("Counter Value") + assert.GreaterOrEqual( + t, count.Int64(), int64(numberOfEvents*duration+1), + fmt.Sprintf("Upkeep %d should have been performed at least %d times", i, numberOfEvents*duration+1)) + } + l.Info().Interface("Upkeep Counters", upkeepCounters).Msg("Upkeep Counters") } - for _, consumerContract := range consumerContracts { + var ( + batchSize = 100 + ) + + for cIter, consumerContract := range consumerContracts { var ( logs []types.Log address = common.HexToAddress(consumerContract.Address()) @@ -359,7 +436,7 @@ func TestLogTrigger(t *testing.T) { Msg("Error getting logs") } else { delay := make([]int64, 0) - for _, log := range logs { + for lIter, log := range logs { eventDetails, err := consumerABI.EventByID(log.Topics[0]) require.NoError(t, err, "Error getting event details") consumer, err := simple_log_upkeep_counter_wrapper.NewSimpleLogUpkeepCounter( @@ -371,14 +448,40 @@ func TestLogTrigger(t *testing.T) { require.NoError(t, err, "Error parsing log") delay = append(delay, parsedLog.TimeToPerform.Int64()) } + if (lIter+1)%batchSize == 0 { + err = chainClient.WaitForEvents() + require.NoError(t, err, "Failed to wait for chain events") + time.Sleep(time.Second * 1) + } } upkeepDelays = append(upkeepDelays, delay) } + if (cIter+1)%batchSize == 0 { + err = chainClient.WaitForEvents() + require.NoError(t, err, "Failed to wait for chain events") + time.Sleep(time.Second * 1) + } } - l.Info().Interface("Upkeep Counters", upkeepCounters).Msg("Upkeep Counters") l.Info().Interface("Upkeep Delays", upkeepDelays).Msg("Upkeep Delays") + var allUpkeepDelays []int64 + + for _, upkeepDelay := range upkeepDelays { + allUpkeepDelays = append(allUpkeepDelays, upkeepDelay...) + } + + avg, median, ninetyPct, ninetyNinePct, maximum := testreporters.IntListStats(allUpkeepDelays) + l.Info(). + Float64("Average", avg).Int64("Median", median). + Int64("90th Percentile", ninetyPct).Int64("99th Percentile", ninetyNinePct). + Int64("Max", maximum).Msg("Upkeep Delays in seconds") + + l.Info(). + Int("Total Perform Count", len(allUpkeepDelays)). + Int("Total Events Emitted", numberOfEvents*numberOfUpkeeps*duration+numberOfUpkeeps). + Msg("Test completed") + t.Cleanup(func() { if err = actions.TeardownRemoteSuite(t, testEnvironment.Cfg.Namespace, chainlinkNodes, nil, chainClient); err != nil { l.Error().Err(err).Msg("Error when tearing down remote suite") diff --git a/integration-tests/load/automationv2_1/gun.go b/integration-tests/load/automationv2_1/gun.go index c95d9ec43e8..35ac83151f0 100644 --- a/integration-tests/load/automationv2_1/gun.go +++ b/integration-tests/load/automationv2_1/gun.go @@ -10,28 +10,30 @@ type LogTriggerGun struct { triggerContract contracts.LogEmitter upkeepContract contracts.KeeperConsumer logger zerolog.Logger + numberOfEvents int } func NewLogTriggerUser( triggerContract contracts.LogEmitter, upkeepContract contracts.KeeperConsumer, logger zerolog.Logger, + numberOfEvents int, ) *LogTriggerGun { return &LogTriggerGun{ triggerContract: triggerContract, upkeepContract: upkeepContract, logger: logger, + numberOfEvents: numberOfEvents, } } func (m *LogTriggerGun) Call(l *wasp.Generator) *wasp.CallResult { m.logger.Debug().Str("Trigger address", m.triggerContract.Address().String()).Msg("Triggering upkeep") - //initialCount, err := m.upkeepContract.Counter(context.Background()) - //m.logger.Debug().Int64("Initial count", initialCount.Int64()).Msg("Initial count") - //if err != nil { - // return &wasp.CallResult{Error: err.Error(), Failed: true} - //} - _, err := m.triggerContract.EmitLogInt(1) + payload := make([]int, 0) + for i := 0; i < m.numberOfEvents; i++ { + payload = append(payload, 1) + } + _, err := m.triggerContract.EmitLogInts(payload) if err != nil { return &wasp.CallResult{Error: err.Error(), Failed: true} } diff --git a/integration-tests/testreporters/keeper_benchmark.go b/integration-tests/testreporters/keeper_benchmark.go index c800eb37be2..7c0b3bd6c14 100644 --- a/integration-tests/testreporters/keeper_benchmark.go +++ b/integration-tests/testreporters/keeper_benchmark.go @@ -133,7 +133,7 @@ func (k *KeeperBenchmarkTestReporter) WriteReport(folderLocation string) error { if err != nil { return err } - avg, median, ninetyPct, ninetyNinePct, max := intListStats(allDelays) + avg, median, ninetyPct, ninetyNinePct, max := IntListStats(allDelays) err = keeperReportWriter.Write([]string{ fmt.Sprint(totalEligibleCount), fmt.Sprint(totalPerformed), @@ -183,7 +183,7 @@ func (k *KeeperBenchmarkTestReporter) WriteReport(folderLocation string) error { } for contractIndex, report := range k.Reports { - avg, median, ninetyPct, ninetyNinePct, max := intListStats(report.AllCheckDelays) + avg, median, ninetyPct, ninetyNinePct, max := IntListStats(report.AllCheckDelays) err = keeperReportWriter.Write([]string{ fmt.Sprint(contractIndex), report.RegistryAddress, @@ -304,8 +304,8 @@ func (k *KeeperBenchmarkTestReporter) SendSlackNotification(t *testing.T, slackC }) } -// intListStats helper calculates some statistics on an int list: avg, median, 90pct, 99pct, max -func intListStats(in []int64) (float64, int64, int64, int64, int64) { +// IntListStats helper calculates some statistics on an int list: avg, median, 90pct, 99pct, max +func IntListStats(in []int64) (float64, int64, int64, int64, int64) { length := len(in) if length == 0 { return 0, 0, 0, 0, 0