Skip to content

Commit

Permalink
updates
Browse files Browse the repository at this point in the history
  • Loading branch information
@anirudhwarrier
anirudhwarrier committed Nov 13, 2023
1 parent 1751ba4 commit 6c416c2
Show file tree
Hide file tree
Showing 4 changed files with 136 additions and 29 deletions.
2 changes: 2 additions & 0 deletions .github/workflows/automation-benchmark-tests.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -112,6 +112,8 @@ jobs:
- name: Run Tests
uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/run-tests@7d541cbbca52d45b8a718257af86d9cf49774d1f # v2.2.15
env:
RR_CPU: 2000m
RR_MEM: 2Gi
DETACH_RUNNER: false
TEST_SUITE: automationv2_1
TEST_ARGS: -test.timeout 720h
Expand Down
141 changes: 122 additions & 19 deletions integration-tests/load/automationv2_1/automationv2_1_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ package automationv2_1

import (
"context"
"encoding/json"
"fmt"
geth "github.com/ethereum/go-ethereum"
"github.com/ethereum/go-ethereum/common"
Expand All @@ -16,11 +17,15 @@ import (
"github.com/smartcontractkit/chainlink/integration-tests/client"
"github.com/smartcontractkit/chainlink/integration-tests/contracts"
contractseth "github.com/smartcontractkit/chainlink/integration-tests/contracts/ethereum"
"github.com/smartcontractkit/chainlink/integration-tests/testreporters"
registrar21 "github.com/smartcontractkit/chainlink/v2/core/gethwrappers/generated/automation_registrar_wrapper2_1"
"github.com/smartcontractkit/chainlink/v2/core/gethwrappers/generated/automation_utils_2_1"
"github.com/smartcontractkit/chainlink/v2/core/gethwrappers/generated/log_emitter"
"github.com/smartcontractkit/chainlink/v2/core/gethwrappers/generated/simple_log_upkeep_counter_wrapper"
ocr3 "github.com/smartcontractkit/libocr/offchainreporting2plus/ocr3confighelper"
ocr2keepers30config "github.com/smartcontractkit/ocr2keepers/pkg/v3/config"
"github.com/smartcontractkit/wasp"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"math/big"
"os"
Expand Down Expand Up @@ -58,9 +63,11 @@ func getEnv(key, fallback string) string {

var (
numberofNodes, _ = strconv.Atoi(getEnv("NUMBEROFNODES", "6"))
numberOfUpkeeps, _ = strconv.Atoi(getEnv("NUMBEROFUPKEEPS", "500"))
duration, _ = strconv.Atoi(getEnv("DURATION", "300"))
numberOfUpkeeps, _ = strconv.Atoi(getEnv("NUMBEROFUPKEEPS", "100"))
duration, _ = strconv.Atoi(getEnv("DURATION", "900"))
blockTime, _ = strconv.Atoi(getEnv("BLOCKTIME", "1"))
numberOfEvents, _ = strconv.Atoi(getEnv("NUMBEROFEVENTS", "1"))
debug, _ = strconv.ParseBool(getEnv("DEBUG", "false"))
)

func TestLogTrigger(t *testing.T) {
Expand Down Expand Up @@ -184,8 +191,63 @@ func TestLogTrigger(t *testing.T) {
contractseth.RegistryVersion_2_1,
)

ocrConfig, err := actions.BuildAutoOCR2ConfigVars(t, chainlinkNodes[1:], *registrySettings, registrar.Address(), time.Second*15)
require.NoError(t, err, "Error building OCR config vars")
S, oracleIdentities, err := actions.GetOracleIdentities(chainlinkNodes)
require.NoError(t, err, "Error getting oracle identities")
offC, err := json.Marshal(ocr2keepers30config.OffchainConfig{
TargetProbability: "0.999",
TargetInRounds: 1,
PerformLockoutWindow: 3600000, // Intentionally set to be higher than in prod for testing purpose
GasLimitPerReport: 5_300_000,
GasOverheadPerUpkeep: 300_000,
MinConfirmations: 0,
MaxUpkeepBatchSize: 10,
})
require.NoError(t, err, "Error marshalling offchain config")

signerOnchainPublicKeys, transmitterAccounts, f, _, offchainConfigVersion, offchainConfig, err := ocr3.ContractSetConfigArgsForTests(
10*time.Second, // deltaProgress time.Duration,
15*time.Second, // deltaResend time.Duration,
500*time.Millisecond, // deltaInitial time.Duration,
1000*time.Millisecond, // deltaRound time.Duration,
200*time.Millisecond, // deltaGrace time.Duration,
300*time.Millisecond, // deltaCertifiedCommitRequest time.Duration
15*time.Second, // deltaStage time.Duration,
24, // rMax uint64,
S, // s []int,
oracleIdentities, // oracles []OracleIdentityExtra,
offC, // reportingPluginConfig []byte,
20*time.Millisecond, // maxDurationQuery time.Duration,
20*time.Millisecond, // maxDurationObservation time.Duration, // good to here
1200*time.Millisecond, // maxDurationShouldAcceptAttestedReport time.Duration,
20*time.Millisecond, // maxDurationShouldTransmitAcceptedReport time.Duration,
1, // f int,
nil, // onchainConfig []byte,
)
require.NoError(t, err, "Error setting OCR config vars")

var signers []common.Address
for _, signer := range signerOnchainPublicKeys {
require.Equal(t, 20, len(signer), "OnChainPublicKey '%v' has wrong length for address", signer)
signers = append(signers, common.BytesToAddress(signer))
}

var transmitters []common.Address
for _, transmitter := range transmitterAccounts {
require.True(t, common.IsHexAddress(string(transmitter)), "TransmitAccount '%s' is not a valid Ethereum address", string(transmitter))
transmitters = append(transmitters, common.HexToAddress(string(transmitter)))
}

onchainConfig, err := registrySettings.EncodeOnChainConfig(registrar.Address(), common.HexToAddress(chainClient.GetDefaultWallet().Address()))
require.NoError(t, err, "Error encoding onchain config")
l.Info().Msg("Done building OCR config")
ocrConfig := contracts.OCRv2Config{
Signers: signers,
Transmitters: transmitters,
F: f,
OnchainConfig: onchainConfig,
OffchainConfigVersion: offchainConfigVersion,
OffchainConfig: offchainConfig,
}

err = registry.SetConfig(*registrySettings, ocrConfig)
require.NoError(t, err, "Error setting registry config")
Expand Down Expand Up @@ -287,6 +349,7 @@ func TestLogTrigger(t *testing.T) {
}
l.Info().Msg("Successfully registered all Automation Consumer Contracts")
l.Info().Interface("Upkeep IDs", upkeepIds).Msg("Upkeep IDs")
l.Info().Msg("Waiting 30s for plugin to start")
time.Sleep(time.Second * 30)

startingBlock, err := chainClient.LatestBlockNumber(context.Background())
Expand All @@ -299,7 +362,7 @@ func TestLogTrigger(t *testing.T) {
T: t,
LoadType: wasp.RPS,
GenName: fmt.Sprintf("log_trigger_gen_%s", triggerContract.Address().String()),
CallTimeout: time.Minute * 3,
CallTimeout: time.Second * 10,
Schedule: wasp.Plain(
1,
loadDuration,
Expand All @@ -308,36 +371,50 @@ func TestLogTrigger(t *testing.T) {
triggerContract,
consumerContracts[i],
l,
numberOfEvents,
),
})
p.Add(g, err)
}

l.Info().Msg("Starting load generators")
startTime := time.Now()
_, err = p.Run(true)
require.NoError(t, err, "Error running load generators")

l.Info().Msg("Finished load generators")
l.Info().Msg("Waiting for upkeeps to be performed")
time.Sleep(time.Second * 60)
l.Info().Msg("Finished waiting for upkeeps to be performed")
l.Info().Msg("Finished waiting 60s for upkeeps to be performed")
endTime := time.Now()
testDuration := endTime.Sub(startTime)
l.Info().Dur("Duration", testDuration).Msg("Test Duration")

upkeepCounters := make([]int64, 0)
upkeepDelays := make([][]int64, 0)

for i, consumerContract := range consumerContracts {
count, err := consumerContract.Counter(nil)
require.NoError(t, err, "Error getting counter value")
upkeepCounters = append(upkeepCounters, count.Int64())
l.Debug().
Int("Count", int(count.Int64())).
Int("Number", i+1).
Int("Out Of", numberOfUpkeeps).
Msg("Counter Value")
//assert.GreaterOrEqual(t, count.Int64(), int64(durationInSeconds+1), "Counter should be greater than 2")
if debug {
for i, consumerContract := range consumerContracts {
count, err := consumerContract.Counter(nil)
require.NoError(t, err, "Error getting counter value")
upkeepCounters = append(upkeepCounters, count.Int64())
l.Debug().
Int("Count", int(count.Int64())).
Int("Number", i+1).
Int("Out Of", numberOfUpkeeps).
Msg("Counter Value")
assert.GreaterOrEqual(
t, count.Int64(), int64(numberOfEvents*duration+1),
fmt.Sprintf("Upkeep %d should have been performed at least %d times", i, numberOfEvents*duration+1))
}
l.Info().Interface("Upkeep Counters", upkeepCounters).Msg("Upkeep Counters")
}

for _, consumerContract := range consumerContracts {
var (
batchSize = 100
)

for cIter, consumerContract := range consumerContracts {
var (
logs []types.Log
address = common.HexToAddress(consumerContract.Address())
Expand All @@ -359,7 +436,7 @@ func TestLogTrigger(t *testing.T) {
Msg("Error getting logs")
} else {
delay := make([]int64, 0)
for _, log := range logs {
for lIter, log := range logs {
eventDetails, err := consumerABI.EventByID(log.Topics[0])
require.NoError(t, err, "Error getting event details")
consumer, err := simple_log_upkeep_counter_wrapper.NewSimpleLogUpkeepCounter(
Expand All @@ -371,14 +448,40 @@ func TestLogTrigger(t *testing.T) {
require.NoError(t, err, "Error parsing log")
delay = append(delay, parsedLog.TimeToPerform.Int64())
}
if (lIter+1)%batchSize == 0 {
err = chainClient.WaitForEvents()
require.NoError(t, err, "Failed to wait for chain events")
time.Sleep(time.Second * 1)
}
}
upkeepDelays = append(upkeepDelays, delay)
}
if (cIter+1)%batchSize == 0 {
err = chainClient.WaitForEvents()
require.NoError(t, err, "Failed to wait for chain events")
time.Sleep(time.Second * 1)
}
}

l.Info().Interface("Upkeep Counters", upkeepCounters).Msg("Upkeep Counters")
l.Info().Interface("Upkeep Delays", upkeepDelays).Msg("Upkeep Delays")

var allUpkeepDelays []int64

for _, upkeepDelay := range upkeepDelays {
allUpkeepDelays = append(allUpkeepDelays, upkeepDelay...)
}

avg, median, ninetyPct, ninetyNinePct, maximum := testreporters.IntListStats(allUpkeepDelays)
l.Info().
Float64("Average", avg).Int64("Median", median).
Int64("90th Percentile", ninetyPct).Int64("99th Percentile", ninetyNinePct).
Int64("Max", maximum).Msg("Upkeep Delays in seconds")

l.Info().
Int("Total Perform Count", len(allUpkeepDelays)).
Int("Total Events Emitted", numberOfEvents*numberOfUpkeeps*duration+numberOfUpkeeps).
Msg("Test completed")

t.Cleanup(func() {
if err = actions.TeardownRemoteSuite(t, testEnvironment.Cfg.Namespace, chainlinkNodes, nil, chainClient); err != nil {
l.Error().Err(err).Msg("Error when tearing down remote suite")
Expand Down
14 changes: 8 additions & 6 deletions integration-tests/load/automationv2_1/gun.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,28 +10,30 @@ type LogTriggerGun struct {
triggerContract contracts.LogEmitter
upkeepContract contracts.KeeperConsumer
logger zerolog.Logger
numberOfEvents int
}

func NewLogTriggerUser(
triggerContract contracts.LogEmitter,
upkeepContract contracts.KeeperConsumer,
logger zerolog.Logger,
numberOfEvents int,
) *LogTriggerGun {
return &LogTriggerGun{
triggerContract: triggerContract,
upkeepContract: upkeepContract,
logger: logger,
numberOfEvents: numberOfEvents,
}
}

func (m *LogTriggerGun) Call(l *wasp.Generator) *wasp.CallResult {
m.logger.Debug().Str("Trigger address", m.triggerContract.Address().String()).Msg("Triggering upkeep")
//initialCount, err := m.upkeepContract.Counter(context.Background())
//m.logger.Debug().Int64("Initial count", initialCount.Int64()).Msg("Initial count")
//if err != nil {
// return &wasp.CallResult{Error: err.Error(), Failed: true}
//}
_, err := m.triggerContract.EmitLogInt(1)
payload := make([]int, 0)
for i := 0; i < m.numberOfEvents; i++ {
payload = append(payload, 1)
}
_, err := m.triggerContract.EmitLogInts(payload)
if err != nil {
return &wasp.CallResult{Error: err.Error(), Failed: true}
}
Expand Down
8 changes: 4 additions & 4 deletions integration-tests/testreporters/keeper_benchmark.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -133,7 +133,7 @@ func (k *KeeperBenchmarkTestReporter) WriteReport(folderLocation string) error {
if err != nil {
return err
}
avg, median, ninetyPct, ninetyNinePct, max := intListStats(allDelays)
avg, median, ninetyPct, ninetyNinePct, max := IntListStats(allDelays)
err = keeperReportWriter.Write([]string{
fmt.Sprint(totalEligibleCount),
fmt.Sprint(totalPerformed),
Expand Down Expand Up @@ -183,7 +183,7 @@ func (k *KeeperBenchmarkTestReporter) WriteReport(folderLocation string) error {
}

for contractIndex, report := range k.Reports {
avg, median, ninetyPct, ninetyNinePct, max := intListStats(report.AllCheckDelays)
avg, median, ninetyPct, ninetyNinePct, max := IntListStats(report.AllCheckDelays)
err = keeperReportWriter.Write([]string{
fmt.Sprint(contractIndex),
report.RegistryAddress,
Expand Down Expand Up @@ -304,8 +304,8 @@ func (k *KeeperBenchmarkTestReporter) SendSlackNotification(t *testing.T, slackC
})
}

// intListStats helper calculates some statistics on an int list: avg, median, 90pct, 99pct, max
func intListStats(in []int64) (float64, int64, int64, int64, int64) {
// IntListStats helper calculates some statistics on an int list: avg, median, 90pct, 99pct, max
func IntListStats(in []int64) (float64, int64, int64, int64, int64) {
length := len(in)
if length == 0 {
return 0, 0, 0, 0, 0
Expand Down

0 comments on commit 6c416c2

Please sign in to comment.