From c13fdc3c80efe26c638e9441a8cae5c0e801c2f5 Mon Sep 17 00:00:00 2001 From: Awbrey Hughlett Date: Tue, 7 Nov 2023 15:22:00 -0500 Subject: [PATCH] This commit introduces some new functionality and some refactoring: - new: log trigger events and log trigger upkeeps - new: simulation progress tracking to the console - change: RunBook changed to SimulationPlan - change: SimulationPlan json altered for cleaner configurations - change: loaders moved to loader package due to import cycles - change: refactored main process for cleaner simulation exit To run a new simulation: ``` make simulator && ./bin/simulator --simulate -f ./tools/simulator/plans/simplan_fast_check.json ``` Verification of expected performs is still not fully tested, but the updates above create the baseline for log trigger upkeeps. --- .gitignore | 4 +- SIMULATOR.md | 281 ++++++++++++++---- cmd/simulator/config/runbook.go | 112 ------- cmd/simulator/config/runbook_test.go | 32 -- cmd/simulator/main.go | 62 ++-- cmd/simulator/node/group.go | 78 ----- cmd/simulator/run/runbook.go | 16 - cmd/simulator/simulate/upkeep/generate.go | 161 ---------- cmd/simulator/simulate/upkeep/loader.go | 107 ------- cmd/simulator/simulate/upkeep/loader_test.go | 45 --- .../v2/runbook_arbitrum_mild.json | 53 ---- .../v2/runbook_eth_goerli_mild.json | 80 ----- .../v2/runbook_matic_mild.json | 53 ---- .../v3/runbook_eth_goerli_mild.json | 80 ----- .../v3/runbook_fast_check.json | 46 --- {cmd => tools}/simulator/config/duration.go | 0 .../simulator/config/duration_test.go | 0 tools/simulator/config/event.go | 91 ++++++ {cmd => tools}/simulator/config/keyring.go | 52 ++-- .../simulator/config/keyring_test.go | 2 +- tools/simulator/config/simulation.go | 184 ++++++++++++ tools/simulator/config/simulation_test.go | 44 +++ {cmd => tools}/simulator/io/log.go | 0 {cmd => tools}/simulator/io/monitor.go | 0 {cmd => tools}/simulator/node/active.go | 0 {cmd => tools}/simulator/node/add.go | 26 +- tools/simulator/node/group.go | 78 +++++ {cmd => tools}/simulator/node/report.go | 6 +- {cmd => tools}/simulator/node/statistics.go | 0 {cmd => tools}/simulator/node/stats.go | 2 +- tools/simulator/plans/only_log_trigger.json | 91 ++++++ tools/simulator/plans/simplan_fast_check.json | 79 +++++ {cmd => tools}/simulator/run/output.go | 25 +- {cmd => tools}/simulator/run/profile.go | 0 tools/simulator/run/runbook.go | 16 + .../simulator/simulate/chain/block.go | 16 +- .../simulator/simulate/chain/broadcaster.go | 28 +- .../simulate/chain/broadcaster_test.go | 8 +- tools/simulator/simulate/chain/generate.go | 244 +++++++++++++++ .../simulate/chain}/generate_test.go | 34 ++- .../simulator/simulate/chain/history.go | 2 +- .../simulator/simulate/chain/history_test.go | 6 +- .../simulator/simulate/chain/listener.go | 0 .../simulator/simulate/chain/listener_test.go | 6 +- {cmd => tools}/simulator/simulate/db/ocr3.go | 0 .../simulator/simulate/db/upkeep.go | 0 {cmd => tools}/simulator/simulate/hydrator.go | 19 +- tools/simulator/simulate/loader/logtrigger.go | 75 +++++ .../simulate/loader/logtrigger_test.go | 1 + .../simulator/simulate/loader/ocr3config.go | 54 ++-- .../simulate/loader/ocr3config_test.go | 20 +- .../simulator/simulate/loader/ocr3transmit.go | 111 ++++--- .../simulate/loader/ocr3transmit_test.go | 13 +- tools/simulator/simulate/loader/upkeep.go | 74 +++++ .../simulator/simulate/loader/upkeep_test.go | 68 +++++ .../simulator/simulate/net/network.go | 0 .../simulator/simulate/net/network_test.go | 2 +- .../simulator/simulate/net/service.go | 2 +- .../simulator/simulate/net/service_test.go | 2 +- .../simulator/simulate/ocr/config.go | 2 +- .../simulator/simulate/ocr/config_test.go | 8 +- .../simulator/simulate/ocr/report.go | 4 +- .../simulator/simulate/ocr/report_test.go | 10 +- tools/simulator/simulate/ocr/transmit.go | 49 +++ tools/simulator/simulate/ocr/transmit_test.go | 1 + .../simulator/simulate/upkeep/active.go | 4 +- .../simulator/simulate/upkeep/active_test.go | 25 +- .../simulator/simulate/upkeep/log.go | 2 +- .../simulator/simulate/upkeep/log_test.go | 14 +- .../simulator/simulate/upkeep/perform.go | 4 +- .../simulator/simulate/upkeep/perform_test.go | 14 +- .../simulator/simulate/upkeep/pipeline.go | 20 +- .../simulate/upkeep/pipeline_test.go | 18 +- .../simulator/simulate/upkeep/source.go | 4 +- .../simulator/simulate/upkeep/source_test.go | 16 +- .../simulator/simulate/upkeep/util.go | 2 +- .../simulator/simulate/upkeep/util_test.go | 2 +- {cmd => tools}/simulator/telemetry/base.go | 0 .../simulator/telemetry/contract.go | 0 {cmd => tools}/simulator/telemetry/log.go | 0 tools/simulator/telemetry/progress.go | 129 ++++++++ {cmd => tools}/simulator/telemetry/rpc.go | 0 {cmd => tools}/simulator/util/encode.go | 0 {cmd => tools}/simulator/util/rand.go | 0 {cmd => tools}/simulator/util/sort.go | 0 85 files changed, 1839 insertions(+), 1180 deletions(-) delete mode 100644 cmd/simulator/config/runbook.go delete mode 100644 cmd/simulator/config/runbook_test.go delete mode 100644 cmd/simulator/node/group.go delete mode 100644 cmd/simulator/run/runbook.go delete mode 100644 cmd/simulator/simulate/upkeep/generate.go delete mode 100644 cmd/simulator/simulate/upkeep/loader.go delete mode 100644 cmd/simulator/simulate/upkeep/loader_test.go delete mode 100644 simulation_runbooks/v2/runbook_arbitrum_mild.json delete mode 100644 simulation_runbooks/v2/runbook_eth_goerli_mild.json delete mode 100644 simulation_runbooks/v2/runbook_matic_mild.json delete mode 100644 simulation_runbooks/v3/runbook_eth_goerli_mild.json delete mode 100644 simulation_runbooks/v3/runbook_fast_check.json rename {cmd => tools}/simulator/config/duration.go (100%) rename {cmd => tools}/simulator/config/duration_test.go (100%) create mode 100644 tools/simulator/config/event.go rename {cmd => tools}/simulator/config/keyring.go (62%) rename {cmd => tools}/simulator/config/keyring_test.go (97%) create mode 100644 tools/simulator/config/simulation.go create mode 100644 tools/simulator/config/simulation_test.go rename {cmd => tools}/simulator/io/log.go (100%) rename {cmd => tools}/simulator/io/monitor.go (100%) rename {cmd => tools}/simulator/node/active.go (100%) rename {cmd => tools}/simulator/node/add.go (84%) create mode 100644 tools/simulator/node/group.go rename {cmd => tools}/simulator/node/report.go (96%) rename {cmd => tools}/simulator/node/statistics.go (100%) rename {cmd => tools}/simulator/node/stats.go (98%) create mode 100644 tools/simulator/plans/only_log_trigger.json create mode 100644 tools/simulator/plans/simplan_fast_check.json rename {cmd => tools}/simulator/run/output.go (69%) rename {cmd => tools}/simulator/run/profile.go (100%) create mode 100644 tools/simulator/run/runbook.go rename {cmd => tools}/simulator/simulate/chain/block.go (80%) rename {cmd => tools}/simulator/simulate/chain/broadcaster.go (85%) rename {cmd => tools}/simulator/simulate/chain/broadcaster_test.go (88%) create mode 100644 tools/simulator/simulate/chain/generate.go rename {cmd/simulator/simulate/upkeep => tools/simulator/simulate/chain}/generate_test.go (59%) rename {cmd => tools}/simulator/simulate/chain/history.go (97%) rename {cmd => tools}/simulator/simulate/chain/history_test.go (83%) rename {cmd => tools}/simulator/simulate/chain/listener.go (100%) rename {cmd => tools}/simulator/simulate/chain/listener_test.go (81%) rename {cmd => tools}/simulator/simulate/db/ocr3.go (100%) rename {cmd => tools}/simulator/simulate/db/upkeep.go (100%) rename {cmd => tools}/simulator/simulate/hydrator.go (69%) create mode 100644 tools/simulator/simulate/loader/logtrigger.go create mode 100644 tools/simulator/simulate/loader/logtrigger_test.go rename cmd/simulator/simulate/ocr/loader.go => tools/simulator/simulate/loader/ocr3config.go (73%) rename cmd/simulator/simulate/ocr/loader_test.go => tools/simulator/simulate/loader/ocr3config_test.go (70%) rename cmd/simulator/simulate/ocr/transmit.go => tools/simulator/simulate/loader/ocr3transmit.go (53%) rename cmd/simulator/simulate/ocr/transmit_test.go => tools/simulator/simulate/loader/ocr3transmit_test.go (77%) create mode 100644 tools/simulator/simulate/loader/upkeep.go create mode 100644 tools/simulator/simulate/loader/upkeep_test.go rename {cmd => tools}/simulator/simulate/net/network.go (100%) rename {cmd => tools}/simulator/simulate/net/network_test.go (97%) rename {cmd => tools}/simulator/simulate/net/service.go (98%) rename {cmd => tools}/simulator/simulate/net/service_test.go (91%) rename {cmd => tools}/simulator/simulate/ocr/config.go (97%) rename {cmd => tools}/simulator/simulate/ocr/config_test.go (84%) rename {cmd => tools}/simulator/simulate/ocr/report.go (95%) rename {cmd => tools}/simulator/simulate/ocr/report_test.go (81%) create mode 100644 tools/simulator/simulate/ocr/transmit.go create mode 100644 tools/simulator/simulate/ocr/transmit_test.go rename {cmd => tools}/simulator/simulate/upkeep/active.go (94%) rename {cmd => tools}/simulator/simulate/upkeep/active_test.go (61%) rename {cmd => tools}/simulator/simulate/upkeep/log.go (98%) rename {cmd => tools}/simulator/simulate/upkeep/log_test.go (86%) rename {cmd => tools}/simulator/simulate/upkeep/perform.go (94%) rename {cmd => tools}/simulator/simulate/upkeep/perform_test.go (87%) rename {cmd => tools}/simulator/simulate/upkeep/pipeline.go (91%) rename {cmd => tools}/simulator/simulate/upkeep/pipeline_test.go (89%) rename {cmd => tools}/simulator/simulate/upkeep/source.go (97%) rename {cmd => tools}/simulator/simulate/upkeep/source_test.go (87%) rename {cmd => tools}/simulator/simulate/upkeep/util.go (94%) rename {cmd => tools}/simulator/simulate/upkeep/util_test.go (85%) rename {cmd => tools}/simulator/telemetry/base.go (100%) rename {cmd => tools}/simulator/telemetry/contract.go (100%) rename {cmd => tools}/simulator/telemetry/log.go (100%) create mode 100644 tools/simulator/telemetry/progress.go rename {cmd => tools}/simulator/telemetry/rpc.go (100%) rename {cmd => tools}/simulator/util/encode.go (100%) rename {cmd => tools}/simulator/util/rand.go (100%) rename {cmd => tools}/simulator/util/sort.go (100%) diff --git a/.gitignore b/.gitignore index 197d2357..f2cc42fd 100644 --- a/.gitignore +++ b/.gitignore @@ -29,5 +29,5 @@ bin/ # default simulation output folders reports/ -runbook_logs/ -private_runbooks/ \ No newline at end of file +simulation_plan_logs/ +simulation_plans/ \ No newline at end of file diff --git a/SIMULATOR.md b/SIMULATOR.md index 61d8357d..2ab26707 100644 --- a/SIMULATOR.md +++ b/SIMULATOR.md @@ -14,11 +14,13 @@ as p2p network latency, block production, upkeep schedules, and more. ## Profiling -Start the service in one terminal window and run the pprof tool in another. For more information on pprof, view some docs [here](https://github.com/google/pprof/blob/main/doc/README.md) to get started. +Start the service in one terminal window and run the pprof tool in another. For +more information on pprof, view some docs +[here](https://github.com/google/pprof/blob/main/doc/README.md) to get started. ``` # terminal 1 -$ ./bin/simulator --pprof --simulate -f ./simulation_runbooks/runbook_eth_goerli_mild.json +$ ./bin/simulator --pprof --simulate -f ./tools/simulator/plans/simplan_fast_check.json # terminal 2 $ go tool pprof -top http://localhost:6060/debug/pprof/heap @@ -42,126 +44,285 @@ p2p and RPC. The charts are provided by an HTTP endpoint on localhost. *Example* ``` -$ ./bin/simulator --simulate -f ./simulation_runbooks/runbook_eth_goerli_mild.json +$ ./bin/simulator --simulate -f ./tools/simulator/plans/simplan_fast_check.json ``` *Options* -- `--simulation-file | -f [string]`: default ./runbook.json, path to JSON file defining simulation parameters -- `--output-directory | -o [string]`: default ./runbook_logs, path to output directory where run logs are written +- `--simulation-file | -f [string]`: default ./simulation_plan.json, path to JSON file defining simulation parameters +- `--output-directory | -o [string]`: default ./simulation_logs, path to output directory where run logs are written - `--simulate [bool]`: default false, run simulation and output results - `--charts [bool]`: default false, start and run charts service to display results - `--pprof [bool]`: default false, run pprof server on simulation startup - `--pprof-port [int]`: default 6060, port to serve pprof profiler on -### Runbook Options +## Simulation Plan -A runbook is a set of configurations for the simulator defined in a JSON file. -Each property is described below. +A simulation plan is a set of configurations for the simulator defined in a JSON file and is designed to produce a +consistent simulation between runs. Be aware that there is some variance in simulation outcomes from the same simulation +plan due to randomness in the `rpc`, `blocks`, or `p2pNetwork` configurations. Events are precise relative to block +production. -*nodes* +### Node + +The instantiation of a node involves creating simulated dependencies and providing them to the delegate constructor for +the plugin. + +*node* +[object] + +Configuration values that apply to setting up nodes in the network. + +*node.totalNodeCount* [int] -Total number of nodes to run in the simulation. Each node is connected via a -simulated p2p network and provided an isolated contract/RPC simulation. +Total number of nodes to run in the simulation. Each node is connected via a simulated p2p network and provided an +isolated contract/RPC simulation. -*maxNodeServiceWorkers* +*node.maxNodeServiceWorkers* [int] -Service workers are used to parallelize RPC calls to be able to process more in -a short time. This number is to set the upper limit on the number of service -workers per simulated node. +Service workers are used to parallelize RPC calls to be able to process more in a short time. This number is to set the +upper limit on the number of service workers per simulated node. -*maxNodeServiceQueueSize* +*node.maxNodeServiceQueueSize* [int] -Max queue size for sending work to service workers. This should be deprecated -soon. +Max queue size for sending work to service workers. This should be deprecated soon. + +### P2PNetwork + +The p2p network simulation does not include any tcp/udp networking layers and only serves the perpose of inter-node +communication. The simulated network can be configured to simulate nodes operating on the same hardware or in close +proximity or configured to simulate nodes spread across a large physical distance. A `maxLatency` of `300ms` might +simulate the physical distance of nodes operating in Paris and Singapore, for example. + +*p2pNetwork* +[object] + +Configure the simulated p2p network. -*avgNetworkLatency* +*p2pNetwork.maxLatency* [int] -The total amount of time a message should take to be sent in the simulated p2p -network. This is an average and is calculated by taking a random number between -0 and the defined latency. +The maximum amount of time a message should take to be sent in the simulated p2p network. This is calculated by taking +a random number between 0 and the provided latency. + +### RPC + +A simulated RPC is the connection layer between the block producer and the node. In a real-world environment, the block +production can be imagined as a singular source and RPCs independently read the state of block production. In this way, +each RPC can have a different 'view' of the singular block source. -*rpcDetail* +Each node gets an isolated instance of a simulated RPC. The role the RPC plays is to surface changes made to the +singular block source such as new upkeeps being created, ocr configs being committed, or logs being emitted. + +*rpc* [object] -This object is a container for RPC related configurations. There is currently a -limit of a single RPC simulation configuration and applies to all instances. +Configure the behavior of the simulated RPC. There is currently a limit of a single RPC simulation configuration and +applies to all instances. -*rpcDetail.maxBlockDelay* +*rpc.maxBlockDelay* [int] The maximum delay in in milliseconds that an RPC would deliver a new block. -*rpcDetail.averageLatency* +*rpc.averageLatency* [int] -The average response latency of a simulated RPC call. All latency calculations -have a baseline of 50 milliseconds with an added latency calculated as a -binomial distribution of the configuration where `N = conf * 2` and `P = 0.4`. +The average response latency of a simulated RPC call. All latency calculations have a baseline of 50 milliseconds with +an added latency calculated as a binomial distribution of the configuration where `N = conf * 2` and `P = 0.4`. -*rpcDetail.errorRate* +*rpc.errorRate* [float] -The probability that an RPC call will return an error. `0.02` is `2%` +The probability that an RPC call will return an error. `0.02` is `2%`. RPC providers are essentially cloud services that +have potential failures. Use this configuration to simulate a flaky RPC provider. -*rpcDetail.rateLimitThreshold* +*rpc.rateLimitThreshold* [int] -Total number of calls per second before returning a rate limit response from the -simulated RPC provider. +Total number of calls per second before returning a rate limit response from the simulated RPC provider. + +### Blocks -*blockDetail* +Simulated blocks in the context of the simulator are only containers of events that apply to the network of nodes and +that are provided to the network on a defined cadence. The concept of signatures, and hashes doesn't apply. Where the +term `hash` is used, the value is likely either a randomly generated value or a value derived from some block data. + +*blocks* [object] -Configuration object for simulated chain. The chain is a coordinated block -producer that feeds each simulated RPC by a dedicated channel. +Configuration object for simulated chain. The chain is a coordinated block producer that feeds each simulated RPC by a +dedicated channel. Each simulated RPC can receive blocks at different times. -*blockDetail.genesisBlock* +*blocks.genesisBlock* [int] -The block number for the first simulated block. Formatted as time. +The block number for the first simulated block. -*blockDetail.blockCadence* +*blocks.blockCadence* [string] The rate at which new blocks are created. Formatted as time. -*blockDetail.blockCadenceJitter* +*blocks.blockCadenceJitter* [string] -Some chains produce blocks on a well defined cadence. Most do not. This -parameter allows some jitter to be applied to the block cadence. +Block cadenece jitter is applied to block production such that each block is not produced exactly on the cadence. -*blockDetail.durationInBlocks* +*blocks.durationInBlocks* [int] -A simulation only runs for this defined number of blocks. The configured upkeeps -are applied within this range. +A simulation only runs for this defined number of blocks. The configured upkeeps are applied within this range. -*blockDetail.endPadding* +*blocks.endPadding* [int] -The simulated chain continues to broadcast blocks for the end padding duration -to allow all performs to have time to be completed. The configured upkeeps do -not apply to this block set. +The simulated chain continues to broadcast blocks for the end padding duration to allow all performs to have time to be +completed. The configured upkeeps do not apply to this block set. + +### Events + +All events are applied to blocks as they are produced. Each event contains at least a `type` that describes how to +process the event and a `eventBlockNumber` which defines the block in which to apply the event. Many events are +singular. Some events are generative in that multiple events are generated from a single configuration. + +Generative events, such as `generateUpkeeps`, allows a more collapsed JSON config. The specific case of generating +upkeeps will create `count` upkeep create events for the same `eventBlockNumber`. -*configEvents* +*events* [array[object]] -Config events change the state of the network and at least 1 is required to -start the network configuration. Each event is broadcast by the simulated chain -at the block defined. +Config events change the state of the network and at least 1 is required to start the network configuration. Each event +is broadcast by the simulated chain at the block defined. + +Every event has some common properties: + +*type* +[string:required] + +Determines the event type. Options include `ocr3config`, `generateUpkeeps`, `logTrigger` + +*eventBlockNumber* +[int:required] + +Block number to commit this event to block history. + +*comment* +[string:optional] + +Optional reference value. Not output on logs. + + +#### OCR 3 Config + +- type: `ocr3config` + +An event with the type `ocr3config` indicates that a new network configuration was committed to the block history. In a +real-world scenario, this would be an on-chain transaction that emits a log. In the simulation, the event is a specific +type and is recognized by the simulated RPC. + +TODO: describe OCR config values and how they are provided + +*encodedOffchainConfig* +[string] + +The encoded config does not currently enforce an encoding type. An OCR off-chain config is an array of bytes allowing +the encoding to be anything. This configuration property should be the value already encoded as the plugin expects. In +the case of JSON, include character escaping such as `{\"version\":\"v3\"}`. + +#### Generate Upkeeps + +- type: `generateUpkeeps` -*configEvents.triggerBlockNumber* +Multiple upkeep events can be generated by using this event type. An upkeep event simulates an upkeep being added to a +registry and made active. The only states relevant to the simulator regarding registered upkeeps are: is it active, and +is it eligible? + +An upkeep becomes active on the `eventBlockNumber` where it is committed to the block history. From that point, +eligibility begins to apply, which is defined by the `eligibilityFunc`. No other events will apply to an upkeep until +after the upkeep becomes active in the block history, which includes `logTrigger` type events. + +The `eligibilityFunc` allows a basic linear function to be supplied indicating when, relative to the trigger block, an +upkeep should be eligible. + +Example: + +func: `2x + 1` +active at block: `100` +final block: `500` + +``` +let start = 100; +let end = 500; +let i = 0; +let next = 0; + +let eligible = []; + +while next < end { + if next > start { + eligible.push(next); + } + + let y = (2 * i) + 1; + + next = start + Math.round(y); + + i++; +} + +// the eligibility function makes the upkeep eligible every 2 blocks with a relative +// offset of 1 to the start block +// eligible: [101, 103, 105, 107, ...] +``` + +The `offsetFunc` advances the `start` point for each generated upkeep to ensure eligibility doesn't overlap for each +generated upkeep. + +Special options such as `always` and `never` are also available. + +*count* +[int] + +Total number of upkeeps to generate for the event configuration. + +*startID* [int] -The block to broadcast the event on. This block should be after the genesis -block and before the final simulation block. +ID to reference the upkeep in the config. The UpkeepID output will be different. + +*eligibilityFunc* +[string] + +Simple linear equation for generating eligibility. Also allowed are `always` and `never`. + +*offsetFunc* +[string] + +Simple linear equation for eligibility start offset. Allowed to be empty when `eligibilityFunc` is `always` or `never`. + +*upkeepType* +[string] + +Options are `conditional` and `logTrigger`. + +*logTriggeredBy* +[string] + +This value applies to a log trigger type upkeep and is the reference point for a `logTrigger` event. If this value +matches the `triggerValue` of a `logTrigger` event, this upkeep is 'triggered' by the log trigger event. + +#### Log Events + +- type: `logTrigger` + +A simulated log event does only simulates the existence of a real-world log and the value for matching to an upkeep +trigger. Once a log event is active in the block history, it can 'trigger' any active and eligible `logTrigger` type +upkeeps with a matching `triggerValue`. -*configEvents.offchainConfigJSON* +*triggerValue* [string] -Stringified JSON for off-chain configuration. \ No newline at end of file +Value used to 'trigger' upkeeps. \ No newline at end of file diff --git a/cmd/simulator/config/runbook.go b/cmd/simulator/config/runbook.go deleted file mode 100644 index d58317b7..00000000 --- a/cmd/simulator/config/runbook.go +++ /dev/null @@ -1,112 +0,0 @@ -package config - -import ( - "encoding/json" - "math/big" - - "github.com/smartcontractkit/libocr/offchainreporting2plus/types" -) - -// TODO: change the name of this to SimulationPlan -type RunBook struct { - Nodes int `json:"nodes"` - MaxServiceWorkers int `json:"maxNodeServiceWorkers"` - MaxQueueSize int `json:"maxNodeServiceQueueSize"` - AvgNetworkLatency Duration `json:"avgNetworkLatency"` - RPCDetail RPC `json:"rpcDetail"` - BlockCadence Blocks `json:"blockDetail"` - ConfigEvents []ConfigEvent `json:"configEvents"` - Upkeeps []Upkeep `json:"upkeeps"` -} - -func (rb RunBook) Encode() ([]byte, error) { - return json.Marshal(rb) -} - -func LoadRunBook(b []byte) (RunBook, error) { - var rb RunBook - - err := json.Unmarshal(b, &rb) - if err != nil { - return rb, err - } - - return rb, nil -} - -type Blocks struct { - Genesis *big.Int `json:"genesisBlock"` - Cadence Duration `json:"blockCadence"` - // Jitter is the average amount of variance applied to the cadence - Jitter Duration `json:"blockCadenceJitter"` - // Duration is the number of blocks to simulate before blocks should stop - // broadcasting - Duration int `json:"durationInBlocks"` - // EndPadding is the number of blocks to add to the end of the process to - // allow all transmits to close up for the simulated test - EndPadding int `json:"endPadding"` -} - -type RPC struct { - // MaxBlockDelay is the maximum amount of time in ms that a block would take - // to be viewed by the node - MaxBlockDelay int `json:"maxBlockDelay"` - // AverageLatency is the average amount of time in ms that an RPC network - // call can take - AverageLatency int `json:"averageLatency"` - // ErrorRate is the chance that any RPC call will return an error. This helps - // simulated heavily loaded RPC servers. - ErrorRate float64 `json:"errorRate"` - // RateLimitThreshold is the point at which rate limiting occurs for RPC calls. - // this limit is calls per second - RateLimitThreshold int `json:"rateLimitThreshold"` -} - -// ConfigEvent is an event that indicates a new config should be broadcast -type ConfigEvent struct { - // Block is the block number where this event is triggered - Block *big.Int `json:"triggerBlockNumber"` - // F is the configurable faulty number of nodes - F int `json:"maxFaultyNodes"` - // Offchain is the json encoded off chain config data - Offchain string `json:"offchainConfigJSON"` - // Rmax is the maximum number of rounds in an epoch - Rmax uint64 `json:"maxRoundsPerEpoch"` - // DeltaProgress is the OCR setting for round leader progress before forcing - // a new epoch and leader - DeltaProgress Duration `json:"deltaProgress"` - // DeltaResend ... - DeltaResend Duration `json:"deltaResend"` - // DeltaInitial ... - DeltaInitial Duration `json:"deltaInitial"` - // DeltaRound is the approximate time a round should complete in - DeltaRound Duration `json:"deltaRound"` - // DeltaGrace ... - DeltaGrace Duration `json:"deltaGrace"` - // DeltaRequest ... - DeltaRequest Duration `json:"deltaCertifiedCommitRequest"` - // DeltaStage is the time OCR waits before attempting a followup transmit - DeltaStage Duration `json:"deltaStage"` - // MaxQuery ... - MaxQuery Duration `json:"maxQueryTime"` - // MaxObservation is the maximum amount of time to provide observation to complete - MaxObservation Duration `json:"maxObservationTime"` - // MaxAccept ... - MaxAccept Duration `json:"maxShouldAcceptTime"` - // MaxTransmit ... - MaxTransmit Duration `json:"maxShouldTransmitTime"` -} - -type Upkeep struct { - Count int `json:"count"` - StartID *big.Int `json:"startID"` - GenerateFunc string `json:"generateFunc"` - OffsetFunc string `json:"offsetFunc"` -} - -type SymBlock struct { - BlockNumber *big.Int - TransmittedData [][]byte - LatestEpoch *uint32 - Config *types.ContractConfig -} diff --git a/cmd/simulator/config/runbook_test.go b/cmd/simulator/config/runbook_test.go deleted file mode 100644 index 37c284b8..00000000 --- a/cmd/simulator/config/runbook_test.go +++ /dev/null @@ -1,32 +0,0 @@ -package config - -import ( - "math/big" - "testing" - "time" - - "github.com/stretchr/testify/assert" -) - -func TestRunbook_Encode(t *testing.T) { - rb := RunBook{ - Nodes: 4, - MaxServiceWorkers: 10, - MaxQueueSize: 1000, - AvgNetworkLatency: Duration(300 * time.Millisecond), - RPCDetail: RPC{}, - BlockCadence: Blocks{ - Genesis: big.NewInt(3), - Cadence: Duration(1 * time.Second), - Jitter: Duration(200 * time.Millisecond), - Duration: 20, - EndPadding: 20, - }, - ConfigEvents: []ConfigEvent{}, - Upkeeps: []Upkeep{}, - } - - _, err := rb.Encode() - - assert.NoError(t, err, "no error expected from encoding the runbook") -} diff --git a/cmd/simulator/main.go b/cmd/simulator/main.go index c0437e56..d4210adc 100644 --- a/cmd/simulator/main.go +++ b/cmd/simulator/main.go @@ -3,6 +3,7 @@ package main import ( "context" "errors" + "fmt" "log" "math/big" "net" @@ -15,16 +16,16 @@ import ( "github.com/smartcontractkit/libocr/offchainreporting2plus/chains/evmutil" flag "github.com/spf13/pflag" - "github.com/smartcontractkit/ocr2keepers/cmd/simulator/config" - "github.com/smartcontractkit/ocr2keepers/cmd/simulator/node" - "github.com/smartcontractkit/ocr2keepers/cmd/simulator/run" - "github.com/smartcontractkit/ocr2keepers/cmd/simulator/simulate/upkeep" - "github.com/smartcontractkit/ocr2keepers/cmd/simulator/telemetry" + "github.com/smartcontractkit/ocr2keepers/tools/simulator/config" + "github.com/smartcontractkit/ocr2keepers/tools/simulator/node" + "github.com/smartcontractkit/ocr2keepers/tools/simulator/run" + "github.com/smartcontractkit/ocr2keepers/tools/simulator/simulate/chain" + "github.com/smartcontractkit/ocr2keepers/tools/simulator/telemetry" ) var ( - simulationFile = flag.StringP("simulation-file", "f", "./runbook.json", "file path to read simulation config from") - outputDirectory = flag.StringP("output-directory", "o", "./runbook_logs", "directory path to output log files") + simulationFile = flag.StringP("simulation-file", "f", "./simulation_plan.json", "file path to read simulation config from") + outputDirectory = flag.StringP("output-directory", "o", "./simulation_plan_logs", "directory path to output log files") simulate = flag.Bool("simulate", false, "run simulation") serveCharts = flag.Bool("charts", false, "create and serve charts") profiler = flag.Bool("pprof", false, "run pprof server on startup") @@ -45,30 +46,28 @@ func main() { }, procLog) // ----- read simulation file - procLog.Println("loading simulation assets ...") - rb, err := run.LoadRunBook(*simulationFile) + plan, err := run.LoadSimulationPlan(*simulationFile) if err != nil { - procLog.Printf("failed to initialize runbook: %s", err) + procLog.Printf("failed to initialize simulation plan: %s", err) os.Exit(1) } // ----- setup simulation output directory and file handles - outputs, err := run.SetupOutput(*outputDirectory, *simulate, rb) + outputs, err := run.SetupOutput(*outputDirectory, *simulate, plan) if err != nil { procLog.Printf("failed to setup output directory: %s", err) os.Exit(1) } - // ----- create simulated upkeeps from runbook - procLog.Println("generating simulated upkeeps ...") - upkeeps, err := upkeep.GenerateConditionals(rb) + // ----- create simulated upkeeps from simulation plan + upkeeps, err := chain.GenerateAllUpkeeps(plan) if err != nil { procLog.Printf("failed to generate simulated upkeeps: %s", err) os.Exit(1) } ngConf := node.GroupConfig{ - Runbook: rb, + SimulationPlan: plan, // Digester is a generic offchain digester Digester: evmutil.EVMOffchainConfigDigester{ ChainID: 1, @@ -83,23 +82,33 @@ func main() { Logger: outputs.SimulationLog, } - ng := node.NewGroup(ngConf) + fmt.Printf("Starting simulation ...\n\n") + + progress := telemetry.NewProgressTelemetry(os.Stdout) + progress.Start() + + ng, err := node.NewGroup(ngConf, progress) + if err != nil { + procLog.Printf("failed to create node group: %s", err) + os.Exit(1) + } + ctx, cancel := contextWithInterrupt(context.Background()) var wg sync.WaitGroup if *simulate { - procLog.Println("starting simulation") - wg.Add(1) - go func(ct context.Context, b config.RunBook, logger *log.Logger) { - if err := ng.Start(ct, b.Nodes, b.MaxServiceWorkers, b.MaxQueueSize); err != nil { - logger.Printf("%s", err) + go func(serviceCtx context.Context, simPlan config.SimulationPlan, logger *log.Logger) { + if err := ng.Start(serviceCtx, simPlan.Node); err != nil { + logger.Printf("node group closed with error: %s", err) } - logger.Println("simulation complete") + if err := progress.Close(); err != nil { + logger.Printf("failed to close progress tracker: %s", err) + } wg.Done() - }(ctx, rb, procLog) + }(ctx, plan, procLog) } if *serveCharts { @@ -154,4 +163,11 @@ func main() { } wg.Wait() + + fmt.Printf("\nSimulation done") + + if !progress.AllProgressComplete() { + fmt.Println("\nsimulation failed") + os.Exit(1) + } } diff --git a/cmd/simulator/node/group.go b/cmd/simulator/node/group.go deleted file mode 100644 index a9576a0f..00000000 --- a/cmd/simulator/node/group.go +++ /dev/null @@ -1,78 +0,0 @@ -package node - -import ( - "io" - "log" - - "github.com/smartcontractkit/libocr/commontypes" - "github.com/smartcontractkit/libocr/offchainreporting2/types" - - "github.com/smartcontractkit/ocr2keepers/cmd/simulator/config" - simio "github.com/smartcontractkit/ocr2keepers/cmd/simulator/io" - "github.com/smartcontractkit/ocr2keepers/cmd/simulator/simulate/chain" - "github.com/smartcontractkit/ocr2keepers/cmd/simulator/simulate/net" - "github.com/smartcontractkit/ocr2keepers/cmd/simulator/simulate/ocr" - "github.com/smartcontractkit/ocr2keepers/cmd/simulator/simulate/upkeep" - "github.com/smartcontractkit/ocr2keepers/cmd/simulator/telemetry" -) - -type GroupConfig struct { - Runbook config.RunBook - Digester types.OffchainConfigDigester - Upkeeps []chain.SimulatedUpkeep - Collectors []telemetry.Collector - Logger *log.Logger -} - -type Group struct { - conf config.RunBook - nodes map[string]*Simulator - network *net.SimulatedNetwork - digester types.OffchainConfigDigester - blockSrc *chain.BlockBroadcaster - transmitter *ocr.OCR3TransmitLoader - confLoader *ocr.OCR3ConfigLoader - upkeeps []chain.SimulatedUpkeep - monitor commontypes.MonitoringEndpoint - collectors []telemetry.Collector - logger *log.Logger -} - -func NewGroup(conf GroupConfig) *Group { - // TODO: monitor data is not text so not sure what to do with this yet - monitor := simio.NewMonitorToWriter(io.Discard) - - lTransmit := ocr.NewOCR3TransmitLoader(conf.Runbook, conf.Logger) - lOCR3Config := ocr.NewOCR3ConfigLoader(conf.Runbook, conf.Digester, conf.Logger) - - lUpkeep, err := upkeep.NewUpkeepConfigLoader(conf.Runbook) - if err != nil { - panic(err) - } - - lLogTriggers, err := upkeep.NewLogTriggerLoader(conf.Runbook) - if err != nil { - panic(err) - } - - loaders := []chain.BlockLoaderFunc{ - lTransmit.Load, - lOCR3Config.Load, - lUpkeep.Load, - lLogTriggers.Load, - } - - return &Group{ - conf: conf.Runbook, - nodes: make(map[string]*Simulator), - network: net.NewSimulatedNetwork(conf.Runbook.AvgNetworkLatency.Value()), - digester: conf.Digester, - blockSrc: chain.NewBlockBroadcaster(conf.Runbook.BlockCadence, conf.Runbook.RPCDetail.MaxBlockDelay, conf.Logger, loaders...), - transmitter: lTransmit, - confLoader: lOCR3Config, - upkeeps: conf.Upkeeps, - monitor: monitor, - collectors: conf.Collectors, - logger: conf.Logger, - } -} diff --git a/cmd/simulator/run/runbook.go b/cmd/simulator/run/runbook.go deleted file mode 100644 index 1ac1569d..00000000 --- a/cmd/simulator/run/runbook.go +++ /dev/null @@ -1,16 +0,0 @@ -package run - -import ( - "os" - - "github.com/smartcontractkit/ocr2keepers/cmd/simulator/config" -) - -func LoadRunBook(path string) (config.RunBook, error) { - data, err := os.ReadFile(path) - if err != nil { - return config.RunBook{}, err - } - - return config.LoadRunBook(data) -} diff --git a/cmd/simulator/simulate/upkeep/generate.go b/cmd/simulator/simulate/upkeep/generate.go deleted file mode 100644 index b368961b..00000000 --- a/cmd/simulator/simulate/upkeep/generate.go +++ /dev/null @@ -1,161 +0,0 @@ -package upkeep - -import ( - "crypto/sha256" - "fmt" - "math/big" - - "github.com/Maldris/mathparse" - "github.com/shopspring/decimal" - - "github.com/smartcontractkit/ocr2keepers/cmd/simulator/config" - "github.com/smartcontractkit/ocr2keepers/cmd/simulator/simulate/chain" - ocr2keepers "github.com/smartcontractkit/ocr2keepers/pkg/v3/types" -) - -func GenerateConditionals(rb config.RunBook) ([]chain.SimulatedUpkeep, error) { - generated := []chain.SimulatedUpkeep{} - limit := new(big.Int).Add(rb.BlockCadence.Genesis, big.NewInt(int64(rb.BlockCadence.Duration))) - - for _, upkeep := range rb.Upkeeps { - p := mathparse.NewParser(upkeep.OffsetFunc) - p.Resolve() - - for y := 1; y <= upkeep.Count; y++ { - id := new(big.Int).Add(upkeep.StartID, big.NewInt(int64(y))) - sym := chain.SimulatedUpkeep{ - ID: id, - UpkeepID: newUpkeepID(id.Bytes(), uint8(ocr2keepers.ConditionTrigger)), - EligibleAt: make([]*big.Int, 0), - } - - var genesis *big.Int - if p.FoundResult() { - // create upkeep at id == result - genesis = big.NewInt(int64(p.GetValueResult())) - } else { - // create upkeep genesis relative to upkeep count - g, err := calcFromTokens(p.GetTokens(), big.NewInt(int64(y))) - if err != nil { - return nil, err - } - - genesis = new(big.Int).Add(rb.BlockCadence.Genesis, g.BigInt()) - } - - if err := generateEligibles(&sym, genesis, limit, upkeep.GenerateFunc); err != nil { - return nil, err - } - - generated = append(generated, sym) - } - } - - return generated, nil -} - -// TODO: complete this -func GenerateLogTriggeredUpkeeps(rb config.RunBook) ([]chain.SimulatedUpkeep, error) { - - return nil, nil -} - -// TODO: complete this -func GenerateLogTriggers(rb config.RunBook) ([]chain.SimulatedLog, error) { - - return nil, nil -} - -func operate(a, b decimal.Decimal, op string) decimal.Decimal { - switch op { - case "+": - return a.Add(b) - case "*": - return a.Mul(b) - case "-": - return a.Sub(b) - default: - } - - return decimal.Zero -} - -func generateEligibles(upkeep *chain.SimulatedUpkeep, genesis *big.Int, limit *big.Int, f string) error { - p := mathparse.NewParser(f) - p.Resolve() - - if p.FoundResult() { - return fmt.Errorf("simple value unsupported") - } else { - // create upkeep from offset function - var i int64 = 0 - nextValue := big.NewInt(0) - tokens := p.GetTokens() - - for nextValue.Cmp(limit) < 0 { - if nextValue.Cmp(genesis) >= 0 { - upkeep.EligibleAt = append(upkeep.EligibleAt, nextValue) - } - - value, err := calcFromTokens(tokens, big.NewInt(i)) - if err != nil { - return err - } - - biValue := value.Round(0).BigInt() - nextValue = new(big.Int).Add(genesis, biValue) - i++ - } - } - - return nil -} - -func calcFromTokens(tokens []mathparse.Token, x *big.Int) (decimal.Decimal, error) { - value := decimal.NewFromInt(0) - action := "+" - - for i := 0; i < len(tokens); i++ { - token := tokens[i] - - switch token.Type { - case 2, 3: - var tVal decimal.Decimal - - if token.Value == "x" { - tVal = decimal.NewFromBigInt(x, int32(0)) - } else { - tVal = decimal.NewFromFloat(token.ParseValue) - } - - value = operate(value, tVal, action) - case 4: - action = token.Value - // case 1, 5, 6, 7, 8: - // log.Printf("unused token: %s", token.Value) - default: - } - } - - return value, nil -} - -func newUpkeepID(entropy []byte, uType uint8) [32]byte { - /* - Following the contract convention, an identifier is composed of 32 bytes: - - - 4 bytes of entropy - - 11 bytes of zeros - - 1 identifying byte for the trigger type - - 16 bytes of entropy - */ - hashedValue := sha256.Sum256(entropy) - - for x := 4; x < 15; x++ { - hashedValue[x] = uint8(0) - } - - hashedValue[15] = uType - - return hashedValue -} diff --git a/cmd/simulator/simulate/upkeep/loader.go b/cmd/simulator/simulate/upkeep/loader.go deleted file mode 100644 index 02fad46e..00000000 --- a/cmd/simulator/simulate/upkeep/loader.go +++ /dev/null @@ -1,107 +0,0 @@ -package upkeep - -import ( - "sync" - - "github.com/smartcontractkit/ocr2keepers/cmd/simulator/config" - "github.com/smartcontractkit/ocr2keepers/cmd/simulator/simulate/chain" -) - -// UpkeepConfigLoader provides upkeep configurations to a block broadcaster. Use -// this loader to introduce upkeeps or change upkeep configs at specific block -// numbers. -type UpkeepConfigLoader struct { - mu sync.RWMutex - conditionals []chain.SimulatedUpkeep - events map[string][]interface{} -} - -// NewUpkeepConfigLoader ... -func NewUpkeepConfigLoader(rb config.RunBook) (*UpkeepConfigLoader, error) { - // generate conditionals - conditionals, err := GenerateConditionals(rb) - if err != nil { - return nil, err - } - - logTriggered, err := GenerateLogTriggeredUpkeeps(rb) - if err != nil { - return nil, err - } - - allUpkeeps := append(conditionals, logTriggered...) - - // TODO: create more event types (create, cancel, pause, etc) - // the only currently supported type is create and will create on the genesis - // block - events := make(map[string][]interface{}) - for _, upkeep := range allUpkeeps { - evts, ok := events[rb.BlockCadence.Genesis.String()] - if !ok { - evts = []interface{}{} - } - - events[rb.BlockCadence.Genesis.String()] = append(evts, chain.UpkeepCreatedTransaction{ - Upkeep: upkeep, - }) - } - - return &UpkeepConfigLoader{ - conditionals: conditionals, - events: events, - }, nil -} - -// Load implements the chain.BlockLoaderFunc type and loads configured upkeep -// events into blocks. -func (ucl *UpkeepConfigLoader) Load(block *chain.Block) { - ucl.mu.RLock() - defer ucl.mu.RUnlock() - - if events, ok := ucl.events[block.Number.String()]; ok { - block.Transactions = append(block.Transactions, events...) - } -} - -// LogTriggerLoader ... -type LogTriggerLoader struct { - mu sync.RWMutex - triggers map[string][]interface{} -} - -// NewLogTriggerLoader ... -func NewLogTriggerLoader(rb config.RunBook) (*LogTriggerLoader, error) { - logs, err := GenerateLogTriggers(rb) - if err != nil { - return nil, err - } - - events := make(map[string][]interface{}) - for _, logEvt := range logs { - for _, trigger := range logEvt.TriggerAt { - existing, ok := events[trigger.String()] - if !ok { - existing = []interface{}{} - } - - events[trigger.String()] = append(existing, chain.Log{ - TriggerValue: logEvt.TriggerValue, - }) - } - } - - return &LogTriggerLoader{ - triggers: events, - }, nil -} - -// Load implements the chain.BlockLoaderFunc type and loads log trigger events -// into blocks -func (ltl *LogTriggerLoader) Load(block *chain.Block) { - ltl.mu.RLock() - defer ltl.mu.RUnlock() - - if events, ok := ltl.triggers[block.Number.String()]; ok { - block.Transactions = append(block.Transactions, events...) - } -} diff --git a/cmd/simulator/simulate/upkeep/loader_test.go b/cmd/simulator/simulate/upkeep/loader_test.go deleted file mode 100644 index 17a448d1..00000000 --- a/cmd/simulator/simulate/upkeep/loader_test.go +++ /dev/null @@ -1,45 +0,0 @@ -package upkeep_test - -import ( - "math/big" - "testing" - "time" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - - "github.com/smartcontractkit/ocr2keepers/cmd/simulator/config" - "github.com/smartcontractkit/ocr2keepers/cmd/simulator/simulate/chain" - "github.com/smartcontractkit/ocr2keepers/cmd/simulator/simulate/upkeep" -) - -func TestUpkeepConfigLoader(t *testing.T) { - runbook := config.RunBook{ - BlockCadence: config.Blocks{ - Genesis: big.NewInt(1), - Cadence: config.Duration(time.Second), - Duration: 10, - }, - Upkeeps: []config.Upkeep{ - { - Count: 1, - StartID: big.NewInt(1), - GenerateFunc: "2x", - OffsetFunc: "x", - }, - }, - } - - loader, err := upkeep.NewUpkeepConfigLoader(runbook) - - require.NoError(t, err) - - block := chain.Block{ - Number: runbook.BlockCadence.Genesis, - Transactions: []interface{}{}, - } - - loader.Load(&block) - - assert.Len(t, block.Transactions, 1) -} diff --git a/simulation_runbooks/v2/runbook_arbitrum_mild.json b/simulation_runbooks/v2/runbook_arbitrum_mild.json deleted file mode 100644 index 0cff840b..00000000 --- a/simulation_runbooks/v2/runbook_arbitrum_mild.json +++ /dev/null @@ -1,53 +0,0 @@ -{ - "nodes": 8, - "maxNodeServiceWorkers": 100, - "maxNodeServiceQueueSize": 1000, - "avgNetworkLatency": "100ms", - "rpcDetail": { - "maxBlockDelay": 2000, - "averageLatency": 300, - "errorRate": 0.02, - "rateLimitThreshold": 1000 - }, - "blockDetail": { - "genesisBlock": 128943862, - "blockCadence": "1s", - "blockCadenceJitter": "100ms", - "durationInBlocks": 300, - "endPadding": 20 - }, - "configEvents": [ - { - "triggerBlockNumber": 128943863, - "maxFaultyNodes": 2, - "offchainConfigJSON": "{\"targetProbability\":\"0.999\",\"targetInRounds\":2,\"uniqueReports\":false,\"gasLimitPerReport\":1000000,\"gasOverheadPerUpkeep\":300000,\"maxUpkeepBatchSize\":10}", - "maxRoundsPerEpoch": 7, - "deltaProgress": "10s", - "deltaResend": "10s", - "deltaRound": "2500ms", - "deltaGrace": "500ms", - "deltaStage": "20s", - "maxQueryTime": "50ms", - "maxObservationTime": "1200ms", - "maxReportTime": "800ms", - "maxShouldAcceptTime": "50ms", - "maxShouldTransmitTime": "50ms" - } - ], - "upkeeps": [ - { - "_comment": "upkeeps that have no performs", - "count": 900, - "startID": 1000, - "generateFunc": "x + 1000", - "offsetFunc": "x" - }, - { - "_comment": "~3 performs per upkeep spaced at every 7 blocks", - "count": 30, - "startID": 200, - "generateFunc": "100x", - "offsetFunc": "7x + 2" - } - ] -} \ No newline at end of file diff --git a/simulation_runbooks/v2/runbook_eth_goerli_mild.json b/simulation_runbooks/v2/runbook_eth_goerli_mild.json deleted file mode 100644 index 013806a3..00000000 --- a/simulation_runbooks/v2/runbook_eth_goerli_mild.json +++ /dev/null @@ -1,80 +0,0 @@ -{ - "nodes": 8, - "maxNodeServiceWorkers": 100, - "maxNodeServiceQueueSize": 1000, - "avgNetworkLatency": "100ms", - "rpcDetail": { - "maxBlockDelay": 2000, - "averageLatency": 300, - "errorRate": 0.02, - "rateLimitThreshold": 1000 - }, - "blockDetail": { - "genesisBlock": 128943862, - "blockCadence": "12s", - "durationInBlocks": 50, - "endPadding": 5 - }, - "configEvents": [ - { - "triggerBlockNumber": 128943863, - "maxFaultyNodes": 2, - "offchainConfigJSON": "{\"targetProbability\":\"0.999\",\"targetInRounds\":4,\"uniqueReports\":false,\"gasLimitPerReport\":1000000,\"gasOverheadPerUpkeep\":300000,\"maxUpkeepBatchSize\":10}", - "maxRoundsPerEpoch": 7, - "deltaProgress": "10s", - "deltaResend": "10s", - "deltaRound": "2500ms", - "deltaGrace": "500ms", - "deltaStage": "20s", - "maxQueryTime": "50ms", - "maxObservationTime": "1200ms", - "maxReportTime": "800ms", - "maxShouldAcceptTime": "50ms", - "maxShouldTransmitTime": "50ms" - } - ], - "upkeeps": [ - { - "_comment": "upkeeps that have no performs", - "count": 800, - "startID": 1000, - "generateFunc": "x + 1000", - "offsetFunc": "x" - }, - { - "_comment": "2 performs per upkeep", - "count": 50, - "startID": 200, - "generateFunc": "50x - 25", - "offsetFunc": "2x + 1" - }, - { - "_comment": "2 performs per upkeep; offset from previous", - "count": 50, - "startID": 400, - "generateFunc": "50x - 20", - "offsetFunc": "2x + 1" - }, - { - "_comment": "2 performs per upkeep; offset from previous", - "count": 50, - "startID": 600, - "generateFunc": "50x - 15", - "offsetFunc": "2x + 1" - }, - { - "_comment": "2 performs per upkeep; offset from previous", - "count": 50, - "startID": 600, - "generateFunc": "50x - 10", - "offsetFunc": "2x + 1" - }, - { - "_comment": "2 performs per upkeep; offset from previous", - "count": 50, - "startID": 600, - "generateFunc": "50x - 5", - "offsetFunc": "2x + 1" - } - ] -} \ No newline at end of file diff --git a/simulation_runbooks/v2/runbook_matic_mild.json b/simulation_runbooks/v2/runbook_matic_mild.json deleted file mode 100644 index 96215a76..00000000 --- a/simulation_runbooks/v2/runbook_matic_mild.json +++ /dev/null @@ -1,53 +0,0 @@ -{ - "nodes": 8, - "maxNodeServiceWorkers": 100, - "maxNodeServiceQueueSize": 1000, - "avgNetworkLatency": "100ms", - "rpcDetail": { - "maxBlockDelay": 2000, - "averageLatency": 300, - "errorRate": 0.02, - "rateLimitThreshold": 1000 - }, - "blockDetail": { - "genesisBlock": 128943862, - "blockCadence": "3s", - "blockCadenceJitter": "200ms", - "durationInBlocks": 100, - "endPadding": 20 - }, - "configEvents": [ - { - "triggerBlockNumber": 128943863, - "maxFaultyNodes": 2, - "offchainConfigJSON": "{\"targetProbability\":\"0.999\",\"targetInRounds\":2,\"uniqueReports\":false,\"gasLimitPerReport\":1000000,\"gasOverheadPerUpkeep\":300000,\"maxUpkeepBatchSize\":10}", - "maxRoundsPerEpoch": 7, - "deltaProgress": "5s", - "deltaResend": "10s", - "deltaRound": "2100ms", - "deltaGrace": "50ms", - "deltaStage": "20s", - "maxQueryTime": "20ms", - "maxObservationTime": "1200ms", - "maxReportTime": "700ms", - "maxShouldAcceptTime": "20ms", - "maxShouldTransmitTime": "20ms" - } - ], - "upkeeps": [ - { - "_comment": "upkeeps that have no performs", - "count": 980, - "startID": 1000, - "generateFunc": "x + 1000", - "offsetFunc": "x" - }, - { - "_comment": "2 performs per upkeep", - "count": 30, - "startID": 200, - "generateFunc": "50x - 25", - "offsetFunc": "4x + 2" - } - ] -} \ No newline at end of file diff --git a/simulation_runbooks/v3/runbook_eth_goerli_mild.json b/simulation_runbooks/v3/runbook_eth_goerli_mild.json deleted file mode 100644 index 013806a3..00000000 --- a/simulation_runbooks/v3/runbook_eth_goerli_mild.json +++ /dev/null @@ -1,80 +0,0 @@ -{ - "nodes": 8, - "maxNodeServiceWorkers": 100, - "maxNodeServiceQueueSize": 1000, - "avgNetworkLatency": "100ms", - "rpcDetail": { - "maxBlockDelay": 2000, - "averageLatency": 300, - "errorRate": 0.02, - "rateLimitThreshold": 1000 - }, - "blockDetail": { - "genesisBlock": 128943862, - "blockCadence": "12s", - "durationInBlocks": 50, - "endPadding": 5 - }, - "configEvents": [ - { - "triggerBlockNumber": 128943863, - "maxFaultyNodes": 2, - "offchainConfigJSON": "{\"targetProbability\":\"0.999\",\"targetInRounds\":4,\"uniqueReports\":false,\"gasLimitPerReport\":1000000,\"gasOverheadPerUpkeep\":300000,\"maxUpkeepBatchSize\":10}", - "maxRoundsPerEpoch": 7, - "deltaProgress": "10s", - "deltaResend": "10s", - "deltaRound": "2500ms", - "deltaGrace": "500ms", - "deltaStage": "20s", - "maxQueryTime": "50ms", - "maxObservationTime": "1200ms", - "maxReportTime": "800ms", - "maxShouldAcceptTime": "50ms", - "maxShouldTransmitTime": "50ms" - } - ], - "upkeeps": [ - { - "_comment": "upkeeps that have no performs", - "count": 800, - "startID": 1000, - "generateFunc": "x + 1000", - "offsetFunc": "x" - }, - { - "_comment": "2 performs per upkeep", - "count": 50, - "startID": 200, - "generateFunc": "50x - 25", - "offsetFunc": "2x + 1" - }, - { - "_comment": "2 performs per upkeep; offset from previous", - "count": 50, - "startID": 400, - "generateFunc": "50x - 20", - "offsetFunc": "2x + 1" - }, - { - "_comment": "2 performs per upkeep; offset from previous", - "count": 50, - "startID": 600, - "generateFunc": "50x - 15", - "offsetFunc": "2x + 1" - }, - { - "_comment": "2 performs per upkeep; offset from previous", - "count": 50, - "startID": 600, - "generateFunc": "50x - 10", - "offsetFunc": "2x + 1" - }, - { - "_comment": "2 performs per upkeep; offset from previous", - "count": 50, - "startID": 600, - "generateFunc": "50x - 5", - "offsetFunc": "2x + 1" - } - ] -} \ No newline at end of file diff --git a/simulation_runbooks/v3/runbook_fast_check.json b/simulation_runbooks/v3/runbook_fast_check.json deleted file mode 100644 index 0ee66f12..00000000 --- a/simulation_runbooks/v3/runbook_fast_check.json +++ /dev/null @@ -1,46 +0,0 @@ -{ - "nodes": 4, - "maxNodeServiceWorkers": 100, - "maxNodeServiceQueueSize": 1000, - "avgNetworkLatency": "100ms", - "rpcDetail": { - "maxBlockDelay": 600, - "averageLatency": 300, - "errorRate": 0.02, - "rateLimitThreshold": 1000 - }, - "blockDetail": { - "genesisBlock": 128943862, - "blockCadence": "1s", - "durationInBlocks": 60, - "endPadding": 20 - }, - "configEvents": [ - { - "triggerBlockNumber": 128943863, - "maxFaultyNodes": 1, - "offchainConfigJSON": "{\"version\":\"v3\",\"performLockoutWindow\":100000,\"targetProbability\":\"0.999\",\"targetInRounds\":4,\"minConfirmations\":1,\"gasLimitPerReport\":1000000,\"gasOverheadPerUpkeep\":300000,\"maxUpkeepBatchSize\":10}", - "maxRoundsPerEpoch": 7, - "deltaProgress": "10s", - "deltaResend": "10s", - "deltaInitial": "300ms", - "deltaRound": "1100ms", - "deltaGrace": "300ms", - "deltaCertifiedCommitRequest": "200ms", - "deltaStage": "20s", - "maxQueryTime": "50ms", - "maxObservationTime": "100ms", - "maxShouldAcceptTime": "50ms", - "maxShouldTransmitTime": "50ms" - } - ], - "upkeeps": [ - { - "_comment": "~3 performs per upkeep", - "count": 10, - "startID": 200, - "generateFunc": "30x - 15", - "offsetFunc": "2x + 1" - } - ] -} \ No newline at end of file diff --git a/cmd/simulator/config/duration.go b/tools/simulator/config/duration.go similarity index 100% rename from cmd/simulator/config/duration.go rename to tools/simulator/config/duration.go diff --git a/cmd/simulator/config/duration_test.go b/tools/simulator/config/duration_test.go similarity index 100% rename from cmd/simulator/config/duration_test.go rename to tools/simulator/config/duration_test.go diff --git a/tools/simulator/config/event.go b/tools/simulator/config/event.go new file mode 100644 index 00000000..84b4a221 --- /dev/null +++ b/tools/simulator/config/event.go @@ -0,0 +1,91 @@ +package config + +import "math/big" + +type EventType string + +const ( + OCR3ConfigEventType EventType = "ocr3config" + GenerateUpkeepEventType EventType = "generateUpkeeps" + LogTriggerEventType EventType = "logTrigger" +) + +type Event struct { + Type EventType `json:"type"` + TriggerBlock *big.Int `json:"eventBlockNumber"` + Comment string `json:"comment,omitempty"` +} + +type UpkeepType string + +const ( + ConditionalUpkeepType UpkeepType = "conditional" + LogTriggerUpkeepType UpkeepType = "logTrigger" +) + +// OCR3ConfigEvent is an event that indicates a new OCR config should be +// broadcast. Consult libOCR for descriptions of each config var. +type OCR3ConfigEvent struct { + Event + // MaxFaultyNodesF is the configurable faulty number of nodes + MaxFaultyNodesF int `json:"maxFaultyNodes"` + // Offchain is the encoded off chain config data. Typically this is JSON + // encoded for the CLAutomation plugin. + Offchain string `json:"encodedOffchainConfig"` + // Rmax is the maximum number of rounds in an epoch + Rmax uint64 `json:"maxRoundsPerEpoch"` + // DeltaProgress is the OCR setting for round leader progress before forcing + // a new epoch and leader + DeltaProgress Duration `json:"deltaProgress"` + DeltaResend Duration `json:"deltaResend"` + DeltaInitial Duration `json:"deltaInitial"` + // DeltaRound is the approximate time a round should complete in + DeltaRound Duration `json:"deltaRound"` + DeltaGrace Duration `json:"deltaGrace"` + DeltaRequest Duration `json:"deltaCertifiedCommitRequest"` + // DeltaStage is the time OCR waits before attempting a followup transmit + DeltaStage Duration `json:"deltaStage"` + MaxQuery Duration `json:"maxQueryTime"` + // MaxObservation is the maximum amount of time to provide observation to complete + MaxObservation Duration `json:"maxObservationTime"` + MaxAccept Duration `json:"maxShouldAcceptTime"` + MaxTransmit Duration `json:"maxShouldTransmitTime"` +} + +// GenerateUpkeepEvent is a configuration for creating upkeeps in bulk. +type GenerateUpkeepEvent struct { + Event + // Count is the total number of upkeeps to create for this event. + Count int `json:"count"` + // StartID is the numeric id on which to begin incrementing for the upkeep + // id. Conflicting ids with multiple generate events will result in a + // config error. + StartID *big.Int `json:"startID"` + // EligibilityFunc is a basic linear function for which to indicate + // eligibility. This can be seen as the cadence on which each upkeep becomes + // eligible. The values 'always' and 'never' are also valid. Empty is + // assumed to be 'never'. Using 'always' for conditional upkeeps is invalid. + EligibilityFunc string `json:"eligibilityFunc,omitempty"` + // OffsetFunc is a basic linear function that determines the block reference + // on which to apply the eligibility function. Each generated upkeep can + // follow the same eligibility function, but start at different blocks + // determined by the offset function. For eligibility 'always' or 'never' it + // is preferable to leave this field empty. + OffsetFunc string `json:"offsetFunc,omitempty"` + // UpkeepType defines whether the generated upkeeps will be configured as + // conditional or log trigger upkeeps. + UpkeepType UpkeepType `json:"upkeepType"` + // LogTriggeredBy is the log value on which to trigger the set of generated + // upkeeps. Only applies to log trigger type upkeeps. An empty value for a + // log triggered upkeep will result in the upkeep never being triggered. + LogTriggeredBy string `json:"logTriggeredBy,omitempty"` +} + +// LogTriggerEvent is a configuration for simulating logs emitted from a chain +// source. Each log originates in a specified block and is defined by the +// trigger value. +type LogTriggerEvent struct { + Event + // TriggerValue corresponds to log trigger upkeeps with 'LogTriggeredBy' set. + TriggerValue string `json:"triggerValue"` +} diff --git a/cmd/simulator/config/keyring.go b/tools/simulator/config/keyring.go similarity index 62% rename from cmd/simulator/config/keyring.go rename to tools/simulator/config/keyring.go index d2f047c3..a2a5d2e6 100644 --- a/cmd/simulator/config/keyring.go +++ b/tools/simulator/config/keyring.go @@ -54,14 +54,14 @@ func NewOffchainKeyring(encryptionMaterial, signingMaterial io.Reader) (*Offchai } // OffchainSign signs message using private key -func (keyring *OffchainKeyring) OffchainSign(msg []byte) (signature []byte, err error) { - return ed25519.Sign(ed25519.PrivateKey(keyring.signingKey), msg), nil +func (k *OffchainKeyring) OffchainSign(msg []byte) (signature []byte, err error) { + return ed25519.Sign(ed25519.PrivateKey(k.signingKey), msg), nil } // ConfigDiffieHellman returns the shared point obtained by multiplying someone's // public key by a secret scalar ( in this case, the offchain key ring's encryption key.) -func (keyring *OffchainKeyring) ConfigDiffieHellman(point [curve25519.PointSize]byte) ([curve25519.PointSize]byte, error) { - p, err := curve25519.X25519(keyring.encryptionKey[:], point[:]) +func (k *OffchainKeyring) ConfigDiffieHellman(point [curve25519.PointSize]byte) ([curve25519.PointSize]byte, error) { + p, err := curve25519.X25519(k.encryptionKey[:], point[:]) if err != nil { return [curve25519.PointSize]byte{}, err } @@ -71,20 +71,20 @@ func (keyring *OffchainKeyring) ConfigDiffieHellman(point [curve25519.PointSize] } // OffchainPublicKey returns the public component of this offchain keyring. -func (keyring *OffchainKeyring) OffchainPublicKey() types.OffchainPublicKey { +func (k *OffchainKeyring) OffchainPublicKey() types.OffchainPublicKey { var offchainPubKey [ed25519.PublicKeySize]byte - copy(offchainPubKey[:], keyring.signingKey.Public().(ed25519.PublicKey)[:]) + copy(offchainPubKey[:], k.signingKey.Public().(ed25519.PublicKey)[:]) return offchainPubKey } // ConfigEncryptionPublicKey returns config public key -func (keyring *OffchainKeyring) ConfigEncryptionPublicKey() types.ConfigEncryptionPublicKey { - cpk, _ := keyring.configEncryptionPublicKey() +func (k *OffchainKeyring) ConfigEncryptionPublicKey() types.ConfigEncryptionPublicKey { + cpk, _ := k.configEncryptionPublicKey() return cpk } -func (keyring *OffchainKeyring) configEncryptionPublicKey() (types.ConfigEncryptionPublicKey, error) { - rv, err := curve25519.X25519(keyring.encryptionKey[:], curve25519.Basepoint) +func (k *OffchainKeyring) configEncryptionPublicKey() (types.ConfigEncryptionPublicKey, error) { + rv, err := curve25519.X25519(k.encryptionKey[:], curve25519.Basepoint) if err != nil { return [curve25519.PointSize]byte{}, err } @@ -110,16 +110,16 @@ func NewEVMKeyring(material io.Reader) (*EvmKeyring, error) { } // PublicKey returns the address of the public key not the public key itself -func (keyring *EvmKeyring) PublicKey() types.OnchainPublicKey { - return keyring.signingAddress().Bytes() +func (k *EvmKeyring) PublicKey() types.OnchainPublicKey { + return k.signingAddress().Bytes() } // XXX: PublicKey returns the address of the public key not the public key itself -func (keyring *EvmKeyring) PKString() string { - return keyring.signingAddress().String() +func (k *EvmKeyring) PKString() string { + return k.signingAddress().String() } -func (keyring *EvmKeyring) reportToSigData(digest types.ConfigDigest, v uint64, r ocr3types.ReportWithInfo[plugin.AutomationReportInfo]) []byte { +func (k *EvmKeyring) reportToSigData(digest types.ConfigDigest, v uint64, r ocr3types.ReportWithInfo[plugin.AutomationReportInfo]) []byte { rawRepctx := [3][32]byte{} // first is the digest @@ -136,12 +136,12 @@ func (keyring *EvmKeyring) reportToSigData(digest types.ConfigDigest, v uint64, return crypto.Keccak256(sigData) } -func (keyring *EvmKeyring) Sign(digest types.ConfigDigest, v uint64, r ocr3types.ReportWithInfo[plugin.AutomationReportInfo]) ([]byte, error) { - return crypto.Sign(keyring.reportToSigData(digest, v, r), &keyring.privateKey) +func (k *EvmKeyring) Sign(digest types.ConfigDigest, v uint64, r ocr3types.ReportWithInfo[plugin.AutomationReportInfo]) ([]byte, error) { + return crypto.Sign(k.reportToSigData(digest, v, r), &k.privateKey) } -func (keyring *EvmKeyring) Verify(publicKey types.OnchainPublicKey, digest types.ConfigDigest, v uint64, r ocr3types.ReportWithInfo[plugin.AutomationReportInfo], signature []byte) bool { - hash := keyring.reportToSigData(digest, v, r) +func (k *EvmKeyring) Verify(publicKey types.OnchainPublicKey, digest types.ConfigDigest, v uint64, r ocr3types.ReportWithInfo[plugin.AutomationReportInfo], signature []byte) bool { + hash := k.reportToSigData(digest, v, r) authorPubkey, err := crypto.SigToPub(hash, signature) if err != nil { return false @@ -150,25 +150,25 @@ func (keyring *EvmKeyring) Verify(publicKey types.OnchainPublicKey, digest types return bytes.Equal(publicKey[:], authorAddress[:]) } -func (keyring *EvmKeyring) MaxSignatureLength() int { +func (k *EvmKeyring) MaxSignatureLength() int { return 65 } -func (keyring *EvmKeyring) signingAddress() common.Address { - return crypto.PubkeyToAddress(*(&keyring.privateKey).Public().(*ecdsa.PublicKey)) +func (k *EvmKeyring) signingAddress() common.Address { + return crypto.PubkeyToAddress(*(&k.privateKey).Public().(*ecdsa.PublicKey)) } -func (keyring *EvmKeyring) Marshal() ([]byte, error) { - return crypto.FromECDSA(&keyring.privateKey), nil +func (k *EvmKeyring) Marshal() ([]byte, error) { + return crypto.FromECDSA(&k.privateKey), nil } -func (keyring *EvmKeyring) Unmarshal(in []byte) error { +func (k *EvmKeyring) Unmarshal(in []byte) error { privateKey, err := crypto.ToECDSA(in) if err != nil { return err } - keyring.privateKey = *privateKey + k.privateKey = *privateKey return nil } diff --git a/cmd/simulator/config/keyring_test.go b/tools/simulator/config/keyring_test.go similarity index 97% rename from cmd/simulator/config/keyring_test.go rename to tools/simulator/config/keyring_test.go index 927897fc..b3f9a21e 100644 --- a/cmd/simulator/config/keyring_test.go +++ b/tools/simulator/config/keyring_test.go @@ -7,8 +7,8 @@ import ( "testing" "github.com/smartcontractkit/libocr/offchainreporting2plus/ocr3types" - "github.com/smartcontractkit/ocr2keepers/cmd/simulator/config" "github.com/smartcontractkit/ocr2keepers/pkg/v3/plugin" + "github.com/smartcontractkit/ocr2keepers/tools/simulator/config" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "golang.org/x/crypto/curve25519" diff --git a/tools/simulator/config/simulation.go b/tools/simulator/config/simulation.go new file mode 100644 index 00000000..c6f423d5 --- /dev/null +++ b/tools/simulator/config/simulation.go @@ -0,0 +1,184 @@ +package config + +import ( + "encoding/json" + "fmt" + "math/big" + + "github.com/smartcontractkit/libocr/offchainreporting2plus/types" +) + +var ( + ErrEncoding = fmt.Errorf("encoding/decoding failure") +) + +// SimulationPlan is a collection of configurations with which to run a +// simulation. +type SimulationPlan struct { + Node Node `json:"node"` + Network Network `json:"p2pNetwork"` + RPC RPC `json:"rpc"` + Blocks Blocks `json:"blocks"` + ConfigEvents []OCR3ConfigEvent `json:"-"` + GenerateUpkeeps []GenerateUpkeepEvent `json:"-"` + LogEvents []LogTriggerEvent `json:"-"` +} + +// Encode applies JSON encoding of a simulation plan to bytes. +func (p SimulationPlan) Encode() ([]byte, error) { + type encodedOutput struct { + SimulationPlan + Events []interface{} `json:"events"` + } + + encodable := encodedOutput{ + SimulationPlan: p, + Events: make([]interface{}, len(p.ConfigEvents)+len(p.GenerateUpkeeps)), + } + + for _, event := range p.ConfigEvents { + // ensure the type is set properly + event.Type = OCR3ConfigEventType + encodable.Events = append(encodable.Events, event) + } + + for _, event := range p.GenerateUpkeeps { + // ensure the type is set properly + event.Type = GenerateUpkeepEventType + encodable.Events = append(encodable.Events, event) + } + + for _, event := range p.LogEvents { + // ensure the type is set properly + event.Type = LogTriggerEventType + encodable.Events = append(encodable.Events, event) + } + + return json.Marshal(encodable) +} + +// DecodeSimulationPlan uses JSON encoding to decode bytes to a simulation plan. +func DecodeSimulationPlan(encoded []byte) (SimulationPlan, error) { + var plan SimulationPlan + + if err := json.Unmarshal(encoded, &plan); err != nil { + return plan, fmt.Errorf("%w: failed to decode simulation plan: %s", ErrEncoding, err.Error()) + } + + plan.ConfigEvents = make([]OCR3ConfigEvent, 0) + plan.GenerateUpkeeps = make([]GenerateUpkeepEvent, 0) + plan.LogEvents = make([]LogTriggerEvent, 0) + + type eventCollection struct { + Events []json.RawMessage `json:"events"` + } + + var events eventCollection + + if err := json.Unmarshal(encoded, &events); err != nil { + return plan, fmt.Errorf("%w: failed to decode events in simulation plan: %s", ErrEncoding, err.Error()) + } + + for idx, rawEvent := range events.Events { + var event Event + if err := json.Unmarshal(rawEvent, &event); err != nil { + return plan, fmt.Errorf("%w: failed to decode event in simulation plan: %s", ErrEncoding, err.Error()) + } + + switch event.Type { + case OCR3ConfigEventType: + var configEvent OCR3ConfigEvent + if err := json.Unmarshal(rawEvent, &configEvent); err != nil { + return plan, fmt.Errorf("%w: failed to decode ocr3config event in simulation plan at index %d: %s", ErrEncoding, idx, err.Error()) + } + + plan.ConfigEvents = append(plan.ConfigEvents, configEvent) + case GenerateUpkeepEventType: + var generateEvent GenerateUpkeepEvent + if err := json.Unmarshal(rawEvent, &generateEvent); err != nil { + return plan, fmt.Errorf("%w: failed to decode generateUpkeep event in simulation plan at index %d: %s", ErrEncoding, idx, err.Error()) + } + + plan.GenerateUpkeeps = append(plan.GenerateUpkeeps, generateEvent) + case LogTriggerEventType: + var logEvent LogTriggerEvent + if err := json.Unmarshal(rawEvent, &logEvent); err != nil { + return plan, fmt.Errorf("%w: failed to decode logTrigger event in simulation plan at index %d: %s", ErrEncoding, idx, err.Error()) + } + + plan.LogEvents = append(plan.LogEvents, logEvent) + default: + return plan, fmt.Errorf("%w: unrecognized event at index %d", ErrEncoding, idx) + } + } + + return plan, nil +} + +// Node is a configuration that applies to the simulated nodes. +type Node struct { + // Count defines the total number of nodes added in the simulation. + Count int `json:"totalNodeCount"` + // MaxServiceWorkers is a configuration on the total number of go-routines + // allowed to each node for running parallel pipeline calls. + MaxServiceWorkers int `json:"maxNodeServiceWorkers"` + // MaxQueueSize limits the queue size for incoming check pipeline requests. + MaxQueueSize int `json:"maxNodeServiceQueueSize"` +} + +// Network is a configuration for the simulated p2p network between simulated +// nodes. +type Network struct { + // MaxLatency applies to the amout of time a message takes to be sent + // between peers. This is intended to simulate delay due to physical + // distance between nodes or other network delays. + MaxLatency Duration `json:"maxLatency"` +} + +// RPC is a configuration for a simulated RPC client. Each node recieves their +// own simulated rpc client which allows the configured values to be applied +// independently. +type RPC struct { + // MaxBlockDelay is the maximum amount of time in ms that a block would take + // to be viewed by the node + MaxBlockDelay int `json:"maxBlockDelay"` + // AverageLatency is the average amount of time in ms that an RPC network + // call can take + AverageLatency int `json:"averageLatency"` + // ErrorRate is the chance that any RPC call will return an error. This helps + // simulated heavily loaded RPC servers. + ErrorRate float64 `json:"errorRate"` + // RateLimitThreshold is the point at which rate limiting occurs for RPC calls. + // this limit is calls per second + RateLimitThreshold int `json:"rateLimitThreshold"` +} + +// Blocks is a configuration for simulated block production. +type Blocks struct { + // Genesis is the starting block number. + Genesis *big.Int `json:"genesisBlock"` + // Cadence is how fast blocks are produced. + Cadence Duration `json:"blockCadence"` + // Jitter is the average amount of variance applied to the cadence + Jitter Duration `json:"blockCadenceJitter"` + // Duration is the number of blocks to simulate before blocks should stop + // broadcasting + Duration int `json:"durationInBlocks"` + // EndPadding is the number of blocks to add to the end of the process to + // allow all transmits to close up for the simulated test + EndPadding int `json:"endPadding"` +} + +type Upkeep struct { + Count int `json:"count"` + StartID *big.Int `json:"startID"` + GenerateFunc string `json:"generateFunc"` + OffsetFunc string `json:"offsetFunc"` +} + +type SymBlock struct { + BlockNumber *big.Int + TransmittedData [][]byte + LatestEpoch *uint32 + Config *types.ContractConfig +} diff --git a/tools/simulator/config/simulation_test.go b/tools/simulator/config/simulation_test.go new file mode 100644 index 00000000..8cf1fc9d --- /dev/null +++ b/tools/simulator/config/simulation_test.go @@ -0,0 +1,44 @@ +package config + +import ( + "math/big" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestSimulationPlan_EncodeDecode(t *testing.T) { + plan := SimulationPlan{ + Node: Node{ + Count: 4, + MaxServiceWorkers: 10, + MaxQueueSize: 1000, + }, + Network: Network{ + MaxLatency: Duration(300 * time.Millisecond), + }, + RPC: RPC{}, + Blocks: Blocks{ + Genesis: big.NewInt(3), + Cadence: Duration(1 * time.Second), + Jitter: Duration(200 * time.Millisecond), + Duration: 20, + EndPadding: 20, + }, + ConfigEvents: []OCR3ConfigEvent{}, + GenerateUpkeeps: []GenerateUpkeepEvent{}, + LogEvents: []LogTriggerEvent{}, + } + + encoded, err := plan.Encode() + + require.NoError(t, err, "no error expected from encoding the simulation plan") + + decodedPlan, err := DecodeSimulationPlan(encoded) + + require.NoError(t, err, "no error expected from decoding the simulation plan") + + assert.Equal(t, plan, decodedPlan, "simulation plan should match after encoding and decoding") +} diff --git a/cmd/simulator/io/log.go b/tools/simulator/io/log.go similarity index 100% rename from cmd/simulator/io/log.go rename to tools/simulator/io/log.go diff --git a/cmd/simulator/io/monitor.go b/tools/simulator/io/monitor.go similarity index 100% rename from cmd/simulator/io/monitor.go rename to tools/simulator/io/monitor.go diff --git a/cmd/simulator/node/active.go b/tools/simulator/node/active.go similarity index 100% rename from cmd/simulator/node/active.go rename to tools/simulator/node/active.go diff --git a/cmd/simulator/node/add.go b/tools/simulator/node/add.go similarity index 84% rename from cmd/simulator/node/add.go rename to tools/simulator/node/add.go index 919df28d..5e6a28e5 100644 --- a/cmd/simulator/node/add.go +++ b/tools/simulator/node/add.go @@ -11,17 +11,17 @@ import ( "github.com/smartcontractkit/libocr/commontypes" "github.com/smartcontractkit/libocr/offchainreporting2plus/types" - simio "github.com/smartcontractkit/ocr2keepers/cmd/simulator/io" pluginconfig "github.com/smartcontractkit/ocr2keepers/pkg/v3/config" "github.com/smartcontractkit/ocr2keepers/pkg/v3/plugin" "github.com/smartcontractkit/ocr2keepers/pkg/v3/runner" + simio "github.com/smartcontractkit/ocr2keepers/tools/simulator/io" - "github.com/smartcontractkit/ocr2keepers/cmd/simulator/config" - "github.com/smartcontractkit/ocr2keepers/cmd/simulator/simulate" - "github.com/smartcontractkit/ocr2keepers/cmd/simulator/telemetry" + "github.com/smartcontractkit/ocr2keepers/tools/simulator/config" + "github.com/smartcontractkit/ocr2keepers/tools/simulator/simulate" + "github.com/smartcontractkit/ocr2keepers/tools/simulator/telemetry" ) -func (g *Group) Add(maxWorkers int, maxQueueSize int) { +func (g *Group) Add(conf config.Node) { simNet := g.network.NewFactory() var rpcTel *telemetry.RPCCollector @@ -85,8 +85,8 @@ func (g *Group) Add(maxWorkers int, maxQueueSize int) { OffchainConfigDigester: g.digester, OffchainKeyring: offchainRing, OnchainKeyring: onchainRing, - MaxServiceWorkers: maxWorkers, - ServiceQueueLength: maxQueueSize, + MaxServiceWorkers: conf.MaxServiceWorkers, + ServiceQueueLength: conf.MaxQueueSize, } _ = simulate.HydrateConfig( @@ -104,8 +104,8 @@ func (g *Group) Add(maxWorkers int, maxQueueSize int) { gLogger, dConfig.Runnable, runner.RunnerConfig{ - Workers: maxWorkers, - WorkerQueueLength: maxQueueSize, + Workers: conf.MaxServiceWorkers, + WorkerQueueLength: conf.MaxQueueSize, CacheExpire: pluginconfig.DefaultCacheExpiration, CacheClean: pluginconfig.DefaultCacheClearInterval, }, @@ -129,17 +129,17 @@ func (g *Group) Add(maxWorkers int, maxQueueSize int) { g.confLoader.AddSigner(simNet.PeerID(), onchainRing, offchainRing) } -func (g *Group) Start(ctx context.Context, count int, maxWorkers int, maxQueueSize int) error { +func (g *Group) Start(ctx context.Context, nodeConfig config.Node) error { var err error - for i := 0; i < count; i++ { - g.Add(maxWorkers, maxQueueSize) + for i := 0; i < nodeConfig.Count; i++ { + g.Add(nodeConfig) } g.logger.Print("starting simulation") select { case <-g.blockSrc.Start(): - err = fmt.Errorf("block duration ended") + err = nil case <-ctx.Done(): g.blockSrc.Stop() err = fmt.Errorf("SIGTERM event stopping process") diff --git a/tools/simulator/node/group.go b/tools/simulator/node/group.go new file mode 100644 index 00000000..4a6482fe --- /dev/null +++ b/tools/simulator/node/group.go @@ -0,0 +1,78 @@ +package node + +import ( + "io" + "log" + + "github.com/smartcontractkit/libocr/commontypes" + "github.com/smartcontractkit/libocr/offchainreporting2/types" + + "github.com/smartcontractkit/ocr2keepers/tools/simulator/config" + simio "github.com/smartcontractkit/ocr2keepers/tools/simulator/io" + "github.com/smartcontractkit/ocr2keepers/tools/simulator/simulate/chain" + "github.com/smartcontractkit/ocr2keepers/tools/simulator/simulate/loader" + "github.com/smartcontractkit/ocr2keepers/tools/simulator/simulate/net" + "github.com/smartcontractkit/ocr2keepers/tools/simulator/telemetry" +) + +type GroupConfig struct { + SimulationPlan config.SimulationPlan + Digester types.OffchainConfigDigester + Upkeeps []chain.SimulatedUpkeep + Collectors []telemetry.Collector + Logger *log.Logger +} + +type Group struct { + conf config.SimulationPlan + nodes map[string]*Simulator + network *net.SimulatedNetwork + digester types.OffchainConfigDigester + blockSrc *chain.BlockBroadcaster + transmitter *loader.OCR3TransmitLoader + confLoader *loader.OCR3ConfigLoader + upkeeps []chain.SimulatedUpkeep + monitor commontypes.MonitoringEndpoint + collectors []telemetry.Collector + logger *log.Logger +} + +func NewGroup(conf GroupConfig, progress *telemetry.ProgressTelemetry) (*Group, error) { + lTransmit, err := loader.NewOCR3TransmitLoader(conf.SimulationPlan, progress, conf.Logger) + if err != nil { + return nil, err + } + + lOCR3Config := loader.NewOCR3ConfigLoader(conf.SimulationPlan, progress, conf.Digester, conf.Logger) + + lUpkeep, err := loader.NewUpkeepConfigLoader(conf.SimulationPlan, progress) + if err != nil { + return nil, err + } + + lLogTriggers, err := loader.NewLogTriggerLoader(conf.SimulationPlan, progress) + if err != nil { + return nil, err + } + + loaders := []chain.BlockLoaderFunc{ + lTransmit.Load, + lOCR3Config.Load, + lUpkeep.Load, + lLogTriggers.Load, + } + + return &Group{ + conf: conf.SimulationPlan, + nodes: make(map[string]*Simulator), + network: net.NewSimulatedNetwork(conf.SimulationPlan.Network.MaxLatency.Value()), + digester: conf.Digester, + blockSrc: chain.NewBlockBroadcaster(conf.SimulationPlan.Blocks, conf.SimulationPlan.RPC.MaxBlockDelay, conf.Logger, progress, loaders...), + transmitter: lTransmit, + confLoader: lOCR3Config, + upkeeps: conf.Upkeeps, + monitor: simio.NewMonitorToWriter(io.Discard), // monitor data is not text so not sure what to do with this yet + collectors: conf.Collectors, + logger: conf.Logger, + }, nil +} diff --git a/cmd/simulator/node/report.go b/tools/simulator/node/report.go similarity index 96% rename from cmd/simulator/node/report.go rename to tools/simulator/node/report.go index 2f98fd97..f7f4bacf 100644 --- a/cmd/simulator/node/report.go +++ b/tools/simulator/node/report.go @@ -9,9 +9,9 @@ import ( "github.com/jedib0t/go-pretty/v6/table" - "github.com/smartcontractkit/ocr2keepers/cmd/simulator/simulate/upkeep" - "github.com/smartcontractkit/ocr2keepers/cmd/simulator/telemetry" - "github.com/smartcontractkit/ocr2keepers/cmd/simulator/util" + "github.com/smartcontractkit/ocr2keepers/tools/simulator/simulate/upkeep" + "github.com/smartcontractkit/ocr2keepers/tools/simulator/telemetry" + "github.com/smartcontractkit/ocr2keepers/tools/simulator/util" ) func (g *Group) WriteTransmitChart() { diff --git a/cmd/simulator/node/statistics.go b/tools/simulator/node/statistics.go similarity index 100% rename from cmd/simulator/node/statistics.go rename to tools/simulator/node/statistics.go diff --git a/cmd/simulator/node/stats.go b/tools/simulator/node/stats.go similarity index 98% rename from cmd/simulator/node/stats.go rename to tools/simulator/node/stats.go index 68525abe..659f59a2 100644 --- a/cmd/simulator/node/stats.go +++ b/tools/simulator/node/stats.go @@ -5,8 +5,8 @@ import ( "math/big" "sort" - "github.com/smartcontractkit/ocr2keepers/cmd/simulator/simulate/chain" ocr2keepers "github.com/smartcontractkit/ocr2keepers/pkg/v3/types" + "github.com/smartcontractkit/ocr2keepers/tools/simulator/simulate/chain" ) type upkeepStatsBuilder struct { diff --git a/tools/simulator/plans/only_log_trigger.json b/tools/simulator/plans/only_log_trigger.json new file mode 100644 index 00000000..16abb538 --- /dev/null +++ b/tools/simulator/plans/only_log_trigger.json @@ -0,0 +1,91 @@ +{ + "node": { + "totalNodeCount": 4, + "maxNodeServiceWorkers": 100, + "maxNodeServiceQueueSize": 1000 + }, + "p2pNetwork": { + "maxLatency": "100ms" + }, + "rpc": { + "maxBlockDelay": 600, + "averageLatency": 300, + "errorRate": 0.02, + "rateLimitThreshold": 1000 + }, + "blocks": { + "genesisBlock": 128943862, + "blockCadence": "1s", + "durationInBlocks": 60, + "endPadding": 20 + }, + "events": [ + { + "type": "ocr3config", + "eventBlockNumber": 128943863, + "comment": "initial ocr config (valid)", + "maxFaultyNodes": 1, + "encodedOffchainConfig": "{\"version\":\"v3\",\"performLockoutWindow\":100000,\"targetProbability\":\"0.999\",\"targetInRounds\":4,\"minConfirmations\":1,\"gasLimitPerReport\":1000000,\"gasOverheadPerUpkeep\":300000,\"maxUpkeepBatchSize\":10}", + "maxRoundsPerEpoch": 7, + "deltaProgress": "10s", + "deltaResend": "10s", + "deltaInitial": "300ms", + "deltaRound": "1100ms", + "deltaGrace": "300ms", + "deltaCertifiedCommitRequest": "200ms", + "deltaStage": "20s", + "maxQueryTime": "50ms", + "maxObservationTime": "100ms", + "maxShouldAcceptTime": "50ms", + "maxShouldTransmitTime": "50ms" + }, + { + "type": "generateUpkeeps", + "eventBlockNumber": 128943862, + "comment": "single log triggered upkeep", + "count": 1, + "startID": 300, + "eligibilityFunc": "always", + "upkeepType": "logTrigger", + "logTriggeredBy": "test_trigger_event" + }, + { + "type": "generateUpkeeps", + "eventBlockNumber": 128943864, + "comment": "5 log triggered upkeeps", + "count": 5, + "startID": 400, + "eligibilityFunc": "always", + "upkeepType": "logTrigger", + "logTriggeredBy": "test_trigger_event" + }, + { + "type": "logTrigger", + "eventBlockNumber": 128943872, + "comment": "trigger 10 blocks after trigger upkeep created", + "triggerValue": "test_trigger_event" + }, + { + "type": "generateUpkeeps", + "eventBlockNumber": 128943878, + "comment": "7 log triggered upkeeps", + "count": 7, + "startID": 500, + "eligibilityFunc": "always", + "upkeepType": "logTrigger", + "logTriggeredBy": "test_trigger_event" + }, + { + "type": "logTrigger", + "eventBlockNumber": 128943882, + "comment": "trigger 10 blocks after trigger upkeep created", + "triggerValue": "test_trigger_event" + }, + { + "type": "logTrigger", + "eventBlockNumber": 128943892, + "comment": "trigger 10 blocks after trigger upkeep created", + "triggerValue": "test_trigger_event" + } + ] +} \ No newline at end of file diff --git a/tools/simulator/plans/simplan_fast_check.json b/tools/simulator/plans/simplan_fast_check.json new file mode 100644 index 00000000..7e16be15 --- /dev/null +++ b/tools/simulator/plans/simplan_fast_check.json @@ -0,0 +1,79 @@ +{ + "node": { + "totalNodeCount": 4, + "maxNodeServiceWorkers": 100, + "maxNodeServiceQueueSize": 1000 + }, + "p2pNetwork": { + "maxLatency": "100ms" + }, + "rpc": { + "maxBlockDelay": 600, + "averageLatency": 300, + "errorRate": 0.02, + "rateLimitThreshold": 1000 + }, + "blocks": { + "genesisBlock": 128943862, + "blockCadence": "1s", + "durationInBlocks": 60, + "endPadding": 20 + }, + "events": [ + { + "type": "ocr3config", + "eventBlockNumber": 128943863, + "comment": "initial ocr config (valid)", + "maxFaultyNodes": 1, + "encodedOffchainConfig": "{\"version\":\"v3\",\"performLockoutWindow\":100000,\"targetProbability\":\"0.999\",\"targetInRounds\":4,\"minConfirmations\":1,\"gasLimitPerReport\":1000000,\"gasOverheadPerUpkeep\":300000,\"maxUpkeepBatchSize\":10}", + "maxRoundsPerEpoch": 7, + "deltaProgress": "10s", + "deltaResend": "10s", + "deltaInitial": "300ms", + "deltaRound": "1100ms", + "deltaGrace": "300ms", + "deltaCertifiedCommitRequest": "200ms", + "deltaStage": "20s", + "maxQueryTime": "50ms", + "maxObservationTime": "100ms", + "maxShouldAcceptTime": "50ms", + "maxShouldTransmitTime": "50ms" + }, + { + "type": "generateUpkeeps", + "eventBlockNumber": 128943862, + "comment": "~3 performs per upkeep", + "count": 10, + "startID": 200, + "eligibilityFunc": "30x - 15", + "offsetFunc": "2x + 1", + "upkeepType": "conditional" + }, + { + "type": "generateUpkeeps", + "eventBlockNumber": 128943862, + "comment": "single log triggered upkeep", + "count": 1, + "startID": 300, + "eligibilityFunc": "always", + "upkeepType": "logTrigger", + "logTriggeredBy": "test_trigger_event" + }, + { + "type": "generateUpkeeps", + "eventBlockNumber": 128943882, + "comment": "single log triggered upkeep", + "count": 1, + "startID": 400, + "eligibilityFunc": "never", + "upkeepType": "logTrigger", + "logTriggeredBy": "test_trigger_event" + }, + { + "type": "logTrigger", + "eventBlockNumber": 128943872, + "comment": "trigger 10 blocks after trigger upkeep created", + "triggerValue": "test_trigger_event" + } + ] +} \ No newline at end of file diff --git a/cmd/simulator/run/output.go b/tools/simulator/run/output.go similarity index 69% rename from cmd/simulator/run/output.go rename to tools/simulator/run/output.go index c429ccdf..bf6eb1d4 100644 --- a/cmd/simulator/run/output.go +++ b/tools/simulator/run/output.go @@ -7,8 +7,8 @@ import ( "log" "os" - "github.com/smartcontractkit/ocr2keepers/cmd/simulator/config" - "github.com/smartcontractkit/ocr2keepers/cmd/simulator/telemetry" + "github.com/smartcontractkit/ocr2keepers/tools/simulator/config" + "github.com/smartcontractkit/ocr2keepers/tools/simulator/telemetry" ) type Outputs struct { @@ -29,7 +29,7 @@ func (out *Outputs) Close() error { return err } -func SetupOutput(path string, simulate bool, runbook config.RunBook) (*Outputs, error) { +func SetupOutput(path string, simulate bool, plan config.SimulationPlan) (*Outputs, error) { var ( logger *log.Logger lggF *os.File @@ -44,14 +44,15 @@ func SetupOutput(path string, simulate bool, runbook config.RunBook) (*Outputs, } // only when running a simulation is the simulation log needed - // if a simulation has already been run, don't write out the current runbook + // if a simulation has already been run, don't write out the current + // simulation plan if simulate { logger, lggF, err = openSimulationLog(path) if err != nil { return nil, err } - if err := saveRunbookToOutput(path, runbook); err != nil { + if err := saveSimulationPlanToOutput(path, plan); err != nil { return nil, err } } @@ -65,29 +66,29 @@ func SetupOutput(path string, simulate bool, runbook config.RunBook) (*Outputs, }, nil } -func saveRunbookToOutput(path string, rb config.RunBook) error { - filename := fmt.Sprintf("%s/runbook.json", path) +func saveSimulationPlanToOutput(path string, plan config.SimulationPlan) error { + filename := fmt.Sprintf("%s/simulation_plan.json", path) flags := os.O_RDWR | os.O_CREATE | os.O_TRUNC f, err := os.OpenFile(filename, flags, 0666) if err != nil { - return fmt.Errorf("failed to open runbook file (%s): %v", filename, err) + return fmt.Errorf("failed to open simulation plan file (%s): %v", filename, err) } defer f.Close() - b, err := rb.Encode() + b, err := plan.Encode() if err != nil { - return fmt.Errorf("failed to encode runbook: %w", err) + return fmt.Errorf("failed to encode simulation_plan: %w", err) } l, err := f.Write(b) if err != nil { - return fmt.Errorf("failed to write encoded runbook to file (%s): %w", filename, err) + return fmt.Errorf("failed to write encoded simulation plan to file (%s): %w", filename, err) } if l != len(b) { - return fmt.Errorf("failed to write encoded runbook to file (%s): not all bytes written", filename) + return fmt.Errorf("failed to write encoded simulation plan to file (%s): not all bytes written", filename) } return nil diff --git a/cmd/simulator/run/profile.go b/tools/simulator/run/profile.go similarity index 100% rename from cmd/simulator/run/profile.go rename to tools/simulator/run/profile.go diff --git a/tools/simulator/run/runbook.go b/tools/simulator/run/runbook.go new file mode 100644 index 00000000..a7b57bf4 --- /dev/null +++ b/tools/simulator/run/runbook.go @@ -0,0 +1,16 @@ +package run + +import ( + "os" + + "github.com/smartcontractkit/ocr2keepers/tools/simulator/config" +) + +func LoadSimulationPlan(path string) (config.SimulationPlan, error) { + data, err := os.ReadFile(path) + if err != nil { + return config.SimulationPlan{}, err + } + + return config.DecodeSimulationPlan(data) +} diff --git a/cmd/simulator/simulate/chain/block.go b/tools/simulator/simulate/chain/block.go similarity index 80% rename from cmd/simulator/simulate/chain/block.go rename to tools/simulator/simulate/chain/block.go index 39984841..10860a77 100644 --- a/cmd/simulator/simulate/chain/block.go +++ b/tools/simulator/simulate/chain/block.go @@ -41,16 +41,18 @@ const ( ) type SimulatedUpkeep struct { - ID *big.Int - UpkeepID [32]byte - Type UpkeepType - EligibleAt []*big.Int - TriggeredBy string - CheckData []byte + ID *big.Int + CreateInBlock *big.Int + UpkeepID [32]byte + Type UpkeepType + AlwaysEligible bool + EligibleAt []*big.Int + TriggeredBy string + CheckData []byte } type SimulatedLog struct { - TriggerAt []*big.Int + TriggerAt *big.Int TriggerValue string } diff --git a/cmd/simulator/simulate/chain/broadcaster.go b/tools/simulator/simulate/chain/broadcaster.go similarity index 85% rename from cmd/simulator/simulate/chain/broadcaster.go rename to tools/simulator/simulate/chain/broadcaster.go index 8c2acc53..34228b83 100644 --- a/cmd/simulator/simulate/chain/broadcaster.go +++ b/tools/simulator/simulate/chain/broadcaster.go @@ -11,15 +11,25 @@ import ( "sync" "time" - "github.com/smartcontractkit/ocr2keepers/cmd/simulator/config" + "github.com/smartcontractkit/ocr2keepers/tools/simulator/config" ) +const ( + progressTelemetryNamespace = "Broadcasting simulated blocks" +) + +type ProgressTelemetry interface { + Register(string, int64) error + Increment(string, int64) +} + type BlockLoaderFunc func(*Block) type BlockBroadcaster struct { // provided dependencies - loaders []BlockLoaderFunc - logger *log.Logger + progress ProgressTelemetry + loaders []BlockLoaderFunc + logger *log.Logger // configuration maxDelay int @@ -40,13 +50,19 @@ type BlockBroadcaster struct { done chan struct{} } -func NewBlockBroadcaster(conf config.Blocks, maxDelay int, logger *log.Logger, loaders ...BlockLoaderFunc) *BlockBroadcaster { +func NewBlockBroadcaster(conf config.Blocks, maxDelay int, logger *log.Logger, progress ProgressTelemetry, loaders ...BlockLoaderFunc) *BlockBroadcaster { limit := new(big.Int).Add(conf.Genesis, big.NewInt(int64(conf.Duration))) // add a block padding to allow all transmits to come through limit = new(big.Int).Add(limit, big.NewInt(int64(conf.EndPadding))) + total := conf.Duration + conf.EndPadding + + if progress != nil { + _ = progress.Register(progressTelemetryNamespace, int64(total)) + } return &BlockBroadcaster{ + progress: progress, loaders: loaders, logger: log.New(logger.Writer(), "[block-broadcaster] ", log.Ldate|log.Ltime|log.Lshortfile), maxDelay: maxDelay, @@ -168,6 +184,10 @@ func (bb *BlockBroadcaster) broadcast() { msg.Hash = sha256.Sum256(bts.Bytes()) + if bb.progress != nil { + bb.progress.Increment(progressTelemetryNamespace, 1) + } + for sub, chSub := range bb.subscriptions { go func(subID int, ch chan Block, delay bool, logger *log.Logger) { defer func() { diff --git a/cmd/simulator/simulate/chain/broadcaster_test.go b/tools/simulator/simulate/chain/broadcaster_test.go similarity index 88% rename from cmd/simulator/simulate/chain/broadcaster_test.go rename to tools/simulator/simulate/chain/broadcaster_test.go index 90f4fa24..b6a630a0 100644 --- a/cmd/simulator/simulate/chain/broadcaster_test.go +++ b/tools/simulator/simulate/chain/broadcaster_test.go @@ -9,8 +9,8 @@ import ( "github.com/stretchr/testify/assert" - "github.com/smartcontractkit/ocr2keepers/cmd/simulator/config" - "github.com/smartcontractkit/ocr2keepers/cmd/simulator/simulate/chain" + "github.com/smartcontractkit/ocr2keepers/tools/simulator/config" + "github.com/smartcontractkit/ocr2keepers/tools/simulator/simulate/chain" ) func TestBlockBroadcaster(t *testing.T) { @@ -26,7 +26,7 @@ func TestBlockBroadcaster(t *testing.T) { maxDelay := 10 logger := log.New(io.Discard, "", 0) loader := new(mockBlockLoader) - broadcaster := chain.NewBlockBroadcaster(conf, maxDelay, logger, loader.Load) + broadcaster := chain.NewBlockBroadcaster(conf, maxDelay, logger, nil, loader.Load) sub1ID, chBlocks1 := broadcaster.Subscribe(true) sub2ID, chBlocks2 := broadcaster.Subscribe(true) @@ -54,7 +54,7 @@ func TestBlockBroadcaster_Close_After_Limit(t *testing.T) { } maxDelay := 10 logger := log.New(io.Discard, "", 0) - broadcaster := chain.NewBlockBroadcaster(conf, maxDelay, logger) + broadcaster := chain.NewBlockBroadcaster(conf, maxDelay, logger, nil) closed := broadcaster.Start() diff --git a/tools/simulator/simulate/chain/generate.go b/tools/simulator/simulate/chain/generate.go new file mode 100644 index 00000000..679077bc --- /dev/null +++ b/tools/simulator/simulate/chain/generate.go @@ -0,0 +1,244 @@ +package chain + +import ( + "crypto/sha256" + "fmt" + "math/big" + + "github.com/Maldris/mathparse" + "github.com/shopspring/decimal" + + ocr2keepers "github.com/smartcontractkit/ocr2keepers/pkg/v3/types" + "github.com/smartcontractkit/ocr2keepers/tools/simulator/config" +) + +var ( + ErrUpkeepGeneration = fmt.Errorf("failed to generate upkeep") +) + +func GenerateAllUpkeeps(plan config.SimulationPlan) ([]SimulatedUpkeep, error) { + generated := make([]SimulatedUpkeep, 0) + limit := new(big.Int).Add(plan.Blocks.Genesis, big.NewInt(int64(plan.Blocks.Duration))) + + for idx, event := range plan.GenerateUpkeeps { + simulated, err := generateSimulatedUpkeeps(event, plan.Blocks.Genesis, limit) + if err != nil { + return nil, fmt.Errorf("%w at index %d", err, idx) + } + + generated = append(generated, simulated...) + } + + return generated, nil +} + +func GenerateLogTriggers(plan config.SimulationPlan) ([]SimulatedLog, error) { + logs := make([]SimulatedLog, len(plan.LogEvents)) + + for idx, event := range plan.LogEvents { + logs[idx] = SimulatedLog{ + TriggerAt: event.TriggerBlock, + TriggerValue: event.TriggerValue, + } + } + + return logs, nil +} + +func generateSimulatedUpkeeps(event config.GenerateUpkeepEvent, start *big.Int, limit *big.Int) ([]SimulatedUpkeep, error) { + applyFunctions := event.EligibilityFunc != "always" && event.EligibilityFunc != "never" + + if !applyFunctions { + simulated, err := generateBasicSimulatedUpkeeps(event, event.EligibilityFunc == "always") + if err != nil { + return nil, err + } + + return simulated, nil + } + + simulated, err := generateEligibilityFuncSimulatedUpkeeps(event, start, limit) + if err != nil { + return nil, err + } + + return simulated, nil +} + +func generateBasicSimulatedUpkeeps(event config.GenerateUpkeepEvent, alwaysEligible bool) ([]SimulatedUpkeep, error) { + simulationType, pluginTriggerType, err := getTriggerType(event.UpkeepType) + if err != nil { + return nil, err + } + + generated := make([]SimulatedUpkeep, 0) + + for y := 1; y <= event.Count; y++ { + id := new(big.Int).Add(event.StartID, big.NewInt(int64(y))) + simulated := SimulatedUpkeep{ + ID: id, + Type: simulationType, + CreateInBlock: event.TriggerBlock, + UpkeepID: newUpkeepID(id.Bytes(), pluginTriggerType), + AlwaysEligible: alwaysEligible, + EligibleAt: make([]*big.Int, 0), + TriggeredBy: event.LogTriggeredBy, + } + + generated = append(generated, simulated) + } + + return generated, nil +} + +func generateEligibilityFuncSimulatedUpkeeps(event config.GenerateUpkeepEvent, start *big.Int, limit *big.Int) ([]SimulatedUpkeep, error) { + simulationType, pluginTriggerType, err := getTriggerType(event.UpkeepType) + if err != nil { + return nil, err + } + + generated := make([]SimulatedUpkeep, 0) + offset := mathparse.NewParser(event.OffsetFunc) + + offset.Resolve() + + for y := 1; y <= event.Count; y++ { + id := new(big.Int).Add(event.StartID, big.NewInt(int64(y))) + sym := SimulatedUpkeep{ + ID: id, + Type: simulationType, + CreateInBlock: event.TriggerBlock, + UpkeepID: newUpkeepID(id.Bytes(), pluginTriggerType), + AlwaysEligible: false, + EligibleAt: make([]*big.Int, 0), + TriggeredBy: event.LogTriggeredBy, + } + + var genesis *big.Int + if offset.FoundResult() { + // create upkeep at id == result + genesis = big.NewInt(int64(offset.GetValueResult())) + } else { + // create upkeep genesis relative to upkeep count + g, err := calcFromTokens(offset.GetTokens(), big.NewInt(int64(y))) + if err != nil { + return nil, err + } + + genesis = new(big.Int).Add(start, g.BigInt()) + } + + if err := generateEligibles(&sym, genesis, limit, event.EligibilityFunc); err != nil { + return nil, err + } + + generated = append(generated, sym) + } + + return generated, nil +} + +func getTriggerType(configType config.UpkeepType) (UpkeepType, uint8, error) { + switch configType { + case config.ConditionalUpkeepType: + return ConditionalType, uint8(ocr2keepers.ConditionTrigger), nil + case config.LogTriggerUpkeepType: + return LogTriggerType, uint8(ocr2keepers.LogTrigger), nil + default: + return 0, 0, fmt.Errorf("%w: trigger type '%s' unrecognized", ErrUpkeepGeneration, configType) + } +} + +func operate(a, b decimal.Decimal, op string) decimal.Decimal { + switch op { + case "+": + return a.Add(b) + case "*": + return a.Mul(b) + case "-": + return a.Sub(b) + default: + } + + return decimal.Zero +} + +func generateEligibles(upkeep *SimulatedUpkeep, genesis *big.Int, limit *big.Int, f string) error { + p := mathparse.NewParser(f) + p.Resolve() + + if p.FoundResult() { + return fmt.Errorf("simple value unsupported") + } else { + // create upkeep from offset function + var i int64 = 0 + nextValue := big.NewInt(0) + tokens := p.GetTokens() + + for nextValue.Cmp(limit) < 0 { + if nextValue.Cmp(genesis) >= 0 { + upkeep.EligibleAt = append(upkeep.EligibleAt, nextValue) + } + + value, err := calcFromTokens(tokens, big.NewInt(i)) + if err != nil { + return err + } + + biValue := value.Round(0).BigInt() + nextValue = new(big.Int).Add(genesis, biValue) + i++ + } + } + + return nil +} + +func calcFromTokens(tokens []mathparse.Token, x *big.Int) (decimal.Decimal, error) { + value := decimal.NewFromInt(0) + action := "+" + + for i := 0; i < len(tokens); i++ { + token := tokens[i] + + switch token.Type { + case 2, 3: + var tVal decimal.Decimal + + if token.Value == "x" { + tVal = decimal.NewFromBigInt(x, int32(0)) + } else { + tVal = decimal.NewFromFloat(token.ParseValue) + } + + value = operate(value, tVal, action) + case 4: + action = token.Value + // case 1, 5, 6, 7, 8: + // log.Printf("unused token: %s", token.Value) + default: + } + } + + return value, nil +} + +func newUpkeepID(entropy []byte, uType uint8) [32]byte { + /* + Following the contract convention, an identifier is composed of 32 bytes: + + - 4 bytes of entropy + - 11 bytes of zeros + - 1 identifying byte for the trigger type + - 16 bytes of entropy + */ + hashedValue := sha256.Sum256(entropy) + + for x := 4; x < 15; x++ { + hashedValue[x] = uint8(0) + } + + hashedValue[15] = uType + + return hashedValue +} diff --git a/cmd/simulator/simulate/upkeep/generate_test.go b/tools/simulator/simulate/chain/generate_test.go similarity index 59% rename from cmd/simulator/simulate/upkeep/generate_test.go rename to tools/simulator/simulate/chain/generate_test.go index a8023d37..3262917a 100644 --- a/cmd/simulator/simulate/upkeep/generate_test.go +++ b/tools/simulator/simulate/chain/generate_test.go @@ -1,33 +1,45 @@ -package upkeep +package chain import ( "math/big" "testing" "github.com/shopspring/decimal" - "github.com/smartcontractkit/ocr2keepers/cmd/simulator/config" - "github.com/smartcontractkit/ocr2keepers/cmd/simulator/simulate/chain" + "github.com/smartcontractkit/ocr2keepers/tools/simulator/config" "github.com/stretchr/testify/assert" ) -func TestGenerateConditionals(t *testing.T) { - rb := config.RunBook{ - BlockCadence: config.Blocks{ +func TestGenerateAllUpkeeps(t *testing.T) { + plan := config.SimulationPlan{ + Blocks: config.Blocks{ Genesis: big.NewInt(128_943_862), Duration: 10, }, - Upkeeps: []config.Upkeep{ - {Count: 15, StartID: big.NewInt(200), GenerateFunc: "24x - 3", OffsetFunc: "3x - 4"}, + GenerateUpkeeps: []config.GenerateUpkeepEvent{ + { + Count: 15, + StartID: big.NewInt(200), + EligibilityFunc: "24x - 3", + OffsetFunc: "3x - 4", + UpkeepType: config.ConditionalUpkeepType, + }, + { + Count: 4, + StartID: big.NewInt(200), + EligibilityFunc: "always", + UpkeepType: config.LogTriggerUpkeepType, + }, }, } - gu, err := GenerateConditionals(rb) + generated, err := GenerateAllUpkeeps(plan) + assert.NoError(t, err) - assert.Len(t, gu, 15) + assert.Len(t, generated, 19) } func TestGenerateEligibles(t *testing.T) { - up := chain.SimulatedUpkeep{} + up := SimulatedUpkeep{} err := generateEligibles(&up, big.NewInt(9), big.NewInt(50), "4x + 5") expected := []int64{14, 18, 22, 26, 30, 34, 38, 42, 46} diff --git a/cmd/simulator/simulate/chain/history.go b/tools/simulator/simulate/chain/history.go similarity index 97% rename from cmd/simulator/simulate/chain/history.go rename to tools/simulator/simulate/chain/history.go index a5a55346..a9eac11d 100644 --- a/cmd/simulator/simulate/chain/history.go +++ b/tools/simulator/simulate/chain/history.go @@ -5,8 +5,8 @@ import ( "runtime" "sync" - "github.com/smartcontractkit/ocr2keepers/cmd/simulator/util" ocr2keepers "github.com/smartcontractkit/ocr2keepers/pkg/v3/types" + "github.com/smartcontractkit/ocr2keepers/tools/simulator/util" ) const ( diff --git a/cmd/simulator/simulate/chain/history_test.go b/tools/simulator/simulate/chain/history_test.go similarity index 83% rename from cmd/simulator/simulate/chain/history_test.go rename to tools/simulator/simulate/chain/history_test.go index ecfd873c..1b5908eb 100644 --- a/cmd/simulator/simulate/chain/history_test.go +++ b/tools/simulator/simulate/chain/history_test.go @@ -8,8 +8,8 @@ import ( "testing" "time" - "github.com/smartcontractkit/ocr2keepers/cmd/simulator/config" - "github.com/smartcontractkit/ocr2keepers/cmd/simulator/simulate/chain" + "github.com/smartcontractkit/ocr2keepers/tools/simulator/config" + "github.com/smartcontractkit/ocr2keepers/tools/simulator/simulate/chain" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -25,7 +25,7 @@ func TestBlockHistoryTracker(t *testing.T) { Duration: 10, } - broadcaster := chain.NewBlockBroadcaster(conf, 1, logger, loadTestUpkeep) + broadcaster := chain.NewBlockBroadcaster(conf, 1, logger, nil, loadTestUpkeep) listener := chain.NewListener(broadcaster, logger) deadline, ok := t.Deadline() diff --git a/cmd/simulator/simulate/chain/listener.go b/tools/simulator/simulate/chain/listener.go similarity index 100% rename from cmd/simulator/simulate/chain/listener.go rename to tools/simulator/simulate/chain/listener.go diff --git a/cmd/simulator/simulate/chain/listener_test.go b/tools/simulator/simulate/chain/listener_test.go similarity index 81% rename from cmd/simulator/simulate/chain/listener_test.go rename to tools/simulator/simulate/chain/listener_test.go index 9afc4eb2..c742a1cd 100644 --- a/cmd/simulator/simulate/chain/listener_test.go +++ b/tools/simulator/simulate/chain/listener_test.go @@ -8,8 +8,8 @@ import ( "testing" "time" - "github.com/smartcontractkit/ocr2keepers/cmd/simulator/config" - "github.com/smartcontractkit/ocr2keepers/cmd/simulator/simulate/chain" + "github.com/smartcontractkit/ocr2keepers/tools/simulator/config" + "github.com/smartcontractkit/ocr2keepers/tools/simulator/simulate/chain" ) func TestListener(t *testing.T) { @@ -23,7 +23,7 @@ func TestListener(t *testing.T) { Duration: 10, } - broadcaster := chain.NewBlockBroadcaster(conf, 1, logger, loadTestUpkeep) + broadcaster := chain.NewBlockBroadcaster(conf, 1, logger, nil, loadTestUpkeep) listener := chain.NewListener(broadcaster, logger) deadline, ok := t.Deadline() diff --git a/cmd/simulator/simulate/db/ocr3.go b/tools/simulator/simulate/db/ocr3.go similarity index 100% rename from cmd/simulator/simulate/db/ocr3.go rename to tools/simulator/simulate/db/ocr3.go diff --git a/cmd/simulator/simulate/db/upkeep.go b/tools/simulator/simulate/db/upkeep.go similarity index 100% rename from cmd/simulator/simulate/db/upkeep.go rename to tools/simulator/simulate/db/upkeep.go diff --git a/cmd/simulator/simulate/hydrator.go b/tools/simulator/simulate/hydrator.go similarity index 69% rename from cmd/simulator/simulate/hydrator.go rename to tools/simulator/simulate/hydrator.go index 907fd3b4..5180a9b7 100644 --- a/cmd/simulator/simulate/hydrator.go +++ b/tools/simulator/simulate/hydrator.go @@ -3,14 +3,15 @@ package simulate import ( "log" - "github.com/smartcontractkit/ocr2keepers/cmd/simulator/config" - "github.com/smartcontractkit/ocr2keepers/cmd/simulator/simulate/chain" - "github.com/smartcontractkit/ocr2keepers/cmd/simulator/simulate/db" - "github.com/smartcontractkit/ocr2keepers/cmd/simulator/simulate/net" - "github.com/smartcontractkit/ocr2keepers/cmd/simulator/simulate/ocr" - "github.com/smartcontractkit/ocr2keepers/cmd/simulator/simulate/upkeep" - "github.com/smartcontractkit/ocr2keepers/cmd/simulator/telemetry" "github.com/smartcontractkit/ocr2keepers/pkg/v3/plugin" + "github.com/smartcontractkit/ocr2keepers/tools/simulator/config" + "github.com/smartcontractkit/ocr2keepers/tools/simulator/simulate/chain" + "github.com/smartcontractkit/ocr2keepers/tools/simulator/simulate/db" + "github.com/smartcontractkit/ocr2keepers/tools/simulator/simulate/loader" + "github.com/smartcontractkit/ocr2keepers/tools/simulator/simulate/net" + "github.com/smartcontractkit/ocr2keepers/tools/simulator/simulate/ocr" + "github.com/smartcontractkit/ocr2keepers/tools/simulator/simulate/upkeep" + "github.com/smartcontractkit/ocr2keepers/tools/simulator/telemetry" ) const ( @@ -21,8 +22,8 @@ func HydrateConfig( name string, config *plugin.DelegateConfig, blocks *chain.BlockBroadcaster, - transmitter *ocr.OCR3TransmitLoader, - conf config.RunBook, + transmitter *loader.OCR3TransmitLoader, + conf config.SimulationPlan, netTelemetry net.NetTelemetry, conTelemetry *telemetry.WrappedContractCollector, logger *log.Logger, diff --git a/tools/simulator/simulate/loader/logtrigger.go b/tools/simulator/simulate/loader/logtrigger.go new file mode 100644 index 00000000..8d0375f5 --- /dev/null +++ b/tools/simulator/simulate/loader/logtrigger.go @@ -0,0 +1,75 @@ +package loader + +import ( + "crypto/sha256" + "sync" + "time" + + "github.com/smartcontractkit/ocr2keepers/tools/simulator/config" + "github.com/smartcontractkit/ocr2keepers/tools/simulator/simulate/chain" +) + +// LogTriggerLoader ... +type LogTriggerLoader struct { + // provided dependencies + progress ProgressTelemetry + + // internal state values + mu sync.RWMutex + triggers map[string][]chain.Log +} + +// NewLogTriggerLoader ... +func NewLogTriggerLoader(plan config.SimulationPlan, progress ProgressTelemetry) (*LogTriggerLoader, error) { + logs, err := chain.GenerateLogTriggers(plan) + if err != nil { + return nil, err + } + + events := make(map[string][]chain.Log) + for _, logEvt := range logs { + trigger := logEvt.TriggerAt + + existing, ok := events[trigger.String()] + if !ok { + existing = []chain.Log{} + } + + events[trigger.String()] = append(existing, chain.Log{ + TriggerValue: logEvt.TriggerValue, + }) + } + + if progress != nil { + if err := progress.Register(emitLogNamespace, int64(len(logs))); err != nil { + return nil, err + } + } + + return &LogTriggerLoader{ + progress: progress, + triggers: events, + }, nil +} + +// Load implements the chain.BlockLoaderFunc type and loads log trigger events +// into blocks +func (ltl *LogTriggerLoader) Load(block *chain.Block) { + ltl.mu.RLock() + defer ltl.mu.RUnlock() + + if events, ok := ltl.triggers[block.Number.String()]; ok { + for _, event := range events { + event.BlockNumber = block.Number + event.BlockHash = block.Hash + event.Idx = uint32(len(block.Transactions)) + event.TxHash = sha256.Sum256([]byte(time.Now().Format(time.RFC3339Nano))) + + block.Transactions = append(block.Transactions, event) + } + + if ltl.progress != nil { + ltl.progress.Increment(emitLogNamespace, int64(len(events))) + } + } +} diff --git a/tools/simulator/simulate/loader/logtrigger_test.go b/tools/simulator/simulate/loader/logtrigger_test.go new file mode 100644 index 00000000..aaa3653b --- /dev/null +++ b/tools/simulator/simulate/loader/logtrigger_test.go @@ -0,0 +1 @@ +package loader_test diff --git a/cmd/simulator/simulate/ocr/loader.go b/tools/simulator/simulate/loader/ocr3config.go similarity index 73% rename from cmd/simulator/simulate/ocr/loader.go rename to tools/simulator/simulate/loader/ocr3config.go index 93a98f4c..01a4b906 100644 --- a/cmd/simulator/simulate/ocr/loader.go +++ b/tools/simulator/simulate/loader/ocr3config.go @@ -1,4 +1,4 @@ -package ocr +package loader import ( "log" @@ -8,8 +8,12 @@ import ( "github.com/smartcontractkit/libocr/offchainreporting2plus/ocr3confighelper" "github.com/smartcontractkit/libocr/offchainreporting2plus/types" - "github.com/smartcontractkit/ocr2keepers/cmd/simulator/config" - "github.com/smartcontractkit/ocr2keepers/cmd/simulator/simulate/chain" + "github.com/smartcontractkit/ocr2keepers/tools/simulator/config" + "github.com/smartcontractkit/ocr2keepers/tools/simulator/simulate/chain" +) + +const ( + ocr3ConfigProgressNamespace = "Emitting OCR3 config transactions" ) type KeySourcer interface { @@ -26,32 +30,44 @@ type Digester interface { ConfigDigest(config types.ContractConfig) (types.ConfigDigest, error) } +type ProgressTelemetry interface { + Register(string, int64) error + Increment(string, int64) +} + // OCR3ConfigLoader ... type OCR3ConfigLoader struct { // provided dependencies - digest Digester - logger *log.Logger + digest Digester + progress ProgressTelemetry + logger *log.Logger // internal state values mu sync.Mutex count uint64 oracles []ocr2config.OracleIdentityExtra - events map[string]config.ConfigEvent + events map[string]config.OCR3ConfigEvent } -// NewOCR3ConfigLoader ... -func NewOCR3ConfigLoader(rb config.RunBook, digest Digester, logger *log.Logger) *OCR3ConfigLoader { - eventLookup := make(map[string]config.ConfigEvent) +// NewOCR3ConfigLoader adds OCR3 config transactions to incoming blocks as +// defined in a provided simulation plan. +func NewOCR3ConfigLoader(plan config.SimulationPlan, progress ProgressTelemetry, digest Digester, logger *log.Logger) *OCR3ConfigLoader { + eventLookup := make(map[string]config.OCR3ConfigEvent) - for _, event := range rb.ConfigEvents { - eventLookup[event.Block.String()] = event + for _, event := range plan.ConfigEvents { + eventLookup[event.Event.TriggerBlock.String()] = event + } + + if progress != nil { + _ = progress.Register(ocr3ConfigProgressNamespace, int64(len(plan.ConfigEvents))) } return &OCR3ConfigLoader{ - logger: log.New(logger.Writer(), "[ocr3-config-loader] ", log.Ldate|log.Ltime|log.Lshortfile), - digest: digest, - events: eventLookup, - oracles: []ocr2config.OracleIdentityExtra{}, + digest: digest, + progress: progress, + logger: log.New(logger.Writer(), "[ocr3-config-loader] ", log.Ldate|log.Ltime|log.Lshortfile), + events: eventLookup, + oracles: []ocr2config.OracleIdentityExtra{}, } } @@ -76,6 +92,10 @@ func (l *OCR3ConfigLoader) Load(block *chain.Block) { Config: conf, }) + if l.progress != nil { + l.progress.Increment(ocr3ConfigProgressNamespace, 1) + } + l.count++ } } @@ -97,7 +117,7 @@ func (l *OCR3ConfigLoader) AddSigner(id string, onKey KeySourcer, offKey Offchai l.oracles = append(l.oracles, newOracle) } -func buildConfig(conf config.ConfigEvent, oracles []ocr2config.OracleIdentityExtra, digester Digester, count uint64) (types.ContractConfig, error) { +func buildConfig(conf config.OCR3ConfigEvent, oracles []ocr2config.OracleIdentityExtra, digester Digester, count uint64) (types.ContractConfig, error) { // S is a slice of values that indicate the number of oracles involved // in attempting to transmit. For the simulator, all nodes will be involved // in transmit attempts. @@ -122,7 +142,7 @@ func buildConfig(conf config.ConfigEvent, oracles []ocr2config.OracleIdentityExt conf.MaxObservation.Value(), // maxDurationObservation time.Duration, conf.MaxAccept.Value(), // maxDurationShouldAcceptAttestedReport time.Duration conf.MaxTransmit.Value(), // maxDurationShouldTransmitAcceptedReport time.Duration, - conf.F, // f int, + conf.MaxFaultyNodesF, // f int, nil, // onchainConfig []byte, ) if err != nil { diff --git a/cmd/simulator/simulate/ocr/loader_test.go b/tools/simulator/simulate/loader/ocr3config_test.go similarity index 70% rename from cmd/simulator/simulate/ocr/loader_test.go rename to tools/simulator/simulate/loader/ocr3config_test.go index e3865e46..0e53de34 100644 --- a/cmd/simulator/simulate/ocr/loader_test.go +++ b/tools/simulator/simulate/loader/ocr3config_test.go @@ -1,4 +1,4 @@ -package ocr_test +package loader_test import ( "crypto/rand" @@ -13,9 +13,9 @@ import ( "github.com/stretchr/testify/require" "github.com/smartcontractkit/libocr/offchainreporting2plus/types" - "github.com/smartcontractkit/ocr2keepers/cmd/simulator/config" - "github.com/smartcontractkit/ocr2keepers/cmd/simulator/simulate/chain" - "github.com/smartcontractkit/ocr2keepers/cmd/simulator/simulate/ocr" + "github.com/smartcontractkit/ocr2keepers/tools/simulator/config" + "github.com/smartcontractkit/ocr2keepers/tools/simulator/simulate/chain" + "github.com/smartcontractkit/ocr2keepers/tools/simulator/simulate/loader" ) func TestOCR3ConfigLoader(t *testing.T) { @@ -24,11 +24,17 @@ func TestOCR3ConfigLoader(t *testing.T) { logger := log.New(io.Discard, "", 0) digester := new(mockDigester) - runbook := config.RunBook{ - ConfigEvents: []config.ConfigEvent{{Block: big.NewInt(2)}}, + plan := config.SimulationPlan{ + ConfigEvents: []config.OCR3ConfigEvent{ + { + Event: config.Event{ + TriggerBlock: big.NewInt(2), + }, + }, + }, } - loader := ocr.NewOCR3ConfigLoader(runbook, digester, logger) + loader := loader.NewOCR3ConfigLoader(plan, nil, digester, logger) block := chain.Block{ Number: big.NewInt(1), } diff --git a/cmd/simulator/simulate/ocr/transmit.go b/tools/simulator/simulate/loader/ocr3transmit.go similarity index 53% rename from cmd/simulator/simulate/ocr/transmit.go rename to tools/simulator/simulate/loader/ocr3transmit.go index 002b2379..4746156f 100644 --- a/cmd/simulator/simulate/ocr/transmit.go +++ b/tools/simulator/simulate/loader/ocr3transmit.go @@ -1,8 +1,7 @@ -package ocr +package loader import ( "bytes" - "context" "crypto/sha256" "encoding/base64" "encoding/gob" @@ -10,17 +9,19 @@ import ( "log" "sync" - "github.com/smartcontractkit/libocr/offchainreporting2/types" - "github.com/smartcontractkit/libocr/offchainreporting2plus/ocr3types" + "github.com/smartcontractkit/ocr2keepers/tools/simulator/config" + "github.com/smartcontractkit/ocr2keepers/tools/simulator/simulate/chain" + "github.com/smartcontractkit/ocr2keepers/tools/simulator/util" +) - "github.com/smartcontractkit/ocr2keepers/cmd/simulator/config" - "github.com/smartcontractkit/ocr2keepers/cmd/simulator/simulate/chain" - "github.com/smartcontractkit/ocr2keepers/pkg/v3/plugin" +const ( + transmitProgressNamespace = "Collecting upkeep perform events" ) type OCR3TransmitLoader struct { // provided dependencies - logger *log.Logger + progress ProgressTelemetry + logger *log.Logger // internal state values mu sync.RWMutex @@ -28,13 +29,26 @@ type OCR3TransmitLoader struct { transmitted map[string]*chain.TransmitEvent } -// NewOCR3TransmitLoader ... -func NewOCR3TransmitLoader(_ config.RunBook, logger *log.Logger) *OCR3TransmitLoader { +// NewOCR3TransmitLoader accepts report bytes and adds them to incoming blocks +// as TransmitEvent transactions. +func NewOCR3TransmitLoader(plan config.SimulationPlan, progress ProgressTelemetry, logger *log.Logger) (*OCR3TransmitLoader, error) { + if progress != nil { + expected, err := calculateExpectedPerformEvents(plan) + if err != nil { + return nil, err + } + + if err := progress.Register(transmitProgressNamespace, expected); err != nil { + return nil, err + } + } + return &OCR3TransmitLoader{ + progress: progress, logger: log.New(logger.Writer(), "[ocr3-transmit-loader] ", log.Ldate|log.Ltime|log.Lshortfile), queue: make([]*chain.TransmitEvent, 0), transmitted: make(map[string]*chain.TransmitEvent), - } + }, nil } func (tl *OCR3TransmitLoader) Load(block *chain.Block) { @@ -46,11 +60,14 @@ func (tl *OCR3TransmitLoader) Load(block *chain.Block) { } transmits := make([]chain.TransmitEvent, 0, len(tl.queue)) + var performs int64 for i := range tl.queue { tl.queue[i].BlockNumber = block.Number tl.queue[i].BlockHash = block.Hash + performs += countPerformEvents(tl.queue[i].Report) + transmits = append(transmits, *tl.queue[i]) } @@ -59,6 +76,10 @@ func (tl *OCR3TransmitLoader) Load(block *chain.Block) { block.Transactions = append(block.Transactions, chain.PerformUpkeepTransaction{ Transmits: transmits, }) + + if tl.progress != nil { + tl.progress.Increment(transmitProgressNamespace, performs) + } } func (tl *OCR3TransmitLoader) Transmit(from string, reportBytes []byte, round uint64) error { @@ -112,38 +133,58 @@ func (tl *OCR3TransmitLoader) Results() []chain.TransmitEvent { return events } -type OCR3Transmitter struct { - // configured values - transmitterID string - loader *OCR3TransmitLoader +func calculateExpectedPerformEvents(plan config.SimulationPlan) (int64, error) { + var count int64 - // internal state values - mu sync.RWMutex -} + upkeeps, err := chain.GenerateAllUpkeeps(plan) + if err != nil { + return count, err + } + + logs, err := chain.GenerateLogTriggers(plan) + if err != nil { + return count, err + } -func NewOCR3Transmitter(id string, loader *OCR3TransmitLoader) *OCR3Transmitter { - return &OCR3Transmitter{ - transmitterID: id, - loader: loader, + for _, upkeep := range upkeeps { + switch upkeep.Type { + case chain.ConditionalType: + count += int64(len(upkeep.EligibleAt)) + case chain.LogTriggerType: + for _, log := range logs { + if logTriggersUpkeep(log, upkeep) { + count++ + } + } + } } + + return count, nil } -func (tr *OCR3Transmitter) Transmit( - ctx context.Context, - digest types.ConfigDigest, - v uint64, - r ocr3types.ReportWithInfo[plugin.AutomationReportInfo], - s []types.AttributedOnchainSignature, -) error { - return tr.loader.Transmit(tr.transmitterID, []byte(r.Report), v) +func logTriggersUpkeep(log chain.SimulatedLog, upkeep chain.SimulatedUpkeep) bool { + if log.TriggerAt.Cmp(upkeep.CreateInBlock) >= 0 && log.TriggerValue == upkeep.TriggeredBy { + if upkeep.AlwaysEligible { + return true + } else { + for _, block := range upkeep.EligibleAt { + if block.Cmp(log.TriggerAt) >= 0 { + return true + } + } + } + } + + return false } -// Account from which the transmitter invokes the contract -func (tr *OCR3Transmitter) FromAccount() (types.Account, error) { - tr.mu.RLock() - defer tr.mu.RUnlock() +func countPerformEvents(report []byte) int64 { + results, err := util.DecodeCheckResultsFromReportBytes(report) + if err != nil { + return 0 + } - return types.Account(tr.transmitterID), nil + return int64(len(results)) } func rawHash(b []byte) [32]byte { diff --git a/cmd/simulator/simulate/ocr/transmit_test.go b/tools/simulator/simulate/loader/ocr3transmit_test.go similarity index 77% rename from cmd/simulator/simulate/ocr/transmit_test.go rename to tools/simulator/simulate/loader/ocr3transmit_test.go index 98d2f0b5..a70049a7 100644 --- a/cmd/simulator/simulate/ocr/transmit_test.go +++ b/tools/simulator/simulate/loader/ocr3transmit_test.go @@ -1,4 +1,4 @@ -package ocr_test +package loader_test import ( "io" @@ -9,16 +9,19 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - "github.com/smartcontractkit/ocr2keepers/cmd/simulator/config" - "github.com/smartcontractkit/ocr2keepers/cmd/simulator/simulate/chain" - "github.com/smartcontractkit/ocr2keepers/cmd/simulator/simulate/ocr" + "github.com/smartcontractkit/ocr2keepers/tools/simulator/config" + "github.com/smartcontractkit/ocr2keepers/tools/simulator/simulate/chain" + "github.com/smartcontractkit/ocr2keepers/tools/simulator/simulate/loader" ) func TestOCR3TransmitLoader(t *testing.T) { t.Parallel() logger := log.New(io.Discard, "", 0) - loader := ocr.NewOCR3TransmitLoader(config.RunBook{}, logger) + loader, err := loader.NewOCR3TransmitLoader(config.SimulationPlan{}, nil, logger) + + require.NoError(t, err) + block := chain.Block{ Number: big.NewInt(1), Transactions: []interface{}{}, diff --git a/tools/simulator/simulate/loader/upkeep.go b/tools/simulator/simulate/loader/upkeep.go new file mode 100644 index 00000000..4e3c0eb0 --- /dev/null +++ b/tools/simulator/simulate/loader/upkeep.go @@ -0,0 +1,74 @@ +package loader + +import ( + "sync" + + "github.com/smartcontractkit/ocr2keepers/tools/simulator/config" + "github.com/smartcontractkit/ocr2keepers/tools/simulator/simulate/chain" +) + +const ( + CreateUpkeepNamespace = "Emitting create upkeep transactions" + emitLogNamespace = "Emitting log events" +) + +// UpkeepConfigLoader provides upkeep configurations to a block broadcaster. Use +// this loader to introduce upkeeps or change upkeep configs at specific block +// numbers. +type UpkeepConfigLoader struct { + // provided dependencies + progress ProgressTelemetry + + // internal state values + mu sync.RWMutex + create map[string][]chain.UpkeepCreatedTransaction +} + +// NewUpkeepConfigLoader ... +func NewUpkeepConfigLoader(plan config.SimulationPlan, progress ProgressTelemetry) (*UpkeepConfigLoader, error) { + // combine all upkeeps together for transmit + allUpkeeps, err := chain.GenerateAllUpkeeps(plan) + if err != nil { + return nil, err + } + + create := make(map[string][]chain.UpkeepCreatedTransaction) + for _, upkeep := range allUpkeeps { + evts, ok := create[upkeep.CreateInBlock.String()] + if !ok { + evts = []chain.UpkeepCreatedTransaction{} + } + + create[upkeep.CreateInBlock.String()] = append(evts, chain.UpkeepCreatedTransaction{ + Upkeep: upkeep, + }) + } + + if progress != nil { + if err := progress.Register(CreateUpkeepNamespace, int64(len(allUpkeeps))); err != nil { + return nil, err + } + } + + return &UpkeepConfigLoader{ + create: create, + progress: progress, + }, nil +} + +// Load implements the chain.BlockLoaderFunc type and loads configured upkeep +// events into blocks. +func (ucl *UpkeepConfigLoader) Load(block *chain.Block) { + ucl.mu.RLock() + defer ucl.mu.RUnlock() + + if events, ok := ucl.create[block.Number.String()]; ok { + for _, event := range events { + block.Transactions = append(block.Transactions, event) + } + + if ucl.progress != nil { + ucl.progress.Increment(CreateUpkeepNamespace, int64(len(events))) + } + } +} diff --git a/tools/simulator/simulate/loader/upkeep_test.go b/tools/simulator/simulate/loader/upkeep_test.go new file mode 100644 index 00000000..53b44a2e --- /dev/null +++ b/tools/simulator/simulate/loader/upkeep_test.go @@ -0,0 +1,68 @@ +package loader_test + +import ( + "math/big" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/require" + + "github.com/smartcontractkit/ocr2keepers/tools/simulator/config" + "github.com/smartcontractkit/ocr2keepers/tools/simulator/simulate/chain" + "github.com/smartcontractkit/ocr2keepers/tools/simulator/simulate/loader" +) + +func TestUpkeepConfigLoader(t *testing.T) { + plan := config.SimulationPlan{ + Blocks: config.Blocks{ + Genesis: big.NewInt(1), + Cadence: config.Duration(time.Second), + Duration: 10, + }, + GenerateUpkeeps: []config.GenerateUpkeepEvent{ + { + Event: config.Event{ + TriggerBlock: big.NewInt(2), + }, + Count: 10, + StartID: big.NewInt(1), + EligibilityFunc: "2x", + OffsetFunc: "x", + UpkeepType: config.ConditionalUpkeepType, + }, + }, + } + telemetry := new(mockProgressTelemetry) + + telemetry.On("Register", loader.CreateUpkeepNamespace, int64(10)).Return(nil) + telemetry.On("Increment", loader.CreateUpkeepNamespace, int64(10)) + + loader, err := loader.NewUpkeepConfigLoader(plan, telemetry) + + require.NoError(t, err) + + block := chain.Block{ + Number: big.NewInt(2), + Transactions: []interface{}{}, + } + + loader.Load(&block) + + assert.Len(t, block.Transactions, 10) +} + +type mockProgressTelemetry struct { + mock.Mock +} + +func (_m *mockProgressTelemetry) Register(namespace string, total int64) error { + res := _m.Called(namespace, total) + + return res.Error(0) +} + +func (_m *mockProgressTelemetry) Increment(namespace string, count int64) { + _m.Called(namespace, count) +} diff --git a/cmd/simulator/simulate/net/network.go b/tools/simulator/simulate/net/network.go similarity index 100% rename from cmd/simulator/simulate/net/network.go rename to tools/simulator/simulate/net/network.go diff --git a/cmd/simulator/simulate/net/network_test.go b/tools/simulator/simulate/net/network_test.go similarity index 97% rename from cmd/simulator/simulate/net/network_test.go rename to tools/simulator/simulate/net/network_test.go index f0926b8a..c89ac996 100644 --- a/cmd/simulator/simulate/net/network_test.go +++ b/tools/simulator/simulate/net/network_test.go @@ -9,7 +9,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - "github.com/smartcontractkit/ocr2keepers/cmd/simulator/simulate/net" + "github.com/smartcontractkit/ocr2keepers/tools/simulator/simulate/net" ) func TestSimulatedNetwork(t *testing.T) { diff --git a/cmd/simulator/simulate/net/service.go b/tools/simulator/simulate/net/service.go similarity index 98% rename from cmd/simulator/simulate/net/service.go rename to tools/simulator/simulate/net/service.go index 34c7d32d..2d9d0b0a 100644 --- a/cmd/simulator/simulate/net/service.go +++ b/tools/simulator/simulate/net/service.go @@ -11,7 +11,7 @@ import ( "gonum.org/v1/gonum/stat/distuv" - "github.com/smartcontractkit/ocr2keepers/cmd/simulator/util" + "github.com/smartcontractkit/ocr2keepers/tools/simulator/util" ) const ( diff --git a/cmd/simulator/simulate/net/service_test.go b/tools/simulator/simulate/net/service_test.go similarity index 91% rename from cmd/simulator/simulate/net/service_test.go rename to tools/simulator/simulate/net/service_test.go index 18528ea1..9db85235 100644 --- a/cmd/simulator/simulate/net/service_test.go +++ b/tools/simulator/simulate/net/service_test.go @@ -8,7 +8,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" - "github.com/smartcontractkit/ocr2keepers/cmd/simulator/simulate/net" + "github.com/smartcontractkit/ocr2keepers/tools/simulator/simulate/net" ) func TestSimulatedNetworkService(t *testing.T) { diff --git a/cmd/simulator/simulate/ocr/config.go b/tools/simulator/simulate/ocr/config.go similarity index 97% rename from cmd/simulator/simulate/ocr/config.go rename to tools/simulator/simulate/ocr/config.go index c1dc3e76..67ecc461 100644 --- a/cmd/simulator/simulate/ocr/config.go +++ b/tools/simulator/simulate/ocr/config.go @@ -9,7 +9,7 @@ import ( "github.com/smartcontractkit/libocr/offchainreporting2plus/types" - "github.com/smartcontractkit/ocr2keepers/cmd/simulator/simulate/chain" + "github.com/smartcontractkit/ocr2keepers/tools/simulator/simulate/chain" ) // OCR3ConfigTracker ... diff --git a/cmd/simulator/simulate/ocr/config_test.go b/tools/simulator/simulate/ocr/config_test.go similarity index 84% rename from cmd/simulator/simulate/ocr/config_test.go rename to tools/simulator/simulate/ocr/config_test.go index 0993163c..777a59e5 100644 --- a/cmd/simulator/simulate/ocr/config_test.go +++ b/tools/simulator/simulate/ocr/config_test.go @@ -13,9 +13,9 @@ import ( "github.com/stretchr/testify/require" "github.com/smartcontractkit/libocr/offchainreporting2plus/types" - "github.com/smartcontractkit/ocr2keepers/cmd/simulator/config" - "github.com/smartcontractkit/ocr2keepers/cmd/simulator/simulate/chain" - "github.com/smartcontractkit/ocr2keepers/cmd/simulator/simulate/ocr" + "github.com/smartcontractkit/ocr2keepers/tools/simulator/config" + "github.com/smartcontractkit/ocr2keepers/tools/simulator/simulate/chain" + "github.com/smartcontractkit/ocr2keepers/tools/simulator/simulate/ocr" ) func TestOCR3ConfigTracker(t *testing.T) { @@ -33,7 +33,7 @@ func TestOCR3ConfigTracker(t *testing.T) { ConfigDigest: sha256.Sum256([]byte("some config data")), } - broadcaster := chain.NewBlockBroadcaster(conf, 1, logger, loadConfigAt(ocrConfig, 2)) + broadcaster := chain.NewBlockBroadcaster(conf, 1, logger, nil, loadConfigAt(ocrConfig, 2)) listener := chain.NewListener(broadcaster, logger) tracker := ocr.NewOCR3ConfigTracker(listener, logger) diff --git a/cmd/simulator/simulate/ocr/report.go b/tools/simulator/simulate/ocr/report.go similarity index 95% rename from cmd/simulator/simulate/ocr/report.go rename to tools/simulator/simulate/ocr/report.go index a0d2c7f4..744d9293 100644 --- a/cmd/simulator/simulate/ocr/report.go +++ b/tools/simulator/simulate/ocr/report.go @@ -8,9 +8,9 @@ import ( "runtime" "sync" - "github.com/smartcontractkit/ocr2keepers/cmd/simulator/simulate/chain" - "github.com/smartcontractkit/ocr2keepers/cmd/simulator/util" ocr2keepers "github.com/smartcontractkit/ocr2keepers/pkg/v3/types" + "github.com/smartcontractkit/ocr2keepers/tools/simulator/simulate/chain" + "github.com/smartcontractkit/ocr2keepers/tools/simulator/util" ) const ( diff --git a/cmd/simulator/simulate/ocr/report_test.go b/tools/simulator/simulate/ocr/report_test.go similarity index 81% rename from cmd/simulator/simulate/ocr/report_test.go rename to tools/simulator/simulate/ocr/report_test.go index d4efbb8e..583345ec 100644 --- a/cmd/simulator/simulate/ocr/report_test.go +++ b/tools/simulator/simulate/ocr/report_test.go @@ -11,11 +11,11 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - "github.com/smartcontractkit/ocr2keepers/cmd/simulator/config" - "github.com/smartcontractkit/ocr2keepers/cmd/simulator/simulate/chain" - "github.com/smartcontractkit/ocr2keepers/cmd/simulator/simulate/ocr" - "github.com/smartcontractkit/ocr2keepers/cmd/simulator/util" ocr2keepers "github.com/smartcontractkit/ocr2keepers/pkg/v3/types" + "github.com/smartcontractkit/ocr2keepers/tools/simulator/config" + "github.com/smartcontractkit/ocr2keepers/tools/simulator/simulate/chain" + "github.com/smartcontractkit/ocr2keepers/tools/simulator/simulate/ocr" + "github.com/smartcontractkit/ocr2keepers/tools/simulator/util" ) func TestReportTracker(t *testing.T) { @@ -54,7 +54,7 @@ func TestReportTracker(t *testing.T) { }, } - broadcaster := chain.NewBlockBroadcaster(conf, 1, logger, loadTransmitsAt(transmits, 2)) + broadcaster := chain.NewBlockBroadcaster(conf, 1, logger, nil, loadTransmitsAt(transmits, 2)) listener := chain.NewListener(broadcaster, logger) tracker := ocr.NewReportTracker(listener, logger) diff --git a/tools/simulator/simulate/ocr/transmit.go b/tools/simulator/simulate/ocr/transmit.go new file mode 100644 index 00000000..99cc8273 --- /dev/null +++ b/tools/simulator/simulate/ocr/transmit.go @@ -0,0 +1,49 @@ +package ocr + +import ( + "context" + "sync" + + "github.com/smartcontractkit/libocr/offchainreporting2/types" + "github.com/smartcontractkit/libocr/offchainreporting2plus/ocr3types" + + "github.com/smartcontractkit/ocr2keepers/pkg/v3/plugin" +) + +type Transmitter interface { + Transmit(string, []byte, uint64) error +} + +type OCR3Transmitter struct { + // configured values + transmitterID string + loader Transmitter + + // internal state values + mu sync.RWMutex +} + +func NewOCR3Transmitter(id string, loader Transmitter) *OCR3Transmitter { + return &OCR3Transmitter{ + transmitterID: id, + loader: loader, + } +} + +func (tr *OCR3Transmitter) Transmit( + ctx context.Context, + digest types.ConfigDigest, + v uint64, + r ocr3types.ReportWithInfo[plugin.AutomationReportInfo], + s []types.AttributedOnchainSignature, +) error { + return tr.loader.Transmit(tr.transmitterID, []byte(r.Report), v) +} + +// Account from which the transmitter invokes the contract +func (tr *OCR3Transmitter) FromAccount() (types.Account, error) { + tr.mu.RLock() + defer tr.mu.RUnlock() + + return types.Account(tr.transmitterID), nil +} diff --git a/tools/simulator/simulate/ocr/transmit_test.go b/tools/simulator/simulate/ocr/transmit_test.go new file mode 100644 index 00000000..041af1aa --- /dev/null +++ b/tools/simulator/simulate/ocr/transmit_test.go @@ -0,0 +1 @@ +package ocr_test diff --git a/cmd/simulator/simulate/upkeep/active.go b/tools/simulator/simulate/upkeep/active.go similarity index 94% rename from cmd/simulator/simulate/upkeep/active.go rename to tools/simulator/simulate/upkeep/active.go index f1f37d4f..83bff7a5 100644 --- a/cmd/simulator/simulate/upkeep/active.go +++ b/tools/simulator/simulate/upkeep/active.go @@ -7,7 +7,7 @@ import ( "runtime" "sync" - "github.com/smartcontractkit/ocr2keepers/cmd/simulator/simulate/chain" + "github.com/smartcontractkit/ocr2keepers/tools/simulator/simulate/chain" ) type ActiveTracker struct { @@ -120,6 +120,8 @@ func (at *ActiveTracker) addSimulatedUpkeep(upkeep chain.SimulatedUpkeep) { upkeepIDs = append(upkeepIDs, key) + at.logger.Printf("upkeep id %d registered as active as type %d", upkeep.ID.Int64(), upkeep.Type) + at.active[upkeep.Type] = upkeepIDs at.idLookup[key] = upkeep } diff --git a/cmd/simulator/simulate/upkeep/active_test.go b/tools/simulator/simulate/upkeep/active_test.go similarity index 61% rename from cmd/simulator/simulate/upkeep/active_test.go rename to tools/simulator/simulate/upkeep/active_test.go index 14665e0a..6fe049e6 100644 --- a/cmd/simulator/simulate/upkeep/active_test.go +++ b/tools/simulator/simulate/upkeep/active_test.go @@ -7,11 +7,14 @@ import ( "testing" "time" - "github.com/smartcontractkit/ocr2keepers/cmd/simulator/config" - "github.com/smartcontractkit/ocr2keepers/cmd/simulator/simulate/chain" - "github.com/smartcontractkit/ocr2keepers/cmd/simulator/simulate/upkeep" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + + ocr2keepers "github.com/smartcontractkit/ocr2keepers/pkg/v3/types" + "github.com/smartcontractkit/ocr2keepers/tools/simulator/config" + "github.com/smartcontractkit/ocr2keepers/tools/simulator/simulate/chain" + "github.com/smartcontractkit/ocr2keepers/tools/simulator/simulate/upkeep" + "github.com/smartcontractkit/ocr2keepers/tools/simulator/util" ) func TestActiveTracker(t *testing.T) { @@ -26,12 +29,18 @@ func TestActiveTracker(t *testing.T) { } upkeep1 := chain.SimulatedUpkeep{ - ID: big.NewInt(10), - UpkeepID: [32]byte{}, + ID: big.NewInt(8), + UpkeepID: util.NewUpkeepID(big.NewInt(8).Bytes(), uint8(ocr2keepers.ConditionTrigger)), Type: chain.ConditionalType, } - broadcaster := chain.NewBlockBroadcaster(conf, 1, logger, loadUpkeepAt(upkeep1, 2)) + upkeep2 := chain.SimulatedUpkeep{ + ID: big.NewInt(10), + UpkeepID: util.NewUpkeepID(big.NewInt(10).Bytes(), uint8(ocr2keepers.LogTrigger)), + Type: chain.LogTriggerType, + } + + broadcaster := chain.NewBlockBroadcaster(conf, 1, logger, nil, loadUpkeepAt(upkeep1, 2), loadUpkeepAt(upkeep2, 2)) listener := chain.NewListener(broadcaster, logger) tracker := upkeep.NewActiveTracker(listener, logger) @@ -39,8 +48,8 @@ func TestActiveTracker(t *testing.T) { <-broadcaster.Start() broadcaster.Stop() - assert.Len(t, tracker.GetAllByType(chain.ConditionalType), 1, "should only have 1 conditional upkeep") - assert.Len(t, tracker.GetAllByType(chain.LogTriggerType), 0, "should have 0 log upkeeps") + assert.Len(t, tracker.GetAllByType(chain.ConditionalType), 1, "should have 1 conditional upkeep") + assert.Len(t, tracker.GetAllByType(chain.LogTriggerType), 1, "should have 1 log upkeeps") trackedUpkeep, ok := tracker.GetByUpkeepID(upkeep1.UpkeepID) diff --git a/cmd/simulator/simulate/upkeep/log.go b/tools/simulator/simulate/upkeep/log.go similarity index 98% rename from cmd/simulator/simulate/upkeep/log.go rename to tools/simulator/simulate/upkeep/log.go index e699e3a7..c641385e 100644 --- a/cmd/simulator/simulate/upkeep/log.go +++ b/tools/simulator/simulate/upkeep/log.go @@ -6,7 +6,7 @@ import ( "runtime" "sync" - "github.com/smartcontractkit/ocr2keepers/cmd/simulator/simulate/chain" + "github.com/smartcontractkit/ocr2keepers/tools/simulator/simulate/chain" ) // LogTriggerTracker ... diff --git a/cmd/simulator/simulate/upkeep/log_test.go b/tools/simulator/simulate/upkeep/log_test.go similarity index 86% rename from cmd/simulator/simulate/upkeep/log_test.go rename to tools/simulator/simulate/upkeep/log_test.go index 4e69a4f7..f40d7305 100644 --- a/cmd/simulator/simulate/upkeep/log_test.go +++ b/tools/simulator/simulate/upkeep/log_test.go @@ -7,11 +7,11 @@ import ( "testing" "time" - "github.com/smartcontractkit/ocr2keepers/cmd/simulator/config" - "github.com/smartcontractkit/ocr2keepers/cmd/simulator/simulate/chain" - "github.com/smartcontractkit/ocr2keepers/cmd/simulator/simulate/upkeep" - "github.com/smartcontractkit/ocr2keepers/cmd/simulator/util" ocr2keepers "github.com/smartcontractkit/ocr2keepers/pkg/v3/types" + "github.com/smartcontractkit/ocr2keepers/tools/simulator/config" + "github.com/smartcontractkit/ocr2keepers/tools/simulator/simulate/chain" + "github.com/smartcontractkit/ocr2keepers/tools/simulator/simulate/upkeep" + "github.com/smartcontractkit/ocr2keepers/tools/simulator/util" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -42,7 +42,7 @@ func TestLogTriggerTracker(t *testing.T) { TriggerValue: "test_trigger", } - broadcaster := chain.NewBlockBroadcaster(conf, 1, logger, loadUpkeepAt(upkeep1, 3), loadLogAt(chainLog1, 4)) + broadcaster := chain.NewBlockBroadcaster(conf, 1, logger, nil, loadUpkeepAt(upkeep1, 3), loadLogAt(chainLog1, 4)) listener := chain.NewListener(broadcaster, logger) active := upkeep.NewActiveTracker(listener, logger) performs := upkeep.NewPerformTracker(listener, logger) @@ -77,7 +77,7 @@ func TestLogTriggerTracker_NoUpkeeps(t *testing.T) { TriggerValue: "test_trigger", } - broadcaster := chain.NewBlockBroadcaster(conf, 1, logger, loadLogAt(chainLog1, 4)) + broadcaster := chain.NewBlockBroadcaster(conf, 1, logger, nil, loadLogAt(chainLog1, 4)) listener := chain.NewListener(broadcaster, logger) active := upkeep.NewActiveTracker(listener, logger) performs := upkeep.NewPerformTracker(listener, logger) @@ -142,7 +142,7 @@ func TestLogTriggerTracker_Performed(t *testing.T) { }, } - broadcaster := chain.NewBlockBroadcaster(conf, 1, logger, loadUpkeepAt(upkeep1, 3), loadLogAt(chainLog1, 4), loadPerformAt(perform1, 5)) + broadcaster := chain.NewBlockBroadcaster(conf, 1, logger, nil, loadUpkeepAt(upkeep1, 3), loadLogAt(chainLog1, 4), loadPerformAt(perform1, 5)) listener := chain.NewListener(broadcaster, logger) active := upkeep.NewActiveTracker(listener, logger) performs := upkeep.NewPerformTracker(listener, logger) diff --git a/cmd/simulator/simulate/upkeep/perform.go b/tools/simulator/simulate/upkeep/perform.go similarity index 94% rename from cmd/simulator/simulate/upkeep/perform.go rename to tools/simulator/simulate/upkeep/perform.go index 06025679..8d6268b7 100644 --- a/cmd/simulator/simulate/upkeep/perform.go +++ b/tools/simulator/simulate/upkeep/perform.go @@ -6,9 +6,9 @@ import ( "runtime" "sync" - "github.com/smartcontractkit/ocr2keepers/cmd/simulator/simulate/chain" - "github.com/smartcontractkit/ocr2keepers/cmd/simulator/util" ocr2keepers "github.com/smartcontractkit/ocr2keepers/pkg/v3/types" + "github.com/smartcontractkit/ocr2keepers/tools/simulator/simulate/chain" + "github.com/smartcontractkit/ocr2keepers/tools/simulator/util" ) type PerformTracker struct { diff --git a/cmd/simulator/simulate/upkeep/perform_test.go b/tools/simulator/simulate/upkeep/perform_test.go similarity index 87% rename from cmd/simulator/simulate/upkeep/perform_test.go rename to tools/simulator/simulate/upkeep/perform_test.go index 23372e30..92f95aa4 100644 --- a/cmd/simulator/simulate/upkeep/perform_test.go +++ b/tools/simulator/simulate/upkeep/perform_test.go @@ -7,11 +7,11 @@ import ( "testing" "time" - "github.com/smartcontractkit/ocr2keepers/cmd/simulator/config" - "github.com/smartcontractkit/ocr2keepers/cmd/simulator/simulate/chain" - "github.com/smartcontractkit/ocr2keepers/cmd/simulator/simulate/upkeep" - "github.com/smartcontractkit/ocr2keepers/cmd/simulator/util" ocr2keepers "github.com/smartcontractkit/ocr2keepers/pkg/v3/types" + "github.com/smartcontractkit/ocr2keepers/tools/simulator/config" + "github.com/smartcontractkit/ocr2keepers/tools/simulator/simulate/chain" + "github.com/smartcontractkit/ocr2keepers/tools/simulator/simulate/upkeep" + "github.com/smartcontractkit/ocr2keepers/tools/simulator/util" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -53,7 +53,7 @@ func TestPerformTracker_LogTrigger(t *testing.T) { }, } - broadcaster := chain.NewBlockBroadcaster(conf, 1, logger, loadPerformAt(perform1, 5)) + broadcaster := chain.NewBlockBroadcaster(conf, 1, logger, nil, loadPerformAt(perform1, 5)) listener := chain.NewListener(broadcaster, logger) tracker := upkeep.NewPerformTracker(listener, logger) @@ -103,7 +103,7 @@ func TestPerformTracker_Conditional(t *testing.T) { }, } - broadcaster := chain.NewBlockBroadcaster(conf, 1, logger, loadPerformAt(perform1, 5)) + broadcaster := chain.NewBlockBroadcaster(conf, 1, logger, nil, loadPerformAt(perform1, 5)) listener := chain.NewListener(broadcaster, logger) tracker := upkeep.NewPerformTracker(listener, logger) @@ -144,7 +144,7 @@ func TestPerformTracker_DecodeReportFailure(t *testing.T) { }, } - broadcaster := chain.NewBlockBroadcaster(conf, 1, logger, loadPerformAt(perform1, 5)) + broadcaster := chain.NewBlockBroadcaster(conf, 1, logger, nil, loadPerformAt(perform1, 5)) listener := chain.NewListener(broadcaster, logger) tracker := upkeep.NewPerformTracker(listener, logger) diff --git a/cmd/simulator/simulate/upkeep/pipeline.go b/tools/simulator/simulate/upkeep/pipeline.go similarity index 91% rename from cmd/simulator/simulate/upkeep/pipeline.go rename to tools/simulator/simulate/upkeep/pipeline.go index fc32ace3..c37e4c74 100644 --- a/cmd/simulator/simulate/upkeep/pipeline.go +++ b/tools/simulator/simulate/upkeep/pipeline.go @@ -6,9 +6,9 @@ import ( "math/big" "sync" - "github.com/smartcontractkit/ocr2keepers/cmd/simulator/config" - "github.com/smartcontractkit/ocr2keepers/cmd/simulator/simulate/net" ocr2keepers "github.com/smartcontractkit/ocr2keepers/pkg/v3/types" + "github.com/smartcontractkit/ocr2keepers/tools/simulator/config" + "github.com/smartcontractkit/ocr2keepers/tools/simulator/simulate/net" ) type CheckTelemetry interface { @@ -27,7 +27,7 @@ type CheckPipeline struct { // TODO: provide upkeep configurations to this component // NewCheckPipeline ... func NewCheckPipeline( - conf config.RunBook, + conf config.SimulationPlan, active *ActiveTracker, performs *PerformTracker, netTelemetry net.NetTelemetry, @@ -35,9 +35,9 @@ func NewCheckPipeline( logger *log.Logger, ) *CheckPipeline { service := net.NewSimulatedNetworkService( - conf.RPCDetail.ErrorRate, - conf.RPCDetail.RateLimitThreshold, - conf.RPCDetail.AverageLatency, + conf.RPC.ErrorRate, + conf.RPC.RateLimitThreshold, + conf.RPC.AverageLatency, netTelemetry, ) @@ -123,7 +123,11 @@ func (cp *CheckPipeline) CheckUpkeeps(ctx context.Context, payloads ...ocr2keepe } if simulated, ok := cp.active.GetByUpkeepID(key.UpkeepID); ok { - results[resultIdx].Eligible = isConditionalEligible(simulated.EligibleAt, performs, block) + if simulated.AlwaysEligible { + results[resultIdx].Eligible = true + } else { + results[resultIdx].Eligible = isEligible(simulated.EligibleAt, performs, block) + } cp.logger.Printf("%s eligibility %t at block %d", key.UpkeepID, results[resultIdx].Eligible, block) } else { @@ -156,7 +160,7 @@ func (cp *CheckPipeline) CheckUpkeeps(ctx context.Context, payloads ...ocr2keepe return output, nil } -func isConditionalEligible(eligibles, performs []*big.Int, block *big.Int) bool { +func isEligible(eligibles, performs []*big.Int, block *big.Int) bool { var eligible bool // start at the highest blocks eligible. the first eligible will be a block diff --git a/cmd/simulator/simulate/upkeep/pipeline_test.go b/tools/simulator/simulate/upkeep/pipeline_test.go similarity index 89% rename from cmd/simulator/simulate/upkeep/pipeline_test.go rename to tools/simulator/simulate/upkeep/pipeline_test.go index 9c63d7a8..6a511367 100644 --- a/cmd/simulator/simulate/upkeep/pipeline_test.go +++ b/tools/simulator/simulate/upkeep/pipeline_test.go @@ -12,24 +12,24 @@ import ( "github.com/stretchr/testify/mock" "github.com/stretchr/testify/require" - "github.com/smartcontractkit/ocr2keepers/cmd/simulator/config" - "github.com/smartcontractkit/ocr2keepers/cmd/simulator/simulate/chain" - "github.com/smartcontractkit/ocr2keepers/cmd/simulator/util" ocr2keepers "github.com/smartcontractkit/ocr2keepers/pkg/v3/types" + "github.com/smartcontractkit/ocr2keepers/tools/simulator/config" + "github.com/smartcontractkit/ocr2keepers/tools/simulator/simulate/chain" + "github.com/smartcontractkit/ocr2keepers/tools/simulator/util" ) func TestCheckPipeline(t *testing.T) { t.Parallel() logger := log.New(io.Discard, "", 0) - conf := config.RunBook{ - BlockCadence: config.Blocks{ + conf := config.SimulationPlan{ + Blocks: config.Blocks{ Genesis: new(big.Int).SetInt64(1), Cadence: config.Duration(50 * time.Millisecond), Jitter: config.Duration(0), Duration: 5, }, - RPCDetail: config.RPC{ + RPC: config.RPC{ ErrorRate: 0.0, RateLimitThreshold: 10, AverageLatency: 10, @@ -51,7 +51,7 @@ func TestCheckPipeline(t *testing.T) { netTel := new(mockNetTelemetry) conTel := new(mockCheckTelemetry) - broadcaster := chain.NewBlockBroadcaster(conf.BlockCadence, 1, logger, loadUpkeepAt(upkeep1, 2)) + broadcaster := chain.NewBlockBroadcaster(conf.Blocks, 1, logger, nil, loadUpkeepAt(upkeep1, 2)) listener := chain.NewListener(broadcaster, logger) active := NewActiveTracker(listener, logger) performs := NewPerformTracker(listener, logger) @@ -79,7 +79,7 @@ func TestCheckPipeline(t *testing.T) { conTel.AssertExpectations(t) } -func TestIsConditionalEligible(t *testing.T) { +func TestIsEligible(t *testing.T) { t.Parallel() tests := []struct { @@ -134,7 +134,7 @@ func TestIsConditionalEligible(t *testing.T) { } for i := range tests { - eligible := isConditionalEligible(tests[i].eligibles, tests[i].performs, tests[i].block) + eligible := isEligible(tests[i].eligibles, tests[i].performs, tests[i].block) if eligible != tests[i].expected { t.Logf("%s was %t but was not expected", tests[i].name, eligible) diff --git a/cmd/simulator/simulate/upkeep/source.go b/tools/simulator/simulate/upkeep/source.go similarity index 97% rename from cmd/simulator/simulate/upkeep/source.go rename to tools/simulator/simulate/upkeep/source.go index 29cb026e..f04483fd 100644 --- a/cmd/simulator/simulate/upkeep/source.go +++ b/tools/simulator/simulate/upkeep/source.go @@ -5,9 +5,9 @@ import ( "log" "math/big" - "github.com/smartcontractkit/ocr2keepers/cmd/simulator/simulate/chain" - "github.com/smartcontractkit/ocr2keepers/cmd/simulator/util" ocr2keepers "github.com/smartcontractkit/ocr2keepers/pkg/v3/types" + "github.com/smartcontractkit/ocr2keepers/tools/simulator/simulate/chain" + "github.com/smartcontractkit/ocr2keepers/tools/simulator/util" ) // Source maintains delivery of active upkeeps based on type and repeat diff --git a/cmd/simulator/simulate/upkeep/source_test.go b/tools/simulator/simulate/upkeep/source_test.go similarity index 87% rename from cmd/simulator/simulate/upkeep/source_test.go rename to tools/simulator/simulate/upkeep/source_test.go index f598d85f..65047d61 100644 --- a/cmd/simulator/simulate/upkeep/source_test.go +++ b/tools/simulator/simulate/upkeep/source_test.go @@ -11,11 +11,11 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - "github.com/smartcontractkit/ocr2keepers/cmd/simulator/config" - "github.com/smartcontractkit/ocr2keepers/cmd/simulator/simulate/chain" - "github.com/smartcontractkit/ocr2keepers/cmd/simulator/simulate/upkeep" - "github.com/smartcontractkit/ocr2keepers/cmd/simulator/util" ocr2keepers "github.com/smartcontractkit/ocr2keepers/pkg/v3/types" + "github.com/smartcontractkit/ocr2keepers/tools/simulator/config" + "github.com/smartcontractkit/ocr2keepers/tools/simulator/simulate/chain" + "github.com/smartcontractkit/ocr2keepers/tools/simulator/simulate/upkeep" + "github.com/smartcontractkit/ocr2keepers/tools/simulator/util" ) func TestSource_GetActiveUpkeeps(t *testing.T) { @@ -35,7 +35,7 @@ func TestSource_GetActiveUpkeeps(t *testing.T) { Type: chain.ConditionalType, } - broadcaster := chain.NewBlockBroadcaster(conf, 1, logger, loadUpkeepAt(upkeep1, 2)) + broadcaster := chain.NewBlockBroadcaster(conf, 1, logger, nil, loadUpkeepAt(upkeep1, 2)) listener := chain.NewListener(broadcaster, logger) active := upkeep.NewActiveTracker(listener, logger) @@ -81,7 +81,7 @@ func TestSource_GetLatestPayloads(t *testing.T) { TriggerValue: "test_trigger", } - broadcaster := chain.NewBlockBroadcaster(conf, 1, logger, loadUpkeepAt(upkeep1, 3), loadLogAt(chainLog1, 4)) + broadcaster := chain.NewBlockBroadcaster(conf, 1, logger, nil, loadUpkeepAt(upkeep1, 3), loadLogAt(chainLog1, 4)) listener := chain.NewListener(broadcaster, logger) active := upkeep.NewActiveTracker(listener, logger) performs := upkeep.NewPerformTracker(listener, logger) @@ -129,7 +129,7 @@ func TestSource_GetRecoveryProposals(t *testing.T) { TriggerValue: "test_trigger", } - broadcaster := chain.NewBlockBroadcaster(conf, 1, logger, loadUpkeepAt(upkeep1, 3), loadLogAt(chainLog1, 4)) + broadcaster := chain.NewBlockBroadcaster(conf, 1, logger, nil, loadUpkeepAt(upkeep1, 3), loadLogAt(chainLog1, 4)) listener := chain.NewListener(broadcaster, logger) active := upkeep.NewActiveTracker(listener, logger) performs := upkeep.NewPerformTracker(listener, logger) @@ -181,7 +181,7 @@ func TestSource_BuildPayloads(t *testing.T) { workID := util.UpkeepWorkID(upkeep1.UpkeepID, trigger1) - broadcaster := chain.NewBlockBroadcaster(conf, 1, logger, loadUpkeepAt(upkeep1, 2)) + broadcaster := chain.NewBlockBroadcaster(conf, 1, logger, nil, loadUpkeepAt(upkeep1, 2)) listener := chain.NewListener(broadcaster, logger) active := upkeep.NewActiveTracker(listener, logger) diff --git a/cmd/simulator/simulate/upkeep/util.go b/tools/simulator/simulate/upkeep/util.go similarity index 94% rename from cmd/simulator/simulate/upkeep/util.go rename to tools/simulator/simulate/upkeep/util.go index 8b68623c..86fecdd7 100644 --- a/cmd/simulator/simulate/upkeep/util.go +++ b/tools/simulator/simulate/upkeep/util.go @@ -3,8 +3,8 @@ package upkeep import ( "fmt" - "github.com/smartcontractkit/ocr2keepers/cmd/simulator/util" ocr2keepers "github.com/smartcontractkit/ocr2keepers/pkg/v3/types" + "github.com/smartcontractkit/ocr2keepers/tools/simulator/util" ) var ( diff --git a/cmd/simulator/simulate/upkeep/util_test.go b/tools/simulator/simulate/upkeep/util_test.go similarity index 85% rename from cmd/simulator/simulate/upkeep/util_test.go rename to tools/simulator/simulate/upkeep/util_test.go index ec3a2814..3c934ab1 100644 --- a/cmd/simulator/simulate/upkeep/util_test.go +++ b/tools/simulator/simulate/upkeep/util_test.go @@ -6,8 +6,8 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - "github.com/smartcontractkit/ocr2keepers/cmd/simulator/simulate/upkeep" ocr2keepers "github.com/smartcontractkit/ocr2keepers/pkg/v3/types" + "github.com/smartcontractkit/ocr2keepers/tools/simulator/simulate/upkeep" ) func TestUtil_EncodeDecode(t *testing.T) { diff --git a/cmd/simulator/telemetry/base.go b/tools/simulator/telemetry/base.go similarity index 100% rename from cmd/simulator/telemetry/base.go rename to tools/simulator/telemetry/base.go diff --git a/cmd/simulator/telemetry/contract.go b/tools/simulator/telemetry/contract.go similarity index 100% rename from cmd/simulator/telemetry/contract.go rename to tools/simulator/telemetry/contract.go diff --git a/cmd/simulator/telemetry/log.go b/tools/simulator/telemetry/log.go similarity index 100% rename from cmd/simulator/telemetry/log.go rename to tools/simulator/telemetry/log.go diff --git a/tools/simulator/telemetry/progress.go b/tools/simulator/telemetry/progress.go new file mode 100644 index 00000000..1d16ac99 --- /dev/null +++ b/tools/simulator/telemetry/progress.go @@ -0,0 +1,129 @@ +package telemetry + +import ( + "io" + "sync/atomic" + "time" + + "github.com/jedib0t/go-pretty/v6/progress" +) + +const ( + trackerProgressCheck = 100 * time.Millisecond +) + +type ProgressTelemetry struct { + success atomic.Bool + writer progress.Writer + incrementMap map[string]chan int64 + chDone chan struct{} +} + +func NewProgressTelemetry(wOutput io.Writer) *ProgressTelemetry { + writer := progress.NewWriter() + + writer.SetOutputWriter(wOutput) + + writer.LengthDone() + writer.SetAutoStop(false) + writer.SetTrackerLength(25) + writer.SetMessageWidth(44) + writer.SetSortBy(progress.SortByPercentDsc) + writer.SetStyle(progress.StyleDefault) + writer.SetTrackerPosition(progress.PositionRight) + writer.SetUpdateFrequency(time.Millisecond * 100) + + writer.Style().Colors = progress.StyleColorsExample + writer.Style().Options.PercentFormat = "%4.1f%%" + writer.Style().Visibility.ETA = true + writer.Style().Visibility.ETAOverall = true + writer.Style().Visibility.Percentage = true + writer.Style().Visibility.Speed = true + writer.Style().Visibility.SpeedOverall = true + writer.Style().Visibility.Time = true + writer.Style().Visibility.TrackerOverall = true + writer.Style().Visibility.Value = true + writer.Style().Visibility.Pinned = true + + return &ProgressTelemetry{ + writer: writer, + incrementMap: make(map[string]chan int64), + chDone: make(chan struct{}), + } +} + +func (t *ProgressTelemetry) Register(namespace string, total int64) error { + chIncrements := make(chan int64, 100) + t.incrementMap[namespace] = chIncrements + + go t.track(namespace, total, chIncrements) + + return nil +} + +func (t *ProgressTelemetry) Increment(namespace string, count int64) { + if chIncrements, exists := t.incrementMap[namespace]; exists { + go func() { + chIncrements <- count + }() + } +} + +func (t *ProgressTelemetry) AllProgressComplete() bool { + return t.success.Load() +} + +func (t *ProgressTelemetry) Start() { + go t.writer.Render() + go t.checkProgress() +} + +func (t *ProgressTelemetry) Close() error { + close(t.chDone) + + return nil +} + +func (t *ProgressTelemetry) track(namespace string, total int64, chIncrements chan int64) { + tracker := progress.Tracker{ + Message: namespace, + Total: total, + Units: progress.UnitsDefault, + DeferStart: true, + } + + t.writer.AppendTracker(&tracker) + + for !tracker.IsDone() { + select { + case increment := <-chIncrements: + tracker.Increment(increment) + + if tracker.Value() == total { + tracker.MarkAsDone() + } + case <-t.chDone: + tracker.MarkAsErrored() + } + } +} + +func (t *ProgressTelemetry) checkProgress() { + ticker := time.NewTicker(trackerProgressCheck) + + for t.writer.IsRenderInProgress() { + select { + case <-ticker.C: + if t.writer.LengthActive() == 0 { + t.writer.Stop() + } + case <-t.chDone: + // wait for all trackers to complete + time.Sleep(500 * time.Millisecond) + + t.writer.Stop() + } + } + + t.success.Store(t.writer.Length() == t.writer.LengthDone()) +} diff --git a/cmd/simulator/telemetry/rpc.go b/tools/simulator/telemetry/rpc.go similarity index 100% rename from cmd/simulator/telemetry/rpc.go rename to tools/simulator/telemetry/rpc.go diff --git a/cmd/simulator/util/encode.go b/tools/simulator/util/encode.go similarity index 100% rename from cmd/simulator/util/encode.go rename to tools/simulator/util/encode.go diff --git a/cmd/simulator/util/rand.go b/tools/simulator/util/rand.go similarity index 100% rename from cmd/simulator/util/rand.go rename to tools/simulator/util/rand.go diff --git a/cmd/simulator/util/sort.go b/tools/simulator/util/sort.go similarity index 100% rename from cmd/simulator/util/sort.go rename to tools/simulator/util/sort.go