diff --git a/.changeset/fuzzy-hairs-appear.md b/.changeset/fuzzy-hairs-appear.md new file mode 100644 index 00000000000..a4797462546 --- /dev/null +++ b/.changeset/fuzzy-hairs-appear.md @@ -0,0 +1,5 @@ +--- +"chainlink": patch +--- + +Prometheus observability layer added to OCR3 Reporting Plugins #internal diff --git a/.github/e2e-tests.yml b/.github/e2e-tests.yml index a671c081c1a..3b394293378 100644 --- a/.github/e2e-tests.yml +++ b/.github/e2e-tests.yml @@ -948,19 +948,6 @@ runner-test-matrix: E2E_TEST_SELECTED_NETWORK: SIMULATED_1,SIMULATED_2 E2E_JD_VERSION: 0.6.0 - - id: smoke/ccip/ccip_batching_test.go:* - path: integration-tests/smoke/ccip/ccip_batching_test.go - test_env_type: docker - runs_on: ubuntu-latest - triggers: - - PR E2E Core Tests - - Nightly E2E Tests - test_cmd: cd integration-tests/ && go test smoke/ccip/ccip_batching_test.go -timeout 12m -test.parallel=1 -count=1 -json - pyroscope_env: ci-smoke-ccipv1_6-evm-simulated - test_env_vars: - E2E_TEST_SELECTED_NETWORK: SIMULATED_1,SIMULATED_2,SIMULATED_3 - E2E_JD_VERSION: 0.6.0 - - id: smoke/ccip/ccip_token_transfer_test.go:* path: integration-tests/smoke/ccip/ccip_token_transfer_test.go test_env_type: docker diff --git a/.github/integration-in-memory-tests.yml b/.github/integration-in-memory-tests.yml index c5e0f088afe..4b4fd71258d 100644 --- a/.github/integration-in-memory-tests.yml +++ b/.github/integration-in-memory-tests.yml @@ -31,6 +31,14 @@ runner-test-matrix: triggers: - PR Integration CCIP Tests test_cmd: cd integration-tests/smoke/ccip && go test ccip_fee_boosting_test.go -timeout 12m -test.parallel=2 -count=1 -json + + - id: smoke/ccip/ccip_batching_test.go:* + path: integration-tests/smoke/ccip/ccip_batching_test.go + test_env_type: in-memory + runs_on: ubuntu-latest + triggers: + - PR Integration CCIP Tests + test_cmd: cd integration-tests/smoke/ccip && go test ccip_batching_test.go -timeout 12m -test.parallel=2 -count=1 -json - id: contracts/ccipreader_test.go:* path: integration-tests/contracts/ccipreader_test.go diff --git a/.github/workflows/changeset.yml b/.github/workflows/changeset.yml index 5d6b2deafe2..331492eb74f 100644 --- a/.github/workflows/changeset.yml +++ b/.github/workflows/changeset.yml @@ -182,7 +182,7 @@ jobs: - added|modified: 'contracts/.changeset/*.md' - name: Setup node - uses: ./.github/actions/setup-node + uses: ./.github/actions/setup-nodejs if: ${{ steps.files-changed.outputs.contracts-changeset == 'true' }} - name: Validate changeset files diff --git a/contracts/src/v0.8/ccip/capability/CCIPHome.sol b/contracts/src/v0.8/ccip/capability/CCIPHome.sol index 7a425566c33..e43e4b0d03f 100644 --- a/contracts/src/v0.8/ccip/capability/CCIPHome.sol +++ b/contracts/src/v0.8/ccip/capability/CCIPHome.sol @@ -62,7 +62,7 @@ import {EnumerableSet} from "../../vendor/openzeppelin-solidity/v5.0.2/contracts /// │ Active │ revokeCandidate │ Candidate │◄───────────┐ /// │ [1,0] │◄───────────────────┤ [1,1] │────────────┘ /// │ ├───────────────────►│ │ -/// └─────────────┘ setSecondary └─────────────┘ +/// └─────────────┘ setCandidate └─────────────┘ /// contract CCIPHome is Ownable2StepMsgSender, ITypeAndVersion, ICapabilityConfiguration, IERC165 { using EnumerableSet for EnumerableSet.UintSet; diff --git a/contracts/src/v0.8/ccip/rmn/RMNHome.sol b/contracts/src/v0.8/ccip/rmn/RMNHome.sol index 684dc3e994e..92aab25ae8e 100644 --- a/contracts/src/v0.8/ccip/rmn/RMNHome.sol +++ b/contracts/src/v0.8/ccip/rmn/RMNHome.sol @@ -54,7 +54,7 @@ import {Ownable2StepMsgSender} from "../../shared/access/Ownable2StepMsgSender.s /// │ Active │ revokeCandidate │ Candidate │◄───────────┐ /// │ [1,0] │◄───────────────────┤ [1,1] │────────────┘ /// │ ├───────────────────►│ │ -/// └─────────────┘ setSecondary └─────────────┘ +/// └─────────────┘ setCandidate └─────────────┘ /// contract RMNHome is Ownable2StepMsgSender, ITypeAndVersion { event ConfigSet(bytes32 indexed configDigest, uint32 version, StaticConfig staticConfig, DynamicConfig dynamicConfig); diff --git a/core/capabilities/ccip/oraclecreator/plugin.go b/core/capabilities/ccip/oraclecreator/plugin.go index f8868b5d9b9..1b8c6344349 100644 --- a/core/capabilities/ccip/oraclecreator/plugin.go +++ b/core/capabilities/ccip/oraclecreator/plugin.go @@ -22,6 +22,7 @@ import ( cctypes "github.com/smartcontractkit/chainlink/v2/core/capabilities/ccip/types" "github.com/smartcontractkit/chainlink/v2/core/chains/evm/assets" "github.com/smartcontractkit/chainlink/v2/core/chains/evm/config/toml" + "github.com/smartcontractkit/chainlink/v2/core/services/ocr3/promwrapper" "github.com/smartcontractkit/libocr/commontypes" libocr3 "github.com/smartcontractkit/libocr/offchainreporting2plus" @@ -229,6 +230,12 @@ func (i *pluginOracleCreator) createFactoryAndTransmitter( ) (ocr3types.ReportingPluginFactory[[]byte], ocr3types.ContractTransmitter[[]byte], error) { var factory ocr3types.ReportingPluginFactory[[]byte] var transmitter ocr3types.ContractTransmitter[[]byte] + + chainID, err := chainsel.GetChainIDFromSelector(uint64(config.Config.ChainSelector)) + if err != nil { + return nil, nil, fmt.Errorf("unsupported chain selector %d %w", config.Config.ChainSelector, err) + } + if config.Config.PluginType == uint8(cctypes.PluginTypeCCIPCommit) { if !i.peerWrapper.IsStarted() { return nil, nil, fmt.Errorf("peer wrapper is not started") @@ -263,6 +270,7 @@ func (i *pluginOracleCreator) createFactoryAndTransmitter( rmnPeerClient, rmnCrypto, ) + factory = promwrapper.NewReportingPluginFactory[[]byte](factory, chainID, "CCIPCommit") transmitter = ocrimpls.NewCommitContractTransmitter[[]byte](destChainWriter, ocrtypes.Account(destFromAccounts[0]), hexutil.Encode(config.Config.OfframpAddress), // TODO: this works for evm only, how about non-evm? @@ -283,6 +291,7 @@ func (i *pluginOracleCreator) createFactoryAndTransmitter( contractReaders, chainWriters, ) + factory = promwrapper.NewReportingPluginFactory[[]byte](factory, chainID, "CCIPExec") transmitter = ocrimpls.NewExecContractTransmitter[[]byte](destChainWriter, ocrtypes.Account(destFromAccounts[0]), hexutil.Encode(config.Config.OfframpAddress), // TODO: this works for evm only, how about non-evm? diff --git a/core/capabilities/compute/compute.go b/core/capabilities/compute/compute.go index 32e43e8d62e..316e4f00eea 100644 --- a/core/capabilities/compute/compute.go +++ b/core/capabilities/compute/compute.go @@ -273,9 +273,19 @@ func (c *Compute) worker(ctx context.Context) { } func (c *Compute) Close() error { + ctx, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() + c.modules.close() close(c.stopCh) + + err := c.registry.Remove(ctx, CapabilityIDCompute) + if err != nil { + return err + } + c.wg.Wait() + return nil } diff --git a/core/capabilities/registry.go b/core/capabilities/registry.go index 47285505805..7038dcdb4b7 100644 --- a/core/capabilities/registry.go +++ b/core/capabilities/registry.go @@ -193,6 +193,21 @@ func (r *Registry) Add(ctx context.Context, c capabilities.BaseCapability) error return nil } +// Add adds a capability to the registry. +func (r *Registry) Remove(ctx context.Context, id string) error { + r.mu.Lock() + defer r.mu.Unlock() + + _, ok := r.m[id] + if !ok { + return fmt.Errorf("unable to remove, capability not found: %s", id) + } + + delete(r.m, id) + r.lggr.Infow("capability removed", "id", id) + return nil +} + // NewRegistry returns a new Registry. func NewRegistry(lggr logger.Logger) *Registry { return &Registry{ diff --git a/core/capabilities/webapi/target/target.go b/core/capabilities/webapi/target/target.go index 4576f95a54e..b211e0fe837 100644 --- a/core/capabilities/webapi/target/target.go +++ b/core/capabilities/webapi/target/target.go @@ -5,6 +5,7 @@ import ( "encoding/json" "fmt" "strings" + "time" "github.com/smartcontractkit/chainlink-common/pkg/capabilities" "github.com/smartcontractkit/chainlink-common/pkg/logger" @@ -57,6 +58,12 @@ func (c *Capability) Start(ctx context.Context) error { } func (c *Capability) Close() error { + ctx, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() + err := c.registry.Remove(ctx, c.capabilityInfo.ID) + if err != nil { + return err + } return nil } diff --git a/core/capabilities/webapi/trigger/trigger.go b/core/capabilities/webapi/trigger/trigger.go index c607f0dbb6f..712cf38a4cc 100644 --- a/core/capabilities/webapi/trigger/trigger.go +++ b/core/capabilities/webapi/trigger/trigger.go @@ -6,6 +6,7 @@ import ( "errors" "fmt" "sync" + "time" ethCommon "github.com/ethereum/go-ethereum/common" @@ -256,6 +257,12 @@ func (h *triggerConnectorHandler) Start(ctx context.Context) error { } func (h *triggerConnectorHandler) Close() error { return h.StopOnce("GatewayConnectorServiceWrapper", func() error { + ctx, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() + err := h.registry.Remove(ctx, h.ID) + if err != nil { + return err + } return nil }) } diff --git a/core/cmd/shell.go b/core/cmd/shell.go index 788e4da6f69..53ba5e3f82f 100644 --- a/core/cmd/shell.go +++ b/core/cmd/shell.go @@ -53,6 +53,7 @@ import ( "github.com/smartcontractkit/chainlink/v2/core/services/relay/evm/mercury/wsrpc/cache" "github.com/smartcontractkit/chainlink/v2/core/services/versioning" "github.com/smartcontractkit/chainlink/v2/core/services/webhook" + "github.com/smartcontractkit/chainlink/v2/core/services/workflows" "github.com/smartcontractkit/chainlink/v2/core/sessions" "github.com/smartcontractkit/chainlink/v2/core/static" "github.com/smartcontractkit/chainlink/v2/core/store/migrate" @@ -111,6 +112,10 @@ func initGlobals(cfgProm config.Prometheus, cfgTracing config.Tracing, cfgTeleme AuthPublicKeyHex: csaPubKeyHex, AuthHeaders: beholderAuthHeaders, } + // note: due to the OTEL specification, all histogram buckets + // must be defined when the beholder client is created + clientCfg.MetricViews = append(clientCfg.MetricViews, workflows.MetricViews()...) + if tracingCfg.Enabled { clientCfg.TraceSpanExporter, err = tracingCfg.NewSpanExporter() if err != nil { diff --git a/core/scripts/go.mod b/core/scripts/go.mod index 94eea2b4b3a..d468907c550 100644 --- a/core/scripts/go.mod +++ b/core/scripts/go.mod @@ -2,6 +2,8 @@ module github.com/smartcontractkit/chainlink/core/scripts go 1.23.3 +toolchain go1.23.4 + // Make sure we're working with the latest chainlink libs replace github.com/smartcontractkit/chainlink/v2 => ../../ @@ -24,7 +26,7 @@ require ( github.com/prometheus/client_golang v1.20.5 github.com/shopspring/decimal v1.4.0 github.com/smartcontractkit/chainlink-automation v0.8.1 - github.com/smartcontractkit/chainlink-common v0.3.1-0.20241127162636-07aa781ee1f4 + github.com/smartcontractkit/chainlink-common v0.3.1-0.20241206011233-b6684ee6508f github.com/smartcontractkit/chainlink/deployment v0.0.0-00010101000000-000000000000 github.com/smartcontractkit/chainlink/v2 v2.14.0-mercury-20240807.0.20241106193309-5560cd76211a github.com/smartcontractkit/libocr v0.0.0-20241007185508-adbe57025f12 @@ -304,6 +306,7 @@ require ( github.com/smartcontractkit/chainlink-protos/orchestrator v0.3.2 // indirect github.com/smartcontractkit/chainlink-solana v1.1.1-0.20241204153209-c3a71b0eef99 // indirect github.com/smartcontractkit/chainlink-starknet/relayer v0.1.1-0.20241202202529-2033490e77b8 // indirect + github.com/smartcontractkit/chainlink-testing-framework/lib v1.50.13 // indirect github.com/smartcontractkit/grpc-proxy v0.0.0-20240830132753-a7e17fec5ab7 // indirect github.com/smartcontractkit/tdh2/go/ocr2/decryptionplugin v0.0.0-20241009055228-33d0c0bf38de // indirect github.com/smartcontractkit/tdh2/go/tdh2 v0.0.0-20241009055228-33d0c0bf38de // indirect diff --git a/core/scripts/go.sum b/core/scripts/go.sum index e312f248ceb..225ae3b9191 100644 --- a/core/scripts/go.sum +++ b/core/scripts/go.sum @@ -1142,8 +1142,8 @@ github.com/smartcontractkit/chainlink-automation v0.8.1 h1:sTc9LKpBvcKPc1JDYAmgB github.com/smartcontractkit/chainlink-automation v0.8.1/go.mod h1:Iij36PvWZ6blrdC5A/nrQUBuf3MH3JvsBB9sSyc9W08= github.com/smartcontractkit/chainlink-ccip v0.0.0-20241204015713-8956bb614e9e h1:GnM6ZWV6vlk2+n6c6o+v/R1LtXzBGVVx7r37nt/h6Uc= github.com/smartcontractkit/chainlink-ccip v0.0.0-20241204015713-8956bb614e9e/go.mod h1:80vGBbOfertJig0xFKsRfm+i17FkjdKkk1dAaGE45Os= -github.com/smartcontractkit/chainlink-common v0.3.1-0.20241127162636-07aa781ee1f4 h1:atCZ1jol7a+tdtgU/wNqXgliBun5H7BjGBicGL8Tj6o= -github.com/smartcontractkit/chainlink-common v0.3.1-0.20241127162636-07aa781ee1f4/go.mod h1:bQktEJf7sJ0U3SmIcXvbGUox7SmXcnSEZ4kUbT8R5Nk= +github.com/smartcontractkit/chainlink-common v0.3.1-0.20241206011233-b6684ee6508f h1:hH+cAG2zt+WK4I2m572LXAnAJg3wtGEAwzBKR8FiXo8= +github.com/smartcontractkit/chainlink-common v0.3.1-0.20241206011233-b6684ee6508f/go.mod h1:bQktEJf7sJ0U3SmIcXvbGUox7SmXcnSEZ4kUbT8R5Nk= github.com/smartcontractkit/chainlink-cosmos v0.5.2-0.20241202195413-82468150ac1e h1:PRoeby6ZlTuTkv2f+7tVU4+zboTfRzI+beECynF4JQ0= github.com/smartcontractkit/chainlink-cosmos v0.5.2-0.20241202195413-82468150ac1e/go.mod h1:mUh5/woemsVaHgTorA080hrYmO3syBCmPdnWc/5dOqk= github.com/smartcontractkit/chainlink-data-streams v0.1.1-0.20241202141438-a90db35252db h1:N1RH1hSr2ACzOFc9hkCcjE8pRBTdcU3p8nsTJByaLes= @@ -1158,6 +1158,8 @@ github.com/smartcontractkit/chainlink-solana v1.1.1-0.20241204153209-c3a71b0eef9 github.com/smartcontractkit/chainlink-solana v1.1.1-0.20241204153209-c3a71b0eef99/go.mod h1:p8aUDfJeley6oer7y+Ucd3edOtRlMTnWg3mN6rhaLWo= github.com/smartcontractkit/chainlink-starknet/relayer v0.1.1-0.20241202202529-2033490e77b8 h1:tNS7U9lrxkFvEuyxQv11HHOiV9LPDGC9wYEy+yM/Jv4= github.com/smartcontractkit/chainlink-starknet/relayer v0.1.1-0.20241202202529-2033490e77b8/go.mod h1:EBrEgcdIbwepqguClkv8Ohy7CbyWSJaE4EC9aBJlQK0= +github.com/smartcontractkit/chainlink-testing-framework/lib v1.50.13 h1:T0kbw07Vb6xUyA9MIJZfErMgWseWi1zf7cYvRpoq7ug= +github.com/smartcontractkit/chainlink-testing-framework/lib v1.50.13/go.mod h1:1CKUOzoK+Ga19WuhRH9pxZ+qUUnrlIx108VEA6qSzeQ= github.com/smartcontractkit/grpc-proxy v0.0.0-20240830132753-a7e17fec5ab7 h1:12ijqMM9tvYVEm+nR826WsrNi6zCKpwBhuApq127wHs= github.com/smartcontractkit/grpc-proxy v0.0.0-20240830132753-a7e17fec5ab7/go.mod h1:FX7/bVdoep147QQhsOPkYsPEXhGZjeYx6lBSaSXtZOA= github.com/smartcontractkit/libocr v0.0.0-20241007185508-adbe57025f12 h1:NzZGjaqez21I3DU7objl3xExTH4fxYvzTqar8DC6360= diff --git a/core/services/chainlink/application.go b/core/services/chainlink/application.go index 863c5d915e9..29473c4d932 100644 --- a/core/services/chainlink/application.go +++ b/core/services/chainlink/application.go @@ -300,18 +300,19 @@ func NewApplication(opts ApplicationOpts) (Application, error) { return nil, fmt.Errorf("expected 1 key, got %d", len(keys)) } - fetcher := syncer.NewFetcherService(globalLogger, gatewayConnectorWrapper) + lggr := globalLogger.Named("WorkflowRegistrySyncer") + fetcher := syncer.NewFetcherService(lggr, gatewayConnectorWrapper) - eventHandler := syncer.NewEventHandler(globalLogger, syncer.NewWorkflowRegistryDS(opts.DS, globalLogger), - fetcher.Fetch, workflowstore.NewDBStore(opts.DS, globalLogger, clockwork.NewRealClock()), opts.CapabilitiesRegistry, + eventHandler := syncer.NewEventHandler(lggr, syncer.NewWorkflowRegistryDS(opts.DS, globalLogger), + fetcher.Fetch, workflowstore.NewDBStore(opts.DS, lggr, clockwork.NewRealClock()), opts.CapabilitiesRegistry, custmsg.NewLabeler(), clockwork.NewRealClock(), keys[0]) - loader := syncer.NewWorkflowRegistryContractLoader(globalLogger, cfg.Capabilities().WorkflowRegistry().Address(), func(ctx context.Context, bytes []byte) (syncer.ContractReader, error) { + loader := syncer.NewWorkflowRegistryContractLoader(lggr, cfg.Capabilities().WorkflowRegistry().Address(), func(ctx context.Context, bytes []byte) (syncer.ContractReader, error) { return relayer.NewContractReader(ctx, bytes) }, eventHandler) globalLogger.Debugw("Creating WorkflowRegistrySyncer") - wfSyncer := syncer.NewWorkflowRegistry(globalLogger, func(ctx context.Context, bytes []byte) (syncer.ContractReader, error) { + wfSyncer := syncer.NewWorkflowRegistry(lggr, func(ctx context.Context, bytes []byte) (syncer.ContractReader, error) { return relayer.NewContractReader(ctx, bytes) }, cfg.Capabilities().WorkflowRegistry().Address(), syncer.WorkflowEventPollerConfig{ diff --git a/core/services/ocr3/promwrapper/factory.go b/core/services/ocr3/promwrapper/factory.go new file mode 100644 index 00000000000..0dabd346112 --- /dev/null +++ b/core/services/ocr3/promwrapper/factory.go @@ -0,0 +1,42 @@ +package promwrapper + +import ( + "context" + + "github.com/smartcontractkit/libocr/offchainreporting2plus/ocr3types" +) + +var _ ocr3types.ReportingPluginFactory[any] = &ReportingPluginFactory[any]{} + +type ReportingPluginFactory[RI any] struct { + origin ocr3types.ReportingPluginFactory[RI] + chainID string + plugin string +} + +func NewReportingPluginFactory[RI any]( + origin ocr3types.ReportingPluginFactory[RI], + chainID string, + plugin string, +) *ReportingPluginFactory[RI] { + return &ReportingPluginFactory[RI]{ + origin: origin, + chainID: chainID, + plugin: plugin, + } +} + +func (r ReportingPluginFactory[RI]) NewReportingPlugin(ctx context.Context, config ocr3types.ReportingPluginConfig) (ocr3types.ReportingPlugin[RI], ocr3types.ReportingPluginInfo, error) { + plugin, info, err := r.origin.NewReportingPlugin(ctx, config) + if err != nil { + return nil, ocr3types.ReportingPluginInfo{}, err + } + wrapped := newReportingPlugin( + plugin, + r.chainID, + r.plugin, + promOCR3ReportsGenerated, + promOCR3Durations, + ) + return wrapped, info, err +} diff --git a/core/services/ocr3/promwrapper/factory_test.go b/core/services/ocr3/promwrapper/factory_test.go new file mode 100644 index 00000000000..72f35aad172 --- /dev/null +++ b/core/services/ocr3/promwrapper/factory_test.go @@ -0,0 +1,41 @@ +package promwrapper + +import ( + "context" + "errors" + "testing" + + "github.com/stretchr/testify/require" + + "github.com/smartcontractkit/libocr/offchainreporting2plus/ocr3types" + + "github.com/smartcontractkit/chainlink-common/pkg/utils/tests" +) + +func Test_WrapperFactory(t *testing.T) { + validFactory := NewReportingPluginFactory(fakeFactory[uint]{}, "solana", "plugin") + failingFactory := NewReportingPluginFactory(fakeFactory[uint]{err: errors.New("error")}, "123", "plugin") + + plugin, _, err := validFactory.NewReportingPlugin(tests.Context(t), ocr3types.ReportingPluginConfig{}) + require.NoError(t, err) + + _, err = plugin.Outcome(tests.Context(t), ocr3types.OutcomeContext{}, nil, nil) + require.NoError(t, err) + + require.Equal(t, 1, counterFromHistogramByLabels(t, promOCR3Durations, "solana", "plugin", "outcome", "true")) + require.Equal(t, 0, counterFromHistogramByLabels(t, promOCR3Durations, "solana", "plugin", "outcome", "false")) + + _, _, err = failingFactory.NewReportingPlugin(tests.Context(t), ocr3types.ReportingPluginConfig{}) + require.Error(t, err) +} + +type fakeFactory[RI any] struct { + err error +} + +func (f fakeFactory[RI]) NewReportingPlugin(context.Context, ocr3types.ReportingPluginConfig) (ocr3types.ReportingPlugin[RI], ocr3types.ReportingPluginInfo, error) { + if f.err != nil { + return nil, ocr3types.ReportingPluginInfo{}, f.err + } + return fakePlugin[RI]{}, ocr3types.ReportingPluginInfo{}, nil +} diff --git a/core/services/ocr3/promwrapper/plugin.go b/core/services/ocr3/promwrapper/plugin.go new file mode 100644 index 00000000000..e4e0c3d35d5 --- /dev/null +++ b/core/services/ocr3/promwrapper/plugin.go @@ -0,0 +1,122 @@ +package promwrapper + +import ( + "context" + "strconv" + "time" + + "github.com/prometheus/client_golang/prometheus" + "github.com/smartcontractkit/libocr/offchainreporting2plus/ocr3types" + ocrtypes "github.com/smartcontractkit/libocr/offchainreporting2plus/types" +) + +var _ ocr3types.ReportingPlugin[any] = &reportingPlugin[any]{} + +type reportingPlugin[RI any] struct { + ocr3types.ReportingPlugin[RI] + chainID string + plugin string + + // Prometheus components for tracking metrics + reportsGenerated *prometheus.CounterVec + durations *prometheus.HistogramVec +} + +func newReportingPlugin[RI any]( + origin ocr3types.ReportingPlugin[RI], + chainID string, + plugin string, + reportsGenerated *prometheus.CounterVec, + durations *prometheus.HistogramVec, +) *reportingPlugin[RI] { + return &reportingPlugin[RI]{ + ReportingPlugin: origin, + chainID: chainID, + plugin: plugin, + reportsGenerated: reportsGenerated, + durations: durations, + } +} + +func (p *reportingPlugin[RI]) Query(ctx context.Context, outctx ocr3types.OutcomeContext) (ocrtypes.Query, error) { + return withObservedExecution(p, query, func() (ocrtypes.Query, error) { + return p.ReportingPlugin.Query(ctx, outctx) + }) +} + +func (p *reportingPlugin[RI]) Observation(ctx context.Context, outctx ocr3types.OutcomeContext, query ocrtypes.Query) (ocrtypes.Observation, error) { + return withObservedExecution(p, observation, func() (ocrtypes.Observation, error) { + return p.ReportingPlugin.Observation(ctx, outctx, query) + }) +} + +func (p *reportingPlugin[RI]) ValidateObservation(ctx context.Context, outctx ocr3types.OutcomeContext, query ocrtypes.Query, ao ocrtypes.AttributedObservation) error { + _, err := withObservedExecution(p, validateObservation, func() (any, error) { + err := p.ReportingPlugin.ValidateObservation(ctx, outctx, query, ao) + return nil, err + }) + return err +} + +func (p *reportingPlugin[RI]) Outcome(ctx context.Context, outctx ocr3types.OutcomeContext, query ocrtypes.Query, aos []ocrtypes.AttributedObservation) (ocr3types.Outcome, error) { + return withObservedExecution(p, outcome, func() (ocr3types.Outcome, error) { + return p.ReportingPlugin.Outcome(ctx, outctx, query, aos) + }) +} + +func (p *reportingPlugin[RI]) Reports(ctx context.Context, seqNr uint64, outcome ocr3types.Outcome) ([]ocr3types.ReportPlus[RI], error) { + result, err := withObservedExecution(p, reports, func() ([]ocr3types.ReportPlus[RI], error) { + return p.ReportingPlugin.Reports(ctx, seqNr, outcome) + }) + p.trackReports(reports, len(result)) + return result, err +} + +func (p *reportingPlugin[RI]) ShouldAcceptAttestedReport(ctx context.Context, seqNr uint64, reportWithInfo ocr3types.ReportWithInfo[RI]) (bool, error) { + result, err := withObservedExecution(p, shouldAccept, func() (bool, error) { + return p.ReportingPlugin.ShouldAcceptAttestedReport(ctx, seqNr, reportWithInfo) + }) + p.trackReports(shouldAccept, boolToInt(result)) + return result, err +} + +func (p *reportingPlugin[RI]) ShouldTransmitAcceptedReport(ctx context.Context, seqNr uint64, reportWithInfo ocr3types.ReportWithInfo[RI]) (bool, error) { + result, err := withObservedExecution(p, shouldTransmit, func() (bool, error) { + return p.ReportingPlugin.ShouldTransmitAcceptedReport(ctx, seqNr, reportWithInfo) + }) + p.trackReports(shouldTransmit, boolToInt(result)) + return result, err +} + +func (p *reportingPlugin[RI]) trackReports( + function functionType, + count int, +) { + p.reportsGenerated. + WithLabelValues(p.chainID, p.plugin, string(function)). + Add(float64(count)) +} + +func boolToInt(arg bool) int { + if arg { + return 1 + } + return 0 +} + +func withObservedExecution[RI, R any]( + p *reportingPlugin[RI], + function functionType, + exec func() (R, error), +) (R, error) { + start := time.Now() + result, err := exec() + + success := err == nil + + p.durations. + WithLabelValues(p.chainID, p.plugin, string(function), strconv.FormatBool(success)). + Observe(float64(time.Since(start))) + + return result, err +} diff --git a/core/services/ocr3/promwrapper/plugin_test.go b/core/services/ocr3/promwrapper/plugin_test.go new file mode 100644 index 00000000000..35a97d109aa --- /dev/null +++ b/core/services/ocr3/promwrapper/plugin_test.go @@ -0,0 +1,171 @@ +package promwrapper + +import ( + "context" + "errors" + "testing" + + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/testutil" + io_prometheus_client "github.com/prometheus/client_model/go" + "github.com/stretchr/testify/require" + + "github.com/smartcontractkit/chainlink-common/pkg/utils/tests" + + "github.com/smartcontractkit/libocr/offchainreporting2plus/ocr3types" + ocrtypes "github.com/smartcontractkit/libocr/offchainreporting2plus/types" +) + +func Test_ReportsGeneratedGauge(t *testing.T) { + plugin1 := newReportingPlugin( + fakePlugin[uint]{reports: make([]ocr3types.ReportPlus[uint], 2)}, + "123", "empty", promOCR3ReportsGenerated, promOCR3Durations, + ) + plugin2 := newReportingPlugin( + fakePlugin[bool]{reports: make([]ocr3types.ReportPlus[bool], 10)}, + "solana", "different_plugin", promOCR3ReportsGenerated, promOCR3Durations, + ) + plugin3 := newReportingPlugin( + fakePlugin[string]{err: errors.New("error")}, + "1234", "empty", promOCR3ReportsGenerated, promOCR3Durations, + ) + + r1, err := plugin1.Reports(tests.Context(t), 1, nil) + require.NoError(t, err) + require.Len(t, r1, 2) + + for i := 0; i < 10; i++ { + r2, err1 := plugin2.Reports(tests.Context(t), 1, nil) + require.NoError(t, err1) + require.Len(t, r2, 10) + } + + _, err = plugin2.ShouldAcceptAttestedReport(tests.Context(t), 1, ocr3types.ReportWithInfo[bool]{}) + require.NoError(t, err) + + _, err = plugin3.Reports(tests.Context(t), 1, nil) + require.Error(t, err) + + g1 := testutil.ToFloat64(promOCR3ReportsGenerated.WithLabelValues("123", "empty", "reports")) + require.Equal(t, 2, int(g1)) + + g2 := testutil.ToFloat64(promOCR3ReportsGenerated.WithLabelValues("solana", "different_plugin", "reports")) + require.Equal(t, 100, int(g2)) + + g3 := testutil.ToFloat64(promOCR3ReportsGenerated.WithLabelValues("solana", "different_plugin", "shouldAccept")) + require.Equal(t, 1, int(g3)) + + g4 := testutil.ToFloat64(promOCR3ReportsGenerated.WithLabelValues("1234", "empty", "reports")) + require.Equal(t, 0, int(g4)) +} + +func Test_DurationHistograms(t *testing.T) { + plugin1 := newReportingPlugin( + fakePlugin[uint]{}, + "123", "empty", promOCR3ReportsGenerated, promOCR3Durations, + ) + plugin2 := newReportingPlugin( + fakePlugin[uint]{err: errors.New("error")}, + "123", "empty", promOCR3ReportsGenerated, promOCR3Durations, + ) + plugin3 := newReportingPlugin( + fakePlugin[uint]{}, + "solana", "commit", promOCR3ReportsGenerated, promOCR3Durations, + ) + + for _, p := range []*reportingPlugin[uint]{plugin1, plugin2, plugin3} { + _, _ = p.Query(tests.Context(t), ocr3types.OutcomeContext{}) + for i := 0; i < 2; i++ { + _, _ = p.Observation(tests.Context(t), ocr3types.OutcomeContext{}, nil) + } + _ = p.ValidateObservation(tests.Context(t), ocr3types.OutcomeContext{}, nil, ocrtypes.AttributedObservation{}) + _, _ = p.Outcome(tests.Context(t), ocr3types.OutcomeContext{}, nil, nil) + _, _ = p.Reports(tests.Context(t), 0, nil) + _, _ = p.ShouldAcceptAttestedReport(tests.Context(t), 0, ocr3types.ReportWithInfo[uint]{}) + _, _ = p.ShouldTransmitAcceptedReport(tests.Context(t), 0, ocr3types.ReportWithInfo[uint]{}) + } + + require.Equal(t, 1, counterFromHistogramByLabels(t, promOCR3Durations, "123", "empty", "query", "true")) + require.Equal(t, 1, counterFromHistogramByLabels(t, promOCR3Durations, "123", "empty", "query", "false")) + require.Equal(t, 1, counterFromHistogramByLabels(t, promOCR3Durations, "solana", "commit", "query", "true")) + + require.Equal(t, 2, counterFromHistogramByLabels(t, promOCR3Durations, "123", "empty", "observation", "true")) + require.Equal(t, 2, counterFromHistogramByLabels(t, promOCR3Durations, "123", "empty", "observation", "false")) + require.Equal(t, 2, counterFromHistogramByLabels(t, promOCR3Durations, "solana", "commit", "observation", "true")) +} + +type fakePlugin[RI any] struct { + reports []ocr3types.ReportPlus[RI] + err error +} + +func (f fakePlugin[RI]) Query(context.Context, ocr3types.OutcomeContext) (ocrtypes.Query, error) { + if f.err != nil { + return nil, f.err + } + return ocrtypes.Query{}, nil +} + +func (f fakePlugin[RI]) Observation(context.Context, ocr3types.OutcomeContext, ocrtypes.Query) (ocrtypes.Observation, error) { + if f.err != nil { + return nil, f.err + } + return ocrtypes.Observation{}, nil +} + +func (f fakePlugin[RI]) ValidateObservation(context.Context, ocr3types.OutcomeContext, ocrtypes.Query, ocrtypes.AttributedObservation) error { + return f.err +} + +func (f fakePlugin[RI]) ObservationQuorum(context.Context, ocr3types.OutcomeContext, ocrtypes.Query, []ocrtypes.AttributedObservation) (quorumReached bool, err error) { + return false, f.err +} + +func (f fakePlugin[RI]) Outcome(context.Context, ocr3types.OutcomeContext, ocrtypes.Query, []ocrtypes.AttributedObservation) (ocr3types.Outcome, error) { + if f.err != nil { + return nil, f.err + } + return ocr3types.Outcome{}, nil +} + +func (f fakePlugin[RI]) Reports(context.Context, uint64, ocr3types.Outcome) ([]ocr3types.ReportPlus[RI], error) { + if f.err != nil { + return nil, f.err + } + return f.reports, nil +} + +func (f fakePlugin[RI]) ShouldAcceptAttestedReport(context.Context, uint64, ocr3types.ReportWithInfo[RI]) (bool, error) { + if f.err != nil { + return false, f.err + } + return true, nil +} + +func (f fakePlugin[RI]) ShouldTransmitAcceptedReport(context.Context, uint64, ocr3types.ReportWithInfo[RI]) (bool, error) { + if f.err != nil { + return false, f.err + } + return true, nil +} + +func (f fakePlugin[RI]) Close() error { + return f.err +} + +func counterFromHistogramByLabels(t *testing.T, histogramVec *prometheus.HistogramVec, labels ...string) int { + observer, err := histogramVec.GetMetricWithLabelValues(labels...) + require.NoError(t, err) + + metricCh := make(chan prometheus.Metric, 1) + observer.(prometheus.Histogram).Collect(metricCh) + close(metricCh) + + metric := <-metricCh + pb := &io_prometheus_client.Metric{} + err = metric.Write(pb) + require.NoError(t, err) + + //nolint:gosec // we don't care about that in tests + return int(pb.GetHistogram().GetSampleCount()) +} diff --git a/core/services/ocr3/promwrapper/types.go b/core/services/ocr3/promwrapper/types.go new file mode 100644 index 00000000000..bf6a1b2a39c --- /dev/null +++ b/core/services/ocr3/promwrapper/types.go @@ -0,0 +1,51 @@ +package promwrapper + +import ( + "time" + + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promauto" +) + +type functionType string + +const ( + query functionType = "query" + observation functionType = "observation" + validateObservation functionType = "validateObservation" + outcome functionType = "outcome" + reports functionType = "reports" + shouldAccept functionType = "shouldAccept" + shouldTransmit functionType = "shouldTransmit" +) + +var ( + buckets = []float64{ + float64(10 * time.Millisecond), + float64(50 * time.Millisecond), + float64(100 * time.Millisecond), + float64(200 * time.Millisecond), + float64(500 * time.Millisecond), + float64(700 * time.Millisecond), + float64(time.Second), + float64(2 * time.Second), + float64(5 * time.Second), + float64(10 * time.Second), + } + + promOCR3ReportsGenerated = promauto.NewCounterVec( + prometheus.CounterOpts{ + Name: "ocr3_reporting_plugin_reports_processed", + Help: "Tracks number of reports processed/generated within by different OCR3 functions", + }, + []string{"chainID", "plugin", "function"}, + ) + promOCR3Durations = promauto.NewHistogramVec( + prometheus.HistogramOpts{ + Name: "ocr3_reporting_plugin_duration", + Help: "The amount of time elapsed during the OCR3 plugin's function", + Buckets: buckets, + }, + []string{"chainID", "plugin", "function", "success"}, + ) +) diff --git a/core/services/relay/evm/evm.go b/core/services/relay/evm/evm.go index 847b5bb72d9..e60dbe1bfdb 100644 --- a/core/services/relay/evm/evm.go +++ b/core/services/relay/evm/evm.go @@ -12,6 +12,7 @@ import ( "net/http" "strings" "sync" + "time" "github.com/ethereum/go-ethereum/accounts/abi" "github.com/ethereum/go-ethereum/common" @@ -259,6 +260,14 @@ func (r *Relayer) Close() error { cs := make([]io.Closer, 0, 2) if r.triggerCapability != nil { cs = append(cs, r.triggerCapability) + + ctx, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() + + err := r.capabilitiesRegistry.Remove(ctx, r.triggerCapability.ID) + if err != nil { + return err + } } cs = append(cs, r.chain) return services.MultiCloser(cs).Close() diff --git a/core/services/workflows/monitoring.go b/core/services/workflows/monitoring.go index 205ce529c28..8457dadeb60 100644 --- a/core/services/workflows/monitoring.go +++ b/core/services/workflows/monitoring.go @@ -5,6 +5,7 @@ import ( "fmt" "go.opentelemetry.io/otel/metric" + sdkmetric "go.opentelemetry.io/otel/sdk/metric" "github.com/smartcontractkit/chainlink-common/pkg/beholder" "github.com/smartcontractkit/chainlink-common/pkg/metrics" @@ -135,6 +136,37 @@ func initMonitoringResources() (em *engineMetrics, err error) { return em, nil } +// Note: due to the OTEL specification, all histogram buckets +// Must be defined when the beholder client is created +func MetricViews() []sdkmetric.View { + return []sdkmetric.View{ + sdkmetric.NewView( + sdkmetric.Instrument{Name: "platform_engine_workflow_earlyexit_time_seconds"}, + sdkmetric.Stream{Aggregation: sdkmetric.AggregationExplicitBucketHistogram{ + Boundaries: []float64{0, 1, 10, 100}, + }}, + ), + sdkmetric.NewView( + sdkmetric.Instrument{Name: "platform_engine_workflow_completed_time_seconds"}, + sdkmetric.Stream{Aggregation: sdkmetric.AggregationExplicitBucketHistogram{ + Boundaries: []float64{0, 100, 1000, 10_000, 50_000, 100_0000, 500_000}, + }}, + ), + sdkmetric.NewView( + sdkmetric.Instrument{Name: "platform_engine_workflow_error_time_seconds"}, + sdkmetric.Stream{Aggregation: sdkmetric.AggregationExplicitBucketHistogram{ + Boundaries: []float64{0, 20, 60, 120, 240}, + }}, + ), + sdkmetric.NewView( + sdkmetric.Instrument{Name: "platform_engine_workflow_step_time_seconds"}, + sdkmetric.Stream{Aggregation: sdkmetric.AggregationExplicitBucketHistogram{ + Boundaries: []float64{0, 20, 60, 120, 240}, + }}, + ), + } +} + // workflowsMetricLabeler wraps monitoring.MetricsLabeler to provide workflow specific utilities // for monitoring resources type workflowsMetricLabeler struct { diff --git a/core/services/workflows/syncer/fetcher.go b/core/services/workflows/syncer/fetcher.go index 357f7518635..fdd0134909d 100644 --- a/core/services/workflows/syncer/fetcher.go +++ b/core/services/workflows/syncer/fetcher.go @@ -44,7 +44,7 @@ func (s *FetcherService) Start(ctx context.Context) error { return s.StartOnce("FetcherService", func() error { connector := s.wrapper.GetGatewayConnector() - outgoingConnectorLggr := s.lggr.Named("WorkflowSyncer") + outgoingConnectorLggr := s.lggr.Named("OutgoingConnectorHandler") webAPIConfig := webapi.ServiceConfig{ RateLimiter: common.RateLimiterConfig{ diff --git a/core/services/workflows/syncer/handler.go b/core/services/workflows/syncer/handler.go index 5cfce71d56c..46dcd21ed90 100644 --- a/core/services/workflows/syncer/handler.go +++ b/core/services/workflows/syncer/handler.go @@ -1,8 +1,8 @@ package syncer import ( + "bytes" "context" - "crypto/sha256" "encoding/hex" "encoding/json" "errors" @@ -14,6 +14,7 @@ import ( "github.com/smartcontractkit/chainlink-common/pkg/custmsg" "github.com/smartcontractkit/chainlink-common/pkg/types/core" + pkgworkflows "github.com/smartcontractkit/chainlink-common/pkg/workflows" "github.com/smartcontractkit/chainlink-common/pkg/workflows/secrets" "github.com/smartcontractkit/chainlink-common/pkg/workflows/wasm/host" "github.com/smartcontractkit/chainlink/v2/core/logger" @@ -263,6 +264,7 @@ func (h *eventHandler) Handle(ctx context.Context, event Event) error { return err } + h.lggr.Debugw("handled force update secrets events for URL hash", "urlHash", payload.SecretsURLHash) return nil case WorkflowRegisteredEvent: payload, ok := event.GetData().(WorkflowRegistryWorkflowRegisteredV1) @@ -282,7 +284,7 @@ func (h *eventHandler) Handle(ctx context.Context, event Event) error { return err } - h.lggr.Debugf("workflow 0x%x registered and started", wfID) + h.lggr.Debugw("handled workflow registration event", "workflowID", wfID) return nil case WorkflowUpdatedEvent: payload, ok := event.GetData().(WorkflowRegistryWorkflowUpdatedV1) @@ -302,6 +304,7 @@ func (h *eventHandler) Handle(ctx context.Context, event Event) error { return err } + h.lggr.Debugw("handled workflow updated event", "workflowID", newWorkflowID) return nil case WorkflowPausedEvent: payload, ok := event.GetData().(WorkflowRegistryWorkflowPausedV1) @@ -321,6 +324,7 @@ func (h *eventHandler) Handle(ctx context.Context, event Event) error { logCustMsg(ctx, cma, fmt.Sprintf("failed to handle workflow paused event: %v", err), h.lggr) return err } + h.lggr.Debugw("handled workflow paused event", "workflowID", wfID) return nil case WorkflowActivatedEvent: payload, ok := event.GetData().(WorkflowRegistryWorkflowActivatedV1) @@ -340,6 +344,7 @@ func (h *eventHandler) Handle(ctx context.Context, event Event) error { return err } + h.lggr.Debugw("handled workflow activated event", "workflowID", wfID) return nil case WorkflowDeletedEvent: payload, ok := event.GetData().(WorkflowRegistryWorkflowDeletedV1) @@ -360,6 +365,7 @@ func (h *eventHandler) Handle(ctx context.Context, event Event) error { return err } + h.lggr.Debugw("handled workflow deleted event", "workflowID", wfID) return nil default: return fmt.Errorf("event type unsupported: %v", event.GetEventType()) @@ -371,8 +377,6 @@ func (h *eventHandler) workflowRegisteredEvent( ctx context.Context, payload WorkflowRegistryWorkflowRegisteredV1, ) error { - wfID := hex.EncodeToString(payload.WorkflowID[:]) - // Download the contents of binaryURL, configURL and secretsURL and cache them locally. binary, err := h.fetcher(ctx, payload.BinaryURL) if err != nil { @@ -390,11 +394,14 @@ func (h *eventHandler) workflowRegisteredEvent( } // Calculate the hash of the binary and config files - hash := workflowID(binary, config, []byte(payload.SecretsURL)) + hash, err := pkgworkflows.GenerateWorkflowID(payload.Owner, binary, config, payload.SecretsURL) + if err != nil { + return fmt.Errorf("failed to generate workflow id: %w", err) + } // Pre-check: verify that the workflowID matches; if it doesn’t abort and log an error via Beholder. - if hash != wfID { - return fmt.Errorf("workflowID mismatch: %s != %s", hash, wfID) + if !bytes.Equal(hash[:], payload.WorkflowID[:]) { + return fmt.Errorf("workflowID mismatch: %x != %x", hash, payload.WorkflowID) } // Save the workflow secrets @@ -409,6 +416,7 @@ func (h *eventHandler) workflowRegisteredEvent( status = job.WorkflowSpecStatusPaused } + wfID := hex.EncodeToString(payload.WorkflowID[:]) entry := &job.WorkflowSpec{ Workflow: hex.EncodeToString(binary), Config: string(config), @@ -425,6 +433,7 @@ func (h *eventHandler) workflowRegisteredEvent( } if status != job.WorkflowSpecStatusActive { + h.lggr.Debugw("workflow is marked as paused, so not starting it", "workflow", wfID) return nil } @@ -611,15 +620,6 @@ func (h *eventHandler) tryEngineCleanup(wfID string) error { return nil } -// workflowID returns a hex encoded sha256 hash of the wasm, config and secretsURL. -func workflowID(wasm, config, secretsURL []byte) string { - sum := sha256.New() - sum.Write(wasm) - sum.Write(config) - sum.Write(secretsURL) - return hex.EncodeToString(sum.Sum(nil)) -} - // logCustMsg emits a custom message to the external sink and logs an error if that fails. func logCustMsg(ctx context.Context, cma custmsg.MessageEmitter, msg string, log logger.Logger) { err := cma.Emit(ctx, msg) diff --git a/core/services/workflows/syncer/handler_test.go b/core/services/workflows/syncer/handler_test.go index f5a915e48ab..bb0a61aea4d 100644 --- a/core/services/workflows/syncer/handler_test.go +++ b/core/services/workflows/syncer/handler_test.go @@ -10,6 +10,7 @@ import ( "time" "github.com/smartcontractkit/chainlink-common/pkg/custmsg" + pkgworkflows "github.com/smartcontractkit/chainlink-common/pkg/workflows" "github.com/smartcontractkit/chainlink-common/pkg/workflows/secrets" "github.com/smartcontractkit/chainlink/v2/core/capabilities" "github.com/smartcontractkit/chainlink/v2/core/internal/testutils" @@ -194,16 +195,12 @@ func Test_workflowRegisteredHandler(t *testing.T) { }) ) - giveWFID := workflowID(binary, config, []byte(secretsURL)) - - b, err := hex.DecodeString(giveWFID) + giveWFID, err := pkgworkflows.GenerateWorkflowID(wfOwner, binary, config, secretsURL) require.NoError(t, err) - wfID := make([]byte, 32) - copy(wfID, b) paused := WorkflowRegistryWorkflowRegisteredV1{ Status: uint8(1), - WorkflowID: [32]byte(wfID), + WorkflowID: giveWFID, Owner: wfOwner, WorkflowName: "workflow-name", BinaryURL: binaryURL, @@ -250,16 +247,14 @@ func Test_workflowRegisteredHandler(t *testing.T) { }) ) - giveWFID := workflowID(binary, config, []byte(secretsURL)) + giveWFID, err := pkgworkflows.GenerateWorkflowID(wfOwner, binary, config, secretsURL) + require.NoError(t, err) - b, err := hex.DecodeString(giveWFID) require.NoError(t, err) - wfID := make([]byte, 32) - copy(wfID, b) active := WorkflowRegistryWorkflowRegisteredV1{ Status: uint8(0), - WorkflowID: [32]byte(wfID), + WorkflowID: giveWFID, Owner: wfOwner, WorkflowName: "workflow-name", BinaryURL: binaryURL, @@ -291,7 +286,7 @@ func Test_workflowRegisteredHandler(t *testing.T) { require.Equal(t, job.WorkflowSpecStatusActive, dbSpec.Status) // Verify the engine is started - engine, err := h.engineRegistry.Get(giveWFID) + engine, err := h.engineRegistry.Get(hex.EncodeToString(giveWFID[:])) require.NoError(t, err) err = engine.Ready() require.NoError(t, err) @@ -321,16 +316,14 @@ func Test_workflowDeletedHandler(t *testing.T) { }) ) - giveWFID := workflowID(binary, config, []byte(secretsURL)) + giveWFID, err := pkgworkflows.GenerateWorkflowID(wfOwner, binary, config, secretsURL) - b, err := hex.DecodeString(giveWFID) require.NoError(t, err) - wfID := make([]byte, 32) - copy(wfID, b) + wfIDs := hex.EncodeToString(giveWFID[:]) active := WorkflowRegistryWorkflowRegisteredV1{ Status: uint8(0), - WorkflowID: [32]byte(wfID), + WorkflowID: giveWFID, Owner: wfOwner, WorkflowName: "workflow-name", BinaryURL: binaryURL, @@ -362,13 +355,13 @@ func Test_workflowDeletedHandler(t *testing.T) { require.Equal(t, job.WorkflowSpecStatusActive, dbSpec.Status) // Verify the engine is started - engine, err := h.engineRegistry.Get(giveWFID) + engine, err := h.engineRegistry.Get(wfIDs) require.NoError(t, err) err = engine.Ready() require.NoError(t, err) deleteEvent := WorkflowRegistryWorkflowDeletedV1{ - WorkflowID: [32]byte(wfID), + WorkflowID: giveWFID, WorkflowOwner: wfOwner, WorkflowName: "workflow-name", DonID: 1, @@ -381,7 +374,7 @@ func Test_workflowDeletedHandler(t *testing.T) { require.Error(t, err) // Verify the engine is deleted - _, err = h.engineRegistry.Get(giveWFID) + _, err = h.engineRegistry.Get(wfIDs) require.Error(t, err) }) } @@ -412,22 +405,20 @@ func Test_workflowPausedActivatedUpdatedHandler(t *testing.T) { }) ) - giveWFID := workflowID(binary, config, []byte(secretsURL)) - updatedWFID := workflowID(binary, updateConfig, []byte(secretsURL)) + giveWFID, err := pkgworkflows.GenerateWorkflowID(wfOwner, binary, config, secretsURL) + require.NoError(t, err) + updatedWFID, err := pkgworkflows.GenerateWorkflowID(wfOwner, binary, updateConfig, secretsURL) + require.NoError(t, err) - b, err := hex.DecodeString(giveWFID) require.NoError(t, err) - wfID := make([]byte, 32) - copy(wfID, b) + wfIDs := hex.EncodeToString(giveWFID[:]) - b, err = hex.DecodeString(updatedWFID) require.NoError(t, err) - newWFID := make([]byte, 32) - copy(newWFID, b) + newWFIDs := hex.EncodeToString(updatedWFID[:]) active := WorkflowRegistryWorkflowRegisteredV1{ Status: uint8(0), - WorkflowID: [32]byte(wfID), + WorkflowID: giveWFID, Owner: wfOwner, WorkflowName: "workflow-name", BinaryURL: binaryURL, @@ -459,14 +450,14 @@ func Test_workflowPausedActivatedUpdatedHandler(t *testing.T) { require.Equal(t, job.WorkflowSpecStatusActive, dbSpec.Status) // Verify the engine is started - engine, err := h.engineRegistry.Get(giveWFID) + engine, err := h.engineRegistry.Get(wfIDs) require.NoError(t, err) err = engine.Ready() require.NoError(t, err) // create a paused event pauseEvent := WorkflowRegistryWorkflowPausedV1{ - WorkflowID: [32]byte(wfID), + WorkflowID: giveWFID, WorkflowOwner: wfOwner, WorkflowName: "workflow-name", DonID: 1, @@ -482,12 +473,12 @@ func Test_workflowPausedActivatedUpdatedHandler(t *testing.T) { require.Equal(t, job.WorkflowSpecStatusPaused, dbSpec.Status) // Verify the engine is removed - _, err = h.engineRegistry.Get(giveWFID) + _, err = h.engineRegistry.Get(wfIDs) require.Error(t, err) // create an activated workflow event activatedEvent := WorkflowRegistryWorkflowActivatedV1{ - WorkflowID: [32]byte(wfID), + WorkflowID: giveWFID, WorkflowOwner: wfOwner, WorkflowName: "workflow-name", DonID: 1, @@ -504,15 +495,15 @@ func Test_workflowPausedActivatedUpdatedHandler(t *testing.T) { require.Equal(t, job.WorkflowSpecStatusActive, dbSpec.Status) // Verify the engine is started - engine, err = h.engineRegistry.Get(giveWFID) + engine, err = h.engineRegistry.Get(wfIDs) require.NoError(t, err) err = engine.Ready() require.NoError(t, err) // create an updated event updatedEvent := WorkflowRegistryWorkflowUpdatedV1{ - OldWorkflowID: [32]byte(wfID), - NewWorkflowID: [32]byte(newWFID), + OldWorkflowID: giveWFID, + NewWorkflowID: updatedWFID, WorkflowOwner: wfOwner, WorkflowName: "workflow-name", BinaryURL: binaryURL, @@ -529,16 +520,16 @@ func Test_workflowPausedActivatedUpdatedHandler(t *testing.T) { require.Equal(t, hex.EncodeToString(wfOwner), dbSpec.WorkflowOwner) require.Equal(t, "workflow-name", dbSpec.WorkflowName) require.Equal(t, job.WorkflowSpecStatusActive, dbSpec.Status) - require.Equal(t, hex.EncodeToString(newWFID), dbSpec.WorkflowID) + require.Equal(t, newWFIDs, dbSpec.WorkflowID) require.Equal(t, newConfigURL, dbSpec.ConfigURL) require.Equal(t, string(updateConfig), dbSpec.Config) // old engine is no longer running - _, err = h.engineRegistry.Get(giveWFID) + _, err = h.engineRegistry.Get(wfIDs) require.Error(t, err) // new engine is started - engine, err = h.engineRegistry.Get(updatedWFID) + engine, err = h.engineRegistry.Get(newWFIDs) require.NoError(t, err) err = engine.Ready() require.NoError(t, err) diff --git a/core/services/workflows/syncer/workflow_registry.go b/core/services/workflows/syncer/workflow_registry.go index 6fc319da76b..024975539af 100644 --- a/core/services/workflows/syncer/workflow_registry.go +++ b/core/services/workflows/syncer/workflow_registry.go @@ -173,7 +173,7 @@ func NewWorkflowRegistry( ) *workflowRegistry { ets := []WorkflowRegistryEventType{ForceUpdateSecretsEvent} wr := &workflowRegistry{ - lggr: lggr.Named(name), + lggr: lggr, newContractReaderFn: newContractReaderFn, workflowRegistryAddress: addr, eventPollerCfg: eventPollerConfig, @@ -633,7 +633,7 @@ func (l *workflowRegistryContractLoader) LoadWorkflows(ctx context.Context, don Data: workflow, EventType: WorkflowRegisteredEvent, }); err != nil { - return nil, fmt.Errorf("failed to handle workflow registration: %w", err) + l.lggr.Errorf("failed to handle workflow registration: %s", err) } } diff --git a/deployment/ccip/changeset/test_assertions.go b/deployment/ccip/changeset/test_assertions.go index ad2ea4257ea..a7d3ecf61f8 100644 --- a/deployment/ccip/changeset/test_assertions.go +++ b/deployment/ccip/changeset/test_assertions.go @@ -259,6 +259,40 @@ func (c *commitReportTracker) allCommited(sourceChainSelector uint64) bool { return true } +// ConfirmMultipleCommits waits for multiple ccipocr3.SeqNumRange to be committed by the Offramp. +// Waiting is done in parallel per every sourceChain/destChain (lane) passed as argument. +func ConfirmMultipleCommits( + t *testing.T, + chains map[uint64]deployment.Chain, + state map[uint64]CCIPChainState, + startBlocks map[uint64]*uint64, + enforceSingleCommit bool, + expectedSeqNums map[SourceDestPair]ccipocr3.SeqNumRange, +) error { + errGrp := &errgroup.Group{} + + for sourceDest, seqRange := range expectedSeqNums { + seqRange := seqRange + srcChain := sourceDest.SourceChainSelector + destChain := sourceDest.DestChainSelector + + errGrp.Go(func() error { + _, err := ConfirmCommitWithExpectedSeqNumRange( + t, + chains[srcChain], + chains[destChain], + state[destChain].OffRamp, + startBlocks[destChain], + seqRange, + enforceSingleCommit, + ) + return err + }) + } + + return errGrp.Wait() +} + // ConfirmCommitWithExpectedSeqNumRange waits for a commit report on the destination chain with the expected sequence number range. // startBlock is the block number to start watching from. // If startBlock is nil, it will start watching from the latest block. @@ -449,7 +483,7 @@ func ConfirmExecWithSeqNrs( return nil, fmt.Errorf("no expected sequence numbers provided") } - timer := time.NewTimer(3 * time.Minute) + timer := time.NewTimer(8 * time.Minute) defer timer.Stop() tick := time.NewTicker(3 * time.Second) defer tick.Stop() @@ -564,6 +598,22 @@ func RequireConsistently(t *testing.T, condition func() bool, duration time.Dura } } +func SeqNumberRageToSlice(seqRanges map[SourceDestPair]ccipocr3.SeqNumRange) map[SourceDestPair][]uint64 { + flatten := make(map[SourceDestPair][]uint64) + + for srcDst, seqRange := range seqRanges { + if _, ok := flatten[srcDst]; !ok { + flatten[srcDst] = make([]uint64, 0, seqRange.End()-seqRange.Start()+1) + } + + for i := seqRange.Start(); i <= seqRange.End(); i++ { + flatten[srcDst] = append(flatten[srcDst], uint64(i)) + } + } + + return flatten +} + const ( EXECUTION_STATE_UNTOUCHED = 0 EXECUTION_STATE_INPROGRESS = 1 diff --git a/deployment/ccip/changeset/test_helpers.go b/deployment/ccip/changeset/test_helpers.go index 742fe39200a..1ee8b0d0e42 100644 --- a/deployment/ccip/changeset/test_helpers.go +++ b/deployment/ccip/changeset/test_helpers.go @@ -286,9 +286,15 @@ func NewMemoryEnvironmentWithJobsAndContracts(t *testing.T, lggr logger.Logger, TimelockMinDelay: big.NewInt(0), } } - var usdcChains []uint64 - if tCfg != nil && tCfg.IsUSDC { - usdcChains = allChains + var ( + usdcChains []uint64 + isMulticall3 bool + ) + if tCfg != nil { + if tCfg.IsUSDC { + usdcChains = allChains + } + isMulticall3 = tCfg.IsMultiCall3 } // Need to deploy prerequisites first so that we can form the USDC config // no proposals to be made, timelock can be passed as nil here @@ -303,6 +309,7 @@ func NewMemoryEnvironmentWithJobsAndContracts(t *testing.T, lggr logger.Logger, ChainSelectors: allChains, Opts: []PrerequisiteOpt{ WithUSDCChains(usdcChains), + WithMulticall3(isMulticall3), }, }, }, @@ -1225,6 +1232,78 @@ func Transfer( return msgSentEvent, startBlocks } +type TestTransferRequest struct { + Name string + SourceChain, DestChain uint64 + Receiver common.Address + ExpectedStatus int + // optional + Tokens []router.ClientEVMTokenAmount + Data []byte + ExtraArgs []byte + ExpectedTokenBalances map[common.Address]*big.Int +} + +// TransferMultiple sends multiple CCIPMessages (represented as TestTransferRequest) sequentially. +// It verifies whether message is not reverted on the source and proper event is emitted by OnRamp. +// However, it doesn't wait for message to be committed or executed. Therefore, you can send multiple messages very fast, +// but you need to make sure they are committed/executed on your own (if that's the intention). +// It saves some time during test execution, because we let plugins batch instead of executing one by one +// If you want to wait for execution in a "batch" manner you will need to pass maps returned by TransferMultiple to +// either ConfirmMultipleCommits (for commit) or ConfirmExecWithSeqNrsForAll (for exec). Check example usage in the tests. +func TransferMultiple( + ctx context.Context, + t *testing.T, + env deployment.Environment, + state CCIPOnChainState, + requests []TestTransferRequest, +) ( + map[uint64]*uint64, + map[SourceDestPair]cciptypes.SeqNumRange, + map[SourceDestPair]map[uint64]int, + map[uint64]map[TokenReceiverIdentifier]*big.Int, +) { + startBlocks := make(map[uint64]*uint64) + expectedSeqNums := make(map[SourceDestPair]cciptypes.SeqNumRange) + expectedExecutionStates := make(map[SourceDestPair]map[uint64]int) + expectedTokenBalances := make(TokenBalanceAccumulator) + + for _, tt := range requests { + t.Run(tt.Name, func(t *testing.T) { + expectedTokenBalances.add(tt.DestChain, tt.Receiver, tt.ExpectedTokenBalances) + + pairId := SourceDestPair{ + SourceChainSelector: tt.SourceChain, + DestChainSelector: tt.DestChain, + } + + msg, blocks := Transfer( + ctx, t, env, state, tt.SourceChain, tt.DestChain, tt.Tokens, tt.Receiver, tt.Data, tt.ExtraArgs) + if _, ok := expectedExecutionStates[pairId]; !ok { + expectedExecutionStates[pairId] = make(map[uint64]int) + } + expectedExecutionStates[pairId][msg.SequenceNumber] = tt.ExpectedStatus + + if _, ok := startBlocks[tt.DestChain]; !ok { + startBlocks[tt.DestChain] = blocks[tt.DestChain] + } + + seqNr, ok := expectedSeqNums[pairId] + if ok { + expectedSeqNums[pairId] = cciptypes.NewSeqNumRange( + seqNr.Start(), cciptypes.SeqNum(msg.SequenceNumber), + ) + } else { + expectedSeqNums[pairId] = cciptypes.NewSeqNumRange( + cciptypes.SeqNum(msg.SequenceNumber), cciptypes.SeqNum(msg.SequenceNumber), + ) + } + }) + } + + return startBlocks, expectedSeqNums, expectedExecutionStates, expectedTokenBalances +} + // TransferAndWaitForSuccess sends a message from sourceChain to destChain and waits for it to be executed func TransferAndWaitForSuccess( ctx context.Context, @@ -1258,6 +1337,60 @@ func TransferAndWaitForSuccess( require.Equal(t, expectedStatus, states[identifier][msgSentEvent.SequenceNumber]) } +// TokenBalanceAccumulator is a convenient accumulator to aggregate expected balances of different tokens +// used across the tests. You can iterate over your test cases and build the final "expected" balances for tokens (per chain, per sender) +// For instance, if your test runs multiple transfers for the same token, and you want to verify the balance of tokens at +// the end of the execution, you can simply use that struct for aggregating expected tokens +// Please also see WaitForTokenBalances to better understand how you can assert token balances +type TokenBalanceAccumulator map[uint64]map[TokenReceiverIdentifier]*big.Int + +func (t TokenBalanceAccumulator) add( + destChain uint64, + receiver common.Address, + expectedBalance map[common.Address]*big.Int) { + for token, balance := range expectedBalance { + tkIdentifier := TokenReceiverIdentifier{token, receiver} + + if _, ok := t[destChain]; !ok { + t[destChain] = make(map[TokenReceiverIdentifier]*big.Int) + } + actual, ok := t[destChain][tkIdentifier] + if !ok { + actual = big.NewInt(0) + } + t[destChain][tkIdentifier] = new(big.Int).Add(actual, balance) + } +} + +type TokenReceiverIdentifier struct { + token common.Address + receiver common.Address +} + +// WaitForTokenBalances waits for multiple ERC20 tokens to reach a particular balance +// It works in a batch manner, so you can pass and exhaustive list of different tokens (per senders and chains) +// and it would work concurrently for the balance to be met. Check WaitForTheTokenBalance to see how balance +// checking is made for a token/receiver pair +func WaitForTokenBalances( + ctx context.Context, + t *testing.T, + chains map[uint64]deployment.Chain, + expectedBalances map[uint64]map[TokenReceiverIdentifier]*big.Int, +) { + errGrp := &errgroup.Group{} + for chainID, tokens := range expectedBalances { + for id, balance := range tokens { + id := id + balance := balance + errGrp.Go(func() error { + WaitForTheTokenBalance(ctx, t, id.token, id.receiver, chains[chainID], balance) + return nil + }) + } + } + require.NoError(t, errGrp.Wait()) +} + func WaitForTheTokenBalance( ctx context.Context, t *testing.T, diff --git a/deployment/environment.go b/deployment/environment.go index bdf9fe6d5de..2de16a32cab 100644 --- a/deployment/environment.go +++ b/deployment/environment.go @@ -54,6 +54,9 @@ type Chain struct { // Note the Sign function can be abstract supporting a variety of key storage mechanisms (e.g. KMS etc). DeployerKey *bind.TransactOpts Confirm func(tx *types.Transaction) (uint64, error) + // Users are a set of keys that can be used to interact with the chain. + // These are distinct from the deployer key. + Users []*bind.TransactOpts } // Environment represents an instance of a deployed product diff --git a/deployment/environment/memory/chain.go b/deployment/environment/memory/chain.go index cbb3e67df7a..40a20a02416 100644 --- a/deployment/environment/memory/chain.go +++ b/deployment/environment/memory/chain.go @@ -9,12 +9,12 @@ import ( "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/ethclient/simulated" - "github.com/ethereum/go-ethereum/params" "github.com/stretchr/testify/require" chainsel "github.com/smartcontractkit/chain-selectors" "github.com/smartcontractkit/chainlink-common/pkg/utils/tests" + "github.com/smartcontractkit/chainlink/v2/core/chains/evm/assets" ) type EVMChain struct { @@ -47,30 +47,7 @@ func GenerateChains(t *testing.T, numChains int, numUsers int) map[uint64]EVMCha chains := make(map[uint64]EVMChain) for i := 0; i < numChains; i++ { chainID := chainsel.TEST_90000001.EvmChainID + uint64(i) - key, err := crypto.GenerateKey() - require.NoError(t, err) - owner, err := bind.NewKeyedTransactorWithChainID(key, big.NewInt(1337)) - require.NoError(t, err) - genesis := types.GenesisAlloc{ - owner.From: {Balance: big.NewInt(0).Mul(big.NewInt(7000), big.NewInt(params.Ether))}} - // create a set of user keys - var users []*bind.TransactOpts - for j := 0; j < numUsers; j++ { - key, err := crypto.GenerateKey() - require.NoError(t, err) - user, err := bind.NewKeyedTransactorWithChainID(key, big.NewInt(1337)) - require.NoError(t, err) - users = append(users, user) - genesis[user.From] = types.Account{Balance: big.NewInt(0).Mul(big.NewInt(7000), big.NewInt(params.Ether))} - } - // there have to be enough initial funds on each chain to allocate for all the nodes that share the given chain in the test - backend := simulated.NewBackend(genesis, simulated.WithBlockGasLimit(50000000)) - backend.Commit() // ts will be now. - chains[chainID] = EVMChain{ - Backend: backend, - DeployerKey: owner, - Users: users, - } + chains[chainID] = evmChain(t, numUsers) } return chains } @@ -78,18 +55,34 @@ func GenerateChains(t *testing.T, numChains int, numUsers int) map[uint64]EVMCha func GenerateChainsWithIds(t *testing.T, chainIDs []uint64) map[uint64]EVMChain { chains := make(map[uint64]EVMChain) for _, chainID := range chainIDs { + chains[chainID] = evmChain(t, 1) + } + return chains +} + +func evmChain(t *testing.T, numUsers int) EVMChain { + key, err := crypto.GenerateKey() + require.NoError(t, err) + owner, err := bind.NewKeyedTransactorWithChainID(key, big.NewInt(1337)) + require.NoError(t, err) + genesis := types.GenesisAlloc{ + owner.From: {Balance: assets.Ether(1_000_000).ToInt()}} + // create a set of user keys + var users []*bind.TransactOpts + for j := 0; j < numUsers; j++ { key, err := crypto.GenerateKey() require.NoError(t, err) - owner, err := bind.NewKeyedTransactorWithChainID(key, big.NewInt(1337)) + user, err := bind.NewKeyedTransactorWithChainID(key, big.NewInt(1337)) require.NoError(t, err) - backend := simulated.NewBackend(types.GenesisAlloc{ - owner.From: {Balance: big.NewInt(0).Mul(big.NewInt(700000), big.NewInt(params.Ether))}}, - simulated.WithBlockGasLimit(10000000)) - backend.Commit() // Note initializes block timestamp to now(). - chains[chainID] = EVMChain{ - Backend: backend, - DeployerKey: owner, - } + users = append(users, user) + genesis[user.From] = types.Account{Balance: assets.Ether(1_000_000).ToInt()} + } + // there have to be enough initial funds on each chain to allocate for all the nodes that share the given chain in the test + backend := simulated.NewBackend(genesis, simulated.WithBlockGasLimit(50000000)) + backend.Commit() // ts will be now. + return EVMChain{ + Backend: backend, + DeployerKey: owner, + Users: users, } - return chains } diff --git a/deployment/environment/memory/environment.go b/deployment/environment/memory/environment.go index f4692998d34..e8f0c265101 100644 --- a/deployment/environment/memory/environment.go +++ b/deployment/environment/memory/environment.go @@ -94,6 +94,7 @@ func generateMemoryChain(t *testing.T, inputs map[uint64]EVMChain) map[uint64]de return receipt.BlockNumber.Uint64(), nil } }, + Users: chain.Users, } } return chains diff --git a/deployment/go.mod b/deployment/go.mod index e1d51c46433..7908b6a8048 100644 --- a/deployment/go.mod +++ b/deployment/go.mod @@ -2,6 +2,8 @@ module github.com/smartcontractkit/chainlink/deployment go 1.23.3 +toolchain go1.23.4 + // Make sure we're working with the latest chainlink libs replace github.com/smartcontractkit/chainlink/v2 => ../ @@ -23,7 +25,7 @@ require ( github.com/smartcontractkit/ccip-owner-contracts v0.0.0-20240926212305-a6deabdfce86 github.com/smartcontractkit/chain-selectors v1.0.31 github.com/smartcontractkit/chainlink-ccip v0.0.0-20241204015713-8956bb614e9e - github.com/smartcontractkit/chainlink-common v0.3.1-0.20241127162636-07aa781ee1f4 + github.com/smartcontractkit/chainlink-common v0.3.1-0.20241206011233-b6684ee6508f github.com/smartcontractkit/chainlink-protos/job-distributor v0.6.0 github.com/smartcontractkit/chainlink-testing-framework/lib v1.50.13 github.com/smartcontractkit/chainlink/v2 v2.0.0-00010101000000-000000000000 diff --git a/deployment/go.sum b/deployment/go.sum index a2dc68f9dc6..fec529e51c6 100644 --- a/deployment/go.sum +++ b/deployment/go.sum @@ -1411,8 +1411,8 @@ github.com/smartcontractkit/chainlink-automation v0.8.1 h1:sTc9LKpBvcKPc1JDYAmgB github.com/smartcontractkit/chainlink-automation v0.8.1/go.mod h1:Iij36PvWZ6blrdC5A/nrQUBuf3MH3JvsBB9sSyc9W08= github.com/smartcontractkit/chainlink-ccip v0.0.0-20241204015713-8956bb614e9e h1:GnM6ZWV6vlk2+n6c6o+v/R1LtXzBGVVx7r37nt/h6Uc= github.com/smartcontractkit/chainlink-ccip v0.0.0-20241204015713-8956bb614e9e/go.mod h1:80vGBbOfertJig0xFKsRfm+i17FkjdKkk1dAaGE45Os= -github.com/smartcontractkit/chainlink-common v0.3.1-0.20241127162636-07aa781ee1f4 h1:atCZ1jol7a+tdtgU/wNqXgliBun5H7BjGBicGL8Tj6o= -github.com/smartcontractkit/chainlink-common v0.3.1-0.20241127162636-07aa781ee1f4/go.mod h1:bQktEJf7sJ0U3SmIcXvbGUox7SmXcnSEZ4kUbT8R5Nk= +github.com/smartcontractkit/chainlink-common v0.3.1-0.20241206011233-b6684ee6508f h1:hH+cAG2zt+WK4I2m572LXAnAJg3wtGEAwzBKR8FiXo8= +github.com/smartcontractkit/chainlink-common v0.3.1-0.20241206011233-b6684ee6508f/go.mod h1:bQktEJf7sJ0U3SmIcXvbGUox7SmXcnSEZ4kUbT8R5Nk= github.com/smartcontractkit/chainlink-cosmos v0.5.2-0.20241202195413-82468150ac1e h1:PRoeby6ZlTuTkv2f+7tVU4+zboTfRzI+beECynF4JQ0= github.com/smartcontractkit/chainlink-cosmos v0.5.2-0.20241202195413-82468150ac1e/go.mod h1:mUh5/woemsVaHgTorA080hrYmO3syBCmPdnWc/5dOqk= github.com/smartcontractkit/chainlink-data-streams v0.1.1-0.20241202141438-a90db35252db h1:N1RH1hSr2ACzOFc9hkCcjE8pRBTdcU3p8nsTJByaLes= diff --git a/deployment/keystone/changeset/accept_ownership.go b/deployment/keystone/changeset/accept_ownership.go index 7dffc5a70c4..662a4c2dcfa 100644 --- a/deployment/keystone/changeset/accept_ownership.go +++ b/deployment/keystone/changeset/accept_ownership.go @@ -5,6 +5,8 @@ import ( "github.com/ethereum/go-ethereum/common" + kslib "github.com/smartcontractkit/chainlink/deployment/keystone" + "github.com/smartcontractkit/chainlink/deployment" "github.com/smartcontractkit/chainlink/deployment/common/changeset" ) @@ -23,39 +25,21 @@ func AcceptAllOwnershipsProposal(e deployment.Environment, req *AcceptAllOwnersh chain := e.Chains[chainSelector] addrBook := e.ExistingAddresses - capRegs, err := capRegistriesFromAddrBook(addrBook, chain) - if err != nil { - return deployment.ChangesetOutput{}, err - } - ocr3, err := ocr3FromAddrBook(addrBook, chain) - if err != nil { - return deployment.ChangesetOutput{}, err - } - forwarders, err := forwardersFromAddrBook(addrBook, chain) - if err != nil { - return deployment.ChangesetOutput{}, err - } - consumers, err := feedsConsumersFromAddrBook(addrBook, chain) + r, err := kslib.GetContractSets(e.Logger, &kslib.GetContractSetsRequest{ + Chains: map[uint64]deployment.Chain{ + req.ChainSelector: chain, + }, + AddressBook: addrBook, + }) if err != nil { return deployment.ChangesetOutput{}, err } - var addrsToTransfer []common.Address - for _, consumer := range consumers { - addrsToTransfer = append(addrsToTransfer, consumer.Address()) - } - for _, o := range ocr3 { - addrsToTransfer = append(addrsToTransfer, o.Address()) - } - for _, f := range forwarders { - addrsToTransfer = append(addrsToTransfer, f.Address()) - } - for _, c := range capRegs { - addrsToTransfer = append(addrsToTransfer, c.Address()) - } + contracts := r.ContractSets[chainSelector] + // Construct the configuration cfg := changeset.TransferToMCMSWithTimelockConfig{ ContractsByChain: map[uint64][]common.Address{ - chainSelector: addrsToTransfer, + chainSelector: contracts.TransferableContracts(), }, MinDelay: minDelay, } diff --git a/deployment/keystone/changeset/accept_ownership_test.go b/deployment/keystone/changeset/accept_ownership_test.go index 997f0b7e163..ec65ef920ac 100644 --- a/deployment/keystone/changeset/accept_ownership_test.go +++ b/deployment/keystone/changeset/accept_ownership_test.go @@ -38,10 +38,6 @@ func TestAcceptAllOwnership(t *testing.T) { Changeset: commonchangeset.WrapChangeSet(changeset.DeployForwarder), Config: registrySel, }, - { - Changeset: commonchangeset.WrapChangeSet(changeset.DeployFeedsConsumer), - Config: &changeset.DeployFeedsConsumerRequest{ChainSelector: registrySel}, - }, { Changeset: commonchangeset.WrapChangeSet(commonchangeset.DeployMCMSWithTimelock), Config: map[uint64]types.MCMSWithTimelockConfig{ diff --git a/deployment/keystone/changeset/configure_contracts.go b/deployment/keystone/changeset/configure_contracts.go index d5bcb32243b..17635a42ed1 100644 --- a/deployment/keystone/changeset/configure_contracts.go +++ b/deployment/keystone/changeset/configure_contracts.go @@ -9,6 +9,25 @@ import ( kslib "github.com/smartcontractkit/chainlink/deployment/keystone" ) +var _ deployment.ChangeSet[InitialContractsCfg] = ConfigureInitialContractsChangeset + +type InitialContractsCfg struct { + RegistryChainSel uint64 + Dons []kslib.DonCapabilities + OCR3Config *kslib.OracleConfigWithSecrets +} + +func ConfigureInitialContractsChangeset(e deployment.Environment, cfg InitialContractsCfg) (deployment.ChangesetOutput, error) { + req := &kslib.ConfigureContractsRequest{ + Env: &e, + RegistryChainSel: cfg.RegistryChainSel, + Dons: cfg.Dons, + OCR3Config: cfg.OCR3Config, + } + return ConfigureInitialContracts(e.Logger, req) +} + +// Deprecated: Use ConfigureInitialContractsChangeset instead. func ConfigureInitialContracts(lggr logger.Logger, req *kslib.ConfigureContractsRequest) (deployment.ChangesetOutput, error) { if err := req.Validate(); err != nil { return deployment.ChangesetOutput{}, fmt.Errorf("failed to validate request: %w", err) diff --git a/deployment/keystone/changeset/deploy_forwarder.go b/deployment/keystone/changeset/deploy_forwarder.go index 2dc160dcf4c..5207d99aacd 100644 --- a/deployment/keystone/changeset/deploy_forwarder.go +++ b/deployment/keystone/changeset/deploy_forwarder.go @@ -9,16 +9,10 @@ import ( var _ deployment.ChangeSet[uint64] = DeployForwarder +// DeployForwarder deploys the KeystoneForwarder contract to all chains in the environment +// callers must merge the output addressbook with the existing one func DeployForwarder(env deployment.Environment, registryChainSel uint64) (deployment.ChangesetOutput, error) { lggr := env.Logger - // expect OCR3 to be deployed & capabilities registry - regAddrs, err := env.ExistingAddresses.AddressesForChain(registryChainSel) - if err != nil { - return deployment.ChangesetOutput{}, fmt.Errorf("no addresses found for chain %d: %w", registryChainSel, err) - } - if len(regAddrs) != 2 { - return deployment.ChangesetOutput{}, fmt.Errorf("expected 2 addresses for chain %d, got %d", registryChainSel, len(regAddrs)) - } ab := deployment.NewMemoryAddressBook() for _, chain := range env.Chains { lggr.Infow("deploying forwarder", "chainSelector", chain.Selector) diff --git a/deployment/keystone/changeset/deploy_forwarder_test.go b/deployment/keystone/changeset/deploy_forwarder_test.go index 8d73134fc1d..b6d8ec8f753 100644 --- a/deployment/keystone/changeset/deploy_forwarder_test.go +++ b/deployment/keystone/changeset/deploy_forwarder_test.go @@ -10,7 +10,6 @@ import ( "github.com/smartcontractkit/chainlink-common/pkg/logger" "github.com/smartcontractkit/chainlink/deployment" "github.com/smartcontractkit/chainlink/deployment/environment/memory" - kslb "github.com/smartcontractkit/chainlink/deployment/keystone" "github.com/smartcontractkit/chainlink/deployment/keystone/changeset" ) @@ -24,43 +23,11 @@ func TestDeployForwarder(t *testing.T) { } env := memory.NewMemoryEnvironment(t, lggr, zapcore.DebugLevel, cfg) - var ( - ocrTV = deployment.NewTypeAndVersion(kslb.OCR3Capability, deployment.Version1_0_0) - crTV = deployment.NewTypeAndVersion(kslb.CapabilitiesRegistry, deployment.Version1_0_0) - ) - registrySel := env.AllChainSelectors()[0] - t.Run("err if no capabilities registry on registry chain", func(t *testing.T) { - m := make(map[uint64]map[string]deployment.TypeAndVersion) - m[registrySel] = map[string]deployment.TypeAndVersion{ - "0x0000000000000000000000000000000000000002": ocrTV, - } - env.ExistingAddresses = deployment.NewMemoryAddressBookFromMap(m) - // capabilities registry and ocr3 must be deployed on registry chain - _, err := changeset.DeployForwarder(env, registrySel) - require.Error(t, err) - }) - - t.Run("err if no ocr3 on registry chain", func(t *testing.T) { - m := make(map[uint64]map[string]deployment.TypeAndVersion) - m[registrySel] = map[string]deployment.TypeAndVersion{ - "0x0000000000000000000000000000000000000001": crTV, - } - env.ExistingAddresses = deployment.NewMemoryAddressBookFromMap(m) - // capabilities registry and ocr3 must be deployed on registry chain - _, err := changeset.DeployForwarder(env, registrySel) - require.Error(t, err) - }) t.Run("should deploy forwarder", func(t *testing.T) { ab := deployment.NewMemoryAddressBook() - // fake capabilities registry - err := ab.Save(registrySel, "0x0000000000000000000000000000000000000001", crTV) - require.NoError(t, err) - // fake ocr3 - err = ab.Save(registrySel, "0x0000000000000000000000000000000000000002", ocrTV) - require.NoError(t, err) // deploy forwarder env.ExistingAddresses = ab resp, err := changeset.DeployForwarder(env, registrySel) diff --git a/deployment/keystone/changeset/deploy_ocr3.go b/deployment/keystone/changeset/deploy_ocr3.go index 40d9e558584..fdf51e644be 100644 --- a/deployment/keystone/changeset/deploy_ocr3.go +++ b/deployment/keystone/changeset/deploy_ocr3.go @@ -1,10 +1,11 @@ package changeset import ( + "encoding/json" "fmt" + "io" - "github.com/smartcontractkit/chainlink-common/pkg/logger" - + "github.com/smartcontractkit/ccip-owner-contracts/pkg/proposal/timelock" "github.com/smartcontractkit/chainlink/deployment" kslib "github.com/smartcontractkit/chainlink/deployment/keystone" ) @@ -27,12 +28,49 @@ func DeployOCR3(env deployment.Environment, registryChainSel uint64) (deployment return deployment.ChangesetOutput{AddressBook: ab}, nil } -func ConfigureOCR3Contract(lggr logger.Logger, env deployment.Environment, cfg kslib.ConfigureOCR3Config) (deployment.ChangesetOutput, error) { +var _ deployment.ChangeSet[ConfigureOCR3Config] = ConfigureOCR3Contract + +type ConfigureOCR3Config struct { + ChainSel uint64 + NodeIDs []string + OCR3Config *kslib.OracleConfigWithSecrets + DryRun bool + WriteGeneratedConfig io.Writer // if not nil, write the generated config to this writer as JSON [OCR2OracleConfig] + + UseMCMS bool +} - _, err := kslib.ConfigureOCR3ContractFromJD(&env, cfg) +func ConfigureOCR3Contract(env deployment.Environment, cfg ConfigureOCR3Config) (deployment.ChangesetOutput, error) { + resp, err := kslib.ConfigureOCR3ContractFromJD(&env, kslib.ConfigureOCR3Config{ + ChainSel: cfg.ChainSel, + NodeIDs: cfg.NodeIDs, + OCR3Config: cfg.OCR3Config, + DryRun: cfg.DryRun, + UseMCMS: cfg.UseMCMS, + }) if err != nil { return deployment.ChangesetOutput{}, fmt.Errorf("failed to configure OCR3Capability: %w", err) } + if w := cfg.WriteGeneratedConfig; w != nil { + b, err := json.MarshalIndent(&resp.OCR2OracleConfig, "", " ") + if err != nil { + return deployment.ChangesetOutput{}, fmt.Errorf("failed to marshal response output: %w", err) + } + env.Logger.Infof("Generated OCR3 config: %s", string(b)) + n, err := w.Write(b) + if err != nil { + return deployment.ChangesetOutput{}, fmt.Errorf("failed to write response output: %w", err) + } + if n != len(b) { + return deployment.ChangesetOutput{}, fmt.Errorf("failed to write all bytes") + } + } // does not create any new addresses - return deployment.ChangesetOutput{}, nil + var proposals []timelock.MCMSWithTimelockProposal + if cfg.UseMCMS { + proposals = append(proposals, *resp.Proposal) + } + return deployment.ChangesetOutput{ + Proposals: proposals, + }, nil } diff --git a/deployment/keystone/changeset/deploy_ocr3_test.go b/deployment/keystone/changeset/deploy_ocr3_test.go index d3fdf118f8b..0d49af68823 100644 --- a/deployment/keystone/changeset/deploy_ocr3_test.go +++ b/deployment/keystone/changeset/deploy_ocr3_test.go @@ -1,6 +1,8 @@ package changeset_test import ( + "bytes" + "encoding/json" "testing" "go.uber.org/zap/zapcore" @@ -8,8 +10,12 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "github.com/smartcontractkit/ccip-owner-contracts/pkg/gethwrappers" "github.com/smartcontractkit/chainlink-common/pkg/logger" + "github.com/smartcontractkit/chainlink/deployment" + commonchangeset "github.com/smartcontractkit/chainlink/deployment/common/changeset" "github.com/smartcontractkit/chainlink/deployment/environment/memory" + kslib "github.com/smartcontractkit/chainlink/deployment/keystone" "github.com/smartcontractkit/chainlink/deployment/keystone/changeset" ) @@ -37,3 +43,103 @@ func TestDeployOCR3(t *testing.T) { oaddrs, _ := resp.AddressBook.AddressesForChain(env.AllChainSelectors()[1]) assert.Len(t, oaddrs, 0) } + +func TestConfigureOCR3(t *testing.T) { + t.Parallel() + lggr := logger.Test(t) + + c := kslib.OracleConfigWithSecrets{ + OracleConfig: kslib.OracleConfig{ + MaxFaultyOracles: 1, + DeltaProgressMillis: 12345, + }, + OCRSecrets: deployment.XXXGenerateTestOCRSecrets(), + } + + t.Run("no mcms", func(t *testing.T) { + + te := SetupTestEnv(t, TestConfig{ + WFDonConfig: DonConfig{N: 4}, + AssetDonConfig: DonConfig{N: 4}, + WriterDonConfig: DonConfig{N: 4}, + NumChains: 1, + }) + + var wfNodes []string + for id, _ := range te.WFNodes { + wfNodes = append(wfNodes, id) + } + + w := &bytes.Buffer{} + cfg := changeset.ConfigureOCR3Config{ + ChainSel: te.RegistrySelector, + NodeIDs: wfNodes, + OCR3Config: &c, + WriteGeneratedConfig: w, + UseMCMS: false, + } + + csOut, err := changeset.ConfigureOCR3Contract(te.Env, cfg) + require.NoError(t, err) + var got kslib.OCR2OracleConfig + err = json.Unmarshal(w.Bytes(), &got) + require.NoError(t, err) + assert.Len(t, got.Signers, 4) + assert.Len(t, got.Transmitters, 4) + assert.Nil(t, csOut.Proposals) + }) + + t.Run("mcms", func(t *testing.T) { + te := SetupTestEnv(t, TestConfig{ + WFDonConfig: DonConfig{N: 4}, + AssetDonConfig: DonConfig{N: 4}, + WriterDonConfig: DonConfig{N: 4}, + NumChains: 1, + UseMCMS: true, + }) + + var wfNodes []string + for id, _ := range te.WFNodes { + wfNodes = append(wfNodes, id) + } + + w := &bytes.Buffer{} + cfg := changeset.ConfigureOCR3Config{ + ChainSel: te.RegistrySelector, + NodeIDs: wfNodes, + OCR3Config: &c, + WriteGeneratedConfig: w, + UseMCMS: true, + } + + csOut, err := changeset.ConfigureOCR3Contract(te.Env, cfg) + require.NoError(t, err) + var got kslib.OCR2OracleConfig + err = json.Unmarshal(w.Bytes(), &got) + require.NoError(t, err) + assert.Len(t, got.Signers, 4) + assert.Len(t, got.Transmitters, 4) + assert.NotNil(t, csOut.Proposals) + t.Logf("got: %v", csOut.Proposals[0]) + + contractSetsResp, err := kslib.GetContractSets(lggr, &kslib.GetContractSetsRequest{ + Chains: te.Env.Chains, + AddressBook: te.Env.ExistingAddresses, + }) + require.NoError(t, err) + var timelocks = map[uint64]*gethwrappers.RBACTimelock{ + te.RegistrySelector: contractSetsResp.ContractSets[te.RegistrySelector].Timelock, + } + // now apply the changeset such that the proposal is signed and execed + w2 := &bytes.Buffer{} + cfg.WriteGeneratedConfig = w2 + _, err = commonchangeset.ApplyChangesets(t, te.Env, timelocks, []commonchangeset.ChangesetApplication{ + { + Changeset: commonchangeset.WrapChangeSet(changeset.ConfigureOCR3Contract), + Config: cfg, + }, + }) + require.NoError(t, err) + }) + +} diff --git a/deployment/keystone/changeset/helpers_test.go b/deployment/keystone/changeset/helpers_test.go new file mode 100644 index 00000000000..85e69507009 --- /dev/null +++ b/deployment/keystone/changeset/helpers_test.go @@ -0,0 +1,419 @@ +package changeset_test + +import ( + "bytes" + "context" + "crypto/sha256" + "encoding/hex" + "errors" + "fmt" + "math/big" + "sort" + + "math" + "testing" + + "github.com/smartcontractkit/ccip-owner-contracts/pkg/gethwrappers" + "github.com/smartcontractkit/chainlink-common/pkg/logger" + "github.com/smartcontractkit/chainlink-common/pkg/utils/tests" + "github.com/smartcontractkit/chainlink-protos/job-distributor/v1/node" + "github.com/smartcontractkit/chainlink/deployment" + commonchangeset "github.com/smartcontractkit/chainlink/deployment/common/changeset" + commontypes "github.com/smartcontractkit/chainlink/deployment/common/types" + "github.com/smartcontractkit/chainlink/deployment/environment/memory" + "github.com/smartcontractkit/chainlink/deployment/keystone" + kslib "github.com/smartcontractkit/chainlink/deployment/keystone" + kschangeset "github.com/smartcontractkit/chainlink/deployment/keystone/changeset" + kcr "github.com/smartcontractkit/chainlink/v2/core/gethwrappers/keystone/generated/capabilities_registry" + "github.com/smartcontractkit/chainlink/v2/core/services/keystore/keys/p2pkey" + "github.com/stretchr/testify/require" + "go.uber.org/zap/zapcore" + "golang.org/x/exp/maps" +) + +func TestSetupTestEnv(t *testing.T) { + t.Parallel() + ctx := tests.Context(t) + for _, useMCMS := range []bool{true, false} { + te := SetupTestEnv(t, TestConfig{ + WFDonConfig: DonConfig{N: 4}, + AssetDonConfig: DonConfig{N: 4}, + WriterDonConfig: DonConfig{N: 4}, + NumChains: 3, + UseMCMS: useMCMS, + }) + t.Run(fmt.Sprintf("set up test env using MCMS: %T", useMCMS), func(t *testing.T) { + require.NotNil(t, te.Env.ExistingAddresses) + require.Len(t, te.Env.Chains, 3) + require.NotEmpty(t, te.RegistrySelector) + require.NotNil(t, te.Env.Offchain) + r, err := te.Env.Offchain.ListNodes(ctx, &node.ListNodesRequest{}) + require.NoError(t, err) + require.Len(t, r.Nodes, 12) + }) + } +} + +type DonConfig struct { + N int +} + +func (c DonConfig) Validate() error { + if c.N < 4 { + return errors.New("N must be at least 4") + } + return nil +} + +// TODO: separate the config into different types; wf should expand to types of ocr keybundles; writer to target chains; ... +type WFDonConfig = DonConfig +type AssetDonConfig = DonConfig +type WriterDonConfig = DonConfig + +type TestConfig struct { + WFDonConfig + AssetDonConfig + WriterDonConfig + NumChains int + + UseMCMS bool +} + +func (c TestConfig) Validate() error { + if err := c.WFDonConfig.Validate(); err != nil { + return err + } + if err := c.AssetDonConfig.Validate(); err != nil { + return err + } + if err := c.WriterDonConfig.Validate(); err != nil { + return err + } + if c.NumChains < 1 { + return errors.New("NumChains must be at least 1") + } + return nil +} + +type TestEnv struct { + Env deployment.Environment + RegistrySelector uint64 + + WFNodes map[string]memory.Node + CWNodes map[string]memory.Node + AssetNodes map[string]memory.Node +} + +// SetupTestEnv sets up a keystone test environment with the given configuration +func SetupTestEnv(t *testing.T, c TestConfig) TestEnv { + require.NoError(t, c.Validate()) + lggr := logger.Test(t) + ctx := tests.Context(t) + chains, _ := memory.NewMemoryChains(t, c.NumChains, 1) + registryChainSel := registryChain(t, chains) + // note that all the nodes require TOML configuration of the cap registry address + // and writers need forwarder address as TOML config + // we choose to use changesets to deploy the initial contracts because that's how it's done in the real world + // this requires a initial environment to house the address book + e := deployment.Environment{ + Logger: lggr, + Chains: chains, + ExistingAddresses: deployment.NewMemoryAddressBook(), + } + e, err := commonchangeset.ApplyChangesets(t, e, nil, []commonchangeset.ChangesetApplication{ + { + Changeset: commonchangeset.WrapChangeSet(kschangeset.DeployCapabilityRegistry), + Config: registryChainSel, + }, + { + Changeset: commonchangeset.WrapChangeSet(kschangeset.DeployOCR3), + Config: registryChainSel, + }, + { + Changeset: commonchangeset.WrapChangeSet(kschangeset.DeployForwarder), + Config: registryChainSel, + }, + }) + require.NoError(t, err) + require.NotNil(t, e) + require.Len(t, e.Chains, c.NumChains) + validateInitialChainState(t, e, registryChainSel) + // now that we have the initial contracts deployed, we can configure the nodes with the addresses + // TODO: configure the nodes with the correct override functions + crConfig := deployment.CapabilityRegistryConfig{ + EVMChainID: registryChainSel, + Contract: [20]byte{}, + } + + wfChains := map[uint64]deployment.Chain{} + wfChains[registryChainSel] = chains[registryChainSel] + wfNodes := memory.NewNodes(t, zapcore.InfoLevel, wfChains, c.WFDonConfig.N, 0, crConfig) + require.Len(t, wfNodes, c.WFDonConfig.N) + + writerChains := map[uint64]deployment.Chain{} + maps.Copy(writerChains, chains) + cwNodes := memory.NewNodes(t, zapcore.InfoLevel, writerChains, c.WriterDonConfig.N, 0, crConfig) + require.Len(t, cwNodes, c.WriterDonConfig.N) + + assetChains := map[uint64]deployment.Chain{} + assetChains[registryChainSel] = chains[registryChainSel] + assetNodes := memory.NewNodes(t, zapcore.InfoLevel, assetChains, c.AssetDonConfig.N, 0, crConfig) + require.Len(t, assetNodes, c.AssetDonConfig.N) + + // TODO: partition nodes into multiple nops + + wfDon := keystone.DonCapabilities{ + Name: keystone.WFDonName, + Nops: []keystone.NOP{ + { + Name: "nop 1", + Nodes: maps.Keys(wfNodes), + }, + }, + Capabilities: []kcr.CapabilitiesRegistryCapability{keystone.OCR3Cap}, + } + cwDon := keystone.DonCapabilities{ + Name: keystone.TargetDonName, + Nops: []keystone.NOP{ + { + Name: "nop 2", + Nodes: maps.Keys(cwNodes), + }, + }, + Capabilities: []kcr.CapabilitiesRegistryCapability{keystone.WriteChainCap}, + } + assetDon := keystone.DonCapabilities{ + Name: keystone.StreamDonName, + Nops: []keystone.NOP{ + { + Name: "nop 3", + Nodes: maps.Keys(assetNodes), + }, + }, + Capabilities: []kcr.CapabilitiesRegistryCapability{keystone.StreamTriggerCap}, + } + + allChains := make(map[uint64]deployment.Chain) + maps.Copy(allChains, chains) + + allNodes := make(map[string]memory.Node) + maps.Copy(allNodes, wfNodes) + maps.Copy(allNodes, cwNodes) + maps.Copy(allNodes, assetNodes) + env := memory.NewMemoryEnvironmentFromChainsNodes(func() context.Context { return ctx }, lggr, allChains, allNodes) + // set the env addresses to the deployed addresses that were created prior to configuring the nodes + err = env.ExistingAddresses.Merge(e.ExistingAddresses) + require.NoError(t, err) + + var ocr3Config = keystone.OracleConfigWithSecrets{ + OracleConfig: keystone.OracleConfig{ + MaxFaultyOracles: len(wfNodes) / 3, + }, + OCRSecrets: deployment.XXXGenerateTestOCRSecrets(), + } + var allDons = []keystone.DonCapabilities{wfDon, cwDon, assetDon} + + _, err = kschangeset.ConfigureInitialContractsChangeset(env, kschangeset.InitialContractsCfg{ + RegistryChainSel: registryChainSel, + Dons: allDons, + OCR3Config: &ocr3Config, + }) + require.NoError(t, err) + // TODO: KS-rm_deploy_opt + //require.Nil(t, csOut.AddressBook, "no new addresses should be created in configure initial contracts") + //require.NoError(t, env.ExistingAddresses.Merge(csOut.AddressBook)) + + req := &keystone.GetContractSetsRequest{ + Chains: env.Chains, + AddressBook: env.ExistingAddresses, + } + + contractSetsResp, err := keystone.GetContractSets(lggr, req) + require.NoError(t, err) + require.Len(t, contractSetsResp.ContractSets, len(env.Chains)) + // check the registry + gotRegistry := contractSetsResp.ContractSets[registryChainSel].CapabilitiesRegistry + require.NotNil(t, gotRegistry) + // validate the registry + // check the nodes + gotNodes, err := gotRegistry.GetNodes(nil) + require.NoError(t, err) + require.Len(t, gotNodes, len(allNodes)) + validateNodes(t, gotRegistry, wfNodes, expectedHashedCapabilities(t, gotRegistry, wfDon)) + validateNodes(t, gotRegistry, cwNodes, expectedHashedCapabilities(t, gotRegistry, cwDon)) + validateNodes(t, gotRegistry, assetNodes, expectedHashedCapabilities(t, gotRegistry, assetDon)) + + // check the dons + validateDon(t, gotRegistry, wfNodes, wfDon) + validateDon(t, gotRegistry, cwNodes, cwDon) + validateDon(t, gotRegistry, assetNodes, assetDon) + + if c.UseMCMS { + // TODO: mcms on all the chains, currently only on the registry chain. need to fix this for forwarders + t.Logf("Enabling MCMS registry chain %d", registryChainSel) + // deploy, configure and xfer ownership of MCMS + env, err = commonchangeset.ApplyChangesets(t, env, nil, []commonchangeset.ChangesetApplication{ + { + Changeset: commonchangeset.WrapChangeSet(commonchangeset.DeployMCMSWithTimelock), + Config: map[uint64]commontypes.MCMSWithTimelockConfig{ + registryChainSel: { + Canceller: commonchangeset.SingleGroupMCMS(t), + Bypasser: commonchangeset.SingleGroupMCMS(t), + Proposer: commonchangeset.SingleGroupMCMS(t), + TimelockExecutors: env.AllDeployerKeys(), + TimelockMinDelay: big.NewInt(0), + }, + }, + }, + }) + require.NoError(t, err) + // extract the MCMS address + r, err := kslib.GetContractSets(lggr, &kslib.GetContractSetsRequest{ + Chains: map[uint64]deployment.Chain{ + registryChainSel: env.Chains[registryChainSel], + }, + AddressBook: env.ExistingAddresses, + }) + require.NoError(t, err) + mcms := r.ContractSets[registryChainSel].MCMSWithTimelockState + require.NotNil(t, mcms) + // transfer ownership of all contracts to the MCMS + env, err = commonchangeset.ApplyChangesets(t, env, map[uint64]*gethwrappers.RBACTimelock{registryChainSel: mcms.Timelock}, []commonchangeset.ChangesetApplication{ + { + Changeset: commonchangeset.WrapChangeSet(kschangeset.AcceptAllOwnershipsProposal), + Config: &kschangeset.AcceptAllOwnershipRequest{ + ChainSelector: registryChainSel, + MinDelay: 0, + }, + }, + }) + require.NoError(t, err) + // ensure the MCMS is deployed + req = &keystone.GetContractSetsRequest{ + Chains: env.Chains, + AddressBook: env.ExistingAddresses, + } + contractSetsResp, err = keystone.GetContractSets(lggr, req) + require.NoError(t, err) + require.Len(t, contractSetsResp.ContractSets, len(env.Chains)) + // check the mcms contract on registry chain + gotMCMS := contractSetsResp.ContractSets[registryChainSel].MCMSWithTimelockState + require.NoError(t, gotMCMS.Validate()) + } + return TestEnv{ + Env: env, + RegistrySelector: registryChainSel, + WFNodes: wfNodes, + CWNodes: cwNodes, + AssetNodes: assetNodes, + } +} + +func registryChain(t *testing.T, chains map[uint64]deployment.Chain) uint64 { + var registryChainSel uint64 = math.MaxUint64 + for sel := range chains { + if sel < registryChainSel { + registryChainSel = sel + } + } + return registryChainSel +} + +// validateInitialChainState checks that the initial chain state +// has the expected contracts deployed +func validateInitialChainState(t *testing.T, env deployment.Environment, registryChainSel uint64) { + ad := env.ExistingAddresses + // all contracts on registry chain + registryChainAddrs, err := ad.AddressesForChain(registryChainSel) + require.NoError(t, err) + require.Len(t, registryChainAddrs, 3) // registry, ocr3, forwarder + // only forwarder on non-home chain + for sel := range env.Chains { + chainAddrs, err := ad.AddressesForChain(sel) + require.NoError(t, err) + if sel != registryChainSel { + require.Len(t, chainAddrs, 1) + } else { + require.Len(t, chainAddrs, 3) + } + containsForwarder := false + for _, tv := range chainAddrs { + if tv.Type == keystone.KeystoneForwarder { + containsForwarder = true + break + } + } + require.True(t, containsForwarder, "no forwarder found in %v on chain %d for target don", chainAddrs, sel) + } +} + +// validateNodes checks that the nodes exist and have the expected capabilities +func validateNodes(t *testing.T, gotRegistry *kcr.CapabilitiesRegistry, nodes map[string]memory.Node, expectedHashedCaps [][32]byte) { + gotNodes, err := gotRegistry.GetNodesByP2PIds(nil, p2pIDs(t, maps.Keys(nodes))) + require.NoError(t, err) + require.Len(t, gotNodes, len(nodes)) + for _, n := range gotNodes { + require.Equal(t, expectedHashedCaps, n.HashedCapabilityIds) + } +} + +// validateDon checks that the don exists and has the expected capabilities +func validateDon(t *testing.T, gotRegistry *kcr.CapabilitiesRegistry, nodes map[string]memory.Node, don kslib.DonCapabilities) { + gotDons, err := gotRegistry.GetDONs(nil) + require.NoError(t, err) + wantP2PID := sortedHash(p2pIDs(t, maps.Keys(nodes))) + found := false + for _, have := range gotDons { + gotP2PID := sortedHash(have.NodeP2PIds) + if gotP2PID == wantP2PID { + found = true + gotCapIDs := capIDs(t, have.CapabilityConfigurations) + require.Equal(t, expectedHashedCapabilities(t, gotRegistry, don), gotCapIDs) + break + } + } + require.True(t, found, "don not found in registry") +} + +func capIDs(t *testing.T, cfgs []kcr.CapabilitiesRegistryCapabilityConfiguration) [][32]byte { + var out [][32]byte + for _, cfg := range cfgs { + out = append(out, cfg.CapabilityId) + } + return out +} + +func ptr[T any](t T) *T { + return &t +} + +func p2pIDs(t *testing.T, vals []string) [][32]byte { + var out [][32]byte + for _, v := range vals { + id, err := p2pkey.MakePeerID(v) + require.NoError(t, err) + out = append(out, id) + } + return out +} + +func expectedHashedCapabilities(t *testing.T, registry *kcr.CapabilitiesRegistry, don kslib.DonCapabilities) [][32]byte { + out := make([][32]byte, len(don.Capabilities)) + var err error + for i, cap := range don.Capabilities { + out[i], err = registry.GetHashedCapabilityId(nil, cap.LabelledName, cap.Version) + require.NoError(t, err) + } + return out +} + +func sortedHash(p2pids [][32]byte) string { + sha256Hash := sha256.New() + sort.Slice(p2pids, func(i, j int) bool { + return bytes.Compare(p2pids[i][:], p2pids[j][:]) < 0 + }) + for _, id := range p2pids { + sha256Hash.Write(id[:]) + } + return hex.EncodeToString(sha256Hash.Sum(nil)) +} diff --git a/deployment/keystone/changeset/internal/test/utils.go b/deployment/keystone/changeset/internal/test/utils.go index df01266043a..6fe4a8f4a2e 100644 --- a/deployment/keystone/changeset/internal/test/utils.go +++ b/deployment/keystone/changeset/internal/test/utils.go @@ -158,7 +158,6 @@ func addDons(t *testing.T, lggr logger.Logger, chain deployment.Chain, registry cc.Config = defaultCapConfig(t, ccfg.Capability) } var exists bool - //var cc kcr.CapabilitiesRegistryCapabilityConfiguration{} cc.CapabilityId, exists = capCache.Get(ccfg.Capability) require.True(t, exists, "capability not found in cache %v", ccfg.Capability) capConfigs = append(capConfigs, cc) diff --git a/deployment/keystone/deploy.go b/deployment/keystone/deploy.go index 374f7f06460..0370cc54ba9 100644 --- a/deployment/keystone/deploy.go +++ b/deployment/keystone/deploy.go @@ -16,6 +16,8 @@ import ( "github.com/ethereum/go-ethereum/rpc" "golang.org/x/exp/maps" + "github.com/smartcontractkit/ccip-owner-contracts/pkg/proposal/timelock" + "github.com/smartcontractkit/chainlink/deployment" "google.golang.org/protobuf/proto" @@ -40,6 +42,7 @@ type ConfigureContractsRequest struct { Dons []DonCapabilities // externally sourced based on the environment OCR3Config *OracleConfigWithSecrets // TODO: probably should be a map of don to config; but currently we only have one wf don therefore one config + // TODO rm this option; unused DoContractDeploy bool // if false, the contracts are assumed to be deployed and the address book is used } @@ -75,6 +78,7 @@ func ConfigureContracts(ctx context.Context, lggr logger.Logger, req ConfigureCo } addrBook := req.Env.ExistingAddresses + // TODO: KS-rm_deploy_opt remove this option; it's not used if req.DoContractDeploy { contractDeployCS, err := DeployContracts(req.Env, req.RegistryChainSel) if err != nil { @@ -149,6 +153,7 @@ func DeployContracts(e *deployment.Environment, chainSel uint64) (*deployment.Ch // DonInfo is DonCapabilities, but expanded to contain node information type DonInfo struct { Name string + F uint8 Nodes []deployment.Node Capabilities []kcr.CapabilitiesRegistryCapability // every capability is hosted on each node } @@ -166,6 +171,7 @@ func DonInfos(dons []DonCapabilities, jd deployment.OffchainClient) ([]DonInfo, } donInfos = append(donInfos, DonInfo{ Name: don.Name, + F: don.F, Nodes: nodes, Capabilities: don.Capabilities, }) @@ -204,11 +210,6 @@ func GetRegistryContract(e *deployment.Environment, registryChainSel uint64, add // ConfigureRegistry configures the registry contract with the given DONS and their capabilities // the address book is required to contain the addresses of the deployed registry contract func ConfigureRegistry(ctx context.Context, lggr logger.Logger, req ConfigureContractsRequest, addrBook deployment.AddressBook) (*ConfigureContractsResponse, error) { - registry, registryChain, err := GetRegistryContract(req.Env, req.RegistryChainSel, addrBook) - if err != nil { - return nil, fmt.Errorf("failed to get registry: %w", err) - } - donInfos, err := DonInfos(req.Dons, req.Env.Offchain) if err != nil { return nil, fmt.Errorf("failed to get don infos: %w", err) @@ -253,13 +254,13 @@ func ConfigureRegistry(ctx context.Context, lggr logger.Logger, req ConfigureCon lggr.Infow("registered node operators", "nops", nopsResp.Nops) // register nodes - nodesResp, err := registerNodes(lggr, ®isterNodesRequest{ - registry: registry, - chain: registryChain, - nopToNodeIDs: nopsToNodeIDs, - donToNodes: donToNodes, - donToCapabilities: capabilitiesResp.DonToCapabilities, - nops: nopsResp.Nops, + nodesResp, err := RegisterNodes(lggr, &RegisterNodesRequest{ + Env: req.Env, + RegistryChainSelector: req.RegistryChainSel, + NopToNodeIDs: nopsToNodeIDs, + DonToNodes: donToNodes, + DonToCapabilities: capabilitiesResp.DonToCapabilities, + Nops: nopsResp.Nops, }) if err != nil { return nil, fmt.Errorf("failed to register nodes: %w", err) @@ -268,24 +269,47 @@ func ConfigureRegistry(ctx context.Context, lggr logger.Logger, req ConfigureCon // TODO: annotate nodes with node_operator_id in JD? + donsToRegister := []DONToRegister{} + for _, don := range req.Dons { + nodes, ok := donToNodes[don.Name] + if !ok { + return nil, fmt.Errorf("nodes not found for don %s", don.Name) + } + f := don.F + if f == 0 { + // TODO: fallback to a default value for compatibility - change to error + f = uint8(len(nodes) / 3) + lggr.Warnw("F not set for don - falling back to default", "don", don.Name, "f", f) + } + donsToRegister = append(donsToRegister, DONToRegister{ + Name: don.Name, + F: f, + Nodes: nodes, + }) + } + + nodeIdToP2PID := map[string][32]byte{} + for nodeID, params := range nodesResp.nodeIDToParams { + nodeIdToP2PID[nodeID] = params.P2pId + } // register DONS - donsResp, err := registerDons(lggr, registerDonsRequest{ - registry: registry, - chain: registryChain, - nodeIDToParams: nodesResp.nodeIDToParams, - donToCapabilities: capabilitiesResp.DonToCapabilities, - donToNodes: donToNodes, + donsResp, err := RegisterDons(lggr, RegisterDonsRequest{ + Env: req.Env, + RegistryChainSelector: req.RegistryChainSel, + NodeIDToP2PID: nodeIdToP2PID, + DonToCapabilities: capabilitiesResp.DonToCapabilities, + DonsToRegister: donsToRegister, }) if err != nil { return nil, fmt.Errorf("failed to register DONS: %w", err) } - lggr.Infow("registered DONs", "dons", len(donsResp.donInfos)) + lggr.Infow("registered DONs", "dons", len(donsResp.DonInfos)) return &ConfigureContractsResponse{ Changeset: &deployment.ChangesetOutput{ AddressBook: addrBook, }, - DonInfos: donsResp.donInfos, + DonInfos: donsResp.DonInfos, }, nil } @@ -320,6 +344,7 @@ func ConfigureForwardContracts(env *deployment.Environment, dons []RegisteredDon return nil } +// Depreciated: use changeset.ConfigureOCR3Contract instead // ocr3 contract on the registry chain for the wf dons func ConfigureOCR3Contract(env *deployment.Environment, chainSel uint64, dons []RegisteredDon, addrBook deployment.AddressBook, cfg *OracleConfigWithSecrets) error { registryChain, ok := env.Chains[chainSel] @@ -350,10 +375,11 @@ func ConfigureOCR3Contract(env *deployment.Environment, chainSel uint64, dons [] } _, err := configureOCR3contract(configureOCR3Request{ - cfg: cfg, - chain: registryChain, - contract: contract, - nodes: don.Nodes, + cfg: cfg, + chain: registryChain, + contract: contract, + nodes: don.Nodes, + contractSet: &contracts, }) if err != nil { return fmt.Errorf("failed to configure OCR3 contract for don %s: %w", don.Name, err) @@ -364,6 +390,7 @@ func ConfigureOCR3Contract(env *deployment.Environment, chainSel uint64, dons [] type ConfigureOCR3Resp struct { OCR2OracleConfig + Proposal *timelock.MCMSWithTimelockProposal } type ConfigureOCR3Config struct { @@ -371,8 +398,11 @@ type ConfigureOCR3Config struct { NodeIDs []string OCR3Config *OracleConfigWithSecrets DryRun bool + + UseMCMS bool } +// Depreciated: use changeset.ConfigureOCR3Contract instead func ConfigureOCR3ContractFromJD(env *deployment.Environment, cfg ConfigureOCR3Config) (*ConfigureOCR3Resp, error) { prefix := "" if cfg.DryRun { @@ -403,17 +433,20 @@ func ConfigureOCR3ContractFromJD(env *deployment.Environment, cfg ConfigureOCR3C return nil, err } r, err := configureOCR3contract(configureOCR3Request{ - cfg: cfg.OCR3Config, - chain: registryChain, - contract: contract, - nodes: nodes, - dryRun: cfg.DryRun, + cfg: cfg.OCR3Config, + chain: registryChain, + contract: contract, + nodes: nodes, + dryRun: cfg.DryRun, + contractSet: &contracts, + useMCMS: cfg.UseMCMS, }) if err != nil { return nil, err } return &ConfigureOCR3Resp{ OCR2OracleConfig: r.ocrConfig, + Proposal: r.proposal, }, nil } @@ -433,6 +466,21 @@ type RegisteredCapability struct { ID [32]byte } +func FromCapabilitiesRegistryCapability(cap *kcr.CapabilitiesRegistryCapability, e deployment.Environment, registryChainSelector uint64) (*RegisteredCapability, error) { + registry, _, err := GetRegistryContract(&e, registryChainSelector, e.ExistingAddresses) + if err != nil { + return nil, fmt.Errorf("failed to get registry: %w", err) + } + id, err := registry.GetHashedCapabilityId(&bind.CallOpts{}, cap.LabelledName, cap.Version) + if err != nil { + return nil, fmt.Errorf("failed to call GetHashedCapabilityId for capability %v: %w", cap, err) + } + return &RegisteredCapability{ + CapabilitiesRegistryCapability: *cap, + ID: id, + }, nil +} + // RegisterCapabilities add computes the capability id, adds it to the registry and associates the registered capabilities with appropriate don(s) func RegisterCapabilities(lggr logger.Logger, req RegisterCapabilitiesRequest) (*RegisterCapabilitiesResponse, error) { if len(req.DonToCapabilities) == 0 { @@ -617,34 +665,39 @@ func DecodeErr(encodedABI string, err error) error { } // register nodes -type registerNodesRequest struct { - registry *kcr.CapabilitiesRegistry - chain deployment.Chain - nopToNodeIDs map[kcr.CapabilitiesRegistryNodeOperator][]string - donToNodes map[string][]deployment.Node - donToCapabilities map[string][]RegisteredCapability - nops []*kcr.CapabilitiesRegistryNodeOperatorAdded -} -type registerNodesResponse struct { +type RegisterNodesRequest struct { + Env *deployment.Environment + RegistryChainSelector uint64 + NopToNodeIDs map[kcr.CapabilitiesRegistryNodeOperator][]string + DonToNodes map[string][]deployment.Node + DonToCapabilities map[string][]RegisteredCapability + Nops []*kcr.CapabilitiesRegistryNodeOperatorAdded +} +type RegisterNodesResponse struct { nodeIDToParams map[string]kcr.CapabilitiesRegistryNodeParams } // registerNodes registers the nodes with the registry. it assumes that the deployer key in the Chain // can sign the transactions update the contract state // TODO: 467 refactor to support MCMS. Specifically need to separate the call data generation from the actual contract call -func registerNodes(lggr logger.Logger, req *registerNodesRequest) (*registerNodesResponse, error) { +func RegisterNodes(lggr logger.Logger, req *RegisterNodesRequest) (*RegisterNodesResponse, error) { + registry, registryChain, err := GetRegistryContract(req.Env, req.RegistryChainSelector, req.Env.ExistingAddresses) + if err != nil { + return nil, fmt.Errorf("failed to get registry: %w", err) + } + var count int - for _, nodes := range req.nopToNodeIDs { + for _, nodes := range req.NopToNodeIDs { count += len(nodes) } lggr.Infow("registering nodes...", "len", count) nodeToRegisterNop := make(map[string]*kcr.CapabilitiesRegistryNodeOperatorAdded) - for _, nop := range req.nops { + for _, nop := range req.Nops { n := kcr.CapabilitiesRegistryNodeOperator{ Name: nop.Name, Admin: nop.Admin, } - nodeIDs := req.nopToNodeIDs[n] + nodeIDs := req.NopToNodeIDs[n] for _, nodeID := range nodeIDs { _, exists := nodeToRegisterNop[nodeID] if !exists { @@ -654,7 +707,7 @@ func registerNodes(lggr logger.Logger, req *registerNodesRequest) (*registerNode } // TODO: deduplicate everywhere - registryChainID, err := chainsel.ChainIdFromSelector(req.chain.Selector) + registryChainID, err := chainsel.ChainIdFromSelector(registryChain.Selector) if err != nil { return nil, err } @@ -664,10 +717,10 @@ func registerNodes(lggr logger.Logger, req *registerNodesRequest) (*registerNode } nodeIDToParams := make(map[string]kcr.CapabilitiesRegistryNodeParams) - for don, nodes := range req.donToNodes { - caps, ok := req.donToCapabilities[don] + for don, nodes := range req.DonToNodes { + caps, ok := req.DonToCapabilities[don] if !ok { - return nil, fmt.Errorf("capabilities not found for node operator %s", don) + return nil, fmt.Errorf("capabilities not found for don %s", don) } var hashedCapabilityIds [][32]byte for _, cap := range caps { @@ -688,7 +741,7 @@ func registerNodes(lggr logger.Logger, req *registerNodesRequest) (*registerNode if !ok { evmCC, exists := n.SelToOCRConfig[registryChainDetails] if !exists { - return nil, fmt.Errorf("config for selector not found on node: %v", req.chain.Selector) + return nil, fmt.Errorf("config for selector %v not found on node (id: %s, name: %s)", registryChain.Selector, n.NodeID, n.Name) } var signer [32]byte copy(signer[:], evmCC.OnchainPublicKey) @@ -726,8 +779,8 @@ func registerNodes(lggr logger.Logger, req *registerNodesRequest) (*registerNode for _, v := range nodeIDToParams { uniqueNodeParams = append(uniqueNodeParams, v) } - lggr.Debugw("unique node params to add", "count", len(uniqueNodeParams)) - tx, err := req.registry.AddNodes(req.chain.DeployerKey, uniqueNodeParams) + lggr.Debugw("unique node params to add", "count", len(uniqueNodeParams), "params", uniqueNodeParams) + tx, err := registry.AddNodes(registryChain.DeployerKey, uniqueNodeParams) if err != nil { err = DecodeErr(kcr.CapabilitiesRegistryABI, err) // no typed errors in the abi, so we have to do string matching @@ -737,7 +790,7 @@ func registerNodes(lggr logger.Logger, req *registerNodesRequest) (*registerNode } lggr.Warn("nodes already exist, falling back to 1-by-1") for _, singleNodeParams := range uniqueNodeParams { - tx, err = req.registry.AddNodes(req.chain.DeployerKey, []kcr.CapabilitiesRegistryNodeParams{singleNodeParams}) + tx, err = registry.AddNodes(registryChain.DeployerKey, []kcr.CapabilitiesRegistryNodeParams{singleNodeParams}) if err != nil { err = DecodeErr(kcr.CapabilitiesRegistryABI, err) if strings.Contains(err.Error(), "NodeAlreadyExists") { @@ -747,7 +800,7 @@ func registerNodes(lggr logger.Logger, req *registerNodesRequest) (*registerNode return nil, fmt.Errorf("failed to call AddNode for node with p2pid %v: %w", singleNodeParams.P2pId, err) } // 1-by-1 tx is pending and we need to wait for it to be mined - _, err = req.chain.Confirm(tx) + _, err = registryChain.Confirm(tx) if err != nil { return nil, fmt.Errorf("failed to confirm AddNode of p2pid node %v transaction %s: %w", singleNodeParams.P2pId, tx.Hash().String(), err) } @@ -755,27 +808,33 @@ func registerNodes(lggr logger.Logger, req *registerNodesRequest) (*registerNode } } else { // the bulk add tx is pending and we need to wait for it to be mined - _, err = req.chain.Confirm(tx) + _, err = registryChain.Confirm(tx) if err != nil { return nil, fmt.Errorf("failed to confirm AddNode confirm transaction %s: %w", tx.Hash().String(), err) } } - return ®isterNodesResponse{ + return &RegisterNodesResponse{ nodeIDToParams: nodeIDToParams, }, nil } -type registerDonsRequest struct { - registry *kcr.CapabilitiesRegistry - chain deployment.Chain +type DONToRegister struct { + Name string + F uint8 + Nodes []deployment.Node +} - nodeIDToParams map[string]kcr.CapabilitiesRegistryNodeParams - donToCapabilities map[string][]RegisteredCapability - donToNodes map[string][]deployment.Node +type RegisterDonsRequest struct { + Env *deployment.Environment + RegistryChainSelector uint64 + + NodeIDToP2PID map[string][32]byte + DonToCapabilities map[string][]RegisteredCapability + DonsToRegister []DONToRegister } -type registerDonsResponse struct { - donInfos map[string]kcr.CapabilitiesRegistryDONInfo +type RegisterDonsResponse struct { + DonInfos map[string]kcr.CapabilitiesRegistryDONInfo } func sortedHash(p2pids [][32]byte) string { @@ -789,14 +848,18 @@ func sortedHash(p2pids [][32]byte) string { return hex.EncodeToString(sha256Hash.Sum(nil)) } -func registerDons(lggr logger.Logger, req registerDonsRequest) (*registerDonsResponse, error) { - lggr.Infow("registering DONs...", "len", len(req.donToNodes)) +func RegisterDons(lggr logger.Logger, req RegisterDonsRequest) (*RegisterDonsResponse, error) { + registry, registryChain, err := GetRegistryContract(req.Env, req.RegistryChainSelector, req.Env.ExistingAddresses) + if err != nil { + return nil, fmt.Errorf("failed to get registry: %w", err) + } + lggr.Infow("registering DONs...", "len", len(req.DonsToRegister)) // track hash of sorted p2pids to don name because the registry return value does not include the don name // and we need to map it back to the don name to access the other mapping data such as the don's capabilities & nodes p2pIdsToDon := make(map[string]string) var addedDons = 0 - donInfos, err := req.registry.GetDONs(&bind.CallOpts{}) + donInfos, err := registry.GetDONs(&bind.CallOpts{}) if err != nil { err = DecodeErr(kcr.CapabilitiesRegistryABI, err) return nil, fmt.Errorf("failed to call GetDONs: %w", err) @@ -807,30 +870,30 @@ func registerDons(lggr logger.Logger, req registerDonsRequest) (*registerDonsRes } lggr.Infow("fetched existing DONs...", "len", len(donInfos), "lenByNodesHash", len(existingDONs)) - for don, nodes := range req.donToNodes { + for _, don := range req.DonsToRegister { var p2pIds [][32]byte - for _, n := range nodes { + for _, n := range don.Nodes { if n.IsBootstrap { continue } - params, ok := req.nodeIDToParams[n.NodeID] + p2pID, ok := req.NodeIDToP2PID[n.NodeID] if !ok { return nil, fmt.Errorf("node params not found for non-bootstrap node %s", n.NodeID) } - p2pIds = append(p2pIds, params.P2pId) + p2pIds = append(p2pIds, p2pID) } p2pSortedHash := sortedHash(p2pIds) - p2pIdsToDon[p2pSortedHash] = don + p2pIdsToDon[p2pSortedHash] = don.Name if _, ok := existingDONs[p2pSortedHash]; ok { lggr.Debugw("don already exists, ignoring", "don", don, "p2p sorted hash", p2pSortedHash) continue } - caps, ok := req.donToCapabilities[don] + caps, ok := req.DonToCapabilities[don.Name] if !ok { - return nil, fmt.Errorf("capabilities not found for node operator %s", don) + return nil, fmt.Errorf("capabilities not found for DON %s", don.Name) } wfSupported := false var cfgs []kcr.CapabilitiesRegistryCapabilityConfiguration @@ -850,17 +913,16 @@ func registerDons(lggr logger.Logger, req registerDonsRequest) (*registerDonsRes }) } - f := len(p2pIds) / 3 // assuming n=3f+1. TODO should come for some config. - tx, err := req.registry.AddDON(req.chain.DeployerKey, p2pIds, cfgs, true, wfSupported, uint8(f)) + tx, err := registry.AddDON(registryChain.DeployerKey, p2pIds, cfgs, true, wfSupported, don.F) if err != nil { err = DecodeErr(kcr.CapabilitiesRegistryABI, err) - return nil, fmt.Errorf("failed to call AddDON for don '%s' p2p2Id hash %s capability %v: %w", don, p2pSortedHash, cfgs, err) + return nil, fmt.Errorf("failed to call AddDON for don '%s' p2p2Id hash %s capability %v: %w", don.Name, p2pSortedHash, cfgs, err) } - _, err = req.chain.Confirm(tx) + _, err = registryChain.Confirm(tx) if err != nil { - return nil, fmt.Errorf("failed to confirm AddDON transaction %s for don %s: %w", tx.Hash().String(), don, err) + return nil, fmt.Errorf("failed to confirm AddDON transaction %s for don %s: %w", tx.Hash().String(), don.Name, err) } - lggr.Debugw("registered DON", "don", don, "p2p sorted hash", p2pSortedHash, "cgs", cfgs, "wfSupported", wfSupported, "f", f) + lggr.Debugw("registered DON", "don", don.Name, "p2p sorted hash", p2pSortedHash, "cgs", cfgs, "wfSupported", wfSupported, "f", don.F) addedDons++ } lggr.Debugf("Registered all DONs (new=%d), waiting for registry to update", addedDons) @@ -870,7 +932,7 @@ func registerDons(lggr logger.Logger, req registerDonsRequest) (*registerDonsRes foundAll := false for i := 0; i < 10; i++ { lggr.Debugw("attempting to get DONs from registry", "attempt#", i) - donInfos, err = req.registry.GetDONs(&bind.CallOpts{}) + donInfos, err = registry.GetDONs(&bind.CallOpts{}) if !containsAllDONs(donInfos, p2pIdsToDon) { lggr.Debugw("some expected dons not registered yet, re-checking after a delay ...") time.Sleep(2 * time.Second) @@ -887,8 +949,8 @@ func registerDons(lggr logger.Logger, req registerDonsRequest) (*registerDonsRes return nil, fmt.Errorf("did not find all desired DONS") } - resp := registerDonsResponse{ - donInfos: make(map[string]kcr.CapabilitiesRegistryDONInfo), + resp := RegisterDonsResponse{ + DonInfos: make(map[string]kcr.CapabilitiesRegistryDONInfo), } for i, donInfo := range donInfos { donName, ok := p2pIdsToDon[sortedHash(donInfo.NodeP2PIds)] @@ -897,7 +959,7 @@ func registerDons(lggr logger.Logger, req registerDonsRequest) (*registerDonsRes continue } lggr.Debugw("adding don info to the reponse (keyed by DON name)", "don", donName) - resp.donInfos[donName] = donInfos[i] + resp.DonInfos[donName] = donInfos[i] } return &resp, nil } diff --git a/deployment/keystone/deploy_test.go b/deployment/keystone/deploy_test.go index b02497c22fa..fd59f3007fd 100644 --- a/deployment/keystone/deploy_test.go +++ b/deployment/keystone/deploy_test.go @@ -1,7 +1,6 @@ package keystone_test import ( - "context" "encoding/json" "fmt" "os" @@ -22,192 +21,9 @@ import ( kcr "github.com/smartcontractkit/chainlink/v2/core/gethwrappers/keystone/generated/capabilities_registry" "github.com/stretchr/testify/assert" - "github.com/test-go/testify/require" - "go.uber.org/zap/zapcore" - "golang.org/x/exp/maps" + "github.com/stretchr/testify/require" ) -func TestDeploy(t *testing.T) { - lggr := logger.Test(t) - ctx := tests.Context(t) - - // sepolia; all nodes are on the this chain - sepoliaChainId := uint64(11155111) - sepoliaArbitrumChainId := uint64(421614) - - sepoliaChainSel, err := chainsel.SelectorFromChainId(sepoliaChainId) - require.NoError(t, err) - // sepoliaArbitrumChainSel, err := chainsel.SelectorFromChainId(sepoliaArbitrumChainId) - // require.NoError(t, err) - // aptos-testnet - aptosChainSel := chainsel.AptosChainIdToChainSelector()[2] - - crConfig := deployment.CapabilityRegistryConfig{ - EVMChainID: sepoliaChainId, - Contract: [20]byte{}, - } - - evmChains := memory.NewMemoryChainsWithChainIDs(t, []uint64{sepoliaChainId, sepoliaArbitrumChainId}) - aptosChain := memory.NewMemoryChain(t, aptosChainSel) - - wfChains := map[uint64]deployment.Chain{} - wfChains[sepoliaChainSel] = evmChains[sepoliaChainSel] - wfChains[aptosChainSel] = aptosChain - wfNodes := memory.NewNodes(t, zapcore.InfoLevel, wfChains, 4, 0, crConfig) - require.Len(t, wfNodes, 4) - - cwNodes := memory.NewNodes(t, zapcore.InfoLevel, evmChains, 4, 0, crConfig) - - assetChains := map[uint64]deployment.Chain{} - assetChains[sepoliaChainSel] = evmChains[sepoliaChainSel] - assetNodes := memory.NewNodes(t, zapcore.InfoLevel, assetChains, 4, 0, crConfig) - require.Len(t, assetNodes, 4) - - // TODO: partition nodes into multiple nops - - wfDon := keystone.DonCapabilities{ - Name: keystone.WFDonName, - Nops: []keystone.NOP{ - { - Name: "nop 1", - Nodes: maps.Keys(wfNodes), - }, - }, - Capabilities: []kcr.CapabilitiesRegistryCapability{keystone.OCR3Cap}, - } - cwDon := keystone.DonCapabilities{ - Name: keystone.TargetDonName, - Nops: []keystone.NOP{ - { - Name: "nop 2", - Nodes: maps.Keys(cwNodes), - }, - }, - Capabilities: []kcr.CapabilitiesRegistryCapability{keystone.WriteChainCap}, - } - assetDon := keystone.DonCapabilities{ - Name: keystone.StreamDonName, - Nops: []keystone.NOP{ - { - Name: "nop 3", - Nodes: maps.Keys(assetNodes), - }, - }, - Capabilities: []kcr.CapabilitiesRegistryCapability{keystone.StreamTriggerCap}, - } - - allChains := make(map[uint64]deployment.Chain) - maps.Copy(allChains, evmChains) - // allChains[aptosChainSel] = aptosChain - - allNodes := make(map[string]memory.Node) - maps.Copy(allNodes, wfNodes) - maps.Copy(allNodes, cwNodes) - maps.Copy(allNodes, assetNodes) - env := memory.NewMemoryEnvironmentFromChainsNodes(func() context.Context { return ctx }, lggr, allChains, allNodes) - - var ocr3Config = keystone.OracleConfigWithSecrets{ - OracleConfig: keystone.OracleConfig{ - MaxFaultyOracles: len(wfNodes) / 3, - }, - OCRSecrets: deployment.XXXGenerateTestOCRSecrets(), - } - - // explicitly deploy the contracts - cs, err := keystone.DeployContracts(&env, sepoliaChainSel) - require.NoError(t, err) - env.ExistingAddresses = cs.AddressBook - deployReq := keystone.ConfigureContractsRequest{ - RegistryChainSel: sepoliaChainSel, - Env: &env, - OCR3Config: &ocr3Config, - Dons: []keystone.DonCapabilities{wfDon, cwDon, assetDon}, - DoContractDeploy: false, - } - deployResp, err := keystone.ConfigureContracts(ctx, lggr, deployReq) - require.NoError(t, err) - ad := deployResp.Changeset.AddressBook - addrs, err := ad.Addresses() - require.NoError(t, err) - lggr.Infow("Deployed Keystone contracts", "address book", addrs) - - // all contracts on home chain - homeChainAddrs, err := ad.AddressesForChain(sepoliaChainSel) - require.NoError(t, err) - require.Len(t, homeChainAddrs, 3) - // only forwarder on non-home chain - for sel := range env.Chains { - chainAddrs, err := ad.AddressesForChain(sel) - require.NoError(t, err) - if sel != sepoliaChainSel { - require.Len(t, chainAddrs, 1) - } else { - require.Len(t, chainAddrs, 3) - } - containsForwarder := false - for _, tv := range chainAddrs { - if tv.Type == keystone.KeystoneForwarder { - containsForwarder = true - break - } - } - require.True(t, containsForwarder, "no forwarder found in %v on chain %d for target don", chainAddrs, sel) - } - req := &keystone.GetContractSetsRequest{ - Chains: env.Chains, - AddressBook: ad, - } - - contractSetsResp, err := keystone.GetContractSets(lggr, req) - require.NoError(t, err) - require.Len(t, contractSetsResp.ContractSets, len(env.Chains)) - // check the registry - regChainContracts, ok := contractSetsResp.ContractSets[sepoliaChainSel] - require.True(t, ok) - gotRegistry := regChainContracts.CapabilitiesRegistry - require.NotNil(t, gotRegistry) - // contract reads - gotDons, err := gotRegistry.GetDONs(&bind.CallOpts{}) - if err != nil { - err = keystone.DecodeErr(kcr.CapabilitiesRegistryABI, err) - require.Fail(t, fmt.Sprintf("failed to get Dons from registry at %s: %s", gotRegistry.Address().String(), err)) - } - require.NoError(t, err) - assert.Len(t, gotDons, len(deployReq.Dons)) - - for n, info := range deployResp.DonInfos { - found := false - for _, gdon := range gotDons { - if gdon.Id == info.Id { - found = true - assert.EqualValues(t, info, gdon) - break - } - } - require.True(t, found, "don %s not found in registry", n) - } - // check the forwarder - for _, cs := range contractSetsResp.ContractSets { - forwarder := cs.Forwarder - require.NotNil(t, forwarder) - // any read to ensure that the contract is deployed correctly - _, err := forwarder.Owner(&bind.CallOpts{}) - require.NoError(t, err) - // TODO expand this test; there is no get method on the forwarder so unclear how to test it - } - // check the ocr3 contract - for chainSel, cs := range contractSetsResp.ContractSets { - if chainSel != sepoliaChainSel { - require.Nil(t, cs.OCR3) - continue - } - require.NotNil(t, cs.OCR3) - // any read to ensure that the contract is deployed correctly - _, err := cs.OCR3.LatestConfigDetails(&bind.CallOpts{}) - require.NoError(t, err) - } -} - // TODO: Deprecated, remove everything below that leverages CLO func nodeOperatorsToIDs(t *testing.T, nops []*models.NodeOperator) (nodeIDs []keystone.NOP) { diff --git a/deployment/keystone/ocr3config.go b/deployment/keystone/ocr3config.go index a281a69ed8a..c4f8714b113 100644 --- a/deployment/keystone/ocr3config.go +++ b/deployment/keystone/ocr3config.go @@ -8,6 +8,8 @@ import ( "encoding/json" "errors" "fmt" + "math/big" + "strings" "time" "github.com/ethereum/go-ethereum/common" @@ -16,7 +18,11 @@ import ( "github.com/smartcontractkit/libocr/offchainreporting2plus/ocr3confighelper" "github.com/smartcontractkit/libocr/offchainreporting2plus/types" + "github.com/smartcontractkit/ccip-owner-contracts/pkg/gethwrappers" + "github.com/smartcontractkit/ccip-owner-contracts/pkg/proposal/mcms" + "github.com/smartcontractkit/ccip-owner-contracts/pkg/proposal/timelock" "github.com/smartcontractkit/chainlink/deployment" + "github.com/smartcontractkit/chainlink/deployment/common/proposalutils" kocr3 "github.com/smartcontractkit/chainlink/v2/core/gethwrappers/keystone/generated/ocr3_capability" "github.com/smartcontractkit/chainlink/v2/core/services/keystore/chaintype" "github.com/smartcontractkit/chainlink/v2/core/services/ocrcommon" @@ -107,6 +113,44 @@ func (c OCR2OracleConfig) MarshalJSON() ([]byte, error) { return json.Marshal(alias) } +func (c *OCR2OracleConfig) UnmarshalJSON(data []byte) error { + type aliasT struct { + Signers []string + Transmitters []string + F uint8 + OnchainConfig string + OffchainConfigVersion uint64 + OffchainConfig string + } + var alias aliasT + err := json.Unmarshal(data, &alias) + if err != nil { + return fmt.Errorf("failed to unmarshal OCR2OracleConfig alias: %w", err) + } + c.F = alias.F + c.OffchainConfigVersion = alias.OffchainConfigVersion + c.Signers = make([][]byte, len(alias.Signers)) + for i, signer := range alias.Signers { + c.Signers[i], err = hex.DecodeString(strings.TrimPrefix(signer, "0x")) + if err != nil { + return fmt.Errorf("failed to decode signer: %w", err) + } + } + c.Transmitters = make([]common.Address, len(alias.Transmitters)) + for i, transmitter := range alias.Transmitters { + c.Transmitters[i] = common.HexToAddress(transmitter) + } + c.OnchainConfig, err = hex.DecodeString(strings.TrimPrefix(alias.OnchainConfig, "0x")) + if err != nil { + return fmt.Errorf("failed to decode onchain config: %w", err) + } + c.OffchainConfig, err = hex.DecodeString(strings.TrimPrefix(alias.OffchainConfig, "0x")) + if err != nil { + return fmt.Errorf("failed to decode offchain config: %w", err) + } + return nil +} + func GenerateOCR3Config(cfg OracleConfigWithSecrets, nca []NodeKeys) (OCR2OracleConfig, error) { onchainPubKeys := [][]byte{} allPubKeys := map[string]any{} @@ -245,6 +289,9 @@ type configureOCR3Request struct { contract *kocr3.OCR3Capability nodes []deployment.Node dryRun bool + + useMCMS bool + contractSet *ContractSet } func (r configureOCR3Request) generateOCR3Config() (OCR2OracleConfig, error) { @@ -254,6 +301,7 @@ func (r configureOCR3Request) generateOCR3Config() (OCR2OracleConfig, error) { type configureOCR3Response struct { ocrConfig OCR2OracleConfig + proposal *timelock.MCMSWithTimelockProposal } func configureOCR3contract(req configureOCR3Request) (*configureOCR3Response, error) { @@ -265,9 +313,15 @@ func configureOCR3contract(req configureOCR3Request) (*configureOCR3Response, er return nil, fmt.Errorf("failed to generate OCR3 config: %w", err) } if req.dryRun { - return &configureOCR3Response{ocrConfig}, nil + return &configureOCR3Response{ocrConfig, nil}, nil } - tx, err := req.contract.SetConfig(req.chain.DeployerKey, + + txOpt := req.chain.DeployerKey + if req.useMCMS { + txOpt = deployment.SimTransactOpts() + } + + tx, err := req.contract.SetConfig(txOpt, ocrConfig.Signers, ocrConfig.Transmitters, ocrConfig.F, @@ -277,12 +331,45 @@ func configureOCR3contract(req configureOCR3Request) (*configureOCR3Response, er ) if err != nil { err = DecodeErr(kocr3.OCR3CapabilityABI, err) - return nil, fmt.Errorf("failed to call SetConfig for OCR3 contract %s: %w", req.contract.Address().String(), err) + return nil, fmt.Errorf("failed to call SetConfig for OCR3 contract %s using mcms: %T: %w", req.contract.Address().String(), req.useMCMS, err) } - _, err = req.chain.Confirm(tx) - if err != nil { - err = DecodeErr(kocr3.OCR3CapabilityABI, err) - return nil, fmt.Errorf("failed to confirm SetConfig for OCR3 contract %s: %w", req.contract.Address().String(), err) + + var proposal *timelock.MCMSWithTimelockProposal + if !req.useMCMS { + _, err = req.chain.Confirm(tx) + if err != nil { + err = DecodeErr(kocr3.OCR3CapabilityABI, err) + return nil, fmt.Errorf("failed to confirm SetConfig for OCR3 contract %s: %w", req.contract.Address().String(), err) + } + } else { + ops := timelock.BatchChainOperation{ + ChainIdentifier: mcms.ChainIdentifier(req.chain.Selector), + Batch: []mcms.Operation{ + { + To: req.contract.Address(), + Data: tx.Data(), + Value: big.NewInt(0), + }, + }, + } + timelocksPerChain := map[uint64]common.Address{ + req.chain.Selector: req.contractSet.Timelock.Address(), + } + proposerMCMSes := map[uint64]*gethwrappers.ManyChainMultiSig{ + req.chain.Selector: req.contractSet.ProposerMcm, + } + + proposal, err = proposalutils.BuildProposalFromBatches( + timelocksPerChain, + proposerMCMSes, + []timelock.BatchChainOperation{ops}, + "proposal to set ocr3 config", + 0, + ) + if err != nil { + return nil, fmt.Errorf("failed to build proposal: %w", err) + } } - return &configureOCR3Response{ocrConfig}, nil + + return &configureOCR3Response{ocrConfig, proposal}, nil } diff --git a/deployment/keystone/state.go b/deployment/keystone/state.go index 68f2ab97a5d..1e6ffdd895f 100644 --- a/deployment/keystone/state.go +++ b/deployment/keystone/state.go @@ -8,6 +8,7 @@ import ( "github.com/smartcontractkit/chainlink-common/pkg/logger" "github.com/smartcontractkit/chainlink/deployment" + commonchangeset "github.com/smartcontractkit/chainlink/deployment/common/changeset" common_v1_0 "github.com/smartcontractkit/chainlink/deployment/common/view/v1_0" "github.com/smartcontractkit/chainlink/deployment/keystone/view" "github.com/smartcontractkit/chainlink/v2/core/gethwrappers/keystone/generated/capabilities_registry" @@ -25,11 +26,26 @@ type GetContractSetsResponse struct { } type ContractSet struct { + commonchangeset.MCMSWithTimelockState OCR3 *ocr3_capability.OCR3Capability Forwarder *forwarder.KeystoneForwarder CapabilitiesRegistry *capabilities_registry.CapabilitiesRegistry } +func (cs ContractSet) TransferableContracts() []common.Address { + var out []common.Address + if cs.OCR3 != nil { + out = append(out, cs.OCR3.Address()) + } + if cs.Forwarder != nil { + out = append(out, cs.Forwarder.Address()) + } + if cs.CapabilitiesRegistry != nil { + out = append(out, cs.CapabilitiesRegistry.Address()) + } + return out +} + func (cs ContractSet) View() (view.KeystoneChainView, error) { out := view.NewKeystoneChainView() if cs.CapabilitiesRegistry != nil { @@ -62,6 +78,11 @@ func GetContractSets(lggr logger.Logger, req *GetContractSetsRequest) (*GetContr func loadContractSet(lggr logger.Logger, chain deployment.Chain, addresses map[string]deployment.TypeAndVersion) (*ContractSet, error) { var out ContractSet + mcmsWithTimelock, err := commonchangeset.LoadMCMSWithTimelockState(chain, addresses) + if err != nil { + return nil, fmt.Errorf("failed to load mcms contract: %w", err) + } + out.MCMSWithTimelockState = *mcmsWithTimelock for addr, tv := range addresses { // todo handle versions diff --git a/deployment/keystone/types.go b/deployment/keystone/types.go index 4b00e5e4dc0..52620a52a0e 100644 --- a/deployment/keystone/types.go +++ b/deployment/keystone/types.go @@ -127,6 +127,7 @@ func (v NOP) Validate() error { // in is in a convenient form to handle the CLO representation of the nop data type DonCapabilities struct { Name string + F uint8 Nops []NOP Capabilities []kcr.CapabilitiesRegistryCapability // every capability is hosted on each nop } diff --git a/go.mod b/go.mod index 75557e56d2b..ad71fa75be3 100644 --- a/go.mod +++ b/go.mod @@ -2,6 +2,8 @@ module github.com/smartcontractkit/chainlink/v2 go 1.23.3 +toolchain go1.23.4 + require ( github.com/Depado/ginprom v1.8.0 github.com/Masterminds/semver/v3 v3.3.0 @@ -77,7 +79,7 @@ require ( github.com/smartcontractkit/chain-selectors v1.0.31 github.com/smartcontractkit/chainlink-automation v0.8.1 github.com/smartcontractkit/chainlink-ccip v0.0.0-20241204015713-8956bb614e9e - github.com/smartcontractkit/chainlink-common v0.3.1-0.20241127162636-07aa781ee1f4 + github.com/smartcontractkit/chainlink-common v0.3.1-0.20241206011233-b6684ee6508f github.com/smartcontractkit/chainlink-cosmos v0.5.2-0.20241202195413-82468150ac1e github.com/smartcontractkit/chainlink-data-streams v0.1.1-0.20241202141438-a90db35252db github.com/smartcontractkit/chainlink-feeds v0.1.1 @@ -104,6 +106,7 @@ require ( go.opentelemetry.io/contrib/instrumentation/github.com/gin-gonic/gin/otelgin v0.49.0 go.opentelemetry.io/otel v1.31.0 go.opentelemetry.io/otel/metric v1.31.0 + go.opentelemetry.io/otel/sdk/metric v1.31.0 go.opentelemetry.io/otel/trace v1.31.0 go.uber.org/multierr v1.11.0 go.uber.org/zap v1.27.0 @@ -362,7 +365,6 @@ require ( go.opentelemetry.io/otel/log v0.6.0 // indirect go.opentelemetry.io/otel/sdk v1.31.0 // indirect go.opentelemetry.io/otel/sdk/log v0.6.0 // indirect - go.opentelemetry.io/otel/sdk/metric v1.31.0 // indirect go.opentelemetry.io/proto/otlp v1.3.1 // indirect go.uber.org/ratelimit v0.3.1 // indirect golang.org/x/arch v0.11.0 // indirect diff --git a/go.sum b/go.sum index fb8eab948e4..42de8ab4a72 100644 --- a/go.sum +++ b/go.sum @@ -1125,8 +1125,8 @@ github.com/smartcontractkit/chainlink-automation v0.8.1 h1:sTc9LKpBvcKPc1JDYAmgB github.com/smartcontractkit/chainlink-automation v0.8.1/go.mod h1:Iij36PvWZ6blrdC5A/nrQUBuf3MH3JvsBB9sSyc9W08= github.com/smartcontractkit/chainlink-ccip v0.0.0-20241204015713-8956bb614e9e h1:GnM6ZWV6vlk2+n6c6o+v/R1LtXzBGVVx7r37nt/h6Uc= github.com/smartcontractkit/chainlink-ccip v0.0.0-20241204015713-8956bb614e9e/go.mod h1:80vGBbOfertJig0xFKsRfm+i17FkjdKkk1dAaGE45Os= -github.com/smartcontractkit/chainlink-common v0.3.1-0.20241127162636-07aa781ee1f4 h1:atCZ1jol7a+tdtgU/wNqXgliBun5H7BjGBicGL8Tj6o= -github.com/smartcontractkit/chainlink-common v0.3.1-0.20241127162636-07aa781ee1f4/go.mod h1:bQktEJf7sJ0U3SmIcXvbGUox7SmXcnSEZ4kUbT8R5Nk= +github.com/smartcontractkit/chainlink-common v0.3.1-0.20241206011233-b6684ee6508f h1:hH+cAG2zt+WK4I2m572LXAnAJg3wtGEAwzBKR8FiXo8= +github.com/smartcontractkit/chainlink-common v0.3.1-0.20241206011233-b6684ee6508f/go.mod h1:bQktEJf7sJ0U3SmIcXvbGUox7SmXcnSEZ4kUbT8R5Nk= github.com/smartcontractkit/chainlink-cosmos v0.5.2-0.20241202195413-82468150ac1e h1:PRoeby6ZlTuTkv2f+7tVU4+zboTfRzI+beECynF4JQ0= github.com/smartcontractkit/chainlink-cosmos v0.5.2-0.20241202195413-82468150ac1e/go.mod h1:mUh5/woemsVaHgTorA080hrYmO3syBCmPdnWc/5dOqk= github.com/smartcontractkit/chainlink-data-streams v0.1.1-0.20241202141438-a90db35252db h1:N1RH1hSr2ACzOFc9hkCcjE8pRBTdcU3p8nsTJByaLes= diff --git a/integration-tests/go.mod b/integration-tests/go.mod index 621c1c0941b..b9c6f213277 100644 --- a/integration-tests/go.mod +++ b/integration-tests/go.mod @@ -2,6 +2,8 @@ module github.com/smartcontractkit/chainlink/integration-tests go 1.23.3 +toolchain go1.23.4 + // Make sure we're working with the latest chainlink libs replace github.com/smartcontractkit/chainlink/v2 => ../ @@ -39,7 +41,7 @@ require ( github.com/smartcontractkit/chain-selectors v1.0.31 github.com/smartcontractkit/chainlink-automation v0.8.1 github.com/smartcontractkit/chainlink-ccip v0.0.0-20241204015713-8956bb614e9e - github.com/smartcontractkit/chainlink-common v0.3.1-0.20241127162636-07aa781ee1f4 + github.com/smartcontractkit/chainlink-common v0.3.1-0.20241206011233-b6684ee6508f github.com/smartcontractkit/chainlink-protos/job-distributor v0.6.0 github.com/smartcontractkit/chainlink-testing-framework/havoc v1.50.2 github.com/smartcontractkit/chainlink-testing-framework/lib v1.50.18 diff --git a/integration-tests/go.sum b/integration-tests/go.sum index 272cc05e0a5..6914a06501f 100644 --- a/integration-tests/go.sum +++ b/integration-tests/go.sum @@ -1432,8 +1432,8 @@ github.com/smartcontractkit/chainlink-automation v0.8.1 h1:sTc9LKpBvcKPc1JDYAmgB github.com/smartcontractkit/chainlink-automation v0.8.1/go.mod h1:Iij36PvWZ6blrdC5A/nrQUBuf3MH3JvsBB9sSyc9W08= github.com/smartcontractkit/chainlink-ccip v0.0.0-20241204015713-8956bb614e9e h1:GnM6ZWV6vlk2+n6c6o+v/R1LtXzBGVVx7r37nt/h6Uc= github.com/smartcontractkit/chainlink-ccip v0.0.0-20241204015713-8956bb614e9e/go.mod h1:80vGBbOfertJig0xFKsRfm+i17FkjdKkk1dAaGE45Os= -github.com/smartcontractkit/chainlink-common v0.3.1-0.20241127162636-07aa781ee1f4 h1:atCZ1jol7a+tdtgU/wNqXgliBun5H7BjGBicGL8Tj6o= -github.com/smartcontractkit/chainlink-common v0.3.1-0.20241127162636-07aa781ee1f4/go.mod h1:bQktEJf7sJ0U3SmIcXvbGUox7SmXcnSEZ4kUbT8R5Nk= +github.com/smartcontractkit/chainlink-common v0.3.1-0.20241206011233-b6684ee6508f h1:hH+cAG2zt+WK4I2m572LXAnAJg3wtGEAwzBKR8FiXo8= +github.com/smartcontractkit/chainlink-common v0.3.1-0.20241206011233-b6684ee6508f/go.mod h1:bQktEJf7sJ0U3SmIcXvbGUox7SmXcnSEZ4kUbT8R5Nk= github.com/smartcontractkit/chainlink-cosmos v0.5.2-0.20241202195413-82468150ac1e h1:PRoeby6ZlTuTkv2f+7tVU4+zboTfRzI+beECynF4JQ0= github.com/smartcontractkit/chainlink-cosmos v0.5.2-0.20241202195413-82468150ac1e/go.mod h1:mUh5/woemsVaHgTorA080hrYmO3syBCmPdnWc/5dOqk= github.com/smartcontractkit/chainlink-data-streams v0.1.1-0.20241202141438-a90db35252db h1:N1RH1hSr2ACzOFc9hkCcjE8pRBTdcU3p8nsTJByaLes= diff --git a/integration-tests/load/go.mod b/integration-tests/load/go.mod index 30eb132d24f..34717eea023 100644 --- a/integration-tests/load/go.mod +++ b/integration-tests/load/go.mod @@ -2,6 +2,8 @@ module github.com/smartcontractkit/chainlink/load-tests go 1.23.3 +toolchain go1.23.4 + // Make sure we're working with the latest chainlink libs replace github.com/smartcontractkit/chainlink/v2 => ../../ @@ -17,7 +19,7 @@ require ( github.com/pkg/errors v0.9.1 github.com/rs/zerolog v1.33.0 github.com/slack-go/slack v0.15.0 - github.com/smartcontractkit/chainlink-common v0.3.1-0.20241127162636-07aa781ee1f4 + github.com/smartcontractkit/chainlink-common v0.3.1-0.20241206011233-b6684ee6508f github.com/smartcontractkit/chainlink-testing-framework/lib v1.50.18 github.com/smartcontractkit/chainlink-testing-framework/seth v1.50.9 github.com/smartcontractkit/chainlink-testing-framework/wasp v1.50.2 diff --git a/integration-tests/load/go.sum b/integration-tests/load/go.sum index 5fbafb202bc..bcb3680e9c5 100644 --- a/integration-tests/load/go.sum +++ b/integration-tests/load/go.sum @@ -1423,8 +1423,8 @@ github.com/smartcontractkit/chainlink-automation v0.8.1 h1:sTc9LKpBvcKPc1JDYAmgB github.com/smartcontractkit/chainlink-automation v0.8.1/go.mod h1:Iij36PvWZ6blrdC5A/nrQUBuf3MH3JvsBB9sSyc9W08= github.com/smartcontractkit/chainlink-ccip v0.0.0-20241204015713-8956bb614e9e h1:GnM6ZWV6vlk2+n6c6o+v/R1LtXzBGVVx7r37nt/h6Uc= github.com/smartcontractkit/chainlink-ccip v0.0.0-20241204015713-8956bb614e9e/go.mod h1:80vGBbOfertJig0xFKsRfm+i17FkjdKkk1dAaGE45Os= -github.com/smartcontractkit/chainlink-common v0.3.1-0.20241127162636-07aa781ee1f4 h1:atCZ1jol7a+tdtgU/wNqXgliBun5H7BjGBicGL8Tj6o= -github.com/smartcontractkit/chainlink-common v0.3.1-0.20241127162636-07aa781ee1f4/go.mod h1:bQktEJf7sJ0U3SmIcXvbGUox7SmXcnSEZ4kUbT8R5Nk= +github.com/smartcontractkit/chainlink-common v0.3.1-0.20241206011233-b6684ee6508f h1:hH+cAG2zt+WK4I2m572LXAnAJg3wtGEAwzBKR8FiXo8= +github.com/smartcontractkit/chainlink-common v0.3.1-0.20241206011233-b6684ee6508f/go.mod h1:bQktEJf7sJ0U3SmIcXvbGUox7SmXcnSEZ4kUbT8R5Nk= github.com/smartcontractkit/chainlink-cosmos v0.5.2-0.20241202195413-82468150ac1e h1:PRoeby6ZlTuTkv2f+7tVU4+zboTfRzI+beECynF4JQ0= github.com/smartcontractkit/chainlink-cosmos v0.5.2-0.20241202195413-82468150ac1e/go.mod h1:mUh5/woemsVaHgTorA080hrYmO3syBCmPdnWc/5dOqk= github.com/smartcontractkit/chainlink-data-streams v0.1.1-0.20241202141438-a90db35252db h1:N1RH1hSr2ACzOFc9hkCcjE8pRBTdcU3p8nsTJByaLes= diff --git a/integration-tests/smoke/ccip/ccip_batching_test.go b/integration-tests/smoke/ccip/ccip_batching_test.go index c801e899585..8c3615fbb20 100644 --- a/integration-tests/smoke/ccip/ccip_batching_test.go +++ b/integration-tests/smoke/ccip/ccip_batching_test.go @@ -9,20 +9,15 @@ import ( "github.com/ethereum/go-ethereum/accounts/abi/bind" "github.com/ethereum/go-ethereum/common" - gethtypes "github.com/ethereum/go-ethereum/core/types" - "github.com/ethereum/go-ethereum/crypto" "github.com/stretchr/testify/require" "golang.org/x/exp/maps" - chainsel "github.com/smartcontractkit/chain-selectors" - "github.com/smartcontractkit/chainlink-ccip/pkg/types/ccipocr3" "github.com/smartcontractkit/chainlink-common/pkg/merklemulti" + "github.com/smartcontractkit/chainlink/deployment/environment/memory" "github.com/smartcontractkit/chainlink/deployment" "github.com/smartcontractkit/chainlink/deployment/ccip/changeset" - testsetups "github.com/smartcontractkit/chainlink/integration-tests/testsetups/ccip" - "github.com/smartcontractkit/chainlink/v2/core/chains/evm/assets" "github.com/smartcontractkit/chainlink/v2/core/gethwrappers/ccip/generated/offramp" "github.com/smartcontractkit/chainlink/v2/core/gethwrappers/ccip/generated/onramp" "github.com/smartcontractkit/chainlink/v2/core/gethwrappers/ccip/generated/router" @@ -30,16 +25,33 @@ import ( "github.com/smartcontractkit/chainlink/v2/core/logger" ) -func Test_CCIPBatching(t *testing.T) { +const ( + numMessages = 40 +) + +type batchTestSetup struct { + e changeset.DeployedEnv + state changeset.CCIPOnChainState + sourceChain1 uint64 + sourceChain2 uint64 + destChain uint64 +} + +func newBatchTestSetup(t *testing.T) batchTestSetup { // Setup 3 chains, with 2 lanes going to the dest. - lggr := logger.TestLogger(t) - ctx := changeset.Context(t) - // Will load 3 chains when specified by the overrides.toml or env vars (E2E_TEST_SELECTED_NETWORK). - // See e2e-tests.yml. - e, _, _ := testsetups.NewLocalDevEnvironmentWithDefaultPrice(t, lggr, &changeset.TestConfigs{ - IsUSDC: false, - IsMultiCall3: true, // needed for this test - }) + e := changeset.NewMemoryEnvironmentWithJobsAndContracts( + t, + logger.TestLogger(t), + memory.MemoryEnvironmentConfig{ + Chains: 3, + Nodes: 4, + Bootstraps: 1, + NumOfUsersPerChain: 2, + }, + &changeset.TestConfigs{ + IsMultiCall3: true, + }, + ) state, err := changeset.LoadOnchainState(e.Env) require.NoError(t, err) @@ -61,251 +73,260 @@ func Test_CCIPBatching(t *testing.T) { require.NoError(t, changeset.AddLaneWithDefaultPricesAndFeeQuoterConfig(e.Env, state, sourceChain1, destChain, false)) require.NoError(t, changeset.AddLaneWithDefaultPricesAndFeeQuoterConfig(e.Env, state, sourceChain2, destChain, false)) - const ( - numMessages = 5 - ) + return batchTestSetup{e, state, sourceChain1, sourceChain2, destChain} +} + +func Test_CCIPBatching_MaxBatchSizeEVM(t *testing.T) { + t.Parallel() + + ctx := changeset.Context(t) + setup := newBatchTestSetup(t) + sourceChain1, sourceChain2, destChain, e, state := setup.sourceChain1, setup.sourceChain2, setup.destChain, setup.e, setup.state + var ( startSeqNum = map[uint64]ccipocr3.SeqNum{ sourceChain1: 1, sourceChain2: 1, } - ) - - t.Run("batch data only messages from single source", func(t *testing.T) { - var ( - sourceChain = sourceChain1 - ) - err := sendMessages( - ctx, - t, - e.Env.Chains[sourceChain], + sourceChain = sourceChain1 + transactors = []*bind.TransactOpts{ e.Env.Chains[sourceChain].DeployerKey, - state.Chains[sourceChain].OnRamp, - state.Chains[sourceChain].Router, - state.Chains[sourceChain].Multicall3, - destChain, - numMessages, - common.LeftPadBytes(state.Chains[destChain].Receiver.Address().Bytes(), 32), - ) - require.NoError(t, err) - - _, err = changeset.ConfirmCommitWithExpectedSeqNumRange( - t, - e.Env.Chains[sourceChain], - e.Env.Chains[destChain], - state.Chains[destChain].OffRamp, - nil, - ccipocr3.NewSeqNumRange(startSeqNum[sourceChain], startSeqNum[sourceChain]+numMessages-1), - true, - ) - require.NoErrorf(t, err, "failed to confirm commit from chain %d", sourceChain) - - states, err := changeset.ConfirmExecWithSeqNrs( - t, - e.Env.Chains[sourceChain], - e.Env.Chains[destChain], - state.Chains[destChain].OffRamp, - nil, - genSeqNrRange(startSeqNum[sourceChain], startSeqNum[sourceChain]+numMessages-1), - ) - require.NoError(t, err) - // assert that all states are successful - for _, state := range states { - require.Equal(t, changeset.EXECUTION_STATE_SUCCESS, state) + e.Env.Chains[sourceChain].Users[0], } + errs = make(chan error, len(transactors)) + ) - startSeqNum[sourceChain] = startSeqNum[sourceChain] + numMessages - }) - - t.Run("batch data only messages from multiple sources", func(t *testing.T) { - t.Skipf("skipping - failing consistently in CI") - var ( - wg sync.WaitGroup - sourceChains = []uint64{sourceChain1, sourceChain2} - errs = make(chan error, len(sourceChains)) - ) - - for _, srcChain := range sourceChains { - wg.Add(1) - go sendMessagesAsync( + for _, transactor := range transactors { + go func() { + err := sendMessages( ctx, t, - e, - state, - srcChain, + e.Env.Chains[sourceChain], + transactor, + state.Chains[sourceChain].OnRamp, + state.Chains[sourceChain].Router, + state.Chains[sourceChain].Multicall3, destChain, - numMessages, - &wg, - errs, + merklemulti.MaxNumberTreeLeaves/2, + common.LeftPadBytes(state.Chains[destChain].Receiver.Address().Bytes(), 32), ) - } - - wg.Wait() - - var i int - for i < len(sourceChains) { - select { - case err := <-errs: - require.NoError(t, err) - i++ - case <-ctx.Done(): - require.FailNow(t, "didn't get all errors before test context was done") - } - } + t.Log("sendMessages error:", err, ", writing to channel") + errs <- err + t.Log("sent error to channel") + }() + } - // confirm the commit reports - outputErrs := make(chan outputErr[*offramp.OffRampCommitReportAccepted], len(sourceChains)) - for _, srcChain := range sourceChains { - wg.Add(1) - go assertCommitReportsAsync( - t, - e, - state, - srcChain, - destChain, - startSeqNum[srcChain], - startSeqNum[srcChain]+ccipocr3.SeqNum(numMessages)-1, - &wg, - outputErrs, - ) + var i = 0 + for i < len(transactors) { + select { + case err := <-errs: + require.NoError(t, err) + i++ + case <-ctx.Done(): + require.FailNow(t, "didn't get all errors before test context was done") } + } - t.Log("waiting for commit report") - wg.Wait() - - i = 0 - var reports []*offramp.OffRampCommitReportAccepted - for i < len(sourceChains) { - select { - case outputErr := <-outputErrs: - require.NoError(t, outputErr.err) - reports = append(reports, outputErr.output) - i++ - case <-ctx.Done(): - require.FailNow(t, "didn't get all commit reports before test context was done") - } - } + _, err := changeset.ConfirmCommitWithExpectedSeqNumRange( + t, + e.Env.Chains[sourceChain], + e.Env.Chains[destChain], + state.Chains[destChain].OffRamp, + nil, // startBlock + ccipocr3.NewSeqNumRange( + startSeqNum[sourceChain], + startSeqNum[sourceChain]+ccipocr3.SeqNum(merklemulti.MaxNumberTreeLeaves)-1, + ), + true, + ) + require.NoErrorf(t, err, "failed to confirm commit from chain %d", sourceChain) +} - // the reports should be the same for both, since both roots should be batched within - // that one report. - require.Lenf(t, reports, len(sourceChains), "expected %d commit reports", len(sourceChains)) - require.NotNil(t, reports[0], "commit report should not be nil") - require.NotNil(t, reports[1], "commit report should not be nil") - // TODO: this assertion is failing, despite messages being sent at the same time. - // require.Equal(t, reports[0], reports[1], "commit reports should be the same") - - // confirm execution - execErrs := make(chan outputErr[map[uint64]int], len(sourceChains)) - for _, srcChain := range sourceChains { - wg.Add(1) - go assertExecAsync( - t, - e, - state, - srcChain, - destChain, - genSeqNrRange(startSeqNum[srcChain], startSeqNum[srcChain]+ccipocr3.SeqNum(numMessages)-1), - &wg, - execErrs, - ) - } +func Test_CCIPBatching_MultiSource(t *testing.T) { + // t.Skip("Exec not working, boosting not working correctly") - t.Log("waiting for exec reports") - wg.Wait() - - i = 0 - var execStates []map[uint64]int - for i < len(sourceChains) { - select { - case outputErr := <-execErrs: - require.NoError(t, outputErr.err) - execStates = append(execStates, outputErr.output) - i++ - case <-ctx.Done(): - require.FailNow(t, "didn't get all exec reports before test context was done") - } - } + t.Parallel() - // assert that all states are successful - for _, states := range execStates { - for _, state := range states { - require.Equal(t, changeset.EXECUTION_STATE_SUCCESS, state) - } - } + // Setup 3 chains, with 2 lanes going to the dest. + ctx := changeset.Context(t) + setup := newBatchTestSetup(t) + sourceChain1, sourceChain2, destChain, e, state := setup.sourceChain1, setup.sourceChain2, setup.destChain, setup.e, setup.state - // update the start and end seq nums - for _, srcChain := range sourceChains { - startSeqNum[srcChain] = startSeqNum[srcChain] + numMessages + var ( + wg sync.WaitGroup + sourceChains = []uint64{sourceChain1, sourceChain2} + errs = make(chan error, len(sourceChains)) + startSeqNum = map[uint64]ccipocr3.SeqNum{ + sourceChain1: 1, + sourceChain2: 1, } - }) - - t.Run("max evm batch size", func(t *testing.T) { - t.Skipf("This test is flaky, skipping until the issue related to fee calculation is resolved") - var ( - sourceChain = sourceChain1 - otherSender = mustNewTransactor(t, e.Env.Chains[sourceChain]) - transactors = []*bind.TransactOpts{ - e.Env.Chains[sourceChain].DeployerKey, - otherSender, - } - errs = make(chan error, len(transactors)) - ) + ) - // transfer some eth to the other sender from the DeployerKey - sendEth( + for _, srcChain := range sourceChains { + wg.Add(1) + go sendMessagesAsync( ctx, t, - e.Env.Chains[sourceChain], - e.Env.Chains[sourceChain].DeployerKey, - otherSender.From, - assets.Ether(20).ToInt(), + e, + state, + srcChain, + destChain, + numMessages, + &wg, + errs, ) + } - for _, transactor := range transactors { - go func() { - err := sendMessages( - ctx, - t, - e.Env.Chains[sourceChain], - transactor, - state.Chains[sourceChain].OnRamp, - state.Chains[sourceChain].Router, - state.Chains[sourceChain].Multicall3, - destChain, - merklemulti.MaxNumberTreeLeaves/2, - common.LeftPadBytes(state.Chains[destChain].Receiver.Address().Bytes(), 32), - ) - t.Log("sendMessages error:", err, ", writing to channel") - errs <- err - t.Log("sent error to channel") - }() + wg.Wait() + + var i int + for i < len(sourceChains) { + select { + case err := <-errs: + require.NoError(t, err) + i++ + case <-ctx.Done(): + require.FailNow(t, "didn't get all errors before test context was done") } + } - var i = 0 - for i < len(transactors) { - select { - case err := <-errs: - require.NoError(t, err) - i++ - case <-ctx.Done(): - require.FailNow(t, "didn't get all errors before test context was done") - } + // confirm the commit reports + outputErrs := make(chan outputErr[*offramp.OffRampCommitReportAccepted], len(sourceChains)) + for _, srcChain := range sourceChains { + wg.Add(1) + go assertCommitReportsAsync( + t, + e, + state, + srcChain, + destChain, + startSeqNum[srcChain], + startSeqNum[srcChain]+ccipocr3.SeqNum(numMessages)-1, + &wg, + outputErrs, + ) + } + + t.Log("waiting for commit report") + wg.Wait() + + i = 0 + var reports []*offramp.OffRampCommitReportAccepted + for i < len(sourceChains) { + select { + case outputErr := <-outputErrs: + require.NoError(t, outputErr.err) + reports = append(reports, outputErr.output) + i++ + case <-ctx.Done(): + require.FailNow(t, "didn't get all commit reports before test context was done") } + } - _, err = changeset.ConfirmCommitWithExpectedSeqNumRange( + // the reports should be the same for both, since both roots should be batched within + // that one report. + require.Lenf(t, reports, len(sourceChains), "expected %d commit reports", len(sourceChains)) + require.NotNil(t, reports[0], "commit report should not be nil") + require.NotNil(t, reports[1], "commit report should not be nil") + // TODO: this assertion is failing, despite messages being sent at the same time. + // require.Equal(t, reports[0], reports[1], "commit reports should be the same") + + // confirm execution + execErrs := make(chan outputErr[map[uint64]int], len(sourceChains)) + for _, srcChain := range sourceChains { + wg.Add(1) + go assertExecAsync( t, - e.Env.Chains[sourceChain], - e.Env.Chains[destChain], - state.Chains[destChain].OffRamp, - nil, // startBlock - ccipocr3.NewSeqNumRange( - startSeqNum[sourceChain], - startSeqNum[sourceChain]+ccipocr3.SeqNum(merklemulti.MaxNumberTreeLeaves)-1, - ), - true, + e, + state, + srcChain, + destChain, + genSeqNrRange(startSeqNum[srcChain], startSeqNum[srcChain]+ccipocr3.SeqNum(numMessages)-1), + &wg, + execErrs, ) - require.NoErrorf(t, err, "failed to confirm commit from chain %d", sourceChain) - }) + } + + t.Log("waiting for exec reports") + wg.Wait() + + i = 0 + var execStates []map[uint64]int + for i < len(sourceChains) { + select { + case outputErr := <-execErrs: + require.NoError(t, outputErr.err) + execStates = append(execStates, outputErr.output) + i++ + case <-ctx.Done(): + require.FailNow(t, "didn't get all exec reports before test context was done") + } + } + + // assert that all states are successful + for _, states := range execStates { + for _, state := range states { + require.Equal(t, changeset.EXECUTION_STATE_SUCCESS, state) + } + } +} + +func Test_CCIPBatching_SingleSource(t *testing.T) { + t.Parallel() + + // Setup 3 chains, with 2 lanes going to the dest. + ctx := changeset.Context(t) + setup := newBatchTestSetup(t) + sourceChain1, sourceChain2, destChain, e, state := setup.sourceChain1, setup.sourceChain2, setup.destChain, setup.e, setup.state + + var ( + startSeqNum = map[uint64]ccipocr3.SeqNum{ + sourceChain1: 1, + sourceChain2: 1, + } + ) + + var ( + sourceChain = sourceChain1 + ) + err := sendMessages( + ctx, + t, + e.Env.Chains[sourceChain], + e.Env.Chains[sourceChain].DeployerKey, + state.Chains[sourceChain].OnRamp, + state.Chains[sourceChain].Router, + state.Chains[sourceChain].Multicall3, + destChain, + numMessages, + common.LeftPadBytes(state.Chains[destChain].Receiver.Address().Bytes(), 32), + ) + require.NoError(t, err) + + _, err = changeset.ConfirmCommitWithExpectedSeqNumRange( + t, + e.Env.Chains[sourceChain], + e.Env.Chains[destChain], + state.Chains[destChain].OffRamp, + nil, + ccipocr3.NewSeqNumRange(startSeqNum[sourceChain], startSeqNum[sourceChain]+numMessages-1), + true, + ) + require.NoErrorf(t, err, "failed to confirm commit from chain %d", sourceChain) + + states, err := changeset.ConfirmExecWithSeqNrs( + t, + e.Env.Chains[sourceChain], + e.Env.Chains[destChain], + state.Chains[destChain].OffRamp, + nil, + genSeqNrRange(startSeqNum[sourceChain], startSeqNum[sourceChain]+numMessages-1), + ) + require.NoError(t, err) + // assert that all states are successful + for _, state := range states { + require.Equal(t, changeset.EXECUTION_STATE_SUCCESS, state) + } } type outputErr[T any] struct { @@ -373,18 +394,34 @@ func sendMessagesAsync( out chan<- error, ) { defer wg.Done() - err := sendMessages( - ctx, - t, - e.Env.Chains[sourceChainSelector], - e.Env.Chains[sourceChainSelector].DeployerKey, - state.Chains[sourceChainSelector].OnRamp, - state.Chains[sourceChainSelector].Router, - state.Chains[sourceChainSelector].Multicall3, - destChainSelector, - numMessages, - common.LeftPadBytes(state.Chains[destChainSelector].Receiver.Address().Bytes(), 32), + var err error + + const ( + numRetries = 3 ) + + // we retry a bunch of times just in case there is a race b/w the prices being + // posted and the messages being sent. + for i := 0; i < numRetries; i++ { + err = sendMessages( + ctx, + t, + e.Env.Chains[sourceChainSelector], + e.Env.Chains[sourceChainSelector].DeployerKey, + state.Chains[sourceChainSelector].OnRamp, + state.Chains[sourceChainSelector].Router, + state.Chains[sourceChainSelector].Multicall3, + destChainSelector, + numMessages, + common.LeftPadBytes(state.Chains[destChainSelector].Receiver.Address().Bytes(), 32), + ) + if err == nil { + break + } + + t.Log("sendMessagesAsync error is non-nil:", err, ", retrying") + } + t.Log("sendMessagesAsync error:", err, ", writing to channel") out <- err } @@ -412,8 +449,13 @@ func sendMessages( return fmt.Errorf("generate messages: %w", err) } + currBalance, err := sourceChain.Client.BalanceAt(ctx, sourceTransactOpts.From, nil) + if err != nil { + return fmt.Errorf("get balance: %w", err) + } + // Send the tx with the messages through the multicall - t.Logf("Sending %d messages with total value %s", numMessages, totalValue.String()) + t.Logf("Sending %d messages with total value %s, current balance: %s", numMessages, totalValue.String(), currBalance.String()) tx, err := sourceMulticall3.Aggregate3Value( &bind.TransactOpts{ From: sourceTransactOpts.From, @@ -434,7 +476,9 @@ func sendMessages( if err != nil { return fmt.Errorf("get message sent event: %w", err) } - defer iter.Close() + defer func() { + require.NoError(t, iter.Close()) + }() // there should be numMessages messages emitted for i := 0; i < numMessages; i++ { @@ -461,7 +505,7 @@ func genMessages( Data: []byte(fmt.Sprintf("hello world %d", i)), TokenAmounts: nil, FeeToken: common.HexToAddress("0x0"), - ExtraArgs: nil, + ExtraArgs: changeset.MakeEVMExtraArgsV2(50_000, false), } fee, err := sourceRouter.GetFee(&bind.CallOpts{Context: ctx}, destChainSelector, msg) @@ -495,51 +539,3 @@ func genSeqNrRange(start, end ccipocr3.SeqNum) []uint64 { } return seqNrs } - -func mustNewTransactor(t *testing.T, chain deployment.Chain) *bind.TransactOpts { - chainID, err := chainsel.GetChainIDFromSelector(chain.Selector) - require.NoError(t, err) - chainIDBig, ok := new(big.Int).SetString(chainID, 10) - require.True(t, ok, "evm chainID must be integral") - key, err := crypto.GenerateKey() - require.NoError(t, err) - transactor, err := bind.NewKeyedTransactorWithChainID(key, chainIDBig) - require.NoError(t, err) - return transactor -} - -func sendEth( - ctx context.Context, - t *testing.T, - chain deployment.Chain, - from *bind.TransactOpts, - to common.Address, - value *big.Int, -) { - balance, err := chain.Client.BalanceAt(ctx, from.From, nil) - require.NoError(t, err) - if balance.Cmp(value) < 0 { - t.Fatalf("insufficient balance: %s < %s", balance.String(), value.String()) - } - t.Logf("balance of from account %s: %s", from.From.String(), balance.String()) - - nonce, err := chain.Client.PendingNonceAt(ctx, from.From) - require.NoError(t, err) - gp, err := chain.Client.SuggestGasPrice(ctx) - require.NoError(t, err) - tx := gethtypes.NewTx(&gethtypes.LegacyTx{ - Nonce: nonce, - GasPrice: gp, - Gas: 21_000, - To: &to, - Value: value, - Data: nil, - }) - signedTx, err := from.Signer(from.From, tx) - require.NoError(t, err) - err = chain.Client.SendTransaction(ctx, signedTx) - require.NoError(t, err) - t.Log("sent funding tx:", signedTx.Hash().Hex()) - _, err = deployment.ConfirmIfNoError(chain, signedTx, err) - require.NoError(t, err) -} diff --git a/integration-tests/smoke/ccip/ccip_token_transfer_test.go b/integration-tests/smoke/ccip/ccip_token_transfer_test.go index 06ee06297ae..81920246bed 100644 --- a/integration-tests/smoke/ccip/ccip_token_transfer_test.go +++ b/integration-tests/smoke/ccip/ccip_token_transfer_test.go @@ -96,54 +96,44 @@ func TestTokenTransfer(t *testing.T) { }, ) - tcs := []struct { - name string - srcChain uint64 - dstChain uint64 - tokenAmounts []router.ClientEVMTokenAmount - receiver common.Address - data []byte - extraData []byte - expectedTokenBalances map[common.Address]*big.Int - expectedExecutionState int - }{ + tcs := []changeset.TestTransferRequest{ { - name: "Send token to EOA", - srcChain: sourceChain, - dstChain: destChain, - tokenAmounts: []router.ClientEVMTokenAmount{ + Name: "Send token to EOA", + SourceChain: sourceChain, + DestChain: destChain, + Tokens: []router.ClientEVMTokenAmount{ { Token: srcToken.Address(), Amount: oneE18, }, }, - receiver: utils.RandomAddress(), - expectedTokenBalances: map[common.Address]*big.Int{ + Receiver: utils.RandomAddress(), + ExpectedTokenBalances: map[common.Address]*big.Int{ destToken.Address(): oneE18, }, - expectedExecutionState: changeset.EXECUTION_STATE_SUCCESS, + ExpectedStatus: changeset.EXECUTION_STATE_SUCCESS, }, { - name: "Send token to contract", - srcChain: sourceChain, - dstChain: destChain, - tokenAmounts: []router.ClientEVMTokenAmount{ + Name: "Send token to contract", + SourceChain: sourceChain, + DestChain: destChain, + Tokens: []router.ClientEVMTokenAmount{ { Token: srcToken.Address(), Amount: oneE18, }, }, - receiver: state.Chains[destChain].Receiver.Address(), - expectedTokenBalances: map[common.Address]*big.Int{ + Receiver: state.Chains[destChain].Receiver.Address(), + ExpectedTokenBalances: map[common.Address]*big.Int{ destToken.Address(): oneE18, }, - expectedExecutionState: changeset.EXECUTION_STATE_SUCCESS, + ExpectedStatus: changeset.EXECUTION_STATE_SUCCESS, }, { - name: "Send N tokens to contract", - srcChain: destChain, - dstChain: sourceChain, - tokenAmounts: []router.ClientEVMTokenAmount{ + Name: "Send N tokens to contract", + SourceChain: destChain, + DestChain: sourceChain, + Tokens: []router.ClientEVMTokenAmount{ { Token: selfServeDestToken.Address(), Amount: oneE18, @@ -157,19 +147,19 @@ func TestTokenTransfer(t *testing.T) { Amount: oneE18, }, }, - receiver: state.Chains[sourceChain].Receiver.Address(), - extraData: changeset.MakeEVMExtraArgsV2(300_000, false), - expectedTokenBalances: map[common.Address]*big.Int{ + Receiver: state.Chains[sourceChain].Receiver.Address(), + ExtraArgs: changeset.MakeEVMExtraArgsV2(300_000, false), + ExpectedTokenBalances: map[common.Address]*big.Int{ selfServeSrcToken.Address(): new(big.Int).Add(oneE18, oneE18), srcToken.Address(): oneE18, }, - expectedExecutionState: changeset.EXECUTION_STATE_SUCCESS, + ExpectedStatus: changeset.EXECUTION_STATE_SUCCESS, }, { - name: "Sending token transfer with custom gasLimits to the EOA is successful", - srcChain: destChain, - dstChain: sourceChain, - tokenAmounts: []router.ClientEVMTokenAmount{ + Name: "Sending token transfer with custom gasLimits to the EOA is successful", + SourceChain: destChain, + DestChain: sourceChain, + Tokens: []router.ClientEVMTokenAmount{ { Token: selfServeDestToken.Address(), Amount: oneE18, @@ -179,19 +169,19 @@ func TestTokenTransfer(t *testing.T) { Amount: new(big.Int).Add(oneE18, oneE18), }, }, - receiver: utils.RandomAddress(), - extraData: changeset.MakeEVMExtraArgsV2(1, false), - expectedTokenBalances: map[common.Address]*big.Int{ + Receiver: utils.RandomAddress(), + ExtraArgs: changeset.MakeEVMExtraArgsV2(1, false), + ExpectedTokenBalances: map[common.Address]*big.Int{ selfServeSrcToken.Address(): oneE18, srcToken.Address(): new(big.Int).Add(oneE18, oneE18), }, - expectedExecutionState: changeset.EXECUTION_STATE_SUCCESS, + ExpectedStatus: changeset.EXECUTION_STATE_SUCCESS, }, { - name: "Sending PTT with too low gas limit leads to the revert when receiver is a contract", - srcChain: destChain, - dstChain: sourceChain, - tokenAmounts: []router.ClientEVMTokenAmount{ + Name: "Sending PTT with too low gas limit leads to the revert when receiver is a contract", + SourceChain: destChain, + DestChain: sourceChain, + Tokens: []router.ClientEVMTokenAmount{ { Token: selfServeDestToken.Address(), Amount: oneE18, @@ -201,45 +191,40 @@ func TestTokenTransfer(t *testing.T) { Amount: oneE18, }, }, - receiver: state.Chains[sourceChain].Receiver.Address(), - data: []byte("this should be reverted because gasLimit is too low, no tokens are transferred as well"), - extraData: changeset.MakeEVMExtraArgsV2(1, false), - expectedTokenBalances: map[common.Address]*big.Int{ + Receiver: state.Chains[sourceChain].Receiver.Address(), + Data: []byte("this should be reverted because gasLimit is too low, no tokens are transferred as well"), + ExtraArgs: changeset.MakeEVMExtraArgsV2(1, false), + ExpectedTokenBalances: map[common.Address]*big.Int{ selfServeSrcToken.Address(): big.NewInt(0), srcToken.Address(): big.NewInt(0), }, - expectedExecutionState: changeset.EXECUTION_STATE_FAILURE, + ExpectedStatus: changeset.EXECUTION_STATE_FAILURE, }, } - for _, tt := range tcs { - t.Run(tt.name, func(t *testing.T) { - initialBalances := map[common.Address]*big.Int{} - for token := range tt.expectedTokenBalances { - initialBalance := changeset.GetTokenBalance(ctx, t, token, tt.receiver, e.Chains[tt.dstChain]) - initialBalances[token] = initialBalance - } + startBlocks, expectedSeqNums, expectedExecutionStates, expectedTokenBalances := + changeset.TransferMultiple(ctx, t, e, state, tcs) - changeset.TransferAndWaitForSuccess( - ctx, - t, - e, - state, - tt.srcChain, - tt.dstChain, - tt.tokenAmounts, - tt.receiver, - tt.data, - tt.expectedExecutionState, - tt.extraData, - ) + err = changeset.ConfirmMultipleCommits( + t, + e.Chains, + state.Chains, + startBlocks, + false, + expectedSeqNums, + ) + require.NoError(t, err) - for token, balance := range tt.expectedTokenBalances { - expected := new(big.Int).Add(initialBalances[token], balance) - changeset.WaitForTheTokenBalance(ctx, t, token, tt.receiver, e.Chains[tt.dstChain], expected) - } - }) - } + execStates := changeset.ConfirmExecWithSeqNrsForAll( + t, + e, + state, + changeset.SeqNumberRageToSlice(expectedSeqNums), + startBlocks, + ) + require.Equal(t, expectedExecutionStates, execStates) + + changeset.WaitForTokenBalances(ctx, t, e.Chains, expectedTokenBalances) } func createAndFundSelfServeActor( diff --git a/integration-tests/smoke/ccip/ccip_usdc_test.go b/integration-tests/smoke/ccip/ccip_usdc_test.go index d1df2a6da92..eacf03df926 100644 --- a/integration-tests/smoke/ccip/ccip_usdc_test.go +++ b/integration-tests/smoke/ccip/ccip_usdc_test.go @@ -11,12 +11,10 @@ import ( "golang.org/x/sync/errgroup" "github.com/smartcontractkit/chainlink-common/pkg/utils/tests" - "github.com/smartcontractkit/chainlink-testing-framework/lib/utils/testcontext" "github.com/smartcontractkit/chainlink/deployment" "github.com/smartcontractkit/chainlink/deployment/ccip/changeset" testsetups "github.com/smartcontractkit/chainlink/integration-tests/testsetups/ccip" "github.com/smartcontractkit/chainlink/v2/core/chains/evm/utils" - "github.com/smartcontractkit/chainlink/v2/core/gethwrappers/ccip/generated/onramp" "github.com/smartcontractkit/chainlink/v2/core/gethwrappers/ccip/generated/router" "github.com/smartcontractkit/chainlink/v2/core/gethwrappers/shared/generated/burn_mint_erc677" "github.com/smartcontractkit/chainlink/v2/core/logger" @@ -37,7 +35,12 @@ func TestUSDCTokenTransfer(t *testing.T) { IsUSDC: true, } tenv, _, _ := testsetups.NewLocalDevEnvironmentWithDefaultPrice(t, lggr, config) - //tenv := changeset.NewMemoryEnvironmentWithJobsAndContracts(t, lggr, 3, 4, config) + //tenv := changeset.NewMemoryEnvironmentWithJobsAndContracts(t, lggr, memory.MemoryEnvironmentConfig{ + // Chains: 3, + // NumOfUsersPerChain: 3, + // Nodes: 5, + // Bootstraps: 1, + //}, config) e := tenv.Env state, err := changeset.LoadOnchainState(e) @@ -97,37 +100,28 @@ func TestUSDCTokenTransfer(t *testing.T) { // MockE2EUSDCTransmitter always mint 1, see MockE2EUSDCTransmitter.sol for more details tinyOneCoin := new(big.Int).SetUint64(1) - tcs := []struct { - name string - receiver common.Address - sourceChain uint64 - destChain uint64 - tokens []router.ClientEVMTokenAmount - data []byte - expectedTokenBalances map[common.Address]*big.Int - expectedExecutionState int - }{ + tcs := []changeset.TestTransferRequest{ { - name: "single USDC token transfer to EOA", - receiver: utils.RandomAddress(), - sourceChain: chainC, - destChain: chainA, - tokens: []router.ClientEVMTokenAmount{ + Name: "single USDC token transfer to EOA", + Receiver: utils.RandomAddress(), + SourceChain: chainC, + DestChain: chainA, + Tokens: []router.ClientEVMTokenAmount{ { Token: cChainUSDC.Address(), Amount: tinyOneCoin, }}, - expectedTokenBalances: map[common.Address]*big.Int{ + ExpectedTokenBalances: map[common.Address]*big.Int{ aChainUSDC.Address(): tinyOneCoin, }, - expectedExecutionState: changeset.EXECUTION_STATE_SUCCESS, + ExpectedStatus: changeset.EXECUTION_STATE_SUCCESS, }, { - name: "multiple USDC tokens within the same message", - receiver: utils.RandomAddress(), - sourceChain: chainC, - destChain: chainA, - tokens: []router.ClientEVMTokenAmount{ + Name: "multiple USDC tokens within the same message", + Receiver: utils.RandomAddress(), + SourceChain: chainC, + DestChain: chainA, + Tokens: []router.ClientEVMTokenAmount{ { Token: cChainUSDC.Address(), Amount: tinyOneCoin, @@ -137,18 +131,18 @@ func TestUSDCTokenTransfer(t *testing.T) { Amount: tinyOneCoin, }, }, - expectedTokenBalances: map[common.Address]*big.Int{ - // 2 coins because of the same receiver + ExpectedTokenBalances: map[common.Address]*big.Int{ + // 2 coins because of the same Receiver aChainUSDC.Address(): new(big.Int).Add(tinyOneCoin, tinyOneCoin), }, - expectedExecutionState: changeset.EXECUTION_STATE_SUCCESS, + ExpectedStatus: changeset.EXECUTION_STATE_SUCCESS, }, { - name: "USDC token together with another token transferred to EOA", - receiver: utils.RandomAddress(), - sourceChain: chainA, - destChain: chainC, - tokens: []router.ClientEVMTokenAmount{ + Name: "USDC token together with another token transferred to EOA", + Receiver: utils.RandomAddress(), + SourceChain: chainA, + DestChain: chainC, + Tokens: []router.ClientEVMTokenAmount{ { Token: aChainUSDC.Address(), Amount: tinyOneCoin, @@ -158,105 +152,89 @@ func TestUSDCTokenTransfer(t *testing.T) { Amount: new(big.Int).Mul(tinyOneCoin, big.NewInt(10)), }, }, - expectedTokenBalances: map[common.Address]*big.Int{ + ExpectedTokenBalances: map[common.Address]*big.Int{ cChainUSDC.Address(): tinyOneCoin, cChainToken.Address(): new(big.Int).Mul(tinyOneCoin, big.NewInt(10)), }, - expectedExecutionState: changeset.EXECUTION_STATE_SUCCESS, + ExpectedStatus: changeset.EXECUTION_STATE_SUCCESS, }, { - name: "programmable token transfer to valid contract receiver", - receiver: state.Chains[chainC].Receiver.Address(), - sourceChain: chainA, - destChain: chainC, - tokens: []router.ClientEVMTokenAmount{ + Name: "USDC programmable token transfer to valid contract receiver", + Receiver: state.Chains[chainC].Receiver.Address(), + SourceChain: chainA, + DestChain: chainC, + Tokens: []router.ClientEVMTokenAmount{ { Token: aChainUSDC.Address(), Amount: tinyOneCoin, }, }, - data: []byte("hello world"), - expectedTokenBalances: map[common.Address]*big.Int{ + Data: []byte("hello world"), + ExpectedTokenBalances: map[common.Address]*big.Int{ cChainUSDC.Address(): tinyOneCoin, }, - expectedExecutionState: changeset.EXECUTION_STATE_SUCCESS, + ExpectedStatus: changeset.EXECUTION_STATE_SUCCESS, + }, + { + Name: "USDC programmable token transfer with too little gas", + Receiver: state.Chains[chainB].Receiver.Address(), + SourceChain: chainC, + DestChain: chainB, + Tokens: []router.ClientEVMTokenAmount{ + { + Token: cChainUSDC.Address(), + Amount: tinyOneCoin, + }, + }, + Data: []byte("gimme more gas to execute that!"), + ExpectedTokenBalances: map[common.Address]*big.Int{ + bChainUSDC.Address(): new(big.Int).SetUint64(0), + }, + ExtraArgs: changeset.MakeEVMExtraArgsV2(1, false), + ExpectedStatus: changeset.EXECUTION_STATE_FAILURE, + }, + { + Name: "USDC token transfer from a different source chain", + Receiver: utils.RandomAddress(), + SourceChain: chainB, + DestChain: chainC, + Tokens: []router.ClientEVMTokenAmount{ + { + Token: bChainUSDC.Address(), + Amount: tinyOneCoin, + }, + }, + Data: nil, + ExpectedTokenBalances: map[common.Address]*big.Int{ + cChainUSDC.Address(): tinyOneCoin, + }, + ExpectedStatus: changeset.EXECUTION_STATE_SUCCESS, }, } - for _, tt := range tcs { - t.Run(tt.name, func(t *testing.T) { - initialBalances := map[common.Address]*big.Int{} - for token := range tt.expectedTokenBalances { - initialBalance := changeset.GetTokenBalance(ctx, t, token, tt.receiver, e.Chains[tt.destChain]) - initialBalances[token] = initialBalance - } - - changeset.TransferAndWaitForSuccess( - ctx, - t, - e, - state, - tt.sourceChain, - tt.destChain, - tt.tokens, - tt.receiver, - tt.data, - tt.expectedExecutionState, - nil, - ) - - for token, balance := range tt.expectedTokenBalances { - expected := new(big.Int).Add(initialBalances[token], balance) - changeset.WaitForTheTokenBalance(ctx, t, token, tt.receiver, e.Chains[tt.destChain], expected) - } - }) - } - - t.Run("multi-source USDC transfer targeting the same dest receiver", func(t *testing.T) { - sendSingleTokenTransfer := func(source, dest uint64, token common.Address, receiver common.Address) (*onramp.OnRampCCIPMessageSent, changeset.SourceDestPair) { - msg := changeset.TestSendRequest(t, e, state, source, dest, false, router.ClientEVM2AnyMessage{ - Receiver: common.LeftPadBytes(receiver.Bytes(), 32), - Data: []byte{}, - TokenAmounts: []router.ClientEVMTokenAmount{{Token: token, Amount: tinyOneCoin}}, - FeeToken: common.HexToAddress("0x0"), - ExtraArgs: nil, - }) - return msg, changeset.SourceDestPair{ - SourceChainSelector: source, - DestChainSelector: dest, - } - } - - receiver := utils.RandomAddress() - - startBlocks := make(map[uint64]*uint64) - expectedSeqNum := make(map[changeset.SourceDestPair]uint64) - expectedSeqNumExec := make(map[changeset.SourceDestPair][]uint64) + startBlocks, expectedSeqNums, expectedExecutionStates, expectedTokenBalances := + changeset.TransferMultiple(ctx, t, e, state, tcs) - latesthdr, err := e.Chains[chainC].Client.HeaderByNumber(testcontext.Get(t), nil) - require.NoError(t, err) - block := latesthdr.Number.Uint64() - startBlocks[chainC] = &block - - message1, message1ID := sendSingleTokenTransfer(chainA, chainC, aChainUSDC.Address(), receiver) - expectedSeqNum[message1ID] = message1.SequenceNumber - expectedSeqNumExec[message1ID] = []uint64{message1.SequenceNumber} - - message2, message2ID := sendSingleTokenTransfer(chainB, chainC, bChainUSDC.Address(), receiver) - expectedSeqNum[message2ID] = message2.SequenceNumber - expectedSeqNumExec[message2ID] = []uint64{message2.SequenceNumber} - - changeset.ConfirmCommitForAllWithExpectedSeqNums(t, e, state, expectedSeqNum, startBlocks) - states := changeset.ConfirmExecWithSeqNrsForAll(t, e, state, expectedSeqNumExec, startBlocks) + err = changeset.ConfirmMultipleCommits( + t, + e.Chains, + state.Chains, + startBlocks, + false, + expectedSeqNums, + ) + require.NoError(t, err) - require.Equal(t, changeset.EXECUTION_STATE_SUCCESS, states[message1ID][message1.SequenceNumber]) - require.Equal(t, changeset.EXECUTION_STATE_SUCCESS, states[message2ID][message2.SequenceNumber]) + execStates := changeset.ConfirmExecWithSeqNrsForAll( + t, + e, + state, + changeset.SeqNumberRageToSlice(expectedSeqNums), + startBlocks, + ) + require.Equal(t, expectedExecutionStates, execStates) - // We sent 1 coin from each source chain, so we should have 2 coins on the destination chain - // Receiver is randomly generated so we don't need to get the initial balance first - expectedBalance := new(big.Int).Add(tinyOneCoin, tinyOneCoin) - changeset.WaitForTheTokenBalance(ctx, t, cChainUSDC.Address(), receiver, e.Chains[chainC], expectedBalance) - }) + changeset.WaitForTokenBalances(ctx, t, e.Chains, expectedTokenBalances) } func updateFeeQuoters( @@ -274,7 +252,11 @@ func updateFeeQuoters( return changeset.UpdateFeeQuoterForUSDC(lggr, e.Chains[chainB], state.Chains[chainB], chainC, bChainUSDC) }) updateFeeQtrGrp.Go(func() error { - return changeset.UpdateFeeQuoterForUSDC(lggr, e.Chains[chainC], state.Chains[chainC], chainA, cChainUSDC) + err1 := changeset.UpdateFeeQuoterForUSDC(lggr, e.Chains[chainC], state.Chains[chainC], chainA, cChainUSDC) + if err1 != nil { + return err1 + } + return changeset.UpdateFeeQuoterForUSDC(lggr, e.Chains[chainC], state.Chains[chainC], chainB, cChainUSDC) }) return updateFeeQtrGrp.Wait() }