From cecb0e1ada32235adbddcbab87163b4123e4ec8c Mon Sep 17 00:00:00 2001 From: Dimitris Grigoriou Date: Wed, 27 Nov 2024 15:26:50 +0200 Subject: [PATCH] Txmv2 stuck tx detection (#15436) * Stuck tx detector alpha * Update stuck tx detection * Add stuck_tx_detection and dual broadcast client * Add support for TXMv2 * Fix orchestrator's monitoring call * Fix AttemptBuilder * Enable DualBroadcast client * Switch DualBroadcast params to pointers * Add context to client * Fix lint * Fix DualBroadcast client * More lint fixes * Fix lint --- core/chains/evm/config/chain_scoped.go | 4 + core/chains/evm/config/chain_scoped_txmv2.go | 25 ++ core/chains/evm/config/chaintype/chaintype.go | 6 +- core/chains/evm/config/config.go | 7 + core/chains/evm/config/toml/config.go | 27 ++- core/chains/evm/config/toml/defaults.go | 1 + .../evm/config/toml/defaults/fallback.toml | 3 + core/chains/evm/keystore/eth.go | 1 + core/chains/evm/keystore/mocks/eth.go | 60 +++++ .../evm/txm/clientwrappers/chain_client.go | 31 +++ .../clientwrappers/dual_broadcast_client.go | 130 ++++++++++ .../evm/txm/clientwrappers/geth_client.go | 51 ++++ core/chains/evm/txm/dummy_keystore.go | 13 + core/chains/evm/txm/mocks/client.go | 27 ++- core/chains/evm/txm/stuck_tx_detector.go | 115 +++++++++ core/chains/evm/txm/txm.go | 30 +-- core/chains/evm/txm/txm_test.go | 26 +- core/chains/evm/txm/types/transaction.go | 4 + core/chains/evm/txmgr/builder.go | 32 ++- core/chains/legacyevm/evm_txm.go | 43 ++-- core/config/docs/chains-evm.toml | 8 + docs/CONFIG.md | 222 ++++++++++++++++++ 22 files changed, 801 insertions(+), 65 deletions(-) create mode 100644 core/chains/evm/config/chain_scoped_txmv2.go create mode 100644 core/chains/evm/txm/clientwrappers/chain_client.go create mode 100644 core/chains/evm/txm/clientwrappers/dual_broadcast_client.go create mode 100644 core/chains/evm/txm/clientwrappers/geth_client.go create mode 100644 core/chains/evm/txm/stuck_tx_detector.go diff --git a/core/chains/evm/config/chain_scoped.go b/core/chains/evm/config/chain_scoped.go index de89272b5e2..3a7ff43d8a6 100644 --- a/core/chains/evm/config/chain_scoped.go +++ b/core/chains/evm/config/chain_scoped.go @@ -52,6 +52,10 @@ func (e *EVMConfig) BalanceMonitor() BalanceMonitor { return &balanceMonitorConfig{c: e.C.BalanceMonitor} } +func (e *EVMConfig) TxmV2() TxmV2 { + return &txmv2Config{c: e.C.TxmV2} +} + func (e *EVMConfig) Transactions() Transactions { return &transactionsConfig{c: e.C.Transactions} } diff --git a/core/chains/evm/config/chain_scoped_txmv2.go b/core/chains/evm/config/chain_scoped_txmv2.go new file mode 100644 index 00000000000..e50148cfae4 --- /dev/null +++ b/core/chains/evm/config/chain_scoped_txmv2.go @@ -0,0 +1,25 @@ +package config + +import ( + "net/url" + "time" + + "github.com/smartcontractkit/chainlink/v2/core/chains/evm/config/toml" +) + +type txmv2Config struct { + c toml.TxmV2 +} + +func (t *txmv2Config) Enabled() bool { + return *t.c.Enabled +} + +func (t *txmv2Config) BlockTime() *time.Duration { + d := t.c.BlockTime.Duration() + return &d +} + +func (t *txmv2Config) CustomURL() *url.URL { + return t.c.CustomURL.URL() +} diff --git a/core/chains/evm/config/chaintype/chaintype.go b/core/chains/evm/config/chaintype/chaintype.go index b2eff02834b..bc2eace8ca3 100644 --- a/core/chains/evm/config/chaintype/chaintype.go +++ b/core/chains/evm/config/chaintype/chaintype.go @@ -23,6 +23,7 @@ const ( ChainZkEvm ChainType = "zkevm" ChainZkSync ChainType = "zksync" ChainZircuit ChainType = "zircuit" + ChainDualBroadcast ChainType = "dualBroadcast" ) // IsL2 returns true if this chain is a Layer 2 chain. Notably: @@ -39,7 +40,7 @@ func (c ChainType) IsL2() bool { func (c ChainType) IsValid() bool { switch c { - case "", ChainArbitrum, ChainAstar, ChainCelo, ChainGnosis, ChainHedera, ChainKroma, ChainMantle, ChainMetis, ChainOptimismBedrock, ChainScroll, ChainWeMix, ChainXLayer, ChainZkEvm, ChainZkSync, ChainZircuit: + case "", ChainArbitrum, ChainAstar, ChainCelo, ChainGnosis, ChainHedera, ChainKroma, ChainMantle, ChainMetis, ChainOptimismBedrock, ChainScroll, ChainWeMix, ChainXLayer, ChainZkEvm, ChainZkSync, ChainZircuit, ChainDualBroadcast: return true } return false @@ -77,6 +78,8 @@ func FromSlug(slug string) ChainType { return ChainZkSync case "zircuit": return ChainZircuit + case "dualBroadcast": + return ChainDualBroadcast default: return ChainType(slug) } @@ -144,4 +147,5 @@ var ErrInvalid = fmt.Errorf("must be one of %s or omitted", strings.Join([]strin string(ChainZkEvm), string(ChainZkSync), string(ChainZircuit), + string(ChainDualBroadcast), }, ", ")) diff --git a/core/chains/evm/config/config.go b/core/chains/evm/config/config.go index f2a571f94b0..be61bfe02ee 100644 --- a/core/chains/evm/config/config.go +++ b/core/chains/evm/config/config.go @@ -18,6 +18,7 @@ import ( type EVM interface { HeadTracker() HeadTracker BalanceMonitor() BalanceMonitor + TxmV2() TxmV2 Transactions() Transactions GasEstimator() GasEstimator OCR() OCR @@ -102,6 +103,12 @@ type ClientErrors interface { TooManyResults() string } +type TxmV2 interface { + Enabled() bool + BlockTime() *time.Duration + CustomURL() *url.URL +} + type Transactions interface { ForwardersEnabled() bool ReaperInterval() time.Duration diff --git a/core/chains/evm/config/toml/config.go b/core/chains/evm/config/toml/config.go index 0505449943e..0f8b1eceee5 100644 --- a/core/chains/evm/config/toml/config.go +++ b/core/chains/evm/config/toml/config.go @@ -300,8 +300,10 @@ func (c *EVMConfig) ValidateConfig() (err error) { is := c.ChainType.ChainType() if is != must { if must == "" { - err = multierr.Append(err, commonconfig.ErrInvalid{Name: "ChainType", Value: c.ChainType.ChainType(), - Msg: "must not be set with this chain id"}) + if c.ChainType.ChainType() != chaintype.ChainDualBroadcast { + err = multierr.Append(err, commonconfig.ErrInvalid{Name: "ChainType", Value: c.ChainType.ChainType(), + Msg: "must not be set with this chain id"}) + } } else { err = multierr.Append(err, commonconfig.ErrInvalid{Name: "ChainType", Value: c.ChainType.ChainType(), Msg: fmt.Sprintf("only %q can be used with this chain id", must)}) @@ -387,6 +389,7 @@ type Chain struct { FinalizedBlockOffset *uint32 NoNewFinalizedHeadsThreshold *commonconfig.Duration + TxmV2 TxmV2 `toml:",omitempty"` Transactions Transactions `toml:",omitempty"` BalanceMonitor BalanceMonitor `toml:",omitempty"` GasEstimator GasEstimator `toml:",omitempty"` @@ -471,6 +474,26 @@ func (c *Chain) ValidateConfig() (err error) { return } +type TxmV2 struct { + Enabled *bool `toml:",omitempty"` + BlockTime *commonconfig.Duration `toml:",omitempty"` + CustomURL *commonconfig.URL `toml:",omitempty"` +} + +func (t *TxmV2) setFrom(f *TxmV2) { + if v := f.Enabled; v != nil { + t.Enabled = f.Enabled + } + + if v := f.BlockTime; v != nil { + t.BlockTime = f.BlockTime + } + + if v := f.CustomURL; v != nil { + t.CustomURL = f.CustomURL + } +} + type Transactions struct { ForwardersEnabled *bool MaxInFlight *uint32 diff --git a/core/chains/evm/config/toml/defaults.go b/core/chains/evm/config/toml/defaults.go index 0885d94e6df..5ce014921f4 100644 --- a/core/chains/evm/config/toml/defaults.go +++ b/core/chains/evm/config/toml/defaults.go @@ -241,6 +241,7 @@ func (c *Chain) SetFrom(f *Chain) { c.NoNewFinalizedHeadsThreshold = v } + c.TxmV2.setFrom(&f.TxmV2) c.Transactions.setFrom(&f.Transactions) c.BalanceMonitor.setFrom(&f.BalanceMonitor) c.GasEstimator.setFrom(&f.GasEstimator) diff --git a/core/chains/evm/config/toml/defaults/fallback.toml b/core/chains/evm/config/toml/defaults/fallback.toml index ab349ee4688..44228968fc6 100644 --- a/core/chains/evm/config/toml/defaults/fallback.toml +++ b/core/chains/evm/config/toml/defaults/fallback.toml @@ -18,6 +18,9 @@ FinalizedBlockOffset = 0 NoNewFinalizedHeadsThreshold = '0' LogBroadcasterEnabled = true +[TxmV2] +Enabled = false + [Transactions] ForwardersEnabled = false MaxInFlight = 16 diff --git a/core/chains/evm/keystore/eth.go b/core/chains/evm/keystore/eth.go index ff71e0a4f18..9c0986d9c3d 100644 --- a/core/chains/evm/keystore/eth.go +++ b/core/chains/evm/keystore/eth.go @@ -13,5 +13,6 @@ type Eth interface { CheckEnabled(ctx context.Context, address common.Address, chainID *big.Int) error EnabledAddressesForChain(ctx context.Context, chainID *big.Int) (addresses []common.Address, err error) SignTx(ctx context.Context, fromAddress common.Address, tx *types.Transaction, chainID *big.Int) (*types.Transaction, error) + SignMessage(ctx context.Context, address common.Address, message []byte) ([]byte, error) SubscribeToKeyChanges(ctx context.Context) (ch chan struct{}, unsub func()) } diff --git a/core/chains/evm/keystore/mocks/eth.go b/core/chains/evm/keystore/mocks/eth.go index b481be1b5c8..bfc85fc672c 100644 --- a/core/chains/evm/keystore/mocks/eth.go +++ b/core/chains/evm/keystore/mocks/eth.go @@ -133,6 +133,66 @@ func (_c *Eth_EnabledAddressesForChain_Call) RunAndReturn(run func(context.Conte return _c } +// SignMessage provides a mock function with given fields: ctx, address, message +func (_m *Eth) SignMessage(ctx context.Context, address common.Address, message []byte) ([]byte, error) { + ret := _m.Called(ctx, address, message) + + if len(ret) == 0 { + panic("no return value specified for SignMessage") + } + + var r0 []byte + var r1 error + if rf, ok := ret.Get(0).(func(context.Context, common.Address, []byte) ([]byte, error)); ok { + return rf(ctx, address, message) + } + if rf, ok := ret.Get(0).(func(context.Context, common.Address, []byte) []byte); ok { + r0 = rf(ctx, address, message) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).([]byte) + } + } + + if rf, ok := ret.Get(1).(func(context.Context, common.Address, []byte) error); ok { + r1 = rf(ctx, address, message) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// Eth_SignMessage_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'SignMessage' +type Eth_SignMessage_Call struct { + *mock.Call +} + +// SignMessage is a helper method to define mock.On call +// - ctx context.Context +// - address common.Address +// - message []byte +func (_e *Eth_Expecter) SignMessage(ctx interface{}, address interface{}, message interface{}) *Eth_SignMessage_Call { + return &Eth_SignMessage_Call{Call: _e.mock.On("SignMessage", ctx, address, message)} +} + +func (_c *Eth_SignMessage_Call) Run(run func(ctx context.Context, address common.Address, message []byte)) *Eth_SignMessage_Call { + _c.Call.Run(func(args mock.Arguments) { + run(args[0].(context.Context), args[1].(common.Address), args[2].([]byte)) + }) + return _c +} + +func (_c *Eth_SignMessage_Call) Return(_a0 []byte, _a1 error) *Eth_SignMessage_Call { + _c.Call.Return(_a0, _a1) + return _c +} + +func (_c *Eth_SignMessage_Call) RunAndReturn(run func(context.Context, common.Address, []byte) ([]byte, error)) *Eth_SignMessage_Call { + _c.Call.Return(run) + return _c +} + // SignTx provides a mock function with given fields: ctx, fromAddress, tx, chainID func (_m *Eth) SignTx(ctx context.Context, fromAddress common.Address, tx *types.Transaction, chainID *big.Int) (*types.Transaction, error) { ret := _m.Called(ctx, fromAddress, tx, chainID) diff --git a/core/chains/evm/txm/clientwrappers/chain_client.go b/core/chains/evm/txm/clientwrappers/chain_client.go new file mode 100644 index 00000000000..7638cc53443 --- /dev/null +++ b/core/chains/evm/txm/clientwrappers/chain_client.go @@ -0,0 +1,31 @@ +package clientwrappers + +import ( + "context" + "math/big" + + "github.com/ethereum/go-ethereum/common" + + "github.com/smartcontractkit/chainlink/v2/core/chains/evm/client" + "github.com/smartcontractkit/chainlink/v2/core/chains/evm/txm/types" +) + +type ChainClient struct { + c client.Client +} + +func NewChainClient(client client.Client) *ChainClient { + return &ChainClient{c: client} +} + +func (c *ChainClient) NonceAt(ctx context.Context, address common.Address, blockNumber *big.Int) (uint64, error) { + return c.c.NonceAt(ctx, address, blockNumber) +} + +func (c *ChainClient) PendingNonceAt(ctx context.Context, address common.Address) (uint64, error) { + return c.c.PendingNonceAt(ctx, address) +} + +func (c *ChainClient) SendTransaction(ctx context.Context, _ *types.Transaction, attempt *types.Attempt) error { + return c.c.SendTransaction(ctx, attempt.SignedTransaction) +} diff --git a/core/chains/evm/txm/clientwrappers/dual_broadcast_client.go b/core/chains/evm/txm/clientwrappers/dual_broadcast_client.go new file mode 100644 index 00000000000..0bbd2530765 --- /dev/null +++ b/core/chains/evm/txm/clientwrappers/dual_broadcast_client.go @@ -0,0 +1,130 @@ +package clientwrappers + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "io" + "math/big" + "net/http" + "net/url" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/common/hexutil" + "github.com/ethereum/go-ethereum/crypto" + + "github.com/smartcontractkit/chainlink/v2/core/chains/evm/client" + "github.com/smartcontractkit/chainlink/v2/core/chains/evm/txm/types" +) + +type DualBroadcastClientKeystore interface { + SignMessage(ctx context.Context, address common.Address, message []byte) ([]byte, error) +} + +type DualBroadcastClient struct { + c client.Client + keystore DualBroadcastClientKeystore + customURL *url.URL +} + +func NewDualBroadcastClient(c client.Client, keystore DualBroadcastClientKeystore, customURL *url.URL) *DualBroadcastClient { + return &DualBroadcastClient{ + c: c, + keystore: keystore, + customURL: customURL, + } +} + +func (d *DualBroadcastClient) NonceAt(ctx context.Context, address common.Address, blockNumber *big.Int) (uint64, error) { + return d.c.NonceAt(ctx, address, blockNumber) +} + +func (d *DualBroadcastClient) PendingNonceAt(ctx context.Context, address common.Address) (uint64, error) { + body := []byte(fmt.Sprintf(`{"jsonrpc":"2.0","method":"eth_getTransactionCount","params":["%s","pending"]}`, address.String())) + response, err := d.signAndPostMessage(ctx, address, body, "") + if err != nil { + return 0, err + } + + nonce, err := hexutil.DecodeUint64(response) + if err != nil { + return 0, fmt.Errorf("failed to decode response %v into uint64: %w", response, err) + } + return nonce, nil +} + +func (d *DualBroadcastClient) SendTransaction(ctx context.Context, tx *types.Transaction, attempt *types.Attempt) error { + meta, err := tx.GetMeta() + if err != nil { + return err + } + //nolint:revive //linter nonsense + if meta != nil && meta.DualBroadcast != nil && *meta.DualBroadcast && !tx.IsPurgeable { + data, err := attempt.SignedTransaction.MarshalBinary() + if err != nil { + return err + } + params := "" + if meta.DualBroadcastParams != nil { + params = *meta.DualBroadcastParams + } + body := []byte(fmt.Sprintf(`{"jsonrpc":"2.0","method":"eth_sendRawTransaction","params":["%s"]}`, hexutil.Encode(data))) + if _, err = d.signAndPostMessage(ctx, tx.FromAddress, body, params); err != nil { + return err + } + return nil + } else { + return d.c.SendTransaction(ctx, attempt.SignedTransaction) + } +} + +func (d *DualBroadcastClient) signAndPostMessage(ctx context.Context, address common.Address, body []byte, urlParams string) (result string, err error) { + bodyReader := bytes.NewReader(body) + postReq, err := http.NewRequestWithContext(ctx, http.MethodPost, d.customURL.String()+"?"+urlParams, bodyReader) + if err != nil { + return + } + + hashedBody := crypto.Keccak256Hash(body).Hex() + signedMessage, err := d.keystore.SignMessage(ctx, address, []byte(hashedBody)) + if err != nil { + return + } + + postReq.Header.Add("X-Flashbots-signature", address.String()+":"+hexutil.Encode(signedMessage)) + postReq.Header.Add("Content-Type", "application/json") + + resp, err := http.DefaultClient.Do(postReq) + if err != nil { + return result, fmt.Errorf("request %v failed: %w", postReq, err) + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + return result, fmt.Errorf("request %v failed with status: %d", postReq, resp.StatusCode) + } + + keyJSON, err := io.ReadAll(resp.Body) + if err != nil { + return + } + var response postResponse + err = json.Unmarshal(keyJSON, &response) + if err != nil { + return result, fmt.Errorf("failed to unmarshal response into struct: %w: %s", err, string(keyJSON)) + } + if response.Error.Message != "" { + return result, errors.New(response.Error.Message) + } + return response.Result, nil +} + +type postResponse struct { + Result string `json:"result,omitempty"` + Error postError +} + +type postError struct { + Message string `json:"message,omitempty"` +} diff --git a/core/chains/evm/txm/clientwrappers/geth_client.go b/core/chains/evm/txm/clientwrappers/geth_client.go new file mode 100644 index 00000000000..d97e5cfae35 --- /dev/null +++ b/core/chains/evm/txm/clientwrappers/geth_client.go @@ -0,0 +1,51 @@ +package clientwrappers + +import ( + "context" + "math/big" + + "github.com/ethereum/go-ethereum" + "github.com/ethereum/go-ethereum/common/hexutil" + "github.com/ethereum/go-ethereum/ethclient" + "github.com/ethereum/go-ethereum/rpc" + + "github.com/smartcontractkit/chainlink/v2/core/chains/evm/client" + "github.com/smartcontractkit/chainlink/v2/core/chains/evm/txm/types" + evmtypes "github.com/smartcontractkit/chainlink/v2/core/chains/evm/types" +) + +type GethClient struct { + *ethclient.Client +} + +func NewGethClient(client *ethclient.Client) *GethClient { + return &GethClient{ + Client: client, + } +} + +func (g *GethClient) BatchCallContext(ctx context.Context, b []rpc.BatchElem) error { + return g.Client.Client().BatchCallContext(ctx, b) +} + +func (g *GethClient) CallContext(ctx context.Context, result interface{}, method string, args ...interface{}) error { + return g.Client.Client().CallContext(ctx, result, method, args...) +} + +func (g *GethClient) CallContract(ctx context.Context, message ethereum.CallMsg, blockNumber *big.Int) ([]byte, error) { + var hex hexutil.Bytes + err := g.CallContext(ctx, &hex, "eth_call", client.ToBackwardCompatibleCallArg(message), client.ToBackwardCompatibleBlockNumArg(blockNumber)) + return hex, err +} + +func (g *GethClient) HeadByNumber(ctx context.Context, number *big.Int) (*evmtypes.Head, error) { + hexNumber := client.ToBlockNumArg(number) + args := []interface{}{hexNumber, false} + head := new(evmtypes.Head) + err := g.CallContext(ctx, head, "eth_getBlockByNumber", args...) + return head, err +} + +func (g *GethClient) SendTransaction(ctx context.Context, _ *types.Transaction, attempt *types.Attempt) error { + return g.Client.SendTransaction(ctx, attempt.SignedTransaction) +} diff --git a/core/chains/evm/txm/dummy_keystore.go b/core/chains/evm/txm/dummy_keystore.go index 1161f34058b..01816dfcbbd 100644 --- a/core/chains/evm/txm/dummy_keystore.go +++ b/core/chains/evm/txm/dummy_keystore.go @@ -6,6 +6,7 @@ import ( "fmt" "math/big" + "github.com/ethereum/go-ethereum/accounts" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/crypto" @@ -43,6 +44,18 @@ func (k *DummyKeystore) SignTx(_ context.Context, fromAddress common.Address, tx return nil, fmt.Errorf("private key for address: %v not found", fromAddress) } +func (k *DummyKeystore) SignMessage(ctx context.Context, address common.Address, data []byte) ([]byte, error) { + key, exists := k.privateKeyMap[address] + if !exists { + return nil, fmt.Errorf("private key for address: %v not found", address) + } + signature, err := crypto.Sign(accounts.TextHash(data), key) + if err != nil { + return nil, fmt.Errorf("failed to sign message for address: %v", address) + } + return signature, nil +} + func (k *DummyKeystore) EnabledAddressesForChain(_ context.Context, _ *big.Int) (addresses []common.Address, err error) { for address := range k.privateKeyMap { addresses = append(addresses, address) diff --git a/core/chains/evm/txm/mocks/client.go b/core/chains/evm/txm/mocks/client.go index 533298625e5..03849ad7e82 100644 --- a/core/chains/evm/txm/mocks/client.go +++ b/core/chains/evm/txm/mocks/client.go @@ -10,7 +10,7 @@ import ( mock "github.com/stretchr/testify/mock" - types "github.com/ethereum/go-ethereum/core/types" + types "github.com/smartcontractkit/chainlink/v2/core/chains/evm/txm/types" ) // Client is an autogenerated mock type for the Client type @@ -141,17 +141,17 @@ func (_c *Client_PendingNonceAt_Call) RunAndReturn(run func(context.Context, com return _c } -// SendTransaction provides a mock function with given fields: _a0, _a1 -func (_m *Client) SendTransaction(_a0 context.Context, _a1 *types.Transaction) error { - ret := _m.Called(_a0, _a1) +// SendTransaction provides a mock function with given fields: ctx, tx, attempt +func (_m *Client) SendTransaction(ctx context.Context, tx *types.Transaction, attempt *types.Attempt) error { + ret := _m.Called(ctx, tx, attempt) if len(ret) == 0 { panic("no return value specified for SendTransaction") } var r0 error - if rf, ok := ret.Get(0).(func(context.Context, *types.Transaction) error); ok { - r0 = rf(_a0, _a1) + if rf, ok := ret.Get(0).(func(context.Context, *types.Transaction, *types.Attempt) error); ok { + r0 = rf(ctx, tx, attempt) } else { r0 = ret.Error(0) } @@ -165,15 +165,16 @@ type Client_SendTransaction_Call struct { } // SendTransaction is a helper method to define mock.On call -// - _a0 context.Context -// - _a1 *types.Transaction -func (_e *Client_Expecter) SendTransaction(_a0 interface{}, _a1 interface{}) *Client_SendTransaction_Call { - return &Client_SendTransaction_Call{Call: _e.mock.On("SendTransaction", _a0, _a1)} +// - ctx context.Context +// - tx *types.Transaction +// - attempt *types.Attempt +func (_e *Client_Expecter) SendTransaction(ctx interface{}, tx interface{}, attempt interface{}) *Client_SendTransaction_Call { + return &Client_SendTransaction_Call{Call: _e.mock.On("SendTransaction", ctx, tx, attempt)} } -func (_c *Client_SendTransaction_Call) Run(run func(_a0 context.Context, _a1 *types.Transaction)) *Client_SendTransaction_Call { +func (_c *Client_SendTransaction_Call) Run(run func(ctx context.Context, tx *types.Transaction, attempt *types.Attempt)) *Client_SendTransaction_Call { _c.Call.Run(func(args mock.Arguments) { - run(args[0].(context.Context), args[1].(*types.Transaction)) + run(args[0].(context.Context), args[1].(*types.Transaction), args[2].(*types.Attempt)) }) return _c } @@ -183,7 +184,7 @@ func (_c *Client_SendTransaction_Call) Return(_a0 error) *Client_SendTransaction return _c } -func (_c *Client_SendTransaction_Call) RunAndReturn(run func(context.Context, *types.Transaction) error) *Client_SendTransaction_Call { +func (_c *Client_SendTransaction_Call) RunAndReturn(run func(context.Context, *types.Transaction, *types.Attempt) error) *Client_SendTransaction_Call { _c.Call.Return(run) return _c } diff --git a/core/chains/evm/txm/stuck_tx_detector.go b/core/chains/evm/txm/stuck_tx_detector.go new file mode 100644 index 00000000000..68d8caf0ed1 --- /dev/null +++ b/core/chains/evm/txm/stuck_tx_detector.go @@ -0,0 +1,115 @@ +package txm + +import ( + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "time" + + "github.com/ethereum/go-ethereum/common" + + "github.com/smartcontractkit/chainlink-common/pkg/logger" + + "github.com/smartcontractkit/chainlink/v2/core/chains/evm/config/chaintype" + "github.com/smartcontractkit/chainlink/v2/core/chains/evm/txm/types" +) + +type StuckTxDetectorConfig struct { + BlockTime time.Duration + StuckTxBlockThreshold uint32 + DetectionURL string +} + +type stuckTxDetector struct { + lggr logger.Logger + chainType chaintype.ChainType + config StuckTxDetectorConfig +} + +func NewStuckTxDetector(lggr logger.Logger, chaintype chaintype.ChainType, config StuckTxDetectorConfig) *stuckTxDetector { + return &stuckTxDetector{ + lggr: lggr, + chainType: chaintype, + config: config, + } +} + +func (s *stuckTxDetector) DetectStuckTransaction(ctx context.Context, tx *types.Transaction) (bool, error) { + switch s.chainType { + // TODO: rename + case chaintype.ChainDualBroadcast: + result, err := s.dualBroadcastDetection(ctx, tx) + if result || err != nil { + return result, err + } + return s.timeBasedDetection(tx), nil + default: + return s.timeBasedDetection(tx), nil + } +} + +func (s *stuckTxDetector) timeBasedDetection(tx *types.Transaction) bool { + threshold := (s.config.BlockTime * time.Duration(s.config.StuckTxBlockThreshold)) + if time.Since(tx.LastBroadcastAt) > threshold && !tx.LastBroadcastAt.IsZero() { + s.lggr.Debugf("TxID: %v last broadcast was: %v which is more than the max configured duration: %v. Transaction is now considered stuck and will be purged.", + tx.ID, tx.LastBroadcastAt, threshold) + return true + } + return false +} + +type APIResponse struct { + Status string `json:"status,omitempty"` + Hash common.Hash `json:"hash,omitempty"` +} + +const ( + APIStatusPending = "PENDING" + APIStatusIncluded = "INCLUDED" + APIStatusFailed = "FAILED" + APIStatusCancelled = "CANCELLED" + APIStatusUnknown = "UNKNOWN" +) + +func (s *stuckTxDetector) dualBroadcastDetection(ctx context.Context, tx *types.Transaction) (bool, error) { + for _, attempt := range tx.Attempts { + req, err := http.NewRequestWithContext(ctx, http.MethodGet, s.config.DetectionURL+attempt.Hash.String(), nil) + if err != nil { + return false, fmt.Errorf("failed to make request for txID: %v, attemptHash: %v - %w", tx.ID, attempt.Hash, err) + } + resp, err := http.DefaultClient.Do(req) + if err != nil { + return false, fmt.Errorf("failed to get transaction status for txID: %v, attemptHash: %v - %w", tx.ID, attempt.Hash, err) + } + if resp.StatusCode != http.StatusOK { + resp.Body.Close() + return false, fmt.Errorf("request %v failed with status: %d", req, resp.StatusCode) + } + body, err := io.ReadAll(resp.Body) + resp.Body.Close() + if err != nil { + return false, err + } + + var apiResponse APIResponse + err = json.Unmarshal(body, &apiResponse) + if err != nil { + return false, fmt.Errorf("failed to unmarshal response for txID: %v, attemptHash: %v - %w: %s", tx.ID, attempt.Hash, err, string(body)) + } + switch apiResponse.Status { + case APIStatusPending, APIStatusIncluded: + return false, nil + case APIStatusFailed, APIStatusCancelled: + s.lggr.Debugf("TxID: %v with attempHash: %v was marked as failed/cancelled by the RPC. Transaction is now considered stuck and will be purged.", + tx.ID, attempt.Hash) + return true, nil + case APIStatusUnknown: + continue + default: + continue + } + } + return false, nil +} diff --git a/core/chains/evm/txm/txm.go b/core/chains/evm/txm/txm.go index 4153f8072f1..ef8e682ab6a 100644 --- a/core/chains/evm/txm/txm.go +++ b/core/chains/evm/txm/txm.go @@ -8,7 +8,6 @@ import ( "time" "github.com/ethereum/go-ethereum/common" - evmtypes "github.com/ethereum/go-ethereum/core/types" "github.com/jpillora/backoff" "github.com/smartcontractkit/chainlink-common/pkg/logger" @@ -28,7 +27,7 @@ const ( type Client interface { PendingNonceAt(context.Context, common.Address) (uint64, error) NonceAt(context.Context, common.Address, *big.Int) (uint64, error) - SendTransaction(context.Context, *evmtypes.Transaction) error + SendTransaction(ctx context.Context, tx *types.Transaction, attempt *types.Attempt) error } type TxStore interface { @@ -57,7 +56,7 @@ type ErrorHandler interface { } type StuckTxDetector interface { - DetectStuckTransaction(tx *types.Transaction) (bool, error) + DetectStuckTransaction(ctx context.Context, tx *types.Transaction) (bool, error) } type Keystore interface { @@ -91,17 +90,18 @@ type Txm struct { wg sync.WaitGroup } -func NewTxm(lggr logger.Logger, chainID *big.Int, client Client, attemptBuilder AttemptBuilder, txStore TxStore, config Config, keystore Keystore) *Txm { +func NewTxm(lggr logger.Logger, chainID *big.Int, client Client, attemptBuilder AttemptBuilder, txStore TxStore, stuckTxDetector StuckTxDetector, config Config, keystore Keystore) *Txm { return &Txm{ - lggr: logger.Sugared(logger.Named(lggr, "Txm")), - keystore: keystore, - chainID: chainID, - client: client, - attemptBuilder: attemptBuilder, - txStore: txStore, - config: config, - nonceMap: make(map[common.Address]uint64), - triggerCh: make(map[common.Address]chan struct{}), + lggr: logger.Sugared(logger.Named(lggr, "Txm")), + keystore: keystore, + chainID: chainID, + client: client, + attemptBuilder: attemptBuilder, + txStore: txStore, + stuckTxDetector: stuckTxDetector, + config: config, + nonceMap: make(map[common.Address]uint64), + triggerCh: make(map[common.Address]chan struct{}), } } @@ -311,7 +311,7 @@ func (t *Txm) createAndSendAttempt(ctx context.Context, tx *types.Transaction, a func (t *Txm) sendTransactionWithError(ctx context.Context, tx *types.Transaction, attempt *types.Attempt, address common.Address) (err error) { start := time.Now() - txErr := t.client.SendTransaction(ctx, attempt.SignedTransaction) + txErr := t.client.SendTransaction(ctx, tx, attempt) tx.AttemptCount++ t.lggr.Infow("Broadcasted attempt", "tx", tx, "attempt", attempt, "duration", time.Since(start), "txErr: ", txErr) if txErr != nil && t.errorHandler != nil { @@ -360,7 +360,7 @@ func (t *Txm) backfillTransactions(ctx context.Context, address common.Address) return false, t.createAndSendEmptyTx(ctx, latestNonce, address) } else { //nolint:revive //linter nonsense if !tx.IsPurgeable && t.stuckTxDetector != nil { - isStuck, err := t.stuckTxDetector.DetectStuckTransaction(tx) + isStuck, err := t.stuckTxDetector.DetectStuckTransaction(ctx, tx) if err != nil { return false, err } diff --git a/core/chains/evm/txm/txm_test.go b/core/chains/evm/txm/txm_test.go index 8314bddb811..93742924d58 100644 --- a/core/chains/evm/txm/txm_test.go +++ b/core/chains/evm/txm/txm_test.go @@ -37,7 +37,7 @@ func TestLifecycle(t *testing.T) { keystore.On("EnabledAddressesForChain", mock.Anything, mock.Anything).Return(addresses, nil) t.Run("fails to start if initial pending nonce call fails", func(t *testing.T) { - txm := NewTxm(logger.Test(t), testutils.FixtureChainID, client, ab, nil, config, keystore) + txm := NewTxm(logger.Test(t), testutils.FixtureChainID, client, ab, nil, nil, config, keystore) client.On("PendingNonceAt", mock.Anything, address1).Return(uint64(0), errors.New("error")).Once() require.Error(t, txm.Start(tests.Context(t))) }) @@ -46,7 +46,7 @@ func TestLifecycle(t *testing.T) { lggr, observedLogs := logger.TestObserved(t, zap.DebugLevel) txStore := storage.NewInMemoryStoreManager(lggr, testutils.FixtureChainID) require.NoError(t, txStore.Add(addresses...)) - txm := NewTxm(lggr, testutils.FixtureChainID, client, ab, txStore, config, keystore) + txm := NewTxm(lggr, testutils.FixtureChainID, client, ab, txStore, nil, config, keystore) var nonce uint64 // Start client.On("PendingNonceAt", mock.Anything, address1).Return(nonce, nil).Once() @@ -67,7 +67,7 @@ func TestTrigger(t *testing.T) { keystore := mocks.NewKeystore(t) t.Run("Trigger fails if Txm is unstarted", func(t *testing.T) { lggr, observedLogs := logger.TestObserved(t, zap.ErrorLevel) - txm := NewTxm(lggr, nil, nil, nil, nil, Config{}, keystore) + txm := NewTxm(lggr, nil, nil, nil, nil, nil, Config{}, keystore) txm.Trigger(address) tests.AssertLogEventually(t, observedLogs, "Txm unstarted") }) @@ -80,7 +80,7 @@ func TestTrigger(t *testing.T) { ab := mocks.NewAttemptBuilder(t) config := Config{BlockTime: 1 * time.Minute, RetryBlockThreshold: 10} keystore.On("EnabledAddressesForChain", mock.Anything, mock.Anything).Return([]common.Address{address}, nil) - txm := NewTxm(lggr, testutils.FixtureChainID, client, ab, txStore, config, keystore) + txm := NewTxm(lggr, testutils.FixtureChainID, client, ab, txStore, nil, config, keystore) var nonce uint64 // Start client.On("PendingNonceAt", mock.Anything, address).Return(nonce, nil).Once() @@ -101,7 +101,7 @@ func TestBroadcastTransaction(t *testing.T) { t.Run("fails if FetchUnconfirmedTransactionAtNonceWithCount for unconfirmed transactions fails", func(t *testing.T) { mTxStore := mocks.NewTxStore(t) mTxStore.On("FetchUnconfirmedTransactionAtNonceWithCount", mock.Anything, mock.Anything, mock.Anything).Return(nil, 0, errors.New("call failed")).Once() - txm := NewTxm(logger.Test(t), testutils.FixtureChainID, client, ab, mTxStore, config, keystore) + txm := NewTxm(logger.Test(t), testutils.FixtureChainID, client, ab, mTxStore, nil, config, keystore) bo, err := txm.broadcastTransaction(ctx, address) require.Error(t, err) assert.False(t, bo) @@ -112,7 +112,7 @@ func TestBroadcastTransaction(t *testing.T) { lggr, observedLogs := logger.TestObserved(t, zap.DebugLevel) mTxStore := mocks.NewTxStore(t) mTxStore.On("FetchUnconfirmedTransactionAtNonceWithCount", mock.Anything, mock.Anything, mock.Anything).Return(nil, maxInFlightTransactions+1, nil).Once() - txm := NewTxm(lggr, testutils.FixtureChainID, client, ab, mTxStore, config, keystore) + txm := NewTxm(lggr, testutils.FixtureChainID, client, ab, mTxStore, nil, config, keystore) bo, err := txm.broadcastTransaction(ctx, address) assert.True(t, bo) require.NoError(t, err) @@ -122,7 +122,7 @@ func TestBroadcastTransaction(t *testing.T) { t.Run("checks pending nonce if unconfirmed transactions are more than 1/3 of maxInFlightTransactions", func(t *testing.T) { lggr, observedLogs := logger.TestObserved(t, zap.DebugLevel) mTxStore := mocks.NewTxStore(t) - txm := NewTxm(lggr, testutils.FixtureChainID, client, ab, mTxStore, config, keystore) + txm := NewTxm(lggr, testutils.FixtureChainID, client, ab, mTxStore, nil, config, keystore) txm.setNonce(address, 1) mTxStore.On("FetchUnconfirmedTransactionAtNonceWithCount", mock.Anything, mock.Anything, mock.Anything).Return(nil, maxInFlightTransactions/3, nil).Twice() @@ -142,7 +142,7 @@ func TestBroadcastTransaction(t *testing.T) { t.Run("fails if UpdateUnstartedTransactionWithNonce fails", func(t *testing.T) { mTxStore := mocks.NewTxStore(t) mTxStore.On("FetchUnconfirmedTransactionAtNonceWithCount", mock.Anything, mock.Anything, mock.Anything).Return(nil, 0, nil).Once() - txm := NewTxm(logger.Test(t), testutils.FixtureChainID, client, ab, mTxStore, config, keystore) + txm := NewTxm(logger.Test(t), testutils.FixtureChainID, client, ab, mTxStore, nil, config, keystore) mTxStore.On("UpdateUnstartedTransactionWithNonce", mock.Anything, mock.Anything, mock.Anything).Return(nil, errors.New("call failed")).Once() bo, err := txm.broadcastTransaction(ctx, address) assert.False(t, bo) @@ -154,7 +154,7 @@ func TestBroadcastTransaction(t *testing.T) { lggr := logger.Test(t) txStore := storage.NewInMemoryStoreManager(lggr, testutils.FixtureChainID) require.NoError(t, txStore.Add(address)) - txm := NewTxm(lggr, testutils.FixtureChainID, client, ab, txStore, config, keystore) + txm := NewTxm(lggr, testutils.FixtureChainID, client, ab, txStore, nil, config, keystore) bo, err := txm.broadcastTransaction(ctx, address) require.NoError(t, err) assert.False(t, bo) @@ -165,7 +165,7 @@ func TestBroadcastTransaction(t *testing.T) { lggr := logger.Test(t) txStore := storage.NewInMemoryStoreManager(lggr, testutils.FixtureChainID) require.NoError(t, txStore.Add(address)) - txm := NewTxm(lggr, testutils.FixtureChainID, client, ab, txStore, config, keystore) + txm := NewTxm(lggr, testutils.FixtureChainID, client, ab, txStore, nil, config, keystore) txm.setNonce(address, 8) IDK := "IDK" txRequest := &types.TxRequest{ @@ -183,7 +183,7 @@ func TestBroadcastTransaction(t *testing.T) { GasLimit: 22000, } ab.On("NewAttempt", mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(attempt, nil).Once() - client.On("SendTransaction", mock.Anything, mock.Anything).Return(nil).Once() + client.On("SendTransaction", mock.Anything, mock.Anything, mock.Anything).Return(nil).Once() bo, err := txm.broadcastTransaction(ctx, address) require.NoError(t, err) @@ -210,7 +210,7 @@ func TestBackfillTransactions(t *testing.T) { keystore := mocks.NewKeystore(t) t.Run("fails if latest nonce fetching fails", func(t *testing.T) { - txm := NewTxm(logger.Test(t), testutils.FixtureChainID, client, ab, storage, config, keystore) + txm := NewTxm(logger.Test(t), testutils.FixtureChainID, client, ab, storage, nil, config, keystore) client.On("NonceAt", mock.Anything, address, mock.Anything).Return(uint64(0), errors.New("latest nonce fail")).Once() bo, err := txm.backfillTransactions(ctx, address) require.Error(t, err) @@ -219,7 +219,7 @@ func TestBackfillTransactions(t *testing.T) { }) t.Run("fails if MarkTransactionsConfirmed fails", func(t *testing.T) { - txm := NewTxm(logger.Test(t), testutils.FixtureChainID, client, ab, storage, config, keystore) + txm := NewTxm(logger.Test(t), testutils.FixtureChainID, client, ab, storage, nil, config, keystore) client.On("NonceAt", mock.Anything, address, mock.Anything).Return(uint64(0), nil).Once() storage.On("MarkTransactionsConfirmed", mock.Anything, mock.Anything, address).Return([]uint64{}, []uint64{}, errors.New("marking transactions confirmed failed")).Once() bo, err := txm.backfillTransactions(ctx, address) diff --git a/core/chains/evm/txm/types/transaction.go b/core/chains/evm/txm/types/transaction.go index 94ddabce81b..d32b56628e3 100644 --- a/core/chains/evm/txm/types/transaction.go +++ b/core/chains/evm/txm/types/transaction.go @@ -170,6 +170,10 @@ type TxMeta struct { // CCIP MessageIDs []string `json:"MessageIDs,omitempty"` SeqNumbers []uint64 `json:"SeqNumbers,omitempty"` + + // Dual Broadcast + DualBroadcast *bool `json:"DualBroadcast,omitempty"` + DualBroadcastParams *string `json:"DualBroadcastParams,omitempty"` } type QueueingTxStrategy struct { diff --git a/core/chains/evm/txmgr/builder.go b/core/chains/evm/txmgr/builder.go index 85d2a894ddc..8e1644f3a73 100644 --- a/core/chains/evm/txmgr/builder.go +++ b/core/chains/evm/txmgr/builder.go @@ -19,6 +19,7 @@ import ( "github.com/smartcontractkit/chainlink/v2/core/chains/evm/keystore" "github.com/smartcontractkit/chainlink/v2/core/chains/evm/logpoller" "github.com/smartcontractkit/chainlink/v2/core/chains/evm/txm" + "github.com/smartcontractkit/chainlink/v2/core/chains/evm/txm/clientwrappers" "github.com/smartcontractkit/chainlink/v2/core/chains/evm/txm/storage" evmtypes "github.com/smartcontractkit/chainlink/v2/core/chains/evm/types" ) @@ -93,12 +94,12 @@ func NewEvmTxm( return txmgr.NewTxm(chainId, cfg, txCfg, keyStore, lggr, checkerFactory, fwdMgr, txAttemptBuilder, txStore, broadcaster, confirmer, resender, tracker, finalizer, client.NewTxError) } -func NewTxmv2( +func NewTxmV2( ds sqlutil.DataSource, chainConfig ChainConfig, fCfg FeeConfig, - blockTime time.Duration, - fwdEnabled bool, + txConfig config.Transactions, + txmV2Config config.TxmV2, client client.Client, lggr logger.Logger, logPoller logpoller.LogPoller, @@ -106,23 +107,40 @@ func NewTxmv2( estimator gas.EvmFeeEstimator, ) (TxManager, error) { var fwdMgr *forwarders.FwdMgr - if fwdEnabled { + if txConfig.ForwardersEnabled() { fwdMgr = forwarders.NewFwdMgr(ds, client, logPoller, lggr, chainConfig) } else { lggr.Info("ForwarderManager: Disabled") } chainID := client.ConfiguredChainID() + + var stuckTxDetector txm.StuckTxDetector + if txConfig.AutoPurge().Enabled() { + stuckTxDetectorConfig := txm.StuckTxDetectorConfig{ + BlockTime: *txmV2Config.BlockTime(), + StuckTxBlockThreshold: *txConfig.AutoPurge().Threshold(), + DetectionURL: txConfig.AutoPurge().DetectionApiUrl().String(), + } + stuckTxDetector = txm.NewStuckTxDetector(lggr, chainConfig.ChainType(), stuckTxDetectorConfig) + } + attemptBuilder := txm.NewAttemptBuilder(chainID, fCfg.PriceMax(), estimator, keyStore) inMemoryStoreManager := storage.NewInMemoryStoreManager(lggr, chainID) config := txm.Config{ EIP1559: fCfg.EIP1559DynamicFees(), - BlockTime: blockTime, //TODO: create new config - //nolint:gosec // we want to reuse the existing config until migrations + BlockTime: *txmV2Config.BlockTime(), + //nolint:gosec // reuse existing config until migration RetryBlockThreshold: uint16(fCfg.BumpThreshold()), EmptyTxLimitDefault: fCfg.LimitDefault(), } - t := txm.NewTxm(lggr, chainID, client, attemptBuilder, inMemoryStoreManager, config, keyStore) + var c txm.Client + if chainConfig.ChainType() == chaintype.ChainDualBroadcast { + c = clientwrappers.NewDualBroadcastClient(client, keyStore, txmV2Config.CustomURL()) + } else { + c = clientwrappers.NewChainClient(client) + } + t := txm.NewTxm(lggr, chainID, c, attemptBuilder, inMemoryStoreManager, stuckTxDetector, config, keyStore) return txm.NewTxmOrchestrator(lggr, chainID, t, inMemoryStoreManager, fwdMgr, keyStore, attemptBuilder), nil } diff --git a/core/chains/legacyevm/evm_txm.go b/core/chains/legacyevm/evm_txm.go index 3a96a9da937..1192462156e 100644 --- a/core/chains/legacyevm/evm_txm.go +++ b/core/chains/legacyevm/evm_txm.go @@ -55,20 +55,35 @@ func newEvmTxm( } if opts.GenTxManager == nil { - txm, err = txmgr.NewTxm( - ds, - cfg, - txmgr.NewEvmTxmFeeConfig(cfg.GasEstimator()), - cfg.Transactions(), - cfg.NodePool().Errors(), - databaseConfig, - listenerConfig, - client, - lggr, - logPoller, - opts.KeyStore, - estimator, - headTracker) + if cfg.TxmV2().Enabled() { + txm, err = txmgr.NewTxmV2( + ds, + cfg, + txmgr.NewEvmTxmFeeConfig(cfg.GasEstimator()), + cfg.Transactions(), + cfg.TxmV2(), + client, + lggr, + logPoller, + opts.KeyStore, + estimator, + ) + } else { + txm, err = txmgr.NewTxm( + ds, + cfg, + txmgr.NewEvmTxmFeeConfig(cfg.GasEstimator()), + cfg.Transactions(), + cfg.NodePool().Errors(), + databaseConfig, + listenerConfig, + client, + lggr, + logPoller, + opts.KeyStore, + estimator, + headTracker) + } } else { txm = opts.GenTxManager(chainID) } diff --git a/core/config/docs/chains-evm.toml b/core/config/docs/chains-evm.toml index 62360cb02cb..cd99027e611 100644 --- a/core/config/docs/chains-evm.toml +++ b/core/config/docs/chains-evm.toml @@ -129,6 +129,14 @@ ReaperThreshold = '168h' # Default # ResendAfterThreshold controls how long to wait before re-broadcasting a transaction that has not yet been confirmed. ResendAfterThreshold = '1m' # Default +[EVM.TxmV2] +# Enabled enables TxmV2. +Enabled = false # Default +# BlockTime controls the frequency of the backfill loop of TxmV2. +BlockTime = '10s' # Example +# CustomURL configures the base url of a custom endpoint used by the ChainDualBroadcast chain type. +CustomURL = 'https://example.api.io' # Example + [EVM.Transactions.AutoPurge] # Enabled enables or disables automatically purging transactions that have been idenitified as terminally stuck (will never be included on-chain). This feature is only expected to be used by ZK chains. Enabled = false # Default diff --git a/docs/CONFIG.md b/docs/CONFIG.md index 30ada2455ca..baf5832f9d7 100644 --- a/docs/CONFIG.md +++ b/docs/CONFIG.md @@ -2033,6 +2033,9 @@ RPCBlockQueryDelay = 1 FinalizedBlockOffset = 0 NoNewFinalizedHeadsThreshold = '9m0s' +[TxmV2] +Enabled = false + [Transactions] ForwardersEnabled = false MaxInFlight = 16 @@ -2137,6 +2140,9 @@ RPCBlockQueryDelay = 1 FinalizedBlockOffset = 0 NoNewFinalizedHeadsThreshold = '0s' +[TxmV2] +Enabled = false + [Transactions] ForwardersEnabled = false MaxInFlight = 16 @@ -2241,6 +2247,9 @@ RPCBlockQueryDelay = 1 FinalizedBlockOffset = 0 NoNewFinalizedHeadsThreshold = '0s' +[TxmV2] +Enabled = false + [Transactions] ForwardersEnabled = false MaxInFlight = 16 @@ -2345,6 +2354,9 @@ RPCBlockQueryDelay = 1 FinalizedBlockOffset = 0 NoNewFinalizedHeadsThreshold = '0s' +[TxmV2] +Enabled = false + [Transactions] ForwardersEnabled = false MaxInFlight = 16 @@ -2450,6 +2462,9 @@ RPCBlockQueryDelay = 1 FinalizedBlockOffset = 0 NoNewFinalizedHeadsThreshold = '13m0s' +[TxmV2] +Enabled = false + [Transactions] ForwardersEnabled = false MaxInFlight = 16 @@ -2558,6 +2573,9 @@ RPCBlockQueryDelay = 1 FinalizedBlockOffset = 0 NoNewFinalizedHeadsThreshold = '0s' +[TxmV2] +Enabled = false + [Transactions] ForwardersEnabled = false MaxInFlight = 16 @@ -2662,6 +2680,9 @@ RPCBlockQueryDelay = 1 FinalizedBlockOffset = 0 NoNewFinalizedHeadsThreshold = '0s' +[TxmV2] +Enabled = false + [Transactions] ForwardersEnabled = false MaxInFlight = 16 @@ -2767,6 +2788,9 @@ RPCBlockQueryDelay = 1 FinalizedBlockOffset = 0 NoNewFinalizedHeadsThreshold = '0s' +[TxmV2] +Enabled = false + [Transactions] ForwardersEnabled = false MaxInFlight = 16 @@ -2871,6 +2895,9 @@ RPCBlockQueryDelay = 2 FinalizedBlockOffset = 2 NoNewFinalizedHeadsThreshold = '45s' +[TxmV2] +Enabled = false + [Transactions] ForwardersEnabled = false MaxInFlight = 16 @@ -2974,6 +3001,9 @@ RPCBlockQueryDelay = 1 FinalizedBlockOffset = 0 NoNewFinalizedHeadsThreshold = '0s' +[TxmV2] +Enabled = false + [Transactions] ForwardersEnabled = false MaxInFlight = 16 @@ -3077,6 +3107,9 @@ RPCBlockQueryDelay = 1 FinalizedBlockOffset = 0 NoNewFinalizedHeadsThreshold = '0s' +[TxmV2] +Enabled = false + [Transactions] ForwardersEnabled = false MaxInFlight = 16 @@ -3181,6 +3214,9 @@ RPCBlockQueryDelay = 2 FinalizedBlockOffset = 2 NoNewFinalizedHeadsThreshold = '40s' +[TxmV2] +Enabled = false + [Transactions] ForwardersEnabled = false MaxInFlight = 16 @@ -3286,6 +3322,9 @@ RPCBlockQueryDelay = 1 FinalizedBlockOffset = 2 NoNewFinalizedHeadsThreshold = '2m0s' +[TxmV2] +Enabled = false + [Transactions] ForwardersEnabled = false MaxInFlight = 16 @@ -3390,6 +3429,9 @@ RPCBlockQueryDelay = 2 FinalizedBlockOffset = 0 NoNewFinalizedHeadsThreshold = '0s' +[TxmV2] +Enabled = false + [Transactions] ForwardersEnabled = false MaxInFlight = 16 @@ -3494,6 +3536,9 @@ RPCBlockQueryDelay = 10 FinalizedBlockOffset = 0 NoNewFinalizedHeadsThreshold = '6m0s' +[TxmV2] +Enabled = false + [Transactions] ForwardersEnabled = false MaxInFlight = 16 @@ -3598,6 +3643,9 @@ RPCBlockQueryDelay = 15 FinalizedBlockOffset = 2 NoNewFinalizedHeadsThreshold = '0s' +[TxmV2] +Enabled = false + [Transactions] ForwardersEnabled = false MaxInFlight = 16 @@ -3703,6 +3751,9 @@ RPCBlockQueryDelay = 15 FinalizedBlockOffset = 2 NoNewFinalizedHeadsThreshold = '0s' +[TxmV2] +Enabled = false + [Transactions] ForwardersEnabled = false MaxInFlight = 16 @@ -3916,6 +3967,9 @@ RPCBlockQueryDelay = 2 FinalizedBlockOffset = 0 NoNewFinalizedHeadsThreshold = '0s' +[TxmV2] +Enabled = false + [Transactions] ForwardersEnabled = false MaxInFlight = 16 @@ -4020,6 +4074,9 @@ RPCBlockQueryDelay = 1 FinalizedBlockOffset = 2 NoNewFinalizedHeadsThreshold = '0s' +[TxmV2] +Enabled = false + [Transactions] ForwardersEnabled = false MaxInFlight = 16 @@ -4128,6 +4185,9 @@ RPCBlockQueryDelay = 1 FinalizedBlockOffset = 0 NoNewFinalizedHeadsThreshold = '0s' +[TxmV2] +Enabled = false + [Transactions] ForwardersEnabled = false MaxInFlight = 16 @@ -4235,6 +4295,9 @@ RPCBlockQueryDelay = 1 FinalizedBlockOffset = 2 NoNewFinalizedHeadsThreshold = '0s' +[TxmV2] +Enabled = false + [Transactions] ForwardersEnabled = false MaxInFlight = 16 @@ -4339,6 +4402,9 @@ RPCBlockQueryDelay = 1 FinalizedBlockOffset = 2 NoNewFinalizedHeadsThreshold = '0s' +[TxmV2] +Enabled = false + [Transactions] ForwardersEnabled = false MaxInFlight = 16 @@ -4443,6 +4509,9 @@ RPCBlockQueryDelay = 1 FinalizedBlockOffset = 2 NoNewFinalizedHeadsThreshold = '0s' +[TxmV2] +Enabled = false + [Transactions] ForwardersEnabled = false MaxInFlight = 16 @@ -4550,6 +4619,9 @@ RPCBlockQueryDelay = 1 FinalizedBlockOffset = 2 NoNewFinalizedHeadsThreshold = '0s' +[TxmV2] +Enabled = false + [Transactions] ForwardersEnabled = false MaxInFlight = 16 @@ -4658,6 +4730,9 @@ RPCBlockQueryDelay = 1 FinalizedBlockOffset = 0 NoNewFinalizedHeadsThreshold = '0s' +[TxmV2] +Enabled = false + [Transactions] ForwardersEnabled = false MaxInFlight = 16 @@ -4874,6 +4949,9 @@ RPCBlockQueryDelay = 1 FinalizedBlockOffset = 0 NoNewFinalizedHeadsThreshold = '0s' +[TxmV2] +Enabled = false + [Transactions] ForwardersEnabled = false MaxInFlight = 16 @@ -4977,6 +5055,9 @@ RPCBlockQueryDelay = 1 FinalizedBlockOffset = 0 NoNewFinalizedHeadsThreshold = '0s' +[TxmV2] +Enabled = false + [Transactions] ForwardersEnabled = false MaxInFlight = 16 @@ -5081,6 +5162,9 @@ RPCBlockQueryDelay = 1 FinalizedBlockOffset = 0 NoNewFinalizedHeadsThreshold = '0s' +[TxmV2] +Enabled = false + [Transactions] ForwardersEnabled = false MaxInFlight = 16 @@ -5185,6 +5269,9 @@ RPCBlockQueryDelay = 15 FinalizedBlockOffset = 0 NoNewFinalizedHeadsThreshold = '0s' +[TxmV2] +Enabled = false + [Transactions] ForwardersEnabled = false MaxInFlight = 16 @@ -5290,6 +5377,9 @@ RPCBlockQueryDelay = 1 FinalizedBlockOffset = 2 NoNewFinalizedHeadsThreshold = '40s' +[TxmV2] +Enabled = false + [Transactions] ForwardersEnabled = false MaxInFlight = 16 @@ -5394,6 +5484,9 @@ RPCBlockQueryDelay = 1 FinalizedBlockOffset = 2 NoNewFinalizedHeadsThreshold = '40s' +[TxmV2] +Enabled = false + [Transactions] ForwardersEnabled = false MaxInFlight = 16 @@ -5713,6 +5806,9 @@ RPCBlockQueryDelay = 1 FinalizedBlockOffset = 0 NoNewFinalizedHeadsThreshold = '0s' +[TxmV2] +Enabled = false + [Transactions] ForwardersEnabled = false MaxInFlight = 16 @@ -5818,6 +5914,9 @@ RPCBlockQueryDelay = 1 FinalizedBlockOffset = 0 NoNewFinalizedHeadsThreshold = '2h0m0s' +[TxmV2] +Enabled = false + [Transactions] ForwardersEnabled = false MaxInFlight = 16 @@ -5926,6 +6025,9 @@ RPCBlockQueryDelay = 1 FinalizedBlockOffset = 2 NoNewFinalizedHeadsThreshold = '0s' +[TxmV2] +Enabled = false + [Transactions] ForwardersEnabled = false MaxInFlight = 16 @@ -6034,6 +6136,9 @@ RPCBlockQueryDelay = 1 FinalizedBlockOffset = 0 NoNewFinalizedHeadsThreshold = '0s' +[TxmV2] +Enabled = false + [Transactions] ForwardersEnabled = false MaxInFlight = 16 @@ -6139,6 +6244,9 @@ RPCBlockQueryDelay = 2 FinalizedBlockOffset = 0 NoNewFinalizedHeadsThreshold = '0s' +[TxmV2] +Enabled = false + [Transactions] ForwardersEnabled = false MaxInFlight = 16 @@ -6350,6 +6458,9 @@ RPCBlockQueryDelay = 1 FinalizedBlockOffset = 0 NoNewFinalizedHeadsThreshold = '0s' +[TxmV2] +Enabled = false + [Transactions] ForwardersEnabled = false MaxInFlight = 16 @@ -6454,6 +6565,9 @@ RPCBlockQueryDelay = 1 FinalizedBlockOffset = 0 NoNewFinalizedHeadsThreshold = '15m0s' +[TxmV2] +Enabled = false + [Transactions] ForwardersEnabled = false MaxInFlight = 16 @@ -6562,6 +6676,9 @@ RPCBlockQueryDelay = 1 FinalizedBlockOffset = 2 NoNewFinalizedHeadsThreshold = '2m0s' +[TxmV2] +Enabled = false + [Transactions] ForwardersEnabled = false MaxInFlight = 16 @@ -6667,6 +6784,9 @@ RPCBlockQueryDelay = 1 FinalizedBlockOffset = 0 NoNewFinalizedHeadsThreshold = '0s' +[TxmV2] +Enabled = false + [Transactions] ForwardersEnabled = false MaxInFlight = 16 @@ -6775,6 +6895,9 @@ RPCBlockQueryDelay = 1 FinalizedBlockOffset = 0 NoNewFinalizedHeadsThreshold = '0s' +[TxmV2] +Enabled = false + [Transactions] ForwardersEnabled = false MaxInFlight = 16 @@ -6883,6 +7006,9 @@ RPCBlockQueryDelay = 1 FinalizedBlockOffset = 0 NoNewFinalizedHeadsThreshold = '0s' +[TxmV2] +Enabled = false + [Transactions] ForwardersEnabled = false MaxInFlight = 16 @@ -6990,6 +7116,9 @@ RPCBlockQueryDelay = 1 FinalizedBlockOffset = 2 NoNewFinalizedHeadsThreshold = '1m0s' +[TxmV2] +Enabled = false + [Transactions] ForwardersEnabled = false MaxInFlight = 16 @@ -7094,6 +7223,9 @@ RPCBlockQueryDelay = 2 FinalizedBlockOffset = 2 NoNewFinalizedHeadsThreshold = '1m0s' +[TxmV2] +Enabled = false + [Transactions] ForwardersEnabled = false MaxInFlight = 16 @@ -7198,6 +7330,9 @@ RPCBlockQueryDelay = 2 FinalizedBlockOffset = 2 NoNewFinalizedHeadsThreshold = '1m0s' +[TxmV2] +Enabled = false + [Transactions] ForwardersEnabled = false MaxInFlight = 16 @@ -7302,6 +7437,9 @@ RPCBlockQueryDelay = 1 FinalizedBlockOffset = 0 NoNewFinalizedHeadsThreshold = '45m0s' +[TxmV2] +Enabled = false + [Transactions] ForwardersEnabled = false MaxInFlight = 16 @@ -7407,6 +7545,9 @@ RPCBlockQueryDelay = 1 FinalizedBlockOffset = 0 NoNewFinalizedHeadsThreshold = '15m0s' +[TxmV2] +Enabled = false + [Transactions] ForwardersEnabled = false MaxInFlight = 16 @@ -7518,6 +7659,9 @@ RPCBlockQueryDelay = 1 FinalizedBlockOffset = 0 NoNewFinalizedHeadsThreshold = '15m0s' +[TxmV2] +Enabled = false + [Transactions] ForwardersEnabled = false MaxInFlight = 16 @@ -7627,6 +7771,9 @@ RPCBlockQueryDelay = 1 FinalizedBlockOffset = 0 NoNewFinalizedHeadsThreshold = '0s' +[TxmV2] +Enabled = false + [Transactions] ForwardersEnabled = false MaxInFlight = 16 @@ -7730,6 +7877,9 @@ RPCBlockQueryDelay = 1 FinalizedBlockOffset = 0 NoNewFinalizedHeadsThreshold = '0s' +[TxmV2] +Enabled = false + [Transactions] ForwardersEnabled = false MaxInFlight = 16 @@ -7835,6 +7985,9 @@ RPCBlockQueryDelay = 1 FinalizedBlockOffset = 0 NoNewFinalizedHeadsThreshold = '0s' +[TxmV2] +Enabled = false + [Transactions] ForwardersEnabled = false MaxInFlight = 16 @@ -7941,6 +8094,9 @@ RPCBlockQueryDelay = 1 FinalizedBlockOffset = 0 NoNewFinalizedHeadsThreshold = '0s' +[TxmV2] +Enabled = false + [Transactions] ForwardersEnabled = false MaxInFlight = 16 @@ -8153,6 +8309,9 @@ RPCBlockQueryDelay = 10 FinalizedBlockOffset = 0 NoNewFinalizedHeadsThreshold = '0s' +[TxmV2] +Enabled = false + [Transactions] ForwardersEnabled = false MaxInFlight = 16 @@ -8256,6 +8415,9 @@ RPCBlockQueryDelay = 10 FinalizedBlockOffset = 0 NoNewFinalizedHeadsThreshold = '12m0s' +[TxmV2] +Enabled = false + [Transactions] ForwardersEnabled = false MaxInFlight = 16 @@ -8463,6 +8625,9 @@ RPCBlockQueryDelay = 1 FinalizedBlockOffset = 0 NoNewFinalizedHeadsThreshold = '0s' +[TxmV2] +Enabled = false + [Transactions] ForwardersEnabled = false MaxInFlight = 16 @@ -8571,6 +8736,9 @@ RPCBlockQueryDelay = 1 FinalizedBlockOffset = 0 NoNewFinalizedHeadsThreshold = '12m0s' +[TxmV2] +Enabled = false + [Transactions] ForwardersEnabled = false MaxInFlight = 16 @@ -8680,6 +8848,9 @@ RPCBlockQueryDelay = 1 FinalizedBlockOffset = 0 NoNewFinalizedHeadsThreshold = '0s' +[TxmV2] +Enabled = false + [Transactions] ForwardersEnabled = false MaxInFlight = 16 @@ -8788,6 +8959,9 @@ RPCBlockQueryDelay = 1 FinalizedBlockOffset = 0 NoNewFinalizedHeadsThreshold = '0s' +[TxmV2] +Enabled = false + [Transactions] ForwardersEnabled = false MaxInFlight = 16 @@ -8895,6 +9069,9 @@ RPCBlockQueryDelay = 1 FinalizedBlockOffset = 0 NoNewFinalizedHeadsThreshold = '0s' +[TxmV2] +Enabled = false + [Transactions] ForwardersEnabled = false MaxInFlight = 16 @@ -9002,6 +9179,9 @@ RPCBlockQueryDelay = 1 FinalizedBlockOffset = 0 NoNewFinalizedHeadsThreshold = '0s' +[TxmV2] +Enabled = false + [Transactions] ForwardersEnabled = false MaxInFlight = 16 @@ -9111,6 +9291,9 @@ RPCBlockQueryDelay = 1 FinalizedBlockOffset = 0 NoNewFinalizedHeadsThreshold = '0s' +[TxmV2] +Enabled = false + [Transactions] ForwardersEnabled = false MaxInFlight = 16 @@ -9328,6 +9511,9 @@ RPCBlockQueryDelay = 1 FinalizedBlockOffset = 0 NoNewFinalizedHeadsThreshold = '0s' +[TxmV2] +Enabled = false + [Transactions] ForwardersEnabled = false MaxInFlight = 16 @@ -9432,6 +9618,9 @@ RPCBlockQueryDelay = 1 FinalizedBlockOffset = 0 NoNewFinalizedHeadsThreshold = '15m0s' +[TxmV2] +Enabled = false + [Transactions] ForwardersEnabled = false MaxInFlight = 16 @@ -9540,6 +9729,9 @@ RPCBlockQueryDelay = 1 FinalizedBlockOffset = 0 NoNewFinalizedHeadsThreshold = '0s' +[TxmV2] +Enabled = false + [Transactions] ForwardersEnabled = false MaxInFlight = 16 @@ -9644,6 +9836,9 @@ RPCBlockQueryDelay = 1 FinalizedBlockOffset = 0 NoNewFinalizedHeadsThreshold = '0s' +[TxmV2] +Enabled = false + [Transactions] ForwardersEnabled = false MaxInFlight = 16 @@ -9985,6 +10180,33 @@ ResendAfterThreshold = '1m' # Default ``` ResendAfterThreshold controls how long to wait before re-broadcasting a transaction that has not yet been confirmed. +## EVM.TxmV2 +```toml +[EVM.TxmV2] +Enabled = false # Default +BlockTime = '10s' # Example +CustomURL = 'https://example.api.io' # Example +``` + + +### Enabled +```toml +Enabled = false # Default +``` +Enabled enables TxmV2. + +### BlockTime +```toml +BlockTime = '10s' # Example +``` +BlockTime controls the frequency of the backfill loop of TxmV2. + +### CustomURL +```toml +CustomURL = 'https://example.api.io' # Example +``` +CustomURL configures the base url of a custom endpoint used by the ChainDualBroadcast chain type. + ## EVM.Transactions.AutoPurge ```toml [EVM.Transactions.AutoPurge]