From 99cb0b67eaab4acf15b942c23ec5f56c4b6aa471 Mon Sep 17 00:00:00 2001 From: Farber98 Date: Wed, 3 Jul 2024 14:10:52 -0300 Subject: [PATCH 01/37] removing finality within common/txmgr --- common/txmgr/confirmer.go | 16 ++++++++-------- common/txmgr/types/tx_store.go | 2 +- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/common/txmgr/confirmer.go b/common/txmgr/confirmer.go index a9e30ffff1e..3b10d330c3d 100644 --- a/common/txmgr/confirmer.go +++ b/common/txmgr/confirmer.go @@ -297,7 +297,7 @@ func (ec *Confirmer[CHAIN_ID, HEAD, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) pro return fmt.Errorf("CheckConfirmedMissingReceipt failed: %w", err) } - if err := ec.CheckForReceipts(ctx, head.BlockNumber()); err != nil { + if err := ec.CheckForReceipts(ctx, head.BlockNumber(), head.LatestFinalizedHead().BlockNumber()); err != nil { return fmt.Errorf("CheckForReceipts failed: %w", err) } @@ -395,8 +395,8 @@ func (ec *Confirmer[CHAIN_ID, HEAD, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) Che return } -// CheckForReceipts finds attempts that are still pending and checks to see if a receipt is present for the given block number -func (ec *Confirmer[CHAIN_ID, HEAD, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) CheckForReceipts(ctx context.Context, blockNum int64) error { +// CheckForReceipts finds attempts that are still pending and checks to see if a receipt is present for the given block number. +func (ec *Confirmer[CHAIN_ID, HEAD, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) CheckForReceipts(ctx context.Context, blockNum int64, latestFinalizedBlockNum int64) error { attempts, err := ec.txStore.FindTxAttemptsRequiringReceiptFetch(ctx, ec.chainID) if err != nil { return fmt.Errorf("FindTxAttemptsRequiringReceiptFetch failed: %w", err) @@ -443,7 +443,7 @@ func (ec *Confirmer[CHAIN_ID, HEAD, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) Che return fmt.Errorf("unable to mark txes as 'confirmed_missing_receipt': %w", err) } - if err := ec.txStore.MarkOldTxesMissingReceiptAsErrored(ctx, blockNum, ec.chainConfig.FinalityDepth(), ec.chainID); err != nil { + if err := ec.txStore.MarkOldTxesMissingReceiptAsErrored(ctx, blockNum, latestFinalizedBlockNum, ec.chainID); err != nil { return fmt.Errorf("unable to confirm buried unconfirmed txes': %w", err) } return nil @@ -1007,15 +1007,15 @@ func (ec *Confirmer[CHAIN_ID, HEAD, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) han // If any of the confirmed transactions does not have a receipt in the chain, it has been // re-org'd out and will be rebroadcast. func (ec *Confirmer[CHAIN_ID, HEAD, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) EnsureConfirmedTransactionsInLongestChain(ctx context.Context, head types.Head[BLOCK_HASH]) error { - if head.ChainLength() < ec.chainConfig.FinalityDepth() { + if head.ChainLength() < uint32(head.LatestFinalizedHead().BlockNumber()) { logArgs := []interface{}{ - "chainLength", head.ChainLength(), "finalityDepth", ec.chainConfig.FinalityDepth(), + "chainLength", head.ChainLength(), "latestFinalizedHead", head.LatestFinalizedHead().BlockNumber(), } if ec.nConsecutiveBlocksChainTooShort > logAfterNConsecutiveBlocksChainTooShort { - warnMsg := "Chain length supplied for re-org detection was shorter than FinalityDepth. Re-org protection is not working properly. This could indicate a problem with the remote RPC endpoint, a compatibility issue with a particular blockchain, a bug with this particular blockchain, heads table being truncated too early, remote node out of sync, or something else. If this happens a lot please raise a bug with the Chainlink team including a log output sample and details of the chain and RPC endpoint you are using." + warnMsg := "Chain length supplied for re-org detection was shorter than LatestFinalizedHead. Re-org protection is not working properly. This could indicate a problem with the remote RPC endpoint, a compatibility issue with a particular blockchain, a bug with this particular blockchain, heads table being truncated too early, remote node out of sync, or something else. If this happens a lot please raise a bug with the Chainlink team including a log output sample and details of the chain and RPC endpoint you are using." ec.lggr.Warnw(warnMsg, append(logArgs, "nConsecutiveBlocksChainTooShort", ec.nConsecutiveBlocksChainTooShort)...) } else { - logMsg := "Chain length supplied for re-org detection was shorter than FinalityDepth" + logMsg := "Chain length supplied for re-org detection was shorter than LatestFinalizedHead" ec.lggr.Debugw(logMsg, append(logArgs, "nConsecutiveBlocksChainTooShort", ec.nConsecutiveBlocksChainTooShort)...) } ec.nConsecutiveBlocksChainTooShort++ diff --git a/common/txmgr/types/tx_store.go b/common/txmgr/types/tx_store.go index c102fb5912a..e1a694a472a 100644 --- a/common/txmgr/types/tx_store.go +++ b/common/txmgr/types/tx_store.go @@ -91,7 +91,7 @@ type TransactionStore[ HasInProgressTransaction(ctx context.Context, account ADDR, chainID CHAIN_ID) (exists bool, err error) LoadTxAttempts(ctx context.Context, etx *Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]) error MarkAllConfirmedMissingReceipt(ctx context.Context, chainID CHAIN_ID) (err error) - MarkOldTxesMissingReceiptAsErrored(ctx context.Context, blockNum int64, finalityDepth uint32, chainID CHAIN_ID) error + MarkOldTxesMissingReceiptAsErrored(ctx context.Context, blockNum int64, latestFinalizedBlockNum int64, chainID CHAIN_ID) error PreloadTxes(ctx context.Context, attempts []TxAttempt[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]) error SaveConfirmedMissingReceiptAttempt(ctx context.Context, timeout time.Duration, attempt *TxAttempt[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], broadcastAt time.Time) error SaveInProgressAttempt(ctx context.Context, attempt *TxAttempt[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]) error From 72f430e633f3e23f83fa85441b9b53c9fb33e33e Mon Sep 17 00:00:00 2001 From: Farber98 Date: Wed, 3 Jul 2024 14:12:24 -0300 Subject: [PATCH 02/37] remove finality from common/txmgr comment --- common/txmgr/confirmer.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/common/txmgr/confirmer.go b/common/txmgr/confirmer.go index 3b10d330c3d..8f8b86d1e28 100644 --- a/common/txmgr/confirmer.go +++ b/common/txmgr/confirmer.go @@ -1000,7 +1000,7 @@ func (ec *Confirmer[CHAIN_ID, HEAD, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) han } } -// EnsureConfirmedTransactionsInLongestChain finds all confirmed txes up to the depth +// EnsureConfirmedTransactionsInLongestChain finds all confirmed txes up to the LatestFinalizedHead // of the given chain and ensures that every one has a receipt with a block hash that is // in the given chain. // From 2de27e3ebb366da75adf42b5a4de80b0a5245e0c Mon Sep 17 00:00:00 2001 From: Farber98 Date: Wed, 3 Jul 2024 14:27:21 -0300 Subject: [PATCH 03/37] latestFinalizedBlockNum support within evm/txmgr txstore --- core/chains/evm/txmgr/evm_tx_store.go | 15 +++++---------- 1 file changed, 5 insertions(+), 10 deletions(-) diff --git a/core/chains/evm/txmgr/evm_tx_store.go b/core/chains/evm/txmgr/evm_tx_store.go index fd38eb7a8c9..098bcffa223 100644 --- a/core/chains/evm/txmgr/evm_tx_store.go +++ b/core/chains/evm/txmgr/evm_tx_store.go @@ -1446,23 +1446,18 @@ ORDER BY nonce ASC // markOldTxesMissingReceiptAsErrored // -// Once eth_tx has all of its attempts broadcast before some cutoff threshold +// Once eth_tx has all of its attempts broadcast before latestFinalizedBlockNum // without receiving any receipts, we mark it as fatally errored (never sent). // // The job run will also be marked as errored in this case since we never got a // receipt and thus cannot pass on any transaction hash -func (o *evmTxStore) MarkOldTxesMissingReceiptAsErrored(ctx context.Context, blockNum int64, finalityDepth uint32, chainID *big.Int) error { +func (o *evmTxStore) MarkOldTxesMissingReceiptAsErrored(ctx context.Context, blockNum int64, latestFinalizedBlockNum int64, chainID *big.Int) error { var cancel context.CancelFunc ctx, cancel = o.stopCh.Ctx(ctx) defer cancel() - // cutoffBlockNum is a block height - // Any 'confirmed_missing_receipt' eth_tx with all attempts older than this block height will be marked as errored + // Any 'confirmed_missing_receipt' eth_tx with all attempts older than latestFinalizedBlockNum will be marked as errored // We will not try to query for receipts for this transaction any more - cutoff := blockNum - int64(finalityDepth) - if cutoff <= 0 { - return nil - } - if cutoff <= 0 { + if latestFinalizedBlockNum <= 0 { return nil } // note: if QOpt passes in a sql.Tx this will reuse it @@ -1487,7 +1482,7 @@ FROM ( FOR UPDATE OF e1 ) e0 WHERE e0.id = evm.txes.id -RETURNING e0.id, e0.nonce`, ErrCouldNotGetReceipt, cutoff, chainID.String()) +RETURNING e0.id, e0.nonce`, ErrCouldNotGetReceipt, latestFinalizedBlockNum, chainID.String()) if err != nil { return pkgerrors.Wrap(err, "markOldTxesMissingReceiptAsErrored failed to query") From fe429455b847f68df3dbb071b62de1c6dca37c1f Mon Sep 17 00:00:00 2001 From: Farber98 Date: Wed, 3 Jul 2024 14:28:51 -0300 Subject: [PATCH 04/37] refactor tests after evm/txmgr txstore changes --- core/chains/evm/txmgr/confirmer_test.go | 95 ++++++++++++---------- core/chains/evm/txmgr/evm_tx_store_test.go | 7 +- 2 files changed, 56 insertions(+), 46 deletions(-) diff --git a/core/chains/evm/txmgr/confirmer_test.go b/core/chains/evm/txmgr/confirmer_test.go index 6b107b222a6..9ae14e70aa3 100644 --- a/core/chains/evm/txmgr/confirmer_test.go +++ b/core/chains/evm/txmgr/confirmer_test.go @@ -152,9 +152,10 @@ func TestEthConfirmer_Lifecycle(t *testing.T) { Hash: testutils.NewHash(), Number: 9, Parent: &evmtypes.Head{ - Number: 8, - Hash: testutils.NewHash(), - Parent: nil, + Number: 8, + Hash: testutils.NewHash(), + IsFinalized: true, + Parent: nil, }, }, } @@ -199,6 +200,7 @@ func TestEthConfirmer_CheckForReceipts(t *testing.T) { nonce := int64(0) ctx := tests.Context(t) blockNum := int64(0) + latestFinalizedBlockNum := int64(0) t.Run("only finds eth_txes in unconfirmed state with at least one broadcast attempt", func(t *testing.T) { mustInsertFatalErrorEthTx(t, txStore, fromAddress) @@ -211,7 +213,7 @@ func TestEthConfirmer_CheckForReceipts(t *testing.T) { mustCreateUnstartedGeneratedTx(t, txStore, fromAddress, config.EVM().ChainID()) // Do the thing - require.NoError(t, ec.CheckForReceipts(ctx, blockNum)) + require.NoError(t, ec.CheckForReceipts(ctx, blockNum, latestFinalizedBlockNum)) }) etx1 := cltest.MustInsertUnconfirmedEthTxWithBroadcastLegacyAttempt(t, txStore, nonce, fromAddress) @@ -232,7 +234,7 @@ func TestEthConfirmer_CheckForReceipts(t *testing.T) { }).Once() // Do the thing - require.NoError(t, ec.CheckForReceipts(ctx, blockNum)) + require.NoError(t, ec.CheckForReceipts(ctx, blockNum, latestFinalizedBlockNum)) var err error etx1, err = txStore.FindTxWithAttempts(ctx, etx1.ID) @@ -261,7 +263,7 @@ func TestEthConfirmer_CheckForReceipts(t *testing.T) { }).Once() // No error because it is merely logged - require.NoError(t, ec.CheckForReceipts(ctx, blockNum)) + require.NoError(t, ec.CheckForReceipts(ctx, blockNum, latestFinalizedBlockNum)) etx, err := txStore.FindTxWithAttempts(ctx, etx1.ID) require.NoError(t, err) @@ -289,7 +291,7 @@ func TestEthConfirmer_CheckForReceipts(t *testing.T) { }).Once() // No error because it is merely logged - require.NoError(t, ec.CheckForReceipts(ctx, blockNum)) + require.NoError(t, ec.CheckForReceipts(ctx, blockNum, latestFinalizedBlockNum)) etx, err := txStore.FindTxWithAttempts(ctx, etx1.ID) require.NoError(t, err) @@ -326,7 +328,7 @@ func TestEthConfirmer_CheckForReceipts(t *testing.T) { }).Once() // Do the thing - require.NoError(t, ec.CheckForReceipts(ctx, blockNum)) + require.NoError(t, ec.CheckForReceipts(ctx, blockNum, latestFinalizedBlockNum)) // Check that the receipt was saved etx, err := txStore.FindTxWithAttempts(ctx, etx1.ID) @@ -388,7 +390,7 @@ func TestEthConfirmer_CheckForReceipts(t *testing.T) { }).Once() // Do the thing - require.NoError(t, ec.CheckForReceipts(ctx, blockNum)) + require.NoError(t, ec.CheckForReceipts(ctx, blockNum, latestFinalizedBlockNum)) // Check that the state was updated etx, err := txStore.FindTxWithAttempts(ctx, etx2.ID) @@ -416,7 +418,7 @@ func TestEthConfirmer_CheckForReceipts(t *testing.T) { }).Once() // Do the thing - require.NoError(t, ec.CheckForReceipts(ctx, blockNum)) + require.NoError(t, ec.CheckForReceipts(ctx, blockNum, latestFinalizedBlockNum)) // No receipt, but no error either etx, err := txStore.FindTxWithAttempts(ctx, etx3.ID) @@ -443,7 +445,7 @@ func TestEthConfirmer_CheckForReceipts(t *testing.T) { }).Once() // Do the thing - require.NoError(t, ec.CheckForReceipts(ctx, blockNum)) + require.NoError(t, ec.CheckForReceipts(ctx, blockNum, latestFinalizedBlockNum)) // No receipt, but no error either etx, err := txStore.FindTxWithAttempts(ctx, etx3.ID) @@ -472,7 +474,7 @@ func TestEthConfirmer_CheckForReceipts(t *testing.T) { }).Once() // Do the thing - require.NoError(t, ec.CheckForReceipts(ctx, blockNum)) + require.NoError(t, ec.CheckForReceipts(ctx, blockNum, latestFinalizedBlockNum)) // Check that the receipt was unchanged etx, err := txStore.FindTxWithAttempts(ctx, etx3.ID) @@ -523,7 +525,7 @@ func TestEthConfirmer_CheckForReceipts(t *testing.T) { }).Once() // Do the thing - require.NoError(t, ec.CheckForReceipts(ctx, blockNum)) + require.NoError(t, ec.CheckForReceipts(ctx, blockNum, latestFinalizedBlockNum)) // Check that the state was updated var err error @@ -576,7 +578,7 @@ func TestEthConfirmer_CheckForReceipts(t *testing.T) { }).Once() // Do the thing - require.NoError(t, ec.CheckForReceipts(ctx, blockNum)) + require.NoError(t, ec.CheckForReceipts(ctx, blockNum, latestFinalizedBlockNum)) // Check that the state was updated etx5, err = txStore.FindTxWithAttempts(ctx, etx5.ID) @@ -614,6 +616,7 @@ func TestEthConfirmer_CheckForReceipts_batching(t *testing.T) { etx := cltest.MustInsertUnconfirmedEthTx(t, txStore, 0, fromAddress) var attempts []txmgr.TxAttempt + latestFinalizedBlockNum := int64(0) // Total of 5 attempts should lead to 3 batched fetches (2, 2, 1) for i := 0; i < 5; i++ { @@ -650,7 +653,7 @@ func TestEthConfirmer_CheckForReceipts_batching(t *testing.T) { elems[0].Result = &evmtypes.Receipt{} }).Once() - require.NoError(t, ec.CheckForReceipts(ctx, 42)) + require.NoError(t, ec.CheckForReceipts(ctx, 42, latestFinalizedBlockNum)) } func TestEthConfirmer_CheckForReceipts_HandlesNonFwdTxsWithForwardingEnabled(t *testing.T) { @@ -671,6 +674,8 @@ func TestEthConfirmer_CheckForReceipts_HandlesNonFwdTxsWithForwardingEnabled(t * _, fromAddress := cltest.MustInsertRandomKeyReturningState(t, ethKeyStore) ec := newEthConfirmer(t, txStore, ethClient, cfg, evmcfg, ethKeyStore, nil) ctx := tests.Context(t) + latestFinalizedBlockNum := int64(0) + // tx is not forwarded and doesn't have meta set. EthConfirmer should handle nil meta values etx := cltest.MustInsertUnconfirmedEthTx(t, txStore, 0, fromAddress) attempt := newBroadcastLegacyEthTxAttempt(t, etx.ID, 2) @@ -697,7 +702,7 @@ func TestEthConfirmer_CheckForReceipts_HandlesNonFwdTxsWithForwardingEnabled(t * *(elems[0].Result.(*evmtypes.Receipt)) = txmReceipt // confirmed }).Once() - require.NoError(t, ec.CheckForReceipts(ctx, 42)) + require.NoError(t, ec.CheckForReceipts(ctx, 42, latestFinalizedBlockNum)) // Check receipt is inserted correctly. dbtx, err = txStore.FindTxWithAttempts(ctx, etx.ID) @@ -724,6 +729,7 @@ func TestEthConfirmer_CheckForReceipts_only_likely_confirmed(t *testing.T) { ec := newEthConfirmer(t, txStore, ethClient, cfg, evmcfg, ethKeyStore, nil) ctx := tests.Context(t) + latestFinalizedBlockNum := int64(0) var attempts []txmgr.TxAttempt // inserting in DESC nonce order to test DB ASC ordering @@ -755,7 +761,7 @@ func TestEthConfirmer_CheckForReceipts_only_likely_confirmed(t *testing.T) { elems[3].Result = &evmtypes.Receipt{} }).Once() - require.NoError(t, ec.CheckForReceipts(ctx, 42)) + require.NoError(t, ec.CheckForReceipts(ctx, 42, latestFinalizedBlockNum)) cltest.BatchElemMustMatchParams(t, captured[0], attempts[0].Hash, "eth_getTransactionReceipt") cltest.BatchElemMustMatchParams(t, captured[1], attempts[1].Hash, "eth_getTransactionReceipt") @@ -778,6 +784,7 @@ func TestEthConfirmer_CheckForReceipts_should_not_check_for_likely_unconfirmed(t ec := newEthConfirmer(t, txStore, ethClient, gconfig, config, ethKeyStore, nil) ctx := tests.Context(t) + latestFinalizedBlockNum := int64(0) etx := cltest.MustInsertUnconfirmedEthTx(t, txStore, 1, fromAddress) for i := 0; i < 4; i++ { @@ -788,7 +795,7 @@ func TestEthConfirmer_CheckForReceipts_should_not_check_for_likely_unconfirmed(t // latest nonce is lower that all attempts' nonces ethClient.On("SequenceAt", mock.Anything, mock.Anything, mock.Anything).Return(evmtypes.Nonce(0), nil) - require.NoError(t, ec.CheckForReceipts(ctx, 42)) + require.NoError(t, ec.CheckForReceipts(ctx, 42, latestFinalizedBlockNum)) } func TestEthConfirmer_CheckForReceipts_confirmed_missing_receipt_scoped_to_key(t *testing.T) { @@ -809,6 +816,7 @@ func TestEthConfirmer_CheckForReceipts_confirmed_missing_receipt_scoped_to_key(t ec := newEthConfirmer(t, txStore, ethClient, cfg, evmcfg, ethKeyStore, nil) ctx := tests.Context(t) + latestFinalizedBlockNum := int64(0) // STATE // key 1, tx with nonce 0 is unconfirmed @@ -832,7 +840,7 @@ func TestEthConfirmer_CheckForReceipts_confirmed_missing_receipt_scoped_to_key(t *(elems[0].Result.(*evmtypes.Receipt)) = txmReceipt2_9 }).Once() - require.NoError(t, ec.CheckForReceipts(ctx, 10)) + require.NoError(t, ec.CheckForReceipts(ctx, 10, latestFinalizedBlockNum)) mustTxBeInState(t, txStore, etx1_0, txmgrcommon.TxUnconfirmed) mustTxBeInState(t, txStore, etx1_1, txmgrcommon.TxUnconfirmed) @@ -850,7 +858,7 @@ func TestEthConfirmer_CheckForReceipts_confirmed_missing_receipt_scoped_to_key(t *(elems[0].Result.(*evmtypes.Receipt)) = txmReceipt1_1 }).Once() - require.NoError(t, ec.CheckForReceipts(ctx, 11)) + require.NoError(t, ec.CheckForReceipts(ctx, 11, latestFinalizedBlockNum)) mustTxBeInState(t, txStore, etx1_0, txmgrcommon.TxConfirmedMissingReceipt) mustTxBeInState(t, txStore, etx1_1, txmgrcommon.TxConfirmed) @@ -861,9 +869,7 @@ func TestEthConfirmer_CheckForReceipts_confirmed_missing_receipt(t *testing.T) { t.Parallel() db := pgtest.NewSqlxDB(t) - cfg := configtest.NewGeneralConfig(t, func(c *chainlink.Config, s *chainlink.Secrets) { - c.EVM[0].FinalityDepth = ptr[uint32](50) - }) + cfg := configtest.NewGeneralConfig(t, func(c *chainlink.Config, s *chainlink.Secrets) {}) txStore := cltest.NewTestTxStore(t, db) ethKeyStore := cltest.NewKeyStore(t, db).Eth() @@ -876,6 +882,7 @@ func TestEthConfirmer_CheckForReceipts_confirmed_missing_receipt(t *testing.T) { ec := newEthConfirmer(t, txStore, ethClient, cfg, evmcfg, ethKeyStore, nil) ctx := tests.Context(t) + latestFinalizedBlockNum := int64(0) // STATE // eth_txes with nonce 0 has two attempts (broadcast before block 21 and 41) the first of which will get a receipt @@ -949,7 +956,7 @@ func TestEthConfirmer_CheckForReceipts_confirmed_missing_receipt(t *testing.T) { // PERFORM // Block num of 43 is one higher than the receipt (as would generally be expected) - require.NoError(t, ec.CheckForReceipts(ctx, 43)) + require.NoError(t, ec.CheckForReceipts(ctx, 43, latestFinalizedBlockNum)) // Expected state is that the "top" eth_tx is now confirmed, with the // two below it "confirmed_missing_receipt" and the "bottom" eth_tx also confirmed @@ -1009,7 +1016,7 @@ func TestEthConfirmer_CheckForReceipts_confirmed_missing_receipt(t *testing.T) { // PERFORM // Block num of 44 is one higher than the receipt (as would generally be expected) - require.NoError(t, ec.CheckForReceipts(ctx, 44)) + require.NoError(t, ec.CheckForReceipts(ctx, 44, latestFinalizedBlockNum)) // Expected state is that the "top" two eth_txes are now confirmed, with the // one below it still "confirmed_missing_receipt" and the bottom one remains confirmed @@ -1038,7 +1045,7 @@ func TestEthConfirmer_CheckForReceipts_confirmed_missing_receipt(t *testing.T) { // eth_txes with nonce 2 is confirmed // eth_txes with nonce 3 is confirmed - t.Run("continues to leave eth_txes with state 'confirmed_missing_receipt' unchanged if at least one attempt is above EVM.FinalityDepth", func(t *testing.T) { + t.Run("continues to leave eth_txes with state 'confirmed_missing_receipt' unchanged if at least one attempt is above LatestFinalizedBlockNum", func(t *testing.T) { ethClient.On("SequenceAt", mock.Anything, mock.Anything, mock.Anything).Return(evmtypes.Nonce(10), nil) ethClient.On("BatchCallContext", mock.Anything, mock.MatchedBy(func(b []rpc.BatchElem) bool { return len(b) == 2 && @@ -1051,9 +1058,11 @@ func TestEthConfirmer_CheckForReceipts_confirmed_missing_receipt(t *testing.T) { elems[1].Result = &evmtypes.Receipt{} }).Once() + latestFinalizedBlockNum = 30 + // PERFORM // Block num of 80 puts the first attempt (21) below threshold but second attempt (41) still above - require.NoError(t, ec.CheckForReceipts(ctx, 80)) + require.NoError(t, ec.CheckForReceipts(ctx, 80, latestFinalizedBlockNum)) // Expected state is that the "top" two eth_txes are now confirmed, with the // one below it still "confirmed_missing_receipt" and the bottom one remains confirmed @@ -1078,7 +1087,7 @@ func TestEthConfirmer_CheckForReceipts_confirmed_missing_receipt(t *testing.T) { // eth_txes with nonce 2 is confirmed // eth_txes with nonce 3 is confirmed - t.Run("marks eth_Txes with state 'confirmed_missing_receipt' as 'errored' if a receipt fails to show up and all attempts are buried deeper than EVM.FinalityDepth", func(t *testing.T) { + t.Run("marks eth_Txes with state 'confirmed_missing_receipt' as 'errored' if a receipt fails to show up and all attempts are buried deeper than LatestFinalizedBlockNum", func(t *testing.T) { ethClient.On("SequenceAt", mock.Anything, mock.Anything, mock.Anything).Return(evmtypes.Nonce(10), nil) ethClient.On("BatchCallContext", mock.Anything, mock.MatchedBy(func(b []rpc.BatchElem) bool { return len(b) == 2 && @@ -1091,9 +1100,11 @@ func TestEthConfirmer_CheckForReceipts_confirmed_missing_receipt(t *testing.T) { elems[1].Result = &evmtypes.Receipt{} }).Once() + latestFinalizedBlockNum = 50 + // PERFORM // Block num of 100 puts the first attempt (21) and second attempt (41) below threshold - require.NoError(t, ec.CheckForReceipts(ctx, 100)) + require.NoError(t, ec.CheckForReceipts(ctx, 100, latestFinalizedBlockNum)) // Expected state is that the "top" two eth_txes are now confirmed, with the // one below it marked as "fatal_error" and the bottom one remains confirmed @@ -1117,9 +1128,7 @@ func TestEthConfirmer_CheckConfirmedMissingReceipt(t *testing.T) { t.Parallel() db := pgtest.NewSqlxDB(t) - cfg := configtest.NewGeneralConfig(t, func(c *chainlink.Config, s *chainlink.Secrets) { - c.EVM[0].FinalityDepth = ptr[uint32](50) - }) + cfg := configtest.NewGeneralConfig(t, func(c *chainlink.Config, s *chainlink.Secrets) {}) txStore := cltest.NewTestTxStore(t, db) ethKeyStore := cltest.NewKeyStore(t, db).Eth() @@ -1197,9 +1206,7 @@ func TestEthConfirmer_CheckConfirmedMissingReceipt_batchSendTransactions_fails(t t.Parallel() db := pgtest.NewSqlxDB(t) - cfg := configtest.NewGeneralConfig(t, func(c *chainlink.Config, s *chainlink.Secrets) { - c.EVM[0].FinalityDepth = ptr[uint32](50) - }) + cfg := configtest.NewGeneralConfig(t, func(c *chainlink.Config, s *chainlink.Secrets) {}) txStore := cltest.NewTestTxStore(t, db) ethKeyStore := cltest.NewKeyStore(t, db).Eth() @@ -1262,7 +1269,6 @@ func TestEthConfirmer_CheckConfirmedMissingReceipt_smallEvmRPCBatchSize_middleBa db := pgtest.NewSqlxDB(t) cfg := configtest.NewGeneralConfig(t, func(c *chainlink.Config, s *chainlink.Secrets) { - c.EVM[0].FinalityDepth = ptr[uint32](50) c.EVM[0].RPCDefaultBatchSize = ptr[uint32](1) }) txStore := cltest.NewTestTxStore(t, db) @@ -2679,9 +2685,10 @@ func TestEthConfirmer_EnsureConfirmedTransactionsInLongestChain(t *testing.T) { Hash: testutils.NewHash(), Number: 9, Parent: &evmtypes.Head{ - Number: 8, - Hash: testutils.NewHash(), - Parent: nil, + Number: 8, + Hash: testutils.NewHash(), + IsFinalized: true, + Parent: nil, }, }, } @@ -3172,8 +3179,9 @@ func TestEthConfirmer_ProcessStuckTransactions(t *testing.T) { tx := mustInsertUnconfirmedTxWithBroadcastAttempts(t, txStore, nonce, fromAddress, autoPurgeMinAttempts, blockNum-int64(autoPurgeThreshold), marketGasPrice.Add(oneGwei)) head := evmtypes.Head{ - Hash: testutils.NewHash(), - Number: blockNum, + Hash: testutils.NewHash(), + Number: blockNum, + IsFinalized: true, } ethClient.On("SequenceAt", mock.Anything, mock.Anything, mock.Anything).Return(evmtypes.Nonce(0), nil).Once() ethClient.On("BatchCallContext", mock.Anything, mock.Anything).Return(nil).Once() @@ -3196,8 +3204,9 @@ func TestEthConfirmer_ProcessStuckTransactions(t *testing.T) { require.Equal(t, bumpedFee.Legacy, latestAttempt.TxFee.Legacy) head = evmtypes.Head{ - Hash: testutils.NewHash(), - Number: blockNum + 1, + Hash: testutils.NewHash(), + Number: blockNum + 1, + IsFinalized: true, } ethClient.On("SequenceAt", mock.Anything, mock.Anything, mock.Anything).Return(evmtypes.Nonce(1), nil) ethClient.On("BatchCallContext", mock.Anything, mock.MatchedBy(func(b []rpc.BatchElem) bool { diff --git a/core/chains/evm/txmgr/evm_tx_store_test.go b/core/chains/evm/txmgr/evm_tx_store_test.go index afb8de4ca52..db0807daf6a 100644 --- a/core/chains/evm/txmgr/evm_tx_store_test.go +++ b/core/chains/evm/txmgr/evm_tx_store_test.go @@ -1141,13 +1141,14 @@ func TestORM_MarkOldTxesMissingReceiptAsErrored(t *testing.T) { ethKeyStore := cltest.NewKeyStore(t, db).Eth() ethClient := evmtest.NewEthClientMockWithDefaultChain(t) _, fromAddress := cltest.MustInsertRandomKeyReturningState(t, ethKeyStore) + latestFinalizedBlockNum := int64(8) // tx state should be confirmed missing receipt - // attempt should be broadcast before cutoff time + // attempt should be before latestFinalizedBlockNum t.Run("successfully mark errored transactions", func(t *testing.T) { etx := mustInsertConfirmedMissingReceiptEthTxWithLegacyAttempt(t, txStore, 1, 7, time.Now(), fromAddress) - err := txStore.MarkOldTxesMissingReceiptAsErrored(tests.Context(t), 10, 2, ethClient.ConfiguredChainID()) + err := txStore.MarkOldTxesMissingReceiptAsErrored(tests.Context(t), 10, latestFinalizedBlockNum, ethClient.ConfiguredChainID()) require.NoError(t, err) etx, err = txStore.FindTxWithAttempts(ctx, etx.ID) @@ -1157,7 +1158,7 @@ func TestORM_MarkOldTxesMissingReceiptAsErrored(t *testing.T) { t.Run("successfully mark errored transactions w/ qopt passing in sql.Tx", func(t *testing.T) { etx := mustInsertConfirmedMissingReceiptEthTxWithLegacyAttempt(t, txStore, 1, 7, time.Now(), fromAddress) - err := txStore.MarkOldTxesMissingReceiptAsErrored(tests.Context(t), 10, 2, ethClient.ConfiguredChainID()) + err := txStore.MarkOldTxesMissingReceiptAsErrored(tests.Context(t), 10, latestFinalizedBlockNum, ethClient.ConfiguredChainID()) require.NoError(t, err) // must run other query outside of postgres transaction so changes are committed From 1d14c4b2c2384ef10c218263b7070c2470589b68 Mon Sep 17 00:00:00 2001 From: Farber98 Date: Wed, 3 Jul 2024 14:31:11 -0300 Subject: [PATCH 05/37] mocks for both common and core/evm --- common/txmgr/mocks/tx_manager.go | 2 +- core/chains/evm/txmgr/mocks/config.go | 2 +- core/chains/evm/txmgr/mocks/evm_tx_store.go | 12 ++++++------ 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/common/txmgr/mocks/tx_manager.go b/common/txmgr/mocks/tx_manager.go index c7380b76876..502f6fe8a25 100644 --- a/common/txmgr/mocks/tx_manager.go +++ b/common/txmgr/mocks/tx_manager.go @@ -1,4 +1,4 @@ -// Code generated by mockery v2.43.2. DO NOT EDIT. +// Code generated by mockery v2.42.2. DO NOT EDIT. package mocks diff --git a/core/chains/evm/txmgr/mocks/config.go b/core/chains/evm/txmgr/mocks/config.go index efaaab68afa..5ec00e960ab 100644 --- a/core/chains/evm/txmgr/mocks/config.go +++ b/core/chains/evm/txmgr/mocks/config.go @@ -1,4 +1,4 @@ -// Code generated by mockery v2.43.2. DO NOT EDIT. +// Code generated by mockery v2.42.2. DO NOT EDIT. package mocks diff --git a/core/chains/evm/txmgr/mocks/evm_tx_store.go b/core/chains/evm/txmgr/mocks/evm_tx_store.go index 5a699f71bf9..8fc58d31439 100644 --- a/core/chains/evm/txmgr/mocks/evm_tx_store.go +++ b/core/chains/evm/txmgr/mocks/evm_tx_store.go @@ -1,4 +1,4 @@ -// Code generated by mockery v2.43.2. DO NOT EDIT. +// Code generated by mockery v2.42.2. DO NOT EDIT. package mocks @@ -1063,17 +1063,17 @@ func (_m *EvmTxStore) MarkAllConfirmedMissingReceipt(ctx context.Context, chainI return r0 } -// MarkOldTxesMissingReceiptAsErrored provides a mock function with given fields: ctx, blockNum, finalityDepth, chainID -func (_m *EvmTxStore) MarkOldTxesMissingReceiptAsErrored(ctx context.Context, blockNum int64, finalityDepth uint32, chainID *big.Int) error { - ret := _m.Called(ctx, blockNum, finalityDepth, chainID) +// MarkOldTxesMissingReceiptAsErrored provides a mock function with given fields: ctx, blockNum, latestFinalizedBlockNum, chainID +func (_m *EvmTxStore) MarkOldTxesMissingReceiptAsErrored(ctx context.Context, blockNum int64, latestFinalizedBlockNum int64, chainID *big.Int) error { + ret := _m.Called(ctx, blockNum, latestFinalizedBlockNum, chainID) if len(ret) == 0 { panic("no return value specified for MarkOldTxesMissingReceiptAsErrored") } var r0 error - if rf, ok := ret.Get(0).(func(context.Context, int64, uint32, *big.Int) error); ok { - r0 = rf(ctx, blockNum, finalityDepth, chainID) + if rf, ok := ret.Get(0).(func(context.Context, int64, int64, *big.Int) error); ok { + r0 = rf(ctx, blockNum, latestFinalizedBlockNum, chainID) } else { r0 = ret.Error(0) } From 62222914daada792908c06ecbf5932bb4225b3d9 Mon Sep 17 00:00:00 2001 From: Farber98 Date: Thu, 4 Jul 2024 10:47:24 -0300 Subject: [PATCH 06/37] remove comments that are still referencing to finalityDepth within evm/txmgr --- core/chains/evm/txmgr/evm_tx_store.go | 8 ++++---- core/chains/evm/txmgr/reaper_test.go | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/core/chains/evm/txmgr/evm_tx_store.go b/core/chains/evm/txmgr/evm_tx_store.go index 098bcffa223..71896bffe39 100644 --- a/core/chains/evm/txmgr/evm_tx_store.go +++ b/core/chains/evm/txmgr/evm_tx_store.go @@ -34,8 +34,8 @@ import ( var ( ErrKeyNotUpdated = errors.New("evmTxStore: Key not updated") - // ErrCouldNotGetReceipt is the error string we save if we reach our finality depth for a confirmed transaction without ever getting a receipt - // This most likely happened because an external wallet used the account for this nonce + // ErrCouldNotGetReceipt is the error string we save if we reach our LatestFinalizedBlockNum for a confirmed transaction + // without ever getting a receipt. This most likely happened because an external wallet used the account for this nonce ErrCouldNotGetReceipt = "could not get receipt" ) @@ -957,11 +957,11 @@ func (o *evmTxStore) SaveFetchedReceipts(ctx context.Context, r []*evmtypes.Rece // NOTE: We continue to attempt to resend evm.txes in this state on // every head to guard against the extremely rare scenario of nonce gap due to // reorg that excludes the transaction (from another wallet) that had this -// nonce (until finality depth is reached, after which we make the explicit +// nonce (until LatestFinalizedBlockNum is reached, after which we make the explicit // decision to give up). This is done in the EthResender. // // We will continue to try to fetch a receipt for these attempts until all -// attempts are below the finality depth from current head. +// attempts are below the LatestFinalizedBlockNum from current head. func (o *evmTxStore) MarkAllConfirmedMissingReceipt(ctx context.Context, chainID *big.Int) (err error) { var cancel context.CancelFunc ctx, cancel = o.stopCh.Ctx(ctx) diff --git a/core/chains/evm/txmgr/reaper_test.go b/core/chains/evm/txmgr/reaper_test.go index b3ce48b702c..de146fca24b 100644 --- a/core/chains/evm/txmgr/reaper_test.go +++ b/core/chains/evm/txmgr/reaper_test.go @@ -114,7 +114,7 @@ func TestReaper_ReapTxes(t *testing.T) { err = r.ReapTxes(42) assert.NoError(t, err) - // Now it deleted because the eth_tx was past EVM.FinalityDepth + // Now it deleted because the eth_tx was marked as finalized cltest.AssertCount(t, db, "evm.txes", 0) }) From 8e01631b7dd1e01e1b3123c1ffc657189c79cc1c Mon Sep 17 00:00:00 2001 From: Farber98 Date: Tue, 9 Jul 2024 18:41:03 -0300 Subject: [PATCH 07/37] add changeset --- .changeset/chatty-spiders-double.md | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 .changeset/chatty-spiders-double.md diff --git a/.changeset/chatty-spiders-double.md b/.changeset/chatty-spiders-double.md new file mode 100644 index 00000000000..750a11628fe --- /dev/null +++ b/.changeset/chatty-spiders-double.md @@ -0,0 +1,5 @@ +--- +"chainlink": minor +--- + +remove dependency on FinalityDepth in EVM TXM code. #internal From 55b069f147c09a447cfaca2ce36a808053164119 Mon Sep 17 00:00:00 2001 From: Farber98 Date: Thu, 11 Jul 2024 12:28:34 -0300 Subject: [PATCH 08/37] error if no LatestFinalizedHead was found --- common/txmgr/confirmer.go | 8 +++++++ core/chains/evm/txmgr/confirmer_test.go | 28 ++++++++++++++++++------- 2 files changed, 28 insertions(+), 8 deletions(-) diff --git a/common/txmgr/confirmer.go b/common/txmgr/confirmer.go index 8f8b86d1e28..dbd410cce8e 100644 --- a/common/txmgr/confirmer.go +++ b/common/txmgr/confirmer.go @@ -297,6 +297,10 @@ func (ec *Confirmer[CHAIN_ID, HEAD, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) pro return fmt.Errorf("CheckConfirmedMissingReceipt failed: %w", err) } + if head.LatestFinalizedHead() == nil { + return errors.New("There is no LatestFinalizedHead") + } + if err := ec.CheckForReceipts(ctx, head.BlockNumber(), head.LatestFinalizedHead().BlockNumber()); err != nil { return fmt.Errorf("CheckForReceipts failed: %w", err) } @@ -1007,6 +1011,10 @@ func (ec *Confirmer[CHAIN_ID, HEAD, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) han // If any of the confirmed transactions does not have a receipt in the chain, it has been // re-org'd out and will be rebroadcast. func (ec *Confirmer[CHAIN_ID, HEAD, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) EnsureConfirmedTransactionsInLongestChain(ctx context.Context, head types.Head[BLOCK_HASH]) error { + if head.LatestFinalizedHead() == nil { + return errors.New("There is no LatestFinalizedHead") + } + if head.ChainLength() < uint32(head.LatestFinalizedHead().BlockNumber()) { logArgs := []interface{}{ "chainLength", head.ChainLength(), "latestFinalizedHead", head.LatestFinalizedHead().BlockNumber(), diff --git a/core/chains/evm/txmgr/confirmer_test.go b/core/chains/evm/txmgr/confirmer_test.go index 9ae14e70aa3..f15e82da59f 100644 --- a/core/chains/evm/txmgr/confirmer_test.go +++ b/core/chains/evm/txmgr/confirmer_test.go @@ -152,13 +152,19 @@ func TestEthConfirmer_Lifecycle(t *testing.T) { Hash: testutils.NewHash(), Number: 9, Parent: &evmtypes.Head{ - Number: 8, - Hash: testutils.NewHash(), - IsFinalized: true, - Parent: nil, + Number: 8, + Hash: testutils.NewHash(), + Parent: nil, }, }, } + + // If there's no finalized head yet, returns error. + err = ec.ProcessHead(ctx, &head) + require.Error(t, err) + + // Successful after having a head finalized in the chain + head.Parent.Parent.IsFinalized = true err = ec.ProcessHead(ctx, &head) require.NoError(t, err) // Can successfully close once @@ -2685,14 +2691,20 @@ func TestEthConfirmer_EnsureConfirmedTransactionsInLongestChain(t *testing.T) { Hash: testutils.NewHash(), Number: 9, Parent: &evmtypes.Head{ - Number: 8, - Hash: testutils.NewHash(), - IsFinalized: true, - Parent: nil, + Number: 8, + Hash: testutils.NewHash(), + Parent: nil, }, }, } + t.Run("error if there's no LatestFinalizedHead", func(t *testing.T) { + require.Error(t, ec.EnsureConfirmedTransactionsInLongestChain(tests.Context(t), &head)) + }) + + // Finalized a head + head.Parent.Parent.IsFinalized = true + t.Run("does nothing if there aren't any transactions", func(t *testing.T) { require.NoError(t, ec.EnsureConfirmedTransactionsInLongestChain(tests.Context(t), &head)) }) From 9a977632def052f543b35f81e506c50b5d826a27 Mon Sep 17 00:00:00 2001 From: Farber98 Date: Thu, 11 Jul 2024 15:09:34 -0300 Subject: [PATCH 09/37] fix mocks version --- common/txmgr/mocks/tx_manager.go | 2 +- core/chains/evm/txmgr/mocks/config.go | 2 +- core/chains/evm/txmgr/mocks/evm_tx_store.go | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/common/txmgr/mocks/tx_manager.go b/common/txmgr/mocks/tx_manager.go index 502f6fe8a25..c7380b76876 100644 --- a/common/txmgr/mocks/tx_manager.go +++ b/common/txmgr/mocks/tx_manager.go @@ -1,4 +1,4 @@ -// Code generated by mockery v2.42.2. DO NOT EDIT. +// Code generated by mockery v2.43.2. DO NOT EDIT. package mocks diff --git a/core/chains/evm/txmgr/mocks/config.go b/core/chains/evm/txmgr/mocks/config.go index 5ec00e960ab..efaaab68afa 100644 --- a/core/chains/evm/txmgr/mocks/config.go +++ b/core/chains/evm/txmgr/mocks/config.go @@ -1,4 +1,4 @@ -// Code generated by mockery v2.42.2. DO NOT EDIT. +// Code generated by mockery v2.43.2. DO NOT EDIT. package mocks diff --git a/core/chains/evm/txmgr/mocks/evm_tx_store.go b/core/chains/evm/txmgr/mocks/evm_tx_store.go index 8fc58d31439..812f7a5368f 100644 --- a/core/chains/evm/txmgr/mocks/evm_tx_store.go +++ b/core/chains/evm/txmgr/mocks/evm_tx_store.go @@ -1,4 +1,4 @@ -// Code generated by mockery v2.42.2. DO NOT EDIT. +// Code generated by mockery v2.43.2. DO NOT EDIT. package mocks From 8ed35fe6a05822ff42058294c10007daaeb377ba Mon Sep 17 00:00:00 2001 From: Farber98 Date: Thu, 11 Jul 2024 15:30:55 -0300 Subject: [PATCH 10/37] fix mocks version --- common/txmgr/mocks/tx_manager.go | 2 +- core/chains/evm/txmgr/mocks/config.go | 2 +- core/chains/evm/txmgr/mocks/evm_tx_store.go | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/common/txmgr/mocks/tx_manager.go b/common/txmgr/mocks/tx_manager.go index c7380b76876..502f6fe8a25 100644 --- a/common/txmgr/mocks/tx_manager.go +++ b/common/txmgr/mocks/tx_manager.go @@ -1,4 +1,4 @@ -// Code generated by mockery v2.43.2. DO NOT EDIT. +// Code generated by mockery v2.42.2. DO NOT EDIT. package mocks diff --git a/core/chains/evm/txmgr/mocks/config.go b/core/chains/evm/txmgr/mocks/config.go index efaaab68afa..5ec00e960ab 100644 --- a/core/chains/evm/txmgr/mocks/config.go +++ b/core/chains/evm/txmgr/mocks/config.go @@ -1,4 +1,4 @@ -// Code generated by mockery v2.43.2. DO NOT EDIT. +// Code generated by mockery v2.42.2. DO NOT EDIT. package mocks diff --git a/core/chains/evm/txmgr/mocks/evm_tx_store.go b/core/chains/evm/txmgr/mocks/evm_tx_store.go index 812f7a5368f..8fc58d31439 100644 --- a/core/chains/evm/txmgr/mocks/evm_tx_store.go +++ b/core/chains/evm/txmgr/mocks/evm_tx_store.go @@ -1,4 +1,4 @@ -// Code generated by mockery v2.43.2. DO NOT EDIT. +// Code generated by mockery v2.42.2. DO NOT EDIT. package mocks From 1a694f7489c6ad5c1128dead34257461e622b163 Mon Sep 17 00:00:00 2001 From: Farber98 Date: Thu, 11 Jul 2024 17:58:37 -0300 Subject: [PATCH 11/37] mock versioning --- common/txmgr/mocks/tx_manager.go | 2 +- core/chains/evm/txmgr/mocks/config.go | 2 +- core/chains/evm/txmgr/mocks/evm_tx_store.go | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/common/txmgr/mocks/tx_manager.go b/common/txmgr/mocks/tx_manager.go index 502f6fe8a25..c7380b76876 100644 --- a/common/txmgr/mocks/tx_manager.go +++ b/common/txmgr/mocks/tx_manager.go @@ -1,4 +1,4 @@ -// Code generated by mockery v2.42.2. DO NOT EDIT. +// Code generated by mockery v2.43.2. DO NOT EDIT. package mocks diff --git a/core/chains/evm/txmgr/mocks/config.go b/core/chains/evm/txmgr/mocks/config.go index 5ec00e960ab..efaaab68afa 100644 --- a/core/chains/evm/txmgr/mocks/config.go +++ b/core/chains/evm/txmgr/mocks/config.go @@ -1,4 +1,4 @@ -// Code generated by mockery v2.42.2. DO NOT EDIT. +// Code generated by mockery v2.43.2. DO NOT EDIT. package mocks diff --git a/core/chains/evm/txmgr/mocks/evm_tx_store.go b/core/chains/evm/txmgr/mocks/evm_tx_store.go index 8fc58d31439..812f7a5368f 100644 --- a/core/chains/evm/txmgr/mocks/evm_tx_store.go +++ b/core/chains/evm/txmgr/mocks/evm_tx_store.go @@ -1,4 +1,4 @@ -// Code generated by mockery v2.42.2. DO NOT EDIT. +// Code generated by mockery v2.43.2. DO NOT EDIT. package mocks From 7f1b8b51fc15845137858f41d448446d1ee9d4b5 Mon Sep 17 00:00:00 2001 From: Farber98 Date: Thu, 11 Jul 2024 18:15:41 -0300 Subject: [PATCH 12/37] mock version --- common/txmgr/types/mocks/tx_store.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/common/txmgr/types/mocks/tx_store.go b/common/txmgr/types/mocks/tx_store.go index 9c2edfa963f..bb8fabf1a71 100644 --- a/common/txmgr/types/mocks/tx_store.go +++ b/common/txmgr/types/mocks/tx_store.go @@ -1,4 +1,4 @@ -// Code generated by mockery v2.43.2. DO NOT EDIT. +// Code generated by mockery v2.42.2. DO NOT EDIT. package mocks From e9c8daa5b78f2085c9123e2beb100e753a4cdfe1 Mon Sep 17 00:00:00 2001 From: Farber98 Date: Thu, 11 Jul 2024 18:38:43 -0300 Subject: [PATCH 13/37] mock version --- common/txmgr/types/mocks/tx_store.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/common/txmgr/types/mocks/tx_store.go b/common/txmgr/types/mocks/tx_store.go index bb8fabf1a71..9c2edfa963f 100644 --- a/common/txmgr/types/mocks/tx_store.go +++ b/common/txmgr/types/mocks/tx_store.go @@ -1,4 +1,4 @@ -// Code generated by mockery v2.42.2. DO NOT EDIT. +// Code generated by mockery v2.43.2. DO NOT EDIT. package mocks From b151dbef0e055d3c22a2105e8616da76611c3311 Mon Sep 17 00:00:00 2001 From: Farber98 Date: Fri, 12 Jul 2024 12:16:00 -0300 Subject: [PATCH 14/37] mock version --- common/txmgr/types/mocks/tx_store.go | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/common/txmgr/types/mocks/tx_store.go b/common/txmgr/types/mocks/tx_store.go index 9c2edfa963f..0550212a7c1 100644 --- a/common/txmgr/types/mocks/tx_store.go +++ b/common/txmgr/types/mocks/tx_store.go @@ -1,4 +1,4 @@ -// Code generated by mockery v2.43.2. DO NOT EDIT. +// Code generated by mockery v2.42.2. DO NOT EDIT. package mocks @@ -912,17 +912,17 @@ func (_m *TxStore[ADDR, CHAIN_ID, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) MarkAllConf return r0 } -// MarkOldTxesMissingReceiptAsErrored provides a mock function with given fields: ctx, blockNum, finalityDepth, chainID -func (_m *TxStore[ADDR, CHAIN_ID, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) MarkOldTxesMissingReceiptAsErrored(ctx context.Context, blockNum int64, finalityDepth uint32, chainID CHAIN_ID) error { - ret := _m.Called(ctx, blockNum, finalityDepth, chainID) +// MarkOldTxesMissingReceiptAsErrored provides a mock function with given fields: ctx, blockNum, latestFinalizedBlockNum, chainID +func (_m *TxStore[ADDR, CHAIN_ID, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) MarkOldTxesMissingReceiptAsErrored(ctx context.Context, blockNum int64, latestFinalizedBlockNum int64, chainID CHAIN_ID) error { + ret := _m.Called(ctx, blockNum, latestFinalizedBlockNum, chainID) if len(ret) == 0 { panic("no return value specified for MarkOldTxesMissingReceiptAsErrored") } var r0 error - if rf, ok := ret.Get(0).(func(context.Context, int64, uint32, CHAIN_ID) error); ok { - r0 = rf(ctx, blockNum, finalityDepth, chainID) + if rf, ok := ret.Get(0).(func(context.Context, int64, int64, CHAIN_ID) error); ok { + r0 = rf(ctx, blockNum, latestFinalizedBlockNum, chainID) } else { r0 = ret.Error(0) } From 552a89f256444704d79b1d3c3027d6d0951c93c8 Mon Sep 17 00:00:00 2001 From: Farber98 Date: Fri, 12 Jul 2024 12:28:36 -0300 Subject: [PATCH 15/37] mock version --- common/txmgr/types/mocks/tx_store.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/common/txmgr/types/mocks/tx_store.go b/common/txmgr/types/mocks/tx_store.go index 0550212a7c1..c5a229abce1 100644 --- a/common/txmgr/types/mocks/tx_store.go +++ b/common/txmgr/types/mocks/tx_store.go @@ -1,4 +1,4 @@ -// Code generated by mockery v2.42.2. DO NOT EDIT. +// Code generated by mockery v2.43.2. DO NOT EDIT. package mocks From 8e91a7526b5abf9be992158909390feb01ac6462 Mon Sep 17 00:00:00 2001 From: Farber98 Date: Wed, 31 Jul 2024 09:00:22 -0300 Subject: [PATCH 16/37] inject head tracker trimmed API --- common/txmgr/confirmer.go | 28 +++++++++++++++++----------- core/chains/evm/txmgr/builder.go | 11 +++++++++-- core/chains/legacyevm/chain.go | 2 +- core/chains/legacyevm/evm_txm.go | 5 ++++- core/cmd/shell_local.go | 4 +++- 5 files changed, 34 insertions(+), 16 deletions(-) diff --git a/common/txmgr/confirmer.go b/common/txmgr/confirmer.go index dbd410cce8e..7125277e3e4 100644 --- a/common/txmgr/confirmer.go +++ b/common/txmgr/confirmer.go @@ -27,6 +27,7 @@ import ( iutils "github.com/smartcontractkit/chainlink/v2/common/internal/utils" txmgrtypes "github.com/smartcontractkit/chainlink/v2/common/txmgr/types" "github.com/smartcontractkit/chainlink/v2/common/types" + evmtypes "github.com/smartcontractkit/chainlink/v2/core/chains/evm/types" ) const ( @@ -102,6 +103,10 @@ var ( }, []string{"chainID"}) ) +type confirmerHeadTracker interface { + LatestAndFinalizedBlock(ctx context.Context) (latest, finalized *evmtypes.Head, err error) +} + // Confirmer is a broad service which performs four different tasks in sequence on every new longest chain // Step 1: Mark that all currently pending transaction attempts were broadcast before this block // Step 2: Check pending transactions for receipts @@ -141,6 +146,8 @@ type Confirmer[ nConsecutiveBlocksChainTooShort int isReceiptNil func(R) bool + + headTracker confirmerHeadTracker } func NewConfirmer[ @@ -164,6 +171,7 @@ func NewConfirmer[ lggr logger.Logger, isReceiptNil func(R) bool, stuckTxDetector txmgrtypes.StuckTxDetector[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], + headTracker confirmerHeadTracker, ) *Confirmer[CHAIN_ID, HEAD, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE] { lggr = logger.Named(lggr, "Confirmer") return &Confirmer[CHAIN_ID, HEAD, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]{ @@ -181,6 +189,7 @@ func NewConfirmer[ mb: mailbox.NewSingle[HEAD](), isReceiptNil: isReceiptNil, stuckTxDetector: stuckTxDetector, + headTracker: headTracker, } } @@ -297,11 +306,12 @@ func (ec *Confirmer[CHAIN_ID, HEAD, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) pro return fmt.Errorf("CheckConfirmedMissingReceipt failed: %w", err) } - if head.LatestFinalizedHead() == nil { - return errors.New("There is no LatestFinalizedHead") + _, latestFinalizedHead, err := ec.headTracker.LatestAndFinalizedBlock(ctx) + if err != nil { + return fmt.Errorf("failed to retrieve latest finalized head: %w", err) } - if err := ec.CheckForReceipts(ctx, head.BlockNumber(), head.LatestFinalizedHead().BlockNumber()); err != nil { + if err := ec.CheckForReceipts(ctx, head.BlockNumber(), latestFinalizedHead.BlockNumber()); err != nil { return fmt.Errorf("CheckForReceipts failed: %w", err) } @@ -322,7 +332,7 @@ func (ec *Confirmer[CHAIN_ID, HEAD, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) pro ec.lggr.Debugw("Finished RebroadcastWhereNecessary", "headNum", head.BlockNumber(), "time", time.Since(mark), "id", "confirmer") mark = time.Now() - if err := ec.EnsureConfirmedTransactionsInLongestChain(ctx, head); err != nil { + if err := ec.EnsureConfirmedTransactionsInLongestChain(ctx, head, latestFinalizedHead); err != nil { return fmt.Errorf("EnsureConfirmedTransactionsInLongestChain failed: %w", err) } @@ -1010,14 +1020,10 @@ func (ec *Confirmer[CHAIN_ID, HEAD, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) han // // If any of the confirmed transactions does not have a receipt in the chain, it has been // re-org'd out and will be rebroadcast. -func (ec *Confirmer[CHAIN_ID, HEAD, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) EnsureConfirmedTransactionsInLongestChain(ctx context.Context, head types.Head[BLOCK_HASH]) error { - if head.LatestFinalizedHead() == nil { - return errors.New("There is no LatestFinalizedHead") - } - - if head.ChainLength() < uint32(head.LatestFinalizedHead().BlockNumber()) { +func (ec *Confirmer[CHAIN_ID, HEAD, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) EnsureConfirmedTransactionsInLongestChain(ctx context.Context, head types.Head[BLOCK_HASH], latestFinalizedHead *evmtypes.Head) error { + if head.ChainLength() < uint32(latestFinalizedHead.BlockNumber()) { logArgs := []interface{}{ - "chainLength", head.ChainLength(), "latestFinalizedHead", head.LatestFinalizedHead().BlockNumber(), + "chainLength", head.ChainLength(), "latestFinalizedHead", latestFinalizedHead.BlockNumber(), } if ec.nConsecutiveBlocksChainTooShort > logAfterNConsecutiveBlocksChainTooShort { warnMsg := "Chain length supplied for re-org detection was shorter than LatestFinalizedHead. Re-org protection is not working properly. This could indicate a problem with the remote RPC endpoint, a compatibility issue with a particular blockchain, a bug with this particular blockchain, heads table being truncated too early, remote node out of sync, or something else. If this happens a lot please raise a bug with the Chainlink team including a log output sample and details of the chain and RPC endpoint you are using." diff --git a/core/chains/evm/txmgr/builder.go b/core/chains/evm/txmgr/builder.go index dcf15a4fa23..77b8cd260f8 100644 --- a/core/chains/evm/txmgr/builder.go +++ b/core/chains/evm/txmgr/builder.go @@ -1,6 +1,7 @@ package txmgr import ( + "context" "math/big" "time" @@ -17,6 +18,10 @@ import ( evmtypes "github.com/smartcontractkit/chainlink/v2/core/chains/evm/types" ) +type latestAndFinalizedBlockHeadTracker interface { + LatestAndFinalizedBlock(ctx context.Context) (latest, finalized *evmtypes.Head, err error) +} + // NewTxm constructs the necessary dependencies for the EvmTxm (broadcaster, confirmer, etc) and returns a new EvmTxManager func NewTxm( ds sqlutil.DataSource, @@ -31,6 +36,7 @@ func NewTxm( logPoller logpoller.LogPoller, keyStore keystore.Eth, estimator gas.EvmFeeEstimator, + headTracker latestAndFinalizedBlockHeadTracker, ) (txm TxManager, err error, ) { @@ -52,7 +58,7 @@ func NewTxm( evmBroadcaster := NewEvmBroadcaster(txStore, txmClient, txmCfg, feeCfg, txConfig, listenerConfig, keyStore, txAttemptBuilder, lggr, checker, chainConfig.NonceAutoSync()) evmTracker := NewEvmTracker(txStore, keyStore, chainID, lggr) stuckTxDetector := NewStuckTxDetector(lggr, client.ConfiguredChainID(), chainConfig.ChainType(), fCfg.PriceMax(), txConfig.AutoPurge(), estimator, txStore, client) - evmConfirmer := NewEvmConfirmer(txStore, txmClient, txmCfg, feeCfg, txConfig, dbConfig, keyStore, txAttemptBuilder, lggr, stuckTxDetector) + evmConfirmer := NewEvmConfirmer(txStore, txmClient, txmCfg, feeCfg, txConfig, dbConfig, keyStore, txAttemptBuilder, lggr, stuckTxDetector, headTracker) var evmResender *Resender if txConfig.ResendAfterThreshold() > 0 { evmResender = NewEvmResender(lggr, txStore, txmClient, evmTracker, keyStore, txmgr.DefaultResenderPollInterval, chainConfig, txConfig) @@ -111,8 +117,9 @@ func NewEvmConfirmer( txAttemptBuilder TxAttemptBuilder, lggr logger.Logger, stuckTxDetector StuckTxDetector, + headTracker latestAndFinalizedBlockHeadTracker, ) *Confirmer { - return txmgr.NewConfirmer(txStore, client, chainConfig, feeConfig, txConfig, dbConfig, keystore, txAttemptBuilder, lggr, func(r *evmtypes.Receipt) bool { return r == nil }, stuckTxDetector) + return txmgr.NewConfirmer(txStore, client, chainConfig, feeConfig, txConfig, dbConfig, keystore, txAttemptBuilder, lggr, func(r *evmtypes.Receipt) bool { return r == nil }, stuckTxDetector, headTracker) } // NewEvmTracker instantiates a new EVM tracker for abandoned transactions diff --git a/core/chains/legacyevm/chain.go b/core/chains/legacyevm/chain.go index 129c0318820..68ff8d4e111 100644 --- a/core/chains/legacyevm/chain.go +++ b/core/chains/legacyevm/chain.go @@ -247,7 +247,7 @@ func newChain(ctx context.Context, cfg *evmconfig.ChainScoped, nodes []*toml.Nod } // note: gas estimator is started as a part of the txm - txm, gasEstimator, err := newEvmTxm(opts.DS, cfg.EVM(), opts.AppConfig.EVMRPCEnabled(), opts.AppConfig.Database(), opts.AppConfig.Database().Listener(), client, l, logPoller, opts) + txm, gasEstimator, err := newEvmTxm(opts.DS, cfg.EVM(), opts.AppConfig.EVMRPCEnabled(), opts.AppConfig.Database(), opts.AppConfig.Database().Listener(), client, l, logPoller, opts, headTracker) if err != nil { return nil, fmt.Errorf("failed to instantiate EvmTxm for chain with ID %s: %w", chainID.String(), err) } diff --git a/core/chains/legacyevm/evm_txm.go b/core/chains/legacyevm/evm_txm.go index cecfd4ffafe..ab116749665 100644 --- a/core/chains/legacyevm/evm_txm.go +++ b/core/chains/legacyevm/evm_txm.go @@ -7,6 +7,7 @@ import ( evmclient "github.com/smartcontractkit/chainlink/v2/core/chains/evm/client" evmconfig "github.com/smartcontractkit/chainlink/v2/core/chains/evm/config" "github.com/smartcontractkit/chainlink/v2/core/chains/evm/gas" + httypes "github.com/smartcontractkit/chainlink/v2/core/chains/evm/headtracker/types" "github.com/smartcontractkit/chainlink/v2/core/chains/evm/logpoller" "github.com/smartcontractkit/chainlink/v2/core/chains/evm/txmgr" "github.com/smartcontractkit/chainlink/v2/core/logger" @@ -22,6 +23,7 @@ func newEvmTxm( lggr logger.Logger, logPoller logpoller.LogPoller, opts ChainRelayExtenderConfig, + headTracker httypes.HeadTracker, ) (txm txmgr.TxManager, estimator gas.EvmFeeEstimator, err error, @@ -63,7 +65,8 @@ func newEvmTxm( lggr, logPoller, opts.KeyStore, - estimator) + estimator, + headTracker) } else { txm = opts.GenTxManager(chainID) } diff --git a/core/cmd/shell_local.go b/core/cmd/shell_local.go index e19cc485d8b..3f6ccb9ec51 100644 --- a/core/cmd/shell_local.go +++ b/core/cmd/shell_local.go @@ -38,6 +38,7 @@ import ( "github.com/smartcontractkit/chainlink/v2/core/build" "github.com/smartcontractkit/chainlink/v2/core/chains/evm/assets" "github.com/smartcontractkit/chainlink/v2/core/chains/evm/gas" + "github.com/smartcontractkit/chainlink/v2/core/chains/evm/headtracker" "github.com/smartcontractkit/chainlink/v2/core/chains/evm/txmgr" evmtypes "github.com/smartcontractkit/chainlink/v2/core/chains/evm/types" ubig "github.com/smartcontractkit/chainlink/v2/core/chains/evm/utils/big" @@ -668,8 +669,9 @@ func (s *Shell) RebroadcastTransactions(c *cli.Context) (err error) { cfg := txmgr.NewEvmTxmConfig(chain.Config().EVM()) feeCfg := txmgr.NewEvmTxmFeeConfig(chain.Config().EVM().GasEstimator()) stuckTxDetector := txmgr.NewStuckTxDetector(lggr, ethClient.ConfiguredChainID(), "", assets.NewWei(assets.NewEth(100).ToInt()), chain.Config().EVM().Transactions().AutoPurge(), nil, orm, ethClient) + ht := headtracker.NewSimulatedHeadTracker(ethClient, true, 0) ec := txmgr.NewEvmConfirmer(orm, txmgr.NewEvmTxmClient(ethClient, chain.Config().EVM().NodePool().Errors()), - cfg, feeCfg, chain.Config().EVM().Transactions(), app.GetConfig().Database(), keyStore.Eth(), txBuilder, chain.Logger(), stuckTxDetector) + cfg, feeCfg, chain.Config().EVM().Transactions(), app.GetConfig().Database(), keyStore.Eth(), txBuilder, chain.Logger(), stuckTxDetector, ht) totalNonces := endingNonce - beginningNonce + 1 nonces := make([]evmtypes.Nonce, totalNonces) for i := int64(0); i < totalNonces; i++ { From cb57245c0d3e8b4ac512fbff59a9fb779b52e445 Mon Sep 17 00:00:00 2001 From: Farber98 Date: Wed, 31 Jul 2024 09:11:41 -0300 Subject: [PATCH 17/37] fix tests --- core/chains/evm/txmgr/confirmer_test.go | 75 ++++++++++++------- core/chains/evm/txmgr/txmgr_test.go | 4 +- .../promreporter/prom_reporter_test.go | 3 +- core/services/vrf/delegate_test.go | 2 +- 4 files changed, 53 insertions(+), 31 deletions(-) diff --git a/core/chains/evm/txmgr/confirmer_test.go b/core/chains/evm/txmgr/confirmer_test.go index f15e82da59f..250ec617dc7 100644 --- a/core/chains/evm/txmgr/confirmer_test.go +++ b/core/chains/evm/txmgr/confirmer_test.go @@ -34,6 +34,7 @@ import ( evmconfig "github.com/smartcontractkit/chainlink/v2/core/chains/evm/config" "github.com/smartcontractkit/chainlink/v2/core/chains/evm/gas" gasmocks "github.com/smartcontractkit/chainlink/v2/core/chains/evm/gas/mocks" + "github.com/smartcontractkit/chainlink/v2/core/chains/evm/headtracker" "github.com/smartcontractkit/chainlink/v2/core/chains/evm/keystore" ksmocks "github.com/smartcontractkit/chainlink/v2/core/chains/evm/keystore/mocks" "github.com/smartcontractkit/chainlink/v2/core/chains/evm/testutils" @@ -131,7 +132,8 @@ func TestEthConfirmer_Lifecycle(t *testing.T) { feeEstimator := gas.NewEvmFeeEstimator(lggr, newEst, ge.EIP1559DynamicFees(), ge) txBuilder := txmgr.NewEvmTxAttemptBuilder(*ethClient.ConfiguredChainID(), ge, ethKeyStore, feeEstimator) stuckTxDetector := txmgr.NewStuckTxDetector(lggr, testutils.FixtureChainID, "", assets.NewWei(assets.NewEth(100).ToInt()), config.EVM().Transactions().AutoPurge(), feeEstimator, txStore, ethClient) - ec := txmgr.NewEvmConfirmer(txStore, txmgr.NewEvmTxmClient(ethClient, nil), txmgr.NewEvmTxmConfig(config.EVM()), txmgr.NewEvmTxmFeeConfig(ge), config.EVM().Transactions(), gconfig.Database(), ethKeyStore, txBuilder, lggr, stuckTxDetector) + ht := headtracker.NewSimulatedHeadTracker(ethClient, true, 0) + ec := txmgr.NewEvmConfirmer(txStore, txmgr.NewEvmTxmClient(ethClient, nil), txmgr.NewEvmTxmConfig(config.EVM()), txmgr.NewEvmTxmFeeConfig(ge), config.EVM().Transactions(), gconfig.Database(), ethKeyStore, txBuilder, lggr, stuckTxDetector, ht) ctx := tests.Context(t) // Can't close unstarted instance @@ -145,26 +147,27 @@ func TestEthConfirmer_Lifecycle(t *testing.T) { // Can't start an already started instance err = ec.Start(ctx) require.Error(t, err) + + latestFinalizedHead := evmtypes.Head{ + Number: 8, + Hash: testutils.NewHash(), + Parent: nil, + IsFinalized: true, // We are guaranteed to receive a latestFinalizedHead. + } + head := evmtypes.Head{ Hash: testutils.NewHash(), Number: 10, Parent: &evmtypes.Head{ Hash: testutils.NewHash(), Number: 9, - Parent: &evmtypes.Head{ - Number: 8, - Hash: testutils.NewHash(), - Parent: nil, - }, + Parent: &latestFinalizedHead, }, } - // If there's no finalized head yet, returns error. - err = ec.ProcessHead(ctx, &head) - require.Error(t, err) + ethClient.On("HeadByNumber", mock.Anything, (*big.Int)(nil)).Return(&head, nil).Once() + ethClient.On("LatestFinalizedBlock", mock.Anything).Return(&latestFinalizedHead, nil).Once() - // Successful after having a head finalized in the chain - head.Parent.Parent.IsFinalized = true err = ec.ProcessHead(ctx, &head) require.NoError(t, err) // Can successfully close once @@ -1663,8 +1666,9 @@ func TestEthConfirmer_RebroadcastWhereNecessary_WithConnectivityCheck(t *testing addresses := []gethCommon.Address{fromAddress} kst.On("EnabledAddressesForChain", mock.Anything, &cltest.FixtureChainID).Return(addresses, nil).Maybe() stuckTxDetector := txmgr.NewStuckTxDetector(lggr, testutils.FixtureChainID, "", assets.NewWei(assets.NewEth(100).ToInt()), ccfg.EVM().Transactions().AutoPurge(), feeEstimator, txStore, ethClient) + ht := headtracker.NewSimulatedHeadTracker(ethClient, true, 0) // Create confirmer with necessary state - ec := txmgr.NewEvmConfirmer(txStore, txmgr.NewEvmTxmClient(ethClient, nil), ccfg.EVM(), txmgr.NewEvmTxmFeeConfig(ccfg.EVM().GasEstimator()), ccfg.EVM().Transactions(), cfg.Database(), kst, txBuilder, lggr, stuckTxDetector) + ec := txmgr.NewEvmConfirmer(txStore, txmgr.NewEvmTxmClient(ethClient, nil), ccfg.EVM(), txmgr.NewEvmTxmFeeConfig(ccfg.EVM().GasEstimator()), ccfg.EVM().Transactions(), cfg.Database(), kst, txBuilder, lggr, stuckTxDetector, ht) servicetest.Run(t, ec) currentHead := int64(30) oldEnough := int64(15) @@ -1712,7 +1716,8 @@ func TestEthConfirmer_RebroadcastWhereNecessary_WithConnectivityCheck(t *testing addresses := []gethCommon.Address{fromAddress} kst.On("EnabledAddressesForChain", mock.Anything, &cltest.FixtureChainID).Return(addresses, nil).Maybe() stuckTxDetector := txmgr.NewStuckTxDetector(lggr, testutils.FixtureChainID, "", assets.NewWei(assets.NewEth(100).ToInt()), ccfg.EVM().Transactions().AutoPurge(), feeEstimator, txStore, ethClient) - ec := txmgr.NewEvmConfirmer(txStore, txmgr.NewEvmTxmClient(ethClient, nil), ccfg.EVM(), txmgr.NewEvmTxmFeeConfig(ccfg.EVM().GasEstimator()), ccfg.EVM().Transactions(), cfg.Database(), kst, txBuilder, lggr, stuckTxDetector) + ht := headtracker.NewSimulatedHeadTracker(ethClient, true, 0) + ec := txmgr.NewEvmConfirmer(txStore, txmgr.NewEvmTxmClient(ethClient, nil), ccfg.EVM(), txmgr.NewEvmTxmFeeConfig(ccfg.EVM().GasEstimator()), ccfg.EVM().Transactions(), cfg.Database(), kst, txBuilder, lggr, stuckTxDetector, ht) servicetest.Run(t, ec) currentHead := int64(30) oldEnough := int64(15) @@ -2684,6 +2689,13 @@ func TestEthConfirmer_EnsureConfirmedTransactionsInLongestChain(t *testing.T) { gconfig, config := newTestChainScopedConfig(t) ec := newEthConfirmer(t, txStore, ethClient, gconfig, config, ethKeyStore, nil) + latestFinalizedHead := evmtypes.Head{ + Number: 8, + Hash: testutils.NewHash(), + Parent: nil, + IsFinalized: false, // Not finalized yet to check if it's handed gracefully. + } + head := evmtypes.Head{ Hash: testutils.NewHash(), Number: 10, @@ -2693,27 +2705,27 @@ func TestEthConfirmer_EnsureConfirmedTransactionsInLongestChain(t *testing.T) { Parent: &evmtypes.Head{ Number: 8, Hash: testutils.NewHash(), - Parent: nil, + Parent: &latestFinalizedHead, }, }, } - t.Run("error if there's no LatestFinalizedHead", func(t *testing.T) { - require.Error(t, ec.EnsureConfirmedTransactionsInLongestChain(tests.Context(t), &head)) + t.Run("No latest finalized head is handed gracefully.", func(t *testing.T) { + require.NoError(t, ec.EnsureConfirmedTransactionsInLongestChain(tests.Context(t), &head, &latestFinalizedHead)) }) - // Finalized a head - head.Parent.Parent.IsFinalized = true + // Finalize Latest Finalized Head + latestFinalizedHead.IsFinalized = true t.Run("does nothing if there aren't any transactions", func(t *testing.T) { - require.NoError(t, ec.EnsureConfirmedTransactionsInLongestChain(tests.Context(t), &head)) + require.NoError(t, ec.EnsureConfirmedTransactionsInLongestChain(tests.Context(t), &head, &latestFinalizedHead)) }) t.Run("does nothing to unconfirmed transactions", func(t *testing.T) { etx := cltest.MustInsertUnconfirmedEthTxWithBroadcastLegacyAttempt(t, txStore, 0, fromAddress) // Do the thing - require.NoError(t, ec.EnsureConfirmedTransactionsInLongestChain(tests.Context(t), &head)) + require.NoError(t, ec.EnsureConfirmedTransactionsInLongestChain(tests.Context(t), &head, &latestFinalizedHead)) etx, err := txStore.FindTxWithAttempts(ctx, etx.ID) require.NoError(t, err) @@ -2725,7 +2737,7 @@ func TestEthConfirmer_EnsureConfirmedTransactionsInLongestChain(t *testing.T) { mustInsertEthReceipt(t, txStore, head.Number, head.Hash, etx.TxAttempts[0].Hash) // Do the thing - require.NoError(t, ec.EnsureConfirmedTransactionsInLongestChain(tests.Context(t), &head)) + require.NoError(t, ec.EnsureConfirmedTransactionsInLongestChain(tests.Context(t), &head, &latestFinalizedHead)) etx, err := txStore.FindTxWithAttempts(ctx, etx.ID) require.NoError(t, err) @@ -2738,7 +2750,7 @@ func TestEthConfirmer_EnsureConfirmedTransactionsInLongestChain(t *testing.T) { mustInsertEthReceipt(t, txStore, head.Parent.Parent.Number-1, testutils.NewHash(), etx.TxAttempts[0].Hash) // Do the thing - require.NoError(t, ec.EnsureConfirmedTransactionsInLongestChain(tests.Context(t), &head)) + require.NoError(t, ec.EnsureConfirmedTransactionsInLongestChain(tests.Context(t), &head, &latestFinalizedHead)) etx, err := txStore.FindTxWithAttempts(ctx, etx.ID) require.NoError(t, err) @@ -2759,7 +2771,7 @@ func TestEthConfirmer_EnsureConfirmedTransactionsInLongestChain(t *testing.T) { }), fromAddress).Return(commonclient.Successful, nil).Once() // Do the thing - require.NoError(t, ec.EnsureConfirmedTransactionsInLongestChain(tests.Context(t), &head)) + require.NoError(t, ec.EnsureConfirmedTransactionsInLongestChain(tests.Context(t), &head, &latestFinalizedHead)) etx, err := txStore.FindTxWithAttempts(ctx, etx.ID) require.NoError(t, err) @@ -2782,7 +2794,7 @@ func TestEthConfirmer_EnsureConfirmedTransactionsInLongestChain(t *testing.T) { commonclient.Successful, nil).Once() // Do the thing - require.NoError(t, ec.EnsureConfirmedTransactionsInLongestChain(tests.Context(t), &head)) + require.NoError(t, ec.EnsureConfirmedTransactionsInLongestChain(tests.Context(t), &head, &latestFinalizedHead)) etx, err := txStore.FindTxWithAttempts(ctx, etx.ID) require.NoError(t, err) @@ -2817,7 +2829,7 @@ func TestEthConfirmer_EnsureConfirmedTransactionsInLongestChain(t *testing.T) { }), fromAddress).Return(commonclient.Successful, nil).Once() // Do the thing - require.NoError(t, ec.EnsureConfirmedTransactionsInLongestChain(tests.Context(t), &head)) + require.NoError(t, ec.EnsureConfirmedTransactionsInLongestChain(tests.Context(t), &head, &latestFinalizedHead)) etx, err := txStore.FindTxWithAttempts(ctx, etx.ID) require.NoError(t, err) @@ -2837,7 +2849,7 @@ func TestEthConfirmer_EnsureConfirmedTransactionsInLongestChain(t *testing.T) { // Add receipt that is higher than head mustInsertEthReceipt(t, txStore, head.Number+1, testutils.NewHash(), attempt.Hash) - require.NoError(t, ec.EnsureConfirmedTransactionsInLongestChain(tests.Context(t), &head)) + require.NoError(t, ec.EnsureConfirmedTransactionsInLongestChain(tests.Context(t), &head, &latestFinalizedHead)) etx, err := txStore.FindTxWithAttempts(ctx, etx.ID) require.NoError(t, err) @@ -3177,7 +3189,8 @@ func TestEthConfirmer_ProcessStuckTransactions(t *testing.T) { ge := evmcfg.EVM().GasEstimator() txBuilder := txmgr.NewEvmTxAttemptBuilder(*ethClient.ConfiguredChainID(), ge, ethKeyStore, feeEstimator) stuckTxDetector := txmgr.NewStuckTxDetector(lggr, testutils.FixtureChainID, "", assets.NewWei(assets.NewEth(100).ToInt()), evmcfg.EVM().Transactions().AutoPurge(), feeEstimator, txStore, ethClient) - ec := txmgr.NewEvmConfirmer(txStore, txmgr.NewEvmTxmClient(ethClient, nil), txmgr.NewEvmTxmConfig(evmcfg.EVM()), txmgr.NewEvmTxmFeeConfig(ge), evmcfg.EVM().Transactions(), cfg.Database(), ethKeyStore, txBuilder, lggr, stuckTxDetector) + ht := headtracker.NewSimulatedHeadTracker(ethClient, true, 0) + ec := txmgr.NewEvmConfirmer(txStore, txmgr.NewEvmTxmClient(ethClient, nil), txmgr.NewEvmTxmConfig(evmcfg.EVM()), txmgr.NewEvmTxmFeeConfig(ge), evmcfg.EVM().Transactions(), cfg.Database(), ethKeyStore, txBuilder, lggr, stuckTxDetector, ht) servicetest.Run(t, ec) ctx := tests.Context(t) @@ -3195,6 +3208,9 @@ func TestEthConfirmer_ProcessStuckTransactions(t *testing.T) { Number: blockNum, IsFinalized: true, } + + ethClient.On("HeadByNumber", mock.Anything, (*big.Int)(nil)).Return(&head, nil).Once() + ethClient.On("LatestFinalizedBlock", mock.Anything).Return(&head, nil).Once() ethClient.On("SequenceAt", mock.Anything, mock.Anything, mock.Anything).Return(evmtypes.Nonce(0), nil).Once() ethClient.On("BatchCallContext", mock.Anything, mock.Anything).Return(nil).Once() @@ -3220,6 +3236,8 @@ func TestEthConfirmer_ProcessStuckTransactions(t *testing.T) { Number: blockNum + 1, IsFinalized: true, } + ethClient.On("HeadByNumber", mock.Anything, (*big.Int)(nil)).Return(&head, nil).Once() + ethClient.On("LatestFinalizedBlock", mock.Anything).Return(&head, nil).Once() ethClient.On("SequenceAt", mock.Anything, mock.Anything, mock.Anything).Return(evmtypes.Nonce(1), nil) ethClient.On("BatchCallContext", mock.Anything, mock.MatchedBy(func(b []rpc.BatchElem) bool { return len(b) == 4 && cltest.BatchElemMatchesParams(b[0], latestAttempt.Hash, "eth_getTransactionReceipt") @@ -3258,7 +3276,8 @@ func newEthConfirmer(t testing.TB, txStore txmgr.EvmTxStore, ethClient client.Cl }, ge.EIP1559DynamicFees(), ge) txBuilder := txmgr.NewEvmTxAttemptBuilder(*ethClient.ConfiguredChainID(), ge, ks, estimator) stuckTxDetector := txmgr.NewStuckTxDetector(lggr, testutils.FixtureChainID, "", assets.NewWei(assets.NewEth(100).ToInt()), config.EVM().Transactions().AutoPurge(), estimator, txStore, ethClient) - ec := txmgr.NewEvmConfirmer(txStore, txmgr.NewEvmTxmClient(ethClient, nil), txmgr.NewEvmTxmConfig(config.EVM()), txmgr.NewEvmTxmFeeConfig(ge), config.EVM().Transactions(), gconfig.Database(), ks, txBuilder, lggr, stuckTxDetector) + ht := headtracker.NewSimulatedHeadTracker(ethClient, true, 0) + ec := txmgr.NewEvmConfirmer(txStore, txmgr.NewEvmTxmClient(ethClient, nil), txmgr.NewEvmTxmConfig(config.EVM()), txmgr.NewEvmTxmFeeConfig(ge), config.EVM().Transactions(), gconfig.Database(), ks, txBuilder, lggr, stuckTxDetector, ht) ec.SetResumeCallback(fn) servicetest.Run(t, ec) return ec diff --git a/core/chains/evm/txmgr/txmgr_test.go b/core/chains/evm/txmgr/txmgr_test.go index 40df5616c99..93a00682523 100644 --- a/core/chains/evm/txmgr/txmgr_test.go +++ b/core/chains/evm/txmgr/txmgr_test.go @@ -85,7 +85,8 @@ func makeTestEvmTxm( lggr, lp, keyStore, - estimator) + estimator, + ht) } func TestTxm_SendNativeToken_DoesNotSendToZero(t *testing.T) { @@ -612,6 +613,7 @@ func TestTxm_GetTransactionStatus(t *testing.T) { feeEstimator := gasmocks.NewEvmFeeEstimator(t) feeEstimator.On("Start", mock.Anything).Return(nil).Once() feeEstimator.On("Close", mock.Anything).Return(nil).Once() + ethClient.On("HeadByNumber", mock.Anything, (*big.Int)(nil)).Return(nil, nil) feeEstimator.On("OnNewLongestChain", mock.Anything, mock.Anything).Once() txm, err := makeTestEvmTxm(t, db, ethClient, feeEstimator, cfg.EVM(), cfg.EVM().GasEstimator(), cfg.EVM().Transactions(), gcfg.Database(), gcfg.Database().Listener(), ethKeyStore) require.NoError(t, err) diff --git a/core/services/promreporter/prom_reporter_test.go b/core/services/promreporter/prom_reporter_test.go index b61fa25bdc4..a0a4a247c21 100644 --- a/core/services/promreporter/prom_reporter_test.go +++ b/core/services/promreporter/prom_reporter_test.go @@ -62,7 +62,8 @@ func newLegacyChainContainer(t *testing.T, db *sqlx.DB) legacyevm.LegacyChainCon lggr, lp, keyStore, - estimator) + estimator, + ht) require.NoError(t, err) cfg := configtest.NewGeneralConfig(t, nil) diff --git a/core/services/vrf/delegate_test.go b/core/services/vrf/delegate_test.go index 889b19d0e04..9718dc376a7 100644 --- a/core/services/vrf/delegate_test.go +++ b/core/services/vrf/delegate_test.go @@ -83,7 +83,7 @@ func buildVrfUni(t *testing.T, db *sqlx.DB, cfg chainlink.GeneralConfig) vrfUniv btORM := bridges.NewORM(db) ks := keystore.NewInMemory(db, utils.FastScryptParams, lggr) _, dbConfig, evmConfig := txmgr.MakeTestConfigs(t) - txm, err := txmgr.NewTxm(db, evmConfig, evmConfig.GasEstimator(), evmConfig.Transactions(), nil, dbConfig, dbConfig.Listener(), ec, logger.TestLogger(t), nil, ks.Eth(), nil) + txm, err := txmgr.NewTxm(db, evmConfig, evmConfig.GasEstimator(), evmConfig.Transactions(), nil, dbConfig, dbConfig.Listener(), ec, logger.TestLogger(t), nil, ks.Eth(), nil, nil) orm := headtracker.NewORM(*testutils.FixtureChainID, db) require.NoError(t, orm.IdempotentInsertHead(testutils.Context(t), cltest.Head(51))) jrm := job.NewORM(db, prm, btORM, ks, lggr) From 1188b4b7c4dba838fddb5330ed0fce631e2277b6 Mon Sep 17 00:00:00 2001 From: Farber98 Date: Wed, 31 Jul 2024 09:12:13 -0300 Subject: [PATCH 18/37] mocks --- common/txmgr/types/mocks/tx_store.go | 18 +++++++++--------- core/chains/evm/txmgr/mocks/evm_tx_store.go | 18 +++++++++--------- 2 files changed, 18 insertions(+), 18 deletions(-) diff --git a/common/txmgr/types/mocks/tx_store.go b/common/txmgr/types/mocks/tx_store.go index 96a8cd344bb..881d1bb09e1 100644 --- a/common/txmgr/types/mocks/tx_store.go +++ b/common/txmgr/types/mocks/tx_store.go @@ -1913,9 +1913,9 @@ func (_c *TxStore_MarkAllConfirmedMissingReceipt_Call[ADDR, CHAIN_ID, TX_HASH, B return _c } -// MarkOldTxesMissingReceiptAsErrored provides a mock function with given fields: ctx, blockNum, finalityDepth, chainID -func (_m *TxStore[ADDR, CHAIN_ID, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) MarkOldTxesMissingReceiptAsErrored(ctx context.Context, blockNum int64, finalityDepth uint32, chainID CHAIN_ID) error { - ret := _m.Called(ctx, blockNum, finalityDepth, chainID) +// MarkOldTxesMissingReceiptAsErrored provides a mock function with given fields: ctx, blockNum, latestFinalizedBlockNum, chainID +func (_m *TxStore[ADDR, CHAIN_ID, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) MarkOldTxesMissingReceiptAsErrored(ctx context.Context, blockNum int64, latestFinalizedBlockNum int64, chainID CHAIN_ID) error { + ret := _m.Called(ctx, blockNum, latestFinalizedBlockNum, chainID) if len(ret) == 0 { panic("no return value specified for MarkOldTxesMissingReceiptAsErrored") @@ -1939,15 +1939,15 @@ type TxStore_MarkOldTxesMissingReceiptAsErrored_Call[ADDR types.Hashable, CHAIN_ // MarkOldTxesMissingReceiptAsErrored is a helper method to define mock.On call // - ctx context.Context // - blockNum int64 -// - finalityDepth uint32 +// - latestFinalizedBlockNum int64 // - chainID CHAIN_ID -func (_e *TxStore_Expecter[ADDR, CHAIN_ID, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) MarkOldTxesMissingReceiptAsErrored(ctx interface{}, blockNum interface{}, finalityDepth interface{}, chainID interface{}) *TxStore_MarkOldTxesMissingReceiptAsErrored_Call[ADDR, CHAIN_ID, TX_HASH, BLOCK_HASH, R, SEQ, FEE] { - return &TxStore_MarkOldTxesMissingReceiptAsErrored_Call[ADDR, CHAIN_ID, TX_HASH, BLOCK_HASH, R, SEQ, FEE]{Call: _e.mock.On("MarkOldTxesMissingReceiptAsErrored", ctx, blockNum, finalityDepth, chainID)} +func (_e *TxStore_Expecter[ADDR, CHAIN_ID, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) MarkOldTxesMissingReceiptAsErrored(ctx interface{}, blockNum interface{}, latestFinalizedBlockNum interface{}, chainID interface{}) *TxStore_MarkOldTxesMissingReceiptAsErrored_Call[ADDR, CHAIN_ID, TX_HASH, BLOCK_HASH, R, SEQ, FEE] { + return &TxStore_MarkOldTxesMissingReceiptAsErrored_Call[ADDR, CHAIN_ID, TX_HASH, BLOCK_HASH, R, SEQ, FEE]{Call: _e.mock.On("MarkOldTxesMissingReceiptAsErrored", ctx, blockNum, latestFinalizedBlockNum, chainID)} } -func (_c *TxStore_MarkOldTxesMissingReceiptAsErrored_Call[ADDR, CHAIN_ID, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) Run(run func(ctx context.Context, blockNum int64, finalityDepth uint32, chainID CHAIN_ID)) *TxStore_MarkOldTxesMissingReceiptAsErrored_Call[ADDR, CHAIN_ID, TX_HASH, BLOCK_HASH, R, SEQ, FEE] { +func (_c *TxStore_MarkOldTxesMissingReceiptAsErrored_Call[ADDR, CHAIN_ID, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) Run(run func(ctx context.Context, blockNum int64, latestFinalizedBlockNum int64, chainID CHAIN_ID)) *TxStore_MarkOldTxesMissingReceiptAsErrored_Call[ADDR, CHAIN_ID, TX_HASH, BLOCK_HASH, R, SEQ, FEE] { _c.Call.Run(func(args mock.Arguments) { - run(args[0].(context.Context), args[1].(int64), args[2].(uint32), args[3].(CHAIN_ID)) + run(args[0].(context.Context), args[1].(int64), args[2].(int64), args[3].(CHAIN_ID)) }) return _c } @@ -1957,7 +1957,7 @@ func (_c *TxStore_MarkOldTxesMissingReceiptAsErrored_Call[ADDR, CHAIN_ID, TX_HAS return _c } -func (_c *TxStore_MarkOldTxesMissingReceiptAsErrored_Call[ADDR, CHAIN_ID, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) RunAndReturn(run func(context.Context, int64, uint32, CHAIN_ID) error) *TxStore_MarkOldTxesMissingReceiptAsErrored_Call[ADDR, CHAIN_ID, TX_HASH, BLOCK_HASH, R, SEQ, FEE] { +func (_c *TxStore_MarkOldTxesMissingReceiptAsErrored_Call[ADDR, CHAIN_ID, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) RunAndReturn(run func(context.Context, int64, int64, CHAIN_ID) error) *TxStore_MarkOldTxesMissingReceiptAsErrored_Call[ADDR, CHAIN_ID, TX_HASH, BLOCK_HASH, R, SEQ, FEE] { _c.Call.Return(run) return _c } diff --git a/core/chains/evm/txmgr/mocks/evm_tx_store.go b/core/chains/evm/txmgr/mocks/evm_tx_store.go index 54dac40325e..32247a90a68 100644 --- a/core/chains/evm/txmgr/mocks/evm_tx_store.go +++ b/core/chains/evm/txmgr/mocks/evm_tx_store.go @@ -2211,9 +2211,9 @@ func (_c *EvmTxStore_MarkAllConfirmedMissingReceipt_Call) RunAndReturn(run func( return _c } -// MarkOldTxesMissingReceiptAsErrored provides a mock function with given fields: ctx, blockNum, finalityDepth, chainID -func (_m *EvmTxStore) MarkOldTxesMissingReceiptAsErrored(ctx context.Context, blockNum int64, finalityDepth uint32, chainID *big.Int) error { - ret := _m.Called(ctx, blockNum, finalityDepth, chainID) +// MarkOldTxesMissingReceiptAsErrored provides a mock function with given fields: ctx, blockNum, latestFinalizedBlockNum, chainID +func (_m *EvmTxStore) MarkOldTxesMissingReceiptAsErrored(ctx context.Context, blockNum int64, latestFinalizedBlockNum int64, chainID *big.Int) error { + ret := _m.Called(ctx, blockNum, latestFinalizedBlockNum, chainID) if len(ret) == 0 { panic("no return value specified for MarkOldTxesMissingReceiptAsErrored") @@ -2237,15 +2237,15 @@ type EvmTxStore_MarkOldTxesMissingReceiptAsErrored_Call struct { // MarkOldTxesMissingReceiptAsErrored is a helper method to define mock.On call // - ctx context.Context // - blockNum int64 -// - finalityDepth uint32 +// - latestFinalizedBlockNum int64 // - chainID *big.Int -func (_e *EvmTxStore_Expecter) MarkOldTxesMissingReceiptAsErrored(ctx interface{}, blockNum interface{}, finalityDepth interface{}, chainID interface{}) *EvmTxStore_MarkOldTxesMissingReceiptAsErrored_Call { - return &EvmTxStore_MarkOldTxesMissingReceiptAsErrored_Call{Call: _e.mock.On("MarkOldTxesMissingReceiptAsErrored", ctx, blockNum, finalityDepth, chainID)} +func (_e *EvmTxStore_Expecter) MarkOldTxesMissingReceiptAsErrored(ctx interface{}, blockNum interface{}, latestFinalizedBlockNum interface{}, chainID interface{}) *EvmTxStore_MarkOldTxesMissingReceiptAsErrored_Call { + return &EvmTxStore_MarkOldTxesMissingReceiptAsErrored_Call{Call: _e.mock.On("MarkOldTxesMissingReceiptAsErrored", ctx, blockNum, latestFinalizedBlockNum, chainID)} } -func (_c *EvmTxStore_MarkOldTxesMissingReceiptAsErrored_Call) Run(run func(ctx context.Context, blockNum int64, finalityDepth uint32, chainID *big.Int)) *EvmTxStore_MarkOldTxesMissingReceiptAsErrored_Call { +func (_c *EvmTxStore_MarkOldTxesMissingReceiptAsErrored_Call) Run(run func(ctx context.Context, blockNum int64, latestFinalizedBlockNum int64, chainID *big.Int)) *EvmTxStore_MarkOldTxesMissingReceiptAsErrored_Call { _c.Call.Run(func(args mock.Arguments) { - run(args[0].(context.Context), args[1].(int64), args[2].(uint32), args[3].(*big.Int)) + run(args[0].(context.Context), args[1].(int64), args[2].(int64), args[3].(*big.Int)) }) return _c } @@ -2255,7 +2255,7 @@ func (_c *EvmTxStore_MarkOldTxesMissingReceiptAsErrored_Call) Return(_a0 error) return _c } -func (_c *EvmTxStore_MarkOldTxesMissingReceiptAsErrored_Call) RunAndReturn(run func(context.Context, int64, uint32, *big.Int) error) *EvmTxStore_MarkOldTxesMissingReceiptAsErrored_Call { +func (_c *EvmTxStore_MarkOldTxesMissingReceiptAsErrored_Call) RunAndReturn(run func(context.Context, int64, int64, *big.Int) error) *EvmTxStore_MarkOldTxesMissingReceiptAsErrored_Call { _c.Call.Return(run) return _c } From a0673ce78609f84026450125d5b92c5e5f63bf49 Mon Sep 17 00:00:00 2001 From: Farber98 Date: Wed, 31 Jul 2024 09:56:47 -0300 Subject: [PATCH 19/37] remove unnecesary tests as finalized head is guaranteed --- core/chains/evm/txmgr/confirmer_test.go | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/core/chains/evm/txmgr/confirmer_test.go b/core/chains/evm/txmgr/confirmer_test.go index 250ec617dc7..1bdc1ce92d9 100644 --- a/core/chains/evm/txmgr/confirmer_test.go +++ b/core/chains/evm/txmgr/confirmer_test.go @@ -2693,7 +2693,7 @@ func TestEthConfirmer_EnsureConfirmedTransactionsInLongestChain(t *testing.T) { Number: 8, Hash: testutils.NewHash(), Parent: nil, - IsFinalized: false, // Not finalized yet to check if it's handed gracefully. + IsFinalized: false, // We are guaranteed to receive a latestFinalizedHead. } head := evmtypes.Head{ @@ -2709,14 +2709,6 @@ func TestEthConfirmer_EnsureConfirmedTransactionsInLongestChain(t *testing.T) { }, }, } - - t.Run("No latest finalized head is handed gracefully.", func(t *testing.T) { - require.NoError(t, ec.EnsureConfirmedTransactionsInLongestChain(tests.Context(t), &head, &latestFinalizedHead)) - }) - - // Finalize Latest Finalized Head - latestFinalizedHead.IsFinalized = true - t.Run("does nothing if there aren't any transactions", func(t *testing.T) { require.NoError(t, ec.EnsureConfirmedTransactionsInLongestChain(tests.Context(t), &head, &latestFinalizedHead)) }) From b10c57d9639a7f9dfef7604d625e6e635a4bcf52 Mon Sep 17 00:00:00 2001 From: Farber98 Date: Wed, 31 Jul 2024 11:06:44 -0300 Subject: [PATCH 20/37] ht in shell local --- core/cmd/shell_local.go | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/core/cmd/shell_local.go b/core/cmd/shell_local.go index 3f6ccb9ec51..bbe7f46e1a5 100644 --- a/core/cmd/shell_local.go +++ b/core/cmd/shell_local.go @@ -34,6 +34,7 @@ import ( "github.com/jmoiron/sqlx" cutils "github.com/smartcontractkit/chainlink-common/pkg/utils" + "github.com/smartcontractkit/chainlink-common/pkg/utils/mailbox" "github.com/smartcontractkit/chainlink/v2/core/build" "github.com/smartcontractkit/chainlink/v2/core/chains/evm/assets" @@ -669,7 +670,11 @@ func (s *Shell) RebroadcastTransactions(c *cli.Context) (err error) { cfg := txmgr.NewEvmTxmConfig(chain.Config().EVM()) feeCfg := txmgr.NewEvmTxmFeeConfig(chain.Config().EVM().GasEstimator()) stuckTxDetector := txmgr.NewStuckTxDetector(lggr, ethClient.ConfiguredChainID(), "", assets.NewWei(assets.NewEth(100).ToInt()), chain.Config().EVM().Transactions().AutoPurge(), nil, orm, ethClient) - ht := headtracker.NewSimulatedHeadTracker(ethClient, true, 0) + htConfig := chain.Config().EVM().HeadTracker() + hb := headtracker.NewHeadBroadcaster(lggr) + htORM := headtracker.NewORM(*chain.ID(), db) + hs := headtracker.NewHeadSaver(lggr, htORM, chain.Config().EVM(), htConfig) + ht := headtracker.NewHeadTracker(lggr, ethClient, chain.Config().EVM(), htConfig, hb, hs, mailbox.NewMonitor(app.ID().String(), lggr)) ec := txmgr.NewEvmConfirmer(orm, txmgr.NewEvmTxmClient(ethClient, chain.Config().EVM().NodePool().Errors()), cfg, feeCfg, chain.Config().EVM().Transactions(), app.GetConfig().Database(), keyStore.Eth(), txBuilder, chain.Logger(), stuckTxDetector, ht) totalNonces := endingNonce - beginningNonce + 1 From 62278e0dacb2f1655a9188aa58d56b5dfdcf6a58 Mon Sep 17 00:00:00 2001 From: Farber98 Date: Thu, 1 Aug 2024 09:25:44 -0300 Subject: [PATCH 21/37] remove evm dep from confirmer and use generics instead --- common/txmgr/confirmer.go | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/common/txmgr/confirmer.go b/common/txmgr/confirmer.go index 7125277e3e4..acde28d11d9 100644 --- a/common/txmgr/confirmer.go +++ b/common/txmgr/confirmer.go @@ -27,7 +27,6 @@ import ( iutils "github.com/smartcontractkit/chainlink/v2/common/internal/utils" txmgrtypes "github.com/smartcontractkit/chainlink/v2/common/txmgr/types" "github.com/smartcontractkit/chainlink/v2/common/types" - evmtypes "github.com/smartcontractkit/chainlink/v2/core/chains/evm/types" ) const ( @@ -103,8 +102,8 @@ var ( }, []string{"chainID"}) ) -type confirmerHeadTracker interface { - LatestAndFinalizedBlock(ctx context.Context) (latest, finalized *evmtypes.Head, err error) +type confirmerHeadTracker[HEAD types.Head[BLOCK_HASH], BLOCK_HASH types.Hashable] interface { + LatestAndFinalizedBlock(ctx context.Context) (latest, finalized HEAD, err error) } // Confirmer is a broad service which performs four different tasks in sequence on every new longest chain @@ -147,7 +146,7 @@ type Confirmer[ nConsecutiveBlocksChainTooShort int isReceiptNil func(R) bool - headTracker confirmerHeadTracker + headTracker confirmerHeadTracker[HEAD, BLOCK_HASH] } func NewConfirmer[ @@ -171,7 +170,7 @@ func NewConfirmer[ lggr logger.Logger, isReceiptNil func(R) bool, stuckTxDetector txmgrtypes.StuckTxDetector[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], - headTracker confirmerHeadTracker, + headTracker confirmerHeadTracker[HEAD, BLOCK_HASH], ) *Confirmer[CHAIN_ID, HEAD, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE] { lggr = logger.Named(lggr, "Confirmer") return &Confirmer[CHAIN_ID, HEAD, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]{ @@ -1020,7 +1019,7 @@ func (ec *Confirmer[CHAIN_ID, HEAD, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) han // // If any of the confirmed transactions does not have a receipt in the chain, it has been // re-org'd out and will be rebroadcast. -func (ec *Confirmer[CHAIN_ID, HEAD, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) EnsureConfirmedTransactionsInLongestChain(ctx context.Context, head types.Head[BLOCK_HASH], latestFinalizedHead *evmtypes.Head) error { +func (ec *Confirmer[CHAIN_ID, HEAD, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) EnsureConfirmedTransactionsInLongestChain(ctx context.Context, head, latestFinalizedHead types.Head[BLOCK_HASH]) error { if head.ChainLength() < uint32(latestFinalizedHead.BlockNumber()) { logArgs := []interface{}{ "chainLength", head.ChainLength(), "latestFinalizedHead", latestFinalizedHead.BlockNumber(), From 2adcd80cd653c1d18558b53b06fc547667dbdf97 Mon Sep 17 00:00:00 2001 From: Farber98 Date: Thu, 1 Aug 2024 09:28:56 -0300 Subject: [PATCH 22/37] make markOldTxesMissingReceiptAsErrored condition inclusive --- core/chains/evm/txmgr/evm_tx_store.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/chains/evm/txmgr/evm_tx_store.go b/core/chains/evm/txmgr/evm_tx_store.go index 9484d3922ed..b6426c9a029 100644 --- a/core/chains/evm/txmgr/evm_tx_store.go +++ b/core/chains/evm/txmgr/evm_tx_store.go @@ -1475,7 +1475,7 @@ FROM ( WHERE e2.state = 'confirmed_missing_receipt' AND e2.evm_chain_id = $3 GROUP BY e2.id - HAVING max(evm.tx_attempts.broadcast_before_block_num) < $2 + HAVING max(evm.tx_attempts.broadcast_before_block_num) <= $2 ) FOR UPDATE OF e1 ) e0 From 4fb2252ab8d5284e4e7d58e81276454f14d1a6e4 Mon Sep 17 00:00:00 2001 From: Farber98 Date: Thu, 1 Aug 2024 09:31:12 -0300 Subject: [PATCH 23/37] use already initialized head tracker within shell local --- core/cmd/shell_local.go | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/core/cmd/shell_local.go b/core/cmd/shell_local.go index bbe7f46e1a5..ed8b653c05b 100644 --- a/core/cmd/shell_local.go +++ b/core/cmd/shell_local.go @@ -34,12 +34,10 @@ import ( "github.com/jmoiron/sqlx" cutils "github.com/smartcontractkit/chainlink-common/pkg/utils" - "github.com/smartcontractkit/chainlink-common/pkg/utils/mailbox" "github.com/smartcontractkit/chainlink/v2/core/build" "github.com/smartcontractkit/chainlink/v2/core/chains/evm/assets" "github.com/smartcontractkit/chainlink/v2/core/chains/evm/gas" - "github.com/smartcontractkit/chainlink/v2/core/chains/evm/headtracker" "github.com/smartcontractkit/chainlink/v2/core/chains/evm/txmgr" evmtypes "github.com/smartcontractkit/chainlink/v2/core/chains/evm/types" ubig "github.com/smartcontractkit/chainlink/v2/core/chains/evm/utils/big" @@ -670,13 +668,8 @@ func (s *Shell) RebroadcastTransactions(c *cli.Context) (err error) { cfg := txmgr.NewEvmTxmConfig(chain.Config().EVM()) feeCfg := txmgr.NewEvmTxmFeeConfig(chain.Config().EVM().GasEstimator()) stuckTxDetector := txmgr.NewStuckTxDetector(lggr, ethClient.ConfiguredChainID(), "", assets.NewWei(assets.NewEth(100).ToInt()), chain.Config().EVM().Transactions().AutoPurge(), nil, orm, ethClient) - htConfig := chain.Config().EVM().HeadTracker() - hb := headtracker.NewHeadBroadcaster(lggr) - htORM := headtracker.NewORM(*chain.ID(), db) - hs := headtracker.NewHeadSaver(lggr, htORM, chain.Config().EVM(), htConfig) - ht := headtracker.NewHeadTracker(lggr, ethClient, chain.Config().EVM(), htConfig, hb, hs, mailbox.NewMonitor(app.ID().String(), lggr)) ec := txmgr.NewEvmConfirmer(orm, txmgr.NewEvmTxmClient(ethClient, chain.Config().EVM().NodePool().Errors()), - cfg, feeCfg, chain.Config().EVM().Transactions(), app.GetConfig().Database(), keyStore.Eth(), txBuilder, chain.Logger(), stuckTxDetector, ht) + cfg, feeCfg, chain.Config().EVM().Transactions(), app.GetConfig().Database(), keyStore.Eth(), txBuilder, chain.Logger(), stuckTxDetector, chain.HeadTracker()) totalNonces := endingNonce - beginningNonce + 1 nonces := make([]evmtypes.Nonce, totalNonces) for i := int64(0); i < totalNonces; i++ { From 73aa5560506c433b75b94ac22639ab7ea3fa3536 Mon Sep 17 00:00:00 2001 From: Joe Huang Date: Thu, 1 Aug 2024 22:58:44 -0500 Subject: [PATCH 24/37] fix mocking, fix unit test --- core/cmd/shell_local_test.go | 3 +-- core/internal/cltest/mocks.go | 1 + 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/core/cmd/shell_local_test.go b/core/cmd/shell_local_test.go index 60545269e29..8ed48dcaa20 100644 --- a/core/cmd/shell_local_test.go +++ b/core/cmd/shell_local_test.go @@ -11,9 +11,8 @@ import ( commonconfig "github.com/smartcontractkit/chainlink-common/pkg/config" "github.com/smartcontractkit/chainlink-common/pkg/utils/mailbox" - "github.com/smartcontractkit/chainlink/v2/core/capabilities" - "github.com/smartcontractkit/chainlink/v2/common/client" + "github.com/smartcontractkit/chainlink/v2/core/capabilities" "github.com/smartcontractkit/chainlink/v2/core/chains/legacyevm" "github.com/smartcontractkit/chainlink/v2/core/cmd" cmdMocks "github.com/smartcontractkit/chainlink/v2/core/cmd/mocks" diff --git a/core/internal/cltest/mocks.go b/core/internal/cltest/mocks.go index 9e0ee2f3f20..fd01f72c131 100644 --- a/core/internal/cltest/mocks.go +++ b/core/internal/cltest/mocks.go @@ -392,6 +392,7 @@ func NewLegacyChainsWithMockChain(t testing.TB, ethClient evmclient.Client, cfg scopedCfg := evmtest.NewChainScopedConfig(t, cfg) ch.On("ID").Return(scopedCfg.EVM().ChainID()) ch.On("Config").Return(scopedCfg) + ch.On("HeadTracker").Return(nil) return NewLegacyChainsWithChain(ch, cfg) } From 242f41c6be2ec64e7bafd23b5567f390eba9b5c0 Mon Sep 17 00:00:00 2001 From: Joe Huang Date: Sun, 4 Aug 2024 21:00:08 -0500 Subject: [PATCH 25/37] fix a potential bug --- common/txmgr/confirmer.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/common/txmgr/confirmer.go b/common/txmgr/confirmer.go index 0cf4ddfb40c..d7ae31bdf9d 100644 --- a/common/txmgr/confirmer.go +++ b/common/txmgr/confirmer.go @@ -1024,7 +1024,7 @@ func (ec *Confirmer[CHAIN_ID, HEAD, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) han // If any of the confirmed transactions does not have a receipt in the chain, it has been // re-org'd out and will be rebroadcast. func (ec *Confirmer[CHAIN_ID, HEAD, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) EnsureConfirmedTransactionsInLongestChain(ctx context.Context, head, latestFinalizedHead types.Head[BLOCK_HASH]) error { - if head.ChainLength() < uint32(latestFinalizedHead.BlockNumber()) { + if head.BlockNumber() < latestFinalizedHead.BlockNumber() { logArgs := []interface{}{ "chainLength", head.ChainLength(), "latestFinalizedHead", latestFinalizedHead.BlockNumber(), } From 6a361f55086b49c4ec491016d592db969c28a5b5 Mon Sep 17 00:00:00 2001 From: Joe Huang Date: Sun, 4 Aug 2024 21:49:08 -0500 Subject: [PATCH 26/37] fix bug --- common/txmgr/confirmer.go | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/common/txmgr/confirmer.go b/common/txmgr/confirmer.go index d7ae31bdf9d..fdcbaa093d9 100644 --- a/common/txmgr/confirmer.go +++ b/common/txmgr/confirmer.go @@ -1024,21 +1024,26 @@ func (ec *Confirmer[CHAIN_ID, HEAD, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) han // If any of the confirmed transactions does not have a receipt in the chain, it has been // re-org'd out and will be rebroadcast. func (ec *Confirmer[CHAIN_ID, HEAD, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) EnsureConfirmedTransactionsInLongestChain(ctx context.Context, head, latestFinalizedHead types.Head[BLOCK_HASH]) error { + logArgs := []interface{}{ + "chainLength", head.ChainLength(), "latestFinalizedHead number", latestFinalizedHead.BlockNumber(), + } + if head.BlockNumber() < latestFinalizedHead.BlockNumber() { - logArgs := []interface{}{ - "chainLength", head.ChainLength(), "latestFinalizedHead", latestFinalizedHead.BlockNumber(), - } + warnMsg := "current head is shorter than latest finalized head" + ec.lggr.Warnw(warnMsg, append(logArgs, "head block number", head.BlockNumber())...) + } else if head.ChainLength() < uint32(head.BlockNumber()-latestFinalizedHead.BlockNumber()) { if ec.nConsecutiveBlocksChainTooShort > logAfterNConsecutiveBlocksChainTooShort { - warnMsg := "Chain length supplied for re-org detection was shorter than LatestFinalizedHead. Re-org protection is not working properly. This could indicate a problem with the remote RPC endpoint, a compatibility issue with a particular blockchain, a bug with this particular blockchain, heads table being truncated too early, remote node out of sync, or something else. If this happens a lot please raise a bug with the Chainlink team including a log output sample and details of the chain and RPC endpoint you are using." + warnMsg := "Chain length supplied for re-org detection was shorter than chain depth. Re-org protection is not working properly. This could indicate a problem with the remote RPC endpoint, a compatibility issue with a particular blockchain, a bug with this particular blockchain, heads table being truncated too early, remote node out of sync, or something else. If this happens a lot please raise a bug with the Chainlink team including a log output sample and details of the chain and RPC endpoint you are using." ec.lggr.Warnw(warnMsg, append(logArgs, "nConsecutiveBlocksChainTooShort", ec.nConsecutiveBlocksChainTooShort)...) } else { - logMsg := "Chain length supplied for re-org detection was shorter than LatestFinalizedHead" + logMsg := "Chain length supplied for re-org detection was shorter than chain depth" ec.lggr.Debugw(logMsg, append(logArgs, "nConsecutiveBlocksChainTooShort", ec.nConsecutiveBlocksChainTooShort)...) } ec.nConsecutiveBlocksChainTooShort++ } else { ec.nConsecutiveBlocksChainTooShort = 0 } + etxs, err := ec.txStore.FindTransactionsConfirmedInBlockRange(ctx, head.BlockNumber(), head.EarliestHeadInChain().BlockNumber(), ec.chainID) if err != nil { return fmt.Errorf("findTransactionsConfirmedInBlockRange failed: %w", err) From f53f260a4feed1ff57824e699e6556d9c676bd05 Mon Sep 17 00:00:00 2001 From: Joe Huang Date: Sun, 4 Aug 2024 22:02:39 -0500 Subject: [PATCH 27/37] refactor --- common/txmgr/confirmer.go | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/common/txmgr/confirmer.go b/common/txmgr/confirmer.go index fdcbaa093d9..ba0fee1f24d 100644 --- a/common/txmgr/confirmer.go +++ b/common/txmgr/confirmer.go @@ -1029,14 +1029,18 @@ func (ec *Confirmer[CHAIN_ID, HEAD, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) Ens } if head.BlockNumber() < latestFinalizedHead.BlockNumber() { - warnMsg := "current head is shorter than latest finalized head" - ec.lggr.Warnw(warnMsg, append(logArgs, "head block number", head.BlockNumber())...) - } else if head.ChainLength() < uint32(head.BlockNumber()-latestFinalizedHead.BlockNumber()) { + errMsg := "current head is shorter than latest finalized head" + ec.lggr.Errorw(errMsg, append(logArgs, "head block number", head.BlockNumber())...) + return errors.New(errMsg) + } + + unconfirmedChainLength := uint32(head.BlockNumber() - latestFinalizedHead.BlockNumber()) + if head.ChainLength() < unconfirmedChainLength { if ec.nConsecutiveBlocksChainTooShort > logAfterNConsecutiveBlocksChainTooShort { - warnMsg := "Chain length supplied for re-org detection was shorter than chain depth. Re-org protection is not working properly. This could indicate a problem with the remote RPC endpoint, a compatibility issue with a particular blockchain, a bug with this particular blockchain, heads table being truncated too early, remote node out of sync, or something else. If this happens a lot please raise a bug with the Chainlink team including a log output sample and details of the chain and RPC endpoint you are using." + warnMsg := "Chain length supplied for re-org detection was shorter than unconfirmed chain length. Re-org protection is not working properly. This could indicate a problem with the remote RPC endpoint, a compatibility issue with a particular blockchain, a bug with this particular blockchain, heads table being truncated too early, remote node out of sync, or something else. If this happens a lot please raise a bug with the Chainlink team including a log output sample and details of the chain and RPC endpoint you are using." ec.lggr.Warnw(warnMsg, append(logArgs, "nConsecutiveBlocksChainTooShort", ec.nConsecutiveBlocksChainTooShort)...) } else { - logMsg := "Chain length supplied for re-org detection was shorter than chain depth" + logMsg := "Chain length supplied for re-org detection was shorter than unconfirmed chain length" ec.lggr.Debugw(logMsg, append(logArgs, "nConsecutiveBlocksChainTooShort", ec.nConsecutiveBlocksChainTooShort)...) } ec.nConsecutiveBlocksChainTooShort++ From 6e71ff59892178bab313bbc6dfb04289439a99b2 Mon Sep 17 00:00:00 2001 From: Joe Huang Date: Mon, 5 Aug 2024 14:26:21 -0500 Subject: [PATCH 28/37] address comments --- common/txmgr/confirmer.go | 50 ++++++++----------------- core/chains/evm/txmgr/confirmer_test.go | 25 +++++-------- 2 files changed, 25 insertions(+), 50 deletions(-) diff --git a/common/txmgr/confirmer.go b/common/txmgr/confirmer.go index ba0fee1f24d..7d5949a82cb 100644 --- a/common/txmgr/confirmer.go +++ b/common/txmgr/confirmer.go @@ -137,14 +137,12 @@ type Confirmer[ ks txmgrtypes.KeyStore[ADDR, CHAIN_ID, SEQ] enabledAddresses []ADDR - mb *mailbox.Mailbox[HEAD] - stopCh services.StopChan - wg sync.WaitGroup - initSync sync.Mutex - isStarted bool - - nConsecutiveBlocksChainTooShort int - isReceiptNil func(R) bool + mb *mailbox.Mailbox[HEAD] + stopCh services.StopChan + wg sync.WaitGroup + initSync sync.Mutex + isStarted bool + isReceiptNil func(R) bool headTracker confirmerHeadTracker[HEAD, BLOCK_HASH] } @@ -310,6 +308,14 @@ func (ec *Confirmer[CHAIN_ID, HEAD, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) pro return fmt.Errorf("failed to retrieve latest finalized head: %w", err) } + if types.Head[BLOCK_HASH](latestFinalizedHead) == nil || !latestFinalizedHead.IsValid() { + return fmt.Errorf("latest finalized head is not valid") + } + + if latestFinalizedHead.BlockNumber() > head.BlockNumber() { + ec.lggr.Debugw("processHead receives old block", "latestFinalizedHead", latestFinalizedHead.BlockNumber(), "headNum", head.BlockNumber(), "time", time.Since(mark), "id", "confirmer") + } + if err := ec.CheckForReceipts(ctx, head.BlockNumber(), latestFinalizedHead.BlockNumber()); err != nil { return fmt.Errorf("CheckForReceipts failed: %w", err) } @@ -331,7 +337,7 @@ func (ec *Confirmer[CHAIN_ID, HEAD, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) pro ec.lggr.Debugw("Finished RebroadcastWhereNecessary", "headNum", head.BlockNumber(), "time", time.Since(mark), "id", "confirmer") mark = time.Now() - if err := ec.EnsureConfirmedTransactionsInLongestChain(ctx, head, latestFinalizedHead); err != nil { + if err := ec.EnsureConfirmedTransactionsInLongestChain(ctx, head); err != nil { return fmt.Errorf("EnsureConfirmedTransactionsInLongestChain failed: %w", err) } @@ -1023,31 +1029,7 @@ func (ec *Confirmer[CHAIN_ID, HEAD, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) han // // If any of the confirmed transactions does not have a receipt in the chain, it has been // re-org'd out and will be rebroadcast. -func (ec *Confirmer[CHAIN_ID, HEAD, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) EnsureConfirmedTransactionsInLongestChain(ctx context.Context, head, latestFinalizedHead types.Head[BLOCK_HASH]) error { - logArgs := []interface{}{ - "chainLength", head.ChainLength(), "latestFinalizedHead number", latestFinalizedHead.BlockNumber(), - } - - if head.BlockNumber() < latestFinalizedHead.BlockNumber() { - errMsg := "current head is shorter than latest finalized head" - ec.lggr.Errorw(errMsg, append(logArgs, "head block number", head.BlockNumber())...) - return errors.New(errMsg) - } - - unconfirmedChainLength := uint32(head.BlockNumber() - latestFinalizedHead.BlockNumber()) - if head.ChainLength() < unconfirmedChainLength { - if ec.nConsecutiveBlocksChainTooShort > logAfterNConsecutiveBlocksChainTooShort { - warnMsg := "Chain length supplied for re-org detection was shorter than unconfirmed chain length. Re-org protection is not working properly. This could indicate a problem with the remote RPC endpoint, a compatibility issue with a particular blockchain, a bug with this particular blockchain, heads table being truncated too early, remote node out of sync, or something else. If this happens a lot please raise a bug with the Chainlink team including a log output sample and details of the chain and RPC endpoint you are using." - ec.lggr.Warnw(warnMsg, append(logArgs, "nConsecutiveBlocksChainTooShort", ec.nConsecutiveBlocksChainTooShort)...) - } else { - logMsg := "Chain length supplied for re-org detection was shorter than unconfirmed chain length" - ec.lggr.Debugw(logMsg, append(logArgs, "nConsecutiveBlocksChainTooShort", ec.nConsecutiveBlocksChainTooShort)...) - } - ec.nConsecutiveBlocksChainTooShort++ - } else { - ec.nConsecutiveBlocksChainTooShort = 0 - } - +func (ec *Confirmer[CHAIN_ID, HEAD, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) EnsureConfirmedTransactionsInLongestChain(ctx context.Context, head types.Head[BLOCK_HASH]) error { etxs, err := ec.txStore.FindTransactionsConfirmedInBlockRange(ctx, head.BlockNumber(), head.EarliestHeadInChain().BlockNumber(), ec.chainID) if err != nil { return fmt.Errorf("findTransactionsConfirmedInBlockRange failed: %w", err) diff --git a/core/chains/evm/txmgr/confirmer_test.go b/core/chains/evm/txmgr/confirmer_test.go index 1bdc1ce92d9..f5e6e715e57 100644 --- a/core/chains/evm/txmgr/confirmer_test.go +++ b/core/chains/evm/txmgr/confirmer_test.go @@ -2689,13 +2689,6 @@ func TestEthConfirmer_EnsureConfirmedTransactionsInLongestChain(t *testing.T) { gconfig, config := newTestChainScopedConfig(t) ec := newEthConfirmer(t, txStore, ethClient, gconfig, config, ethKeyStore, nil) - latestFinalizedHead := evmtypes.Head{ - Number: 8, - Hash: testutils.NewHash(), - Parent: nil, - IsFinalized: false, // We are guaranteed to receive a latestFinalizedHead. - } - head := evmtypes.Head{ Hash: testutils.NewHash(), Number: 10, @@ -2705,19 +2698,19 @@ func TestEthConfirmer_EnsureConfirmedTransactionsInLongestChain(t *testing.T) { Parent: &evmtypes.Head{ Number: 8, Hash: testutils.NewHash(), - Parent: &latestFinalizedHead, + Parent: nil, }, }, } t.Run("does nothing if there aren't any transactions", func(t *testing.T) { - require.NoError(t, ec.EnsureConfirmedTransactionsInLongestChain(tests.Context(t), &head, &latestFinalizedHead)) + require.NoError(t, ec.EnsureConfirmedTransactionsInLongestChain(tests.Context(t), &head)) }) t.Run("does nothing to unconfirmed transactions", func(t *testing.T) { etx := cltest.MustInsertUnconfirmedEthTxWithBroadcastLegacyAttempt(t, txStore, 0, fromAddress) // Do the thing - require.NoError(t, ec.EnsureConfirmedTransactionsInLongestChain(tests.Context(t), &head, &latestFinalizedHead)) + require.NoError(t, ec.EnsureConfirmedTransactionsInLongestChain(tests.Context(t), &head)) etx, err := txStore.FindTxWithAttempts(ctx, etx.ID) require.NoError(t, err) @@ -2729,7 +2722,7 @@ func TestEthConfirmer_EnsureConfirmedTransactionsInLongestChain(t *testing.T) { mustInsertEthReceipt(t, txStore, head.Number, head.Hash, etx.TxAttempts[0].Hash) // Do the thing - require.NoError(t, ec.EnsureConfirmedTransactionsInLongestChain(tests.Context(t), &head, &latestFinalizedHead)) + require.NoError(t, ec.EnsureConfirmedTransactionsInLongestChain(tests.Context(t), &head)) etx, err := txStore.FindTxWithAttempts(ctx, etx.ID) require.NoError(t, err) @@ -2742,7 +2735,7 @@ func TestEthConfirmer_EnsureConfirmedTransactionsInLongestChain(t *testing.T) { mustInsertEthReceipt(t, txStore, head.Parent.Parent.Number-1, testutils.NewHash(), etx.TxAttempts[0].Hash) // Do the thing - require.NoError(t, ec.EnsureConfirmedTransactionsInLongestChain(tests.Context(t), &head, &latestFinalizedHead)) + require.NoError(t, ec.EnsureConfirmedTransactionsInLongestChain(tests.Context(t), &head)) etx, err := txStore.FindTxWithAttempts(ctx, etx.ID) require.NoError(t, err) @@ -2763,7 +2756,7 @@ func TestEthConfirmer_EnsureConfirmedTransactionsInLongestChain(t *testing.T) { }), fromAddress).Return(commonclient.Successful, nil).Once() // Do the thing - require.NoError(t, ec.EnsureConfirmedTransactionsInLongestChain(tests.Context(t), &head, &latestFinalizedHead)) + require.NoError(t, ec.EnsureConfirmedTransactionsInLongestChain(tests.Context(t), &head)) etx, err := txStore.FindTxWithAttempts(ctx, etx.ID) require.NoError(t, err) @@ -2786,7 +2779,7 @@ func TestEthConfirmer_EnsureConfirmedTransactionsInLongestChain(t *testing.T) { commonclient.Successful, nil).Once() // Do the thing - require.NoError(t, ec.EnsureConfirmedTransactionsInLongestChain(tests.Context(t), &head, &latestFinalizedHead)) + require.NoError(t, ec.EnsureConfirmedTransactionsInLongestChain(tests.Context(t), &head)) etx, err := txStore.FindTxWithAttempts(ctx, etx.ID) require.NoError(t, err) @@ -2821,7 +2814,7 @@ func TestEthConfirmer_EnsureConfirmedTransactionsInLongestChain(t *testing.T) { }), fromAddress).Return(commonclient.Successful, nil).Once() // Do the thing - require.NoError(t, ec.EnsureConfirmedTransactionsInLongestChain(tests.Context(t), &head, &latestFinalizedHead)) + require.NoError(t, ec.EnsureConfirmedTransactionsInLongestChain(tests.Context(t), &head)) etx, err := txStore.FindTxWithAttempts(ctx, etx.ID) require.NoError(t, err) @@ -2841,7 +2834,7 @@ func TestEthConfirmer_EnsureConfirmedTransactionsInLongestChain(t *testing.T) { // Add receipt that is higher than head mustInsertEthReceipt(t, txStore, head.Number+1, testutils.NewHash(), attempt.Hash) - require.NoError(t, ec.EnsureConfirmedTransactionsInLongestChain(tests.Context(t), &head, &latestFinalizedHead)) + require.NoError(t, ec.EnsureConfirmedTransactionsInLongestChain(tests.Context(t), &head)) etx, err := txStore.FindTxWithAttempts(ctx, etx.ID) require.NoError(t, err) From 4663172d0577f5a38e04bf1fa8e9364849c8f18d Mon Sep 17 00:00:00 2001 From: Joe Huang Date: Mon, 5 Aug 2024 14:39:45 -0500 Subject: [PATCH 29/37] fix lint --- common/txmgr/confirmer.go | 42 ++++++++++++++++----------------------- 1 file changed, 17 insertions(+), 25 deletions(-) diff --git a/common/txmgr/confirmer.go b/common/txmgr/confirmer.go index 7d5949a82cb..1f170c0caa6 100644 --- a/common/txmgr/confirmer.go +++ b/common/txmgr/confirmer.go @@ -33,14 +33,6 @@ const ( // processHeadTimeout represents a sanity limit on how long ProcessHead // should take to complete processHeadTimeout = 10 * time.Minute - - // logAfterNConsecutiveBlocksChainTooShort logs a warning if we go at least - // this many consecutive blocks with a re-org protection chain that is too - // short - // - // we don't log every time because on startup it can be lower, only if it - // persists does it indicate a serious problem - logAfterNConsecutiveBlocksChainTooShort = 10 ) var ( @@ -144,7 +136,7 @@ type Confirmer[ isStarted bool isReceiptNil func(R) bool - headTracker confirmerHeadTracker[HEAD, BLOCK_HASH] + confirmerHeadTracker confirmerHeadTracker[HEAD, BLOCK_HASH] } func NewConfirmer[ @@ -172,21 +164,21 @@ func NewConfirmer[ ) *Confirmer[CHAIN_ID, HEAD, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE] { lggr = logger.Named(lggr, "Confirmer") return &Confirmer[CHAIN_ID, HEAD, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]{ - txStore: txStore, - lggr: logger.Sugared(lggr), - client: client, - TxAttemptBuilder: txAttemptBuilder, - resumeCallback: nil, - chainConfig: chainConfig, - feeConfig: feeConfig, - txConfig: txConfig, - dbConfig: dbConfig, - chainID: client.ConfiguredChainID(), - ks: keystore, - mb: mailbox.NewSingle[HEAD](), - isReceiptNil: isReceiptNil, - stuckTxDetector: stuckTxDetector, - headTracker: headTracker, + txStore: txStore, + lggr: logger.Sugared(lggr), + client: client, + TxAttemptBuilder: txAttemptBuilder, + resumeCallback: nil, + chainConfig: chainConfig, + feeConfig: feeConfig, + txConfig: txConfig, + dbConfig: dbConfig, + chainID: client.ConfiguredChainID(), + ks: keystore, + mb: mailbox.NewSingle[HEAD](), + isReceiptNil: isReceiptNil, + stuckTxDetector: stuckTxDetector, + confirmerHeadTracker: headTracker, } } @@ -303,7 +295,7 @@ func (ec *Confirmer[CHAIN_ID, HEAD, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) pro return fmt.Errorf("CheckConfirmedMissingReceipt failed: %w", err) } - _, latestFinalizedHead, err := ec.headTracker.LatestAndFinalizedBlock(ctx) + _, latestFinalizedHead, err := ec.confirmerHeadTracker.LatestAndFinalizedBlock(ctx) if err != nil { return fmt.Errorf("failed to retrieve latest finalized head: %w", err) } From 805975d15415f5feafdc6b9f9fe2bc4e639bf030 Mon Sep 17 00:00:00 2001 From: Joe Huang Date: Tue, 6 Aug 2024 10:53:21 -0500 Subject: [PATCH 30/37] rename --- common/txmgr/confirmer.go | 34 +++++++++++++++++----------------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/common/txmgr/confirmer.go b/common/txmgr/confirmer.go index 1f170c0caa6..dddb4293c56 100644 --- a/common/txmgr/confirmer.go +++ b/common/txmgr/confirmer.go @@ -136,7 +136,7 @@ type Confirmer[ isStarted bool isReceiptNil func(R) bool - confirmerHeadTracker confirmerHeadTracker[HEAD, BLOCK_HASH] + headTracker confirmerHeadTracker[HEAD, BLOCK_HASH] } func NewConfirmer[ @@ -164,21 +164,21 @@ func NewConfirmer[ ) *Confirmer[CHAIN_ID, HEAD, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE] { lggr = logger.Named(lggr, "Confirmer") return &Confirmer[CHAIN_ID, HEAD, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]{ - txStore: txStore, - lggr: logger.Sugared(lggr), - client: client, - TxAttemptBuilder: txAttemptBuilder, - resumeCallback: nil, - chainConfig: chainConfig, - feeConfig: feeConfig, - txConfig: txConfig, - dbConfig: dbConfig, - chainID: client.ConfiguredChainID(), - ks: keystore, - mb: mailbox.NewSingle[HEAD](), - isReceiptNil: isReceiptNil, - stuckTxDetector: stuckTxDetector, - confirmerHeadTracker: headTracker, + txStore: txStore, + lggr: logger.Sugared(lggr), + client: client, + TxAttemptBuilder: txAttemptBuilder, + resumeCallback: nil, + chainConfig: chainConfig, + feeConfig: feeConfig, + txConfig: txConfig, + dbConfig: dbConfig, + chainID: client.ConfiguredChainID(), + ks: keystore, + mb: mailbox.NewSingle[HEAD](), + isReceiptNil: isReceiptNil, + stuckTxDetector: stuckTxDetector, + headTracker: headTracker, } } @@ -295,7 +295,7 @@ func (ec *Confirmer[CHAIN_ID, HEAD, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) pro return fmt.Errorf("CheckConfirmedMissingReceipt failed: %w", err) } - _, latestFinalizedHead, err := ec.confirmerHeadTracker.LatestAndFinalizedBlock(ctx) + _, latestFinalizedHead, err := ec.headTracker.LatestAndFinalizedBlock(ctx) if err != nil { return fmt.Errorf("failed to retrieve latest finalized head: %w", err) } From 31d4569f723fe4ae6a17f6237f3ec998cc2c59e3 Mon Sep 17 00:00:00 2001 From: Joe Huang Date: Tue, 6 Aug 2024 12:43:28 -0500 Subject: [PATCH 31/37] have log back --- common/txmgr/confirmer.go | 49 +++++++++++++++++++++---- core/chains/evm/txmgr/confirmer_test.go | 23 ++++++++---- 2 files changed, 56 insertions(+), 16 deletions(-) diff --git a/common/txmgr/confirmer.go b/common/txmgr/confirmer.go index dddb4293c56..3ce88c6025a 100644 --- a/common/txmgr/confirmer.go +++ b/common/txmgr/confirmer.go @@ -33,6 +33,14 @@ const ( // processHeadTimeout represents a sanity limit on how long ProcessHead // should take to complete processHeadTimeout = 10 * time.Minute + + // logAfterNConsecutiveBlocksChainTooShort logs a warning if we go at least + // this many consecutive blocks with a re-org protection chain that is too + // short + // + // we don't log every time because on startup it can be lower, only if it + // persists does it indicate a serious problem + logAfterNConsecutiveBlocksChainTooShort = 10 ) var ( @@ -129,12 +137,13 @@ type Confirmer[ ks txmgrtypes.KeyStore[ADDR, CHAIN_ID, SEQ] enabledAddresses []ADDR - mb *mailbox.Mailbox[HEAD] - stopCh services.StopChan - wg sync.WaitGroup - initSync sync.Mutex - isStarted bool - isReceiptNil func(R) bool + mb *mailbox.Mailbox[HEAD] + stopCh services.StopChan + wg sync.WaitGroup + initSync sync.Mutex + isStarted bool + nConsecutiveBlocksChainTooShort int + isReceiptNil func(R) bool headTracker confirmerHeadTracker[HEAD, BLOCK_HASH] } @@ -329,7 +338,7 @@ func (ec *Confirmer[CHAIN_ID, HEAD, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) pro ec.lggr.Debugw("Finished RebroadcastWhereNecessary", "headNum", head.BlockNumber(), "time", time.Since(mark), "id", "confirmer") mark = time.Now() - if err := ec.EnsureConfirmedTransactionsInLongestChain(ctx, head); err != nil { + if err := ec.EnsureConfirmedTransactionsInLongestChain(ctx, head, latestFinalizedHead.BlockNumber()); err != nil { return fmt.Errorf("EnsureConfirmedTransactionsInLongestChain failed: %w", err) } @@ -1021,7 +1030,31 @@ func (ec *Confirmer[CHAIN_ID, HEAD, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) han // // If any of the confirmed transactions does not have a receipt in the chain, it has been // re-org'd out and will be rebroadcast. -func (ec *Confirmer[CHAIN_ID, HEAD, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) EnsureConfirmedTransactionsInLongestChain(ctx context.Context, head types.Head[BLOCK_HASH]) error { +func (ec *Confirmer[CHAIN_ID, HEAD, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) EnsureConfirmedTransactionsInLongestChain(ctx context.Context, head types.Head[BLOCK_HASH], latestFinalizedHeadNumber int64) error { + logArgs := []interface{}{ + "chainLength", head.ChainLength(), "latestFinalizedHead number", latestFinalizedHeadNumber, + } + + if head.BlockNumber() < latestFinalizedHeadNumber { + errMsg := "current head is shorter than latest finalized head" + ec.lggr.Errorw(errMsg, append(logArgs, "head block number", head.BlockNumber())...) + return errors.New(errMsg) + } + + calculatedFinalityDepth := uint32(head.BlockNumber() - latestFinalizedHeadNumber) + if head.ChainLength() < calculatedFinalityDepth { + if ec.nConsecutiveBlocksChainTooShort > logAfterNConsecutiveBlocksChainTooShort { + warnMsg := "Chain length supplied for re-org detection was shorter than actual finality depth. Re-org protection is not working properly. This could indicate a problem with the remote RPC endpoint, a compatibility issue with a particular blockchain, a bug with this particular blockchain, heads table being truncated too early, remote node out of sync, or something else. If this happens a lot please raise a bug with the Chainlink team including a log output sample and details of the chain and RPC endpoint you are using." + ec.lggr.Warnw(warnMsg, append(logArgs, "nConsecutiveBlocksChainTooShort", ec.nConsecutiveBlocksChainTooShort)...) + } else { + logMsg := "Chain length supplied for re-org detection was shorter than actual finality depth" + ec.lggr.Debugw(logMsg, append(logArgs, "nConsecutiveBlocksChainTooShort", ec.nConsecutiveBlocksChainTooShort)...) + } + ec.nConsecutiveBlocksChainTooShort++ + } else { + ec.nConsecutiveBlocksChainTooShort = 0 + } + etxs, err := ec.txStore.FindTransactionsConfirmedInBlockRange(ctx, head.BlockNumber(), head.EarliestHeadInChain().BlockNumber(), ec.chainID) if err != nil { return fmt.Errorf("findTransactionsConfirmedInBlockRange failed: %w", err) diff --git a/core/chains/evm/txmgr/confirmer_test.go b/core/chains/evm/txmgr/confirmer_test.go index f5e6e715e57..cce6dc8fc65 100644 --- a/core/chains/evm/txmgr/confirmer_test.go +++ b/core/chains/evm/txmgr/confirmer_test.go @@ -2689,6 +2689,13 @@ func TestEthConfirmer_EnsureConfirmedTransactionsInLongestChain(t *testing.T) { gconfig, config := newTestChainScopedConfig(t) ec := newEthConfirmer(t, txStore, ethClient, gconfig, config, ethKeyStore, nil) + latestFinalizedHead := evmtypes.Head{ + Number: 8, + Hash: testutils.NewHash(), + Parent: nil, + IsFinalized: false, // We are guaranteed to receive a latestFinalizedHead. + } + head := evmtypes.Head{ Hash: testutils.NewHash(), Number: 10, @@ -2703,14 +2710,14 @@ func TestEthConfirmer_EnsureConfirmedTransactionsInLongestChain(t *testing.T) { }, } t.Run("does nothing if there aren't any transactions", func(t *testing.T) { - require.NoError(t, ec.EnsureConfirmedTransactionsInLongestChain(tests.Context(t), &head)) + require.NoError(t, ec.EnsureConfirmedTransactionsInLongestChain(tests.Context(t), &head, latestFinalizedHead.BlockNumber())) }) t.Run("does nothing to unconfirmed transactions", func(t *testing.T) { etx := cltest.MustInsertUnconfirmedEthTxWithBroadcastLegacyAttempt(t, txStore, 0, fromAddress) // Do the thing - require.NoError(t, ec.EnsureConfirmedTransactionsInLongestChain(tests.Context(t), &head)) + require.NoError(t, ec.EnsureConfirmedTransactionsInLongestChain(tests.Context(t), &head, latestFinalizedHead.BlockNumber())) etx, err := txStore.FindTxWithAttempts(ctx, etx.ID) require.NoError(t, err) @@ -2722,7 +2729,7 @@ func TestEthConfirmer_EnsureConfirmedTransactionsInLongestChain(t *testing.T) { mustInsertEthReceipt(t, txStore, head.Number, head.Hash, etx.TxAttempts[0].Hash) // Do the thing - require.NoError(t, ec.EnsureConfirmedTransactionsInLongestChain(tests.Context(t), &head)) + require.NoError(t, ec.EnsureConfirmedTransactionsInLongestChain(tests.Context(t), &head, latestFinalizedHead.BlockNumber())) etx, err := txStore.FindTxWithAttempts(ctx, etx.ID) require.NoError(t, err) @@ -2735,7 +2742,7 @@ func TestEthConfirmer_EnsureConfirmedTransactionsInLongestChain(t *testing.T) { mustInsertEthReceipt(t, txStore, head.Parent.Parent.Number-1, testutils.NewHash(), etx.TxAttempts[0].Hash) // Do the thing - require.NoError(t, ec.EnsureConfirmedTransactionsInLongestChain(tests.Context(t), &head)) + require.NoError(t, ec.EnsureConfirmedTransactionsInLongestChain(tests.Context(t), &head, latestFinalizedHead.BlockNumber())) etx, err := txStore.FindTxWithAttempts(ctx, etx.ID) require.NoError(t, err) @@ -2756,7 +2763,7 @@ func TestEthConfirmer_EnsureConfirmedTransactionsInLongestChain(t *testing.T) { }), fromAddress).Return(commonclient.Successful, nil).Once() // Do the thing - require.NoError(t, ec.EnsureConfirmedTransactionsInLongestChain(tests.Context(t), &head)) + require.NoError(t, ec.EnsureConfirmedTransactionsInLongestChain(tests.Context(t), &head, latestFinalizedHead.BlockNumber())) etx, err := txStore.FindTxWithAttempts(ctx, etx.ID) require.NoError(t, err) @@ -2779,7 +2786,7 @@ func TestEthConfirmer_EnsureConfirmedTransactionsInLongestChain(t *testing.T) { commonclient.Successful, nil).Once() // Do the thing - require.NoError(t, ec.EnsureConfirmedTransactionsInLongestChain(tests.Context(t), &head)) + require.NoError(t, ec.EnsureConfirmedTransactionsInLongestChain(tests.Context(t), &head, latestFinalizedHead.BlockNumber())) etx, err := txStore.FindTxWithAttempts(ctx, etx.ID) require.NoError(t, err) @@ -2814,7 +2821,7 @@ func TestEthConfirmer_EnsureConfirmedTransactionsInLongestChain(t *testing.T) { }), fromAddress).Return(commonclient.Successful, nil).Once() // Do the thing - require.NoError(t, ec.EnsureConfirmedTransactionsInLongestChain(tests.Context(t), &head)) + require.NoError(t, ec.EnsureConfirmedTransactionsInLongestChain(tests.Context(t), &head, latestFinalizedHead.BlockNumber())) etx, err := txStore.FindTxWithAttempts(ctx, etx.ID) require.NoError(t, err) @@ -2834,7 +2841,7 @@ func TestEthConfirmer_EnsureConfirmedTransactionsInLongestChain(t *testing.T) { // Add receipt that is higher than head mustInsertEthReceipt(t, txStore, head.Number+1, testutils.NewHash(), attempt.Hash) - require.NoError(t, ec.EnsureConfirmedTransactionsInLongestChain(tests.Context(t), &head)) + require.NoError(t, ec.EnsureConfirmedTransactionsInLongestChain(tests.Context(t), &head, latestFinalizedHead.BlockNumber())) etx, err := txStore.FindTxWithAttempts(ctx, etx.ID) require.NoError(t, err) From 0f98a461e868341068a8cdf9a8db6840e71d57e8 Mon Sep 17 00:00:00 2001 From: Joe Huang Date: Tue, 6 Aug 2024 13:47:32 -0500 Subject: [PATCH 32/37] update comments --- common/txmgr/confirmer.go | 2 +- core/chains/evm/txmgr/evm_tx_store.go | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/common/txmgr/confirmer.go b/common/txmgr/confirmer.go index 3ce88c6025a..8d554664535 100644 --- a/common/txmgr/confirmer.go +++ b/common/txmgr/confirmer.go @@ -1024,7 +1024,7 @@ func (ec *Confirmer[CHAIN_ID, HEAD, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) han } } -// EnsureConfirmedTransactionsInLongestChain finds all confirmed txes up to the LatestFinalizedHead +// EnsureConfirmedTransactionsInLongestChain finds all confirmed txes up to the earliest head // of the given chain and ensures that every one has a receipt with a block hash that is // in the given chain. // diff --git a/core/chains/evm/txmgr/evm_tx_store.go b/core/chains/evm/txmgr/evm_tx_store.go index b6426c9a029..6d1a8ec1067 100644 --- a/core/chains/evm/txmgr/evm_tx_store.go +++ b/core/chains/evm/txmgr/evm_tx_store.go @@ -959,7 +959,7 @@ func (o *evmTxStore) SaveFetchedReceipts(ctx context.Context, r []*evmtypes.Rece // decision to give up). This is done in the EthResender. // // We will continue to try to fetch a receipt for these attempts until all -// attempts are below the LatestFinalizedBlockNum from current head. +// attempts are equal to or below the LatestFinalizedBlockNum from current head. func (o *evmTxStore) MarkAllConfirmedMissingReceipt(ctx context.Context, chainID *big.Int) (err error) { var cancel context.CancelFunc ctx, cancel = o.stopCh.Ctx(ctx) @@ -1444,7 +1444,7 @@ ORDER BY nonce ASC // markOldTxesMissingReceiptAsErrored // -// Once eth_tx has all of its attempts broadcast before latestFinalizedBlockNum +// Once eth_tx has all of its attempts broadcast equal to or before latestFinalizedBlockNum // without receiving any receipts, we mark it as fatally errored (never sent). // // The job run will also be marked as errored in this case since we never got a @@ -1453,8 +1453,8 @@ func (o *evmTxStore) MarkOldTxesMissingReceiptAsErrored(ctx context.Context, blo var cancel context.CancelFunc ctx, cancel = o.stopCh.Ctx(ctx) defer cancel() - // Any 'confirmed_missing_receipt' eth_tx with all attempts older than latestFinalizedBlockNum will be marked as errored - // We will not try to query for receipts for this transaction any more + // Any 'confirmed_missing_receipt' eth_tx with all attempts equal to or older than latestFinalizedBlockNum will be marked as errored + // We will not try to query for receipts for this transaction anymore if latestFinalizedBlockNum <= 0 { return nil } From dbbac952c958729e6d40d2f7c90ee7d0c697507f Mon Sep 17 00:00:00 2001 From: Joe Huang Date: Tue, 6 Aug 2024 14:02:05 -0500 Subject: [PATCH 33/37] remove nil check --- common/txmgr/confirmer.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/common/txmgr/confirmer.go b/common/txmgr/confirmer.go index 8d554664535..87cfccbb24f 100644 --- a/common/txmgr/confirmer.go +++ b/common/txmgr/confirmer.go @@ -309,7 +309,7 @@ func (ec *Confirmer[CHAIN_ID, HEAD, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) pro return fmt.Errorf("failed to retrieve latest finalized head: %w", err) } - if types.Head[BLOCK_HASH](latestFinalizedHead) == nil || !latestFinalizedHead.IsValid() { + if !latestFinalizedHead.IsValid() { return fmt.Errorf("latest finalized head is not valid") } From 14a5668d1789aeadcde9bd801525702977aa020f Mon Sep 17 00:00:00 2001 From: Joe Huang Date: Tue, 6 Aug 2024 14:15:28 -0500 Subject: [PATCH 34/37] minor --- common/txmgr/confirmer.go | 1 - 1 file changed, 1 deletion(-) diff --git a/common/txmgr/confirmer.go b/common/txmgr/confirmer.go index 87cfccbb24f..7a016925c65 100644 --- a/common/txmgr/confirmer.go +++ b/common/txmgr/confirmer.go @@ -1054,7 +1054,6 @@ func (ec *Confirmer[CHAIN_ID, HEAD, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) Ens } else { ec.nConsecutiveBlocksChainTooShort = 0 } - etxs, err := ec.txStore.FindTransactionsConfirmedInBlockRange(ctx, head.BlockNumber(), head.EarliestHeadInChain().BlockNumber(), ec.chainID) if err != nil { return fmt.Errorf("findTransactionsConfirmedInBlockRange failed: %w", err) From ebe7d592cb2f0efbf6fc8c11308e66c98524616f Mon Sep 17 00:00:00 2001 From: Joe Huang Date: Tue, 6 Aug 2024 16:27:43 -0500 Subject: [PATCH 35/37] fix test --- core/chains/evm/txmgr/txmgr_test.go | 15 +-------------- 1 file changed, 1 insertion(+), 14 deletions(-) diff --git a/core/chains/evm/txmgr/txmgr_test.go b/core/chains/evm/txmgr/txmgr_test.go index 77b32c4f3c6..3d52e6eb4f0 100644 --- a/core/chains/evm/txmgr/txmgr_test.go +++ b/core/chains/evm/txmgr/txmgr_test.go @@ -613,20 +613,6 @@ func TestTxm_GetTransactionStatus(t *testing.T) { gcfg := configtest.NewTestGeneralConfig(t) cfg := evmtest.NewChainScopedConfig(t, gcfg) -<<<<<<< HEAD - ethClient := evmtest.NewEthClientMockWithDefaultChain(t) - ethClient.On("PendingNonceAt", mock.Anything, mock.Anything).Return(uint64(0), nil).Maybe() - feeEstimator := gasmocks.NewEvmFeeEstimator(t) - feeEstimator.On("Start", mock.Anything).Return(nil).Once() - feeEstimator.On("Close", mock.Anything).Return(nil).Once() - ethClient.On("HeadByNumber", mock.Anything, (*big.Int)(nil)).Return(nil, nil) - feeEstimator.On("OnNewLongestChain", mock.Anything, mock.Anything).Once() - txm, err := makeTestEvmTxm(t, db, ethClient, feeEstimator, cfg.EVM(), cfg.EVM().GasEstimator(), cfg.EVM().Transactions(), gcfg.Database(), gcfg.Database().Listener(), ethKeyStore) - require.NoError(t, err) - servicetest.Run(t, txm) - -======= ->>>>>>> 2312827156f24fa4a6e420aec12e5a3aeac81e2b head := &evmtypes.Head{ Hash: utils.NewHash(), Number: 100, @@ -641,6 +627,7 @@ func TestTxm_GetTransactionStatus(t *testing.T) { ethClient.On("PendingNonceAt", mock.Anything, mock.Anything).Return(uint64(0), nil).Maybe() ethClient.On("HeadByNumber", mock.Anything, mock.Anything).Return(head, nil).Once() ethClient.On("HeadByNumber", mock.Anything, mock.Anything).Return(head.Parent, nil).Once() + ethClient.On("HeadByNumber", mock.Anything, mock.Anything).Return(head, nil) feeEstimator := gasmocks.NewEvmFeeEstimator(t) feeEstimator.On("Start", mock.Anything).Return(nil).Once() feeEstimator.On("Close", mock.Anything).Return(nil).Once() From e59e259b8fc4d53df26d0f68eb0330edf76c02b6 Mon Sep 17 00:00:00 2001 From: Joe Huang Date: Tue, 6 Aug 2024 18:38:54 -0500 Subject: [PATCH 36/37] grammar --- common/txmgr/confirmer.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/common/txmgr/confirmer.go b/common/txmgr/confirmer.go index 7a016925c65..9a693a646a0 100644 --- a/common/txmgr/confirmer.go +++ b/common/txmgr/confirmer.go @@ -314,7 +314,7 @@ func (ec *Confirmer[CHAIN_ID, HEAD, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) pro } if latestFinalizedHead.BlockNumber() > head.BlockNumber() { - ec.lggr.Debugw("processHead receives old block", "latestFinalizedHead", latestFinalizedHead.BlockNumber(), "headNum", head.BlockNumber(), "time", time.Since(mark), "id", "confirmer") + ec.lggr.Debugw("processHead received old block", "latestFinalizedHead", latestFinalizedHead.BlockNumber(), "headNum", head.BlockNumber(), "time", time.Since(mark), "id", "confirmer") } if err := ec.CheckForReceipts(ctx, head.BlockNumber(), latestFinalizedHead.BlockNumber()); err != nil { From b0ed1e081b251378eca547c4d9a70abd56aaa2a0 Mon Sep 17 00:00:00 2001 From: Joe Huang Date: Tue, 6 Aug 2024 18:53:55 -0500 Subject: [PATCH 37/37] rephrase --- common/txmgr/confirmer.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/common/txmgr/confirmer.go b/common/txmgr/confirmer.go index 9a693a646a0..3b421191782 100644 --- a/common/txmgr/confirmer.go +++ b/common/txmgr/confirmer.go @@ -1044,10 +1044,10 @@ func (ec *Confirmer[CHAIN_ID, HEAD, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) Ens calculatedFinalityDepth := uint32(head.BlockNumber() - latestFinalizedHeadNumber) if head.ChainLength() < calculatedFinalityDepth { if ec.nConsecutiveBlocksChainTooShort > logAfterNConsecutiveBlocksChainTooShort { - warnMsg := "Chain length supplied for re-org detection was shorter than actual finality depth. Re-org protection is not working properly. This could indicate a problem with the remote RPC endpoint, a compatibility issue with a particular blockchain, a bug with this particular blockchain, heads table being truncated too early, remote node out of sync, or something else. If this happens a lot please raise a bug with the Chainlink team including a log output sample and details of the chain and RPC endpoint you are using." + warnMsg := "Chain length supplied for re-org detection was shorter than the depth from the latest head to the finalized head. Re-org protection is not working properly. This could indicate a problem with the remote RPC endpoint, a compatibility issue with a particular blockchain, a bug with this particular blockchain, heads table being truncated too early, remote node out of sync, or something else. If this happens a lot please raise a bug with the Chainlink team including a log output sample and details of the chain and RPC endpoint you are using." ec.lggr.Warnw(warnMsg, append(logArgs, "nConsecutiveBlocksChainTooShort", ec.nConsecutiveBlocksChainTooShort)...) } else { - logMsg := "Chain length supplied for re-org detection was shorter than actual finality depth" + logMsg := "Chain length supplied for re-org detection was shorter than the depth from the latest head to the finalized head" ec.lggr.Debugw(logMsg, append(logArgs, "nConsecutiveBlocksChainTooShort", ec.nConsecutiveBlocksChainTooShort)...) } ec.nConsecutiveBlocksChainTooShort++