From b276443f7c2d208f1fe3052fb4d0bcec69765c75 Mon Sep 17 00:00:00 2001 From: evgeniy-scherbina Date: Fri, 27 Sep 2024 15:07:27 -0400 Subject: [PATCH 1/8] fix: fix bug in evm-indexer --- go.mod | 2 +- server/indexer_service.go | 31 ++++++++++++++ server/indexer_service_test.go | 75 ++++++++++++++++++++++++++++++++++ 3 files changed, 107 insertions(+), 1 deletion(-) create mode 100644 server/indexer_service_test.go diff --git a/go.mod b/go.mod index c57877c6dd..0683a49ff1 100644 --- a/go.mod +++ b/go.mod @@ -13,6 +13,7 @@ require ( github.com/armon/go-metrics v0.4.1 github.com/btcsuite/btcd v0.23.4 github.com/btcsuite/btcd/btcutil v1.1.3 + github.com/cenkalti/backoff/v4 v4.1.3 github.com/cometbft/cometbft v0.37.4 github.com/cometbft/cometbft-db v0.9.1 github.com/cosmos/cosmos-proto v1.0.0-beta.4 @@ -69,7 +70,6 @@ require ( github.com/bgentry/speakeasy v0.1.1-0.20220910012023-760eaf8b6816 // indirect github.com/btcsuite/btcd/btcec/v2 v2.3.2 // indirect github.com/btcsuite/btcd/chaincfg/chainhash v1.0.1 // indirect - github.com/cenkalti/backoff/v4 v4.1.3 // indirect github.com/cespare/xxhash v1.1.0 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/chzyer/readline v1.5.1 // indirect diff --git a/server/indexer_service.go b/server/indexer_service.go index 2edbc02aa5..9ec7434e06 100644 --- a/server/indexer_service.go +++ b/server/indexer_service.go @@ -17,8 +17,10 @@ package server import ( "context" + "errors" "time" + "github.com/cenkalti/backoff/v4" "github.com/cometbft/cometbft/libs/service" rpcclient "github.com/cometbft/cometbft/rpc/client" "github.com/cometbft/cometbft/types" @@ -54,6 +56,14 @@ func NewEVMIndexerService( // and indexing them by events. func (eis *EVMIndexerService) OnStart() error { ctx := context.Background() + + // when kava in state-sync mode, it returns zero as latest_block_height, which leads to undesired behavior, more + // details here: https://kava-labs.atlassian.net/wiki/spaces/ENG/pages/1623687169/EVM-Indexer+State+Sync+Issue + // to prevent this we wait until state-sync will finish + if err := waitUntilClientReady(ctx, eis.client, backoff.NewConstantBackOff(time.Second)); err != nil { + return err + } + status, err := eis.client.Status(ctx) if err != nil { return err @@ -122,3 +132,24 @@ func (eis *EVMIndexerService) OnStart() error { } } } + +// waitUntilClientReady waits until StatusClient is ready to serve requests +func waitUntilClientReady(ctx context.Context, client rpcclient.StatusClient, b backoff.BackOff) error { + err := backoff.Retry(func() error { + status, err := client.Status(ctx) + if err != nil { + return err + } + + if status.SyncInfo.LatestBlockHeight == 0 { + return errors.New("node isn't ready, possibly in state sync process") + } + + return nil + }, b) + if err != nil { + return err + } + + return nil +} diff --git a/server/indexer_service_test.go b/server/indexer_service_test.go new file mode 100644 index 0000000000..a147f1f869 --- /dev/null +++ b/server/indexer_service_test.go @@ -0,0 +1,75 @@ +package server + +import ( + "context" + "testing" + "time" + + "github.com/cenkalti/backoff/v4" + "github.com/stretchr/testify/require" + + coretypes "github.com/cometbft/cometbft/rpc/core/types" +) + +var ( + failedResponse = &coretypes.ResultStatus{ + SyncInfo: coretypes.SyncInfo{ + LatestBlockHeight: 0, + }, + } + + successfulResponse = &coretypes.ResultStatus{ + SyncInfo: coretypes.SyncInfo{ + LatestBlockHeight: 1, + }, + } +) + +type statusClientMock struct { + // retries left before success response + retriesLeft uint +} + +func newStatusClientMock(retriesLeft uint) *statusClientMock { + return &statusClientMock{ + retriesLeft: retriesLeft, + } +} + +func (m *statusClientMock) Status(context.Context) (*coretypes.ResultStatus, error) { + if m.retriesLeft == 0 { + return successfulResponse, nil + } + + m.retriesLeft-- + return failedResponse, nil +} + +func TestWaitUntilClientReady(t *testing.T) { + for _, tc := range []struct { + desc string + retriesLeft uint + }{ + { + desc: "test case #1", + retriesLeft: 0, + }, + { + desc: "test case #2", + retriesLeft: 1, + }, + { + desc: "test case #3", + retriesLeft: 10, + }, + } { + t.Run(tc.desc, func(t *testing.T) { + ctxb := context.Background() + mock := newStatusClientMock(tc.retriesLeft) + + err := waitUntilClientReady(ctxb, mock, backoff.NewConstantBackOff(time.Nanosecond)) + require.NoError(t, err) + require.Equal(t, uint(0), mock.retriesLeft) + }) + } +} From 0ae55f19d7272a08e19b2af24ee21fe040488680 Mon Sep 17 00:00:00 2001 From: evgeniy-scherbina Date: Mon, 30 Sep 2024 14:15:28 -0400 Subject: [PATCH 2/8] fix: change documentation link --- server/indexer_service.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/server/indexer_service.go b/server/indexer_service.go index 9ec7434e06..d833b609a8 100644 --- a/server/indexer_service.go +++ b/server/indexer_service.go @@ -58,8 +58,7 @@ func (eis *EVMIndexerService) OnStart() error { ctx := context.Background() // when kava in state-sync mode, it returns zero as latest_block_height, which leads to undesired behavior, more - // details here: https://kava-labs.atlassian.net/wiki/spaces/ENG/pages/1623687169/EVM-Indexer+State+Sync+Issue - // to prevent this we wait until state-sync will finish + // details here: https://github.com/Kava-Labs/ethermint/issues/79 to prevent this we wait until state-sync will finish if err := waitUntilClientReady(ctx, eis.client, backoff.NewConstantBackOff(time.Second)); err != nil { return err } From 0b7c21415204d82d1608e6c197ea047783503138 Mon Sep 17 00:00:00 2001 From: evgeniy-scherbina Date: Mon, 30 Sep 2024 14:25:26 -0400 Subject: [PATCH 3/8] docs: improve names for tests --- server/indexer_service_test.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/server/indexer_service_test.go b/server/indexer_service_test.go index a147f1f869..6556762df4 100644 --- a/server/indexer_service_test.go +++ b/server/indexer_service_test.go @@ -51,15 +51,15 @@ func TestWaitUntilClientReady(t *testing.T) { retriesLeft uint }{ { - desc: "test case #1", + desc: "return successful response right away", retriesLeft: 0, }, { - desc: "test case #2", + desc: "return successful response after one retry", retriesLeft: 1, }, { - desc: "test case #3", + desc: "return successful response after 10 retries", retriesLeft: 10, }, } { From 744f04a037c94d216bc1c47621df7d681b2b05de Mon Sep 17 00:00:00 2001 From: evgeniy-scherbina Date: Mon, 30 Sep 2024 17:13:34 -0400 Subject: [PATCH 4/8] chore: update cenkalti/backoff library --- go.mod | 2 +- go.sum | 4 ++-- gomod2nix.toml | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/go.mod b/go.mod index 0683a49ff1..9f4d4104e3 100644 --- a/go.mod +++ b/go.mod @@ -13,7 +13,7 @@ require ( github.com/armon/go-metrics v0.4.1 github.com/btcsuite/btcd v0.23.4 github.com/btcsuite/btcd/btcutil v1.1.3 - github.com/cenkalti/backoff/v4 v4.1.3 + github.com/cenkalti/backoff/v4 v4.3.0 github.com/cometbft/cometbft v0.37.4 github.com/cometbft/cometbft-db v0.9.1 github.com/cosmos/cosmos-proto v1.0.0-beta.4 diff --git a/go.sum b/go.sum index 1724053952..1e26759b2b 100644 --- a/go.sum +++ b/go.sum @@ -338,8 +338,8 @@ github.com/casbin/casbin/v2 v2.1.2/go.mod h1:YcPU1XXisHhLzuxH9coDNf2FbKpjGlbCg3n github.com/cenkalti/backoff v2.2.1+incompatible h1:tNowT99t7UNflLxfYYSlKYsBpXdEet03Pg2g16Swow4= github.com/cenkalti/backoff v2.2.1+incompatible/go.mod h1:90ReRw6GdpyfrHakVjL/QHaoyV4aDUVVkXQJJJ3NXXM= github.com/cenkalti/backoff/v4 v4.1.1/go.mod h1:scbssz8iZGpm3xbr14ovlUdkxfGXNInqkPWOWmG2CLw= -github.com/cenkalti/backoff/v4 v4.1.3 h1:cFAlzYUlVYDysBEH2T5hyJZMh3+5+WCBvSnK6Q8UtC4= -github.com/cenkalti/backoff/v4 v4.1.3/go.mod h1:scbssz8iZGpm3xbr14ovlUdkxfGXNInqkPWOWmG2CLw= +github.com/cenkalti/backoff/v4 v4.3.0 h1:MyRJ/UdXutAwSAT+s3wNd7MfTIcy71VQueUuFK343L8= +github.com/cenkalti/backoff/v4 v4.3.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/cespare/cp v0.1.0 h1:SE+dxFebS7Iik5LK0tsi1k9ZCxEaFX4AjQmoyA+1dJk= github.com/cespare/cp v0.1.0/go.mod h1:SOGHArjBr4JWaSDEVpWpo/hNg6RoKrls6Oh40hiwW+s= diff --git a/gomod2nix.toml b/gomod2nix.toml index 4f5c60a338..01b778d28c 100644 --- a/gomod2nix.toml +++ b/gomod2nix.toml @@ -90,8 +90,8 @@ schema = 3 version = "v1.0.1" hash = "sha256-vix0j/KGNvoKjhlKgVeSLY6un2FHeIEoZWMC4z3yvZ4=" [mod."github.com/cenkalti/backoff/v4"] - version = "v4.1.3" - hash = "sha256-u6MEDopHoTWAZoVvvXOKnAg++xre53YgQx0gmf6t2KU=" + version = "v4.3.0" + hash = "sha256-wfVjNZsGG1WoNC5aL+kdcy6QXPgZo4THAevZ1787md8=" [mod."github.com/cespare/xxhash"] version = "v1.1.0" hash = "sha256-nVDTtXH9PC3yJ0THaQZEN243UP9xgLi/clt5xRqj3+M=" From af2e3340de8dfa9e63ff5706325bbdc3e88c50ac Mon Sep 17 00:00:00 2001 From: evgeniy-scherbina Date: Mon, 30 Sep 2024 18:09:23 -0400 Subject: [PATCH 5/8] refactor: use exponential-back-off instead of constant-back-off --- server/indexer_service.go | 6 +++++- server/indexer_service_test.go | 18 ++++++++++++++++++ 2 files changed, 23 insertions(+), 1 deletion(-) diff --git a/server/indexer_service.go b/server/indexer_service.go index d833b609a8..a25874e397 100644 --- a/server/indexer_service.go +++ b/server/indexer_service.go @@ -59,7 +59,11 @@ func (eis *EVMIndexerService) OnStart() error { // when kava in state-sync mode, it returns zero as latest_block_height, which leads to undesired behavior, more // details here: https://github.com/Kava-Labs/ethermint/issues/79 to prevent this we wait until state-sync will finish - if err := waitUntilClientReady(ctx, eis.client, backoff.NewConstantBackOff(time.Second)); err != nil { + exponentialBackOff := backoff.NewExponentialBackOff( + backoff.WithMaxInterval(time.Second*10), // set max retry interval + backoff.WithMaxElapsedTime(time.Hour*3), // set timeout + ) + if err := waitUntilClientReady(ctx, eis.client, exponentialBackOff); err != nil { return err } diff --git a/server/indexer_service_test.go b/server/indexer_service_test.go index 6556762df4..48393c7c51 100644 --- a/server/indexer_service_test.go +++ b/server/indexer_service_test.go @@ -2,6 +2,7 @@ package server import ( "context" + "math" "testing" "time" @@ -73,3 +74,20 @@ func TestWaitUntilClientReady(t *testing.T) { }) } } + +func TestWaitUntilClientReadyTimeout(t *testing.T) { + ctxb := context.Background() + // create a mock client which always returns an error + mock := newStatusClientMock(math.MaxUint) + + exponentialBackOff := backoff.NewExponentialBackOff( + backoff.WithInitialInterval(time.Millisecond), + backoff.WithMaxInterval(time.Millisecond*10), + backoff.WithMaxElapsedTime(time.Millisecond*100), + ) + + err := waitUntilClientReady(ctxb, mock, exponentialBackOff) + // make sure error is propagated in case of timeout + require.Error(t, err) + require.Contains(t, err.Error(), "node isn't ready, possibly in state sync process") +} From bfc2ac156f97940d859dc213a48965a7f8447a2d Mon Sep 17 00:00:00 2001 From: evgeniy-scherbina Date: Mon, 30 Sep 2024 19:22:31 -0400 Subject: [PATCH 6/8] refactor: use constants for back-off configuration --- server/indexer_service.go | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/server/indexer_service.go b/server/indexer_service.go index a25874e397..e6d113e9d3 100644 --- a/server/indexer_service.go +++ b/server/indexer_service.go @@ -32,6 +32,9 @@ const ( ServiceName = "EVMIndexerService" NewBlockWaitTimeout = 60 * time.Second + + statusClientMaxRetryInterval = time.Second * 10 + statusClientTimeout = time.Hour * 3 ) // EVMIndexerService indexes transactions for json-rpc service. @@ -60,8 +63,8 @@ func (eis *EVMIndexerService) OnStart() error { // when kava in state-sync mode, it returns zero as latest_block_height, which leads to undesired behavior, more // details here: https://github.com/Kava-Labs/ethermint/issues/79 to prevent this we wait until state-sync will finish exponentialBackOff := backoff.NewExponentialBackOff( - backoff.WithMaxInterval(time.Second*10), // set max retry interval - backoff.WithMaxElapsedTime(time.Hour*3), // set timeout + backoff.WithMaxInterval(statusClientMaxRetryInterval), // set max retry interval + backoff.WithMaxElapsedTime(statusClientTimeout), // set timeout ) if err := waitUntilClientReady(ctx, eis.client, exponentialBackOff); err != nil { return err From a7288d256338fd3b8624b049e436727382be5183 Mon Sep 17 00:00:00 2001 From: evgeniy-scherbina Date: Mon, 30 Sep 2024 20:02:17 -0400 Subject: [PATCH 7/8] fix: linter --- .golangci.yml | 2 ++ tests/rpc/utils.go | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/.golangci.yml b/.golangci.yml index 3f76d081a9..988fd8aeda 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -48,6 +48,8 @@ issues: - lll - path: rpc/websockets.go text: 'G114: Use of net/http serve function that has no support for setting timeouts' + exclude: + - "G115: integer overflow conversion" max-same-issues: 50 linters-settings: diff --git a/tests/rpc/utils.go b/tests/rpc/utils.go index 77539af10d..f1c5055c79 100644 --- a/tests/rpc/utils.go +++ b/tests/rpc/utils.go @@ -149,7 +149,7 @@ func CallWithError(method string, params interface{}) (*Response, error) { } if rpcRes.Error != nil { - return nil, fmt.Errorf(rpcRes.Error.Message) + return nil, errors.New(rpcRes.Error.Message) } return rpcRes, nil From 657d774f3347776cef10ed5f3f60b9aa4a25fb2f Mon Sep 17 00:00:00 2001 From: evgeniy-scherbina Date: Mon, 30 Sep 2024 20:08:28 -0400 Subject: [PATCH 8/8] chore: increase status client timeout --- server/indexer_service.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/indexer_service.go b/server/indexer_service.go index e6d113e9d3..88e8c56240 100644 --- a/server/indexer_service.go +++ b/server/indexer_service.go @@ -34,7 +34,7 @@ const ( NewBlockWaitTimeout = 60 * time.Second statusClientMaxRetryInterval = time.Second * 10 - statusClientTimeout = time.Hour * 3 + statusClientTimeout = time.Hour * 48 ) // EVMIndexerService indexes transactions for json-rpc service.