feat: retry send transaction

go.mod

@@ -43,6 +43,7 @@ require (
 	github.com/acomagu/bufpipe v1.0.4 // indirect
 	github.com/agnivade/levenshtein v1.1.1 // indirect
 	github.com/alecthomas/units v0.0.0-20231202071711-9a357b53e9c9 // indirect
+	github.com/avast/retry-go v3.0.0+incompatible // indirect
 	github.com/aws/aws-sdk-go-v2 v1.25.1 // indirect
 	github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.1 // indirect
 	github.com/aws/aws-sdk-go-v2/config v1.27.3 // indirect

go.sum

@@ -76,6 +76,8 @@ github.com/apache/arrow/go/arrow v0.0.0-20210105145422-88aaea5262db/go.mod h1:c9
 github.com/arbovm/levenshtein v0.0.0-20160628152529-48b4e1c0c4d0/go.mod h1:t2tdKJDJF9BV14lnkjHmOQgcvEKgtqs5a1N3LNdJhGE=
 github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio=
 github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs=
+github.com/avast/retry-go v3.0.0+incompatible h1:4SOWQ7Qs+oroOTQOYnAHqelpCO0biHSxpiH9JdtuBj0=
+github.com/avast/retry-go v3.0.0+incompatible/go.mod h1:XtSnn+n/sHqQIpZ10K1qAevBhOOCWBLXXy3hyiqqBrY=
 github.com/awalterschulze/gographviz v0.0.0-20190221210632-1e9ccb565bca/go.mod h1:GEV5wmg4YquNw7v1kkyoX9etIk8yVmXj+AkDHuuETHs=
 github.com/aws/aws-sdk-go-v2 v1.25.1 h1:P7hU6A5qEdmajGwvae/zDkOq+ULLC9tQBTwqqiwFGpI=
 github.com/aws/aws-sdk-go-v2 v1.25.1/go.mod h1:Evoc5AsmtveRt1komDwIsjHFyrP5tDuF1D1U+6z6pNo=

pkg/resourceprovider/resourceprovider.go

@@ -6,8 +6,10 @@ import (
 	"fmt"
 	"math/big"
 	"strconv"
+	"strings"
 	"time"
 
+	retry "github.com/avast/retry-go"
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/google/uuid"
 	"github.com/holiman/uint256"
@@ -118,9 +120,19 @@ func (resourceProvider *ResourceProvider) StartMineLoop(ctx context.Context) cha
 
 	taskCh := make(chan Task)
 	resourceProvider.controller.web3Events.Pow.SubscribenewPowRound(func(newPowRound pow.PowNewPowRound) {
-
-		_, challenge, err := resourceProvider.web3SDK.GetGenerateChallenge(ctx, nodeId)
-
+		var challenge *pow.PowGenerateChallenge
+		var err error
+		err = retrySend(ctx, 28, func() error {
+			_, challenge, err = resourceProvider.web3SDK.GetGenerateChallenge(ctx, nodeId)
+			if err != nil {
+				if isRecoverable(err) {
+					log.Warn().Err(err).Msg("got a error, try to resend get challenge")
+					return err
+				}
+				return retry.Unrecoverable(err)
+			}
+			return nil
+		})
 		if err != nil {
 			log.Err(err).Msgf("Unable to fetch challenge")
 			return
@@ -141,7 +153,6 @@ func (resourceProvider *ResourceProvider) StartMineLoop(ctx context.Context) cha
 			Challenge:  challenge.Challenge,
 			Difficulty: difficulty,
 		}
-
 	})
 
 	submitWork := func(nonce *big.Int, hashrate float64) {
@@ -153,11 +164,24 @@ func (resourceProvider *ResourceProvider) StartMineLoop(ctx context.Context) cha
 			Date:     finishTime,
 			Hashrate: hashrate,
 		})
-		txId, err := resourceProvider.web3SDK.SubmitWork(ctx, nonce, nodeId)
+
+		var txId common.Hash
+		err = retrySend(ctx, 28, func() error {
+			txId, err = resourceProvider.web3SDK.SubmitWork(ctx, nonce, nodeId)
+			if err != nil {
+				if isRecoverable(err) {
+					log.Warn().Err(err).Msg("got a error, try to submit work")
+					return err
+				}
+				return retry.Unrecoverable(err) //others is unrecoverable  or all error regard as recoverable?
+			}
+			return nil
+		})
 		if err != nil {
 			log.Err(err).Msgf("Submit work fail")
 			return
 		}
+
 		log.Info().Str("address", walletAddress.Hex()).
 			Str("nodeid", nodeId).
 			Str("Nonce", nonce.String()).
@@ -187,3 +211,18 @@ func TriggerNewPowRound(ctx context.Context, web3SDK *web3.Web3SDK) (common.Hash
 	}
 	return tx.Hash(), nil
 }
+
+// retrySend retry interval 1s,2s,4s,8s,10s,10s,10s
+// total time cost 15 +10(retryCount-4) for retryCount > 4
+func retrySend(ctx context.Context, retryCount uint, retryableFunc retry.RetryableFunc) error {
+	return retry.Do(retryableFunc, retry.Attempts(retryCount),
+		retry.Context(ctx),
+		retry.DelayType(retry.BackOffDelay),
+		retry.Delay(time.Second),
+		retry.MaxDelay(time.Second*10),
+	)
+}
+
+func isRecoverable(err error) bool {
+	return strings.Contains(err.Error(), "Your app has exceeded its compute units per second capacity")
+}