Use exponential backoff for retries

dkron/execution.go

@@ -2,13 +2,16 @@ package dkron
 
 import (
 	"fmt"
+	"math"
 	"strconv"
 	"time"
 
 	proto "github.com/distribworks/dkron/v3/plugin/types"
 	"google.golang.org/protobuf/types/known/timestamppb"
 )
 
+const defaultRetryInterval = 500 * time.Millisecond
+
 // Execution type holds all of the details of a specific Execution.
 type Execution struct {
 	// Id is the Key for this execution
@@ -91,3 +94,13 @@ func (e *Execution) Key() string {
 func (e *Execution) GetGroup() string {
 	return strconv.FormatInt(e.Group, 10)
 }
+
+func (e *Execution) CalculateExponentialBackoff() time.Duration {
+	now := time.Now()
+	if now.Before(e.StartedAt) {
+		return 0
+	}
+	diff := now.Sub(e.StartedAt)
+	backoff := math.Log2(float64(diff/defaultRetryInterval)) + float64(e.Attempt)
+	return time.Duration(backoff) * defaultRetryInterval
+}

dkron/grpc.go

@@ -217,14 +217,19 @@ func (grpcs *GRPCServer) ExecutionDone(ctx context.Context, execDoneReq *proto.E
 		// Keep all execution properties intact except the last output
 		execution.Output = ""
 
+		eb := execution.CalculateExponentialBackoff()
 		grpcs.logger.WithFields(logrus.Fields{
 			"attempt":   execution.Attempt,
 			"execution": execution,
+			"backoff":   eb,
 		}).Debug("grpc: Retrying execution")
 
+		time.Sleep(eb)
+
 		if _, err := grpcs.agent.Run(job.Name, execution); err != nil {
 			return nil, err
 		}
+
 		return &proto.ExecutionDoneResponse{
 			From:    grpcs.agent.config.NodeName,
 			Payload: []byte("retry"),