Add support for timeout for restic backup and restore

apis/constant.go

@@ -75,7 +75,6 @@ const (
 	KubeStashAppRefKind      = "kubestash.com/app-ref-kind"
 	KubeStashAppRefNamespace = "kubestash.com/app-ref-namespace"
 	KubeStashAppRefName      = "kubestash.com/app-ref-name"
-	KubeDBAppVersion         = "kubedb.com/db-version"
 )
 
 // Keys for structure logging
@@ -153,3 +152,9 @@ const (
 	SnapshotVersionV1 = "v1"
 	DirRepository     = "repository"
 )
+
+// Annotations
+const (
+	AnnKubeDBAppVersion          = "kubedb.com/db-version"
+	AnnRestoreSessionBeneficiary = "restoresession.kubestash.com/beneficiary"
+)

apis/core/v1alpha1/backupconfiguration_types.go

@@ -132,8 +132,8 @@ type SessionConfig struct {
 	// +optional
 	RetryConfig *RetryConfig `json:"retryConfig,omitempty"`
 
-	// Timeout specifies the maximum duration of backup. BackupSession will be considered Failed
-	// if backup does not complete within this time limit. By default, KubeStash don't set any timeout for backup.
+	// Timeout specifies the maximum duration of backup. Backup will be considered Failed
+	// if backup tasks do not complete within this time limit. By default, KubeStash don't set any timeout for backup.
 	// +optional
 	Timeout *metav1.Duration `json:"timeout,omitempty"`
 

apis/core/v1alpha1/backupsession_helpers.go

@@ -243,3 +243,14 @@ func (b *BackupSession) checkFailureInRetentionPolicy() (bool, string) {
 	}
 	return false, ""
 }
+
+func (b *BackupSession) GetRemainingTimeoutDuration() (*metav1.Duration, error) {
+	if b.Spec.Timeout == nil || b.Status.Deadline == nil {
+		return nil, nil
+	}
+	currentTime := metav1.Now()
+	if b.Status.Deadline.Before(&currentTime) {
+		return nil, fmt.Errorf("deadline exceeded")
+	}
+	return &metav1.Duration{Duration: b.Status.Deadline.Sub(currentTime.Time)}, nil
+}

apis/core/v1alpha1/backupsession_types.go

@@ -63,6 +63,11 @@ type BackupSessionSpec struct {
 	// If this set to non-zero, KubeStash will create a new BackupSession if the current one fails.
 	// +optional
 	RetryLeft int32 `json:"retryLeft,omitempty"`
+
+	// Timeout specifies the maximum duration of backup. Backup will be considered Failed
+	// if backup tasks do not complete within this time limit. By default, KubeStash don't set any timeout for backup.
+	// +optional
+	Timeout *metav1.Duration `json:"timeout,omitempty"`
 }
 
 // BackupSessionStatus defines the observed state of BackupSession
@@ -75,10 +80,10 @@ type BackupSessionStatus struct {
 	// +optional
 	Duration string `json:"duration,omitempty"`
 
-	// Deadline specifies the deadline of backup. BackupSession will be
-	// considered Failed if backup does not complete within this deadline
+	// Deadline specifies the deadline of backup. Backup will be
+	// considered Failed if it does not complete within this deadline
 	// +optional
-	Deadline *metav1.Time `json:"sessionDeadline,omitempty"`
+	Deadline *metav1.Time `json:"backupDeadline,omitempty"`
 
 	// TotalSnapshots specifies the total number of snapshots created for this backupSession.
 	// +optional

apis/core/v1alpha1/restoresession_helpers.go

@@ -17,6 +17,7 @@ limitations under the License.
 package v1alpha1
 
 import (
+	"fmt"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	kmapi "kmodules.xyz/client-go/api/v1"
 	"kubestash.dev/apimachinery/apis"
@@ -41,8 +42,7 @@ func (rs *RestoreSession) CalculatePhase() RestorePhase {
 	}
 
 	if cutil.IsConditionTrue(rs.Status.Conditions, TypeMetricsPushed) &&
-		(cutil.IsConditionTrue(rs.Status.Conditions, TypeDeadlineExceeded) ||
-			cutil.IsConditionFalse(rs.Status.Conditions, TypePreRestoreHooksExecutionSucceeded) ||
+		(cutil.IsConditionFalse(rs.Status.Conditions, TypePreRestoreHooksExecutionSucceeded) ||
 			cutil.IsConditionFalse(rs.Status.Conditions, TypePostRestoreHooksExecutionSucceeded) ||
 			cutil.IsConditionFalse(rs.Status.Conditions, TypeRestoreExecutorEnsured)) {
 		return RestoreFailed
@@ -181,3 +181,14 @@ func (rs *RestoreSession) GetDataSourceNamespace() string {
 	}
 	return rs.Spec.DataSource.Namespace
 }
+
+func (rs *RestoreSession) GetRemainingTimeoutDuration() (*metav1.Duration, error) {
+	if rs.Spec.Timeout == nil || rs.Status.Deadline == nil {
+		return nil, nil
+	}
+	currentTime := metav1.Now()
+	if rs.Status.Deadline.Before(&currentTime) {
+		return nil, fmt.Errorf("deadline exceeded")
+	}
+	return &metav1.Duration{Duration: rs.Status.Deadline.Sub(currentTime.Time)}, nil
+}

apis/core/v1alpha1/restoresession_helpers_test.go

@@ -230,25 +230,6 @@ func TestRestoreSessionPhaseIsFailedIfRestoreExecutorEnsuredConditionIsFalse(t *
 	assert.Equal(t, RestoreFailed, rs.CalculatePhase())
 }
 
-func TestRestoreSessionPhaseIsFailedIfDeadlineExceededConditionIsTrue(t *testing.T) {
-	rs := sampleRestoreSession(func(r *RestoreSession) {
-		r.Status.Conditions = append(r.Status.Conditions,
-			kmapi.Condition{
-				Type:   TypeDeadlineExceeded,
-				Status: metav1.ConditionTrue,
-				Reason: ReasonFailedToCompleteWithinDeadline,
-			},
-			kmapi.Condition{
-				Type:   TypeMetricsPushed,
-				Status: metav1.ConditionTrue,
-				Reason: ReasonSuccessfullyPushedMetrics,
-			},
-		)
-	})
-
-	assert.Equal(t, RestoreFailed, rs.CalculatePhase())
-}
-
 func TestRestoreSessionPhaseIsRunningIfPostRestoreHooksNotExecuted(test *testing.T) {
 	rs := sampleRestoreSession(func(r *RestoreSession) {
 		r.Status.Components = map[string]ComponentRestoreStatus{

apis/core/v1alpha1/restoresession_types.go

@@ -64,8 +64,8 @@ type RestoreSessionSpec struct {
 	// +optional
 	Hooks *RestoreHooks `json:"hooks,omitempty"`
 
-	// Timeout specifies a duration that KubeStash should wait for the session execution to be completed.
-	// If the session execution does not finish within this time period, KubeStash will consider this session as a failure.
+	// Timeout specifies a duration that KubeStash should wait for the restore to be completed.
+	// If the restore tasks do not finish within this time period, KubeStash will consider this restore as a failure.
 	// +optional
 	Timeout *metav1.Duration `json:"timeout,omitempty"`
 
@@ -278,10 +278,10 @@ type RestoreSessionStatus struct {
 	// +optional
 	Duration string `json:"duration,omitempty"`
 
-	// Deadline specifies a timestamp till this session is valid. If the session does not complete within this deadline,
-	// it will be considered as failed.
+	// Deadline specifies the deadline of restore. Restore will be
+	// considered Failed if it does not complete within this deadline
 	// +optional
-	Deadline *metav1.Time `json:"deadline,omitempty"`
+	Deadline *metav1.Time `json:"restoreDeadline,omitempty"`
 
 	// TotalComponents represents the number of total components for this RestoreSession
 	// +optional

apis/core/v1alpha1/types.go

@@ -217,9 +217,6 @@ type RetryConfig struct {
 }
 
 const (
-	TypeDeadlineExceeded                 = "DeadlineExceeded"
-	ReasonFailedToCompleteWithinDeadline = "FailedToCompleteWithinDeadline"
-
 	// TypeMetricsPushed indicates whether Metrics are pushed or not
 	TypeMetricsPushed               = "MetricsPushed"
 	ReasonSuccessfullyPushedMetrics = "SuccessfullyPushedMetrics"

crds/core.kubestash.com_backupbatches.yaml

@@ -36196,9 +36196,9 @@ spec:
                       type: array
                     timeout:
                       description: Timeout specifies the maximum duration of backup.
-                        BackupSession will be considered Failed if backup does not
-                        complete within this time limit. By default, KubeStash don't
-                        set any timeout for backup.
+                        Backup will be considered Failed if backup tasks do not complete
+                        within this time limit. By default, KubeStash don't set any
+                        timeout for backup.
                       type: string
                   type: object
                 type: array

crds/core.kubestash.com_backupblueprints.yaml

@@ -37241,8 +37241,8 @@ spec:
                           type: integer
                         timeout:
                           description: Timeout specifies the maximum duration of backup.
-                            BackupSession will be considered Failed if backup does
-                            not complete within this time limit. By default, KubeStash
+                            Backup will be considered Failed if backup tasks do not
+                            complete within this time limit. By default, KubeStash
                             don't set any timeout for backup.
                           type: string
                       type: object

crds/core.kubestash.com_backupconfigurations.yaml

@@ -34375,9 +34375,9 @@ spec:
                       type: integer
                     timeout:
                       description: Timeout specifies the maximum duration of backup.
-                        BackupSession will be considered Failed if backup does not
-                        complete within this time limit. By default, KubeStash don't
-                        set any timeout for backup.
+                        Backup will be considered Failed if backup tasks do not complete
+                        within this time limit. By default, KubeStash don't set any
+                        timeout for backup.
                       type: string
                   type: object
                 type: array

crds/core.kubestash.com_backupsessions.yaml

@@ -87,10 +87,21 @@ spec:
                 description: Session specifies the name of the session that triggered
                   this backup
                 type: string
+              timeout:
+                description: Timeout specifies the maximum duration of backup. Backup
+                  will be considered Failed if backup tasks do not complete within
+                  this time limit. By default, KubeStash don't set any timeout for
+                  backup.
+                type: string
             type: object
           status:
             description: BackupSessionStatus defines the observed state of BackupSession
             properties:
+              backupDeadline:
+                description: Deadline specifies the deadline of backup. Backup will
+                  be considered Failed if it does not complete within this deadline
+                format: date-time
+                type: string
               conditions:
                 description: Conditions represents list of conditions regarding this
                   BackupSession
@@ -249,12 +260,6 @@ spec:
                   not. This field will exist only if the `retryConfig` has been set
                   in the respective backup invoker.
                 type: boolean
-              sessionDeadline:
-                description: Deadline specifies the deadline of backup. BackupSession
-                  will be considered Failed if backup does not complete within this
-                  deadline
-                format: date-time
-                type: string
               snapshots:
                 description: Snapshots specifies the Snapshots status
                 items:

crds/core.kubestash.com_restoresessions.yaml

@@ -24469,9 +24469,9 @@ spec:
                 type: object
               timeout:
                 description: Timeout specifies a duration that KubeStash should wait
-                  for the session execution to be completed. If the session execution
-                  does not finish within this time period, KubeStash will consider
-                  this session as a failure.
+                  for the restore to be completed. If the restore tasks do not finish
+                  within this time period, KubeStash will consider this restore as
+                  a failure.
                 type: string
             type: object
           status:
@@ -24557,12 +24557,6 @@ spec:
                   - type
                   type: object
                 type: array
-              deadline:
-                description: Deadline specifies a timestamp till this session is valid.
-                  If the session does not complete within this deadline, it will be
-                  considered as failed.
-                format: date-time
-                type: string
               dependencies:
                 description: Dependencies specifies whether the objects required by
                   this RestoreSession exist or not
@@ -24667,6 +24661,11 @@ spec:
                 - Invalid
                 - Unknown
                 type: string
+              restoreDeadline:
+                description: Deadline specifies the deadline of restore. Restore will
+                  be considered Failed if it does not complete within this deadline
+                format: date-time
+                type: string
               targetFound:
                 description: TargetFound specifies whether the restore target exist
                   or not

pkg/restic/commands.go

@@ -35,7 +35,8 @@ import (
 )
 
 const (
-	ResticCMD = "restic"
+	ResticCMD  = "restic"
+	TimeoutCMD = "timeout"
 )
 
 type Snapshot struct {
@@ -176,7 +177,10 @@ func (w *ResticWrapper) backup(params backupParams) ([]byte, error) {
 	args = w.appendInsecureTLSFlag(args)
 	args = w.appendMaxConnectionsFlag(args)
 
-	return w.run(Command{Name: ResticCMD, Args: args})
+	command := Command{Name: ResticCMD, Args: args}
+	command = w.wrapWithTimeoutIfConfigured(command)
+
+	return w.run(command)
 }
 
 func (w *ResticWrapper) backupFromStdin(options BackupOptions) ([]byte, error) {
@@ -200,7 +204,10 @@ func (w *ResticWrapper) backupFromStdin(options BackupOptions) ([]byte, error) {
 	args = w.appendInsecureTLSFlag(args)
 	args = w.appendMaxConnectionsFlag(args)
 
-	commands = append(commands, Command{Name: ResticCMD, Args: args})
+	command := Command{Name: ResticCMD, Args: args}
+	command = w.wrapWithTimeoutIfConfigured(command)
+
+	commands = append(commands, command)
 	return w.run(commands...)
 }
 
@@ -246,7 +253,10 @@ func (w *ResticWrapper) restore(params restoreParams) ([]byte, error) {
 	args = w.appendInsecureTLSFlag(args)
 	args = w.appendMaxConnectionsFlag(args)
 
-	return w.run(Command{Name: ResticCMD, Args: args})
+	command := Command{Name: ResticCMD, Args: args}
+	command = w.wrapWithTimeoutIfConfigured(command)
+
+	return w.run(command)
 }
 
 func (w *ResticWrapper) DumpOnce(dumpOptions DumpOptions) ([]byte, error) {
@@ -277,10 +287,11 @@ func (w *ResticWrapper) DumpOnce(dumpOptions DumpOptions) ([]byte, error) {
 	args = w.appendMaxConnectionsFlag(args)
 	args = w.appendInsecureTLSFlag(args)
 
+	command := Command{Name: ResticCMD, Args: args}
+	command = w.wrapWithTimeoutIfConfigured(command)
+
 	// first add restic command, then add StdoutPipeCommands
-	commands := []Command{
-		{Name: ResticCMD, Args: args},
-	}
+	commands := []Command{command}
 	commands = append(commands, dumpOptions.StdoutPipeCommands...)
 	return w.run(commands...)
 }
@@ -368,7 +379,7 @@ func (w *ResticWrapper) run(commands ...Command) ([]byte, error) {
 	w.sh.Stderr = io.MultiWriter(os.Stderr, errBuff)
 
 	for _, cmd := range commands {
-		if cmd.Name == ResticCMD {
+		if cmd.Name == ResticCMD || cmd.Name == TimeoutCMD {
 			// first apply NiceSettings, then apply IONiceSettings
 			cmd, err = w.applyNiceSettings(cmd)
 			if err != nil {
@@ -393,6 +404,9 @@ func (w *ResticWrapper) run(commands ...Command) ([]byte, error) {
 func formatError(err error, stdErr string) error {
 	parts := strings.Split(strings.TrimSuffix(stdErr, "\n"), "\n")
 	if len(parts) > 1 {
+		if strings.Contains(parts[1], "signal terminated") {
+			return errors.New(strings.Join(append([]string{"deadline exceeded or signal terminated"}, parts[2:]...), " "))
+		}
 		return errors.New(strings.Join(parts[1:], " "))
 	}
 	return err
@@ -515,3 +529,12 @@ func (w *ResticWrapper) removeKey(params keyParams) ([]byte, error) {
 
 	return w.run(Command{Name: ResticCMD, Args: args})
 }
+
+func (w *ResticWrapper) wrapWithTimeoutIfConfigured(cmd Command) Command {
+	if w.config.Timeout != nil {
+		timeoutArgs := []interface{}{fmt.Sprintf("%f", w.config.Timeout.Seconds()), cmd.Name}
+		timeoutArgs = append(timeoutArgs, cmd.Args...)
+		return Command{Name: TimeoutCMD, Args: timeoutArgs}
+	}
+	return cmd
+}