feat: support full backup

ref: longhorn/longhorn 7070

Signed-off-by: Jack Lin <[email protected]>

backupstore.go

@@ -23,6 +23,10 @@ type Volume struct {
 	CompressionMethod    string `json:",string"`
 	StorageClassName     string `json:",string"`
 	DataEngine           string `json:",string"`
+
+	// It is not the count of the current backup on the backupstore,
+	// but how many backups of the volume have been created before.
+	BackupCount int64 `json:",string"`
 }
 
 type Snapshot struct {
@@ -37,15 +41,18 @@ type ProcessingBlocks struct {
 
 type Backup struct {
 	sync.Mutex
-	Name              string
-	VolumeName        string
-	SnapshotName      string
-	SnapshotCreatedAt string
-	CreatedTime       string
-	Size              int64 `json:",string"`
-	Labels            map[string]string
-	IsIncremental     bool
-	CompressionMethod string
+	Name                  string
+	VolumeName            string
+	SnapshotName          string
+	SnapshotCreatedAt     string
+	CreatedTime           string
+	Size                  int64 `json:",string"`
+	Labels                map[string]string
+	Parameters            map[string]string
+	IsIncremental         bool
+	CompressionMethod     string
+	NewlyUploadedDataSize int64 `json:",string"`
+	ReUploadedDataSize    int64 `json:",string"`
 
 	ProcessingBlocks *ProcessingBlocks
 
@@ -74,6 +81,9 @@ func addVolume(driver BackupStoreDriver, volume *Volume) error {
 		return fmt.Errorf("invalid volume name %v", volume.Name)
 	}
 
+	// first time create backup volume means first time create backup on this backupstore
+	volume.BackupCount = 0
+
 	if err := saveVolume(driver, volume); err != nil {
 		log.WithError(err).Errorf("Failed to add volume %v", volume.Name)
 		return err

deltablock.go

@@ -16,6 +16,7 @@ import (
 	. "github.com/longhorn/backupstore/logging"
 	"github.com/longhorn/backupstore/types"
 	"github.com/longhorn/backupstore/util"
+	lhbackup "github.com/longhorn/go-common-libs/backup"
 )
 
 type DeltaBackupConfig struct {
@@ -26,6 +27,7 @@ type DeltaBackupConfig struct {
 	DeltaOps        DeltaBlockBackupOperations
 	Labels          map[string]string
 	ConcurrentLimit int32
+	Parameters      map[string]string
 }
 
 type DeltaRestoreConfig struct {
@@ -175,7 +177,7 @@ func CreateDeltaBlockBackup(backupName string, config *DeltaBackupConfig) (isInc
 	}
 
 	backupRequest := &backupRequest{}
-	if volume.LastBackupName != "" {
+	if volume.LastBackupName != "" && !isFullBackup(config) {
 		lastBackupName := volume.LastBackupName
 		var backup, err = loadBackup(bsDriver, lastBackupName, volume.Name)
 		if err != nil {
@@ -208,6 +210,9 @@ func CreateDeltaBlockBackup(backupName string, config *DeltaBackupConfig) (isInc
 		}
 	}
 
+	logrus.Infof("[DEBUG] isFullBackup(config): %v", isFullBackup(config))
+	logrus.Infof("[DEBUG] backupRequest.lastBackup: %v", backupRequest.lastBackup)
+
 	log.WithFields(logrus.Fields{
 		LogFieldReason:       LogReasonStart,
 		LogFieldObject:       LogObjectSnapshot,
@@ -368,13 +373,20 @@ func backupBlock(bsDriver BackupStoreDriver, config *DeltaBackupConfig,
 	}()
 
 	blkFile := getBlockFilePath(volume.Name, checksum)
+	reUpload := false
 	if bsDriver.FileExists(blkFile) {
-		log.Debugf("Found existing block matching at %v", blkFile)
-		return nil
+		if !isFullBackup(config) {
+			log.Debugf("Found existing block matching at %v", blkFile)
+			return nil
+		}
+		log.Debugf("Reupload existing block matching at %v", blkFile)
+		reUpload = true
 	}
 
-	log.Tracef("Creating new block file at %v", blkFile)
-	newBlock = true
+	updateUploadDataSize(reUpload, deltaBackup)
+
+	log.Tracef("Uploading block file at %v", blkFile)
+	newBlock = !reUpload
 	rs, err := util.CompressData(deltaBackup.CompressionMethod, block)
 	if err != nil {
 		return err
@@ -383,6 +395,17 @@ func backupBlock(bsDriver BackupStoreDriver, config *DeltaBackupConfig,
 	return bsDriver.Write(blkFile, rs)
 }
 
+func updateUploadDataSize(reUpload bool, deltaBackup *Backup) {
+	deltaBackup.Lock()
+	defer deltaBackup.Unlock()
+
+	if reUpload {
+		deltaBackup.ReUploadedDataSize += DEFAULT_BLOCK_SIZE
+	} else {
+		deltaBackup.NewlyUploadedDataSize += DEFAULT_BLOCK_SIZE
+	}
+}
+
 func backupMapping(bsDriver BackupStoreDriver, config *DeltaBackupConfig,
 	deltaBackup *Backup, blockSize int64, mapping types.Mapping, progress *progress) error {
 	volume := config.Volume
@@ -532,7 +555,10 @@ func performBackup(bsDriver BackupStoreDriver, config *DeltaBackupConfig, delta
 	backup.CreatedTime = util.Now()
 	backup.Size = int64(len(backup.Blocks)) * DEFAULT_BLOCK_SIZE
 	backup.Labels = config.Labels
+	backup.Parameters = config.Parameters
 	backup.IsIncremental = lastBackup != nil
+	backup.NewlyUploadedDataSize = deltaBackup.NewlyUploadedDataSize
+	backup.ReUploadedDataSize = deltaBackup.ReUploadedDataSize
 
 	if err := saveBackup(bsDriver, backup); err != nil {
 		return progress.progress, "", err
@@ -554,6 +580,7 @@ func performBackup(bsDriver BackupStoreDriver, config *DeltaBackupConfig, delta
 	volume.CompressionMethod = config.Volume.CompressionMethod
 	volume.StorageClassName = config.Volume.StorageClassName
 	volume.DataEngine = config.Volume.DataEngine
+	volume.BackupCount = volume.BackupCount + 1
 
 	if err := saveVolume(bsDriver, volume); err != nil {
 		return progress.progress, "", err
@@ -1303,3 +1330,12 @@ func getBlockNamesForVolume(driver BackupStoreDriver, volumeName string) ([]stri
 
 	return util.ExtractNames(names, "", BLK_SUFFIX), nil
 }
+
+func isFullBackup(config *DeltaBackupConfig) bool {
+	if config.Parameters != nil {
+		if backupMode, exist := config.Parameters[lhbackup.LonghornBackupParameterBackupMode]; exist {
+			return backupMode == lhbackup.LonghornBackupModeFull
+		}
+	}
+	return false
+}

go.mod

@@ -7,20 +7,21 @@ require (
 	github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v0.3.0
 	github.com/aws/aws-sdk-go v1.34.2
 	github.com/gammazero/workerpool v1.1.2
-	github.com/google/uuid v1.3.0
+	github.com/google/uuid v1.6.0
+	github.com/longhorn/go-common-libs v0.0.0-20240319112414-b75404dc7fbc
 	github.com/pierrec/lz4/v4 v4.1.17
 	github.com/pkg/errors v0.9.1
-	github.com/sirupsen/logrus v1.6.0
+	github.com/sirupsen/logrus v1.9.3
 	github.com/slok/goresilience v0.2.0
 	github.com/spf13/afero v1.5.1
-	github.com/stretchr/testify v1.8.0
+	github.com/stretchr/testify v1.8.4
 	github.com/urfave/cli v1.20.0
 	go.uber.org/multierr v1.9.0
-	golang.org/x/net v0.17.0
-	golang.org/x/sys v0.13.0
+	golang.org/x/net v0.20.0
+	golang.org/x/sys v0.18.0
 	gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c
 	k8s.io/apimachinery v0.26.0
-	k8s.io/mount-utils v0.26.0
+	k8s.io/mount-utils v0.29.3
 )
 
 require (
@@ -29,11 +30,10 @@ require (
 	github.com/cespare/xxhash/v2 v2.1.1 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/gammazero/deque v0.1.0 // indirect
-	github.com/go-logr/logr v1.2.3 // indirect
-	github.com/golang/protobuf v1.5.2 // indirect
+	github.com/go-logr/logr v1.3.0 // indirect
+	github.com/golang/protobuf v1.5.3 // indirect
 	github.com/jmespath/go-jmespath v0.3.0 // indirect
-	github.com/konsorten/go-windows-terminal-sequences v1.0.3 // indirect
-	github.com/kr/pretty v0.2.1 // indirect
+	github.com/kr/pretty v0.3.1 // indirect
 	github.com/kr/text v0.2.0 // indirect
 	github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect
 	github.com/moby/sys/mountinfo v0.6.2 // indirect
@@ -42,10 +42,13 @@ require (
 	github.com/prometheus/client_model v0.2.0 // indirect
 	github.com/prometheus/common v0.26.0 // indirect
 	github.com/prometheus/procfs v0.6.0 // indirect
+	github.com/rogpeppe/go-internal v1.10.0 // indirect
 	go.uber.org/atomic v1.7.0 // indirect
-	golang.org/x/text v0.13.0 // indirect
-	google.golang.org/protobuf v1.28.1 // indirect
+	golang.org/x/text v0.14.0 // indirect
+	google.golang.org/protobuf v1.33.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
-	k8s.io/klog/v2 v2.80.1 // indirect
-	k8s.io/utils v0.0.0-20221107191617-1a15be271d1d // indirect
+	k8s.io/klog/v2 v2.110.1 // indirect
+	k8s.io/utils v0.0.0-20230726121419-3b25d923346b // indirect
 )
+
+replace github.com/longhorn/go-common-libs v0.0.0-20240319112414-b75404dc7fbc => github.com/ChanYiLin/go-common-libs v0.0.0-20240329072435-638753f76305