spdk lvol: snapshot management of UserCreated flag

Longhorn 7578

Signed-off-by: Damiano Cipriani <[email protected]>

Dockerfile.dapper

@@ -36,7 +36,7 @@ RUN curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/i
 ENV HUGEMEM=1024
 RUN echo "vm.nr_hugepages=$((HUGEMEM/2))" >> /etc/sysctl.conf
 ENV SPDK_DIR /usr/src/spdk
-ENV SPDK_COMMIT_ID 72b7762674cc35262086b185a455a8a30f46432d
+ENV SPDK_COMMIT_ID 7a25c3c5a102dfb2d8715f9bf090691176de5ef7
 RUN git clone https://github.com/longhorn/spdk.git ${SPDK_DIR} --recursive && \
     cd ${SPDK_DIR} && \
     git checkout ${SPDK_COMMIT_ID} && \

pkg/api/types.go

@@ -7,6 +7,10 @@ import (
 	"github.com/longhorn/longhorn-spdk-engine/proto/spdkrpc"
 )
 
+type SnapshotOptions struct {
+	UserCreated bool
+}
+
 type Replica struct {
 	Name       string           `json:"name"`
 	LvsName    string           `json:"lvs_name"`
@@ -31,6 +35,7 @@ type Lvol struct {
 	Parent       string          `json:"parent"`
 	Children     map[string]bool `json:"children"`
 	CreationTime string          `json:"creation_time"`
+	UserCreated  bool            `json:"user_created"`
 }
 
 func ProtoLvolToLvol(l *spdkrpc.Lvol) *Lvol {
@@ -45,6 +50,7 @@ func ProtoLvolToLvol(l *spdkrpc.Lvol) *Lvol {
 		Parent:       l.Parent,
 		Children:     l.Children,
 		CreationTime: l.CreationTime,
+		UserCreated:  l.UserCreated,
 	}
 }
 
@@ -60,6 +66,7 @@ func LvolToProtoLvol(l *Lvol) *spdkrpc.Lvol {
 		Parent:       l.Parent,
 		Children:     l.Children,
 		CreationTime: l.CreationTime,
+		UserCreated:  l.UserCreated,
 	}
 }
 

pkg/client/client.go

@@ -138,7 +138,7 @@ func (c *SPDKClient) ReplicaWatch(ctx context.Context) (*api.ReplicaStream, erro
 	return api.NewReplicaStream(stream), nil
 }
 
-func (c *SPDKClient) ReplicaSnapshotCreate(name, snapshotName string) error {
+func (c *SPDKClient) ReplicaSnapshotCreate(name, snapshotName string, opts *api.SnapshotOptions) error {
 	if name == "" || snapshotName == "" {
 		return fmt.Errorf("failed to create SPDK replica snapshot: missing required parameter name or snapshot name")
 	}
@@ -150,6 +150,7 @@ func (c *SPDKClient) ReplicaSnapshotCreate(name, snapshotName string) error {
 	_, err := client.ReplicaSnapshotCreate(ctx, &spdkrpc.SnapshotRequest{
 		Name:         name,
 		SnapshotName: snapshotName,
+		UserCreated:  opts != nil && opts.UserCreated,
 	})
 	return errors.Wrapf(err, "failed to create SPDK replica %s snapshot %s", name, snapshotName)
 }
@@ -318,7 +319,7 @@ func (c *SPDKClient) ReplicaRebuildingDstFinish(replicaName string, unexposeRequ
 	return errors.Wrapf(err, "failed to finish replica rebuilding dst %s", replicaName)
 }
 
-func (c *SPDKClient) ReplicaRebuildingDstSnapshotCreate(name, snapshotName string) error {
+func (c *SPDKClient) ReplicaRebuildingDstSnapshotCreate(name, snapshotName string, opts *api.SnapshotOptions) error {
 	if name == "" || snapshotName == "" {
 		return fmt.Errorf("failed to create dst SPDK replica rebuilding snapshot: missing required parameter name or snapshot name")
 	}
@@ -330,6 +331,7 @@ func (c *SPDKClient) ReplicaRebuildingDstSnapshotCreate(name, snapshotName strin
 	_, err := client.ReplicaRebuildingDstSnapshotCreate(ctx, &spdkrpc.SnapshotRequest{
 		Name:         name,
 		SnapshotName: snapshotName,
+		UserCreated:  opts != nil && opts.UserCreated,
 	})
 	return errors.Wrapf(err, "failed to create dst SPDK replica %s rebuilding snapshot %s", name, snapshotName)
 }
@@ -434,7 +436,7 @@ func (c *SPDKClient) EngineWatch(ctx context.Context) (*api.EngineStream, error)
 	return api.NewEngineStream(stream), nil
 }
 
-func (c *SPDKClient) EngineSnapshotCreate(name, snapshotName string) (string, error) {
+func (c *SPDKClient) EngineSnapshotCreate(name, snapshotName string, opts *api.SnapshotOptions) (string, error) {
 	if name == "" {
 		return "", fmt.Errorf("failed to create SPDK engine snapshot: missing required parameter name")
 	}
@@ -446,6 +448,7 @@ func (c *SPDKClient) EngineSnapshotCreate(name, snapshotName string) (string, er
 	resp, err := client.EngineSnapshotCreate(ctx, &spdkrpc.SnapshotRequest{
 		Name:         name,
 		SnapshotName: snapshotName,
+		UserCreated:  opts != nil && opts.UserCreated,
 	})
 	if err != nil {
 		return "", errors.Wrapf(err, "failed to create SPDK engine %s snapshot %s", name, snapshotName)

pkg/spdk/engine.go

@@ -788,7 +788,7 @@ func (e *Engine) ReplicaAddStart(spdkClient *spdkclient.Client, replicaName, rep
 
 	// TODO: For online rebuilding, the IO should be paused first
 	snapshotName := GenerateRebuildingSnapshotName()
-	updateRequired, err = e.snapshotOperationWithoutLock(spdkClient, replicaClients, snapshotName, SnapshotOperationCreate)
+	updateRequired, err = e.snapshotOperationWithoutLock(spdkClient, replicaClients, snapshotName, SnapshotOperationCreate, nil)
 	if err != nil {
 		return err
 	}
@@ -979,7 +979,7 @@ func (e *Engine) ReplicaShallowCopy(dstReplicaName, dstReplicaAddress string) (e
 		if err = srcReplicaServiceCli.ReplicaSnapshotShallowCopy(srcReplicaName, currentSnapshotName); err != nil {
 			return err
 		}
-		if err = dstReplicaServiceCli.ReplicaRebuildingDstSnapshotCreate(dstReplicaName, currentSnapshotName); err != nil {
+		if err = dstReplicaServiceCli.ReplicaRebuildingDstSnapshotCreate(dstReplicaName, currentSnapshotName, &api.SnapshotOptions{UserCreated: rpcSrcReplica.Snapshots[currentSnapshotName].UserCreated}); err != nil {
 			return err
 		}
 		prevSnapshotName = currentSnapshotName
@@ -1067,21 +1067,21 @@ const (
 	SnapshotOperationRevert = SnapshotOperationType("snapshot-revert")
 )
 
-func (e *Engine) SnapshotCreate(spdkClient *spdkclient.Client, inputSnapshotName string) (snapshotName string, err error) {
-	return e.snapshotOperation(spdkClient, inputSnapshotName, SnapshotOperationCreate)
+func (e *Engine) SnapshotCreate(spdkClient *spdkclient.Client, inputSnapshotName string, opts *api.SnapshotOptions) (snapshotName string, err error) {
+	return e.snapshotOperation(spdkClient, inputSnapshotName, SnapshotOperationCreate, opts)
 }
 
 func (e *Engine) SnapshotDelete(spdkClient *spdkclient.Client, snapshotName string) (err error) {
-	_, err = e.snapshotOperation(spdkClient, snapshotName, SnapshotOperationDelete)
+	_, err = e.snapshotOperation(spdkClient, snapshotName, SnapshotOperationDelete, nil)
 	return err
 }
 
 func (e *Engine) SnapshotRevert(spdkClient *spdkclient.Client, snapshotName string) (err error) {
-	_, err = e.snapshotOperation(spdkClient, snapshotName, SnapshotOperationRevert)
+	_, err = e.snapshotOperation(spdkClient, snapshotName, SnapshotOperationRevert, nil)
 	return err
 }
 
-func (e *Engine) snapshotOperation(spdkClient *spdkclient.Client, inputSnapshotName string, snapshotOp SnapshotOperationType) (snapshotName string, err error) {
+func (e *Engine) snapshotOperation(spdkClient *spdkclient.Client, inputSnapshotName string, snapshotOp SnapshotOperationType, opts *api.SnapshotOptions) (snapshotName string, err error) {
 	updateRequired := false
 
 	if snapshotOp == SnapshotOperationCreate {
@@ -1143,7 +1143,7 @@ func (e *Engine) snapshotOperation(spdkClient *spdkclient.Client, inputSnapshotN
 		}
 	}()
 
-	if updateRequired, err = e.snapshotOperationWithoutLock(spdkClient, replicaClients, snapshotName, snapshotOp); err != nil {
+	if updateRequired, err = e.snapshotOperationWithoutLock(spdkClient, replicaClients, snapshotName, snapshotOp, opts); err != nil {
 		return "", err
 	}
 
@@ -1219,7 +1219,7 @@ func (e *Engine) snapshotOperationPreCheckWithoutLock(replicaClients map[string]
 	return snapshotName, nil
 }
 
-func (e *Engine) snapshotOperationWithoutLock(spdkClient *spdkclient.Client, replicaClients map[string]*client.SPDKClient, snapshotName string, snapshotOp SnapshotOperationType) (updated bool, err error) {
+func (e *Engine) snapshotOperationWithoutLock(spdkClient *spdkclient.Client, replicaClients map[string]*client.SPDKClient, snapshotName string, snapshotOp SnapshotOperationType, opts *api.SnapshotOptions) (updated bool, err error) {
 	if snapshotOp == SnapshotOperationRevert {
 		if _, err := spdkClient.BdevRaidDelete(e.Name); err != nil && !jsonrpc.IsJSONRPCRespErrorNoSuchDevice(err) {
 			e.log.WithError(err).Errorf("Failed to delete RAID after snapshot %s revert", snapshotName)
@@ -1228,7 +1228,7 @@ func (e *Engine) snapshotOperationWithoutLock(spdkClient *spdkclient.Client, rep
 	}
 
 	for replicaName := range replicaClients {
-		if err := e.replicaSnapshotOperation(spdkClient, replicaClients[replicaName], replicaName, snapshotName, snapshotOp); err != nil && e.ReplicaModeMap[replicaName] != types.ModeERR {
+		if err := e.replicaSnapshotOperation(spdkClient, replicaClients[replicaName], replicaName, snapshotName, snapshotOp, opts); err != nil && e.ReplicaModeMap[replicaName] != types.ModeERR {
 			e.ReplicaModeMap[replicaName] = types.ModeERR
 			e.ReplicaBdevNameMap[replicaName] = ""
 			e.log.WithError(err).Errorf("Failed to issue operation %s for replica %s snapshot %s, will mark the replica as mode ERR", snapshotOp, replicaName, snapshotName)
@@ -1256,11 +1256,11 @@ func (e *Engine) snapshotOperationWithoutLock(spdkClient *spdkclient.Client, rep
 	return updated, nil
 }
 
-func (e *Engine) replicaSnapshotOperation(spdkClient *spdkclient.Client, replicaClient *client.SPDKClient, replicaName, snapshotName string, snapshotOp SnapshotOperationType) error {
+func (e *Engine) replicaSnapshotOperation(spdkClient *spdkclient.Client, replicaClient *client.SPDKClient, replicaName, snapshotName string, snapshotOp SnapshotOperationType, opts *api.SnapshotOptions) error {
 	switch snapshotOp {
 	case SnapshotOperationCreate:
 		// TODO: execute `sync` for the nvme initiator before snapshot start
-		return replicaClient.ReplicaSnapshotCreate(replicaName, snapshotName)
+		return replicaClient.ReplicaSnapshotCreate(replicaName, snapshotName, opts)
 	case SnapshotOperationDelete:
 		return replicaClient.ReplicaSnapshotDelete(replicaName, snapshotName)
 	case SnapshotOperationRevert:

pkg/spdk/replica.go

@@ -24,6 +24,7 @@ import (
 	helpertypes "github.com/longhorn/go-spdk-helper/pkg/types"
 	helperutil "github.com/longhorn/go-spdk-helper/pkg/util"
 
+	"github.com/longhorn/longhorn-spdk-engine/pkg/api"
 	"github.com/longhorn/longhorn-spdk-engine/pkg/types"
 	"github.com/longhorn/longhorn-spdk-engine/pkg/util"
 	"github.com/longhorn/longhorn-spdk-engine/proto/spdkrpc"
@@ -92,6 +93,7 @@ type Lvol struct {
 	// Children is map[<snapshot lvol name>] rather than map[<snapshot name>]. <snapshot lvol name> consists of `<replica name>-snap-<snapshot name>`
 	Children     map[string]*Lvol
 	CreationTime string
+	UserCreated  bool
 }
 
 func ServiceReplicaToProtoReplica(r *Replica) *spdkrpc.Replica {
@@ -125,6 +127,7 @@ func ServiceLvolToProtoLvol(replicaName string, lvol *Lvol) *spdkrpc.Lvol {
 		Parent:       GetSnapshotNameFromReplicaSnapshotLvolName(replicaName, lvol.Parent),
 		Children:     map[string]bool{},
 		CreationTime: lvol.CreationTime,
+		UserCreated:  lvol.UserCreated,
 	}
 
 	if lvol.Name == replicaName {
@@ -156,6 +159,7 @@ func BdevLvolInfoToServiceLvol(bdev *spdktypes.BdevInfo) *Lvol {
 		// Need to update this separately
 		Children:     map[string]*Lvol{},
 		CreationTime: bdev.CreationTime,
+		UserCreated:  bdev.DriverSpecific.Lvol.Xattrs[spdkclient.UserCreated] == "true",
 	}
 }
 
@@ -782,7 +786,7 @@ func (r *Replica) Get() (pReplica *spdkrpc.Replica) {
 	return ServiceReplicaToProtoReplica(r)
 }
 
-func (r *Replica) SnapshotCreate(spdkClient *spdkclient.Client, snapshotName string) (pReplica *spdkrpc.Replica, err error) {
+func (r *Replica) SnapshotCreate(spdkClient *spdkclient.Client, snapshotName string, opts *api.SnapshotOptions) (pReplica *spdkrpc.Replica, err error) {
 	updateRequired := false
 
 	r.Lock()
@@ -822,7 +826,16 @@ func (r *Replica) SnapshotCreate(spdkClient *spdkclient.Client, snapshotName str
 	}
 	headSvcLvol := r.ActiveChain[r.ChainLength-1]
 
-	snapUUID, err := spdkClient.BdevLvolSnapshot(headSvcLvol.UUID, snapLvolName, nil)
+	var xattrs []spdkclient.Xattr
+	if opts != nil {
+		xattr := spdkclient.Xattr{
+			Name:  spdkclient.UserCreated,
+			Value: strconv.FormatBool(opts.UserCreated),
+		}
+		xattrs = append(xattrs, xattr)
+	}
+
+	snapUUID, err := spdkClient.BdevLvolSnapshot(headSvcLvol.UUID, snapLvolName, xattrs)
 	if err != nil {
 		return nil, err
 	}
@@ -1390,7 +1403,7 @@ func (r *Replica) RebuildingDstFinish(spdkClient *spdkclient.Client, unexposeReq
 // 	return nil
 // }
 
-func (r *Replica) RebuildingDstSnapshotCreate(spdkClient *spdkclient.Client, snapshotName string) (err error) {
+func (r *Replica) RebuildingDstSnapshotCreate(spdkClient *spdkclient.Client, snapshotName string, opts *api.SnapshotOptions) (err error) {
 	updateRequired := false
 
 	r.Lock()
@@ -1426,8 +1439,17 @@ func (r *Replica) RebuildingDstSnapshotCreate(spdkClient *spdkclient.Client, sna
 		}
 	}()
 
+	var xattrs []spdkclient.Xattr
+	if opts != nil {
+		xattr := spdkclient.Xattr{
+			Name:  spdkclient.UserCreated,
+			Value: strconv.FormatBool(opts.UserCreated),
+		}
+		xattrs = append(xattrs, xattr)
+	}
+
 	snapLvolName := GetReplicaSnapshotLvolName(r.Name, snapshotName)
-	snapUUID, err := spdkClient.BdevLvolSnapshot(r.rebuildingLvol.UUID, snapLvolName, nil)
+	snapUUID, err := spdkClient.BdevLvolSnapshot(r.rebuildingLvol.UUID, snapLvolName, xattrs)
 	if err != nil {
 		return err
 	}
@@ -1709,7 +1731,7 @@ func (r *Replica) postFullRestoreOperations(spdkClient *spdkclient.Client, resto
 
 	r.log.Infof("Taking snapshot %v of the restored volume", restore.SnapshotName)
 
-	_, err := r.SnapshotCreate(spdkClient, restore.SnapshotName)
+	_, err := r.SnapshotCreate(spdkClient, restore.SnapshotName, nil)
 	if err != nil {
 		r.log.WithError(err).Error("Failed to take snapshot of the restored volume")
 		return errors.Wrapf(err, "failed to take snapshot of the restored volume")

pkg/spdk/server.go

@@ -451,7 +451,7 @@ func (s *Server) ReplicaSnapshotCreate(ctx context.Context, req *spdkrpc.Snapsho
 		return nil, grpcstatus.Errorf(grpccodes.NotFound, "cannot find replica %s during snapshot create", req.Name)
 	}
 
-	return r.SnapshotCreate(spdkClient, req.SnapshotName)
+	return r.SnapshotCreate(spdkClient, req.SnapshotName, &api.SnapshotOptions{UserCreated: req.UserCreated})
 }
 
 func (s *Server) ReplicaSnapshotDelete(ctx context.Context, req *spdkrpc.SnapshotRequest) (ret *emptypb.Empty, err error) {
@@ -659,7 +659,7 @@ func (s *Server) ReplicaRebuildingDstSnapshotCreate(ctx context.Context, req *sp
 		return nil, grpcstatus.Errorf(grpccodes.NotFound, "cannot find replica %s during rebuilding dst snapshot create", req.Name)
 	}
 
-	if err = r.RebuildingDstSnapshotCreate(spdkClient, req.SnapshotName); err != nil {
+	if err = r.RebuildingDstSnapshotCreate(spdkClient, req.SnapshotName, &api.SnapshotOptions{UserCreated: req.UserCreated}); err != nil {
 		return nil, err
 	}
 	return &emptypb.Empty{}, nil
@@ -893,7 +893,11 @@ func (s *Server) EngineSnapshotCreate(ctx context.Context, req *spdkrpc.Snapshot
 		return nil, grpcstatus.Errorf(grpccodes.NotFound, "cannot find engine %v for snapshot creation", req.Name)
 	}
 
-	snapshotName, err := e.SnapshotCreate(spdkClient, req.SnapshotName)
+	opts := api.SnapshotOptions{
+		UserCreated: req.UserCreated,
+	}
+
+	snapshotName, err := e.SnapshotCreate(spdkClient, req.SnapshotName, &opts)
 	return &spdkrpc.SnapshotResponse{SnapshotName: snapshotName}, err
 }
 

pkg/spdk/types.go

@@ -93,7 +93,7 @@ func GetBdevMap(cli *spdkclient.Client) (map[string]*spdktypes.BdevInfo, error)
 }
 
 func GetBdevLvolMap(cli *spdkclient.Client) (map[string]*spdktypes.BdevInfo, error) {
-	bdevList, err := cli.BdevGetBdevs("", 0)
+	bdevList, err := cli.BdevLvolGet("", 0)
 	if err != nil {
 		return nil, err
 	}