Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

User created flag #115

Merged
merged 2 commits into from
Feb 26, 2024
Merged

User created flag #115

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
0d3593f
proto: add user created flag to snapshot request
DamiaSan Jan 24, 2024
c2ba0e8
spdk lvol: snapshot management of UserCreated flag
DamiaSan Feb 5, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion Dockerfile.dapper
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,7 @@ RUN curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/i
ENV HUGEMEM=1024
RUN echo "vm.nr_hugepages=$((HUGEMEM/2))" >> /etc/sysctl.conf
ENV SPDK_DIR /usr/src/spdk
ENV SPDK_COMMIT_ID 72b7762674cc35262086b185a455a8a30f46432d
ENV SPDK_COMMIT_ID 7a25c3c5a102dfb2d8715f9bf090691176de5ef7
Copy link
Member

@innobead innobead Feb 26, 2024 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@DamiaSan remember to update the latest revision later, 6925ca6543e4ad5a5e3a5db09217ae16bf939831.

RUN git clone https://github.com/longhorn/spdk.git ${SPDK_DIR} --recursive && \
cd ${SPDK_DIR} && \
git checkout ${SPDK_COMMIT_ID} && \
Expand Down
7 changes: 7 additions & 0 deletions pkg/api/types.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,10 @@ import (
"github.com/longhorn/longhorn-spdk-engine/proto/spdkrpc"
)

type SnapshotOptions struct {
UserCreated bool
}

type Replica struct {
Name string `json:"name"`
LvsName string `json:"lvs_name"`
Expand All @@ -31,6 +35,7 @@ type Lvol struct {
Parent string `json:"parent"`
Children map[string]bool `json:"children"`
CreationTime string `json:"creation_time"`
UserCreated bool `json:"user_created"`
}

func ProtoLvolToLvol(l *spdkrpc.Lvol) *Lvol {
Expand All @@ -45,6 +50,7 @@ func ProtoLvolToLvol(l *spdkrpc.Lvol) *Lvol {
Parent: l.Parent,
Children: l.Children,
CreationTime: l.CreationTime,
UserCreated: l.UserCreated,
}
}

Expand All @@ -60,6 +66,7 @@ func LvolToProtoLvol(l *Lvol) *spdkrpc.Lvol {
Parent: l.Parent,
Children: l.Children,
CreationTime: l.CreationTime,
UserCreated: l.UserCreated,
}
}

Expand Down
9 changes: 6 additions & 3 deletions pkg/client/client.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -138,7 +138,7 @@ func (c *SPDKClient) ReplicaWatch(ctx context.Context) (*api.ReplicaStream, erro
return api.NewReplicaStream(stream), nil
}

func (c *SPDKClient) ReplicaSnapshotCreate(name, snapshotName string) error {
func (c *SPDKClient) ReplicaSnapshotCreate(name, snapshotName string, opts *api.SnapshotOptions) error {
if name == "" || snapshotName == "" {
return fmt.Errorf("failed to create SPDK replica snapshot: missing required parameter name or snapshot name")
}
Expand All @@ -150,6 +150,7 @@ func (c *SPDKClient) ReplicaSnapshotCreate(name, snapshotName string) error {
_, err := client.ReplicaSnapshotCreate(ctx, &spdkrpc.SnapshotRequest{
Name: name,
SnapshotName: snapshotName,
UserCreated: opts != nil && opts.UserCreated,
})
return errors.Wrapf(err, "failed to create SPDK replica %s snapshot %s", name, snapshotName)
}
Expand Down Expand Up @@ -318,7 +319,7 @@ func (c *SPDKClient) ReplicaRebuildingDstFinish(replicaName string, unexposeRequ
return errors.Wrapf(err, "failed to finish replica rebuilding dst %s", replicaName)
}

func (c *SPDKClient) ReplicaRebuildingDstSnapshotCreate(name, snapshotName string) error {
func (c *SPDKClient) ReplicaRebuildingDstSnapshotCreate(name, snapshotName string, opts *api.SnapshotOptions) error {
if name == "" || snapshotName == "" {
return fmt.Errorf("failed to create dst SPDK replica rebuilding snapshot: missing required parameter name or snapshot name")
}
Expand All @@ -330,6 +331,7 @@ func (c *SPDKClient) ReplicaRebuildingDstSnapshotCreate(name, snapshotName strin
_, err := client.ReplicaRebuildingDstSnapshotCreate(ctx, &spdkrpc.SnapshotRequest{
Name: name,
SnapshotName: snapshotName,
UserCreated: opts != nil && opts.UserCreated,
})
return errors.Wrapf(err, "failed to create dst SPDK replica %s rebuilding snapshot %s", name, snapshotName)
}
Expand Down Expand Up @@ -434,7 +436,7 @@ func (c *SPDKClient) EngineWatch(ctx context.Context) (*api.EngineStream, error)
return api.NewEngineStream(stream), nil
}

func (c *SPDKClient) EngineSnapshotCreate(name, snapshotName string) (string, error) {
func (c *SPDKClient) EngineSnapshotCreate(name, snapshotName string, opts *api.SnapshotOptions) (string, error) {
if name == "" {
return "", fmt.Errorf("failed to create SPDK engine snapshot: missing required parameter name")
}
Expand All @@ -446,6 +448,7 @@ func (c *SPDKClient) EngineSnapshotCreate(name, snapshotName string) (string, er
resp, err := client.EngineSnapshotCreate(ctx, &spdkrpc.SnapshotRequest{
Name: name,
SnapshotName: snapshotName,
UserCreated: opts != nil && opts.UserCreated,
})
if err != nil {
return "", errors.Wrapf(err, "failed to create SPDK engine %s snapshot %s", name, snapshotName)
Expand Down
24 changes: 12 additions & 12 deletions pkg/spdk/engine.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -788,7 +788,7 @@ func (e *Engine) ReplicaAddStart(spdkClient *spdkclient.Client, replicaName, rep

// TODO: For online rebuilding, the IO should be paused first
snapshotName := GenerateRebuildingSnapshotName()
updateRequired, err = e.snapshotOperationWithoutLock(spdkClient, replicaClients, snapshotName, SnapshotOperationCreate)
updateRequired, err = e.snapshotOperationWithoutLock(spdkClient, replicaClients, snapshotName, SnapshotOperationCreate, nil)
if err != nil {
return err
}
Expand Down Expand Up @@ -979,7 +979,7 @@ func (e *Engine) ReplicaShallowCopy(dstReplicaName, dstReplicaAddress string) (e
if err = srcReplicaServiceCli.ReplicaSnapshotShallowCopy(srcReplicaName, currentSnapshotName); err != nil {
return err
}
if err = dstReplicaServiceCli.ReplicaRebuildingDstSnapshotCreate(dstReplicaName, currentSnapshotName); err != nil {
if err = dstReplicaServiceCli.ReplicaRebuildingDstSnapshotCreate(dstReplicaName, currentSnapshotName, &api.SnapshotOptions{UserCreated: rpcSrcReplica.Snapshots[currentSnapshotName].UserCreated}); err != nil {
return err
}
prevSnapshotName = currentSnapshotName
Expand Down Expand Up @@ -1067,21 +1067,21 @@ const (
SnapshotOperationRevert = SnapshotOperationType("snapshot-revert")
)

func (e *Engine) SnapshotCreate(spdkClient *spdkclient.Client, inputSnapshotName string) (snapshotName string, err error) {
return e.snapshotOperation(spdkClient, inputSnapshotName, SnapshotOperationCreate)
func (e *Engine) SnapshotCreate(spdkClient *spdkclient.Client, inputSnapshotName string, opts *api.SnapshotOptions) (snapshotName string, err error) {
return e.snapshotOperation(spdkClient, inputSnapshotName, SnapshotOperationCreate, opts)
}

func (e *Engine) SnapshotDelete(spdkClient *spdkclient.Client, snapshotName string) (err error) {
_, err = e.snapshotOperation(spdkClient, snapshotName, SnapshotOperationDelete)
_, err = e.snapshotOperation(spdkClient, snapshotName, SnapshotOperationDelete, nil)
return err
}

func (e *Engine) SnapshotRevert(spdkClient *spdkclient.Client, snapshotName string) (err error) {
_, err = e.snapshotOperation(spdkClient, snapshotName, SnapshotOperationRevert)
_, err = e.snapshotOperation(spdkClient, snapshotName, SnapshotOperationRevert, nil)
return err
}

func (e *Engine) snapshotOperation(spdkClient *spdkclient.Client, inputSnapshotName string, snapshotOp SnapshotOperationType) (snapshotName string, err error) {
func (e *Engine) snapshotOperation(spdkClient *spdkclient.Client, inputSnapshotName string, snapshotOp SnapshotOperationType, opts *api.SnapshotOptions) (snapshotName string, err error) {
updateRequired := false

if snapshotOp == SnapshotOperationCreate {
Expand Down Expand Up @@ -1143,7 +1143,7 @@ func (e *Engine) snapshotOperation(spdkClient *spdkclient.Client, inputSnapshotN
}
}()

if updateRequired, err = e.snapshotOperationWithoutLock(spdkClient, replicaClients, snapshotName, snapshotOp); err != nil {
if updateRequired, err = e.snapshotOperationWithoutLock(spdkClient, replicaClients, snapshotName, snapshotOp, opts); err != nil {
return "", err
}

Expand Down Expand Up @@ -1219,7 +1219,7 @@ func (e *Engine) snapshotOperationPreCheckWithoutLock(replicaClients map[string]
return snapshotName, nil
}

func (e *Engine) snapshotOperationWithoutLock(spdkClient *spdkclient.Client, replicaClients map[string]*client.SPDKClient, snapshotName string, snapshotOp SnapshotOperationType) (updated bool, err error) {
func (e *Engine) snapshotOperationWithoutLock(spdkClient *spdkclient.Client, replicaClients map[string]*client.SPDKClient, snapshotName string, snapshotOp SnapshotOperationType, opts *api.SnapshotOptions) (updated bool, err error) {
if snapshotOp == SnapshotOperationRevert {
if _, err := spdkClient.BdevRaidDelete(e.Name); err != nil && !jsonrpc.IsJSONRPCRespErrorNoSuchDevice(err) {
e.log.WithError(err).Errorf("Failed to delete RAID after snapshot %s revert", snapshotName)
Expand All @@ -1228,7 +1228,7 @@ func (e *Engine) snapshotOperationWithoutLock(spdkClient *spdkclient.Client, rep
}

for replicaName := range replicaClients {
if err := e.replicaSnapshotOperation(spdkClient, replicaClients[replicaName], replicaName, snapshotName, snapshotOp); err != nil && e.ReplicaModeMap[replicaName] != types.ModeERR {
if err := e.replicaSnapshotOperation(spdkClient, replicaClients[replicaName], replicaName, snapshotName, snapshotOp, opts); err != nil && e.ReplicaModeMap[replicaName] != types.ModeERR {
e.ReplicaModeMap[replicaName] = types.ModeERR
e.ReplicaBdevNameMap[replicaName] = ""
e.log.WithError(err).Errorf("Failed to issue operation %s for replica %s snapshot %s, will mark the replica as mode ERR", snapshotOp, replicaName, snapshotName)
Expand Down Expand Up @@ -1256,11 +1256,11 @@ func (e *Engine) snapshotOperationWithoutLock(spdkClient *spdkclient.Client, rep
return updated, nil
}

func (e *Engine) replicaSnapshotOperation(spdkClient *spdkclient.Client, replicaClient *client.SPDKClient, replicaName, snapshotName string, snapshotOp SnapshotOperationType) error {
func (e *Engine) replicaSnapshotOperation(spdkClient *spdkclient.Client, replicaClient *client.SPDKClient, replicaName, snapshotName string, snapshotOp SnapshotOperationType, opts *api.SnapshotOptions) error {
switch snapshotOp {
case SnapshotOperationCreate:
// TODO: execute `sync` for the nvme initiator before snapshot start
return replicaClient.ReplicaSnapshotCreate(replicaName, snapshotName)
return replicaClient.ReplicaSnapshotCreate(replicaName, snapshotName, opts)
case SnapshotOperationDelete:
return replicaClient.ReplicaSnapshotDelete(replicaName, snapshotName)
case SnapshotOperationRevert:
Expand Down
32 changes: 27 additions & 5 deletions pkg/spdk/replica.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@ import (
helpertypes "github.com/longhorn/go-spdk-helper/pkg/types"
helperutil "github.com/longhorn/go-spdk-helper/pkg/util"

"github.com/longhorn/longhorn-spdk-engine/pkg/api"
"github.com/longhorn/longhorn-spdk-engine/pkg/types"
"github.com/longhorn/longhorn-spdk-engine/pkg/util"
"github.com/longhorn/longhorn-spdk-engine/proto/spdkrpc"
Expand Down Expand Up @@ -92,6 +93,7 @@ type Lvol struct {
// Children is map[<snapshot lvol name>] rather than map[<snapshot name>]. <snapshot lvol name> consists of `<replica name>-snap-<snapshot name>`
Children map[string]*Lvol
CreationTime string
UserCreated bool
}

func ServiceReplicaToProtoReplica(r *Replica) *spdkrpc.Replica {
Expand Down Expand Up @@ -125,6 +127,7 @@ func ServiceLvolToProtoLvol(replicaName string, lvol *Lvol) *spdkrpc.Lvol {
Parent: GetSnapshotNameFromReplicaSnapshotLvolName(replicaName, lvol.Parent),
Children: map[string]bool{},
CreationTime: lvol.CreationTime,
UserCreated: lvol.UserCreated,
}

if lvol.Name == replicaName {
Expand Down Expand Up @@ -156,6 +159,7 @@ func BdevLvolInfoToServiceLvol(bdev *spdktypes.BdevInfo) *Lvol {
// Need to update this separately
Children: map[string]*Lvol{},
CreationTime: bdev.CreationTime,
UserCreated: bdev.DriverSpecific.Lvol.Xattrs[spdkclient.UserCreated] == "true",
}
}

Expand Down Expand Up @@ -782,7 +786,7 @@ func (r *Replica) Get() (pReplica *spdkrpc.Replica) {
return ServiceReplicaToProtoReplica(r)
}

func (r *Replica) SnapshotCreate(spdkClient *spdkclient.Client, snapshotName string) (pReplica *spdkrpc.Replica, err error) {
func (r *Replica) SnapshotCreate(spdkClient *spdkclient.Client, snapshotName string, opts *api.SnapshotOptions) (pReplica *spdkrpc.Replica, err error) {
updateRequired := false

r.Lock()
Expand Down Expand Up @@ -822,7 +826,16 @@ func (r *Replica) SnapshotCreate(spdkClient *spdkclient.Client, snapshotName str
}
headSvcLvol := r.ActiveChain[r.ChainLength-1]

snapUUID, err := spdkClient.BdevLvolSnapshot(headSvcLvol.UUID, snapLvolName, nil)
var xattrs []spdkclient.Xattr
if opts != nil {
xattr := spdkclient.Xattr{
Name: spdkclient.UserCreated,
Value: strconv.FormatBool(opts.UserCreated),
}
xattrs = append(xattrs, xattr)
}

snapUUID, err := spdkClient.BdevLvolSnapshot(headSvcLvol.UUID, snapLvolName, xattrs)
if err != nil {
return nil, err
}
Expand Down Expand Up @@ -1390,7 +1403,7 @@ func (r *Replica) RebuildingDstFinish(spdkClient *spdkclient.Client, unexposeReq
// return nil
// }

func (r *Replica) RebuildingDstSnapshotCreate(spdkClient *spdkclient.Client, snapshotName string) (err error) {
func (r *Replica) RebuildingDstSnapshotCreate(spdkClient *spdkclient.Client, snapshotName string, opts *api.SnapshotOptions) (err error) {
updateRequired := false

r.Lock()
Expand Down Expand Up @@ -1426,8 +1439,17 @@ func (r *Replica) RebuildingDstSnapshotCreate(spdkClient *spdkclient.Client, sna
}
}()

var xattrs []spdkclient.Xattr
if opts != nil {
xattr := spdkclient.Xattr{
Name: spdkclient.UserCreated,
Value: strconv.FormatBool(opts.UserCreated),
}
xattrs = append(xattrs, xattr)
}

snapLvolName := GetReplicaSnapshotLvolName(r.Name, snapshotName)
snapUUID, err := spdkClient.BdevLvolSnapshot(r.rebuildingLvol.UUID, snapLvolName, nil)
snapUUID, err := spdkClient.BdevLvolSnapshot(r.rebuildingLvol.UUID, snapLvolName, xattrs)
if err != nil {
return err
}
Expand Down Expand Up @@ -1709,7 +1731,7 @@ func (r *Replica) postFullRestoreOperations(spdkClient *spdkclient.Client, resto

r.log.Infof("Taking snapshot %v of the restored volume", restore.SnapshotName)

_, err := r.SnapshotCreate(spdkClient, restore.SnapshotName)
_, err := r.SnapshotCreate(spdkClient, restore.SnapshotName, nil)
if err != nil {
r.log.WithError(err).Error("Failed to take snapshot of the restored volume")
return errors.Wrapf(err, "failed to take snapshot of the restored volume")
Expand Down
10 changes: 7 additions & 3 deletions pkg/spdk/server.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -451,7 +451,7 @@ func (s *Server) ReplicaSnapshotCreate(ctx context.Context, req *spdkrpc.Snapsho
return nil, grpcstatus.Errorf(grpccodes.NotFound, "cannot find replica %s during snapshot create", req.Name)
}

return r.SnapshotCreate(spdkClient, req.SnapshotName)
return r.SnapshotCreate(spdkClient, req.SnapshotName, &api.SnapshotOptions{UserCreated: req.UserCreated})
}

func (s *Server) ReplicaSnapshotDelete(ctx context.Context, req *spdkrpc.SnapshotRequest) (ret *emptypb.Empty, err error) {
Expand Down Expand Up @@ -659,7 +659,7 @@ func (s *Server) ReplicaRebuildingDstSnapshotCreate(ctx context.Context, req *sp
return nil, grpcstatus.Errorf(grpccodes.NotFound, "cannot find replica %s during rebuilding dst snapshot create", req.Name)
}

if err = r.RebuildingDstSnapshotCreate(spdkClient, req.SnapshotName); err != nil {
if err = r.RebuildingDstSnapshotCreate(spdkClient, req.SnapshotName, &api.SnapshotOptions{UserCreated: req.UserCreated}); err != nil {
return nil, err
}
return &emptypb.Empty{}, nil
Expand Down Expand Up @@ -893,7 +893,11 @@ func (s *Server) EngineSnapshotCreate(ctx context.Context, req *spdkrpc.Snapshot
return nil, grpcstatus.Errorf(grpccodes.NotFound, "cannot find engine %v for snapshot creation", req.Name)
}

snapshotName, err := e.SnapshotCreate(spdkClient, req.SnapshotName)
opts := api.SnapshotOptions{
UserCreated: req.UserCreated,
}

snapshotName, err := e.SnapshotCreate(spdkClient, req.SnapshotName, &opts)
return &spdkrpc.SnapshotResponse{SnapshotName: snapshotName}, err
}

Expand Down
2 changes: 1 addition & 1 deletion pkg/spdk/types.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -93,7 +93,7 @@ func GetBdevMap(cli *spdkclient.Client) (map[string]*spdktypes.BdevInfo, error)
}

func GetBdevLvolMap(cli *spdkclient.Client) (map[string]*spdktypes.BdevInfo, error) {
bdevList, err := cli.BdevGetBdevs("", 0)
bdevList, err := cli.BdevLvolGet("", 0)
if err != nil {
return nil, err
}
Expand Down
Loading