From 2b151eca7a23c01c3700b717c6d39bad2f39b191 Mon Sep 17 00:00:00 2001 From: Derek Su Date: Mon, 5 Feb 2024 05:46:46 +0000 Subject: [PATCH] disk: support different drivers Longhorn 7672 Signed-off-by: Derek Su --- pkg/api/types.go | 1 + pkg/client/client.go | 24 +-- pkg/spdk/disk.go | 194 +++++++++++++---------- pkg/spdk/disk/aio/aio.go | 78 +++++++++ pkg/spdk/disk/driver.go | 86 ++++++++++ pkg/spdk/disk/nvme/nvme.go | 80 ++++++++++ pkg/spdk/disk/types.go | 96 +++++++++++ pkg/spdk/disk/virtio-blk/virtio-blk.go | 57 +++++++ pkg/spdk/disk/virtio-scsi/virtio-scsi.go | 57 +++++++ pkg/spdk/engine.go | 2 +- pkg/spdk/replica.go | 4 +- pkg/spdk/server.go | 6 +- pkg/spdk_test.go | 11 +- pkg/util/block.go | 72 +++++++++ 14 files changed, 663 insertions(+), 105 deletions(-) create mode 100644 pkg/spdk/disk/aio/aio.go create mode 100644 pkg/spdk/disk/driver.go create mode 100644 pkg/spdk/disk/nvme/nvme.go create mode 100644 pkg/spdk/disk/types.go create mode 100644 pkg/spdk/disk/virtio-blk/virtio-blk.go create mode 100644 pkg/spdk/disk/virtio-scsi/virtio-scsi.go create mode 100644 pkg/util/block.go diff --git a/pkg/api/types.go b/pkg/api/types.go index 93e096ab2..ff467cc49 100644 --- a/pkg/api/types.go +++ b/pkg/api/types.go @@ -155,6 +155,7 @@ func ProtoEngineToEngine(e *spdkrpc.Engine) *Engine { type DiskInfo struct { ID string + Name string UUID string Path string Type string diff --git a/pkg/client/client.go b/pkg/client/client.go index 9005e1fa8..3c3c52ffc 100644 --- a/pkg/client/client.go +++ b/pkg/client/client.go @@ -682,7 +682,7 @@ func (c *SPDKClient) ReplicaRestoreStatus(replicaName string) (*spdkrpc.ReplicaR // DiskCreate creates a disk with the given name and path. // diskUUID is optional, if not provided, it indicates the disk is newly added. -func (c *SPDKClient) DiskCreate(diskName, diskUUID, diskPath string, blockSize int64) (*spdkrpc.Disk, error) { +func (c *SPDKClient) DiskCreate(diskName, diskUUID, diskPath, diskDriver string, blockSize int64) (*spdkrpc.Disk, error) { if diskName == "" || diskPath == "" { return nil, fmt.Errorf("failed to create disk: missing required parameters") } @@ -692,14 +692,15 @@ func (c *SPDKClient) DiskCreate(diskName, diskUUID, diskPath string, blockSize i defer cancel() return client.DiskCreate(ctx, &spdkrpc.DiskCreateRequest{ - DiskName: diskName, - DiskUuid: diskUUID, - DiskPath: diskPath, - BlockSize: blockSize, + DiskName: diskName, + DiskUuid: diskUUID, + DiskPath: diskPath, + BlockSize: blockSize, + DiskDriver: diskDriver, }) } -func (c *SPDKClient) DiskGet(diskName string) (*spdkrpc.Disk, error) { +func (c *SPDKClient) DiskGet(diskName, diskPath, diskDriver string) (*spdkrpc.Disk, error) { if diskName == "" { return nil, fmt.Errorf("failed to get disk info: missing required parameter") } @@ -709,11 +710,13 @@ func (c *SPDKClient) DiskGet(diskName string) (*spdkrpc.Disk, error) { defer cancel() return client.DiskGet(ctx, &spdkrpc.DiskGetRequest{ - DiskName: diskName, + DiskName: diskName, + DiskPath: diskPath, + DiskDriver: diskDriver, }) } -func (c *SPDKClient) DiskDelete(diskName, diskUUID string) error { +func (c *SPDKClient) DiskDelete(diskName, diskUUID, diskDriver string) error { if diskName == "" || diskUUID == "" { return fmt.Errorf("failed to delete disk: missing required parameters") } @@ -723,8 +726,9 @@ func (c *SPDKClient) DiskDelete(diskName, diskUUID string) error { defer cancel() _, err := client.DiskDelete(ctx, &spdkrpc.DiskDeleteRequest{ - DiskName: diskName, - DiskUuid: diskUUID, + DiskName: diskName, + DiskUuid: diskUUID, + DiskDriver: diskDriver, }) return err } diff --git a/pkg/spdk/disk.go b/pkg/spdk/disk.go index e0d2b236b..a8bcef05f 100644 --- a/pkg/spdk/disk.go +++ b/pkg/spdk/disk.go @@ -2,8 +2,6 @@ package spdk import ( "fmt" - "io" - "os" "path/filepath" "github.com/pkg/errors" @@ -20,6 +18,12 @@ import ( "github.com/longhorn/longhorn-spdk-engine/pkg/util" "github.com/longhorn/longhorn-spdk-engine/proto/spdkrpc" + + "github.com/longhorn/longhorn-spdk-engine/pkg/spdk/disk" + _ "github.com/longhorn/longhorn-spdk-engine/pkg/spdk/disk/aio" + _ "github.com/longhorn/longhorn-spdk-engine/pkg/spdk/disk/nvme" + _ "github.com/longhorn/longhorn-spdk-engine/pkg/spdk/disk/virtio-blk" + _ "github.com/longhorn/longhorn-spdk-engine/pkg/spdk/disk/virtio-scsi" ) const ( @@ -29,12 +33,13 @@ const ( hostPrefix = "/host" ) -func svcDiskCreate(spdkClient *spdkclient.Client, diskName, diskUUID, diskPath string, blockSize int64) (ret *spdkrpc.Disk, err error) { +func svcDiskCreate(spdkClient *spdkclient.Client, diskName, diskUUID, diskPath, diskDriver string, blockSize int64) (ret *spdkrpc.Disk, err error) { log := logrus.WithFields(logrus.Fields{ - "diskName": diskName, - "diskUUID": diskUUID, - "diskPath": diskPath, - "blockSize": blockSize, + "diskName": diskName, + "diskUUID": diskUUID, + "diskPath": diskPath, + "blockSize": blockSize, + "diskDriver": diskDriver, }) log.Info("Creating disk") @@ -50,27 +55,24 @@ func svcDiskCreate(spdkClient *spdkclient.Client, diskName, diskUUID, diskPath s return nil, grpcstatus.Error(grpccodes.InvalidArgument, "disk name and disk path are required") } - if err := validateDiskCreation(spdkClient, diskPath); err != nil { - return nil, grpcstatus.Error(grpccodes.InvalidArgument, errors.Wrap(err, "failed to validate disk create request").Error()) - } - - if blockSize <= 0 { - blockSize = int64(defaultBlockSize) - log.Infof("Using default block size %v", blockSize) + exactDiskDriver, err := disk.GetDiskDriver(diskDriver, diskPath) + if err != nil { + return nil, errors.Wrapf(err, "failed to get disk driver for disk %s", diskName) } - uuid, err := addBlockDevice(spdkClient, diskName, diskUUID, diskPath, blockSize) + lvstoreUUID, err := addBlockDevice(spdkClient, diskName, diskUUID, diskPath, exactDiskDriver, blockSize) if err != nil { - return nil, grpcstatus.Error(grpccodes.Internal, errors.Wrap(err, "failed to add block device").Error()) + return nil, grpcstatus.Error(grpccodes.Internal, errors.Wrap(err, "failed to add disk block device").Error()) } - return lvstoreToDisk(spdkClient, diskPath, "", uuid) + return lvstoreToDisk(spdkClient, diskPath, "", lvstoreUUID, exactDiskDriver) } -func svcDiskDelete(spdkClient *spdkclient.Client, diskName, diskUUID string) (ret *emptypb.Empty, err error) { +func svcDiskDelete(spdkClient *spdkclient.Client, diskName, diskUUID, diskDriver string) (ret *emptypb.Empty, err error) { log := logrus.WithFields(logrus.Fields{ - "diskName": diskName, - "diskUUID": diskUUID, + "diskName": diskName, + "diskUUID": diskUUID, + "diskDriver": diskDriver, }) log.Info("Deleting disk") @@ -103,15 +105,22 @@ func svcDiskDelete(spdkClient *spdkclient.Client, diskName, diskUUID string) (re aioBdevName = lvstore.BaseBdev } - if _, err := spdkClient.BdevAioDelete(aioBdevName); err != nil { - return nil, errors.Wrapf(err, "failed to delete AIO bdev %v", aioBdevName) + if _, err := disk.DiskDelete(spdkClient, aioBdevName, diskDriver); err != nil { + return nil, errors.Wrapf(err, "failed to delete disk %v", diskName) } + return &emptypb.Empty{}, nil } -func svcDiskGet(spdkClient *spdkclient.Client, diskName string) (ret *spdkrpc.Disk, err error) { +type DeviceInfo struct { + DeviceDriver string `json:"device_driver"` +} + +func svcDiskGet(spdkClient *spdkclient.Client, diskName, diskPath, diskDriver string) (ret *spdkrpc.Disk, err error) { log := logrus.WithFields(logrus.Fields{ - "diskName": diskName, + "diskName": diskName, + "diskPath": diskPath, + "diskDriver": diskDriver, }) log.Trace("Getting disk info") @@ -128,30 +137,47 @@ func svcDiskGet(spdkClient *spdkclient.Client, diskName string) (ret *spdkrpc.Di } // Check if the disk exists - bdevs, err := spdkClient.BdevAioGet(diskName, 0) + bdevs, err := disk.DiskGet(spdkClient, diskName, diskPath, diskDriver, 0) if err != nil { if !jsonrpc.IsJSONRPCRespErrorNoSuchDevice(err) { - return nil, grpcstatus.Errorf(grpccodes.Internal, errors.Wrapf(err, "failed to get AIO bdev with name %v", diskName).Error()) + return nil, grpcstatus.Errorf(grpccodes.Internal, errors.Wrapf(err, "failed to get bdev with name %v", diskName).Error()) } } if len(bdevs) == 0 { - return nil, grpcstatus.Errorf(grpccodes.NotFound, "cannot find AIO bdev with name %v", diskName) + return nil, grpcstatus.Errorf(grpccodes.NotFound, "cannot find disk bdev with name %v", diskName) } var targetBdev *spdktypes.BdevInfo + var exactDiskDriver disk.BlockDiskDriver + for i, bdev := range bdevs { - if bdev.DriverSpecific != nil { + switch bdev.ProductName { + case spdktypes.BdevProductNameAio: + if bdev.DriverSpecific != nil { + targetBdev = &bdevs[i] + diskPath = util.RemovePrefix(bdev.DriverSpecific.Aio.FileName, hostPrefix) + } + exactDiskDriver = disk.BlockDiskDriverAio + case spdktypes.BdevProductNameVirtioBlk: + exactDiskDriver = disk.BlockDiskDriverVirtioBlk + targetBdev = &bdevs[i] + case spdktypes.BdevProductNameVirtioScsi: + exactDiskDriver = disk.BlockDiskDriverVirtioScsi targetBdev = &bdevs[i] + case spdktypes.BdevProductNameNvme: + exactDiskDriver = disk.BlockDiskDriverNvme + targetBdev = &bdevs[i] + } + if targetBdev != nil { break } } + if targetBdev == nil { - return nil, grpcstatus.Errorf(grpccodes.NotFound, errors.Wrapf(err, "failed to get AIO bdev for disk %v", diskName).Error()) + return nil, grpcstatus.Errorf(grpccodes.NotFound, errors.Wrapf(err, "failed to get disk bdev for disk %v", diskName).Error()) } - diskPath := util.RemovePrefix(targetBdev.DriverSpecific.Aio.FileName, hostPrefix) - - return lvstoreToDisk(spdkClient, diskPath, diskName, "") + return lvstoreToDisk(spdkClient, diskPath, targetBdev.Name, "", exactDiskDriver) } func getDiskPath(path string) string { @@ -172,45 +198,15 @@ func getDiskID(filename string) (string, error) { return fmt.Sprintf("%d-%d", dev.Export.Major, dev.Export.Minor), nil } -func getDiskDeviceSize(path string) (int64, error) { - file, err := os.Open(path) - if err != nil { - return 0, errors.Wrapf(err, "failed to open %s", path) - } - defer file.Close() - - pos, err := file.Seek(0, io.SeekEnd) - if err != nil { - return 0, errors.Wrapf(err, "failed to seek %s", path) - } - return pos, nil -} - -func validateDiskCreation(spdkClient *spdkclient.Client, diskPath string) error { - ok, err := spdkutil.IsBlockDevice(diskPath) - if err != nil { - return errors.Wrap(err, "failed to check if disk is a block device") - } - if !ok { - return errors.Wrapf(err, "disk %v is not a block device", diskPath) - } - - size, err := getDiskDeviceSize(diskPath) - if err != nil { - return errors.Wrap(err, "failed to get disk size") - } - if size == 0 { - return fmt.Errorf("disk %v size is 0", diskPath) - } - +func validateAioDiskCreation(spdkClient *spdkclient.Client, diskPath string, diskDriver disk.BlockDiskDriver) error { diskID, err := getDiskID(getDiskPath(diskPath)) if err != nil { return errors.Wrap(err, "failed to get disk device number") } - bdevs, err := spdkClient.BdevAioGet("", 0) + bdevs, err := disk.DiskGet(spdkClient, "", "", string(diskDriver), 0) if err != nil { - return errors.Wrap(err, "failed to get AIO bdevs") + return errors.Wrap(err, "failed to get disk bdevs") } for _, bdev := range bdevs { @@ -220,33 +216,56 @@ func validateDiskCreation(spdkClient *spdkclient.Client, diskPath string) error } if id == diskID { - return fmt.Errorf("disk %v is already used by AIO bdev %v", diskPath, bdev.Name) + return fmt.Errorf("disk %v is already used by disk bdev %v", diskPath, bdev.Name) } } return nil } -func addBlockDevice(spdkClient *spdkclient.Client, diskName, diskUUID, diskPath string, blockSize int64) (string, error) { +func addBlockDevice(spdkClient *spdkclient.Client, diskName, diskUUID, originalDiskPath string, diskDriver disk.BlockDiskDriver, blockSize int64) (string, error) { log := logrus.WithFields(logrus.Fields{ - "diskName": diskName, - "diskUUID": diskUUID, - "diskPath": diskPath, - "blockSize": blockSize, + "diskName": diskName, + "diskUUID": diskUUID, + "diskPath": originalDiskPath, + "diskDriver": diskDriver, + "blockSize": blockSize, }) - log.Infof("Creating AIO bdev %v with block size %v", diskName, blockSize) - bdevName, err := spdkClient.BdevAioCreate(getDiskPath(diskPath), diskName, uint64(blockSize)) + diskPath := originalDiskPath + switch diskDriver { + case disk.BlockDiskDriverAio: + if err := validateAioDiskCreation(spdkClient, diskPath, diskDriver); err != nil { + return "", errors.Wrap(err, "failed to validate disk creation") + } + diskPath = getDiskPath(originalDiskPath) + } + + log.Info("Creating disk bdev") + + bdevName, err := disk.DiskCreate(spdkClient, diskName, diskPath, string(diskDriver), uint64(blockSize)) if err != nil { if !jsonrpc.IsJSONRPCRespErrorFileExists(err) { - return "", errors.Wrapf(err, "failed to create AIO bdev") + return "", errors.Wrapf(err, "failed to create disk bdev") } } - log.Infof("Creating lvstore %v", diskName) + bdevs, err := disk.DiskGet(spdkClient, bdevName, diskPath, "", 0) + if err != nil { + return "", errors.Wrapf(err, "failed to get disk bdev") + } + if len(bdevs) == 0 { + return "", fmt.Errorf("cannot find disk bdev with name %v", bdevName) + } + if len(bdevs) > 1 { + return "", fmt.Errorf("found multiple disk bdevs with name %v", bdevName) + } + bdev := bdevs[0] - // Name of the lvstore is the same as the name of the aio bdev - lvstoreName := bdevName + // Name of the lvstore is the same as the name of the disk bdev + lvstoreName := bdev.Name + + log.Infof("Creating lvstore %v", lvstoreName) lvstores, err := spdkClient.BdevLvolGetLvstore("", "") if err != nil { @@ -267,28 +286,28 @@ func addBlockDevice(spdkClient *spdkclient.Client, diskName, diskUUID, diskPath return lvstore.UUID, nil } - // Rename the existing lvstore to the name of the aio bdev if the UUID matches + // Rename the existing lvstore to the name of the disk bdev if the UUID matches log.Infof("Renaming the existing lvstore %v to %v", lvstore.Name, lvstoreName) renamed, err := spdkClient.BdevLvolRenameLvstore(lvstore.Name, lvstoreName) if err != nil { - return "", errors.Wrapf(err, "failed to rename lvstore from %v to %v", lvstore.Name, lvstoreName) + return "", errors.Wrapf(err, "failed to rename lvstore %v to %v", lvstore.Name, lvstoreName) } if !renamed { - return "", fmt.Errorf("failed to rename lvstore from %v to %v", lvstore.Name, lvstoreName) + return "", fmt.Errorf("failed to rename lvstore %v to %v", lvstore.Name, lvstoreName) } return lvstore.UUID, nil } if diskUUID == "" { log.Infof("Creating a new lvstore %v", lvstoreName) - return spdkClient.BdevLvolCreateLvstore(lvstoreName, diskName, defaultClusterSize) + return spdkClient.BdevLvolCreateLvstore(bdev.Name, lvstoreName, defaultClusterSize) } // The lvstore should be created before, but it cannot be found now. return "", grpcstatus.Error(grpccodes.NotFound, fmt.Sprintf("cannot find lvstore with UUID %v", diskUUID)) } -func lvstoreToDisk(spdkClient *spdkclient.Client, diskPath, lvstoreName, lvstoreUUID string) (*spdkrpc.Disk, error) { +func lvstoreToDisk(spdkClient *spdkclient.Client, diskPath, lvstoreName, lvstoreUUID string, diskDriver disk.BlockDiskDriver) (*spdkrpc.Disk, error) { lvstores, err := spdkClient.BdevLvolGetLvstore(lvstoreName, lvstoreUUID) if err != nil { return nil, errors.Wrapf(err, "failed to get lvstore with name %v and UUID %v", lvstoreName, lvstoreUUID) @@ -296,16 +315,21 @@ func lvstoreToDisk(spdkClient *spdkclient.Client, diskPath, lvstoreName, lvstore lvstore := &lvstores[0] // A disk does not have a fsid, so we use the device number as the disk ID - diskID, err := getDiskID(getDiskPath(diskPath)) - if err != nil { - return nil, errors.Wrapf(err, "failed to get disk ID") + diskID := diskPath + if diskDriver == disk.BlockDiskDriverAio { + diskID, err = getDiskID(getDiskPath(diskPath)) + if err != nil { + return nil, errors.Wrapf(err, "failed to get disk ID") + } } return &spdkrpc.Disk{ Id: diskID, + Name: lvstore.Name, Uuid: lvstore.UUID, Path: diskPath, Type: DiskTypeBlock, + Driver: string(diskDriver), TotalSize: int64(lvstore.TotalDataClusters * lvstore.ClusterSize), FreeSize: int64(lvstore.FreeClusters * lvstore.ClusterSize), TotalBlocks: int64(lvstore.TotalDataClusters * lvstore.ClusterSize / lvstore.BlockSize), diff --git a/pkg/spdk/disk/aio/aio.go b/pkg/spdk/disk/aio/aio.go new file mode 100644 index 000000000..fd16efd07 --- /dev/null +++ b/pkg/spdk/disk/aio/aio.go @@ -0,0 +1,78 @@ +package aio + +import ( + "fmt" + "io" + "os" + + "github.com/pkg/errors" + + spdkclient "github.com/longhorn/go-spdk-helper/pkg/spdk/client" + spdktypes "github.com/longhorn/go-spdk-helper/pkg/spdk/types" + spdkutil "github.com/longhorn/go-spdk-helper/pkg/util" + + "github.com/longhorn/longhorn-spdk-engine/pkg/spdk/disk" +) + +type DiskDriverAio struct { +} + +const ( + diskDriver = "aio" +) + +func init() { + driver := &DiskDriverAio{} + if err := disk.RegisterDiskDriver(diskDriver, driver); err != nil { + panic(err) + } +} + +func (d *DiskDriverAio) DiskCreate(spdkClient *spdkclient.Client, diskName, diskPath string, blockSize uint64) (string, error) { + if err := validateDiskCreation(spdkClient, diskPath); err != nil { + return "", errors.Wrap(err, "failed to validate disk creation") + } + + return spdkClient.BdevAioCreate(diskPath, diskName, blockSize) +} + +func (d *DiskDriverAio) DiskDelete(spdkClient *spdkclient.Client, diskName string) (deleted bool, err error) { + return spdkClient.BdevAioDelete(diskName) +} + +func (d *DiskDriverAio) DiskGet(spdkClient *spdkclient.Client, diskName, diskPath string, timeout uint64) ([]spdktypes.BdevInfo, error) { + return spdkClient.BdevAioGet(diskName, timeout) +} + +func validateDiskCreation(spdkClient *spdkclient.Client, diskPath string) error { + ok, err := spdkutil.IsBlockDevice(diskPath) + if err != nil { + return errors.Wrap(err, "failed to check if disk is a block device") + } + if !ok { + return errors.Wrapf(err, "disk %v is not a block device", diskPath) + } + + size, err := getDiskDeviceSize(diskPath) + if err != nil { + return errors.Wrap(err, "failed to get disk size") + } + if size == 0 { + return fmt.Errorf("disk %v size is 0", diskPath) + } + return nil +} + +func getDiskDeviceSize(path string) (int64, error) { + file, err := os.Open(path) + if err != nil { + return 0, errors.Wrapf(err, "failed to open %s", path) + } + defer file.Close() + + pos, err := file.Seek(0, io.SeekEnd) + if err != nil { + return 0, errors.Wrapf(err, "failed to seek %s", path) + } + return pos, nil +} diff --git a/pkg/spdk/disk/driver.go b/pkg/spdk/disk/driver.go new file mode 100644 index 000000000..cc22f9dfc --- /dev/null +++ b/pkg/spdk/disk/driver.go @@ -0,0 +1,86 @@ +package disk + +import ( + "fmt" + + "github.com/pkg/errors" + + spdkclient "github.com/longhorn/go-spdk-helper/pkg/spdk/client" + spdktypes "github.com/longhorn/go-spdk-helper/pkg/spdk/types" +) + +type DiskDriver interface { + DiskCreate(*spdkclient.Client, string, string, uint64) (string, error) + DiskDelete(*spdkclient.Client, string) (bool, error) + DiskGet(*spdkclient.Client, string, string, uint64) ([]spdktypes.BdevInfo, error) +} + +var ( + diskDrivers map[string]DiskDriver +) + +func init() { + diskDrivers = make(map[string]DiskDriver) +} + +func RegisterDiskDriver(diskDriver string, ops DiskDriver) error { + if _, exists := diskDrivers[diskDriver]; exists { + return fmt.Errorf("disk driver %s has already been registered", diskDriver) + } + diskDrivers[diskDriver] = ops + return nil +} + +func DiskCreate(spdkClient *spdkclient.Client, diskName, diskPath, diskDriver string, blockSize uint64) (string, error) { + driver, ok := diskDrivers[diskDriver] + if !ok { + return "", fmt.Errorf("disk driver %s is not registered", diskDriver) + } + + return driver.DiskCreate(spdkClient, diskName, diskPath, blockSize) +} + +func DiskDelete(spdkClient *spdkclient.Client, diskName, diskDriver string) (bool, error) { + driver, ok := diskDrivers[diskDriver] + if !ok { + return false, fmt.Errorf("disk driver %s is not registered", diskDriver) + } + + return driver.DiskDelete(spdkClient, diskName) +} + +func DiskGet(spdkClient *spdkclient.Client, diskName, diskPath, diskDriver string, timeout uint64) ([]spdktypes.BdevInfo, error) { + if diskDriver == "" { + if !isBDF(diskPath) { + return spdkClient.BdevGetBdevs(diskName, 0) + } + bdevs, err := spdkClient.BdevGetBdevs("", 0) + if err != nil { + return nil, errors.Wrapf(err, "failed to get bdevs") + } + foundBdevs := []spdktypes.BdevInfo{} + for _, bdev := range bdevs { + if bdev.DriverSpecific == nil { + continue + } + if bdev.DriverSpecific.Nvme == nil { + continue + } + nvmes := *bdev.DriverSpecific.Nvme + for _, nvme := range nvmes { + + if nvme.PciAddress == diskPath { + foundBdevs = append(foundBdevs, bdev) + } + } + } + return foundBdevs, nil + } + + driver, ok := diskDrivers[diskDriver] + if !ok { + return nil, fmt.Errorf("disk driver %s is not registered", diskDriver) + } + + return driver.DiskGet(spdkClient, diskName, diskPath, timeout) +} diff --git a/pkg/spdk/disk/nvme/nvme.go b/pkg/spdk/disk/nvme/nvme.go new file mode 100644 index 000000000..8c781f893 --- /dev/null +++ b/pkg/spdk/disk/nvme/nvme.go @@ -0,0 +1,80 @@ +package nvme + +import ( + "github.com/pkg/errors" + "github.com/sirupsen/logrus" + + commonTypes "github.com/longhorn/go-common-libs/types" + spdkclient "github.com/longhorn/go-spdk-helper/pkg/spdk/client" + spdksetup "github.com/longhorn/go-spdk-helper/pkg/spdk/setup" + spdktypes "github.com/longhorn/go-spdk-helper/pkg/spdk/types" + helpertypes "github.com/longhorn/go-spdk-helper/pkg/types" + helperutil "github.com/longhorn/go-spdk-helper/pkg/util" + + "github.com/longhorn/longhorn-spdk-engine/pkg/spdk/disk" +) + +type DiskDriverNvme struct { +} + +const ( + diskDriver = "nvme" +) + +func init() { + driver := &DiskDriverNvme{} + if err := disk.RegisterDiskDriver(diskDriver, driver); err != nil { + panic(err) + } +} + +func (d *DiskDriverNvme) DiskCreate(spdkClient *spdkclient.Client, diskName, diskPath string, blockSize uint64) (string, error) { + // TODO: validate the diskPath + executor, err := helperutil.NewExecutor(commonTypes.ProcDirectory) + if err != nil { + return "", errors.Wrapf(err, "failed to get the executor for disk create %v", diskPath) + } + + _, err = spdksetup.Bind(diskPath, "uio_pci_generic", executor) + if err != nil { + return "", errors.Wrapf(err, "failed to bind the disk %v with uio_pci_generic", diskPath) + } + + bdevs, err := spdkClient.BdevNvmeAttachController(diskName, "", diskPath, "", "PCIe", "", + helpertypes.DefaultCtrlrLossTimeoutSec, helpertypes.DefaultReconnectDelaySec, helpertypes.DefaultFastIOFailTimeoutSec) + if err != nil { + return "", errors.Wrapf(err, "failed to attach the disk %v with %v", diskPath, diskDriver) + } + if len(bdevs) == 0 { + return "", errors.Errorf("failed to attach the disk %v with %v", diskPath, diskDriver) + } + return bdevs[0], nil +} + +func (d *DiskDriverNvme) DiskDelete(spdkClient *spdkclient.Client, diskName string) (deleted bool, err error) { + return spdkClient.BdevVirtioDetachController(diskName) +} + +func (d *DiskDriverNvme) DiskGet(spdkClient *spdkclient.Client, diskName, diskPath string, timeout uint64) ([]spdktypes.BdevInfo, error) { + bdevs, err := spdkClient.BdevGetBdevs("", 0) + if err != nil { + return nil, errors.Wrapf(err, "failed to get bdevs") + } + foundBdevs := []spdktypes.BdevInfo{} + for _, bdev := range bdevs { + if bdev.DriverSpecific == nil { + continue + } + if bdev.DriverSpecific.Nvme == nil { + continue + } + nvmes := *bdev.DriverSpecific.Nvme + for _, nvme := range nvmes { + if nvme.PciAddress == diskPath { + logrus.Infof("Found bdev %v for disk %v", bdev, diskName) + foundBdevs = append(foundBdevs, bdev) + } + } + } + return foundBdevs, nil +} diff --git a/pkg/spdk/disk/types.go b/pkg/spdk/disk/types.go new file mode 100644 index 000000000..744c1f006 --- /dev/null +++ b/pkg/spdk/disk/types.go @@ -0,0 +1,96 @@ +package disk + +import ( + "fmt" + "regexp" + "slices" + + "github.com/pkg/errors" + + "github.com/longhorn/longhorn-spdk-engine/pkg/util" +) + +type BlockDiskDriver string + +const ( + BlockDiskDriverAuto = BlockDiskDriver("auto") + BlockDiskDriverAio = BlockDiskDriver("aio") + BlockDiskDriverNvme = BlockDiskDriver("nvme") + BlockDiskDriverVirtio = BlockDiskDriver("virtio") + BlockDiskDriverVirtioScsi = BlockDiskDriver("virtio-scsi") + BlockDiskDriverVirtioBlk = BlockDiskDriver("virtio-blk") +) + +type BlockDiskSubsystem string + +const ( + BlockDiskSubsystemVirtio = BlockDiskSubsystem("virtio") + BlockDiskSubsystemPci = BlockDiskSubsystem("pci") + BlockDiskSubsystemNvme = BlockDiskSubsystem("nvme") + BlockDiskSubsystemScsi = BlockDiskSubsystem("scsi") +) + +type BlockDiskType string + +const ( + BlockDiskTypeDisk = BlockDiskType("disk") + BlockDiskTypeLoop = BlockDiskType("loop") +) + +func GetDiskDriver(diskDriver, diskPathOrBdf string) (BlockDiskDriver, error) { + diskPath := diskPathOrBdf + if isBDF(diskPathOrBdf) { + devName, err := util.GetDevNameFromBDF(diskPathOrBdf) + if err != nil { + return "", errors.Wrapf(err, "failed to get device name from BDF %s", diskPathOrBdf) + } + diskPath = fmt.Sprintf("/dev/%s", devName) + } + + var exactDiskDriver BlockDiskDriver + diskType, err := util.GetBlockDiskType(diskPath) + if err != nil { + return "", errors.Wrapf(err, "failed to get disk type for %s", diskPath) + } + + switch BlockDiskType(diskType) { + case BlockDiskTypeDisk: + subsystems, err := util.GetBlockDiskSubsystems(diskPath) + if err != nil { + return "", errors.Wrapf(err, "failed to get disk subsystems for %s", diskPath) + } + if slices.Contains(subsystems, string(BlockDiskSubsystemVirtio)) && slices.Contains(subsystems, string(BlockDiskSubsystemPci)) { + if diskDriver != string(BlockDiskDriverAuto) && diskDriver != string(BlockDiskDriverVirtio) { + return "", fmt.Errorf("unsupported disk driver %s for disk type %s", diskDriver, diskType) + } + + exactDiskDriver = BlockDiskDriverVirtioBlk + if slices.Contains(subsystems, string(BlockDiskSubsystemScsi)) { + exactDiskDriver = BlockDiskDriverVirtioScsi + } + } else if slices.Contains(subsystems, string(BlockDiskSubsystemNvme)) { + if diskDriver != string(BlockDiskDriverAuto) && diskDriver != string(BlockDiskDriverNvme) { + return "", fmt.Errorf("unsupported disk driver %s for disk type %s", diskDriver, diskType) + } + + exactDiskDriver = BlockDiskDriverNvme + } else { + return "", fmt.Errorf("unsupported disk subsystems %v for disk type %s", subsystems, diskType) + } + case BlockDiskTypeLoop: + if diskDriver != string(BlockDiskDriverAuto) && diskDriver != string(BlockDiskDriverAio) { + return "", fmt.Errorf("unsupported disk driver %s for disk type %s", diskDriver, diskType) + } + exactDiskDriver = BlockDiskDriverAio + default: + return "", fmt.Errorf("unsupported disk type %s", diskType) + } + + return exactDiskDriver, nil +} + +func isBDF(addr string) bool { + bdfFormat := "[a-f0-9]{4}:[a-f0-9]{2}:[a-f0-9]{2}\\.[a-f0-9]{1}" + bdfPattern := regexp.MustCompile(bdfFormat) + return bdfPattern.MatchString(addr) +} diff --git a/pkg/spdk/disk/virtio-blk/virtio-blk.go b/pkg/spdk/disk/virtio-blk/virtio-blk.go new file mode 100644 index 000000000..eb610797f --- /dev/null +++ b/pkg/spdk/disk/virtio-blk/virtio-blk.go @@ -0,0 +1,57 @@ +package virtioblk + +import ( + "github.com/pkg/errors" + + commonTypes "github.com/longhorn/go-common-libs/types" + spdkclient "github.com/longhorn/go-spdk-helper/pkg/spdk/client" + spdksetup "github.com/longhorn/go-spdk-helper/pkg/spdk/setup" + spdktypes "github.com/longhorn/go-spdk-helper/pkg/spdk/types" + helperutil "github.com/longhorn/go-spdk-helper/pkg/util" + + "github.com/longhorn/longhorn-spdk-engine/pkg/spdk/disk" +) + +type DiskDriverVirtioBlk struct { +} + +const ( + diskDriver = "virtio-blk" +) + +func init() { + driver := &DiskDriverVirtioBlk{} + if err := disk.RegisterDiskDriver(diskDriver, driver); err != nil { + panic(err) + } +} + +func (d *DiskDriverVirtioBlk) DiskCreate(spdkClient *spdkclient.Client, diskName, diskPath string, blockSize uint64) (string, error) { + // TODO: validate the diskPath + executor, err := helperutil.NewExecutor(commonTypes.ProcDirectory) + if err != nil { + return "", errors.Wrapf(err, "failed to get the executor for disk create %v", diskPath) + } + + _, err = spdksetup.Bind(diskPath, "uio_pci_generic", executor) + if err != nil { + return "", errors.Wrapf(err, "failed to bind the disk %v with uio_pci_generic", diskPath) + } + + bdevs, err := spdkClient.BdevVirtioAttachController(diskName, "pci", diskPath, "blk") + if err != nil { + return "", errors.Wrapf(err, "failed to attach the disk %v with virtio-blk", diskPath) + } + if len(bdevs) == 0 { + return "", errors.Errorf("failed to attach the disk %v with virtio-blk", diskPath) + } + return bdevs[0], nil +} + +func (d *DiskDriverVirtioBlk) DiskDelete(spdkClient *spdkclient.Client, diskName string) (deleted bool, err error) { + return spdkClient.BdevVirtioDetachController(diskName) +} + +func (d *DiskDriverVirtioBlk) DiskGet(spdkClient *spdkclient.Client, diskName, diskPath string, timeout uint64) ([]spdktypes.BdevInfo, error) { + return spdkClient.BdevGetBdevs(diskName, timeout) +} diff --git a/pkg/spdk/disk/virtio-scsi/virtio-scsi.go b/pkg/spdk/disk/virtio-scsi/virtio-scsi.go new file mode 100644 index 000000000..e742345c3 --- /dev/null +++ b/pkg/spdk/disk/virtio-scsi/virtio-scsi.go @@ -0,0 +1,57 @@ +package virtioscsi + +import ( + "github.com/pkg/errors" + + commonTypes "github.com/longhorn/go-common-libs/types" + spdkclient "github.com/longhorn/go-spdk-helper/pkg/spdk/client" + spdksetup "github.com/longhorn/go-spdk-helper/pkg/spdk/setup" + spdktypes "github.com/longhorn/go-spdk-helper/pkg/spdk/types" + helperutil "github.com/longhorn/go-spdk-helper/pkg/util" + + "github.com/longhorn/longhorn-spdk-engine/pkg/spdk/disk" +) + +type DiskDriverVirtioScsi struct { +} + +const ( + diskDriver = "virtio-scsi" +) + +func init() { + driver := &DiskDriverVirtioScsi{} + if err := disk.RegisterDiskDriver(diskDriver, driver); err != nil { + panic(err) + } +} + +func (d *DiskDriverVirtioScsi) DiskCreate(spdkClient *spdkclient.Client, diskName, diskPath string, blockSize uint64) (string, error) { + // TODO: validate the diskPath + executor, err := helperutil.NewExecutor(commonTypes.ProcDirectory) + if err != nil { + return "", errors.Wrapf(err, "failed to get the executor for disk create %v", diskPath) + } + + _, err = spdksetup.Bind(diskPath, "uio_pci_generic", executor) + if err != nil { + return "", errors.Wrapf(err, "failed to bind the disk %v with uio_pci_generic", diskPath) + } + + bdevs, err := spdkClient.BdevVirtioAttachController(diskName, "pci", diskPath, "scsi") + if err != nil { + return "", errors.Wrapf(err, "failed to attach the disk %v with %v", diskPath, diskDriver) + } + if len(bdevs) == 0 { + return "", errors.Errorf("failed to attach the disk %v with %v", diskPath, diskDriver) + } + return bdevs[0], nil +} + +func (d *DiskDriverVirtioScsi) DiskDelete(spdkClient *spdkclient.Client, diskName string) (deleted bool, err error) { + return spdkClient.BdevVirtioDetachController(diskName) +} + +func (d *DiskDriverVirtioScsi) DiskGet(spdkClient *spdkclient.Client, diskName, diskPath string, timeout uint64) ([]spdktypes.BdevInfo, error) { + return spdkClient.BdevGetBdevs(diskName, timeout) +} diff --git a/pkg/spdk/engine.go b/pkg/spdk/engine.go index 2524898d3..c374f0bc6 100644 --- a/pkg/spdk/engine.go +++ b/pkg/spdk/engine.go @@ -1098,7 +1098,7 @@ func (e *Engine) snapshotOperation(spdkClient *spdkclient.Client, inputSnapshotN } else { e.log.Infof("Requesting system sync %v before snapshot", devicePath) // TODO: only sync the device path rather than all filesystems - if _, err := ne.Execute("sync", []string{}, SyncTimeout); err != nil { + if _, err := ne.Execute(nil, "sync", []string{}, SyncTimeout); err != nil { // sync should never fail though, so it more like due to the nsenter e.log.WithError(err).Errorf("WARNING: failed to sync for snapshot op %v with snapshot %s, will skip the sync and continue", snapshotOp, inputSnapshotName) } diff --git a/pkg/spdk/replica.go b/pkg/spdk/replica.go index 29d792456..8fee259e2 100644 --- a/pkg/spdk/replica.go +++ b/pkg/spdk/replica.go @@ -822,7 +822,7 @@ func (r *Replica) SnapshotCreate(spdkClient *spdkclient.Client, snapshotName str } headSvcLvol := r.ActiveChain[r.ChainLength-1] - snapUUID, err := spdkClient.BdevLvolSnapshot(headSvcLvol.UUID, snapLvolName) + snapUUID, err := spdkClient.BdevLvolSnapshot(headSvcLvol.UUID, snapLvolName, nil) if err != nil { return nil, err } @@ -1427,7 +1427,7 @@ func (r *Replica) RebuildingDstSnapshotCreate(spdkClient *spdkclient.Client, sna }() snapLvolName := GetReplicaSnapshotLvolName(r.Name, snapshotName) - snapUUID, err := spdkClient.BdevLvolSnapshot(r.rebuildingLvol.UUID, snapLvolName) + snapUUID, err := spdkClient.BdevLvolSnapshot(r.rebuildingLvol.UUID, snapLvolName, nil) if err != nil { return err } diff --git a/pkg/spdk/server.go b/pkg/spdk/server.go index a459fbff1..a2ced5e05 100644 --- a/pkg/spdk/server.go +++ b/pkg/spdk/server.go @@ -1176,7 +1176,7 @@ func (s *Server) DiskCreate(ctx context.Context, req *spdkrpc.DiskCreateRequest) spdkClient := s.spdkClient s.RUnlock() - return svcDiskCreate(spdkClient, req.DiskName, req.DiskUuid, req.DiskPath, req.BlockSize) + return svcDiskCreate(spdkClient, req.DiskName, req.DiskUuid, req.DiskPath, req.DiskDriver, req.BlockSize) } func (s *Server) DiskDelete(ctx context.Context, req *spdkrpc.DiskDeleteRequest) (ret *emptypb.Empty, err error) { @@ -1184,7 +1184,7 @@ func (s *Server) DiskDelete(ctx context.Context, req *spdkrpc.DiskDeleteRequest) spdkClient := s.spdkClient s.RUnlock() - return svcDiskDelete(spdkClient, req.DiskName, req.DiskUuid) + return svcDiskDelete(spdkClient, req.DiskName, req.DiskUuid, req.DiskDriver) } func (s *Server) DiskGet(ctx context.Context, req *spdkrpc.DiskGetRequest) (ret *spdkrpc.Disk, err error) { @@ -1192,7 +1192,7 @@ func (s *Server) DiskGet(ctx context.Context, req *spdkrpc.DiskGetRequest) (ret spdkClient := s.spdkClient s.RUnlock() - return svcDiskGet(spdkClient, req.DiskName) + return svcDiskGet(spdkClient, req.DiskName, req.DiskPath, req.DiskDriver) } func (s *Server) VersionDetailGet(context.Context, *emptypb.Empty) (*spdkrpc.VersionDetailGetReply, error) { diff --git a/pkg/spdk_test.go b/pkg/spdk_test.go index ea19aacde..050f6a135 100644 --- a/pkg/spdk_test.go +++ b/pkg/spdk_test.go @@ -159,6 +159,8 @@ func CleanupDiskFile(c *C, loopDevicePath string) { func (s *TestSuite) TestSPDKMultipleThread(c *C) { fmt.Println("Testing SPDK basic operations with multiple threads") + diskDriverName := "aio" + ip, err := commonNet.GetAnyExternalIP() c.Assert(err, IsNil) os.Setenv(commonNet.EnvPodIP, ip) @@ -178,13 +180,13 @@ func (s *TestSuite) TestSPDKMultipleThread(c *C) { spdkCli, err := client.NewSPDKClient(net.JoinHostPort(ip, strconv.Itoa(types.SPDKServicePort))) c.Assert(err, IsNil) - disk, err := spdkCli.DiskCreate(defaultTestDiskName, "", loopDevicePath, int64(defaultTestBlockSize)) + disk, err := spdkCli.DiskCreate(defaultTestDiskName, "", loopDevicePath, diskDriverName, int64(defaultTestBlockSize)) c.Assert(err, IsNil) c.Assert(disk.Path, Equals, loopDevicePath) c.Assert(disk.Uuid, Not(Equals), "") defer func() { - err := spdkCli.DiskDelete(defaultTestDiskName, disk.Uuid) + err := spdkCli.DiskDelete(defaultTestDiskName, disk.Uuid, diskDriverName) c.Assert(err, IsNil) }() @@ -422,6 +424,7 @@ func (s *TestSuite) TestSPDKMultipleThread(c *C) { func (s *TestSuite) TestSPDKMultipleThreadSnapshot(c *C) { fmt.Println("Testing SPDK snapshot operations with multiple threads") + diskDriverName := "aio" ip, err := commonNet.GetAnyExternalIP() c.Assert(err, IsNil) @@ -442,13 +445,13 @@ func (s *TestSuite) TestSPDKMultipleThreadSnapshot(c *C) { spdkCli, err := client.NewSPDKClient(net.JoinHostPort(ip, strconv.Itoa(types.SPDKServicePort))) c.Assert(err, IsNil) - disk, err := spdkCli.DiskCreate(defaultTestDiskName, "", loopDevicePath, int64(defaultTestBlockSize)) + disk, err := spdkCli.DiskCreate(defaultTestDiskName, "", loopDevicePath, diskDriverName, int64(defaultTestBlockSize)) c.Assert(err, IsNil) c.Assert(disk.Path, Equals, loopDevicePath) c.Assert(disk.Uuid, Not(Equals), "") defer func() { - err := spdkCli.DiskDelete(defaultTestDiskName, disk.Uuid) + err := spdkCli.DiskDelete(defaultTestDiskName, disk.Uuid, diskDriverName) c.Assert(err, IsNil) }() diff --git a/pkg/util/block.go b/pkg/util/block.go new file mode 100644 index 000000000..3f559a868 --- /dev/null +++ b/pkg/util/block.go @@ -0,0 +1,72 @@ +package util + +import ( + "fmt" + "os" + "strings" + + "github.com/pkg/errors" + "github.com/sirupsen/logrus" + + commonTypes "github.com/longhorn/go-common-libs/types" + "github.com/longhorn/go-spdk-helper/pkg/types" + helperutil "github.com/longhorn/go-spdk-helper/pkg/util" +) + +func GetDevNameFromBDF(bdf string) (string, error) { + ne, err := helperutil.NewExecutor(commonTypes.ProcDirectory) + if err != nil { + return "", errors.Wrap(err, "failed to create executor") + } + + cmdArgs := []string{"-n", "--nodeps", "--output", "NAME"} + output, err := ne.Execute(nil, "lsblk", cmdArgs, types.ExecuteTimeout) + if err != nil { + return "", errors.Wrap(err, "failed to list block devices") + } + + devices := strings.Fields(string(output)) + for _, dev := range devices { + link, err := os.Readlink("/sys/block/" + dev) + if err != nil { + logrus.WithError(err).Warnf("Failed to read link for %s", dev) + continue + } + + if strings.Contains(link, bdf) { + return dev, nil + } + } + + return "", fmt.Errorf("failed to find device for bdf %s", bdf) +} + +func GetBlockDiskType(devPath string) (string, error) { + ne, err := helperutil.NewExecutor(commonTypes.ProcDirectory) + if err != nil { + return "", errors.Wrap(err, "failed to create executor") + } + + cmdArgs := []string{"-n", "-d", "-o", "type", devPath} + output, err := ne.Execute(nil, "lsblk", cmdArgs, types.ExecuteTimeout) + if err != nil { + return "", errors.Wrap(err, "failed to get disk type") + } + + return strings.TrimSpace(string(output)), nil +} + +func GetBlockDiskSubsystems(devName string) ([]string, error) { + ne, err := helperutil.NewExecutor(commonTypes.ProcDirectory) + if err != nil { + return nil, errors.Wrap(err, "failed to create executor") + } + + cmdArgs := []string{"-n", "-d", "-o", "subsystems", fmt.Sprintf("/dev/%s", devName)} + output, err := ne.Execute(nil, "lsblk", cmdArgs, types.ExecuteTimeout) + if err != nil { + return nil, errors.Wrap(err, "failed to get disk subsystems") + } + + return strings.Split(strings.TrimSpace(string(output)), ":"), nil +}