diff --git a/go.mod b/go.mod index cb8be1c8..6b693796 100644 --- a/go.mod +++ b/go.mod @@ -59,3 +59,5 @@ require ( k8s.io/mount-utils v0.31.3 // indirect k8s.io/utils v0.0.0-20240711033017-18e509b52bc8 // indirect ) + +replace github.com/longhorn/go-spdk-helper v0.0.0-20241208060229-9425f07f800a => github.com/derekbit/go-spdk-helper v0.0.0-20241208073439-a6020c58ab85 diff --git a/go.sum b/go.sum index 04481694..a4b29d8e 100644 --- a/go.sum +++ b/go.sum @@ -18,6 +18,8 @@ github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSs github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/derekbit/go-spdk-helper v0.0.0-20241208073439-a6020c58ab85 h1:Z92nuV0JC+7pWTLu4KAyrjtQo6pQjCa2oaj0hD8sESk= +github.com/derekbit/go-spdk-helper v0.0.0-20241208073439-a6020c58ab85/go.mod h1:JoJT6We5ctsc/I4t/ucRLqqlTgyXAJIVTl0P4KW33do= github.com/gammazero/deque v0.2.0 h1:SkieyNB4bg2/uZZLxvya0Pq6diUlwx7m2TeT7GAIWaA= github.com/gammazero/deque v0.2.0/go.mod h1:LFroj8x4cMYCukHJDbxFCkT+r9AndaJnFMuZDV34tuU= github.com/gammazero/workerpool v1.1.3 h1:WixN4xzukFoN0XSeXF6puqEqFTl2mECI9S6W44HWy9Q= @@ -48,8 +50,6 @@ github.com/longhorn/backupstore v0.0.0-20241208060255-5c474bb003bd h1:lcqAEYf2+Z github.com/longhorn/backupstore v0.0.0-20241208060255-5c474bb003bd/go.mod h1:35urUcY2wO9lgFQnoR4l9i4Hc/x/fNzI8Pn/dloPSd0= github.com/longhorn/go-common-libs v0.0.0-20241208100509-e1932c65c078 h1:QnN9bPRhWr38ziOdknlC+NORk0PgQJcPX1vo20oVi9I= github.com/longhorn/go-common-libs v0.0.0-20241208100509-e1932c65c078/go.mod h1:whDcaYDin1L7uaKTpr86RxpfOT+VJQbubDWdEGPnvVs= -github.com/longhorn/go-spdk-helper v0.0.0-20241208060229-9425f07f800a h1:o8ycbukiUGbBPNGnGbU3H+YSbjNsdk5KgJHoiWkpjmA= -github.com/longhorn/go-spdk-helper v0.0.0-20241208060229-9425f07f800a/go.mod h1:JoJT6We5ctsc/I4t/ucRLqqlTgyXAJIVTl0P4KW33do= github.com/longhorn/types v0.0.0-20241208031854-891e672bc453 h1:NdPh1ARYoBnFqCn3sRNJXa+WeJZcS+J0wXpJBou26KM= github.com/longhorn/types v0.0.0-20241208031854-891e672bc453/go.mod h1:ZElOIs7s/Cjaw7P9kY+uvTzh87mfO34pk39B6TVmg0g= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= diff --git a/vendor/github.com/longhorn/go-spdk-helper/pkg/nvme/initiator.go b/vendor/github.com/longhorn/go-spdk-helper/pkg/nvme/initiator.go index e9fdac98..033b1ac0 100644 --- a/vendor/github.com/longhorn/go-spdk-helper/pkg/nvme/initiator.go +++ b/vendor/github.com/longhorn/go-spdk-helper/pkg/nvme/initiator.go @@ -3,6 +3,7 @@ package nvme import ( "fmt" "os" + "path/filepath" "strconv" "strings" "time" @@ -164,7 +165,7 @@ func (i *Initiator) WaitForDisconnect(maxNumRetries int, retryInterval time.Dura for r := 0; r < maxNumRetries; r++ { err = i.loadNVMeDeviceInfoWithoutLock(i.TransportAddress, i.TransportServiceID, i.SubsystemNQN) - if IsValidNvmeDeviceNotFound(err) { + if types.ErrorIsValidNvmeDeviceNotFound(err) { return nil } time.Sleep(retryInterval) @@ -215,7 +216,7 @@ func (i *Initiator) Resume() error { } func (i *Initiator) resumeLinearDmDevice() error { - logrus.Infof("Resuming linear dm device %s", i.Name) + i.logger.Info("Resuming linear dm device") return util.DmsetupResume(i.Name, i.executor) } @@ -243,24 +244,22 @@ func (i *Initiator) replaceDmDeviceTarget() error { } // Start starts the NVMe initiator with the given transportAddress and transportServiceID -func (i *Initiator) Start(transportAddress, transportServiceID string, dmDeviceAndEndpointCleanupRequired bool) (dmDeviceBusy bool, err error) { +func (i *Initiator) Start(transportAddress, transportServiceID string, dmDeviceAndEndpointCleanupRequired bool) (dmDeviceIsBusy bool, err error) { defer func() { if err != nil { err = errors.Wrapf(err, "failed to start NVMe initiator %s", i.Name) } }() + if transportAddress == "" || transportServiceID == "" { + return false, fmt.Errorf("invalid transportAddress %s and transportServiceID %s for starting initiator %s", transportAddress, transportServiceID, i.Name) + } + i.logger.WithFields(logrus.Fields{ "transportAddress": transportAddress, "transportServiceID": transportServiceID, "dmDeviceAndEndpointCleanupRequired": dmDeviceAndEndpointCleanupRequired, - }) - - i.logger.Info("Starting initiator") - - if transportAddress == "" || transportServiceID == "" { - return false, fmt.Errorf("invalid TransportAddress %s and TransportServiceID %s for initiator %s start", transportAddress, transportServiceID, i.Name) - } + }).Info("Starting NVMe initiator") if i.hostProc != "" { lock, err := i.newLock() @@ -271,104 +270,113 @@ func (i *Initiator) Start(transportAddress, transportServiceID string, dmDeviceA } // Check if the initiator/NVMe device is already launched and matches the params - if err := i.loadNVMeDeviceInfoWithoutLock(i.TransportAddress, i.TransportServiceID, i.SubsystemNQN); err == nil { + err = i.loadNVMeDeviceInfoWithoutLock(i.TransportAddress, i.TransportServiceID, i.SubsystemNQN) + if err == nil { if i.TransportAddress == transportAddress && i.TransportServiceID == transportServiceID { - if err = i.LoadEndpoint(false); err == nil { + err = i.LoadEndpoint(false) + if err == nil { i.logger.Info("NVMe initiator is already launched with correct params") return false, nil } i.logger.WithError(err).Warnf("NVMe initiator is launched with failed to load the endpoint") } else { - i.logger.Warnf("NVMe initiator is launched but with incorrect address, the required one is %s:%s, will try to stop then relaunch it", - transportAddress, transportServiceID) + i.logger.Warnf("NVMe initiator is launched but with incorrect address, the required one is %s:%s, will try to stop then relaunch it", transportAddress, transportServiceID) } } - i.logger.Infof("Stopping NVMe initiator blindly before starting") - dmDeviceBusy, err = i.stopWithoutLock(dmDeviceAndEndpointCleanupRequired, false, false) + i.logger.Info("Stopping NVMe initiator blindly before starting") + dmDeviceIsBusy, err = i.stopWithoutLock(dmDeviceAndEndpointCleanupRequired, false, false) if err != nil { - return dmDeviceBusy, errors.Wrapf(err, "failed to stop the mismatching NVMe initiator %s before starting", i.Name) + return dmDeviceIsBusy, errors.Wrapf(err, "failed to stop the mismatching NVMe initiator %s before starting", i.Name) } - i.logger.WithFields(logrus.Fields{ - "transportAddress": transportAddress, - "transportServiceID": transportServiceID, - }) - i.logger.Infof("Launching NVMe initiator") - - // Setup initiator - for r := 0; r < maxNumRetries; r++ { - // Rerun this API for a discovered target should be fine - subsystemNQN, err := DiscoverTarget(transportAddress, transportServiceID, i.executor) - if err != nil { - i.logger.WithError(err).Warn("Failed to discover target") - time.Sleep(retryInterval) - continue - } - - controllerName, err := ConnectTarget(transportAddress, transportServiceID, subsystemNQN, i.executor) - if err != nil { - i.logger.WithError(err).Warn("Failed to connect target") - time.Sleep(retryInterval) - continue - } - - i.SubsystemNQN = subsystemNQN - i.ControllerName = controllerName - break - } + i.logger.Info("Launching NVMe initiator") + i.connectTarget(transportAddress, transportServiceID) if i.ControllerName == "" { - return dmDeviceBusy, fmt.Errorf("failed to start NVMe initiator %s within %d * %v sec retries", i.Name, maxNumRetries, retryInterval.Seconds()) + return dmDeviceIsBusy, fmt.Errorf("failed to start NVMe initiator %s within %d * %v sec retries", i.Name, maxNumRetries, retryInterval.Seconds()) } - for r := 0; r < maxNumWaitDeviceRetries; r++ { - err = i.loadNVMeDeviceInfoWithoutLock(i.TransportAddress, i.TransportServiceID, i.SubsystemNQN) - if err == nil { - break - } - time.Sleep(waitDeviceInterval) - } + err = i.waitAndLoadNVMeDeviceInfoWithoutLock(transportAddress, transportServiceID) if err != nil { - return dmDeviceBusy, errors.Wrapf(err, "failed to load device info after starting NVMe initiator %s", i.Name) + return dmDeviceIsBusy, errors.Wrapf(err, "failed to load device info after connecting target for NVMe initiator %s", i.Name) } - needMakeEndpoint := true if dmDeviceAndEndpointCleanupRequired { - if dmDeviceBusy { + if dmDeviceIsBusy { // Endpoint is already created, just replace the target device - needMakeEndpoint = false - i.logger.Infof("Linear dm device is busy, trying the best to replace the target device for NVMe initiator %s", i.Name) + i.logger.Info("Linear dm device is busy, trying the best to replace the target device for NVMe initiator") if err := i.replaceDmDeviceTarget(); err != nil { - i.logger.WithError(err).Warnf("Failed to replace the target device for NVMe initiator %s", i.Name) + i.logger.WithError(err).Warnf("Failed to replace the target device for NVMe initiator") } else { - i.logger.Infof("Successfully replaced the target device for NVMe initiator %s", i.Name) - dmDeviceBusy = false + i.logger.Info("Successfully replaced the target device for NVMe initiator") + dmDeviceIsBusy = false } } else { - i.logger.Infof("Creating linear dm device for NVMe initiator %s", i.Name) + i.logger.Info("Creating linear dm device for NVMe initiator") if err := i.createLinearDmDevice(); err != nil { return false, errors.Wrapf(err, "failed to create linear dm device for NVMe initiator %s", i.Name) } } } else { - i.logger.Infof("Skipping creating linear dm device for NVMe initiator %s", i.Name) + i.logger.Info("Skipping creating linear dm device for NVMe initiator") i.dev.Export = i.dev.Nvme } - if needMakeEndpoint { - i.logger.Infof("Creating endpoint %v", i.Endpoint) + i.logger.Infof("Creating endpoint %v", i.Endpoint) + exist, err := i.isEndpointExist() + if err != nil { + return dmDeviceIsBusy, errors.Wrapf(err, "failed to check if endpoint %v exists for NVMe initiator %s", i.Endpoint, i.Name) + } + if exist { + i.logger.Infof("Skipping endpoint %v creation for NVMe initiator", i.Endpoint) + } else { if err := i.makeEndpoint(); err != nil { - return dmDeviceBusy, err + return dmDeviceIsBusy, err } } i.logger.Infof("Launched NVMe initiator: %+v", i) - return dmDeviceBusy, nil + return dmDeviceIsBusy, nil } -func (i *Initiator) Stop(dmDeviceAndEndpointCleanupRequired, deferDmDeviceCleanup, errOnBusyDmDevice bool) (bool, error) { +func (i *Initiator) waitAndLoadNVMeDeviceInfoWithoutLock(transportAddress, transportServiceID string) (err error) { + for r := 0; r < maxNumWaitDeviceRetries; r++ { + //err = i.loadNVMeDeviceInfoWithoutLock(i.TransportAddress, i.TransportServiceID, i.SubsystemNQN) + err = i.loadNVMeDeviceInfoWithoutLock(transportAddress, transportServiceID, i.SubsystemNQN) + if err == nil { + break + } + time.Sleep(waitDeviceInterval) + } + return err +} + +func (i *Initiator) connectTarget(transportAddress, transportServiceID string) { + for r := 0; r < maxNumRetries; r++ { + // Rerun this API for a discovered target should be fine + subsystemNQN, err := DiscoverTarget(transportAddress, transportServiceID, i.executor) + if err != nil { + i.logger.WithError(err).Warn("Failed to discover target") + time.Sleep(retryInterval) + continue + } + + controllerName, err := ConnectTarget(transportAddress, transportServiceID, subsystemNQN, i.executor) + if err != nil { + i.logger.WithError(err).Warn("Failed to connect target") + time.Sleep(retryInterval) + continue + } + + i.SubsystemNQN = subsystemNQN + i.ControllerName = controllerName + break + } +} + +func (i *Initiator) Stop(dmDeviceAndEndpointCleanupRequired, deferDmDeviceCleanup, returnErrorForBusyDevice bool) (bool, error) { if i.hostProc != "" { lock, err := i.newLock() if err != nil { @@ -377,37 +385,36 @@ func (i *Initiator) Stop(dmDeviceAndEndpointCleanupRequired, deferDmDeviceCleanu defer lock.Unlock() } - return i.stopWithoutLock(dmDeviceAndEndpointCleanupRequired, deferDmDeviceCleanup, errOnBusyDmDevice) + return i.stopWithoutLock(dmDeviceAndEndpointCleanupRequired, deferDmDeviceCleanup, returnErrorForBusyDevice) } -func (i *Initiator) removeDmDeviceAndEndpoint(deferDmDeviceCleanup, errOnBusyDmDevice bool) (bool, error) { - if err := i.removeLinearDmDevice(false, deferDmDeviceCleanup); err != nil { - if strings.Contains(err.Error(), "Device or resource busy") { - if errOnBusyDmDevice { - return true, err +func (i *Initiator) stopWithoutLock(dmDeviceAndEndpointCleanupRequired, deferDmDeviceCleanup, returnErrorForBusyDevice bool) (dmDeviceIsBusy bool, err error) { + dmDeviceIsBusy = false + + if dmDeviceAndEndpointCleanupRequired { + err = i.removeLinearDmDevice(false, deferDmDeviceCleanup) + if err != nil { + if !os.IsNotExist(err) { + if types.ErrorIsDeviceOrResourceBusy(err) { + if returnErrorForBusyDevice { + return true, err + } + dmDeviceIsBusy = true + } else { + return false, err + } } - return true, nil } - return false, err - } - if err := i.removeEndpoint(); err != nil { - return false, err - } - return false, nil -} -func (i *Initiator) stopWithoutLock(dmDeviceAndEndpointCleanupRequired, deferDmDeviceCleanup, errOnBusyDmDevice bool) (bool, error) { - dmDeviceBusy := false - if dmDeviceAndEndpointCleanupRequired { - var err error - dmDeviceBusy, err = i.removeDmDeviceAndEndpoint(deferDmDeviceCleanup, errOnBusyDmDevice) + err = i.removeEndpoint() if err != nil { return false, err } } - if err := DisconnectTarget(i.SubsystemNQN, i.executor); err != nil { - return dmDeviceBusy, errors.Wrapf(err, "failed to logout target") + err = DisconnectTarget(i.SubsystemNQN, i.executor) + if err != nil { + return dmDeviceIsBusy, errors.Wrapf(err, "failed to disconnect target for NVMe initiator %s", i.Name) } i.ControllerName = "" @@ -415,7 +422,7 @@ func (i *Initiator) stopWithoutLock(dmDeviceAndEndpointCleanupRequired, deferDmD i.TransportAddress = "" i.TransportServiceID = "" - return dmDeviceBusy, nil + return dmDeviceIsBusy, nil } func (i *Initiator) GetControllerName() string { @@ -467,10 +474,11 @@ func (i *Initiator) loadNVMeDeviceInfoWithoutLock(transportAddress, transportSer if i.ControllerName != "" && i.ControllerName != nvmeDevices[0].Controllers[0].Controller { return fmt.Errorf("found mismatching between the detected controller name %s and the recorded value %s for NVMe initiator %s", nvmeDevices[0].Controllers[0].Controller, i.ControllerName, i.Name) } + i.ControllerName = nvmeDevices[0].Controllers[0].Controller i.NamespaceName = nvmeDevices[0].Namespaces[0].NameSpace i.TransportAddress, i.TransportServiceID = GetIPAndPortFromControllerAddress(nvmeDevices[0].Controllers[0].Address) - i.logger.WithFields(logrus.Fields{ + i.logger = i.logger.WithFields(logrus.Fields{ "controllerName": i.ControllerName, "namespaceName": i.NamespaceName, "transportAddress": i.TransportAddress, @@ -506,7 +514,7 @@ func (i *Initiator) findDependentDevices(devName string) ([]string, error) { return depDevices, nil } -func (i *Initiator) LoadEndpoint(dmDeviceBusy bool) error { +func (i *Initiator) LoadEndpoint(dmDeviceIsBusy bool) error { dev, err := util.DetectDevice(i.Endpoint, i.executor) if err != nil { return err @@ -517,8 +525,8 @@ func (i *Initiator) LoadEndpoint(dmDeviceBusy bool) error { return err } - if dmDeviceBusy { - i.logger.Debugf("Skipping endpoint %v loading for NVMe initiator %v due to device busy", i.Endpoint, i.Name) + if dmDeviceIsBusy { + i.logger.Debugf("Skipping endpoint %v loading due to device busy", i.Endpoint) } else { if i.NamespaceName != "" && !i.isNamespaceExist(depDevices) { return fmt.Errorf("detected device %s name mismatching from endpoint %v for NVMe initiator %s", dev.Name, i.Endpoint, i.Name) @@ -533,6 +541,17 @@ func (i *Initiator) LoadEndpoint(dmDeviceBusy bool) error { return nil } +func (i *Initiator) isEndpointExist() (bool, error) { + _, err := os.Stat(i.Endpoint) + if err != nil { + if os.IsNotExist(err) { + return false, nil + } + return false, err + } + return true, nil +} + func (i *Initiator) makeEndpoint() error { if err := util.DuplicateDevice(i.dev, i.Endpoint); err != nil { return errors.Wrap(err, "failed to duplicate device") @@ -542,26 +561,25 @@ func (i *Initiator) makeEndpoint() error { } func (i *Initiator) removeEndpoint() error { + if i.Endpoint == "" { + return nil + } + if err := util.RemoveDevice(i.Endpoint); err != nil { return err } i.dev = nil i.isUp = false - return nil } func (i *Initiator) removeLinearDmDevice(force, deferred bool) error { - devPath := getDmDevicePath(i.Name) - if _, err := os.Stat(devPath); err != nil { - if os.IsNotExist(err) { - logrus.Infof("Linear dm device %s doesn't exist", devPath) - return nil - } - return errors.Wrapf(err, "failed to stat linear dm device %s", devPath) + dmDevPath := getDmDevicePath(i.Name) + if _, err := os.Stat(dmDevPath); err != nil { + return err } - logrus.Infof("Removing linear dm device %s", i.Name) + i.logger.Info("Removing linear dm device") return util.DmsetupRemove(i.Name, force, deferred, i.executor) } @@ -578,7 +596,8 @@ func (i *Initiator) createLinearDmDevice() error { // Create a device mapper device with the same size as the original device table := fmt.Sprintf("0 %v linear %v 0", sectors, nvmeDevPath) - logrus.Infof("Creating linear dm device %s with table %s", i.Name, table) + + i.logger.Infof("Creating linear dm device with table '%s'", table) if err := util.DmsetupCreate(i.Name, table, i.executor); err != nil { return err } @@ -588,6 +607,7 @@ func (i *Initiator) createLinearDmDevice() error { return err } + // Get the device numbers major, minor, err := util.GetDeviceNumbers(dmDevPath, i.executor) if err != nil { return err @@ -613,7 +633,7 @@ func validateDiskCreation(path string, timeout int) error { } func (i *Initiator) suspendLinearDmDevice(noflush, nolockfs bool) error { - logrus.Infof("Suspending linear dm device %s", i.Name) + i.logger.Info("Suspending linear dm device") return util.DmsetupSuspend(i.Name, noflush, nolockfs, i.executor) } @@ -664,15 +684,27 @@ func (i *Initiator) reloadLinearDmDevice() error { table := fmt.Sprintf("0 %v linear %v 0", sectors, devPath) - logrus.Infof("Reloading linear dm device %s with table '%s'", i.Name, table) + i.logger.Infof("Reloading linear dm device with table '%s'", table) - return util.DmsetupReload(i.Name, table, i.executor) -} + err = util.DmsetupReload(i.Name, table, i.executor) + if err != nil { + return err + } -func getDmDevicePath(name string) string { - return fmt.Sprintf("/dev/mapper/%s", name) + // Reload the device numbers + dmDevPath := getDmDevicePath(i.Name) + major, minor, err := util.GetDeviceNumbers(dmDevPath, i.executor) + if err != nil { + return err + } + + i.dev.Export.Name = i.Name + i.dev.Export.Major = major + i.dev.Export.Minor = minor + + return nil } -func IsValidNvmeDeviceNotFound(err error) bool { - return strings.Contains(err.Error(), ErrorMessageCannotFindValidNvmeDevice) +func getDmDevicePath(name string) string { + return filepath.Join("/dev/mapper", name) } diff --git a/vendor/github.com/longhorn/go-spdk-helper/pkg/nvme/nvme.go b/vendor/github.com/longhorn/go-spdk-helper/pkg/nvme/nvme.go index 75e26da9..764ba670 100644 --- a/vendor/github.com/longhorn/go-spdk-helper/pkg/nvme/nvme.go +++ b/vendor/github.com/longhorn/go-spdk-helper/pkg/nvme/nvme.go @@ -8,10 +8,8 @@ import ( "github.com/sirupsen/logrus" commonns "github.com/longhorn/go-common-libs/ns" -) -const ( - ErrorMessageCannotFindValidNvmeDevice = "cannot find a valid nvme device" + "github.com/longhorn/go-spdk-helper/pkg/types" ) // DiscoverTarget discovers a target @@ -59,7 +57,7 @@ func ConnectTarget(ip, port, nqn string, executor *commonns.Executor) (controlle return connect(hostID, hostNQN, nqn, DefaultTransportType, ip, port, executor) } -// DisconnectTarget disconnects a target +// DisconnectTarget disconnects from a target func DisconnectTarget(nqn string, executor *commonns.Executor) error { return disconnect(nqn, executor) } @@ -72,22 +70,21 @@ func GetDevices(ip, port, nqn string, executor *commonns.Executor) (devices []De devices = []Device{} - nvmeDevices, err := listControllers(executor) + nvmeDevices, err := listRecognizedNvmeDevices(executor) if err != nil { return nil, err } for _, d := range nvmeDevices { - // Get subsystem subsystems, err := listSubsystems(d.DevicePath, executor) if err != nil { - logrus.WithError(err).Warnf("failed to get subsystem for nvme device %s", d.DevicePath) + logrus.WithError(err).Warnf("failed to list subsystems for NVMe device %s", d.DevicePath) continue } if len(subsystems) == 0 { - return nil, fmt.Errorf("no subsystem found for nvme device %s", d.DevicePath) + return nil, fmt.Errorf("no subsystem found for NVMe device %s", d.DevicePath) } if len(subsystems) > 1 { - return nil, fmt.Errorf("multiple subsystems found for nvme device %s", d.DevicePath) + return nil, fmt.Errorf("multiple subsystems found for NVMe device %s", d.DevicePath) } sys := subsystems[0] @@ -166,7 +163,7 @@ func GetDevices(ip, port, nqn string, executor *commonns.Executor) (devices []De } } - return nil, fmt.Errorf(ErrorMessageCannotFindValidNvmeDevice+" with subsystem NQN %s and address %s:%s", nqn, ip, port) + return nil, fmt.Errorf(types.ErrorMessageCannotFindValidNvmeDevice+" with subsystem NQN %s and address %s:%s", nqn, ip, port) } return res, nil } diff --git a/vendor/github.com/longhorn/go-spdk-helper/pkg/nvme/nvmecli.go b/vendor/github.com/longhorn/go-spdk-helper/pkg/nvme/nvmecli.go index adc61996..b656d7d8 100644 --- a/vendor/github.com/longhorn/go-spdk-helper/pkg/nvme/nvmecli.go +++ b/vendor/github.com/longhorn/go-spdk-helper/pkg/nvme/nvmecli.go @@ -149,12 +149,12 @@ func listSubsystems(devicePath string, executor *commonns.Executor) ([]Subsystem } if major == 1 { - return listSubsystemsV1(jsonStr, executor) + return listSubsystemsV1(jsonStr) } - return listSubsystemsV2(jsonStr, executor) + return listSubsystemsV2(jsonStr) } -func listSubsystemsV1(jsonStr string, executor *commonns.Executor) ([]Subsystem, error) { +func listSubsystemsV1(jsonStr string) ([]Subsystem, error) { output := map[string][]Subsystem{} if err := json.Unmarshal([]byte(jsonStr), &output); err != nil { return nil, err @@ -169,7 +169,7 @@ type ListSubsystemsV2Output struct { Subsystems []Subsystem `json:"Subsystems"` } -func listSubsystemsV2(jsonStr string, executor *commonns.Executor) ([]Subsystem, error) { +func listSubsystemsV2(jsonStr string) ([]Subsystem, error) { var output []ListSubsystemsV2Output if err := json.Unmarshal([]byte(jsonStr), &output); err != nil { return nil, err @@ -197,7 +197,7 @@ type CliDevice struct { SectorSize int32 `json:"SectorSize,omitempty"` } -func listControllers(executor *commonns.Executor) ([]CliDevice, error) { +func listRecognizedNvmeDevices(executor *commonns.Executor) ([]CliDevice, error) { opts := []string{ "list", "-o", "json", diff --git a/vendor/github.com/longhorn/go-spdk-helper/pkg/spdk/client/advanced.go b/vendor/github.com/longhorn/go-spdk-helper/pkg/spdk/client/advanced.go index 5fefbe40..3c4fce1b 100644 --- a/vendor/github.com/longhorn/go-spdk-helper/pkg/spdk/client/advanced.go +++ b/vendor/github.com/longhorn/go-spdk-helper/pkg/spdk/client/advanced.go @@ -8,6 +8,7 @@ import ( spdktypes "github.com/longhorn/go-spdk-helper/pkg/spdk/types" ) +// AddDevice adds a device with the given device path, name, and cluster size. func (c *Client) AddDevice(devicePath, name string, clusterSize uint32) (bdevAioName, lvsName, lvsUUID string, err error) { // Use the file name as aio name and lvs name if name is not specified. if name == "" { @@ -39,6 +40,7 @@ func (c *Client) AddDevice(devicePath, name string, clusterSize uint32) (bdevAio return name, name, lvsUUID, nil } +// DeleteDevice deletes the device with the given bdevAioName and lvsName. func (c *Client) DeleteDevice(bdevAioName, lvsName string) (err error) { if _, err := c.BdevLvolDeleteLvstore(lvsName, ""); err != nil { return err @@ -51,6 +53,7 @@ func (c *Client) DeleteDevice(bdevAioName, lvsName string) (err error) { return nil } +// StartExposeBdev exposes the bdev with the given nqn, bdevName, nguid, ip, and port. func (c *Client) StartExposeBdev(nqn, bdevName, nguid, ip, port string) error { nvmfTransportList, err := c.NvmfGetTransports("", "") if err != nil { @@ -77,8 +80,10 @@ func (c *Client) StartExposeBdev(nqn, bdevName, nguid, ip, port string) error { return nil } +// StopExposeBdev stops exposing the bdev with the given nqn. func (c *Client) StopExposeBdev(nqn string) error { var subsystem *spdktypes.NvmfSubsystem + subsystemList, err := c.NvmfGetSubsystems("", "") if err != nil { return err diff --git a/vendor/github.com/longhorn/go-spdk-helper/pkg/types/types.go b/vendor/github.com/longhorn/go-spdk-helper/pkg/types/types.go index 12481635..74a2d0c8 100644 --- a/vendor/github.com/longhorn/go-spdk-helper/pkg/types/types.go +++ b/vendor/github.com/longhorn/go-spdk-helper/pkg/types/types.go @@ -2,6 +2,7 @@ package types import ( "fmt" + "strings" "time" ) @@ -25,6 +26,11 @@ const ( ExecuteTimeout = 60 * time.Second ) +const ( + ErrorMessageCannotFindValidNvmeDevice = "cannot find a valid NVMe device" + ErrorMessageDeviceOrResourceBusy = "device or resource busy" +) + const ( // Sequence of Events: // 1. Issuing I/O Command: The system sends a read command to the NVMe SSD. @@ -69,3 +75,11 @@ type DiskStatus struct { Device string BlockDevices string } + +func ErrorIsDeviceOrResourceBusy(err error) bool { + return strings.Contains(strings.ToLower(err.Error()), ErrorMessageDeviceOrResourceBusy) +} + +func ErrorIsValidNvmeDeviceNotFound(err error) bool { + return strings.Contains(err.Error(), ErrorMessageCannotFindValidNvmeDevice) +} diff --git a/vendor/github.com/longhorn/go-spdk-helper/pkg/util/device.go b/vendor/github.com/longhorn/go-spdk-helper/pkg/util/device.go index 32009875..2610cc04 100644 --- a/vendor/github.com/longhorn/go-spdk-helper/pkg/util/device.go +++ b/vendor/github.com/longhorn/go-spdk-helper/pkg/util/device.go @@ -41,13 +41,15 @@ type LonghornBlockDevice struct { } // RemoveDevice removes the given device -func RemoveDevice(dev string) error { - if _, err := os.Stat(dev); err == nil { - if err := remove(dev); err != nil { - return errors.Wrapf(err, "failed to removing device %s", dev) +func RemoveDevice(devPath string) error { + if _, err := os.Stat(devPath); err != nil { + if os.IsNotExist(err) { + return nil } } - return nil + + // Still try to remove the device node + return remove(devPath) } // GetKnownDevices returns the path of the device with the given major and minor numbers diff --git a/vendor/modules.txt b/vendor/modules.txt index be508518..50715c5c 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -64,7 +64,7 @@ github.com/longhorn/go-common-libs/sync github.com/longhorn/go-common-libs/sys github.com/longhorn/go-common-libs/types github.com/longhorn/go-common-libs/utils -# github.com/longhorn/go-spdk-helper v0.0.0-20241208060229-9425f07f800a +# github.com/longhorn/go-spdk-helper v0.0.0-20241208060229-9425f07f800a => github.com/derekbit/go-spdk-helper v0.0.0-20241208073439-a6020c58ab85 ## explicit; go 1.22.7 github.com/longhorn/go-spdk-helper/pkg/jsonrpc github.com/longhorn/go-spdk-helper/pkg/nvme