diff --git a/config/examples/kustomization.yaml b/config/examples/kustomization.yaml index c75d8fb0d..2780a57c4 100644 --- a/config/examples/kustomization.yaml +++ b/config/examples/kustomization.yaml @@ -9,4 +9,4 @@ kind: Kustomization images: - name: nnf-mfu newName: ghcr.io/nearnodeflash/nnf-mfu - newTag: master + newTag: 0.0.2 diff --git a/config/manager/kustomization.yaml b/config/manager/kustomization.yaml index cfe82999d..d92687c6d 100644 --- a/config/manager/kustomization.yaml +++ b/config/manager/kustomization.yaml @@ -17,4 +17,4 @@ kind: Kustomization images: - name: controller newName: ghcr.io/nearnodeflash/nnf-sos - newTag: 53297ed0c2ce52ddc2113671b3dd3c79f5894dce + newTag: 0.0.0.414-1c1f5 diff --git a/controllers/nnf_node_storage_controller.go b/controllers/nnf_node_storage_controller.go index 003b360e2..aaefa2617 100644 --- a/controllers/nnf_node_storage_controller.go +++ b/controllers/nnf_node_storage_controller.go @@ -22,9 +22,11 @@ package controllers import ( "context" "crypto/md5" + "encoding/json" "fmt" "net/http" "os" + "os/exec" "strconv" "strings" "time" @@ -42,6 +44,7 @@ import ( ec "github.com/NearNodeFlash/nnf-ec/pkg/ec" nnf "github.com/NearNodeFlash/nnf-ec/pkg/manager-nnf" + nnfnvme "github.com/NearNodeFlash/nnf-ec/pkg/manager-nvme" nnfserver "github.com/NearNodeFlash/nnf-ec/pkg/manager-server" openapi "github.com/NearNodeFlash/nnf-ec/pkg/rfsf/pkg/common" @@ -691,7 +694,26 @@ func (r *NnfNodeStorageReconciler) isSpecComplete(nodeStorage *nnfv1alpha1.NnfNo return false } +type nvmeListVerboseNamespaces struct { + Device string `json:"NameSpace"` + NSID uint32 `json:"NSID"` +} + +type nvmeListVerboseControllers struct { + Namespaces []nvmeListVerboseNamespaces `json:"Namespaces"` +} + +type nvmeListVerboseDevice struct { + SubsystemNQN string `json:"SubsystemNQN"` + Controllers []nvmeListVerboseControllers `json:"Controllers"` +} + +type nvmeListVerboseDevices struct { + Devices []nvmeListVerboseDevice `json:"Devices"` +} + func (r *NnfNodeStorageReconciler) createStoragePool(ss nnf.StorageServiceApi, id string, capacity int64) (*sf.StoragePoolV150StoragePool, error) { + sp := &sf.StoragePoolV150StoragePool{ Id: id, CapacityBytes: capacity, @@ -748,6 +770,8 @@ func (r *NnfNodeStorageReconciler) getEndpoint(ss nnf.StorageServiceApi, id stri } func (r *NnfNodeStorageReconciler) createStorageGroup(ss nnf.StorageServiceApi, id string, spID string, epID string) (*sf.StorageGroupV150StorageGroup, error) { + log := r.Log.WithValues("Richerson", spID) + sp, err := r.getStoragePool(ss, spID) if err != nil { return nil, err @@ -770,6 +794,61 @@ func (r *NnfNodeStorageReconciler) createStorageGroup(ss nnf.StorageServiceApi, return nil, err } + _, found := os.LookupEnv("NNF_TEST_ENVIRONMENT") + if found || os.Getenv("ENVIRONMENT") == "kind" { + return sg, nil + } + + vc := &sf.VolumeCollectionVolumeCollection{} + if err := ss.StorageServiceIdStoragePoolIdCapacitySourceIdProvidingVolumesGet(ss.Id(), spID, "0", vc); err != nil { + return nil, err + } + + data, err := exec.Command("bash", "-c", "nvme list -v --output-format=json").Output() + if err != nil { + return nil, err + } + + devices := nvmeListVerboseDevices{} + if err := json.Unmarshal(data, &devices); err != nil { + return nil, err + } + + nvmeSS := nnfnvme.NewDefaultStorageService() + + for _, member := range vc.Members { + components := strings.Split(member.OdataId, "/") + storageId := components[4] + volumeId := components[6] + + storage := &sf.StorageV190Storage{} + if err := nvmeSS.StorageIdGet(storageId, storage); err != nil { + return nil, err + } + + volume := &sf.VolumeV161Volume{} + if err := nvmeSS.StorageIdVolumeIdGet(storageId, volumeId, volume); err != nil { + return nil, err + } + + for _, device := range devices.Devices { + if strings.Replace(storage.Identifiers[0].DurableName, "\u0000", "", -1) != device.SubsystemNQN { + continue + } + log.Info("found NQN match") + for _, controller := range device.Controllers { + for _, namespace := range controller.Namespaces { + if fmt.Sprintf("%v", namespace.NSID) == volume.NVMeNamespaceProperties.NamespaceId { + log.Info("Device", "Storage", storageId, "volume", volumeId, "path", "/dev/"+namespace.Device) + } + } + } + } + + //log.Info("volume", "index", i, "device") + } + + log.Info("Richerson", "devices", devices) return sg, nil }