Merge branch 'master' into release-v0.1.2

Signed-off-by: Dean Roehrich <[email protected]>

Dockerfile

@@ -15,7 +15,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-FROM golang:1.19 as builder
+FROM golang:1.21 AS builder
 
 WORKDIR /workspace
 
@@ -38,12 +38,12 @@ ENTRYPOINT ["sh", "runContainerTest.sh"]
 
 # Setup Static Analysis
 # TODO: These should move to pre-commit check
-FROM builder as codestyle
+FROM builder AS codestyle
 COPY static-analysis/docker_codestyle_entry.sh static-analysis/docker_codestyle_entry.sh
 ENTRYPOINT ["sh", "static-analysis/docker_codestyle_entry.sh"]
 
 # the static-analysis-lint-container
-FROM builder as lint
+FROM builder AS lint
 COPY static-analysis/docker_lint_entry.sh static-analysis/docker_lint_entry.sh
 ENTRYPOINT ["sh", "static-analysis/docker_lint_entry.sh"]
 

pkg/controller.go

@@ -1,5 +1,5 @@
 /*
- * Copyright 2020, 2021, 2022 Hewlett Packard Enterprise Development LP
+ * Copyright 2020-2024 Hewlett Packard Enterprise Development LP
  * Other additional copyright holders may be indicated within.
  *
  * The entirety of this work is licensed under the Apache License,
@@ -52,12 +52,17 @@ const (
 )
 
 type Options struct {
-	mock              bool   // Enable mock interfaces for Switches, NVMe, and NNF
-	cli               bool   // Enable CLI commands instead of binary
-	persistence       bool   // Enable persistent object storage; used during crash/reboot recovery
-	json              string // Initialize the element controller with the provided json file
-	direct            string // Enable direct management of NVMe devices matching this regexp pattern
-	InitializeAndExit bool   // Initialize all controllers then exit without starting the http server (mfg use)
+	mock                 bool   // Enable mock interfaces for Switches, NVMe, and NNF
+	cli                  bool   // Enable CLI commands instead of binary
+	persistence          bool   // Enable persistent object storage; used during crash/reboot recovery
+	json                 string // Initialize the element controller with the provided json file
+	direct               string // Enable direct management of NVMe devices matching this regexp pattern
+	InitializeAndExit    bool   // Initialize all controllers then exit without starting the http server (mfg use)
+	deleteUnknownVolumes bool   // Delete volumes not represented by a storage pool at the end of initialization
+}
+
+func (o *Options) DeleteUnknownVolumes() bool {
+	return o.deleteUnknownVolumes
 }
 
 func newDefaultOptions() *Options {
@@ -77,6 +82,7 @@ func BindFlags(fs *flag.FlagSet) *Options {
 	fs.StringVar(&opts.json, "json", "", "Initialize database with provided json file")
 	fs.StringVar(&opts.direct, "direct", opts.direct, "Enable direct management of NVMe block devices matching this regexp pattern. Implies Mock.")
 	fs.BoolVar(&opts.InitializeAndExit, "initializeAndExit", opts.InitializeAndExit, "Initialize all hardware controllers, then exit without starting the http server. Useful in hardware bringup")
+	fs.BoolVar(&opts.deleteUnknownVolumes, "deleteUnknownVolumes", opts.deleteUnknownVolumes, "Delete volumes not represented by storage pools")
 
 	nvme.BindFlags(fs)
 
@@ -117,16 +123,16 @@ func NewController(opts *Options) *ec.Controller {
 		persistent.StorageProvider = persistent.NewJsonFilePersistentStorageProvider(opts.json)
 	}
 
-	return ec.NewController(Name, Port, Version, NewDefaultApiRouters(switchCtrl, nvmeCtrl, nnfCtrl))
+	return ec.NewController(Name, Port, Version, NewDefaultApiRouters(switchCtrl, nvmeCtrl, nnfCtrl, opts.deleteUnknownVolumes))
 }
 
 // NewDefaultApiRouters -
-func NewDefaultApiRouters(switchCtrl fabric.SwitchtecControllerInterface, nvmeCtrl nvme.NvmeController, nnfCtrl nnf.NnfControllerInterface) ec.Routers {
+func NewDefaultApiRouters(switchCtrl fabric.SwitchtecControllerInterface, nvmeCtrl nvme.NvmeController, nnfCtrl nnf.NnfControllerInterface, nnfUnknownVolumes bool) ec.Routers {
 
 	routers := []ec.Router{
 		fabric.NewDefaultApiRouter(fabric.NewDefaultApiService(), switchCtrl),
 		nvme.NewDefaultApiRouter(nvme.NewDefaultApiService(), nvmeCtrl),
-		nnf.NewDefaultApiRouter(nnf.NewDefaultApiService(nnf.NewDefaultStorageService()), nnfCtrl),
+		nnf.NewDefaultApiRouter(nnf.NewDefaultApiService(nnf.NewDefaultStorageService(nnfUnknownVolumes)), nnfCtrl),
 		telemetry.NewDefaultApiRouter(telemetry.NewDefaultApiService()),
 		event.NewDefaultApiRouter(event.NewDefaultApiService()),
 		msgreg.NewDefaultApiRouter(msgreg.NewDefaultApiService()),

pkg/ec/ec.go

@@ -1,5 +1,5 @@
 /*
- * Copyright 2020, 2021, 2022 Hewlett Packard Enterprise Development LP
+ * Copyright 2020-2024 Hewlett Packard Enterprise Development LP
  * Other additional copyright holders may be indicated within.
  *
  * The entirety of this work is licensed under the Apache License,
@@ -109,7 +109,7 @@ type Options struct {
 }
 
 func NewDefaultOptions() *Options {
-	return &Options{Http: true, Port: 8080, Log: false, Verbose: false}
+	return &Options{Http: false, Port: 8080, Log: false, Verbose: false}
 }
 
 func NewDefaultTestOptions() *Options {
@@ -322,7 +322,7 @@ func (c *Controller) Init(opts *Options) error {
 
 // Run - Run a controller with standard behavior - that is with GRPC server and
 // request handling that operates by unpacking the GRPC request and
-// forwardining it to the element controller's handlers.
+// forwarding it to the element controller's handlers.
 func (c *Controller) Run() error {
 	if c.processor == nil {
 		return fmt.Errorf("controller processor uninitialized")

pkg/manager-nnf/allocation_policy.go

@@ -1,5 +1,5 @@
 /*
- * Copyright 2020, 2021, 2022 Hewlett Packard Enterprise Development LP
+ * Copyright 2020-2024 Hewlett Packard Enterprise Development LP
  * Other additional copyright holders may be indicated within.
  *
  * The entirety of this work is licensed under the Apache License,
@@ -210,7 +210,7 @@ func (p *SpareAllocationPolicy) Allocate(pid uuid.UUID) ([]nvme.ProvidingVolume,
 			return volumes, fmt.Errorf("Create Volume Failure: %s", err)
 		}
 
-		remainingCapacityBytes = remainingCapacityBytes - volume.GetCapaityBytes()
+		remainingCapacityBytes = remainingCapacityBytes - volume.GetCapacityBytes()
 		volumes = append(volumes, nvme.ProvidingVolume{Storage: storage, VolumeId: volume.Id()})
 	}
 

pkg/manager-nnf/manager.go

@@ -1,5 +1,5 @@
 /*
- * Copyright 2020, 2021, 2022 Hewlett Packard Enterprise Development LP
+ * Copyright 2020-2024 Hewlett Packard Enterprise Development LP
  * Other additional copyright holders may be indicated within.
  *
  * The entirety of this work is licensed under the Apache License,
@@ -55,7 +55,8 @@ var storageService = StorageService{
 	health: sf.CRITICAL_RH,
 }
 
-func NewDefaultStorageService() StorageServiceApi {
+func NewDefaultStorageService(unknownVolumes bool) StorageServiceApi {
+	storageService.deleteUnknownVolumes = unknownVolumes
 	return NewAerService(&storageService) // Wrap the default storage service with Advanced Error Reporting capabilities
 }
 
@@ -76,9 +77,12 @@ type StorageService struct {
 
 	// Index of the Id field of any Storage Service resource (Pools, Groups, Endpoints, FileSystems)
 	// That is, given a Storage Service resource OdataId field, ResourceIndex will correspond to the
-	// index within the OdataId splity by "/" i.e.     strings.split(OdataId, "/")[ResourceIndex]
+	// index within the OdataId split by "/" i.e.     strings.split(OdataId, "/")[ResourceIndex]
 	resourceIndex int
 
+	// This flag controls whether we delete volumes that don't appear in storage pools we know about.
+	deleteUnknownVolumes bool
+
 	log ec.Logger
 }
 
@@ -528,8 +532,8 @@ func (s *StorageService) EventHandler(e event.Event) error {
 		return nil
 	}
 
-	// Check if the fabric is ready; that is all devices are enumerated and discovery
-	// is complete.
+	// Check if the fabric is ready;
+	// that is all devices are enumerated and discovery is complete.
 	if e.Is(msgreg.FabricReadyNnf("")) {
 		log.V(1).Info("Fabric ready")
 
@@ -539,8 +543,10 @@ func (s *StorageService) EventHandler(e event.Event) error {
 		}
 
 		// Remove any namespaces that are not part of a Storage Pool
-		log.V(2).Info("Cleanup obsolete volumes")
-		s.cleanupVolumes()
+		if s.deleteUnknownVolumes {
+			log.V(2).Info("Cleanup unknown volumes")
+			s.cleanupVolumes()
+		}
 
 		s.state = sf.ENABLED_RST
 		s.health = sf.OK_RH

pkg/manager-nnf/storage_pool.go

@@ -1,5 +1,5 @@
 /*
- * Copyright 2020, 2021, 2022 Hewlett Packard Enterprise Development LP
+ * Copyright 2020-2024 Hewlett Packard Enterprise Development LP
  * Other additional copyright holders may be indicated within.
  *
  * The entirety of this work is licensed under the Apache License,
@@ -55,7 +55,7 @@ type AllocatedVolume struct {
 
 func (p *StoragePool) GetCapacityBytes() (capacityBytes uint64) {
 	for _, pv := range p.providingVolumes {
-		capacityBytes += pv.Storage.FindVolume(pv.VolumeId).GetCapaityBytes()
+		capacityBytes += pv.Storage.FindVolume(pv.VolumeId).GetCapacityBytes()
 	}
 	return capacityBytes
 }

pkg/manager-nvme/manager.go

@@ -1,5 +1,5 @@
 /*
- * Copyright 2020, 2021, 2022 Hewlett Packard Enterprise Development LP
+ * Copyright 2020-2024 Hewlett Packard Enterprise Development LP
  * Other additional copyright holders may be indicated within.
  *
  * The entirety of this work is licensed under the Apache License,
@@ -82,7 +82,7 @@ type Manager struct {
 
 	// Command-Line Options
 	purge       bool // Purge existing namespaces on storage controllers
-	purgeMockDb bool // Purge the persistent mock databse
+	purgeMockDb bool // Purge the persistent mock database
 
 	log ec.Logger
 }
@@ -635,7 +635,7 @@ func (s *Storage) findVolume(volumeId string) *Volume {
 
 func (v *Volume) Id() string                               { return v.id }
 func (v *Volume) GetOdataId() string                       { return v.storage.OdataId() + "/Volumes/" + v.id }
-func (v *Volume) GetCapaityBytes() uint64                  { return uint64(v.capacityBytes) }
+func (v *Volume) GetCapacityBytes() uint64                 { return uint64(v.capacityBytes) }
 func (v *Volume) GetNamespaceId() nvme.NamespaceIdentifier { return v.namespaceId }
 
 func (v *Volume) GetGloballyUniqueIdentifier() nvme.NamespaceGloballyUniqueIdentifier {

pkg/tests/benchmarks/benchmark_test.go

@@ -39,7 +39,7 @@ func BenchmarkStorage(b *testing.B) {
 		b.Fatalf("Failed to start nnf controller")
 	}
 
-	ss := nnf.NewDefaultStorageService()
+	ss := nnf.NewDefaultStorageService(true /* deleteUnknownVolumes */)
 	b.ResetTimer()
 
 	pools := make([]*sf.StoragePoolV150StoragePool, 0)

pkg/tests/filesystem/file_system_test.go

@@ -43,7 +43,7 @@ func TestFileSystem(t *testing.T) {
 	server.FileSystemRegistry.RegisterFileSystem(testFs)
 	// TODO: defer server.FileSystemRegistry.UnregisterFileSystem(testFs)
 
-	ss := nnf.NewDefaultStorageService()
+	ss := nnf.NewDefaultStorageService(true /* deleteUnknownVolumes */)
 
 	sp := &sf.StoragePoolV150StoragePool{
 		CapacityBytes: 1024 * 1024,

pkg/tests/nnf_test.go

@@ -39,7 +39,7 @@ func TestStoragePools(t *testing.T) {
 		t.Fatalf("Failed to start nnf controller")
 	}
 
-	ss := nnf.NewDefaultStorageService()
+	ss := nnf.NewDefaultStorageService(true /* deleteUnknownVolumes */)
 
 	cs := &sf.CapacityCapacitySource{}
 	if err := ss.StorageServiceIdCapacitySourceGet(ss.Id(), cs); err != nil {

pkg/tests/recovery/recovery_test.go

@@ -83,7 +83,7 @@ var _ = Describe("Reboot Recovery Testing", func() {
 				c = nnfec.NewController(nnfec.NewMockOptions(true))
 				Expect(c.Init(ec.NewDefaultOptions())).NotTo(HaveOccurred())
 
-				ss = nnf.NewDefaultStorageService()
+				ss = nnf.NewDefaultStorageService(true /* deleteUnknownVolumes */)
 			})
 
 			// After each test we close the NNF Element Controller, thereby safely closing