Merge release v0.1.10

Release v0.1.10

config/manager/kustomization.yaml

@@ -17,7 +17,7 @@ kind: Kustomization
 images:
 - name: controller
   newName: ghcr.io/nearnodeflash/nnf-dm
-  newTag: 0.1.9
+  newTag: 0.1.10
 - name: nnf-mfu
   newName: ghcr.io/nearnodeflash/nnf-mfu
   newTag: 0.1.2

go.mod

@@ -3,8 +3,8 @@ module github.com/NearNodeFlash/nnf-dm
 go 1.21
 
 require (
-	github.com/NearNodeFlash/lustre-fs-operator v0.0.1-0.20240913195900-b3285e54755e
-	github.com/NearNodeFlash/nnf-sos v0.0.1-0.20240913201015-2fe131e04231
+	github.com/NearNodeFlash/lustre-fs-operator v0.0.1-0.20240925185149-26d9d6071a1c
+	github.com/NearNodeFlash/nnf-sos v0.0.1-0.20240925191626-b418b8044027
 	github.com/onsi/ginkgo/v2 v2.17.1
 	github.com/onsi/gomega v1.32.0
 	github.com/prometheus/client_golang v1.16.0
@@ -73,7 +73,7 @@ require (
 )
 
 require (
-	github.com/DataWorkflowServices/dws v0.0.1-0.20240913193141-737bcd946a02
+	github.com/DataWorkflowServices/dws v0.0.1-0.20240916174522-9062a91241cd
 	go.openly.dev/pointy v1.3.0
 )
 

go.sum

@@ -1,11 +1,11 @@
-github.com/DataWorkflowServices/dws v0.0.1-0.20240913193141-737bcd946a02 h1:ZidnleTEjdZX6geLAWTw3NMEsO8YLEQ9ubPx1fKGDeg=
-github.com/DataWorkflowServices/dws v0.0.1-0.20240913193141-737bcd946a02/go.mod h1:6MrEEHISskyooSKcKU6R3mFqH6Yh6KzWgajhcw2s+nM=
-github.com/NearNodeFlash/lustre-fs-operator v0.0.1-0.20240913195900-b3285e54755e h1:fvJHwXbEDss0m642zVe3Hra0y/zh49Vnhqk27I3HrRc=
-github.com/NearNodeFlash/lustre-fs-operator v0.0.1-0.20240913195900-b3285e54755e/go.mod h1:VP4+66Hv5XnaJgzaQ0fooB4r4141aM5saoo8nCZQTGA=
+github.com/DataWorkflowServices/dws v0.0.1-0.20240916174522-9062a91241cd h1:/bSL8rV9cB13b3I5SCf20t4q3ab91i533sPcYIISI+8=
+github.com/DataWorkflowServices/dws v0.0.1-0.20240916174522-9062a91241cd/go.mod h1:6MrEEHISskyooSKcKU6R3mFqH6Yh6KzWgajhcw2s+nM=
+github.com/NearNodeFlash/lustre-fs-operator v0.0.1-0.20240925185149-26d9d6071a1c h1:fSuMz3j8UzlYZI59Ded8XuUjYd7C5IyLB55jwgSTIew=
+github.com/NearNodeFlash/lustre-fs-operator v0.0.1-0.20240925185149-26d9d6071a1c/go.mod h1:3wENUqk1b7V0q5L5kNQ2ZE3z/NywL4sqXqVYolsiJ94=
 github.com/NearNodeFlash/nnf-ec v0.0.1-0.20240912200758-f862bc773739 h1:T49ixk8TBeiYQ3CPpx463GL2gKk94dD9nXpoLBfdmqg=
 github.com/NearNodeFlash/nnf-ec v0.0.1-0.20240912200758-f862bc773739/go.mod h1:oxdwMqfttOF9dabJhqrWlirCnMk8/8eyLMwl+hducjk=
-github.com/NearNodeFlash/nnf-sos v0.0.1-0.20240913201015-2fe131e04231 h1:Eg5yY1q99Jna7inXKrE7eaoSYffTOqkK9J1fkAA7GGo=
-github.com/NearNodeFlash/nnf-sos v0.0.1-0.20240913201015-2fe131e04231/go.mod h1:5PFN2heAF9W2RUfKpYQwhcsYIxjhxJFA02DVYmMsfJk=
+github.com/NearNodeFlash/nnf-sos v0.0.1-0.20240925191626-b418b8044027 h1:5mdG1KNxPdN7TRpZBk11TBEigA2HIkgK/9neruqOZSs=
+github.com/NearNodeFlash/nnf-sos v0.0.1-0.20240925191626-b418b8044027/go.mod h1:dZD9Kx+kHdKCig6My2uA3JNU0Hy8Sz7vb4A+RZel3m4=
 github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA=
 github.com/benbjohnson/clock v1.3.0 h1:ip6w0uFQkncKQ979AypyG0ER7mqUSBdKLOgAle/AT8A=
 github.com/benbjohnson/clock v1.3.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA=

internal/controller/datamovement_controller.go

@@ -613,39 +613,42 @@ func buildStatCommand(uid, gid uint32, cmd, hostfile, path string) string {
 }
 
 func (r *DataMovementReconciler) prepareDestination(ctx context.Context, profile *nnfv1alpha2.NnfDataMovementProfile, dm *nnfv1alpha2.NnfDataMovement, mpiHostfile string, log logr.Logger) error {
-	// These functions interact with the filesystem, so they can't run in the test env
-	if !isTestEnv() {
-		// Determine the destination directory based on the source and path
-		log.Info("Determining destination directory based on source/dest file types")
-		destDir, err := getDestinationDir(profile, dm, mpiHostfile, log)
-		if err != nil {
-			return dwsv1alpha2.NewResourceError("could not determine source type").WithError(err).WithFatal()
-		}
+	// These functions interact with the filesystem, so they can't run in the test env.  Also, if
+	// the profile disables destination creation, skip it
+	if isTestEnv() || !profile.Data.CreateDestDir {
+		return nil
+	}
 
-		// See if an index mount directory on the destination is required
-		log.Info("Determining if index mount directory is required")
-		indexMount, err := r.checkIndexMountDir(ctx, dm)
-		if err != nil {
-			return dwsv1alpha2.NewResourceError("could not determine index mount directory").WithError(err).WithFatal()
-		}
+	// Determine the destination directory based on the source and path
+	log.Info("Determining destination directory based on source/dest file types")
+	destDir, err := getDestinationDir(profile, dm, mpiHostfile, log)
+	if err != nil {
+		return dwsv1alpha2.NewResourceError("could not determine source type").WithError(err).WithFatal()
+	}
 
-		// Account for index mount directory on the destDir and the dm dest path
-		// This updates the destination on dm
-		if indexMount != "" {
-			log.Info("Index mount directory is required", "indexMountdir", indexMount)
-			d, err := handleIndexMountDir(profile, dm, destDir, indexMount, mpiHostfile, log)
-			if err != nil {
-				return dwsv1alpha2.NewResourceError("could not handle index mount directory").WithError(err).WithFatal()
-			}
-			destDir = d
-			log.Info("Updated destination for index mount directory", "destDir", destDir, "dm.Spec.Destination.Path", dm.Spec.Destination.Path)
-		}
+	// See if an index mount directory on the destination is required
+	log.Info("Determining if index mount directory is required")
+	indexMount, err := r.checkIndexMountDir(ctx, dm)
+	if err != nil {
+		return dwsv1alpha2.NewResourceError("could not determine index mount directory").WithError(err).WithFatal()
+	}
 
-		// Create the destination directory
-		log.Info("Creating destination directory", "destinationDir", destDir, "indexMountDir", indexMount)
-		if err := createDestinationDir(profile, dm, destDir, mpiHostfile, log); err != nil {
-			return dwsv1alpha2.NewResourceError("could not create destination directory").WithError(err).WithFatal()
+	// Account for index mount directory on the destDir and the dm dest path
+	// This updates the destination on dm
+	if indexMount != "" {
+		log.Info("Index mount directory is required", "indexMountdir", indexMount)
+		d, err := handleIndexMountDir(profile, dm, destDir, indexMount, mpiHostfile, log)
+		if err != nil {
+			return dwsv1alpha2.NewResourceError("could not handle index mount directory").WithError(err).WithFatal()
 		}
+		destDir = d
+		log.Info("Updated destination for index mount directory", "destDir", destDir, "dm.Spec.Destination.Path", dm.Spec.Destination.Path)
+	}
+
+	// Create the destination directory
+	log.Info("Creating destination directory", "destinationDir", destDir, "indexMountDir", indexMount)
+	if err := createDestinationDir(profile, dm, destDir, mpiHostfile, log); err != nil {
+		return dwsv1alpha2.NewResourceError("could not create destination directory").WithError(err).WithFatal()
 	}
 
 	log.Info("Destination prepared", "dm.Spec.Destination", dm.Spec.Destination)

vendor/modules.txt

@@ -1,16 +1,16 @@
-# github.com/DataWorkflowServices/dws v0.0.1-0.20240913193141-737bcd946a02
+# github.com/DataWorkflowServices/dws v0.0.1-0.20240916174522-9062a91241cd
 ## explicit; go 1.21
 github.com/DataWorkflowServices/dws/api/v1alpha2
 github.com/DataWorkflowServices/dws/utils/dwdparse
 github.com/DataWorkflowServices/dws/utils/updater
-# github.com/NearNodeFlash/lustre-fs-operator v0.0.1-0.20240913195900-b3285e54755e
+# github.com/NearNodeFlash/lustre-fs-operator v0.0.1-0.20240925185149-26d9d6071a1c
 ## explicit; go 1.21
 github.com/NearNodeFlash/lustre-fs-operator/api/v1beta1
 github.com/NearNodeFlash/lustre-fs-operator/config/crd/bases
 # github.com/NearNodeFlash/nnf-ec v0.0.1-0.20240912200758-f862bc773739
 ## explicit; go 1.19
 github.com/NearNodeFlash/nnf-ec/pkg/rfsf/pkg/models
-# github.com/NearNodeFlash/nnf-sos v0.0.1-0.20240913201015-2fe131e04231
+# github.com/NearNodeFlash/nnf-sos v0.0.1-0.20240925191626-b418b8044027
 ## explicit; go 1.21
 github.com/NearNodeFlash/nnf-sos/api/v1alpha2
 github.com/NearNodeFlash/nnf-sos/config/crd/bases