From d53f62375662b56ff65f4d05ed93cc110e23cbbb Mon Sep 17 00:00:00 2001 From: Blake Devcich Date: Tue, 24 Sep 2024 12:10:47 -0500 Subject: [PATCH 1/3] Fix createDestDir regression Skipping `prepareDestination` with `CreateDestDir:false` got dropped in a previous change. This puts it back. Signed-off-by: Blake Devcich --- .../controller/datamovement_controller.go | 61 ++++++++++--------- 1 file changed, 32 insertions(+), 29 deletions(-) diff --git a/internal/controller/datamovement_controller.go b/internal/controller/datamovement_controller.go index 2a27909e..771883bf 100644 --- a/internal/controller/datamovement_controller.go +++ b/internal/controller/datamovement_controller.go @@ -613,39 +613,42 @@ func buildStatCommand(uid, gid uint32, cmd, hostfile, path string) string { } func (r *DataMovementReconciler) prepareDestination(ctx context.Context, profile *nnfv1alpha2.NnfDataMovementProfile, dm *nnfv1alpha2.NnfDataMovement, mpiHostfile string, log logr.Logger) error { - // These functions interact with the filesystem, so they can't run in the test env - if !isTestEnv() { - // Determine the destination directory based on the source and path - log.Info("Determining destination directory based on source/dest file types") - destDir, err := getDestinationDir(profile, dm, mpiHostfile, log) - if err != nil { - return dwsv1alpha2.NewResourceError("could not determine source type").WithError(err).WithFatal() - } + // These functions interact with the filesystem, so they can't run in the test env. Also, if + // the profile disables destination creation, skip it + if isTestEnv() || !profile.Data.CreateDestDir { + return nil + } - // See if an index mount directory on the destination is required - log.Info("Determining if index mount directory is required") - indexMount, err := r.checkIndexMountDir(ctx, dm) - if err != nil { - return dwsv1alpha2.NewResourceError("could not determine index mount directory").WithError(err).WithFatal() - } + // Determine the destination directory based on the source and path + log.Info("Determining destination directory based on source/dest file types") + destDir, err := getDestinationDir(profile, dm, mpiHostfile, log) + if err != nil { + return dwsv1alpha2.NewResourceError("could not determine source type").WithError(err).WithFatal() + } - // Account for index mount directory on the destDir and the dm dest path - // This updates the destination on dm - if indexMount != "" { - log.Info("Index mount directory is required", "indexMountdir", indexMount) - d, err := handleIndexMountDir(profile, dm, destDir, indexMount, mpiHostfile, log) - if err != nil { - return dwsv1alpha2.NewResourceError("could not handle index mount directory").WithError(err).WithFatal() - } - destDir = d - log.Info("Updated destination for index mount directory", "destDir", destDir, "dm.Spec.Destination.Path", dm.Spec.Destination.Path) - } + // See if an index mount directory on the destination is required + log.Info("Determining if index mount directory is required") + indexMount, err := r.checkIndexMountDir(ctx, dm) + if err != nil { + return dwsv1alpha2.NewResourceError("could not determine index mount directory").WithError(err).WithFatal() + } - // Create the destination directory - log.Info("Creating destination directory", "destinationDir", destDir, "indexMountDir", indexMount) - if err := createDestinationDir(profile, dm, destDir, mpiHostfile, log); err != nil { - return dwsv1alpha2.NewResourceError("could not create destination directory").WithError(err).WithFatal() + // Account for index mount directory on the destDir and the dm dest path + // This updates the destination on dm + if indexMount != "" { + log.Info("Index mount directory is required", "indexMountdir", indexMount) + d, err := handleIndexMountDir(profile, dm, destDir, indexMount, mpiHostfile, log) + if err != nil { + return dwsv1alpha2.NewResourceError("could not handle index mount directory").WithError(err).WithFatal() } + destDir = d + log.Info("Updated destination for index mount directory", "destDir", destDir, "dm.Spec.Destination.Path", dm.Spec.Destination.Path) + } + + // Create the destination directory + log.Info("Creating destination directory", "destinationDir", destDir, "indexMountDir", indexMount) + if err := createDestinationDir(profile, dm, destDir, mpiHostfile, log); err != nil { + return dwsv1alpha2.NewResourceError("could not create destination directory").WithError(err).WithFatal() } log.Info("Destination prepared", "dm.Spec.Destination", dm.Spec.Destination) From 9946da12aeb2aff16fb4c3caaaf05d36d45efb34 Mon Sep 17 00:00:00 2001 From: Blake Devcich Date: Wed, 25 Sep 2024 14:20:03 -0500 Subject: [PATCH 2/3] Update submodules Signed-off-by: Blake Devcich --- go.mod | 6 +++--- go.sum | 12 ++++++------ vendor/modules.txt | 6 +++--- 3 files changed, 12 insertions(+), 12 deletions(-) diff --git a/go.mod b/go.mod index bffb0b85..8c26c82a 100644 --- a/go.mod +++ b/go.mod @@ -3,8 +3,8 @@ module github.com/NearNodeFlash/nnf-dm go 1.21 require ( - github.com/NearNodeFlash/lustre-fs-operator v0.0.1-0.20240913195900-b3285e54755e - github.com/NearNodeFlash/nnf-sos v0.0.1-0.20240913201015-2fe131e04231 + github.com/NearNodeFlash/lustre-fs-operator v0.0.1-0.20240925185149-26d9d6071a1c + github.com/NearNodeFlash/nnf-sos v0.0.1-0.20240925191626-b418b8044027 github.com/onsi/ginkgo/v2 v2.17.1 github.com/onsi/gomega v1.32.0 github.com/prometheus/client_golang v1.16.0 @@ -73,7 +73,7 @@ require ( ) require ( - github.com/DataWorkflowServices/dws v0.0.1-0.20240913193141-737bcd946a02 + github.com/DataWorkflowServices/dws v0.0.1-0.20240916174522-9062a91241cd go.openly.dev/pointy v1.3.0 ) diff --git a/go.sum b/go.sum index 07e5d909..8cba75a7 100644 --- a/go.sum +++ b/go.sum @@ -1,11 +1,11 @@ -github.com/DataWorkflowServices/dws v0.0.1-0.20240913193141-737bcd946a02 h1:ZidnleTEjdZX6geLAWTw3NMEsO8YLEQ9ubPx1fKGDeg= -github.com/DataWorkflowServices/dws v0.0.1-0.20240913193141-737bcd946a02/go.mod h1:6MrEEHISskyooSKcKU6R3mFqH6Yh6KzWgajhcw2s+nM= -github.com/NearNodeFlash/lustre-fs-operator v0.0.1-0.20240913195900-b3285e54755e h1:fvJHwXbEDss0m642zVe3Hra0y/zh49Vnhqk27I3HrRc= -github.com/NearNodeFlash/lustre-fs-operator v0.0.1-0.20240913195900-b3285e54755e/go.mod h1:VP4+66Hv5XnaJgzaQ0fooB4r4141aM5saoo8nCZQTGA= +github.com/DataWorkflowServices/dws v0.0.1-0.20240916174522-9062a91241cd h1:/bSL8rV9cB13b3I5SCf20t4q3ab91i533sPcYIISI+8= +github.com/DataWorkflowServices/dws v0.0.1-0.20240916174522-9062a91241cd/go.mod h1:6MrEEHISskyooSKcKU6R3mFqH6Yh6KzWgajhcw2s+nM= +github.com/NearNodeFlash/lustre-fs-operator v0.0.1-0.20240925185149-26d9d6071a1c h1:fSuMz3j8UzlYZI59Ded8XuUjYd7C5IyLB55jwgSTIew= +github.com/NearNodeFlash/lustre-fs-operator v0.0.1-0.20240925185149-26d9d6071a1c/go.mod h1:3wENUqk1b7V0q5L5kNQ2ZE3z/NywL4sqXqVYolsiJ94= github.com/NearNodeFlash/nnf-ec v0.0.1-0.20240912200758-f862bc773739 h1:T49ixk8TBeiYQ3CPpx463GL2gKk94dD9nXpoLBfdmqg= github.com/NearNodeFlash/nnf-ec v0.0.1-0.20240912200758-f862bc773739/go.mod h1:oxdwMqfttOF9dabJhqrWlirCnMk8/8eyLMwl+hducjk= -github.com/NearNodeFlash/nnf-sos v0.0.1-0.20240913201015-2fe131e04231 h1:Eg5yY1q99Jna7inXKrE7eaoSYffTOqkK9J1fkAA7GGo= -github.com/NearNodeFlash/nnf-sos v0.0.1-0.20240913201015-2fe131e04231/go.mod h1:5PFN2heAF9W2RUfKpYQwhcsYIxjhxJFA02DVYmMsfJk= +github.com/NearNodeFlash/nnf-sos v0.0.1-0.20240925191626-b418b8044027 h1:5mdG1KNxPdN7TRpZBk11TBEigA2HIkgK/9neruqOZSs= +github.com/NearNodeFlash/nnf-sos v0.0.1-0.20240925191626-b418b8044027/go.mod h1:dZD9Kx+kHdKCig6My2uA3JNU0Hy8Sz7vb4A+RZel3m4= github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= github.com/benbjohnson/clock v1.3.0 h1:ip6w0uFQkncKQ979AypyG0ER7mqUSBdKLOgAle/AT8A= github.com/benbjohnson/clock v1.3.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= diff --git a/vendor/modules.txt b/vendor/modules.txt index 4f6a5882..a2c4a037 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -1,16 +1,16 @@ -# github.com/DataWorkflowServices/dws v0.0.1-0.20240913193141-737bcd946a02 +# github.com/DataWorkflowServices/dws v0.0.1-0.20240916174522-9062a91241cd ## explicit; go 1.21 github.com/DataWorkflowServices/dws/api/v1alpha2 github.com/DataWorkflowServices/dws/utils/dwdparse github.com/DataWorkflowServices/dws/utils/updater -# github.com/NearNodeFlash/lustre-fs-operator v0.0.1-0.20240913195900-b3285e54755e +# github.com/NearNodeFlash/lustre-fs-operator v0.0.1-0.20240925185149-26d9d6071a1c ## explicit; go 1.21 github.com/NearNodeFlash/lustre-fs-operator/api/v1beta1 github.com/NearNodeFlash/lustre-fs-operator/config/crd/bases # github.com/NearNodeFlash/nnf-ec v0.0.1-0.20240912200758-f862bc773739 ## explicit; go 1.19 github.com/NearNodeFlash/nnf-ec/pkg/rfsf/pkg/models -# github.com/NearNodeFlash/nnf-sos v0.0.1-0.20240913201015-2fe131e04231 +# github.com/NearNodeFlash/nnf-sos v0.0.1-0.20240925191626-b418b8044027 ## explicit; go 1.21 github.com/NearNodeFlash/nnf-sos/api/v1alpha2 github.com/NearNodeFlash/nnf-sos/config/crd/bases From 0db1e8e3293c03fe43516b17a80c982e2915c1e6 Mon Sep 17 00:00:00 2001 From: Blake Devcich Date: Wed, 25 Sep 2024 15:44:08 -0500 Subject: [PATCH 3/3] Update own release references Signed-off-by: Blake Devcich --- config/manager/kustomization.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/manager/kustomization.yaml b/config/manager/kustomization.yaml index 07dd5522..54e90d82 100644 --- a/config/manager/kustomization.yaml +++ b/config/manager/kustomization.yaml @@ -17,7 +17,7 @@ kind: Kustomization images: - name: controller newName: ghcr.io/nearnodeflash/nnf-dm - newTag: 0.1.9 + newTag: 0.1.10 - name: nnf-mfu newName: ghcr.io/nearnodeflash/nnf-mfu newTag: 0.1.2