Skip to content

Commit

Permalink
Add lustre component count to NnfStorage/NnfNodeStorage API
Browse files Browse the repository at this point in the history
  • Loading branch information
@bdevcich
bdevcich committed Dec 3, 2024
1 parent 822b70d commit 2ca478c
Show file tree
Hide file tree
Showing 12 changed files with 374 additions and 120 deletions.
21 changes: 19 additions & 2 deletions api/v1alpha2/conversion.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -343,12 +343,20 @@ func (src *NnfNodeStorage) ConvertTo(dstRaw conversion.Hub) error {

// Manually restore data.
restored := &nnfv1alpha4.NnfNodeStorage{}
if ok, err := utilconversion.UnmarshalData(src, restored); err != nil || !ok {
hasAnno, err := utilconversion.UnmarshalData(src, restored)
if err != nil {
return err
}
// EDIT THIS FUNCTION! If the annotation is holding anything that is
// hub-specific then copy it into 'dst' from 'restored'.
// Otherwise, you may comment out UnmarshalData() until it's needed.
if hasAnno {
dst.Spec.LustreStorage.LustreComponents.MDTs = append([]string(nil), restored.Spec.LustreStorage.LustreComponents.MDTs...)
dst.Spec.LustreStorage.LustreComponents.MGTs = append([]string(nil), restored.Spec.LustreStorage.LustreComponents.MGTs...)
dst.Spec.LustreStorage.LustreComponents.MGTMDTs = append([]string(nil), restored.Spec.LustreStorage.LustreComponents.MGTMDTs...)
dst.Spec.LustreStorage.LustreComponents.OSTs = append([]string(nil), restored.Spec.LustreStorage.LustreComponents.OSTs...)
dst.Spec.LustreStorage.LustreComponents.NNFNodes = append([]string(nil), restored.Spec.LustreStorage.LustreComponents.NNFNodes...)
}

return nil
}
Expand Down Expand Up @@ -407,12 +415,21 @@ func (src *NnfStorage) ConvertTo(dstRaw conversion.Hub) error {

// Manually restore data.
restored := &nnfv1alpha4.NnfStorage{}
if ok, err := utilconversion.UnmarshalData(src, restored); err != nil || !ok {
hasAnno, err := utilconversion.UnmarshalData(src, restored)
if err != nil {
return err
}

// EDIT THIS FUNCTION! If the annotation is holding anything that is
// hub-specific then copy it into 'dst' from 'restored'.
// Otherwise, you may comment out UnmarshalData() until it's needed.
if hasAnno {
dst.Status.LustreComponents.MDTs = append([]string(nil), restored.Status.LustreComponents.MDTs...)
dst.Status.LustreComponents.MGTs = append([]string(nil), restored.Status.LustreComponents.MGTs...)
dst.Status.LustreComponents.MGTMDTs = append([]string(nil), restored.Status.LustreComponents.MGTMDTs...)
dst.Status.LustreComponents.OSTs = append([]string(nil), restored.Status.LustreComponents.OSTs...)
dst.Status.LustreComponents.NNFNodes = append([]string(nil), restored.Status.LustreComponents.NNFNodes...)
}

return nil
}
Expand Down
21 changes: 19 additions & 2 deletions api/v1alpha3/conversion.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -342,12 +342,20 @@ func (src *NnfNodeStorage) ConvertTo(dstRaw conversion.Hub) error {

// Manually restore data.
restored := &nnfv1alpha4.NnfNodeStorage{}
if ok, err := utilconversion.UnmarshalData(src, restored); err != nil || !ok {
hasAnno, err := utilconversion.UnmarshalData(src, restored)
if err != nil {
return err
}
// EDIT THIS FUNCTION! If the annotation is holding anything that is
// hub-specific then copy it into 'dst' from 'restored'.
// Otherwise, you may comment out UnmarshalData() until it's needed.
if hasAnno {
dst.Spec.LustreStorage.LustreComponents.MDTs = append([]string(nil), restored.Spec.LustreStorage.LustreComponents.MDTs...)
dst.Spec.LustreStorage.LustreComponents.MGTs = append([]string(nil), restored.Spec.LustreStorage.LustreComponents.MGTs...)
dst.Spec.LustreStorage.LustreComponents.MGTMDTs = append([]string(nil), restored.Spec.LustreStorage.LustreComponents.MGTMDTs...)
dst.Spec.LustreStorage.LustreComponents.OSTs = append([]string(nil), restored.Spec.LustreStorage.LustreComponents.OSTs...)
dst.Spec.LustreStorage.LustreComponents.NNFNodes = append([]string(nil), restored.Spec.LustreStorage.LustreComponents.NNFNodes...)
}

return nil
}
Expand Down Expand Up @@ -406,12 +414,21 @@ func (src *NnfStorage) ConvertTo(dstRaw conversion.Hub) error {

// Manually restore data.
restored := &nnfv1alpha4.NnfStorage{}
if ok, err := utilconversion.UnmarshalData(src, restored); err != nil || !ok {
hasAnno, err := utilconversion.UnmarshalData(src, restored)
if err != nil {
return err
}

// EDIT THIS FUNCTION! If the annotation is holding anything that is
// hub-specific then copy it into 'dst' from 'restored'.
// Otherwise, you may comment out UnmarshalData() until it's needed.
if hasAnno {
dst.Status.LustreComponents.MDTs = append([]string(nil), restored.Status.LustreComponents.MDTs...)
dst.Status.LustreComponents.MGTs = append([]string(nil), restored.Status.LustreComponents.MGTs...)
dst.Status.LustreComponents.MGTMDTs = append([]string(nil), restored.Status.LustreComponents.MGTMDTs...)
dst.Status.LustreComponents.OSTs = append([]string(nil), restored.Status.LustreComponents.OSTs...)
dst.Status.LustreComponents.NNFNodes = append([]string(nil), restored.Status.LustreComponents.NNFNodes...)
}

return nil
}
Expand Down
3 changes: 3 additions & 0 deletions api/v1alpha4/nnfnodestorage_types.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -88,6 +88,9 @@ type LustreStorageSpec struct {
// BackFs is the type of backing filesystem to use.
// +kubebuilder:validation:Enum=ldiskfs;zfs
BackFs string `json:"backFs,omitempty"`

// Component info for the Lustre filesystem
LustreComponents NnfStorageLustreComponents `json:"lustreComponents,omitempty"`
}

// NnfNodeStorageStatus defines the status for NnfNodeStorage
Expand Down
12 changes: 12 additions & 0 deletions api/v1alpha4/nnfstorage_types.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -60,6 +60,15 @@ type NnfStorageLustreSpec struct {
PersistentMgsReference corev1.ObjectReference `json:"persistentMgsReference,omitempty"`
}

// TODO: document these
type NnfStorageLustreComponents struct {
MDTs []string `json:"mdts,omitempty"`
MGTs []string `json:"mgts,omitempty"`
MGTMDTs []string `json:"mgtmdts,omitempty"`
OSTs []string `json:"osts,omitempty"`
NNFNodes []string `json:"nnfNodes,omitempty"`
}

// NnfStorageAllocationSetSpec defines the details for an allocation set
type NnfStorageAllocationSetSpec struct {
// Name is a human readable label for this set of allocations (e.g., xfs)
Expand Down Expand Up @@ -124,6 +133,9 @@ type NnfStorageLustreStatus struct {
// LustgreMgtReference is an object reference to the NnfLustreMGT resource used
// by the NnfStorage
LustreMgtReference corev1.ObjectReference `json:"lustreMgtReference,omitempty"`

// Component info for the Lustre filesystem
LustreComponents NnfStorageLustreComponents `json:"lustreComponents,omitempty"`
}

// NnfStorageStatus defines the observed status of NNF Storage.
Expand Down
48 changes: 45 additions & 3 deletions api/v1alpha4/zz_generated.deepcopy.go
View file

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

24 changes: 24 additions & 0 deletions config/crd/bases/nnf.cray.hpe.com_nnfnodestorages.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -576,6 +576,30 @@ spec:
filesystem.
maxLength: 8
type: string
lustreComponents:
description: Component info for the Lustre filesystem
properties:
mdts:
items:
type: string
type: array
mgtmdts:
items:
type: string
type: array
mgts:
items:
type: string
type: array
nnfNodes:
items:
type: string
type: array
osts:
items:
type: string
type: array
type: object
mgsAddress:
description: |-
MgsAddress is the NID of the MGS to use. This is used only when
Expand Down
24 changes: 24 additions & 0 deletions config/crd/bases/nnf.cray.hpe.com_nnfstorages.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -841,6 +841,30 @@ spec:
filesystem.
maxLength: 8
type: string
lustreComponents:
description: Component info for the Lustre filesystem
properties:
mdts:
items:
type: string
type: array
mgtmdts:
items:
type: string
type: array
mgts:
items:
type: string
type: array
nnfNodes:
items:
type: string
type: array
osts:
items:
type: string
type: array
type: object
lustreMgtReference:
description: |-
LustgreMgtReference is an object reference to the NnfLustreMGT resource used
Expand Down
106 changes: 8 additions & 98 deletions internal/controller/filesystem_helpers.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -450,21 +450,16 @@ func newLustreFileSystem(ctx context.Context, c client.Client, nnfNodeStorage *n
fs.CommandArgs.PreDeactivate = cmdLines.PreDeactivate
fs.TempDir = fmt.Sprintf("/mnt/temp/%s-%d", nnfNodeStorage.Name, index)

log.Info("BLAKE labels - before filesystem", "labels", nnfNodeStorage.Labels)
server, err := getServerForClientMountOrNnfNodeStorage(ctx, c, nnfNodeStorage.Labels)
if err != nil {
return nil, dwsv1alpha2.NewResourceError("could not retrieve corresponding NnfServer resource for this NnfNodeStorage").WithError(err).WithMajor()
}
storages := getLustreMapping(server)
components := nnfNodeStorage.Spec.LustreStorage.LustreComponents

fs.CommandArgs.Vars = map[string]string{
"$USERID": fmt.Sprintf("%d", nnfNodeStorage.Spec.UserID),
"$GROUPID": fmt.Sprintf("%d", nnfNodeStorage.Spec.GroupID),
"$NUM_MGTS": fmt.Sprintf("%d", len(storages["mgt"])),
"$NUM_MDTS": fmt.Sprintf("%d", len(storages["mdt"])),
"$NUM_MGTMDTS": fmt.Sprintf("%d", len(storages["mgtmdt"])),
"$NUM_OSTS": fmt.Sprintf("%d", len(storages["ost"])),
"$NUM_RABBITS": fmt.Sprintf("%d", len(storages["rabbits"])),
"$USERID": fmt.Sprintf("%d", nnfNodeStorage.Spec.UserID),
"$GROUPID": fmt.Sprintf("%d", nnfNodeStorage.Spec.GroupID),
"$NUM_MDTS": fmt.Sprintf("%d", len(components.MDTs)),
"$NUM_MGTS": fmt.Sprintf("%d", len(components.MGTs)),
"$NUM_MGTMDTS": fmt.Sprintf("%d", len(components.MGTMDTs)),
"$NUM_OSTS": fmt.Sprintf("%d", len(components.OSTs)),
"$NUM_NNFNODES": fmt.Sprintf("%d", len(components.NNFNodes)),
}

log.Info("BLAKE", "Vars", fs.CommandArgs.Vars)
Expand Down Expand Up @@ -554,88 +549,3 @@ func logicalVolumeName(ctx context.Context, c client.Client, nnfNodeStorage *nnf

return "lv", nil
}

// Retrieve the ClientMount's corresponding NnfServer resource. To do this, we first need to get the corresponding NnfStorage resource. That is done by
// looking at the owner of the ClientMount resource. It should be NnfStorage. Then, we inspect the NnfStorage resource's owner. In this case, there can
// be two different owners:
//
// 1. Workflow (non-persistent storage case)
// 2. PersistentStorageInstance (persistent storage case)
//
// Once we understand who owns the NnfStorage resource, we can then obtain the NnfServer resource through slightly different methods.
func getServerForClientMountOrNnfNodeStorage(ctx context.Context, c client.Client, labels map[string]string) (*dwsv1alpha2.Servers, error) {
storageKind := "NnfStorage"
persistentKind := "PersistentStorageInstance"
workflowKind := "Workflow"

// Get the owner and directive index from ClientMount's labels
ownerKind, ownerExists := labels[dwsv1alpha2.OwnerKindLabel]
ownerName, ownerNameExists := labels[dwsv1alpha2.OwnerNameLabel]
ownerNS, ownerNSExists := labels[dwsv1alpha2.OwnerNamespaceLabel]
_, idxExists := labels[nnfv1alpha4.DirectiveIndexLabel]

// log.FromContext(ctx).Info("BLAKE labels - after", "labels", labels)

// We should expect the owner to be NnfStorage and have the expected labels
if !ownerExists || !ownerNameExists || !ownerNSExists || !idxExists || ownerKind != storageKind {
return nil, dwsv1alpha2.NewResourceError(fmt.Sprintf("expected owner to be of kind NnfStorage and have the expected labels: %v", labels)).WithMajor()
}

// Retrieve the NnfStorage resource
storage := &nnfv1alpha4.NnfStorage{
ObjectMeta: metav1.ObjectMeta{
Name: ownerName,
Namespace: ownerNS,
},
}
if err := c.Get(ctx, client.ObjectKeyFromObject(storage), storage); err != nil {
return nil, dwsv1alpha2.NewResourceError("unable retrieve NnfStorage resource").WithError(err).WithMajor()
}

// Get the owner and directive index from NnfStorage's labels
ownerKind, ownerExists = storage.Labels[dwsv1alpha2.OwnerKindLabel]
ownerName, ownerNameExists = storage.Labels[dwsv1alpha2.OwnerNameLabel]
ownerNS, ownerNSExists = storage.Labels[dwsv1alpha2.OwnerNamespaceLabel]
idx, idxExists := storage.Labels[nnfv1alpha4.DirectiveIndexLabel]

// We should expect the owner of the NnfStorage to be Workflow or PersistentStorageInstance and
// have the expected labels
if !ownerExists || !ownerNameExists || !ownerNSExists || !idxExists || (ownerKind != workflowKind && ownerKind != persistentKind) {
return nil, dwsv1alpha2.NewResourceError("expected owner to be of kind Workflow or PersistentStorageInstance and have the expected labels").WithMajor()
}

// If the owner is a workflow, then we can use the workflow labels and directive index to get
// the Servers Resource.
var listOptions []client.ListOption
if ownerKind == workflowKind {
listOptions = []client.ListOption{
client.MatchingLabels(map[string]string{
dwsv1alpha2.WorkflowNameLabel: ownerName,
dwsv1alpha2.WorkflowNamespaceLabel: ownerNS,
nnfv1alpha4.DirectiveIndexLabel: idx,
}),
}
} else {
// Otherwise the owner is a PersistentStorageInstance and we'll need to use the owner
// labels. It also will not have a directive index.
listOptions = []client.ListOption{
client.MatchingLabels(map[string]string{
dwsv1alpha2.OwnerKindLabel: ownerKind,
dwsv1alpha2.OwnerNameLabel: ownerName,
dwsv1alpha2.OwnerNamespaceLabel: ownerNS,
}),
}
}

serversList := &dwsv1alpha2.ServersList{}
if err := c.List(ctx, serversList, listOptions...); err != nil {
return nil, dwsv1alpha2.NewResourceError("unable retrieve NnfServers resource").WithError(err).WithMajor()
}

// We should only have 1
if len(serversList.Items) != 1 {
return nil, dwsv1alpha2.NewResourceError(fmt.Sprintf("wrong number of NnfServers resources: expected 1, got %d", len(serversList.Items))).WithMajor()
}

return &serversList.Items[0], nil
}
Loading

0 comments on commit 2ca478c

Please sign in to comment.