Skip to content

Commit

Permalink
RDMA subsystem is implemented via ib_core module config.
Browse files Browse the repository at this point in the history
  • Loading branch information
e0ne committed Aug 28, 2024
1 parent 34dcef5 commit 288e028
Show file tree
Hide file tree
Showing 21 changed files with 337 additions and 100 deletions.
7 changes: 7 additions & 0 deletions api/v1/sriovnetworknodestate_types.go
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@ import (
type SriovNetworkNodeStateSpec struct {
Interfaces Interfaces `json:"interfaces,omitempty"`
Bridges Bridges `json:"bridges,omitempty"`
System System `json:"system,omitempty"`
}

type Interfaces []Interface
Expand Down Expand Up @@ -114,10 +115,16 @@ type OVSUplinkConfigExt struct {
Interface OVSInterfaceConfig `json:"interface,omitempty"`
}

type System struct {
// +kubebuilder:validation:Enum=shared;exclusive
RdmaMode string `json:"rdmaMode,omitempty"`
}

// SriovNetworkNodeStateStatus defines the observed state of SriovNetworkNodeState
type SriovNetworkNodeStateStatus struct {
Interfaces InterfaceExts `json:"interfaces,omitempty"`
Bridges Bridges `json:"bridges,omitempty"`
System System `json:"system,omitempty"`
SyncStatus string `json:"syncStatus,omitempty"`
LastSyncError string `json:"lastSyncError,omitempty"`
}
Expand Down
4 changes: 4 additions & 0 deletions api/v1/sriovnetworkpoolconfig_types.go
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,10 @@ type SriovNetworkPoolConfigSpec struct {
// Drain will respect Pod Disruption Budgets (PDBs) such as etcd quorum guards,
// even if maxUnavailable is greater than one.
MaxUnavailable *intstr.IntOrString `json:"maxUnavailable,omitempty"`

// +kubebuilder:validation:Enum=shared;exclusive
// RDMA subsystem. Allowed value "shared", "exclusive".
RdmaMode string `json:"rdmaMode,omitempty"`
}

type OvsHardwareOffloadConfig struct {
Expand Down
17 changes: 17 additions & 0 deletions api/v1/zz_generated.deepcopy.go

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

1 change: 1 addition & 0 deletions bindata/manifests/ib-core-config/ib_core.conf
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
options ib_core netns_mode={{.Mode}}
Original file line number Diff line number Diff line change
Expand Up @@ -174,6 +174,14 @@ spec:
- pciAddress
type: object
type: array
system:
properties:
rdmaMode:
enum:
- shared
- exclusive
type: string
type: object
type: object
status:
description: SriovNetworkNodeStateStatus defines the observed state of
Expand Down Expand Up @@ -335,6 +343,14 @@ spec:
type: string
syncStatus:
type: string
system:
properties:
rdmaMode:
enum:
- shared
- exclusive
type: string
type: object
type: object
type: object
served: true
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -111,6 +111,12 @@ spec:
Name is the name of MachineConfigPool to be enabled with OVS hardware offload
type: string
type: object
rdmaMode:
description: RDMA subsystem. Allowed value "shared", "exclusive".
enum:
- shared
- exclusive
type: string
type: object
status:
description: SriovNetworkPoolConfigStatus defines the observed state of
Expand Down
99 changes: 1 addition & 98 deletions controllers/drain_controller.go
Original file line number Diff line number Diff line change
Expand Up @@ -24,11 +24,8 @@ import (

corev1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/labels"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/types"
"k8s.io/apimachinery/pkg/util/intstr"
"k8s.io/client-go/tools/record"
"k8s.io/client-go/util/workqueue"
ctrl "sigs.k8s.io/controller-runtime"
Expand All @@ -48,13 +45,6 @@ import (
"github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/vars"
)

var (
oneNode = intstr.FromInt32(1)
defaultNpcl = &sriovnetworkv1.SriovNetworkPoolConfig{Spec: sriovnetworkv1.SriovNetworkPoolConfigSpec{
MaxUnavailable: &oneNode,
NodeSelector: &metav1.LabelSelector{}}}
)

type DrainReconcile struct {
client.Client
Scheme *runtime.Scheme
Expand Down Expand Up @@ -346,94 +336,7 @@ func (dr *DrainReconcile) tryDrainNode(ctx context.Context, node *corev1.Node) (
}

func (dr *DrainReconcile) findNodePoolConfig(ctx context.Context, node *corev1.Node) (*sriovnetworkv1.SriovNetworkPoolConfig, []corev1.Node, error) {
logger := log.FromContext(ctx)
logger.Info("findNodePoolConfig():")
// get all the sriov network pool configs
npcl := &sriovnetworkv1.SriovNetworkPoolConfigList{}
err := dr.List(ctx, npcl)
if err != nil {
logger.Error(err, "failed to list sriovNetworkPoolConfig")
return nil, nil, err
}

selectedNpcl := []*sriovnetworkv1.SriovNetworkPoolConfig{}
nodesInPools := map[string]interface{}{}

for _, npc := range npcl.Items {
// we skip hw offload objects
if npc.Spec.OvsHardwareOffloadConfig.Name != "" {
continue
}

if npc.Spec.NodeSelector == nil {
npc.Spec.NodeSelector = &metav1.LabelSelector{}
}

selector, err := metav1.LabelSelectorAsSelector(npc.Spec.NodeSelector)
if err != nil {
logger.Error(err, "failed to create label selector from nodeSelector", "nodeSelector", npc.Spec.NodeSelector)
return nil, nil, err
}

if selector.Matches(labels.Set(node.Labels)) {
selectedNpcl = append(selectedNpcl, npc.DeepCopy())
}

nodeList := &corev1.NodeList{}
err = dr.List(ctx, nodeList, &client.ListOptions{LabelSelector: selector})
if err != nil {
logger.Error(err, "failed to list all the nodes matching the pool with label selector from nodeSelector",
"machineConfigPoolName", npc,
"nodeSelector", npc.Spec.NodeSelector)
return nil, nil, err
}

for _, nodeName := range nodeList.Items {
nodesInPools[nodeName.Name] = nil
}
}

if len(selectedNpcl) > 1 {
// don't allow the node to be part of multiple pools
err = fmt.Errorf("node is part of more then one pool")
logger.Error(err, "multiple pools founded for a specific node", "numberOfPools", len(selectedNpcl), "pools", selectedNpcl)
return nil, nil, err
} else if len(selectedNpcl) == 1 {
// found one pool for our node
logger.V(2).Info("found sriovNetworkPool", "pool", *selectedNpcl[0])
selector, err := metav1.LabelSelectorAsSelector(selectedNpcl[0].Spec.NodeSelector)
if err != nil {
logger.Error(err, "failed to create label selector from nodeSelector", "nodeSelector", selectedNpcl[0].Spec.NodeSelector)
return nil, nil, err
}

// list all the nodes that are also part of this pool and return them
nodeList := &corev1.NodeList{}
err = dr.List(ctx, nodeList, &client.ListOptions{LabelSelector: selector})
if err != nil {
logger.Error(err, "failed to list nodes using with label selector", "labelSelector", selector)
return nil, nil, err
}

return selectedNpcl[0], nodeList.Items, nil
} else {
// in this case we get all the nodes and remove the ones that already part of any pool
logger.V(1).Info("node doesn't belong to any pool, using default drain configuration with MaxUnavailable of one", "pool", *defaultNpcl)
nodeList := &corev1.NodeList{}
err = dr.List(ctx, nodeList)
if err != nil {
logger.Error(err, "failed to list all the nodes")
return nil, nil, err
}

defaultNodeLists := []corev1.Node{}
for _, nodeObj := range nodeList.Items {
if _, exist := nodesInPools[nodeObj.Name]; !exist {
defaultNodeLists = append(defaultNodeLists, nodeObj)
}
}
return defaultNpcl, defaultNodeLists, nil
}
return utils.FindNodePoolConfig(ctx, node, dr.Client)
}

// SetupWithManager sets up the controller with the Manager.
Expand Down
8 changes: 8 additions & 0 deletions controllers/sriovnetworknodepolicy_controller.go
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,7 @@ import (
sriovnetworkv1 "github.com/k8snetworkplumbingwg/sriov-network-operator/api/v1"
constants "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/consts"
"github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/featuregate"
"github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/utils"
"github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/vars"
)

Expand Down Expand Up @@ -272,6 +273,13 @@ func (r *SriovNetworkNodePolicyReconciler) syncAllSriovNetworkNodeStates(ctx con
ns.Name = node.Name
ns.Namespace = vars.Namespace
j, _ := json.Marshal(ns)
netPoolConfig, _, err := utils.FindNodePoolConfig(ctx, &node, r.Client)
if err != nil {
log.Log.Error(err, "nodeStateSyncHandler(): failed to get SriovNetworkPoolConfig for the current node")
}
if netPoolConfig != nil {
ns.Spec.System.RdmaMode = netPoolConfig.Spec.RdmaMode
}
logger.V(2).Info("SriovNetworkNodeState CR", "content", j)
if err := r.syncSriovNetworkNodeState(ctx, dc, npl, ns, &node); err != nil {
logger.Error(err, "Fail to sync", "SriovNetworkNodeState", ns.Name)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -174,6 +174,14 @@ spec:
- pciAddress
type: object
type: array
system:
properties:
rdmaMode:
enum:
- shared
- exclusive
type: string
type: object
type: object
status:
description: SriovNetworkNodeStateStatus defines the observed state of
Expand Down Expand Up @@ -335,6 +343,14 @@ spec:
type: string
syncStatus:
type: string
system:
properties:
rdmaMode:
enum:
- shared
- exclusive
type: string
type: object
type: object
type: object
served: true
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -111,6 +111,12 @@ spec:
Name is the name of MachineConfigPool to be enabled with OVS hardware offload
type: string
type: object
rdmaMode:
description: RDMA subsystem. Allowed value "shared", "exclusive".
enum:
- shared
- exclusive
type: string
type: object
status:
description: SriovNetworkPoolConfigStatus defines the observed state of
Expand Down
3 changes: 3 additions & 0 deletions pkg/consts/constants.go
Original file line number Diff line number Diff line change
Expand Up @@ -54,6 +54,9 @@ const (
VdpaTypeVirtio = "virtio"
VdpaTypeVhost = "vhost"

RdmaSubsystemModeShared = "shared"
RdmaSubsystemModeExclusive = "exclusive"

ClusterTypeOpenshift = "openshift"
ClusterTypeKubernetes = "kubernetes"

Expand Down
10 changes: 10 additions & 0 deletions pkg/daemon/daemon.go
Original file line number Diff line number Diff line change
Expand Up @@ -429,6 +429,16 @@ func (dn *Daemon) nodeStateSyncHandler() error {
reqReboot = reqReboot || r
}

if dn.currentNodeState.Status.System.RdmaMode != dn.desiredNodeState.Spec.System.RdmaMode {
err = dn.HostHelpers.SetRDMASubsystem(dn.desiredNodeState.Spec.System.RdmaMode)
if err != nil {
log.Log.Error(err, "nodeStateSyncHandler(): failed to set RDMA subsystem")
return err
}
reqReboot = true
reqDrain = true
}

// When running using systemd check if the applied configuration is the latest one
// or there is a new config we need to apply
// When using systemd configuration we write the file
Expand Down
7 changes: 7 additions & 0 deletions pkg/daemon/writer.go
Original file line number Diff line number Diff line change
Expand Up @@ -118,6 +118,7 @@ func (w *NodeStateStatusWriter) pollNicStatus() error {
log.Log.V(2).Info("pollNicStatus()")
var iface []sriovnetworkv1.InterfaceExt
var bridges sriovnetworkv1.Bridges
var rdmaMode string
var err error

if vars.PlatformType == consts.VirtualOpenStack {
Expand All @@ -138,8 +139,14 @@ func (w *NodeStateStatusWriter) pollNicStatus() error {
}
}

rdmaMode, err = w.hostHelper.DiscoverRDMASubsystem()
if err != nil {
return err
}

w.status.Interfaces = iface
w.status.Bridges = bridges
w.status.System.RdmaMode = rdmaMode

return nil
}
Expand Down
29 changes: 29 additions & 0 deletions pkg/helper/mock/mock_helper.go

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

Loading

0 comments on commit 288e028

Please sign in to comment.