Skip to content

Commit

Permalink
feat: Add supports for vDPA interface
Browse files Browse the repository at this point in the history
Signed-off-by: Ziteng Liu <[email protected]>
  • Loading branch information
makoto126 authored and fengye87 committed Jan 10, 2024
1 parent 30780fe commit db3f654
Show file tree
Hide file tree
Showing 8 changed files with 126 additions and 2 deletions.
12 changes: 12 additions & 0 deletions cmd/virt-prerunner/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -252,6 +252,18 @@ func buildVMConfig(ctx context.Context, vm *virtv1alpha1.VirtualMachine) (*cloud
vmConfig.Devices = append(vmConfig.Devices, &sriovDeviceConfig)
}
}
case iface.VDPA != nil:
for _, networkStatus := range networkStatusList {
if networkStatus.Interface == linkName && networkStatus.DeviceInfo != nil && networkStatus.DeviceInfo.Vdpa != nil {
vdpaDeviceConfig := cloudhypervisor.VdpaConfig{
Id: iface.Name,
NumQueues: iface.VDPA.NumQueues,
Iommu: iface.VDPA.IOMMU,
Path: networkStatus.DeviceInfo.Vdpa.Path,
}
vmConfig.Vdpa = append(vmConfig.Vdpa, &vdpaDeviceConfig)
}
}
case iface.VhostUser != nil:
netConfig := cloudhypervisor.NetConfig{
Id: iface.Name,
Expand Down
7 changes: 7 additions & 0 deletions deploy/crd/virt.virtink.smartx.com_virtualmachines.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -910,6 +910,13 @@ spec:
type: string
sriov:
type: object
vdpa:
properties:
iommu:
type: boolean
numQueues:
type: integer
type: object
vhostUser:
type: object
required:
Expand Down
61 changes: 61 additions & 0 deletions docs/interfaces_and_networks.md
Original file line number Diff line number Diff line change
Expand Up @@ -83,6 +83,7 @@ Each interface should declare its type by defining one of the following fields:
| `bridge` | Connect using a linux bridge |
| `masquerade` | Connect using iptables rules to NAT the traffic |
| `sriov` | Passthrough a SR-IOV PCI device via VFIO |
| `vdpa` | Add a vhost-vdpa device |

Each interface may also have additional configuration fields that modify properties "seen" inside guest instances, as listed below:

Expand Down Expand Up @@ -224,3 +225,63 @@ spec:
multus:
networkName: mellanox-sriov-25g
```
### `vdpa` Mode
Similar to the sriov mode, the following components need to be installed:
- [Multus CNI](https://github.com/k8snetworkplumbingwg/multus-cni)
- A vDPA device plugin
- A vDPA CNI Plugin

If your hardware vendor support vDPA over SR-IOV, you can use
the [SR-IOV Network Device Plugin](https://github.com/k8snetworkplumbingwg/sriov-network-device-plugin), here is a sample configuration:
```yaml
apiVersion: v1
kind: ConfigMap
metadata:
name: sriovdp-config
namespace: kube-system
data:
config.json: |
{
"resourceList": [
{
"resourceName": "jaguar",
"resourcePrefix": "jaguarmicro.com",
"selectors": {
"vendors": ["1f53"],
"devices": ["1000"],
"drivers": ["jaguar"],
"vdpaType": "vhost"
}
}
]
}
```
If you don't need complex network configuration, the changes to enable the SR-IOV CNI to also manage vDPA interfaces are in this repository:

https://github.com/amorenoz/sriov-cni/tree/rfe/vdpa

#### Start an vDPA VM

To create a VM that will attach to the aforementioned network, refer to the following VM spec:

```yaml
apiVersion: virt.virtink.smartx.com/v1alpha1
kind: VirtualMachine
spec:
instance:
interfaces:
- name: pod
- name: vdpa1
vdpa: { numQueues: 9, iommu: False }
- name: vdpa2
vdpa: { numQueues: 9, iommu: False }
networks:
- name: pod
pod: {}
- name: vdpa1
multus:
networkName: offload-ovn1
- name: vdpa2
multus:
networkName: offload-ovn2
```
6 changes: 6 additions & 0 deletions pkg/apis/virt/v1alpha1/types.go
Original file line number Diff line number Diff line change
Expand Up @@ -102,6 +102,7 @@ type InterfaceBindingMethod struct {
Bridge *InterfaceBridge `json:"bridge,omitempty"`
Masquerade *InterfaceMasquerade `json:"masquerade,omitempty"`
SRIOV *InterfaceSRIOV `json:"sriov,omitempty"`
VDPA *InterfaceVDPA `json:"vdpa,omitempty"`
VhostUser *InterfaceVhostUser `json:"vhostUser,omitempty"`
}

Expand All @@ -115,6 +116,11 @@ type InterfaceMasquerade struct {
type InterfaceSRIOV struct {
}

type InterfaceVDPA struct {
NumQueues int `json:"numQueues,omitempty"`
IOMMU bool `json:"iommu,omitempty"`
}

type InterfaceVhostUser struct {
}

Expand Down
21 changes: 21 additions & 0 deletions pkg/apis/virt/v1alpha1/zz_generated.deepcopy.go

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

8 changes: 7 additions & 1 deletion pkg/controller/vm_webhook.go
Original file line number Diff line number Diff line change
Expand Up @@ -165,7 +165,7 @@ func MutateVM(ctx context.Context, vm *virtv1alpha1.VirtualMachine, oldVM *virtv
vm.Spec.Instance.Interfaces[i].MAC = macStr
}

if vm.Spec.Instance.Interfaces[i].Bridge == nil && vm.Spec.Instance.Interfaces[i].Masquerade == nil && vm.Spec.Instance.Interfaces[i].SRIOV == nil && vm.Spec.Instance.Interfaces[i].VhostUser == nil {
if vm.Spec.Instance.Interfaces[i].Bridge == nil && vm.Spec.Instance.Interfaces[i].Masquerade == nil && vm.Spec.Instance.Interfaces[i].SRIOV == nil && vm.Spec.Instance.Interfaces[i].VDPA == nil && vm.Spec.Instance.Interfaces[i].VhostUser == nil {
vm.Spec.Instance.Interfaces[i].InterfaceBindingMethod = virtv1alpha1.InterfaceBindingMethod{
Bridge: &virtv1alpha1.InterfaceBridge{},
}
Expand Down Expand Up @@ -504,6 +504,12 @@ func ValidateInterfaceBindingMethod(ctx context.Context, bindingMethod *virtv1al
errs = append(errs, field.Forbidden(fieldPath.Child("sriov"), "may not specify more than 1 binding method"))
}
}
if bindingMethod.VDPA != nil {
cnt++
if cnt > 1 {
errs = append(errs, field.Forbidden(fieldPath.Child("vdpa"), "may not specify more than 1 binding method"))
}
}
if bindingMethod.VhostUser != nil {
cnt++
if cnt > 1 {
Expand Down
11 changes: 11 additions & 0 deletions pkg/controller/vm_webhook_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -365,6 +365,17 @@ func TestValidateVM(t *testing.T) {
return vm
}(),
invalidFields: []string{"spec.instance.interfaces[0].sriov"},
}, {
vm: func() *virtv1alpha1.VirtualMachine {
vm := validVM.DeepCopy()
vm.Spec.Instance.Interfaces[0].InterfaceBindingMethod.Bridge = nil
vm.Spec.Instance.Interfaces[0].InterfaceBindingMethod.VDPA = &virtv1alpha1.InterfaceVDPA{
NumQueues: 9,
IOMMU: false,
}
return vm
}(),
invalidFields: []string{"spec.instance.interfaces[0].vdpa.numQueues", "spec.instance.interfaces[0].vdpa.iommu"},
}, {
vm: func() *virtv1alpha1.VirtualMachine {
vm := validVM.DeepCopy()
Expand Down
2 changes: 1 addition & 1 deletion pkg/daemon/vm_controller.go
Original file line number Diff line number Diff line change
Expand Up @@ -145,7 +145,7 @@ func (r *VMReconciler) reconcile(ctx context.Context, vm *virtv1alpha1.VirtualMa
return err
}

if len(vmConfig.Devices) > 0 {
if len(vmConfig.Devices) > 0 || len(vmConfig.Vdpa) > 0 {
cloudHypervisorPID, err := pid.GetPIDBySocket(filepath.Join(getVMDataDirPath(vm), "ch.sock"))
if err != nil {
return fmt.Errorf("get cloud-hypervisor process pid: %s", err)
Expand Down

0 comments on commit db3f654

Please sign in to comment.