Skip to content

Commit

Permalink
Merge pull request #147 from orelmisan/refactor_vmi_creation
Browse files Browse the repository at this point in the history
[traffic-gen-in-vm] Refactor VMI creation
  • Loading branch information
orelmisan authored Jul 19, 2023
2 parents 69c9281 + a086020 commit c20107a
Show file tree
Hide file tree
Showing 7 changed files with 166 additions and 183 deletions.
61 changes: 20 additions & 41 deletions pkg/internal/checkup/checkup.go
Original file line number Diff line number Diff line change
Expand Up @@ -27,12 +27,12 @@ import (

k8scorev1 "k8s.io/api/core/v1"
k8serrors "k8s.io/apimachinery/pkg/api/errors"
"k8s.io/apimachinery/pkg/util/rand"
"k8s.io/apimachinery/pkg/util/wait"

kvcorev1 "kubevirt.io/api/core/v1"

"github.com/kiagnose/kubevirt-dpdk-checkup/pkg/internal/checkup/configmap"
"github.com/kiagnose/kubevirt-dpdk-checkup/pkg/internal/checkup/vmi"
"github.com/kiagnose/kubevirt-dpdk-checkup/pkg/internal/config"
"github.com/kiagnose/kubevirt-dpdk-checkup/pkg/internal/status"
)
Expand Down Expand Up @@ -63,19 +63,20 @@ type Checkup struct {
}

const (
VMIUnderTestNamePrefix = "vmi-under-test"
TrafficGenNamePrefix = "dpdk-traffic-gen"
TrafficGenConfigMapNamePrefix = "dpdk-traffic-gen-config"
)

func New(client kubeVirtVMIClient, namespace string, checkupConfig config.Config, executor testExecutor) *Checkup {
const randomStringLen = 5
randomSuffix := rand.String(randomStringLen)

return &Checkup{
client: client,
namespace: namespace,
params: checkupConfig,
vmiUnderTest: newVMIUnderTest(checkupConfig),
trafficGen: newTrafficGen(checkupConfig),
trafficGenConfigMap: newTrafficGenConfigMap(checkupConfig),
vmiUnderTest: newVMIUnderTest(vmiUnderTestName(randomSuffix), checkupConfig),
trafficGen: newTrafficGen(trafficGenName(randomSuffix), checkupConfig),
trafficGenConfigMap: newTrafficGenConfigMap(trafficGenConfigMapName(randomSuffix), checkupConfig),
executor: executor,
}
}
Expand Down Expand Up @@ -286,44 +287,22 @@ func ObjectFullName(namespace, name string) string {
return fmt.Sprintf("%s/%s", namespace, name)
}

func newTrafficGenConfigMap(checkupConfig config.Config) *k8scorev1.ConfigMap {
return configmap.New(TrafficGenConfigMapNamePrefix, checkupConfig.PodName, checkupConfig.PodUID)
func newTrafficGenConfigMap(name string, checkupConfig config.Config) *k8scorev1.ConfigMap {
return configmap.New(
name,
checkupConfig.PodName,
checkupConfig.PodUID,
)
}

func newVMIUnderTest(checkupConfig config.Config) *kvcorev1.VirtualMachineInstance {
vmiConfig := vmi.DPDKVMIConfig{
NamePrefix: VMIUnderTestNamePrefix,
OwnerName: checkupConfig.PodName,
OwnerUID: checkupConfig.PodUID,
Affinity: vmi.Affinity(checkupConfig.DPDKNodeLabelSelector, checkupConfig.PodUID),
ContainerDiskImage: checkupConfig.VMContainerDiskImage,
NetworkAttachmentDefinitionName: checkupConfig.NetworkAttachmentDefinitionName,
NICEastMACAddress: checkupConfig.DPDKEastMacAddress.String(),
NICEastPCIAddress: config.VMIEastNICPCIAddress,
NICWestMACAddress: checkupConfig.DPDKWestMacAddress.String(),
NICWestPCIAddress: config.VMIWestNICPCIAddress,
Username: config.VMIUsername,
Password: config.VMIPassword,
}

return vmi.NewDPDKVMI(vmiConfig)
func vmiUnderTestName(suffix string) string {
return VMIUnderTestNamePrefix + "-" + suffix
}

func newTrafficGen(checkupConfig config.Config) *kvcorev1.VirtualMachineInstance {
vmiConfig := vmi.DPDKVMIConfig{
NamePrefix: TrafficGenNamePrefix,
OwnerName: checkupConfig.PodName,
OwnerUID: checkupConfig.PodUID,
Affinity: vmi.Affinity(checkupConfig.TrafficGeneratorNodeLabelSelector, checkupConfig.PodUID),
ContainerDiskImage: checkupConfig.TrafficGeneratorImage,
NetworkAttachmentDefinitionName: checkupConfig.NetworkAttachmentDefinitionName,
NICEastMACAddress: checkupConfig.TrafficGeneratorEastMacAddress.String(),
NICEastPCIAddress: config.VMIEastNICPCIAddress,
NICWestMACAddress: checkupConfig.TrafficGeneratorWestMacAddress.String(),
NICWestPCIAddress: config.VMIWestNICPCIAddress,
Username: config.VMIUsername,
Password: config.VMIPassword,
}
func trafficGenName(suffix string) string {
return TrafficGenNamePrefix + "-" + suffix
}

return vmi.NewDPDKVMI(vmiConfig)
func trafficGenConfigMapName(suffix string) string {
return TrafficGenConfigMapNamePrefix + "-" + suffix
}
3 changes: 1 addition & 2 deletions pkg/internal/checkup/checkup_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,6 @@ import (
kvcorev1 "kubevirt.io/api/core/v1"

"github.com/kiagnose/kubevirt-dpdk-checkup/pkg/internal/checkup"
checkupvmi "github.com/kiagnose/kubevirt-dpdk-checkup/pkg/internal/checkup/vmi"
"github.com/kiagnose/kubevirt-dpdk-checkup/pkg/internal/config"
"github.com/kiagnose/kubevirt-dpdk-checkup/pkg/internal/status"
)
Expand Down Expand Up @@ -263,7 +262,7 @@ func assertPodAntiAffinityExists(t *testing.T, testClient *clientStub, vmiName,
MatchExpressions: []k8smetav1.LabelSelectorRequirement{
{
Operator: k8smetav1.LabelSelectorOpIn,
Key: checkupvmi.DPDKCheckupUIDLabelKey,
Key: checkup.DPDKCheckupUIDLabelKey,
Values: []string{ownerUID},
},
},
Expand Down
125 changes: 125 additions & 0 deletions pkg/internal/checkup/vmi.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,125 @@
/*
* This file is part of the kiagnose project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
* Copyright 2023 Red Hat, Inc.
*
*/

package checkup

import (
"fmt"
"strings"

k8scorev1 "k8s.io/api/core/v1"
kvcorev1 "kubevirt.io/api/core/v1"

"github.com/kiagnose/kubevirt-dpdk-checkup/pkg/internal/checkup/vmi"
"github.com/kiagnose/kubevirt-dpdk-checkup/pkg/internal/config"
)

const (
VMIUnderTestNamePrefix = "vmi-under-test"
TrafficGenNamePrefix = "dpdk-traffic-gen"
)

const DPDKCheckupUIDLabelKey = "kubevirt-dpdk-checkup/uid"

const (
CPUSocketsCount = 1
CPUCoresCount = 4
CPUTreadsCount = 2
hugePageSize = "1Gi"
guestMemory = "4Gi"
rootDiskName = "rootdisk"
cloudInitDiskName = "cloudinitdisk"
eastNetworkName = "nic-east"
westNetworkName = "nic-west"

terminationGracePeriodSeconds = 0
)

func newVMIUnderTest(name string, checkupConfig config.Config) *kvcorev1.VirtualMachineInstance {
optionsToApply := baseOptions(checkupConfig)

optionsToApply = append(optionsToApply,
vmi.WithAffinity(Affinity(checkupConfig.DPDKNodeLabelSelector, checkupConfig.PodUID)),
vmi.WithSRIOVInterface(eastNetworkName, checkupConfig.DPDKEastMacAddress.String(), config.VMIEastNICPCIAddress),
vmi.WithSRIOVInterface(westNetworkName, checkupConfig.DPDKWestMacAddress.String(), config.VMIWestNICPCIAddress),
vmi.WithContainerDisk(rootDiskName, checkupConfig.VMContainerDiskImage),
vmi.WithCloudInitNoCloudVolume(cloudInitDiskName, CloudInit(config.VMIUsername, config.VMIPassword)),
)

return vmi.New(name, optionsToApply...)
}

func newTrafficGen(name string, checkupConfig config.Config) *kvcorev1.VirtualMachineInstance {
optionsToApply := baseOptions(checkupConfig)

optionsToApply = append(optionsToApply,
vmi.WithAffinity(Affinity(checkupConfig.TrafficGeneratorNodeLabelSelector, checkupConfig.PodUID)),
vmi.WithSRIOVInterface(eastNetworkName, checkupConfig.TrafficGeneratorEastMacAddress.String(), config.VMIEastNICPCIAddress),
vmi.WithSRIOVInterface(westNetworkName, checkupConfig.TrafficGeneratorWestMacAddress.String(), config.VMIWestNICPCIAddress),
vmi.WithContainerDisk(rootDiskName, checkupConfig.TrafficGeneratorImage),
vmi.WithCloudInitNoCloudVolume(cloudInitDiskName, CloudInit(config.VMIUsername, config.VMIPassword)),
)

return vmi.New(name, optionsToApply...)
}

func baseOptions(checkupConfig config.Config) []vmi.Option {
labels := map[string]string{
DPDKCheckupUIDLabelKey: checkupConfig.PodUID,
}

return []vmi.Option{
vmi.WithOwnerReference(checkupConfig.PodName, checkupConfig.PodUID),
vmi.WithLabels(labels),
vmi.WithoutCRIOCPULoadBalancing(),
vmi.WithoutCRIOCPUQuota(),
vmi.WithoutCRIOIRQLoadBalancing(),
vmi.WithDedicatedCPU(CPUSocketsCount, CPUCoresCount, CPUTreadsCount),
vmi.WithMemory(hugePageSize, guestMemory),
vmi.WithNetworkInterfaceMultiQueue(),
vmi.WithRandomNumberGenerator(),
vmi.WithTerminationGracePeriodSeconds(terminationGracePeriodSeconds),
vmi.WithMultusNetwork(eastNetworkName, checkupConfig.NetworkAttachmentDefinitionName),
vmi.WithMultusNetwork(westNetworkName, checkupConfig.NetworkAttachmentDefinitionName),
vmi.WithVirtIODisk(rootDiskName),
vmi.WithVirtIODisk(cloudInitDiskName),
}
}

func Affinity(nodeName, ownerUID string) *k8scorev1.Affinity {
var affinity k8scorev1.Affinity
if nodeName != "" {
affinity.NodeAffinity = vmi.NewRequiredNodeAffinity(nodeName)
} else {
affinity.PodAntiAffinity = vmi.NewPreferredPodAntiAffinity(DPDKCheckupUIDLabelKey, ownerUID)
}

return &affinity
}

func CloudInit(username, password string) string {
sb := strings.Builder{}
sb.WriteString("#cloud-config\n")
sb.WriteString(fmt.Sprintf("user: %s\n", username))
sb.WriteString(fmt.Sprintf("password: %s\n", password))
sb.WriteString("chpasswd:\n")
sb.WriteString(" expire: false")

return sb.String()
}
19 changes: 4 additions & 15 deletions pkg/internal/checkup/vmi/affinity.go
Original file line number Diff line number Diff line change
Expand Up @@ -24,20 +24,9 @@ import (
k8smetav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)

func Affinity(nodeName, ownerUID string) *k8scorev1.Affinity {
var affinity k8scorev1.Affinity
if nodeName != "" {
affinity.NodeAffinity = newRequiredNodeAffinity(nodeName)
} else {
affinity.PodAntiAffinity = newPreferredPodAntiAffinity(DPDKCheckupUIDLabelKey, ownerUID)
}

return &affinity
}

// newRequiredNodeAffinity returns new node affinity with node selector of the given node name.
// NewRequiredNodeAffinity returns new node affinity with node selector of the given node name.
// Adding it to a VMI will make sure it will schedule on the given node name.
func newRequiredNodeAffinity(nodeName string) *k8scorev1.NodeAffinity {
func NewRequiredNodeAffinity(nodeName string) *k8scorev1.NodeAffinity {
req := k8scorev1.NodeSelectorRequirement{
Key: k8scorev1.LabelHostname,
Operator: k8scorev1.NodeSelectorOpIn,
Expand All @@ -55,9 +44,9 @@ func newRequiredNodeAffinity(nodeName string) *k8scorev1.NodeAffinity {
}
}

// newPreferredPodAntiAffinity returns new pod anti-affinity with label selector of the given label key and value.
// NewPreferredPodAntiAffinity returns new pod anti-affinity with label selector of the given label key and value.
// Adding it to a VMI will make sure it won't schedule on the same node as other VMIs with the given label.
func newPreferredPodAntiAffinity(labelKey, labelVal string) *k8scorev1.PodAntiAffinity {
func NewPreferredPodAntiAffinity(labelKey, labelVal string) *k8scorev1.PodAntiAffinity {
req := k8smetav1.LabelSelectorRequirement{
Operator: k8smetav1.LabelSelectorOpIn,
Key: labelKey,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -20,15 +20,10 @@
package vmi

import (
"fmt"
"strings"

corev1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/api/resource"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/types"
"k8s.io/apimachinery/pkg/util/rand"

kvcorev1 "kubevirt.io/api/core/v1"
)

Expand All @@ -46,67 +41,8 @@ const (
CRIOIRQLoadBalancingAnnotation = "irq-load-balancing.crio.io"
)

const DPDKCheckupUIDLabelKey = "kubevirt-dpdk-checkup/uid"

const Disable = "disable"

type DPDKVMIConfig struct {
NamePrefix string
OwnerName string
OwnerUID string
Affinity *corev1.Affinity
ContainerDiskImage string
NetworkAttachmentDefinitionName string
NICEastMACAddress string
NICEastPCIAddress string
NICWestMACAddress string
NICWestPCIAddress string
Username string
Password string
}

func NewDPDKVMI(vmiConfig DPDKVMIConfig) *kvcorev1.VirtualMachineInstance {
const (
CPUSocketsCount = 1
CPUCoresCount = 4
CPUTreadsCount = 2
hugePageSize = "1Gi"
guestMemory = "4Gi"
rootDiskName = "rootdisk"
cloudInitDiskName = "cloudinitdisk"
eastNetworkName = "nic-east"
westNetworkName = "nic-west"

terminationGracePeriodSeconds = 0
)

labels := map[string]string{
DPDKCheckupUIDLabelKey: vmiConfig.OwnerUID,
}

return New(RandomizeName(vmiConfig.NamePrefix),
WithOwnerReference(vmiConfig.OwnerName, vmiConfig.OwnerUID),
WithLabels(labels),
WithAffinity(vmiConfig.Affinity),
WithoutCRIOCPULoadBalancing(),
WithoutCRIOCPUQuota(),
WithoutCRIOIRQLoadBalancing(),
WithDedicatedCPU(CPUSocketsCount, CPUCoresCount, CPUTreadsCount),
WithSRIOVInterface(eastNetworkName, vmiConfig.NICEastMACAddress, vmiConfig.NICEastPCIAddress),
WithMultusNetwork(eastNetworkName, vmiConfig.NetworkAttachmentDefinitionName),
WithSRIOVInterface(westNetworkName, vmiConfig.NICWestMACAddress, vmiConfig.NICWestPCIAddress),
WithMultusNetwork(westNetworkName, vmiConfig.NetworkAttachmentDefinitionName),
WithNetworkInterfaceMultiQueue(),
WithRandomNumberGenerator(),
WithMemory(hugePageSize, guestMemory),
WithTerminationGracePeriodSeconds(terminationGracePeriodSeconds),
WithContainerDisk(rootDiskName, vmiConfig.ContainerDiskImage),
WithVirtIODisk(rootDiskName),
WithCloudInitNoCloudVolume(cloudInitDiskName, CloudInit(vmiConfig.Username, vmiConfig.Password)),
WithVirtIODisk(cloudInitDiskName),
)
}

type Option func(vmi *kvcorev1.VirtualMachineInstance)

func New(name string, options ...Option) *kvcorev1.VirtualMachineInstance {
Expand Down Expand Up @@ -300,23 +236,6 @@ func WithAffinity(affinity *corev1.Affinity) Option {
}
}

func CloudInit(username, password string) string {
sb := strings.Builder{}
sb.WriteString("#cloud-config\n")
sb.WriteString(fmt.Sprintf("user: %s\n", username))
sb.WriteString(fmt.Sprintf("password: %s\n", password))
sb.WriteString("chpasswd:\n")
sb.WriteString(" expire: false")

return sb.String()
}

func RandomizeName(prefix string) string {
const randomStringLen = 5

return fmt.Sprintf("%s-%s", prefix, rand.String(randomStringLen))
}

func Pointer[T any](v T) *T {
return &v
}
Loading

0 comments on commit c20107a

Please sign in to comment.