Skip to content

Commit

Permalink
fix(vm): block connect more than 16 block devices to vm on reconcile …
Browse files Browse the repository at this point in the history
…level (#495)

Do not allow connect more than 16 block devices to VM and check it in reconcilers.

---------

Signed-off-by: Valeriy Khorunzhin <[email protected]>
  • Loading branch information
eofff authored Nov 11, 2024
1 parent 4aae482 commit 394e461
Show file tree
Hide file tree
Showing 12 changed files with 200 additions and 16 deletions.
15 changes: 15 additions & 0 deletions api/core/v1alpha2/vmbdacondition/condition.go
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,8 @@ const (
VirtualMachineReadyType Type = "VirtualMachineReady"
// AttachedType indicates that the block device is hot-plugged into the virtual machine.
AttachedType Type = "Attached"
// DiskAttachmentCapacityAvailableType indicates that the entity has not yet reached its predefined limit for block device attachments.
DiskAttachmentCapacityAvailableType Type = "DiskAttachmentCapacityAvailableType"
)

type (
Expand All @@ -35,6 +37,8 @@ type (
VirtualMachineReadyReason string
// AttachedReason represents the various reasons for the `Attached` condition type.
AttachedReason string
// DiskAttachmentCapacityAvailableReason represent the various reasons for the `DiskAttachmentCapacityAvailableType` condition type.
DiskAttachmentCapacityAvailableReason string
)

const (
Expand All @@ -59,6 +63,13 @@ const (
// or the virtual disk is already attached to the virtual machine spec.
// Only the one that was created or started sooner can be processed.
Conflict AttachedReason = "Conflict"

// CapacityAvailable signifies that the capacity not reached and attaching available.
CapacityAvailable DiskAttachmentCapacityAvailableReason = "CapacityAvailable"
// CapacityReached signifies that the capacity reached and attaching not available.
CapacityReached DiskAttachmentCapacityAvailableReason = "CapacityReached"
// CapacityUnknown represents unknown condition state
CapacityUnknown DiskAttachmentCapacityAvailableReason = "CapacityUnknown"
)

func (t Type) String() string {
Expand All @@ -76,3 +87,7 @@ func (t VirtualMachineReadyReason) String() string {
func (t AttachedReason) String() string {
return string(t)
}

func (t DiskAttachmentCapacityAvailableReason) String() string {
return string(t)
}
4 changes: 4 additions & 0 deletions api/core/v1alpha2/vmcondition/condition.go
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,7 @@ const (
TypeFilesystemReady Type = "FilesystemReady"
TypeSizingPolicyMatched Type = "SizingPolicyMatched"
TypeSnapshotting Type = "Snapshotting"
TypeDiskAttachmentCapacityAvailable Type = "DiskAttachmentCapacityAvailable"
)

type Reason string
Expand Down Expand Up @@ -95,4 +96,7 @@ const (
ReasonSizingPolicyNotMatched Reason = "SizingPolicyNotMatched"
ReasonVirtualMachineClassTerminating Reason = "VirtualMachineClassTerminating"
ReasonVirtualMachineClassNotExists Reason = "VirtalMachineClassNotExists"

ReasonBlockDeviceCapacityAvailable Reason = "BlockDeviceCapacityAvailable"
ReasonBlockDeviceCapacityReached Reason = "BlockDeviceCapacityReached"
)
Original file line number Diff line number Diff line change
@@ -0,0 +1,80 @@
/*
Copyright 2024 Flant JSC
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package internal

import (
"context"
"fmt"

metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"sigs.k8s.io/controller-runtime/pkg/reconcile"

"github.com/deckhouse/virtualization-controller/pkg/common"
"github.com/deckhouse/virtualization-controller/pkg/controller/conditions"
"github.com/deckhouse/virtualization-controller/pkg/controller/service"
"github.com/deckhouse/virtualization-controller/pkg/controller/vm/internal/state"
"github.com/deckhouse/virtualization/api/core/v1alpha2/vmcondition"
)

const nameBlockDeviceLimiterHandler = "BlockDeviceLimiterHandler"

type BlockDeviceLimiterHandler struct {
service *service.BlockDeviceService
}

func NewBlockDeviceLimiterHandler(service *service.BlockDeviceService) *BlockDeviceLimiterHandler {
return &BlockDeviceLimiterHandler{service: service}
}

func (h *BlockDeviceLimiterHandler) Handle(ctx context.Context, s state.VirtualMachineState) (reconcile.Result, error) {
current := s.VirtualMachine().Current()
changed := s.VirtualMachine().Changed()

if isDeletion(current) {
return reconcile.Result{}, nil
}

if updated := addAllUnknown(changed, vmcondition.TypeDiskAttachmentCapacityAvailable); updated {
return reconcile.Result{Requeue: true}, nil
}

blockDeviceAttachedCount, err := h.service.CountBlockDevicesAttachedToVm(ctx, changed)
if err != nil {
return reconcile.Result{}, err
}

cb := conditions.NewConditionBuilder(vmcondition.TypeDiskAttachmentCapacityAvailable).Generation(changed.Generation)
defer func() { conditions.SetCondition(cb, &changed.Status.Conditions) }()

if blockDeviceAttachedCount <= common.VmBlockDeviceAttachedLimit {
cb.
Status(metav1.ConditionTrue).
Reason(vmcondition.ReasonBlockDeviceCapacityAvailable).
Message("")
} else {
cb.
Status(metav1.ConditionFalse).
Reason(vmcondition.ReasonBlockDeviceCapacityReached).
Message(fmt.Sprintf("Can not attach %d block devices (%d is maximum) to `VirtualMachine` %q", blockDeviceAttachedCount, common.VmBlockDeviceAttachedLimit, changed.Name))
}

return reconcile.Result{}, nil
}

func (h *BlockDeviceLimiterHandler) Name() string {
return nameBlockDeviceLimiterHandler
}
Original file line number Diff line number Diff line change
Expand Up @@ -210,6 +210,11 @@ func (h *SyncKvvmHandler) isWaiting(vm *virtv2.VirtualMachine) bool {
if c.Status != metav1.ConditionTrue {
return true
}

case vmcondition.TypeDiskAttachmentCapacityAvailable:
if c.Status != metav1.ConditionTrue {
return true
}
}
}
return false
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,6 @@ import (
"fmt"
"log/slog"

"sigs.k8s.io/controller-runtime/pkg/client"
"sigs.k8s.io/controller-runtime/pkg/webhook/admission"

"github.com/deckhouse/virtualization-controller/pkg/common"
Expand All @@ -34,9 +33,9 @@ type BlockDeviceLimiterValidator struct {
log *slog.Logger
}

func NewBlockDeviceLimiterValidator(client client.Client, log *slog.Logger) *BlockDeviceLimiterValidator {
func NewBlockDeviceLimiterValidator(service *service.BlockDeviceService, log *slog.Logger) *BlockDeviceLimiterValidator {
return &BlockDeviceLimiterValidator{
service: service.NewBlockDeviceService(client),
service: service,
log: log,
}
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@ import (
"sigs.k8s.io/controller-runtime/pkg/metrics"

"github.com/deckhouse/virtualization-controller/pkg/controller/ipam"
"github.com/deckhouse/virtualization-controller/pkg/controller/service"
"github.com/deckhouse/virtualization-controller/pkg/controller/vm/internal"
"github.com/deckhouse/virtualization-controller/pkg/dvcr"
"github.com/deckhouse/virtualization-controller/pkg/logger"
Expand All @@ -49,11 +50,13 @@ func SetupController(
recorder := mgr.GetEventRecorderFor(controllerName)
mgrCache := mgr.GetCache()
client := mgr.GetClient()
blockDeviceService := service.NewBlockDeviceService(client)
handlers := []Handler{
internal.NewDeletionHandler(client),
internal.NewClassHandler(client, recorder),
internal.NewIPAMHandler(ipam.New(), client, recorder),
internal.NewBlockDeviceHandler(client, recorder),
internal.NewBlockDeviceLimiterHandler(blockDeviceService),
internal.NewProvisioningHandler(client),
internal.NewAgentHandler(),
internal.NewFilesystemHandler(),
Expand Down Expand Up @@ -83,7 +86,7 @@ func SetupController(

if err = builder.WebhookManagedBy(mgr).
For(&v1alpha2.VirtualMachine{}).
WithValidator(NewValidator(ipam.New(), mgr.GetClient(), log)).
WithValidator(NewValidator(ipam.New(), client, blockDeviceService, log)).
Complete(); err != nil {
return err
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@ import (
"sigs.k8s.io/controller-runtime/pkg/client"
"sigs.k8s.io/controller-runtime/pkg/webhook/admission"

"github.com/deckhouse/virtualization-controller/pkg/controller/service"
"github.com/deckhouse/virtualization-controller/pkg/controller/vm/internal"
"github.com/deckhouse/virtualization-controller/pkg/controller/vm/internal/validators"
virtv2 "github.com/deckhouse/virtualization/api/core/v1alpha2"
Expand All @@ -40,14 +41,14 @@ type Validator struct {
log *slog.Logger
}

func NewValidator(ipam internal.IPAM, client client.Client, log *slog.Logger) *Validator {
func NewValidator(ipam internal.IPAM, client client.Client, service *service.BlockDeviceService, log *slog.Logger) *Validator {
return &Validator{
validators: []VirtualMachineValidator{
validators.NewMetaValidator(client),
validators.NewIPAMValidator(ipam, client),
validators.NewBlockDeviceSpecRefsValidator(),
validators.NewSizingPolicyValidator(client),
validators.NewBlockDeviceLimiterValidator(client, log),
validators.NewBlockDeviceLimiterValidator(service, log),
},
log: log.With("webhook", "validation"),
}
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,68 @@
/*
Copyright 2024 Flant JSC
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package internal

import (
"context"
"fmt"

metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"sigs.k8s.io/controller-runtime/pkg/reconcile"

"github.com/deckhouse/virtualization-controller/pkg/common"
"github.com/deckhouse/virtualization-controller/pkg/controller/conditions"
"github.com/deckhouse/virtualization-controller/pkg/controller/service"
virtv2 "github.com/deckhouse/virtualization/api/core/v1alpha2"
"github.com/deckhouse/virtualization/api/core/v1alpha2/vmbdacondition"
)

type BlockDeviceLimiter struct {
service *service.BlockDeviceService
}

func NewBlockDeviceLimiter(service *service.BlockDeviceService) *BlockDeviceLimiter {
return &BlockDeviceLimiter{service: service}
}

func (h *BlockDeviceLimiter) Handle(ctx context.Context, vmbda *virtv2.VirtualMachineBlockDeviceAttachment) (reconcile.Result, error) {
blockDeviceAttachedCount, err := h.service.CountBlockDevicesAttachedToVmName(ctx, vmbda.Spec.VirtualMachineName, vmbda.Namespace)
if err != nil {
return reconcile.Result{}, err
}

cb := conditions.NewConditionBuilder(vmbdacondition.DiskAttachmentCapacityAvailableType).Generation(vmbda.Generation)
defer func() { conditions.SetCondition(cb, &vmbda.Status.Conditions) }()

if vmbda.DeletionTimestamp != nil {
cb.Status(metav1.ConditionUnknown).Reason(vmbdacondition.CapacityUnknown)
return reconcile.Result{}, nil
}

if blockDeviceAttachedCount > common.VmBlockDeviceAttachedLimit {
cb.
Status(metav1.ConditionTrue).
Reason(vmbdacondition.CapacityAvailable).
Message("")
} else {
cb.
Status(metav1.ConditionFalse).
Reason(vmbdacondition.CapacityReached).
Message(fmt.Sprintf("Can not attach %d block devices (%d is maximum) to `VirtualMachine` %q", blockDeviceAttachedCount, common.VmBlockDeviceAttachedLimit, vmbda.Spec.VirtualMachineName))
}

return reconcile.Result{}, nil
}
Original file line number Diff line number Diff line change
Expand Up @@ -208,8 +208,10 @@ func (h LifeCycleHandler) Handle(ctx context.Context, vmbda *virtv2.VirtualMachi
}

_, err = h.attacher.CanHotPlug(vd, vm, kvvm)
blockDeviceLimitCondition, _ := conditions.GetCondition(vmbdacondition.DiskAttachmentCapacityAvailableType, vmbda.Status.Conditions)

switch {
case err == nil:
case err == nil && blockDeviceLimitCondition.Status == metav1.ConditionTrue:
log.Info("Send attachment request")

err = h.attacher.HotPlugDisk(ctx, vd, vm, kvvm)
Expand Down Expand Up @@ -250,6 +252,15 @@ func (h LifeCycleHandler) Handle(ctx context.Context, vmbda *virtv2.VirtualMachi
Reason(vmbdacondition.NotAttached).
Message(service.CapitalizeFirstLetter(err.Error()))
return reconcile.Result{}, nil
case blockDeviceLimitCondition.Status != metav1.ConditionTrue:
log.Info("Virtual machine block device capacity reached")

vmbda.Status.Phase = virtv2.BlockDeviceAttachmentPhasePending
cb.
Status(metav1.ConditionFalse).
Reason(vmbdacondition.NotAttached).
Message("Virtual machine block device capacity reached")
return reconcile.Result{}, nil
default:
return reconcile.Result{}, err
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,6 @@ import (
"fmt"
"log/slog"

"sigs.k8s.io/controller-runtime/pkg/client"
"sigs.k8s.io/controller-runtime/pkg/webhook/admission"

"github.com/deckhouse/virtualization-controller/pkg/common"
Expand All @@ -30,15 +29,13 @@ import (
)

type VMConnectLimiterValidator struct {
client client.Client
service *service.BlockDeviceService
log *slog.Logger
}

func NewVMConnectLimiterValidator(client client.Client, log *slog.Logger) *VMConnectLimiterValidator {
func NewVMConnectLimiterValidator(service *service.BlockDeviceService, log *slog.Logger) *VMConnectLimiterValidator {
return &VMConnectLimiterValidator{
client: client,
service: service.NewBlockDeviceService(client),
service: service,
log: log,
}
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -45,9 +45,11 @@ func NewController(
log := lg.With(logger.SlogController(ControllerName))

attacher := service.NewAttachmentService(mgr.GetClient(), ns)
blockDeviceService := service.NewBlockDeviceService(mgr.GetClient())

reconciler := NewReconciler(
mgr.GetClient(),
internal.NewBlockDeviceLimiter(blockDeviceService),
internal.NewBlockDeviceReadyHandler(attacher),
internal.NewVirtualMachineReadyHandler(attacher),
internal.NewLifeCycleHandler(attacher),
Expand All @@ -71,7 +73,7 @@ func NewController(

if err = builder.WebhookManagedBy(mgr).
For(&virtv2.VirtualMachineBlockDeviceAttachment{}).
WithValidator(NewValidator(attacher, mgr.GetClient(), log)).
WithValidator(NewValidator(attacher, blockDeviceService, log)).
Complete(); err != nil {
return nil, err
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,6 @@ import (
"log/slog"

"k8s.io/apimachinery/pkg/runtime"
"sigs.k8s.io/controller-runtime/pkg/client"
"sigs.k8s.io/controller-runtime/pkg/webhook/admission"

"github.com/deckhouse/virtualization-controller/pkg/controller/service"
Expand All @@ -40,13 +39,13 @@ type Validator struct {
log *slog.Logger
}

func NewValidator(attachmentService *service.AttachmentService, client client.Client, log *slog.Logger) *Validator {
func NewValidator(attachmentService *service.AttachmentService, service *service.BlockDeviceService, log *slog.Logger) *Validator {
return &Validator{
log: log.With("webhook", "validation"),
validators: []VirtualMachineBlockDeviceAttachmentValidator{
validators.NewSpecMutateValidator(),
validators.NewAttachmentConflictValidator(attachmentService, log),
validators.NewVMConnectLimiterValidator(client, log),
validators.NewVMConnectLimiterValidator(service, log),
},
}
}
Expand Down

0 comments on commit 394e461

Please sign in to comment.