diff --git a/api/core/v1alpha2/events.go b/api/core/v1alpha2/events.go index 8050fb86f..e6af316ea 100644 --- a/api/core/v1alpha2/events.go +++ b/api/core/v1alpha2/events.go @@ -22,8 +22,14 @@ const ( // ReasonVMChangesApplied is event reason that changes applied from VM to underlying KVVM. ReasonVMChangesApplied = "ChangesApplied" - // ReasonVMRestarted is event reason that VM restarted. - ReasonVMRestarted = "VMRestarted" + // ReasonVMStart is event reason that VM is about to start. + ReasonVMStart = "Start" + + // ReasonVMStop is event reason that VM is about to stop. + ReasonVMStop = "Stop" + + // ReasonVMRestart is event reason that VM is about to restart. + ReasonVMRestart = "Restart" // ReasonVMLastAppliedSpecInvalid is event reason that JSON in last-applied-spec annotation is invalid. ReasonVMLastAppliedSpecInvalid = "VMLastAppliedSpecInvalid" diff --git a/images/virtualization-artifact/pkg/controller/vm/internal/block_device.go b/images/virtualization-artifact/pkg/controller/vm/internal/block_device.go index 4b1dab110..9115ede6c 100644 --- a/images/virtualization-artifact/pkg/controller/vm/internal/block_device.go +++ b/images/virtualization-artifact/pkg/controller/vm/internal/block_device.go @@ -25,7 +25,6 @@ import ( corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/client-go/tools/record" virtv1 "kubevirt.io/api/core/v1" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/reconcile" @@ -35,6 +34,7 @@ import ( "github.com/deckhouse/virtualization-controller/pkg/controller/kvbuilder" "github.com/deckhouse/virtualization-controller/pkg/controller/service" "github.com/deckhouse/virtualization-controller/pkg/controller/vm/internal/state" + "github.com/deckhouse/virtualization-controller/pkg/eventrecord" "github.com/deckhouse/virtualization-controller/pkg/logger" virtv2 "github.com/deckhouse/virtualization/api/core/v1alpha2" "github.com/deckhouse/virtualization/api/core/v1alpha2/vmcondition" @@ -42,7 +42,7 @@ import ( const nameBlockDeviceHandler = "BlockDeviceHandler" -func NewBlockDeviceHandler(cl client.Client, recorder record.EventRecorder) *BlockDeviceHandler { +func NewBlockDeviceHandler(cl client.Client, recorder eventrecord.EventRecorderLogger) *BlockDeviceHandler { return &BlockDeviceHandler{ client: cl, recorder: recorder, @@ -55,7 +55,7 @@ func NewBlockDeviceHandler(cl client.Client, recorder record.EventRecorder) *Blo type BlockDeviceHandler struct { client client.Client - recorder record.EventRecorder + recorder eventrecord.EventRecorderLogger viProtection *service.ProtectionService cviProtection *service.ProtectionService diff --git a/images/virtualization-artifact/pkg/controller/vm/internal/class.go b/images/virtualization-artifact/pkg/controller/vm/internal/class.go index 0f86d3d02..759c3ba35 100644 --- a/images/virtualization-artifact/pkg/controller/vm/internal/class.go +++ b/images/virtualization-artifact/pkg/controller/vm/internal/class.go @@ -23,12 +23,12 @@ import ( corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/client-go/tools/record" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/reconcile" "github.com/deckhouse/virtualization-controller/pkg/controller/conditions" "github.com/deckhouse/virtualization-controller/pkg/controller/vm/internal/state" + "github.com/deckhouse/virtualization-controller/pkg/eventrecord" "github.com/deckhouse/virtualization-controller/pkg/logger" virtv2 "github.com/deckhouse/virtualization/api/core/v1alpha2" "github.com/deckhouse/virtualization/api/core/v1alpha2/vmcondition" @@ -36,7 +36,7 @@ import ( const nameClassHandler = "ClassHandler" -func NewClassHandler(client client.Client, recorder record.EventRecorder) *ClassHandler { +func NewClassHandler(client client.Client, recorder eventrecord.EventRecorderLogger) *ClassHandler { return &ClassHandler{ client: client, recorder: recorder, @@ -45,7 +45,7 @@ func NewClassHandler(client client.Client, recorder record.EventRecorder) *Class type ClassHandler struct { client client.Client - recorder record.EventRecorder + recorder eventrecord.EventRecorderLogger } func (h *ClassHandler) Handle(ctx context.Context, s state.VirtualMachineState) (reconcile.Result, error) { diff --git a/images/virtualization-artifact/pkg/controller/vm/internal/ipam.go b/images/virtualization-artifact/pkg/controller/vm/internal/ipam.go index 9cd1c9eb1..df30746ce 100644 --- a/images/virtualization-artifact/pkg/controller/vm/internal/ipam.go +++ b/images/virtualization-artifact/pkg/controller/vm/internal/ipam.go @@ -24,7 +24,6 @@ import ( corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/client-go/tools/record" virtv1 "kubevirt.io/api/core/v1" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/reconcile" @@ -33,6 +32,7 @@ import ( "github.com/deckhouse/virtualization-controller/pkg/controller/kvbuilder" "github.com/deckhouse/virtualization-controller/pkg/controller/service" "github.com/deckhouse/virtualization-controller/pkg/controller/vm/internal/state" + "github.com/deckhouse/virtualization-controller/pkg/eventrecord" "github.com/deckhouse/virtualization-controller/pkg/logger" virtv2 "github.com/deckhouse/virtualization/api/core/v1alpha2" "github.com/deckhouse/virtualization/api/core/v1alpha2/vmcondition" @@ -46,7 +46,7 @@ type IPAM interface { CreateIPAddress(ctx context.Context, vm *virtv2.VirtualMachine, client client.Client) error } -func NewIPAMHandler(ipam IPAM, cl client.Client, recorder record.EventRecorder) *IPAMHandler { +func NewIPAMHandler(ipam IPAM, cl client.Client, recorder eventrecord.EventRecorderLogger) *IPAMHandler { return &IPAMHandler{ ipam: ipam, client: cl, @@ -58,7 +58,7 @@ func NewIPAMHandler(ipam IPAM, cl client.Client, recorder record.EventRecorder) type IPAMHandler struct { ipam IPAM client client.Client - recorder record.EventRecorder + recorder eventrecord.EventRecorderLogger protection *service.ProtectionService } diff --git a/images/virtualization-artifact/pkg/controller/vm/internal/lifecycle.go b/images/virtualization-artifact/pkg/controller/vm/internal/lifecycle.go index 94e1ff1a6..d4197a180 100644 --- a/images/virtualization-artifact/pkg/controller/vm/internal/lifecycle.go +++ b/images/virtualization-artifact/pkg/controller/vm/internal/lifecycle.go @@ -23,7 +23,6 @@ import ( corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/client-go/tools/record" virtv1 "kubevirt.io/api/core/v1" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/reconcile" @@ -32,6 +31,7 @@ import ( "github.com/deckhouse/virtualization-controller/pkg/controller/conditions" "github.com/deckhouse/virtualization-controller/pkg/controller/service" "github.com/deckhouse/virtualization-controller/pkg/controller/vm/internal/state" + "github.com/deckhouse/virtualization-controller/pkg/eventrecord" "github.com/deckhouse/virtualization-controller/pkg/logger" virtv2 "github.com/deckhouse/virtualization/api/core/v1alpha2" "github.com/deckhouse/virtualization/api/core/v1alpha2/vmcondition" @@ -46,13 +46,13 @@ var lifeCycleConditions = []vmcondition.Type{ const nameLifeCycleHandler = "LifeCycleHandler" -func NewLifeCycleHandler(client client.Client, recorder record.EventRecorder) *LifeCycleHandler { +func NewLifeCycleHandler(client client.Client, recorder eventrecord.EventRecorderLogger) *LifeCycleHandler { return &LifeCycleHandler{client: client, recorder: recorder} } type LifeCycleHandler struct { client client.Client - recorder record.EventRecorder + recorder eventrecord.EventRecorderLogger } func (h *LifeCycleHandler) Handle(ctx context.Context, s state.VirtualMachineState) (reconcile.Result, error) { diff --git a/images/virtualization-artifact/pkg/controller/vm/internal/sync_kvvm.go b/images/virtualization-artifact/pkg/controller/vm/internal/sync_kvvm.go index 6360cea4d..143ca5b26 100644 --- a/images/virtualization-artifact/pkg/controller/vm/internal/sync_kvvm.go +++ b/images/virtualization-artifact/pkg/controller/vm/internal/sync_kvvm.go @@ -26,7 +26,6 @@ import ( corev1 "k8s.io/api/core/v1" k8serrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/client-go/tools/record" virtv1 "kubevirt.io/api/core/v1" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/reconcile" @@ -41,6 +40,7 @@ import ( "github.com/deckhouse/virtualization-controller/pkg/controller/vm/internal/state" "github.com/deckhouse/virtualization-controller/pkg/controller/vmchange" "github.com/deckhouse/virtualization-controller/pkg/dvcr" + "github.com/deckhouse/virtualization-controller/pkg/eventrecord" "github.com/deckhouse/virtualization-controller/pkg/logger" virtv2 "github.com/deckhouse/virtualization/api/core/v1alpha2" "github.com/deckhouse/virtualization/api/core/v1alpha2/vmcondition" @@ -48,7 +48,7 @@ import ( const nameSyncKvvmHandler = "SyncKvvmHandler" -func NewSyncKvvmHandler(dvcrSettings *dvcr.Settings, client client.Client, recorder record.EventRecorder) *SyncKvvmHandler { +func NewSyncKvvmHandler(dvcrSettings *dvcr.Settings, client client.Client, recorder eventrecord.EventRecorderLogger) *SyncKvvmHandler { return &SyncKvvmHandler{ dvcrSettings: dvcrSettings, client: client, @@ -58,7 +58,7 @@ func NewSyncKvvmHandler(dvcrSettings *dvcr.Settings, client client.Client, recor type SyncKvvmHandler struct { client client.Client - recorder record.EventRecorder + recorder eventrecord.EventRecorderLogger dvcrSettings *dvcr.Settings } @@ -489,7 +489,7 @@ func (h *SyncKvvmHandler) applyVMChangesToKVVM(ctx context.Context, s state.Virt log.Info("Restart VM to apply changes", "vm.name", current.GetName()) h.recorder.Event(current, corev1.EventTypeNormal, virtv2.ReasonVMChangesApplied, "Apply disruptive changes") - h.recorder.Event(current, corev1.EventTypeNormal, virtv2.ReasonVMRestarted, "") + h.recorder.Event(current, corev1.EventTypeNormal, virtv2.ReasonVMRestart, "") // Update KVVM spec according the current VM spec. if err = h.updateKVVM(ctx, s); err != nil { @@ -568,7 +568,7 @@ func (h *SyncKvvmHandler) syncPowerState(ctx context.Context, s state.VirtualMac // Ensure KVVMI is absent. err = h.client.Delete(ctx, kvvmi) if err != nil && !k8serrors.IsNotFound(err) { - return fmt.Errorf("force AlwaysOff: delete KVVMI: %w", err) + return fmt.Errorf("automatic stop VM for %s policy: delete KVVMI: %w", vmRunPolicy, err) } } err = h.ensureRunStrategy(ctx, kvvm, virtv1.RunStrategyHalted) @@ -580,7 +580,7 @@ func (h *SyncKvvmHandler) syncPowerState(ctx context.Context, s state.VirtualMac strategy, _ := kvvm.RunStrategy() if strategy == virtv1.RunStrategyAlways && kvvmi == nil { if err = powerstate.StartVM(ctx, h.client, kvvm); err != nil { - return fmt.Errorf("failed to start VM: %w", err) + return fmt.Errorf("automatic start VM for %s policy: %w", vmRunPolicy, err) } } if kvvmi != nil && kvvmi.DeletionTimestamp == nil { @@ -605,10 +605,23 @@ func (h *SyncKvvmHandler) syncPowerState(ctx context.Context, s state.VirtualMac } } if kvvmi.Status.Phase == virtv1.Failed { - log.Info("Restart on Failed KVVMI", "obj", kvvmi.GetName()) + //h.recorder.WithLogging(log).RecordEvent(&eventrecord.Event{ + // InvolvedObject: s.VirtualMachine().Current(), + // Type: corev1.EventTypeNormal, + // Reason: "Restart", + // Message: fmt.Sprintf("Restart initiated by controller for %s runPolicy after observing failed VM instance", vmRunPolicy)), + //}) + h.recorder.WithLogging(log).Eventf( + s.VirtualMachine().Current(), + corev1.EventTypeNormal, + virtv2.ReasonVMRestart, + "Restart initiated by controller for %s runPolicy after observing failed VM instance", + vmRunPolicy, + ) + err = powerstate.SafeRestartVM(ctx, h.client, kvvm, kvvmi) if err != nil { - return fmt.Errorf("restart VM on failed: %w", err) + return fmt.Errorf("automatic restart of Failed VM: %w", err) } } } diff --git a/images/virtualization-artifact/pkg/controller/vm/vm_controller.go b/images/virtualization-artifact/pkg/controller/vm/vm_controller.go index 5e2b47200..2c84e8c1e 100644 --- a/images/virtualization-artifact/pkg/controller/vm/vm_controller.go +++ b/images/virtualization-artifact/pkg/controller/vm/vm_controller.go @@ -27,10 +27,12 @@ import ( "sigs.k8s.io/controller-runtime/pkg/metrics" "github.com/deckhouse/deckhouse/pkg/log" + "github.com/deckhouse/virtualization-controller/pkg/controller/ipam" "github.com/deckhouse/virtualization-controller/pkg/controller/service" "github.com/deckhouse/virtualization-controller/pkg/controller/vm/internal" "github.com/deckhouse/virtualization-controller/pkg/dvcr" + "github.com/deckhouse/virtualization-controller/pkg/eventrecord" "github.com/deckhouse/virtualization-controller/pkg/logger" vmmetrics "github.com/deckhouse/virtualization-controller/pkg/monitoring/metrics/virtualmachine" "github.com/deckhouse/virtualization/api/core/v1alpha2" @@ -47,7 +49,7 @@ func SetupController( dvcrSettings *dvcr.Settings, ) error { log := lg.With(logger.SlogController(controllerName)) - recorder := mgr.GetEventRecorderFor(controllerName) + recorder := eventrecord.NewEventRecorderLogger(mgr.GetEventRecorderFor(controllerName)) mgrCache := mgr.GetCache() client := mgr.GetClient() blockDeviceService := service.NewBlockDeviceService(client) diff --git a/images/virtualization-artifact/pkg/eventrecord/event.go b/images/virtualization-artifact/pkg/eventrecord/event.go new file mode 100644 index 000000000..02de6e589 --- /dev/null +++ b/images/virtualization-artifact/pkg/eventrecord/event.go @@ -0,0 +1,49 @@ +package eventrecord + +import ( + corev1 "k8s.io/api/core/v1" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +type Event struct { + // The object that this event is about. + InvolvedObject client.Object + + // The top object that this event is related to. E.g. VirtualMachine for VirtualMachineOperation. + // Only for logging, will not be reflected in the Event resource. + RelatedObject client.Object + + // Type of this event (Normal, Warning), new types could be added in the future + Type string + + // This should be a short, machine understandable string that gives the reason + // for the transition into the object's current status. + Reason string + + // A human-readable description of the status of this operation. + Message string +} + +func NewNormalEvent(involvedObject client.Object, reason, message string) *Event { + return &Event{ + InvolvedObject: involvedObject, + Type: corev1.EventTypeNormal, + Reason: reason, + Message: message, + } +} + +func NewWarningEvent(involvedObject client.Object, reason, message string) *Event { + return &Event{ + InvolvedObject: involvedObject, + Type: corev1.EventTypeWarning, + Reason: reason, + Message: message, + } +} + +func NewNormalEventWithRelated(involvedObject client.Object, relatedObject client.Object, reason, message string) *Event { + ev := NewNormalEvent(involvedObject, reason, message) + ev.RelatedObject = relatedObject + return ev +} diff --git a/images/virtualization-artifact/pkg/eventrecord/eventrecorderlogger.go b/images/virtualization-artifact/pkg/eventrecord/eventrecorderlogger.go new file mode 100644 index 000000000..0c092965d --- /dev/null +++ b/images/virtualization-artifact/pkg/eventrecord/eventrecorderlogger.go @@ -0,0 +1,125 @@ +package eventrecord + +import ( + "fmt" + + "k8s.io/client-go/tools/record" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +const ( + eventTypeLabel = "eventType" + reasonLabel = "reason" + involvedNameLabel = "involvedName" + involvedNamespaceLabel = "involvedNamespace" + involvedKindLabel = "involvedKind" + relatedNameLabel = "relatedName" + relatedNamespaceLabel = "relatedNamespace" + relatedKindLabel = "relatedKind" +) + +// infoLogger is local interface to use Info method from different loggers. +type infoLogger interface { + Info(msg string, args ...any) +} + +// EventRecorderLogger is a wrapper around client-go's EventRecorder to record Events with logging. +type EventRecorderLogger interface { + Event(object client.Object, eventtype, reason, message string) + + // Eventf is just like Event, but with Sprintf for the message field. + Eventf(involved client.Object, eventtype, reason, messageFmt string, args ...interface{}) + + // AnnotatedEventf is just like eventf, but with annotations attached + AnnotatedEventf(involved client.Object, annotations map[string]string, eventtype, reason, messageFmt string, args ...interface{}) + + WithLogging(logger infoLogger) EventRecorderLogger +} + +func NewEventRecorderLogger(recorder record.EventRecorder) EventRecorderLogger { + return &EventRecorderLoggerImpl{recorder: recorder} +} + +// EventRecorderLoggerImpl implements Event recorder that also log event object. +type EventRecorderLoggerImpl struct { + recorder record.EventRecorder + logger infoLogger +} + +func (e *EventRecorderLoggerImpl) WithLogging(logger infoLogger) EventRecorderLogger { + return &EventRecorderLoggerImpl{ + recorder: e.recorder, + logger: logger, + } +} + +//// FromObject records an event from an existing event object. +//// Notice that only involvedObject, type, reason, and message are used. +//func (e *EventRecorderLoggerImpl) FromObject(ev *Event) { +// e.recorder.Event(ev.InvolvedObject, ev.Type, ev.Reason, ev.Message) +//} + +//// EventRelated records an event and adds related object info to log. +//func (e *EventRecorderLoggerImpl) EventRelated(object client.Object, related client.Object, eventtype, reason, message string) { +// e.recorder.Event(object, eventtype, reason, message) +// e.log(&Event{ +// InvolvedObject: object, +// RelatedObject: related, +// Type: eventtype, +// Reason: reason, +// Message: message, +// }) +//} + +// Event calls EventRecorder.Event as-is. +func (e *EventRecorderLoggerImpl) Event(object client.Object, eventtype, reason, message string) { + e.recorder.Event(object, eventtype, reason, message) + e.logf(object, eventtype, reason, message) +} + +// Eventf calls EventRecorder.Eventf as-is. +func (e *EventRecorderLoggerImpl) Eventf(object client.Object, eventtype, reason, messageFmt string, args ...interface{}) { + e.recorder.Eventf(object, eventtype, reason, messageFmt, args) + e.logf(object, eventtype, reason, messageFmt, args...) +} + +// AnnotatedEventf calls EventRecorder.AnnotatedEventf as-is. +func (e *EventRecorderLoggerImpl) AnnotatedEventf(object client.Object, annotations map[string]string, eventtype, reason, messageFmt string, args ...interface{}) { + e.recorder.AnnotatedEventf(object, annotations, eventtype, reason, messageFmt, args) + e.logf(object, eventtype, reason, messageFmt, args...) +} + +func (e *EventRecorderLoggerImpl) log(ev *Event) { + if e.logger == nil { + return + } + args := []any{ + eventTypeLabel, ev.Type, + reasonLabel, ev.Reason, + involvedNameLabel, ev.InvolvedObject.GetName(), + involvedNamespaceLabel, ev.InvolvedObject.GetNamespace(), + involvedKindLabel, ev.InvolvedObject.GetObjectKind().GroupVersionKind().Kind, + } + if ev.RelatedObject != nil { + args = append(args, + relatedNameLabel, ev.RelatedObject.GetName(), + relatedNamespaceLabel, ev.RelatedObject.GetNamespace(), + relatedKindLabel, ev.RelatedObject.GetObjectKind().GroupVersionKind().Kind, + ) + } + e.logger.Info(ev.Message, args...) +} + +func (e *EventRecorderLoggerImpl) logf(involved client.Object, eventtype, reason, messageFmt string, args ...interface{}) { + if e.logger == nil { + return + } + e.logger.Info( + fmt.Sprintf(messageFmt, args...), + eventTypeLabel, eventtype, + reasonLabel, reason, + involvedNameLabel, involved.GetName(), + involvedNamespaceLabel, involved.GetNamespace(), + involvedKindLabel, involved.GetObjectKind().GroupVersionKind().Kind, + ) +}