checkup: Add guest-agent-ping probe and wait for ready state

Currently the checkup setup only waits for the VMI to finish "booting",
i.e. the guest agent service to be ready.
However this is not enough in order to ensure that the VMI has been
configured, a procedure currently done on the cloud-init service.
When the configuration is complete, the script configuring the guest in
the cloud-init service adds a marker file.

This commit:
- introduces a new waiting mechanism, using guest-agent-ping probe [0]
to poll-wait the guest until the file is present, and only then sets the
VMI ready condition to true.
- adds a wait for the VMI ready condition to be true.

[0]
https://kubevirt.io/user-guide/virtual_machines/liveness_and_readiness_probes/#defining-guest-agent-ping-probes

Signed-off-by: Ram Lavi <[email protected]>

pkg/internal/checkup/checkup.go

@@ -238,8 +238,45 @@ func (c *Checkup) setupVMIWaitReady(ctx context.Context, name string) (*kvcorev1
 	if err != nil {
 		return nil, fmt.Errorf("failed to wait on VMI %q boot: %w", vmiFullName, err)
 	}
+
+	log.Printf("Waiting for VMI %q ready condition...", vmiFullName)
+	_, err = c.waitForVMIReadyCondition(ctx, name)
+	if err != nil {
+		return nil, fmt.Errorf("failed to wait on VMI %q ready condition: %w", vmiFullName, err)
+	}
+
 	return updatedVMI, err
 }
+
+func (c *Checkup) waitForVMIReadyCondition(ctx context.Context, name string) (*kvcorev1.VirtualMachineInstance, error) {
+	vmiFullName := ObjectFullName(c.namespace, name)
+	var updatedVMI *kvcorev1.VirtualMachineInstance
+
+	conditionFn := func(ctx context.Context) (bool, error) {
+		var err error
+		updatedVMI, err = c.client.GetVirtualMachineInstance(ctx, c.namespace, name)
+		if err != nil {
+			return false, err
+		}
+
+		for _, condition := range updatedVMI.Status.Conditions {
+			if condition.Type == kvcorev1.VirtualMachineInstanceReady && condition.Status == k8scorev1.ConditionTrue {
+				return true, nil
+			}
+		}
+
+		return false, nil
+	}
+	const pollInterval = 5 * time.Second
+	if err := wait.PollImmediateUntilWithContext(ctx, pollInterval, conditionFn); err != nil {
+		return nil, fmt.Errorf("failed to wait for VMI %q to be ready: %v", vmiFullName, err)
+	}
+
+	log.Printf("VMI %q has successfully reached ready condition", vmiFullName)
+
+	return updatedVMI, nil
+}
+
 func (c *Checkup) waitForVMIToBoot(ctx context.Context, name string) (*kvcorev1.VirtualMachineInstance, error) {
 	vmiFullName := ObjectFullName(c.namespace, name)
 	log.Printf("Waiting for VMI %q to boot...", vmiFullName)

pkg/internal/checkup/checkup_test.go

@@ -417,10 +417,15 @@ func (cs *clientStub) GetVirtualMachineInstance(_ context.Context, namespace, na
 		return nil, k8serrors.NewNotFound(schema.GroupResource{Group: "kubevirt.io", Resource: "virtualmachineinstances"}, name)
 	}
 
-	vmi.Status.Conditions = append(vmi.Status.Conditions, kvcorev1.VirtualMachineInstanceCondition{
-		Type:   kvcorev1.VirtualMachineInstanceAgentConnected,
-		Status: k8scorev1.ConditionTrue,
-	})
+	vmi.Status.Conditions = append(vmi.Status.Conditions,
+		kvcorev1.VirtualMachineInstanceCondition{
+			Type:   kvcorev1.VirtualMachineInstanceAgentConnected,
+			Status: k8scorev1.ConditionTrue,
+		},
+		kvcorev1.VirtualMachineInstanceCondition{
+			Type:   kvcorev1.VirtualMachineInstanceReady,
+			Status: k8scorev1.ConditionTrue,
+		})
 
 	return vmi, nil
 }

pkg/internal/checkup/vmi.go

@@ -70,6 +70,7 @@ func newVMIUnderTest(name string, checkupConfig config.Config, configMapName str
 			CloudInit(config.VMIUsername, config.VMIPassword, vmiUnderTestBootCommands(configDiskSerial))),
 		vmi.WithConfigMapVolume(configVolumeName, configMapName),
 		vmi.WithConfigMapDisk(configVolumeName, configDiskSerial),
+		vmi.WithReadinessFileProbe(config.TunedAdmSetMarkerFileFullPath),
 	)
 
 	return vmi.New(name, optionsToApply...)
@@ -92,6 +93,7 @@ func newTrafficGen(name string, checkupConfig config.Config, configMapName strin
 		),
 		vmi.WithConfigMapVolume(configVolumeName, configMapName),
 		vmi.WithConfigMapDisk(configVolumeName, configDiskSerial),
+		vmi.WithReadinessFileProbe(config.TunedAdmSetMarkerFileFullPath),
 	)
 
 	return vmi.New(name, optionsToApply...)

pkg/internal/checkup/vmi/spec.go

@@ -129,6 +129,22 @@ func WithVirtIODisk(name string) Option {
 	}
 }
 
+func WithReadinessFileProbe(fileName string) Option {
+	return func(vmi *kvcorev1.VirtualMachineInstance) {
+		var readinessProbeCommand = []string{"cat", fileName}
+		vmi.Spec.ReadinessProbe = &kvcorev1.Probe{
+			Handler: kvcorev1.Handler{
+				Exec: &corev1.ExecAction{
+					Command: readinessProbeCommand,
+				},
+			},
+			FailureThreshold:    10,
+			InitialDelaySeconds: 60,
+			PeriodSeconds:       10,
+		}
+	}
+}
+
 func WithConfigMapDisk(name, serial string) Option {
 	return func(vmi *kvcorev1.VirtualMachineInstance) {
 		vmi.Spec.Domain.Devices.Disks = append(vmi.Spec.Domain.Devices.Disks,