From 81c9773c027d9a976dcf33c2503c0906781f38ce Mon Sep 17 00:00:00 2001 From: yaroslavborbat Date: Tue, 10 Dec 2024 10:30:46 +0300 Subject: [PATCH 01/11] add Signed-off-by: yaroslavborbat --- .../patches/031-hotplug-container-disk.patch | 1849 +++++++++++++++++ 1 file changed, 1849 insertions(+) create mode 100644 images/virt-artifact/patches/031-hotplug-container-disk.patch diff --git a/images/virt-artifact/patches/031-hotplug-container-disk.patch b/images/virt-artifact/patches/031-hotplug-container-disk.patch new file mode 100644 index 000000000..d9cf34277 --- /dev/null +++ b/images/virt-artifact/patches/031-hotplug-container-disk.patch @@ -0,0 +1,1849 @@ +diff --git a/api/openapi-spec/swagger.json b/api/openapi-spec/swagger.json +index c4822a0448..d6bb534249 100644 +--- a/api/openapi-spec/swagger.json ++++ b/api/openapi-spec/swagger.json +@@ -12951,6 +12951,9 @@ + "image" + ], + "properties": { ++ "hotpluggable": { ++ "type": "boolean" ++ }, + "image": { + "description": "Image is the name of the image with the embedded disk.", + "type": "string", +@@ -13973,6 +13976,9 @@ + "description": "HotplugVolumeSource Represents the source of a volume to mount which are capable of being hotplugged on a live running VMI. Only one of its members may be specified.", + "type": "object", + "properties": { ++ "containerDisk": { ++ "$ref": "#/definitions/v1.ContainerDiskSource" ++ }, + "dataVolume": { + "description": "DataVolume represents the dynamic creation a PVC for this volume as well as the process of populating that PVC with a disk image.", + "$ref": "#/definitions/v1.DataVolumeSource" +diff --git a/cmd/virt-chroot/main.go b/cmd/virt-chroot/main.go +index e28daa07c7..7a69b7451b 100644 +--- a/cmd/virt-chroot/main.go ++++ b/cmd/virt-chroot/main.go +@@ -20,6 +20,7 @@ var ( + cpuTime uint64 + memoryBytes uint64 + targetUser string ++ targetUserID int + ) + + func init() { +@@ -51,7 +52,12 @@ func main() { + + // Looking up users needs resources, let's do it before we set rlimits. + var u *user.User +- if targetUser != "" { ++ if targetUserID >= 0 { ++ _, _, errno := syscall.Syscall(syscall.SYS_SETUID, uintptr(targetUserID), 0, 0) ++ if errno != 0 { ++ return fmt.Errorf("failed to switch to user: %d. errno: %d", targetUserID, errno) ++ } ++ } else if targetUser != "" { + var err error + u, err = user.Lookup(targetUser) + if err != nil { +@@ -116,6 +122,7 @@ func main() { + rootCmd.PersistentFlags().Uint64Var(&memoryBytes, "memory", 0, "memory in bytes for the process") + rootCmd.PersistentFlags().StringVar(&mntNamespace, "mount", "", "mount namespace to use") + rootCmd.PersistentFlags().StringVar(&targetUser, "user", "", "switch to this targetUser to e.g. drop privileges") ++ rootCmd.PersistentFlags().IntVar(&targetUserID, "userid", -1, "switch to this targetUser to e.g. drop privileges") + + execCmd := &cobra.Command{ + Use: "exec", +@@ -136,16 +143,39 @@ func main() { + Args: cobra.MinimumNArgs(2), + RunE: func(cmd *cobra.Command, args []string) error { + var mntOpts uint = 0 ++ var dataOpts []string + + fsType := cmd.Flag("type").Value.String() + mntOptions := cmd.Flag("options").Value.String() ++ var ( ++ uid = -1 ++ gid = -1 ++ ) + for _, opt := range strings.Split(mntOptions, ",") { + opt = strings.TrimSpace(opt) +- switch opt { +- case "ro": ++ switch { ++ case opt == "ro": + mntOpts = mntOpts | syscall.MS_RDONLY +- case "bind": ++ case opt == "bind": + mntOpts = mntOpts | syscall.MS_BIND ++ case opt == "remount": ++ mntOpts = mntOpts | syscall.MS_REMOUNT ++ case strings.HasPrefix(opt, "uid="): ++ uidS := strings.TrimPrefix(opt, "uid=") ++ uidI, err := strconv.Atoi(uidS) ++ if err != nil { ++ return fmt.Errorf("failed to parse uid: %w", err) ++ } ++ uid = uidI ++ dataOpts = append(dataOpts, opt) ++ case strings.HasPrefix(opt, "gid="): ++ gidS := strings.TrimPrefix(opt, "gid=") ++ gidI, err := strconv.Atoi(gidS) ++ if err != nil { ++ return fmt.Errorf("failed to parse gid: %w", err) ++ } ++ gid = gidI ++ dataOpts = append(dataOpts, opt) + default: + return fmt.Errorf("mount option %s is not supported", opt) + } +@@ -168,8 +198,17 @@ func main() { + return fmt.Errorf("mount target invalid: %v", err) + } + defer targetFile.Close() +- +- return syscall.Mount(sourceFile.SafePath(), targetFile.SafePath(), fsType, uintptr(mntOpts), "") ++ if uid >= 0 && gid >= 0 { ++ err = os.Chown(targetFile.SafePath(), uid, gid) ++ if err != nil { ++ return fmt.Errorf("chown target failed: %w", err) ++ } ++ } ++ var data string ++ if len(dataOpts) > 0 { ++ data = strings.Join(dataOpts, ",") ++ } ++ return syscall.Mount(sourceFile.SafePath(), targetFile.SafePath(), fsType, uintptr(mntOpts), data) + }, + } + mntCmd.Flags().StringP("options", "o", "", "comma separated list of mount options") +diff --git a/manifests/generated/kv-resource.yaml b/manifests/generated/kv-resource.yaml +index 66d1b01dbf..43e36b7195 100644 +--- a/manifests/generated/kv-resource.yaml ++++ b/manifests/generated/kv-resource.yaml +@@ -3307,9 +3307,6 @@ spec: + - jsonPath: .status.phase + name: Phase + type: string +- deprecated: true +- deprecationWarning: kubevirt.io/v1alpha3 is now deprecated and will be removed +- in a future release. + name: v1alpha3 + schema: + openAPIV3Schema: +diff --git a/manifests/generated/operator-csv.yaml.in b/manifests/generated/operator-csv.yaml.in +index 400d118024..05ee099c67 100644 +--- a/manifests/generated/operator-csv.yaml.in ++++ b/manifests/generated/operator-csv.yaml.in +@@ -605,6 +605,13 @@ spec: + - '*' + verbs: + - '*' ++ - apiGroups: ++ - subresources.virtualization.deckhouse.io ++ resources: ++ - virtualmachines/addvolume ++ - virtualmachines/removevolume ++ verbs: ++ - update + - apiGroups: + - subresources.kubevirt.io + resources: +diff --git a/manifests/generated/rbac-operator.authorization.k8s.yaml.in b/manifests/generated/rbac-operator.authorization.k8s.yaml.in +index 10dbb92269..1ccc9e9fa7 100644 +--- a/manifests/generated/rbac-operator.authorization.k8s.yaml.in ++++ b/manifests/generated/rbac-operator.authorization.k8s.yaml.in +@@ -143,7 +143,7 @@ kind: RoleBinding + metadata: + labels: + kubevirt.io: "" +- name: kubevirt-operator-rolebinding ++ name: kubevirt-operator + namespace: {{.Namespace}} + roleRef: + apiGroup: rbac.authorization.k8s.io +@@ -607,6 +607,13 @@ rules: + - '*' + verbs: + - '*' ++- apiGroups: ++ - subresources.virtualization.deckhouse.io ++ resources: ++ - virtualmachines/addvolume ++ - virtualmachines/removevolume ++ verbs: ++ - update + - apiGroups: + - subresources.kubevirt.io + resources: +diff --git a/pkg/container-disk/container-disk.go b/pkg/container-disk/container-disk.go +index 3251d04787..34affe841a 100644 +--- a/pkg/container-disk/container-disk.go ++++ b/pkg/container-disk/container-disk.go +@@ -47,8 +47,10 @@ var containerDiskOwner = "qemu" + var podsBaseDir = util.KubeletPodsDir + + var mountBaseDir = filepath.Join(util.VirtShareDir, "/container-disks") ++var hotplugBaseDir = filepath.Join(util.VirtShareDir, "/hotplug-disks") + + type SocketPathGetter func(vmi *v1.VirtualMachineInstance, volumeIndex int) (string, error) ++type HotplugSocketPathGetter func(vmi *v1.VirtualMachineInstance, volumeName string) (string, error) + type KernelBootSocketPathGetter func(vmi *v1.VirtualMachineInstance) (string, error) + + const KernelBootName = "kernel-boot" +@@ -107,6 +109,10 @@ func GetDiskTargetPathFromLauncherView(volumeIndex int) string { + return filepath.Join(mountBaseDir, GetDiskTargetName(volumeIndex)) + } + ++func GetHotplugContainerDiskTargetPathFromLauncherView(volumeName string) string { ++ return filepath.Join(hotplugBaseDir, fmt.Sprintf("%s.img", volumeName)) ++} ++ + func GetKernelBootArtifactPathFromLauncherView(artifact string) string { + artifactBase := filepath.Base(artifact) + return filepath.Join(mountBaseDir, KernelBootName, artifactBase) +@@ -170,6 +176,23 @@ func NewSocketPathGetter(baseDir string) SocketPathGetter { + } + } + ++func NewHotplugSocketPathGetter(baseDir string) HotplugSocketPathGetter { ++ return func(vmi *v1.VirtualMachineInstance, volumeName string) (string, error) { ++ for _, v := range vmi.Status.VolumeStatus { ++ if v.Name == volumeName && v.HotplugVolume != nil && v.ContainerDiskVolume != nil { ++ basePath := getHotplugContainerDiskSocketBasePath(baseDir, string(v.HotplugVolume.AttachPodUID)) ++ socketPath := filepath.Join(basePath, fmt.Sprintf("hotplug-container-disk-%s.sock", volumeName)) ++ exists, _ := diskutils.FileExists(socketPath) ++ if exists { ++ return socketPath, nil ++ } ++ } ++ } ++ ++ return "", fmt.Errorf("container disk socket path not found for vmi \"%s\"", vmi.Name) ++ } ++} ++ + // NewKernelBootSocketPathGetter get the socket pat of the kernel-boot containerDisk. For testing a baseDir + // can be provided which can for instance point to /tmp. + func NewKernelBootSocketPathGetter(baseDir string) KernelBootSocketPathGetter { +@@ -394,10 +417,37 @@ func CreateEphemeralImages( + return nil + } + ++func CreateEphemeralImagesForHotplug( ++ vmi *v1.VirtualMachineInstance, ++ diskCreator ephemeraldisk.EphemeralDiskCreatorInterface, ++ disksInfo map[string]*DiskInfo, ++) error { ++ for i, volume := range vmi.Spec.Volumes { ++ if volume.VolumeSource.ContainerDisk != nil && volume.VolumeSource.ContainerDisk.Hotpluggable { ++ info, _ := disksInfo[volume.Name] ++ if info == nil { ++ return fmt.Errorf("no disk info provided for volume %s", volume.Name) ++ } ++ ++ if backingFile, err := GetDiskTargetPartFromLauncherView(i); err != nil { ++ return err ++ } else if err := diskCreator.CreateBackedImageForVolume(volume, backingFile, info.Format); err != nil { ++ return err ++ } ++ } ++ } ++ ++ return nil ++} ++ + func getContainerDiskSocketBasePath(baseDir, podUID string) string { + return fmt.Sprintf("%s/pods/%s/volumes/kubernetes.io~empty-dir/container-disks", baseDir, podUID) + } + ++func getHotplugContainerDiskSocketBasePath(baseDir, podUID string) string { ++ return fmt.Sprintf("%s/pods/%s/volumes/kubernetes.io~empty-dir/hotplug-container-disks", baseDir, podUID) ++} ++ + // ExtractImageIDsFromSourcePod takes the VMI and its source pod to determine the exact image used by containerdisks and boot container images, + // which is recorded in the status section of a started pod; if the status section does not contain this info the tag is used. + // It returns a map where the key is the vlume name and the value is the imageID +diff --git a/pkg/controller/controller.go b/pkg/controller/controller.go +index 490cc445ef..4b7dbc12fe 100644 +--- a/pkg/controller/controller.go ++++ b/pkg/controller/controller.go +@@ -278,6 +278,10 @@ func ApplyVolumeRequestOnVMISpec(vmiSpec *v1.VirtualMachineInstanceSpec, request + dvSource := request.AddVolumeOptions.VolumeSource.DataVolume.DeepCopy() + dvSource.Hotpluggable = true + newVolume.VolumeSource.DataVolume = dvSource ++ } else if request.AddVolumeOptions.VolumeSource.ContainerDisk != nil { ++ containerDiskSource := request.AddVolumeOptions.VolumeSource.ContainerDisk.DeepCopy() ++ containerDiskSource.Hotpluggable = true ++ newVolume.VolumeSource.ContainerDisk = containerDiskSource + } + + vmiSpec.Volumes = append(vmiSpec.Volumes, newVolume) +@@ -444,6 +448,9 @@ func VMIHasHotplugVolumes(vmi *v1.VirtualMachineInstance) bool { + if volume.PersistentVolumeClaim != nil && volume.PersistentVolumeClaim.Hotpluggable { + return true + } ++ if volume.ContainerDisk != nil && volume.ContainerDisk.Hotpluggable { ++ return true ++ } + } + return false + } +@@ -557,7 +564,7 @@ func GetHotplugVolumes(vmi *v1.VirtualMachineInstance, virtlauncherPod *k8sv1.Po + podVolumeMap[podVolume.Name] = podVolume + } + for _, vmiVolume := range vmiVolumes { +- if _, ok := podVolumeMap[vmiVolume.Name]; !ok && (vmiVolume.DataVolume != nil || vmiVolume.PersistentVolumeClaim != nil || vmiVolume.MemoryDump != nil) { ++ if _, ok := podVolumeMap[vmiVolume.Name]; !ok && (vmiVolume.DataVolume != nil || vmiVolume.PersistentVolumeClaim != nil || vmiVolume.MemoryDump != nil || vmiVolume.ContainerDisk != nil) { + hotplugVolumes = append(hotplugVolumes, vmiVolume.DeepCopy()) + } + } +diff --git a/pkg/virt-api/rest/subresource.go b/pkg/virt-api/rest/subresource.go +index b5d62f5af5..bf561f00ae 100644 +--- a/pkg/virt-api/rest/subresource.go ++++ b/pkg/virt-api/rest/subresource.go +@@ -1023,7 +1023,8 @@ func volumeSourceName(volumeSource *v1.HotplugVolumeSource) string { + + func volumeSourceExists(volume v1.Volume, volumeName string) bool { + return (volume.DataVolume != nil && volume.DataVolume.Name == volumeName) || +- (volume.PersistentVolumeClaim != nil && volume.PersistentVolumeClaim.ClaimName == volumeName) ++ (volume.PersistentVolumeClaim != nil && volume.PersistentVolumeClaim.ClaimName == volumeName) || ++ (volume.ContainerDisk != nil && volume.ContainerDisk.Image != "") + } + + func volumeExists(volume v1.Volume, volumeName string) bool { +@@ -1125,6 +1126,8 @@ func (app *SubresourceAPIApp) addVolumeRequestHandler(request *restful.Request, + opts.VolumeSource.DataVolume.Hotpluggable = true + } else if opts.VolumeSource.PersistentVolumeClaim != nil { + opts.VolumeSource.PersistentVolumeClaim.Hotpluggable = true ++ } else if opts.VolumeSource.ContainerDisk != nil { ++ opts.VolumeSource.ContainerDisk.Hotpluggable = true + } + + // inject into VMI if ephemeral, else set as a request on the VM to both make permanent and hotplug. +diff --git a/pkg/virt-api/webhooks/validating-webhook/admitters/vmi-update-admitter.go b/pkg/virt-api/webhooks/validating-webhook/admitters/vmi-update-admitter.go +index 0af25f8074..803c0ed4cd 100644 +--- a/pkg/virt-api/webhooks/validating-webhook/admitters/vmi-update-admitter.go ++++ b/pkg/virt-api/webhooks/validating-webhook/admitters/vmi-update-admitter.go +@@ -200,11 +200,11 @@ func verifyHotplugVolumes(newHotplugVolumeMap, oldHotplugVolumeMap map[string]v1 + } + } else { + // This is a new volume, ensure that the volume is either DV, PVC or memoryDumpVolume +- if v.DataVolume == nil && v.PersistentVolumeClaim == nil && v.MemoryDump == nil { ++ if v.DataVolume == nil && v.PersistentVolumeClaim == nil && v.MemoryDump == nil && v.ContainerDisk == nil { + return webhookutils.ToAdmissionResponse([]metav1.StatusCause{ + { + Type: metav1.CauseTypeFieldValueInvalid, +- Message: fmt.Sprintf("volume %s is not a PVC or DataVolume", k), ++ Message: fmt.Sprintf("volume %s is not a PVC,DataVolume,MemoryDumpVolume or ContainerDisk", k), + }, + }) + } +diff --git a/pkg/virt-controller/services/template.go b/pkg/virt-controller/services/template.go +index 76ed7307ec..f607c24786 100644 +--- a/pkg/virt-controller/services/template.go ++++ b/pkg/virt-controller/services/template.go +@@ -64,13 +64,15 @@ import ( + ) + + const ( +- containerDisks = "container-disks" +- hotplugDisks = "hotplug-disks" +- hookSidecarSocks = "hook-sidecar-sockets" +- varRun = "/var/run" +- virtBinDir = "virt-bin-share-dir" +- hotplugDisk = "hotplug-disk" +- virtExporter = "virt-exporter" ++ containerDisks = "container-disks" ++ hotplugDisks = "hotplug-disks" ++ hookSidecarSocks = "hook-sidecar-sockets" ++ varRun = "/var/run" ++ virtBinDir = "virt-bin-share-dir" ++ hotplugDisk = "hotplug-disk" ++ virtExporter = "virt-exporter" ++ hotplugContainerDisks = "hotplug-container-disks" ++ HotplugContainerDisk = "hotplug-container-disk-" + ) + + const KvmDevice = "devices.virtualization.deckhouse.io/kvm" +@@ -846,6 +848,49 @@ func sidecarContainerName(i int) string { + return fmt.Sprintf("hook-sidecar-%d", i) + } + ++func sidecarContainerHotplugContainerdDiskName(name string) string { ++ return fmt.Sprintf("%s%s", HotplugContainerDisk, name) ++} ++ ++func (t *templateService) containerForHotplugContainerDisk(name string, cd *v1.ContainerDiskSource, vmi *v1.VirtualMachineInstance) k8sv1.Container { ++ runUser := int64(util.NonRootUID) ++ sharedMount := k8sv1.MountPropagationHostToContainer ++ path := fmt.Sprintf("/path/%s", name) ++ command := []string{"/init/usr/bin/container-disk"} ++ args := []string{"--copy-path", path} ++ ++ return k8sv1.Container{ ++ Name: name, ++ Image: cd.Image, ++ Command: command, ++ Args: args, ++ Resources: hotplugContainerResourceRequirementsForVMI(vmi, t.clusterConfig), ++ SecurityContext: &k8sv1.SecurityContext{ ++ AllowPrivilegeEscalation: pointer.Bool(false), ++ RunAsNonRoot: pointer.Bool(true), ++ RunAsUser: &runUser, ++ SeccompProfile: &k8sv1.SeccompProfile{ ++ Type: k8sv1.SeccompProfileTypeRuntimeDefault, ++ }, ++ Capabilities: &k8sv1.Capabilities{ ++ Drop: []k8sv1.Capability{"ALL"}, ++ }, ++ SELinuxOptions: &k8sv1.SELinuxOptions{ ++ Type: t.clusterConfig.GetSELinuxLauncherType(), ++ Level: "s0", ++ }, ++ }, ++ VolumeMounts: []k8sv1.VolumeMount{ ++ initContainerVolumeMount(), ++ { ++ Name: hotplugContainerDisks, ++ MountPath: "/path", ++ MountPropagation: &sharedMount, ++ }, ++ }, ++ } ++} ++ + func (t *templateService) RenderHotplugAttachmentPodTemplate(volumes []*v1.Volume, ownerPod *k8sv1.Pod, vmi *v1.VirtualMachineInstance, claimMap map[string]*k8sv1.PersistentVolumeClaim) (*k8sv1.Pod, error) { + zero := int64(0) + runUser := int64(util.NonRootUID) +@@ -924,6 +969,30 @@ func (t *templateService) RenderHotplugAttachmentPodTemplate(volumes []*v1.Volum + TerminationGracePeriodSeconds: &zero, + }, + } ++ first := true ++ for _, vol := range vmi.Spec.Volumes { ++ if vol.ContainerDisk == nil || !vol.ContainerDisk.Hotpluggable { ++ continue ++ } ++ name := sidecarContainerHotplugContainerdDiskName(vol.Name) ++ pod.Spec.Containers = append(pod.Spec.Containers, t.containerForHotplugContainerDisk(name, vol.ContainerDisk, vmi)) ++ if first { ++ first = false ++ userId := int64(util.NonRootUID) ++ initContainerCommand := []string{"/usr/bin/cp", ++ "/usr/bin/container-disk", ++ "/init/usr/bin/container-disk", ++ } ++ pod.Spec.InitContainers = append( ++ pod.Spec.InitContainers, ++ t.newInitContainerRenderer(vmi, ++ initContainerVolumeMount(), ++ initContainerResourceRequirementsForVMI(vmi, v1.ContainerDisk, t.clusterConfig), ++ userId).Render(initContainerCommand)) ++ pod.Spec.Volumes = append(pod.Spec.Volumes, emptyDirVolume(hotplugContainerDisks)) ++ pod.Spec.Volumes = append(pod.Spec.Volumes, emptyDirVolume(virtBinDir)) ++ } ++ } + + err := matchSELinuxLevelOfVMI(pod, vmi) + if err != nil { +diff --git a/pkg/virt-controller/watch/BUILD.bazel b/pkg/virt-controller/watch/BUILD.bazel +index 4fd325ba86..82fcaee0a3 100644 +--- a/pkg/virt-controller/watch/BUILD.bazel ++++ b/pkg/virt-controller/watch/BUILD.bazel +@@ -101,6 +101,7 @@ go_library( + "//vendor/k8s.io/client-go/util/flowcontrol:go_default_library", + "//vendor/k8s.io/client-go/util/workqueue:go_default_library", + "//vendor/k8s.io/utils/pointer:go_default_library", ++ "//vendor/k8s.io/utils/ptr:go_default_library", + "//vendor/k8s.io/utils/trace:go_default_library", + "//vendor/kubevirt.io/containerized-data-importer-api/pkg/apis/core/v1beta1:go_default_library", + ], +diff --git a/pkg/virt-controller/watch/vmi.go b/pkg/virt-controller/watch/vmi.go +index fa4e86ee17..ebe718f90d 100644 +--- a/pkg/virt-controller/watch/vmi.go ++++ b/pkg/virt-controller/watch/vmi.go +@@ -1836,6 +1836,10 @@ func (c *VMIController) handleHotplugVolumes(hotplugVolumes []*virtv1.Volume, ho + readyHotplugVolumes := make([]*virtv1.Volume, 0) + // Find all ready volumes + for _, volume := range hotplugVolumes { ++ if volume.ContainerDisk != nil { ++ readyHotplugVolumes = append(readyHotplugVolumes, volume) ++ continue ++ } + var err error + ready, wffc, err := storagetypes.VolumeReadyToAttachToNode(vmi.Namespace, *volume, dataVolumes, c.dataVolumeIndexer, c.pvcIndexer) + if err != nil { +@@ -1884,7 +1888,15 @@ func (c *VMIController) handleHotplugVolumes(hotplugVolumes []*virtv1.Volume, ho + + func (c *VMIController) podVolumesMatchesReadyVolumes(attachmentPod *k8sv1.Pod, volumes []*virtv1.Volume) bool { + // -2 for empty dir and token +- if len(attachmentPod.Spec.Volumes)-2 != len(volumes) { ++ // -3 if exist container-disk ++ magicNum := len(attachmentPod.Spec.Volumes) - 2 ++ for _, volume := range volumes { ++ if volume.ContainerDisk != nil { ++ magicNum -= 1 ++ break ++ } ++ } ++ if magicNum != len(volumes) { + return false + } + podVolumeMap := make(map[string]k8sv1.Volume) +@@ -1893,10 +1905,20 @@ func (c *VMIController) podVolumesMatchesReadyVolumes(attachmentPod *k8sv1.Pod, + podVolumeMap[volume.Name] = volume + } + } ++ containerDisksNames := make(map[string]struct{}) ++ for _, ctr := range attachmentPod.Spec.Containers { ++ if strings.HasPrefix(ctr.Name, services.HotplugContainerDisk) { ++ containerDisksNames[strings.TrimPrefix(ctr.Name, services.HotplugContainerDisk)] = struct{}{} ++ } ++ } + for _, volume := range volumes { ++ if volume.ContainerDisk != nil { ++ delete(containerDisksNames, volume.Name) ++ continue ++ } + delete(podVolumeMap, volume.Name) + } +- return len(podVolumeMap) == 0 ++ return len(podVolumeMap) == 0 && len(containerDisksNames) == 0 + } + + func (c *VMIController) createAttachmentPod(vmi *virtv1.VirtualMachineInstance, virtLauncherPod *k8sv1.Pod, volumes []*virtv1.Volume) (*k8sv1.Pod, syncError) { +@@ -2007,7 +2029,17 @@ func (c *VMIController) createAttachmentPodTemplate(vmi *virtv1.VirtualMachineIn + var pod *k8sv1.Pod + var err error + +- volumeNamesPVCMap, err := storagetypes.VirtVolumesToPVCMap(volumes, c.pvcIndexer, virtlauncherPod.Namespace) ++ var hasContainerDisk bool ++ var newVolumes []*virtv1.Volume ++ for _, volume := range volumes { ++ if volume.VolumeSource.ContainerDisk != nil { ++ hasContainerDisk = true ++ continue ++ } ++ newVolumes = append(newVolumes, volume) ++ } ++ ++ volumeNamesPVCMap, err := storagetypes.VirtVolumesToPVCMap(newVolumes, c.pvcIndexer, virtlauncherPod.Namespace) + if err != nil { + return nil, fmt.Errorf("failed to get PVC map: %v", err) + } +@@ -2029,7 +2061,7 @@ func (c *VMIController) createAttachmentPodTemplate(vmi *virtv1.VirtualMachineIn + } + } + +- if len(volumeNamesPVCMap) > 0 { ++ if len(volumeNamesPVCMap) > 0 || hasContainerDisk { + pod, err = c.templateService.RenderHotplugAttachmentPodTemplate(volumes, virtlauncherPod, vmi, volumeNamesPVCMap) + } + return pod, err +@@ -2151,23 +2183,39 @@ func (c *VMIController) updateVolumeStatus(vmi *virtv1.VirtualMachineInstance, v + ClaimName: volume.Name, + } + } ++ if volume.ContainerDisk != nil && status.ContainerDiskVolume == nil { ++ status.ContainerDiskVolume = &virtv1.ContainerDiskInfo{} ++ } + if attachmentPod == nil { +- if !c.volumeReady(status.Phase) { +- status.HotplugVolume.AttachPodUID = "" +- // Volume is not hotplugged in VM and Pod is gone, or hasn't been created yet, check for the PVC associated with the volume to set phase and message +- phase, reason, message := c.getVolumePhaseMessageReason(&vmi.Spec.Volumes[i], vmi.Namespace) +- status.Phase = phase +- status.Message = message +- status.Reason = reason ++ if volume.ContainerDisk != nil { ++ if !c.volumeReady(status.Phase) { ++ status.HotplugVolume.AttachPodUID = "" ++ // Volume is not hotplugged in VM and Pod is gone, or hasn't been created yet, check for the PVC associated with the volume to set phase and message ++ phase, reason, message := c.getVolumePhaseMessageReason(&vmi.Spec.Volumes[i], vmi.Namespace) ++ status.Phase = phase ++ status.Message = message ++ status.Reason = reason ++ } + } + } else { + status.HotplugVolume.AttachPodName = attachmentPod.Name +- if len(attachmentPod.Status.ContainerStatuses) == 1 && attachmentPod.Status.ContainerStatuses[0].Ready { ++ if volume.ContainerDisk != nil { ++ uid := types.UID("") ++ for _, cs := range attachmentPod.Status.ContainerStatuses { ++ name := strings.TrimPrefix(cs.Name, "hotplug-container-disk-") ++ if volume.Name == name && cs.Ready { ++ uid = attachmentPod.UID ++ break ++ } ++ } ++ status.HotplugVolume.AttachPodUID = uid ++ } else if len(attachmentPod.Status.ContainerStatuses) == 1 && attachmentPod.Status.ContainerStatuses[0].Ready { + status.HotplugVolume.AttachPodUID = attachmentPod.UID + } else { + // Remove UID of old pod if a new one is available, but not yet ready + status.HotplugVolume.AttachPodUID = "" + } ++ + if c.canMoveToAttachedPhase(status.Phase) { + status.Phase = virtv1.HotplugVolumeAttachedToNode + status.Message = fmt.Sprintf("Created hotplug attachment pod %s, for volume %s", attachmentPod.Name, volume.Name) +@@ -2176,7 +2224,6 @@ func (c *VMIController) updateVolumeStatus(vmi *virtv1.VirtualMachineInstance, v + } + } + } +- + if volume.VolumeSource.PersistentVolumeClaim != nil || volume.VolumeSource.DataVolume != nil || volume.VolumeSource.MemoryDump != nil { + + pvcName := storagetypes.PVCNameFromVirtVolume(&volume) +diff --git a/pkg/virt-handler/container-disk/hotplug.go b/pkg/virt-handler/container-disk/hotplug.go +new file mode 100644 +index 0000000000..f0d3a0607c +--- /dev/null ++++ b/pkg/virt-handler/container-disk/hotplug.go +@@ -0,0 +1,481 @@ ++package container_disk ++ ++import ( ++ "encoding/json" ++ "errors" ++ "fmt" ++ "os" ++ "path/filepath" ++ "strings" ++ "sync" ++ "time" ++ ++ hotplugdisk "kubevirt.io/kubevirt/pkg/hotplug-disk" ++ "kubevirt.io/kubevirt/pkg/unsafepath" ++ ++ "kubevirt.io/kubevirt/pkg/safepath" ++ virtconfig "kubevirt.io/kubevirt/pkg/virt-config" ++ virt_chroot "kubevirt.io/kubevirt/pkg/virt-handler/virt-chroot" ++ ++ "kubevirt.io/client-go/log" ++ ++ containerdisk "kubevirt.io/kubevirt/pkg/container-disk" ++ diskutils "kubevirt.io/kubevirt/pkg/ephemeral-disk-utils" ++ "kubevirt.io/kubevirt/pkg/virt-handler/isolation" ++ ++ "k8s.io/apimachinery/pkg/api/equality" ++ "k8s.io/apimachinery/pkg/types" ++ ++ v1 "kubevirt.io/api/core/v1" ++) ++ ++type HotplugMounter interface { ++ ContainerDisksReady(vmi *v1.VirtualMachineInstance, notInitializedSince time.Time) (bool, error) ++ MountAndVerify(vmi *v1.VirtualMachineInstance) (map[string]*containerdisk.DiskInfo, error) ++ IsMounted(vmi *v1.VirtualMachineInstance, volumeName string) (bool, error) ++ Umount(vmi *v1.VirtualMachineInstance) error ++ UmountAll(vmi *v1.VirtualMachineInstance) error ++ ComputeChecksums(vmi *v1.VirtualMachineInstance) (*DiskChecksums, error) ++} ++ ++type hotplugMounter struct { ++ podIsolationDetector isolation.PodIsolationDetector ++ mountStateDir string ++ mountRecords map[types.UID]*vmiMountTargetRecord ++ mountRecordsLock sync.Mutex ++ suppressWarningTimeout time.Duration ++ clusterConfig *virtconfig.ClusterConfig ++ nodeIsolationResult isolation.IsolationResult ++ ++ hotplugPathGetter containerdisk.HotplugSocketPathGetter ++ hotplugManager hotplugdisk.HotplugDiskManagerInterface ++} ++ ++func (m *hotplugMounter) IsMounted(vmi *v1.VirtualMachineInstance, volumeName string) (bool, error) { ++ virtLauncherUID := m.findVirtlauncherUID(vmi) ++ if virtLauncherUID == "" { ++ return false, nil ++ } ++ target, err := m.hotplugManager.GetFileSystemDiskTargetPathFromHostView(virtLauncherUID, volumeName, false) ++ if err != nil { ++ return false, err ++ } ++ return isolation.IsMounted(target) ++} ++ ++func NewHotplugMounter(isoDetector isolation.PodIsolationDetector, ++ mountStateDir string, ++ clusterConfig *virtconfig.ClusterConfig, ++ hotplugManager hotplugdisk.HotplugDiskManagerInterface, ++) HotplugMounter { ++ return &hotplugMounter{ ++ mountRecords: make(map[types.UID]*vmiMountTargetRecord), ++ podIsolationDetector: isoDetector, ++ mountStateDir: mountStateDir, ++ suppressWarningTimeout: 1 * time.Minute, ++ clusterConfig: clusterConfig, ++ nodeIsolationResult: isolation.NodeIsolationResult(), ++ ++ hotplugPathGetter: containerdisk.NewHotplugSocketPathGetter(""), ++ hotplugManager: hotplugManager, ++ } ++} ++ ++func (m *hotplugMounter) deleteMountTargetRecord(vmi *v1.VirtualMachineInstance) error { ++ if string(vmi.UID) == "" { ++ return fmt.Errorf("unable to find container disk mounted directories for vmi without uid") ++ } ++ ++ recordFile := filepath.Join(m.mountStateDir, string(vmi.UID)) ++ ++ exists, err := diskutils.FileExists(recordFile) ++ if err != nil { ++ return err ++ } ++ ++ if exists { ++ record, err := m.getMountTargetRecord(vmi) ++ if err != nil { ++ return err ++ } ++ ++ for _, target := range record.MountTargetEntries { ++ os.Remove(target.TargetFile) ++ os.Remove(target.SocketFile) ++ } ++ ++ os.Remove(recordFile) ++ } ++ ++ m.mountRecordsLock.Lock() ++ defer m.mountRecordsLock.Unlock() ++ delete(m.mountRecords, vmi.UID) ++ ++ return nil ++} ++ ++func (m *hotplugMounter) getMountTargetRecord(vmi *v1.VirtualMachineInstance) (*vmiMountTargetRecord, error) { ++ var ok bool ++ var existingRecord *vmiMountTargetRecord ++ ++ if string(vmi.UID) == "" { ++ return nil, fmt.Errorf("unable to find container disk mounted directories for vmi without uid") ++ } ++ ++ m.mountRecordsLock.Lock() ++ defer m.mountRecordsLock.Unlock() ++ existingRecord, ok = m.mountRecords[vmi.UID] ++ ++ // first check memory cache ++ if ok { ++ return existingRecord, nil ++ } ++ ++ // if not there, see if record is on disk, this can happen if virt-handler restarts ++ recordFile := filepath.Join(m.mountStateDir, filepath.Clean(string(vmi.UID))) ++ ++ exists, err := diskutils.FileExists(recordFile) ++ if err != nil { ++ return nil, err ++ } ++ ++ if exists { ++ record := vmiMountTargetRecord{} ++ // #nosec No risk for path injection. Using static base and cleaned filename ++ bytes, err := os.ReadFile(recordFile) ++ if err != nil { ++ return nil, err ++ } ++ err = json.Unmarshal(bytes, &record) ++ if err != nil { ++ return nil, err ++ } ++ ++ if !record.UsesSafePaths { ++ record.UsesSafePaths = true ++ for i, entry := range record.MountTargetEntries { ++ safePath, err := safepath.JoinAndResolveWithRelativeRoot("/", entry.TargetFile) ++ if err != nil { ++ return nil, fmt.Errorf("failed converting legacy path to safepath: %v", err) ++ } ++ record.MountTargetEntries[i].TargetFile = unsafepath.UnsafeAbsolute(safePath.Raw()) ++ } ++ } ++ ++ m.mountRecords[vmi.UID] = &record ++ return &record, nil ++ } ++ ++ // not found ++ return nil, nil ++} ++ ++func (m *hotplugMounter) addMountTargetRecord(vmi *v1.VirtualMachineInstance, record *vmiMountTargetRecord) error { ++ return m.setAddMountTargetRecordHelper(vmi, record, true) ++} ++ ++func (m *hotplugMounter) setMountTargetRecord(vmi *v1.VirtualMachineInstance, record *vmiMountTargetRecord) error { ++ return m.setAddMountTargetRecordHelper(vmi, record, false) ++} ++ ++func (m *hotplugMounter) setAddMountTargetRecordHelper(vmi *v1.VirtualMachineInstance, record *vmiMountTargetRecord, addPreviousRules bool) error { ++ if string(vmi.UID) == "" { ++ return fmt.Errorf("unable to set container disk mounted directories for vmi without uid") ++ } ++ ++ record.UsesSafePaths = true ++ ++ recordFile := filepath.Join(m.mountStateDir, string(vmi.UID)) ++ fileExists, err := diskutils.FileExists(recordFile) ++ if err != nil { ++ return err ++ } ++ ++ m.mountRecordsLock.Lock() ++ defer m.mountRecordsLock.Unlock() ++ ++ existingRecord, ok := m.mountRecords[vmi.UID] ++ if ok && fileExists && equality.Semantic.DeepEqual(existingRecord, record) { ++ // already done ++ return nil ++ } ++ ++ if addPreviousRules && existingRecord != nil && len(existingRecord.MountTargetEntries) > 0 { ++ record.MountTargetEntries = append(record.MountTargetEntries, existingRecord.MountTargetEntries...) ++ } ++ ++ bytes, err := json.Marshal(record) ++ if err != nil { ++ return err ++ } ++ ++ err = os.MkdirAll(filepath.Dir(recordFile), 0750) ++ if err != nil { ++ return err ++ } ++ ++ err = os.WriteFile(recordFile, bytes, 0600) ++ if err != nil { ++ return err ++ } ++ ++ m.mountRecords[vmi.UID] = record ++ ++ return nil ++} ++ ++func (m *hotplugMounter) MountAndVerify(vmi *v1.VirtualMachineInstance) (map[string]*containerdisk.DiskInfo, error) { ++ virtLauncherUID := m.findVirtlauncherUID(vmi) ++ if virtLauncherUID == "" { ++ return nil, nil ++ } ++ ++ record := vmiMountTargetRecord{} ++ disksInfo := map[string]*containerdisk.DiskInfo{} ++ ++ for _, volume := range vmi.Spec.Volumes { ++ if volume.ContainerDisk != nil && volume.ContainerDisk.Hotpluggable { ++ target, err := m.hotplugManager.GetFileSystemDiskTargetPathFromHostView(virtLauncherUID, volume.Name, true) ++ if err != nil { ++ return nil, err ++ } ++ ++ sock, err := m.hotplugPathGetter(vmi, volume.Name) ++ if err != nil { ++ return nil, err ++ } ++ ++ record.MountTargetEntries = append(record.MountTargetEntries, vmiMountTargetEntry{ ++ TargetFile: unsafepath.UnsafeAbsolute(target.Raw()), ++ SocketFile: sock, ++ }) ++ } ++ } ++ ++ if len(record.MountTargetEntries) > 0 { ++ err := m.setMountTargetRecord(vmi, &record) ++ if err != nil { ++ return nil, err ++ } ++ } ++ ++ vmiRes, err := m.podIsolationDetector.Detect(vmi) ++ if err != nil { ++ return nil, fmt.Errorf("failed to detect VMI pod: %v", err) ++ } ++ ++ for _, volume := range vmi.Spec.Volumes { ++ if volume.ContainerDisk != nil && volume.ContainerDisk.Hotpluggable { ++ target, err := m.hotplugManager.GetFileSystemDiskTargetPathFromHostView(virtLauncherUID, volume.Name, false) ++ ++ if isMounted, err := isolation.IsMounted(target); err != nil { ++ return nil, fmt.Errorf("failed to determine if %s is already mounted: %v", target, err) ++ } else if !isMounted { ++ ++ sourceFile, err := m.getContainerDiskPath(vmi, &volume, volume.Name) ++ if err != nil { ++ return nil, fmt.Errorf("failed to find a sourceFile in containerDisk %v: %v", volume.Name, err) ++ } ++ ++ log.DefaultLogger().Object(vmi).Infof("Bind mounting container disk at %s to %s", sourceFile, target) ++ opts := []string{ ++ "bind", "ro", "uid=107", "gid=107", ++ } ++ err = virt_chroot.MountChrootWithOptions(sourceFile, target, opts...) ++ if err != nil { ++ return nil, fmt.Errorf("failed to bindmount containerDisk %v. err: %w", volume.Name, err) ++ } ++ } ++ ++ imageInfo, err := isolation.GetImageInfo( ++ containerdisk.GetHotplugContainerDiskTargetPathFromLauncherView(volume.Name), ++ vmiRes, ++ m.clusterConfig.GetDiskVerification(), ++ ) ++ if err != nil { ++ return nil, fmt.Errorf("failed to get image info: %v", err) ++ } ++ if err := containerdisk.VerifyImage(imageInfo); err != nil { ++ return nil, fmt.Errorf("invalid image in containerDisk %v: %v", volume.Name, err) ++ } ++ disksInfo[volume.Name] = imageInfo ++ } ++ } ++ ++ return disksInfo, nil ++} ++ ++func (m *hotplugMounter) Umount(vmi *v1.VirtualMachineInstance) error { ++ record, err := m.getMountTargetRecord(vmi) ++ if err != nil { ++ return err ++ } else if record == nil { ++ // no entries to unmount ++ ++ log.DefaultLogger().Object(vmi).Infof("No container disk mount entries found to unmount") ++ return nil ++ } ++ for _, r := range record.MountTargetEntries { ++ name, err := extractNameFromSocket(r.SocketFile) ++ if err != nil { ++ return err ++ } ++ needUmount := true ++ for _, v := range vmi.Status.VolumeStatus { ++ if v.Name == name { ++ needUmount = false ++ } ++ } ++ if needUmount { ++ file, err := safepath.NewFileNoFollow(r.TargetFile) ++ if err != nil { ++ if errors.Is(err, os.ErrNotExist) { ++ continue ++ } ++ return fmt.Errorf(failedCheckMountPointFmt, r.TargetFile, err) ++ } ++ _ = file.Close() ++ // #nosec No risk for attacket injection. Parameters are predefined strings ++ out, err := virt_chroot.UmountChroot(file.Path()).CombinedOutput() ++ if err != nil { ++ return fmt.Errorf(failedUnmountFmt, file, string(out), err) ++ } ++ } ++ } ++ return nil ++} ++ ++func extractNameFromSocket(socketFile string) (string, error) { ++ base := filepath.Base(socketFile) ++ if strings.HasPrefix(base, "hotplug-container-disk-") && strings.HasSuffix(base, ".sock") { ++ name := strings.TrimPrefix(base, "hotplug-container-disk-") ++ name = strings.TrimSuffix(name, ".sock") ++ return name, nil ++ } ++ return "", fmt.Errorf("name not found in path") ++} ++ ++func (m *hotplugMounter) UmountAll(vmi *v1.VirtualMachineInstance) error { ++ if vmi.UID == "" { ++ return nil ++ } ++ ++ record, err := m.getMountTargetRecord(vmi) ++ if err != nil { ++ return err ++ } else if record == nil { ++ // no entries to unmount ++ ++ log.DefaultLogger().Object(vmi).Infof("No container disk mount entries found to unmount") ++ return nil ++ } ++ ++ log.DefaultLogger().Object(vmi).Infof("Found container disk mount entries") ++ for _, entry := range record.MountTargetEntries { ++ log.DefaultLogger().Object(vmi).Infof("Looking to see if containerdisk is mounted at path %s", entry.TargetFile) ++ file, err := safepath.NewFileNoFollow(entry.TargetFile) ++ if err != nil { ++ if errors.Is(err, os.ErrNotExist) { ++ continue ++ } ++ return fmt.Errorf(failedCheckMountPointFmt, entry.TargetFile, err) ++ } ++ _ = file.Close() ++ if mounted, err := isolation.IsMounted(file.Path()); err != nil { ++ return fmt.Errorf(failedCheckMountPointFmt, file, err) ++ } else if mounted { ++ log.DefaultLogger().Object(vmi).Infof("unmounting container disk at path %s", file) ++ // #nosec No risk for attacket injection. Parameters are predefined strings ++ out, err := virt_chroot.UmountChroot(file.Path()).CombinedOutput() ++ if err != nil { ++ return fmt.Errorf(failedUnmountFmt, file, string(out), err) ++ } ++ } ++ } ++ err = m.deleteMountTargetRecord(vmi) ++ if err != nil { ++ return err ++ } ++ ++ return nil ++} ++ ++func (m *hotplugMounter) ContainerDisksReady(vmi *v1.VirtualMachineInstance, notInitializedSince time.Time) (bool, error) { ++ for _, volume := range vmi.Spec.Volumes { ++ if volume.ContainerDisk != nil && volume.ContainerDisk.Hotpluggable { ++ _, err := m.hotplugPathGetter(vmi, volume.Name) ++ if err != nil { ++ log.DefaultLogger().Object(vmi).Reason(err).Infof("containerdisk %s not yet ready", volume.Name) ++ if time.Now().After(notInitializedSince.Add(m.suppressWarningTimeout)) { ++ return false, fmt.Errorf("containerdisk %s still not ready after one minute", volume.Name) ++ } ++ return false, nil ++ } ++ } ++ } ++ ++ log.DefaultLogger().Object(vmi).V(4).Info("all containerdisks are ready") ++ return true, nil ++} ++ ++func (m *hotplugMounter) getContainerDiskPath(vmi *v1.VirtualMachineInstance, volume *v1.Volume, volumeName string) (*safepath.Path, error) { ++ sock, err := m.hotplugPathGetter(vmi, volumeName) ++ if err != nil { ++ return nil, ErrDiskContainerGone ++ } ++ ++ res, err := m.podIsolationDetector.DetectForSocket(vmi, sock) ++ if err != nil { ++ return nil, fmt.Errorf("failed to detect socket for containerDisk %v: %v", volume.Name, err) ++ } ++ ++ mountPoint, err := isolation.ParentPathForRootMount(m.nodeIsolationResult, res) ++ if err != nil { ++ return nil, fmt.Errorf("failed to detect root mount point of containerDisk %v on the node: %v", volume.Name, err) ++ } ++ ++ return containerdisk.GetImage(mountPoint, volume.ContainerDisk.Path) ++} ++ ++func (m *hotplugMounter) ComputeChecksums(vmi *v1.VirtualMachineInstance) (*DiskChecksums, error) { ++ ++ diskChecksums := &DiskChecksums{ ++ ContainerDiskChecksums: map[string]uint32{}, ++ } ++ ++ for _, volume := range vmi.Spec.Volumes { ++ if volume.VolumeSource.ContainerDisk == nil || !volume.VolumeSource.ContainerDisk.Hotpluggable { ++ continue ++ } ++ ++ path, err := m.getContainerDiskPath(vmi, &volume, volume.Name) ++ if err != nil { ++ return nil, err ++ } ++ ++ checksum, err := getDigest(path) ++ if err != nil { ++ return nil, err ++ } ++ ++ diskChecksums.ContainerDiskChecksums[volume.Name] = checksum ++ } ++ ++ return diskChecksums, nil ++} ++ ++func (m *hotplugMounter) findVirtlauncherUID(vmi *v1.VirtualMachineInstance) (uid types.UID) { ++ cnt := 0 ++ for podUID := range vmi.Status.ActivePods { ++ _, err := m.hotplugManager.GetHotplugTargetPodPathOnHost(podUID) ++ if err == nil { ++ uid = podUID ++ cnt++ ++ } ++ } ++ if cnt == 1 { ++ return ++ } ++ // Either no pods, or multiple pods, skip. ++ return types.UID("") ++} +diff --git a/pkg/virt-handler/container-disk/mount.go b/pkg/virt-handler/container-disk/mount.go +index 953c20f3af..d99bec3a43 100644 +--- a/pkg/virt-handler/container-disk/mount.go ++++ b/pkg/virt-handler/container-disk/mount.go +@@ -54,6 +54,8 @@ type mounter struct { + kernelBootSocketPathGetter containerdisk.KernelBootSocketPathGetter + clusterConfig *virtconfig.ClusterConfig + nodeIsolationResult isolation.IsolationResult ++ ++ hotplugPathGetter containerdisk.HotplugSocketPathGetter + } + + type Mounter interface { +@@ -98,6 +100,8 @@ func NewMounter(isoDetector isolation.PodIsolationDetector, mountStateDir string + kernelBootSocketPathGetter: containerdisk.NewKernelBootSocketPathGetter(""), + clusterConfig: clusterConfig, + nodeIsolationResult: isolation.NodeIsolationResult(), ++ ++ hotplugPathGetter: containerdisk.NewHotplugSocketPathGetter(""), + } + } + +@@ -254,7 +258,7 @@ func (m *mounter) MountAndVerify(vmi *v1.VirtualMachineInstance) (map[string]*co + disksInfo := map[string]*containerdisk.DiskInfo{} + + for i, volume := range vmi.Spec.Volumes { +- if volume.ContainerDisk != nil { ++ if volume.ContainerDisk != nil && !volume.ContainerDisk.Hotpluggable { + diskTargetDir, err := containerdisk.GetDiskTargetDirFromHostView(vmi) + if err != nil { + return nil, err +@@ -296,7 +300,7 @@ func (m *mounter) MountAndVerify(vmi *v1.VirtualMachineInstance) (map[string]*co + } + + for i, volume := range vmi.Spec.Volumes { +- if volume.ContainerDisk != nil { ++ if volume.ContainerDisk != nil && !volume.ContainerDisk.Hotpluggable { + diskTargetDir, err := containerdisk.GetDiskTargetDirFromHostView(vmi) + if err != nil { + return nil, err +@@ -394,7 +398,7 @@ func (m *mounter) Unmount(vmi *v1.VirtualMachineInstance) error { + + func (m *mounter) ContainerDisksReady(vmi *v1.VirtualMachineInstance, notInitializedSince time.Time) (bool, error) { + for i, volume := range vmi.Spec.Volumes { +- if volume.ContainerDisk != nil { ++ if volume.ContainerDisk != nil && !volume.ContainerDisk.Hotpluggable { + _, err := m.socketPathGetter(vmi, i) + if err != nil { + log.DefaultLogger().Object(vmi).Reason(err).Infof("containerdisk %s not yet ready", volume.Name) +@@ -706,7 +710,7 @@ func (m *mounter) ComputeChecksums(vmi *v1.VirtualMachineInstance) (*DiskChecksu + + // compute for containerdisks + for i, volume := range vmi.Spec.Volumes { +- if volume.VolumeSource.ContainerDisk == nil { ++ if volume.VolumeSource.ContainerDisk == nil || volume.VolumeSource.ContainerDisk.Hotpluggable { + continue + } + +diff --git a/pkg/virt-handler/hotplug-disk/mount.go b/pkg/virt-handler/hotplug-disk/mount.go +index 971c8d55fc..034c3d8526 100644 +--- a/pkg/virt-handler/hotplug-disk/mount.go ++++ b/pkg/virt-handler/hotplug-disk/mount.go +@@ -343,7 +343,7 @@ func (m *volumeMounter) mountFromPod(vmi *v1.VirtualMachineInstance, sourceUID t + return err + } + for _, volumeStatus := range vmi.Status.VolumeStatus { +- if volumeStatus.HotplugVolume == nil { ++ if volumeStatus.HotplugVolume == nil || volumeStatus.ContainerDiskVolume != nil { + // Skip non hotplug volumes + continue + } +@@ -649,7 +649,7 @@ func (m *volumeMounter) Unmount(vmi *v1.VirtualMachineInstance, cgroupManager cg + return err + } + for _, volumeStatus := range vmi.Status.VolumeStatus { +- if volumeStatus.HotplugVolume == nil { ++ if volumeStatus.HotplugVolume == nil || volumeStatus.ContainerDiskVolume != nil { + continue + } + var path *safepath.Path +diff --git a/pkg/virt-handler/isolation/detector.go b/pkg/virt-handler/isolation/detector.go +index f83f96ead4..5e38c6cedd 100644 +--- a/pkg/virt-handler/isolation/detector.go ++++ b/pkg/virt-handler/isolation/detector.go +@@ -24,6 +24,8 @@ package isolation + import ( + "fmt" + "net" ++ "os" ++ "path" + "runtime" + "syscall" + "time" +@@ -207,12 +209,45 @@ func setProcessMemoryLockRLimit(pid int, size int64) error { + return nil + } + ++type deferFunc func() ++ ++func (s *socketBasedIsolationDetector) socketHack(socket string) (sock net.Conn, deferFunc deferFunc, err error) { ++ fn := func() {} ++ if len([]rune(socket)) <= 108 { ++ sock, err = net.DialTimeout("unix", socket, time.Duration(isolationDialTimeout)*time.Second) ++ fn = func() { ++ if err == nil { ++ sock.Close() ++ } ++ } ++ return sock, fn, err ++ } ++ base := path.Base(socket) ++ newPath := fmt.Sprintf("/tmp/%s", base) ++ if err = os.Symlink(socket, newPath); err != nil { ++ return nil, fn, err ++ } ++ sock, err = net.DialTimeout("unix", newPath, time.Duration(isolationDialTimeout)*time.Second) ++ fn = func() { ++ if err == nil { ++ sock.Close() ++ } ++ os.Remove(newPath) ++ } ++ return sock, fn, err ++} ++ + func (s *socketBasedIsolationDetector) getPid(socket string) (int, error) { +- sock, err := net.DialTimeout("unix", socket, time.Duration(isolationDialTimeout)*time.Second) ++ sock, defFn, err := s.socketHack(socket) ++ defer defFn() + if err != nil { + return -1, err + } +- defer sock.Close() ++ //sock, err := net.DialTimeout("unix", socket, time.Duration(isolationDialTimeout)*time.Second) ++ //if err != nil { ++ // return -1, err ++ //} ++ //defer sock.Close() + + ufile, err := sock.(*net.UnixConn).File() + if err != nil { +diff --git a/pkg/virt-handler/virt-chroot/virt-chroot.go b/pkg/virt-handler/virt-chroot/virt-chroot.go +index 4160212b7b..580b788acc 100644 +--- a/pkg/virt-handler/virt-chroot/virt-chroot.go ++++ b/pkg/virt-handler/virt-chroot/virt-chroot.go +@@ -20,7 +20,10 @@ + package virt_chroot + + import ( ++ "bytes" ++ "fmt" + "os/exec" ++ "slices" + "strings" + + "kubevirt.io/kubevirt/pkg/safepath" +@@ -48,6 +51,49 @@ func MountChroot(sourcePath, targetPath *safepath.Path, ro bool) *exec.Cmd { + return UnsafeMountChroot(trimProcPrefix(sourcePath), trimProcPrefix(targetPath), ro) + } + ++func MountChrootWithOptions(sourcePath, targetPath *safepath.Path, mountOptions ...string) error { ++ args := append(getBaseArgs(), "mount") ++ remountArgs := slices.Clone(args) ++ ++ mountOptions = slices.DeleteFunc(mountOptions, func(s string) bool { ++ return s == "remount" ++ }) ++ if len(mountOptions) > 0 { ++ opts := strings.Join(mountOptions, ",") ++ remountOpts := "remount," + opts ++ args = append(args, "-o", opts) ++ remountArgs = append(remountArgs, "-o", remountOpts) ++ } ++ ++ sp := trimProcPrefix(sourcePath) ++ tp := trimProcPrefix(targetPath) ++ args = append(args, sp, tp) ++ remountArgs = append(remountArgs, sp, tp) ++ ++ stdout := new(bytes.Buffer) ++ stderr := new(bytes.Buffer) ++ ++ cmd := exec.Command(binaryPath, args...) ++ cmd.Stdout = stdout ++ cmd.Stderr = stderr ++ err := cmd.Run() ++ if err != nil { ++ return fmt.Errorf("mount failed: %w, stdout: %s, stderr: %s", err, stdout.String(), stderr.String()) ++ } ++ ++ stdout = new(bytes.Buffer) ++ stderr = new(bytes.Buffer) ++ ++ remountCmd := exec.Command(binaryPath, remountArgs...) ++ cmd.Stdout = stdout ++ cmd.Stderr = stderr ++ err = remountCmd.Run() ++ if err != nil { ++ return fmt.Errorf("mount failed: %w, stdout: %s, stderr: %s", err, stdout.String(), stderr.String()) ++ } ++ return nil ++} ++ + // Deprecated: UnsafeMountChroot is used to connect to code which needs to be refactored + // to handle mounts securely. + func UnsafeMountChroot(sourcePath, targetPath string, ro bool) *exec.Cmd { +diff --git a/pkg/virt-handler/vm.go b/pkg/virt-handler/vm.go +index 24352cf6e9..f1de8d1149 100644 +--- a/pkg/virt-handler/vm.go ++++ b/pkg/virt-handler/vm.go +@@ -25,6 +25,7 @@ import ( + goerror "errors" + "fmt" + "io" ++ "maps" + "net" + "os" + "path/filepath" +@@ -247,6 +248,13 @@ func NewController( + vmiExpectations: controller.NewUIDTrackingControllerExpectations(controller.NewControllerExpectations()), + sriovHotplugExecutorPool: executor.NewRateLimitedExecutorPool(executor.NewExponentialLimitedBackoffCreator()), + ioErrorRetryManager: NewFailRetryManager("io-error-retry", 10*time.Second, 3*time.Minute, 30*time.Second), ++ ++ hotplugContainerDiskMounter: container_disk.NewHotplugMounter( ++ podIsolationDetector, ++ filepath.Join(virtPrivateDir, "hotplug-container-disk-mount-state"), ++ clusterConfig, ++ hotplugdisk.NewHotplugDiskManager(kubeletPodsDir), ++ ), + } + + _, err := vmiSourceInformer.AddEventHandler(cache.ResourceEventHandlerFuncs{ +@@ -342,6 +350,8 @@ type VirtualMachineController struct { + hostCpuModel string + vmiExpectations *controller.UIDTrackingControllerExpectations + ioErrorRetryManager *FailRetryManager ++ ++ hotplugContainerDiskMounter container_disk.HotplugMounter + } + + type virtLauncherCriticalSecurebootError struct { +@@ -876,7 +886,15 @@ func (d *VirtualMachineController) updateHotplugVolumeStatus(vmi *v1.VirtualMach + needsRefresh := false + if volumeStatus.Target == "" { + needsRefresh = true +- mounted, err := d.hotplugVolumeMounter.IsMounted(vmi, volumeStatus.Name, volumeStatus.HotplugVolume.AttachPodUID) ++ var ( ++ mounted bool ++ err error ++ ) ++ if volumeStatus.ContainerDiskVolume != nil { ++ mounted, err = d.hotplugContainerDiskMounter.IsMounted(vmi, volumeStatus.Name) ++ } else { ++ mounted, err = d.hotplugVolumeMounter.IsMounted(vmi, volumeStatus.Name, volumeStatus.HotplugVolume.AttachPodUID) ++ } + if err != nil { + log.Log.Object(vmi).Errorf("error occurred while checking if volume is mounted: %v", err) + } +@@ -898,6 +916,7 @@ func (d *VirtualMachineController) updateHotplugVolumeStatus(vmi *v1.VirtualMach + volumeStatus.Reason = VolumeUnMountedFromPodReason + } + } ++ + } else { + // Successfully attached to VM. + volumeStatus.Phase = v1.VolumeReady +@@ -2178,6 +2197,11 @@ func (d *VirtualMachineController) processVmCleanup(vmi *v1.VirtualMachineInstan + return err + } + ++ err := d.hotplugContainerDiskMounter.UmountAll(vmi) ++ if err != nil { ++ return err ++ } ++ + // UnmountAll does the cleanup on the "best effort" basis: it is + // safe to pass a nil cgroupManager. + cgroupManager, _ := getCgroupManager(vmi) +@@ -2829,6 +2853,12 @@ func (d *VirtualMachineController) vmUpdateHelperMigrationTarget(origVMI *v1.Vir + return err + } + ++ hotplugDiskInfo, err := d.hotplugContainerDiskMounter.MountAndVerify(vmi) ++ if err != nil { ++ return err ++ } ++ maps.Copy(disksInfo, hotplugDiskInfo) ++ + // Mount hotplug disks + if attachmentPodUID := vmi.Status.MigrationState.TargetAttachmentPodUID; attachmentPodUID != types.UID("") { + cgroupManager, err := getCgroupManager(vmi) +@@ -3051,6 +3081,11 @@ func (d *VirtualMachineController) vmUpdateHelperDefault(origVMI *v1.VirtualMach + if err != nil { + return err + } ++ hotplugDiskInfo, err := d.hotplugContainerDiskMounter.MountAndVerify(vmi) ++ if err != nil { ++ return err ++ } ++ maps.Copy(disksInfo, hotplugDiskInfo) + + // Try to mount hotplug volume if there is any during startup. + if err := d.hotplugVolumeMounter.Mount(vmi, cgroupManager); err != nil { +@@ -3138,6 +3173,11 @@ func (d *VirtualMachineController) vmUpdateHelperDefault(origVMI *v1.VirtualMach + log.Log.Object(vmi).Error(err.Error()) + } + ++ hotplugDiskInfo, err := d.hotplugContainerDiskMounter.MountAndVerify(vmi) ++ if err != nil { ++ return err ++ } ++ maps.Copy(disksInfo, hotplugDiskInfo) + if err := d.hotplugVolumeMounter.Mount(vmi, cgroupManager); err != nil { + return err + } +@@ -3215,6 +3255,9 @@ func (d *VirtualMachineController) vmUpdateHelperDefault(origVMI *v1.VirtualMach + + if vmi.IsRunning() { + // Umount any disks no longer mounted ++ if err := d.hotplugContainerDiskMounter.Umount(vmi); err != nil { ++ return err ++ } + if err := d.hotplugVolumeMounter.Unmount(vmi, cgroupManager); err != nil { + return err + } +diff --git a/pkg/virt-launcher/virtwrap/converter/converter.go b/pkg/virt-launcher/virtwrap/converter/converter.go +index 3318c1c466..ce726edf12 100644 +--- a/pkg/virt-launcher/virtwrap/converter/converter.go ++++ b/pkg/virt-launcher/virtwrap/converter/converter.go +@@ -649,6 +649,9 @@ func Convert_v1_Hotplug_Volume_To_api_Disk(source *v1.Volume, disk *api.Disk, c + if source.DataVolume != nil { + return Convert_v1_Hotplug_DataVolume_To_api_Disk(source.Name, disk, c) + } ++ if source.ContainerDisk != nil { ++ return Convert_v1_Hotplug_ContainerDisk_To_api_Disk(source.Name, disk, c) ++ } + return fmt.Errorf("hotplug disk %s references an unsupported source", disk.Alias.GetName()) + } + +@@ -690,6 +693,10 @@ func GetHotplugBlockDeviceVolumePath(volumeName string) string { + return filepath.Join(string(filepath.Separator), "var", "run", "kubevirt", "hotplug-disks", volumeName) + } + ++func GetHotplugContainerDiskPath(volumeName string) string { ++ return filepath.Join(string(filepath.Separator), "var", "run", "kubevirt", "hotplug-disks", fmt.Sprintf("%s.img", volumeName)) ++} ++ + func Convert_v1_PersistentVolumeClaim_To_api_Disk(name string, disk *api.Disk, c *ConverterContext) error { + if c.IsBlockPVC[name] { + return Convert_v1_BlockVolumeSource_To_api_Disk(name, disk, c.VolumesDiscardIgnore) +@@ -768,6 +775,35 @@ func Convert_v1_Hotplug_BlockVolumeSource_To_api_Disk(volumeName string, disk *a + return nil + } + ++func Convert_v1_Hotplug_ContainerDisk_To_api_Disk(volumeName string, disk *api.Disk, c *ConverterContext) error { ++ if disk.Type == "lun" { ++ return fmt.Errorf(deviceTypeNotCompatibleFmt, disk.Alias.GetName()) ++ } ++ info := c.DisksInfo[volumeName] ++ if info == nil { ++ return fmt.Errorf("no disk info provided for volume %s", volumeName) ++ } ++ ++ disk.Type = "file" ++ disk.Driver.Type = info.Format ++ disk.Driver.ErrorPolicy = v1.DiskErrorPolicyStop ++ disk.ReadOnly = &api.ReadOnly{} ++ if !contains(c.VolumesDiscardIgnore, volumeName) { ++ disk.Driver.Discard = "unmap" ++ } ++ disk.Source.File = GetHotplugContainerDiskPath(volumeName) ++ disk.BackingStore = &api.BackingStore{ ++ Format: &api.BackingStoreFormat{}, ++ Source: &api.DiskSource{}, ++ } ++ ++ //disk.BackingStore.Format.Type = info.Format ++ //disk.BackingStore.Source.File = info.BackingFile ++ //disk.BackingStore.Type = "file" ++ ++ return nil ++} ++ + func Convert_v1_HostDisk_To_api_Disk(volumeName string, path string, disk *api.Disk) error { + disk.Type = "file" + disk.Driver.Type = "raw" +diff --git a/pkg/virt-operator/resource/apply/BUILD.bazel b/pkg/virt-operator/resource/apply/BUILD.bazel +index f6bd9bd4f1..fe6ab54f8c 100644 +--- a/pkg/virt-operator/resource/apply/BUILD.bazel ++++ b/pkg/virt-operator/resource/apply/BUILD.bazel +@@ -4,7 +4,6 @@ go_library( + name = "go_default_library", + srcs = [ + "admissionregistration.go", +- "apiservices.go", + "apps.go", + "certificates.go", + "core.go", +@@ -65,7 +64,6 @@ go_library( + "//vendor/k8s.io/client-go/tools/cache:go_default_library", + "//vendor/k8s.io/client-go/tools/record:go_default_library", + "//vendor/k8s.io/client-go/util/workqueue:go_default_library", +- "//vendor/k8s.io/kube-aggregator/pkg/apis/apiregistration/v1:go_default_library", + "//vendor/k8s.io/utils/pointer:go_default_library", + ], + ) +diff --git a/pkg/virt-operator/resource/generate/components/BUILD.bazel b/pkg/virt-operator/resource/generate/components/BUILD.bazel +index 70d2da0897..affcd3fecd 100644 +--- a/pkg/virt-operator/resource/generate/components/BUILD.bazel ++++ b/pkg/virt-operator/resource/generate/components/BUILD.bazel +@@ -3,7 +3,6 @@ load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test") + go_library( + name = "go_default_library", + srcs = [ +- "apiservices.go", + "crds.go", + "daemonsets.go", + "deployments.go", +@@ -62,7 +61,6 @@ go_library( + "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", + "//vendor/k8s.io/apimachinery/pkg/util/intstr:go_default_library", + "//vendor/k8s.io/apimachinery/pkg/util/yaml:go_default_library", +- "//vendor/k8s.io/kube-aggregator/pkg/apis/apiregistration/v1:go_default_library", + "//vendor/k8s.io/utils/pointer:go_default_library", + ], + ) +@@ -70,7 +68,6 @@ go_library( + go_test( + name = "go_default_test", + srcs = [ +- "apiservices_test.go", + "components_suite_test.go", + "crds_test.go", + "deployments_test.go", +@@ -85,7 +82,6 @@ go_test( + deps = [ + "//pkg/certificates/bootstrap:go_default_library", + "//pkg/certificates/triple/cert:go_default_library", +- "//staging/src/kubevirt.io/api/core/v1:go_default_library", + "//staging/src/kubevirt.io/client-go/testutils:go_default_library", + "//vendor/github.com/onsi/ginkgo/v2:go_default_library", + "//vendor/github.com/onsi/gomega:go_default_library", +diff --git a/pkg/virt-operator/resource/generate/components/validations_generated.go b/pkg/virt-operator/resource/generate/components/validations_generated.go +index 4913dbead0..42225780ba 100644 +--- a/pkg/virt-operator/resource/generate/components/validations_generated.go ++++ b/pkg/virt-operator/resource/generate/components/validations_generated.go +@@ -7723,6 +7723,8 @@ var CRDsValidation map[string]string = map[string]string{ + ContainerDisk references a docker image, embedding a qcow or raw disk. + More info: https://kubevirt.gitbooks.io/user-guide/registry-disk.html + properties: ++ hotpluggable: ++ type: boolean + image: + description: Image is the name of the image with the embedded + disk. +@@ -8355,6 +8357,35 @@ var CRDsValidation map[string]string = map[string]string{ + description: VolumeSource represents the source of the volume + to map to the disk. + properties: ++ containerDisk: ++ description: Represents a docker image with an embedded disk. ++ properties: ++ hotpluggable: ++ type: boolean ++ image: ++ description: Image is the name of the image with the embedded ++ disk. ++ type: string ++ imagePullPolicy: ++ description: |- ++ Image pull policy. ++ One of Always, Never, IfNotPresent. ++ Defaults to Always if :latest tag is specified, or IfNotPresent otherwise. ++ Cannot be updated. ++ More info: https://kubernetes.io/docs/concepts/containers/images#updating-images ++ type: string ++ imagePullSecret: ++ description: ImagePullSecret is the name of the Docker ++ registry secret required to pull the image. The secret ++ must already exist. ++ type: string ++ path: ++ description: Path defines the path to disk file in the ++ container ++ type: string ++ required: ++ - image ++ type: object + dataVolume: + description: |- + DataVolume represents the dynamic creation a PVC for this volume as well as +@@ -12768,6 +12799,8 @@ var CRDsValidation map[string]string = map[string]string{ + ContainerDisk references a docker image, embedding a qcow or raw disk. + More info: https://kubevirt.gitbooks.io/user-guide/registry-disk.html + properties: ++ hotpluggable: ++ type: boolean + image: + description: Image is the name of the image with the embedded + disk. +@@ -18328,6 +18361,8 @@ var CRDsValidation map[string]string = map[string]string{ + ContainerDisk references a docker image, embedding a qcow or raw disk. + More info: https://kubevirt.gitbooks.io/user-guide/registry-disk.html + properties: ++ hotpluggable: ++ type: boolean + image: + description: Image is the name of the image with the embedded + disk. +@@ -22835,6 +22870,8 @@ var CRDsValidation map[string]string = map[string]string{ + ContainerDisk references a docker image, embedding a qcow or raw disk. + More info: https://kubevirt.gitbooks.io/user-guide/registry-disk.html + properties: ++ hotpluggable: ++ type: boolean + image: + description: Image is the name of the image with + the embedded disk. +@@ -28015,6 +28052,8 @@ var CRDsValidation map[string]string = map[string]string{ + ContainerDisk references a docker image, embedding a qcow or raw disk. + More info: https://kubevirt.gitbooks.io/user-guide/registry-disk.html + properties: ++ hotpluggable: ++ type: boolean + image: + description: Image is the name of the image + with the embedded disk. +@@ -28673,6 +28712,36 @@ var CRDsValidation map[string]string = map[string]string{ + description: VolumeSource represents the source of + the volume to map to the disk. + properties: ++ containerDisk: ++ description: Represents a docker image with an ++ embedded disk. ++ properties: ++ hotpluggable: ++ type: boolean ++ image: ++ description: Image is the name of the image ++ with the embedded disk. ++ type: string ++ imagePullPolicy: ++ description: |- ++ Image pull policy. ++ One of Always, Never, IfNotPresent. ++ Defaults to Always if :latest tag is specified, or IfNotPresent otherwise. ++ Cannot be updated. ++ More info: https://kubernetes.io/docs/concepts/containers/images#updating-images ++ type: string ++ imagePullSecret: ++ description: ImagePullSecret is the name of ++ the Docker registry secret required to pull ++ the image. The secret must already exist. ++ type: string ++ path: ++ description: Path defines the path to disk ++ file in the container ++ type: string ++ required: ++ - image ++ type: object + dataVolume: + description: |- + DataVolume represents the dynamic creation a PVC for this volume as well as +diff --git a/pkg/virt-operator/resource/generate/install/generated_mock_strategy.go b/pkg/virt-operator/resource/generate/install/generated_mock_strategy.go +index 5f1e9a3121..1fa1416af0 100644 +--- a/pkg/virt-operator/resource/generate/install/generated_mock_strategy.go ++++ b/pkg/virt-operator/resource/generate/install/generated_mock_strategy.go +@@ -241,16 +241,6 @@ func (_mr *_MockStrategyInterfaceRecorder) MutatingWebhookConfigurations() *gomo + return _mr.mock.ctrl.RecordCall(_mr.mock, "MutatingWebhookConfigurations") + } + +-func (_m *MockStrategyInterface) APIServices() []*v18.APIService { +- ret := _m.ctrl.Call(_m, "APIServices") +- ret0, _ := ret[0].([]*v18.APIService) +- return ret0 +-} +- +-func (_mr *_MockStrategyInterfaceRecorder) APIServices() *gomock.Call { +- return _mr.mock.ctrl.RecordCall(_mr.mock, "APIServices") +-} +- + func (_m *MockStrategyInterface) CertificateSecrets() []*v14.Secret { + ret := _m.ctrl.Call(_m, "CertificateSecrets") + ret0, _ := ret[0].([]*v14.Secret) +diff --git a/pkg/virt-operator/resource/generate/rbac/exportproxy.go b/pkg/virt-operator/resource/generate/rbac/exportproxy.go +index ebc9f2adbd..a0dc0586b4 100644 +--- a/pkg/virt-operator/resource/generate/rbac/exportproxy.go ++++ b/pkg/virt-operator/resource/generate/rbac/exportproxy.go +@@ -23,6 +23,7 @@ import ( + rbacv1 "k8s.io/api/rbac/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" ++ + "kubevirt.io/kubevirt/pkg/virt-operator/resource/generate/components" + + virtv1 "kubevirt.io/api/core/v1" +diff --git a/staging/src/kubevirt.io/api/apitesting/testdata/HEAD/kubevirt.io.v1.VirtualMachine.json b/staging/src/kubevirt.io/api/apitesting/testdata/HEAD/kubevirt.io.v1.VirtualMachine.json +index b651173636..3453dfb0da 100644 +--- a/staging/src/kubevirt.io/api/apitesting/testdata/HEAD/kubevirt.io.v1.VirtualMachine.json ++++ b/staging/src/kubevirt.io/api/apitesting/testdata/HEAD/kubevirt.io.v1.VirtualMachine.json +@@ -754,7 +754,8 @@ + "image": "imageValue", + "imagePullSecret": "imagePullSecretValue", + "path": "pathValue", +- "imagePullPolicy": "imagePullPolicyValue" ++ "imagePullPolicy": "imagePullPolicyValue", ++ "hotpluggable": true + }, + "ephemeral": { + "persistentVolumeClaim": { +@@ -1209,6 +1210,13 @@ + "dataVolume": { + "name": "nameValue", + "hotpluggable": true ++ }, ++ "containerDisk": { ++ "image": "imageValue", ++ "imagePullSecret": "imagePullSecretValue", ++ "path": "pathValue", ++ "imagePullPolicy": "imagePullPolicyValue", ++ "hotpluggable": true + } + }, + "dryRun": [ +diff --git a/staging/src/kubevirt.io/api/apitesting/testdata/HEAD/kubevirt.io.v1.VirtualMachine.yaml b/staging/src/kubevirt.io/api/apitesting/testdata/HEAD/kubevirt.io.v1.VirtualMachine.yaml +index 53dfdacc3b..8b23193158 100644 +--- a/staging/src/kubevirt.io/api/apitesting/testdata/HEAD/kubevirt.io.v1.VirtualMachine.yaml ++++ b/staging/src/kubevirt.io/api/apitesting/testdata/HEAD/kubevirt.io.v1.VirtualMachine.yaml +@@ -719,6 +719,7 @@ spec: + optional: true + volumeLabel: volumeLabelValue + containerDisk: ++ hotpluggable: true + image: imageValue + imagePullPolicy: imagePullPolicyValue + imagePullSecret: imagePullSecretValue +@@ -838,6 +839,12 @@ status: + - dryRunValue + name: nameValue + volumeSource: ++ containerDisk: ++ hotpluggable: true ++ image: imageValue ++ imagePullPolicy: imagePullPolicyValue ++ imagePullSecret: imagePullSecretValue ++ path: pathValue + dataVolume: + hotpluggable: true + name: nameValue +diff --git a/staging/src/kubevirt.io/api/apitesting/testdata/HEAD/kubevirt.io.v1.VirtualMachineInstance.json b/staging/src/kubevirt.io/api/apitesting/testdata/HEAD/kubevirt.io.v1.VirtualMachineInstance.json +index 3be904512c..f595798e89 100644 +--- a/staging/src/kubevirt.io/api/apitesting/testdata/HEAD/kubevirt.io.v1.VirtualMachineInstance.json ++++ b/staging/src/kubevirt.io/api/apitesting/testdata/HEAD/kubevirt.io.v1.VirtualMachineInstance.json +@@ -694,7 +694,8 @@ + "image": "imageValue", + "imagePullSecret": "imagePullSecretValue", + "path": "pathValue", +- "imagePullPolicy": "imagePullPolicyValue" ++ "imagePullPolicy": "imagePullPolicyValue", ++ "hotpluggable": true + }, + "ephemeral": { + "persistentVolumeClaim": { +diff --git a/staging/src/kubevirt.io/api/apitesting/testdata/HEAD/kubevirt.io.v1.VirtualMachineInstance.yaml b/staging/src/kubevirt.io/api/apitesting/testdata/HEAD/kubevirt.io.v1.VirtualMachineInstance.yaml +index 6fd2ab6523..b6457ec94d 100644 +--- a/staging/src/kubevirt.io/api/apitesting/testdata/HEAD/kubevirt.io.v1.VirtualMachineInstance.yaml ++++ b/staging/src/kubevirt.io/api/apitesting/testdata/HEAD/kubevirt.io.v1.VirtualMachineInstance.yaml +@@ -524,6 +524,7 @@ spec: + optional: true + volumeLabel: volumeLabelValue + containerDisk: ++ hotpluggable: true + image: imageValue + imagePullPolicy: imagePullPolicyValue + imagePullSecret: imagePullSecretValue +diff --git a/staging/src/kubevirt.io/api/core/v1/BUILD.bazel b/staging/src/kubevirt.io/api/core/v1/BUILD.bazel +index f8615293a3..0c6c166985 100644 +--- a/staging/src/kubevirt.io/api/core/v1/BUILD.bazel ++++ b/staging/src/kubevirt.io/api/core/v1/BUILD.bazel +@@ -28,7 +28,6 @@ go_library( + "//vendor/k8s.io/apimachinery/pkg/runtime/schema:go_default_library", + "//vendor/k8s.io/apimachinery/pkg/types:go_default_library", + "//vendor/k8s.io/utils/net:go_default_library", +- "//vendor/k8s.io/utils/pointer:go_default_library", + "//vendor/kubevirt.io/containerized-data-importer-api/pkg/apis/core/v1beta1:go_default_library", + ], + ) +diff --git a/staging/src/kubevirt.io/api/core/v1/deepcopy_generated.go b/staging/src/kubevirt.io/api/core/v1/deepcopy_generated.go +index abd5a495d6..7372b22a9a 100644 +--- a/staging/src/kubevirt.io/api/core/v1/deepcopy_generated.go ++++ b/staging/src/kubevirt.io/api/core/v1/deepcopy_generated.go +@@ -1948,6 +1948,11 @@ func (in *HotplugVolumeSource) DeepCopyInto(out *HotplugVolumeSource) { + *out = new(DataVolumeSource) + **out = **in + } ++ if in.ContainerDisk != nil { ++ in, out := &in.ContainerDisk, &out.ContainerDisk ++ *out = new(ContainerDiskSource) ++ **out = **in ++ } + return + } + +diff --git a/staging/src/kubevirt.io/api/core/v1/schema.go b/staging/src/kubevirt.io/api/core/v1/schema.go +index 29aa3932d3..302ed9ffde 100644 +--- a/staging/src/kubevirt.io/api/core/v1/schema.go ++++ b/staging/src/kubevirt.io/api/core/v1/schema.go +@@ -854,6 +854,8 @@ type HotplugVolumeSource struct { + // the process of populating that PVC with a disk image. + // +optional + DataVolume *DataVolumeSource `json:"dataVolume,omitempty"` ++ ++ ContainerDisk *ContainerDiskSource `json:"containerDisk,omitempty"` + } + + type DataVolumeSource struct { +@@ -911,6 +913,8 @@ type ContainerDiskSource struct { + // More info: https://kubernetes.io/docs/concepts/containers/images#updating-images + // +optional + ImagePullPolicy v1.PullPolicy `json:"imagePullPolicy,omitempty"` ++ ++ Hotpluggable bool `json:"hotpluggable,omitempty"` + } + + // Exactly one of its members must be set. +diff --git a/staging/src/kubevirt.io/client-go/api/openapi_generated.go b/staging/src/kubevirt.io/client-go/api/openapi_generated.go +index cc2d743492..b982b1620c 100644 +--- a/staging/src/kubevirt.io/client-go/api/openapi_generated.go ++++ b/staging/src/kubevirt.io/client-go/api/openapi_generated.go +@@ -17772,6 +17772,12 @@ func schema_kubevirtio_api_core_v1_ContainerDiskSource(ref common.ReferenceCallb + Enum: []interface{}{"Always", "IfNotPresent", "Never"}, + }, + }, ++ "hotpluggable": { ++ SchemaProps: spec.SchemaProps{ ++ Type: []string{"boolean"}, ++ Format: "", ++ }, ++ }, + }, + Required: []string{"image"}, + }, +@@ -19645,11 +19651,16 @@ func schema_kubevirtio_api_core_v1_HotplugVolumeSource(ref common.ReferenceCallb + Ref: ref("kubevirt.io/api/core/v1.DataVolumeSource"), + }, + }, ++ "containerDisk": { ++ SchemaProps: spec.SchemaProps{ ++ Ref: ref("kubevirt.io/api/core/v1.ContainerDiskSource"), ++ }, ++ }, + }, + }, + }, + Dependencies: []string{ +- "kubevirt.io/api/core/v1.DataVolumeSource", "kubevirt.io/api/core/v1.PersistentVolumeClaimVolumeSource"}, ++ "kubevirt.io/api/core/v1.ContainerDiskSource", "kubevirt.io/api/core/v1.DataVolumeSource", "kubevirt.io/api/core/v1.PersistentVolumeClaimVolumeSource"}, + } + } + From fdd2e89172e4f089beff9bb011ac6cdea2017a76 Mon Sep 17 00:00:00 2001 From: yaroslavborbat Date: Wed, 15 Jan 2025 04:47:09 +0300 Subject: [PATCH 02/11] inc Signed-off-by: yaroslavborbat --- .../patches/031-hotplug-container-disk.patch | 1849 ----------------- 1 file changed, 1849 deletions(-) delete mode 100644 images/virt-artifact/patches/031-hotplug-container-disk.patch diff --git a/images/virt-artifact/patches/031-hotplug-container-disk.patch b/images/virt-artifact/patches/031-hotplug-container-disk.patch deleted file mode 100644 index d9cf34277..000000000 --- a/images/virt-artifact/patches/031-hotplug-container-disk.patch +++ /dev/null @@ -1,1849 +0,0 @@ -diff --git a/api/openapi-spec/swagger.json b/api/openapi-spec/swagger.json -index c4822a0448..d6bb534249 100644 ---- a/api/openapi-spec/swagger.json -+++ b/api/openapi-spec/swagger.json -@@ -12951,6 +12951,9 @@ - "image" - ], - "properties": { -+ "hotpluggable": { -+ "type": "boolean" -+ }, - "image": { - "description": "Image is the name of the image with the embedded disk.", - "type": "string", -@@ -13973,6 +13976,9 @@ - "description": "HotplugVolumeSource Represents the source of a volume to mount which are capable of being hotplugged on a live running VMI. Only one of its members may be specified.", - "type": "object", - "properties": { -+ "containerDisk": { -+ "$ref": "#/definitions/v1.ContainerDiskSource" -+ }, - "dataVolume": { - "description": "DataVolume represents the dynamic creation a PVC for this volume as well as the process of populating that PVC with a disk image.", - "$ref": "#/definitions/v1.DataVolumeSource" -diff --git a/cmd/virt-chroot/main.go b/cmd/virt-chroot/main.go -index e28daa07c7..7a69b7451b 100644 ---- a/cmd/virt-chroot/main.go -+++ b/cmd/virt-chroot/main.go -@@ -20,6 +20,7 @@ var ( - cpuTime uint64 - memoryBytes uint64 - targetUser string -+ targetUserID int - ) - - func init() { -@@ -51,7 +52,12 @@ func main() { - - // Looking up users needs resources, let's do it before we set rlimits. - var u *user.User -- if targetUser != "" { -+ if targetUserID >= 0 { -+ _, _, errno := syscall.Syscall(syscall.SYS_SETUID, uintptr(targetUserID), 0, 0) -+ if errno != 0 { -+ return fmt.Errorf("failed to switch to user: %d. errno: %d", targetUserID, errno) -+ } -+ } else if targetUser != "" { - var err error - u, err = user.Lookup(targetUser) - if err != nil { -@@ -116,6 +122,7 @@ func main() { - rootCmd.PersistentFlags().Uint64Var(&memoryBytes, "memory", 0, "memory in bytes for the process") - rootCmd.PersistentFlags().StringVar(&mntNamespace, "mount", "", "mount namespace to use") - rootCmd.PersistentFlags().StringVar(&targetUser, "user", "", "switch to this targetUser to e.g. drop privileges") -+ rootCmd.PersistentFlags().IntVar(&targetUserID, "userid", -1, "switch to this targetUser to e.g. drop privileges") - - execCmd := &cobra.Command{ - Use: "exec", -@@ -136,16 +143,39 @@ func main() { - Args: cobra.MinimumNArgs(2), - RunE: func(cmd *cobra.Command, args []string) error { - var mntOpts uint = 0 -+ var dataOpts []string - - fsType := cmd.Flag("type").Value.String() - mntOptions := cmd.Flag("options").Value.String() -+ var ( -+ uid = -1 -+ gid = -1 -+ ) - for _, opt := range strings.Split(mntOptions, ",") { - opt = strings.TrimSpace(opt) -- switch opt { -- case "ro": -+ switch { -+ case opt == "ro": - mntOpts = mntOpts | syscall.MS_RDONLY -- case "bind": -+ case opt == "bind": - mntOpts = mntOpts | syscall.MS_BIND -+ case opt == "remount": -+ mntOpts = mntOpts | syscall.MS_REMOUNT -+ case strings.HasPrefix(opt, "uid="): -+ uidS := strings.TrimPrefix(opt, "uid=") -+ uidI, err := strconv.Atoi(uidS) -+ if err != nil { -+ return fmt.Errorf("failed to parse uid: %w", err) -+ } -+ uid = uidI -+ dataOpts = append(dataOpts, opt) -+ case strings.HasPrefix(opt, "gid="): -+ gidS := strings.TrimPrefix(opt, "gid=") -+ gidI, err := strconv.Atoi(gidS) -+ if err != nil { -+ return fmt.Errorf("failed to parse gid: %w", err) -+ } -+ gid = gidI -+ dataOpts = append(dataOpts, opt) - default: - return fmt.Errorf("mount option %s is not supported", opt) - } -@@ -168,8 +198,17 @@ func main() { - return fmt.Errorf("mount target invalid: %v", err) - } - defer targetFile.Close() -- -- return syscall.Mount(sourceFile.SafePath(), targetFile.SafePath(), fsType, uintptr(mntOpts), "") -+ if uid >= 0 && gid >= 0 { -+ err = os.Chown(targetFile.SafePath(), uid, gid) -+ if err != nil { -+ return fmt.Errorf("chown target failed: %w", err) -+ } -+ } -+ var data string -+ if len(dataOpts) > 0 { -+ data = strings.Join(dataOpts, ",") -+ } -+ return syscall.Mount(sourceFile.SafePath(), targetFile.SafePath(), fsType, uintptr(mntOpts), data) - }, - } - mntCmd.Flags().StringP("options", "o", "", "comma separated list of mount options") -diff --git a/manifests/generated/kv-resource.yaml b/manifests/generated/kv-resource.yaml -index 66d1b01dbf..43e36b7195 100644 ---- a/manifests/generated/kv-resource.yaml -+++ b/manifests/generated/kv-resource.yaml -@@ -3307,9 +3307,6 @@ spec: - - jsonPath: .status.phase - name: Phase - type: string -- deprecated: true -- deprecationWarning: kubevirt.io/v1alpha3 is now deprecated and will be removed -- in a future release. - name: v1alpha3 - schema: - openAPIV3Schema: -diff --git a/manifests/generated/operator-csv.yaml.in b/manifests/generated/operator-csv.yaml.in -index 400d118024..05ee099c67 100644 ---- a/manifests/generated/operator-csv.yaml.in -+++ b/manifests/generated/operator-csv.yaml.in -@@ -605,6 +605,13 @@ spec: - - '*' - verbs: - - '*' -+ - apiGroups: -+ - subresources.virtualization.deckhouse.io -+ resources: -+ - virtualmachines/addvolume -+ - virtualmachines/removevolume -+ verbs: -+ - update - - apiGroups: - - subresources.kubevirt.io - resources: -diff --git a/manifests/generated/rbac-operator.authorization.k8s.yaml.in b/manifests/generated/rbac-operator.authorization.k8s.yaml.in -index 10dbb92269..1ccc9e9fa7 100644 ---- a/manifests/generated/rbac-operator.authorization.k8s.yaml.in -+++ b/manifests/generated/rbac-operator.authorization.k8s.yaml.in -@@ -143,7 +143,7 @@ kind: RoleBinding - metadata: - labels: - kubevirt.io: "" -- name: kubevirt-operator-rolebinding -+ name: kubevirt-operator - namespace: {{.Namespace}} - roleRef: - apiGroup: rbac.authorization.k8s.io -@@ -607,6 +607,13 @@ rules: - - '*' - verbs: - - '*' -+- apiGroups: -+ - subresources.virtualization.deckhouse.io -+ resources: -+ - virtualmachines/addvolume -+ - virtualmachines/removevolume -+ verbs: -+ - update - - apiGroups: - - subresources.kubevirt.io - resources: -diff --git a/pkg/container-disk/container-disk.go b/pkg/container-disk/container-disk.go -index 3251d04787..34affe841a 100644 ---- a/pkg/container-disk/container-disk.go -+++ b/pkg/container-disk/container-disk.go -@@ -47,8 +47,10 @@ var containerDiskOwner = "qemu" - var podsBaseDir = util.KubeletPodsDir - - var mountBaseDir = filepath.Join(util.VirtShareDir, "/container-disks") -+var hotplugBaseDir = filepath.Join(util.VirtShareDir, "/hotplug-disks") - - type SocketPathGetter func(vmi *v1.VirtualMachineInstance, volumeIndex int) (string, error) -+type HotplugSocketPathGetter func(vmi *v1.VirtualMachineInstance, volumeName string) (string, error) - type KernelBootSocketPathGetter func(vmi *v1.VirtualMachineInstance) (string, error) - - const KernelBootName = "kernel-boot" -@@ -107,6 +109,10 @@ func GetDiskTargetPathFromLauncherView(volumeIndex int) string { - return filepath.Join(mountBaseDir, GetDiskTargetName(volumeIndex)) - } - -+func GetHotplugContainerDiskTargetPathFromLauncherView(volumeName string) string { -+ return filepath.Join(hotplugBaseDir, fmt.Sprintf("%s.img", volumeName)) -+} -+ - func GetKernelBootArtifactPathFromLauncherView(artifact string) string { - artifactBase := filepath.Base(artifact) - return filepath.Join(mountBaseDir, KernelBootName, artifactBase) -@@ -170,6 +176,23 @@ func NewSocketPathGetter(baseDir string) SocketPathGetter { - } - } - -+func NewHotplugSocketPathGetter(baseDir string) HotplugSocketPathGetter { -+ return func(vmi *v1.VirtualMachineInstance, volumeName string) (string, error) { -+ for _, v := range vmi.Status.VolumeStatus { -+ if v.Name == volumeName && v.HotplugVolume != nil && v.ContainerDiskVolume != nil { -+ basePath := getHotplugContainerDiskSocketBasePath(baseDir, string(v.HotplugVolume.AttachPodUID)) -+ socketPath := filepath.Join(basePath, fmt.Sprintf("hotplug-container-disk-%s.sock", volumeName)) -+ exists, _ := diskutils.FileExists(socketPath) -+ if exists { -+ return socketPath, nil -+ } -+ } -+ } -+ -+ return "", fmt.Errorf("container disk socket path not found for vmi \"%s\"", vmi.Name) -+ } -+} -+ - // NewKernelBootSocketPathGetter get the socket pat of the kernel-boot containerDisk. For testing a baseDir - // can be provided which can for instance point to /tmp. - func NewKernelBootSocketPathGetter(baseDir string) KernelBootSocketPathGetter { -@@ -394,10 +417,37 @@ func CreateEphemeralImages( - return nil - } - -+func CreateEphemeralImagesForHotplug( -+ vmi *v1.VirtualMachineInstance, -+ diskCreator ephemeraldisk.EphemeralDiskCreatorInterface, -+ disksInfo map[string]*DiskInfo, -+) error { -+ for i, volume := range vmi.Spec.Volumes { -+ if volume.VolumeSource.ContainerDisk != nil && volume.VolumeSource.ContainerDisk.Hotpluggable { -+ info, _ := disksInfo[volume.Name] -+ if info == nil { -+ return fmt.Errorf("no disk info provided for volume %s", volume.Name) -+ } -+ -+ if backingFile, err := GetDiskTargetPartFromLauncherView(i); err != nil { -+ return err -+ } else if err := diskCreator.CreateBackedImageForVolume(volume, backingFile, info.Format); err != nil { -+ return err -+ } -+ } -+ } -+ -+ return nil -+} -+ - func getContainerDiskSocketBasePath(baseDir, podUID string) string { - return fmt.Sprintf("%s/pods/%s/volumes/kubernetes.io~empty-dir/container-disks", baseDir, podUID) - } - -+func getHotplugContainerDiskSocketBasePath(baseDir, podUID string) string { -+ return fmt.Sprintf("%s/pods/%s/volumes/kubernetes.io~empty-dir/hotplug-container-disks", baseDir, podUID) -+} -+ - // ExtractImageIDsFromSourcePod takes the VMI and its source pod to determine the exact image used by containerdisks and boot container images, - // which is recorded in the status section of a started pod; if the status section does not contain this info the tag is used. - // It returns a map where the key is the vlume name and the value is the imageID -diff --git a/pkg/controller/controller.go b/pkg/controller/controller.go -index 490cc445ef..4b7dbc12fe 100644 ---- a/pkg/controller/controller.go -+++ b/pkg/controller/controller.go -@@ -278,6 +278,10 @@ func ApplyVolumeRequestOnVMISpec(vmiSpec *v1.VirtualMachineInstanceSpec, request - dvSource := request.AddVolumeOptions.VolumeSource.DataVolume.DeepCopy() - dvSource.Hotpluggable = true - newVolume.VolumeSource.DataVolume = dvSource -+ } else if request.AddVolumeOptions.VolumeSource.ContainerDisk != nil { -+ containerDiskSource := request.AddVolumeOptions.VolumeSource.ContainerDisk.DeepCopy() -+ containerDiskSource.Hotpluggable = true -+ newVolume.VolumeSource.ContainerDisk = containerDiskSource - } - - vmiSpec.Volumes = append(vmiSpec.Volumes, newVolume) -@@ -444,6 +448,9 @@ func VMIHasHotplugVolumes(vmi *v1.VirtualMachineInstance) bool { - if volume.PersistentVolumeClaim != nil && volume.PersistentVolumeClaim.Hotpluggable { - return true - } -+ if volume.ContainerDisk != nil && volume.ContainerDisk.Hotpluggable { -+ return true -+ } - } - return false - } -@@ -557,7 +564,7 @@ func GetHotplugVolumes(vmi *v1.VirtualMachineInstance, virtlauncherPod *k8sv1.Po - podVolumeMap[podVolume.Name] = podVolume - } - for _, vmiVolume := range vmiVolumes { -- if _, ok := podVolumeMap[vmiVolume.Name]; !ok && (vmiVolume.DataVolume != nil || vmiVolume.PersistentVolumeClaim != nil || vmiVolume.MemoryDump != nil) { -+ if _, ok := podVolumeMap[vmiVolume.Name]; !ok && (vmiVolume.DataVolume != nil || vmiVolume.PersistentVolumeClaim != nil || vmiVolume.MemoryDump != nil || vmiVolume.ContainerDisk != nil) { - hotplugVolumes = append(hotplugVolumes, vmiVolume.DeepCopy()) - } - } -diff --git a/pkg/virt-api/rest/subresource.go b/pkg/virt-api/rest/subresource.go -index b5d62f5af5..bf561f00ae 100644 ---- a/pkg/virt-api/rest/subresource.go -+++ b/pkg/virt-api/rest/subresource.go -@@ -1023,7 +1023,8 @@ func volumeSourceName(volumeSource *v1.HotplugVolumeSource) string { - - func volumeSourceExists(volume v1.Volume, volumeName string) bool { - return (volume.DataVolume != nil && volume.DataVolume.Name == volumeName) || -- (volume.PersistentVolumeClaim != nil && volume.PersistentVolumeClaim.ClaimName == volumeName) -+ (volume.PersistentVolumeClaim != nil && volume.PersistentVolumeClaim.ClaimName == volumeName) || -+ (volume.ContainerDisk != nil && volume.ContainerDisk.Image != "") - } - - func volumeExists(volume v1.Volume, volumeName string) bool { -@@ -1125,6 +1126,8 @@ func (app *SubresourceAPIApp) addVolumeRequestHandler(request *restful.Request, - opts.VolumeSource.DataVolume.Hotpluggable = true - } else if opts.VolumeSource.PersistentVolumeClaim != nil { - opts.VolumeSource.PersistentVolumeClaim.Hotpluggable = true -+ } else if opts.VolumeSource.ContainerDisk != nil { -+ opts.VolumeSource.ContainerDisk.Hotpluggable = true - } - - // inject into VMI if ephemeral, else set as a request on the VM to both make permanent and hotplug. -diff --git a/pkg/virt-api/webhooks/validating-webhook/admitters/vmi-update-admitter.go b/pkg/virt-api/webhooks/validating-webhook/admitters/vmi-update-admitter.go -index 0af25f8074..803c0ed4cd 100644 ---- a/pkg/virt-api/webhooks/validating-webhook/admitters/vmi-update-admitter.go -+++ b/pkg/virt-api/webhooks/validating-webhook/admitters/vmi-update-admitter.go -@@ -200,11 +200,11 @@ func verifyHotplugVolumes(newHotplugVolumeMap, oldHotplugVolumeMap map[string]v1 - } - } else { - // This is a new volume, ensure that the volume is either DV, PVC or memoryDumpVolume -- if v.DataVolume == nil && v.PersistentVolumeClaim == nil && v.MemoryDump == nil { -+ if v.DataVolume == nil && v.PersistentVolumeClaim == nil && v.MemoryDump == nil && v.ContainerDisk == nil { - return webhookutils.ToAdmissionResponse([]metav1.StatusCause{ - { - Type: metav1.CauseTypeFieldValueInvalid, -- Message: fmt.Sprintf("volume %s is not a PVC or DataVolume", k), -+ Message: fmt.Sprintf("volume %s is not a PVC,DataVolume,MemoryDumpVolume or ContainerDisk", k), - }, - }) - } -diff --git a/pkg/virt-controller/services/template.go b/pkg/virt-controller/services/template.go -index 76ed7307ec..f607c24786 100644 ---- a/pkg/virt-controller/services/template.go -+++ b/pkg/virt-controller/services/template.go -@@ -64,13 +64,15 @@ import ( - ) - - const ( -- containerDisks = "container-disks" -- hotplugDisks = "hotplug-disks" -- hookSidecarSocks = "hook-sidecar-sockets" -- varRun = "/var/run" -- virtBinDir = "virt-bin-share-dir" -- hotplugDisk = "hotplug-disk" -- virtExporter = "virt-exporter" -+ containerDisks = "container-disks" -+ hotplugDisks = "hotplug-disks" -+ hookSidecarSocks = "hook-sidecar-sockets" -+ varRun = "/var/run" -+ virtBinDir = "virt-bin-share-dir" -+ hotplugDisk = "hotplug-disk" -+ virtExporter = "virt-exporter" -+ hotplugContainerDisks = "hotplug-container-disks" -+ HotplugContainerDisk = "hotplug-container-disk-" - ) - - const KvmDevice = "devices.virtualization.deckhouse.io/kvm" -@@ -846,6 +848,49 @@ func sidecarContainerName(i int) string { - return fmt.Sprintf("hook-sidecar-%d", i) - } - -+func sidecarContainerHotplugContainerdDiskName(name string) string { -+ return fmt.Sprintf("%s%s", HotplugContainerDisk, name) -+} -+ -+func (t *templateService) containerForHotplugContainerDisk(name string, cd *v1.ContainerDiskSource, vmi *v1.VirtualMachineInstance) k8sv1.Container { -+ runUser := int64(util.NonRootUID) -+ sharedMount := k8sv1.MountPropagationHostToContainer -+ path := fmt.Sprintf("/path/%s", name) -+ command := []string{"/init/usr/bin/container-disk"} -+ args := []string{"--copy-path", path} -+ -+ return k8sv1.Container{ -+ Name: name, -+ Image: cd.Image, -+ Command: command, -+ Args: args, -+ Resources: hotplugContainerResourceRequirementsForVMI(vmi, t.clusterConfig), -+ SecurityContext: &k8sv1.SecurityContext{ -+ AllowPrivilegeEscalation: pointer.Bool(false), -+ RunAsNonRoot: pointer.Bool(true), -+ RunAsUser: &runUser, -+ SeccompProfile: &k8sv1.SeccompProfile{ -+ Type: k8sv1.SeccompProfileTypeRuntimeDefault, -+ }, -+ Capabilities: &k8sv1.Capabilities{ -+ Drop: []k8sv1.Capability{"ALL"}, -+ }, -+ SELinuxOptions: &k8sv1.SELinuxOptions{ -+ Type: t.clusterConfig.GetSELinuxLauncherType(), -+ Level: "s0", -+ }, -+ }, -+ VolumeMounts: []k8sv1.VolumeMount{ -+ initContainerVolumeMount(), -+ { -+ Name: hotplugContainerDisks, -+ MountPath: "/path", -+ MountPropagation: &sharedMount, -+ }, -+ }, -+ } -+} -+ - func (t *templateService) RenderHotplugAttachmentPodTemplate(volumes []*v1.Volume, ownerPod *k8sv1.Pod, vmi *v1.VirtualMachineInstance, claimMap map[string]*k8sv1.PersistentVolumeClaim) (*k8sv1.Pod, error) { - zero := int64(0) - runUser := int64(util.NonRootUID) -@@ -924,6 +969,30 @@ func (t *templateService) RenderHotplugAttachmentPodTemplate(volumes []*v1.Volum - TerminationGracePeriodSeconds: &zero, - }, - } -+ first := true -+ for _, vol := range vmi.Spec.Volumes { -+ if vol.ContainerDisk == nil || !vol.ContainerDisk.Hotpluggable { -+ continue -+ } -+ name := sidecarContainerHotplugContainerdDiskName(vol.Name) -+ pod.Spec.Containers = append(pod.Spec.Containers, t.containerForHotplugContainerDisk(name, vol.ContainerDisk, vmi)) -+ if first { -+ first = false -+ userId := int64(util.NonRootUID) -+ initContainerCommand := []string{"/usr/bin/cp", -+ "/usr/bin/container-disk", -+ "/init/usr/bin/container-disk", -+ } -+ pod.Spec.InitContainers = append( -+ pod.Spec.InitContainers, -+ t.newInitContainerRenderer(vmi, -+ initContainerVolumeMount(), -+ initContainerResourceRequirementsForVMI(vmi, v1.ContainerDisk, t.clusterConfig), -+ userId).Render(initContainerCommand)) -+ pod.Spec.Volumes = append(pod.Spec.Volumes, emptyDirVolume(hotplugContainerDisks)) -+ pod.Spec.Volumes = append(pod.Spec.Volumes, emptyDirVolume(virtBinDir)) -+ } -+ } - - err := matchSELinuxLevelOfVMI(pod, vmi) - if err != nil { -diff --git a/pkg/virt-controller/watch/BUILD.bazel b/pkg/virt-controller/watch/BUILD.bazel -index 4fd325ba86..82fcaee0a3 100644 ---- a/pkg/virt-controller/watch/BUILD.bazel -+++ b/pkg/virt-controller/watch/BUILD.bazel -@@ -101,6 +101,7 @@ go_library( - "//vendor/k8s.io/client-go/util/flowcontrol:go_default_library", - "//vendor/k8s.io/client-go/util/workqueue:go_default_library", - "//vendor/k8s.io/utils/pointer:go_default_library", -+ "//vendor/k8s.io/utils/ptr:go_default_library", - "//vendor/k8s.io/utils/trace:go_default_library", - "//vendor/kubevirt.io/containerized-data-importer-api/pkg/apis/core/v1beta1:go_default_library", - ], -diff --git a/pkg/virt-controller/watch/vmi.go b/pkg/virt-controller/watch/vmi.go -index fa4e86ee17..ebe718f90d 100644 ---- a/pkg/virt-controller/watch/vmi.go -+++ b/pkg/virt-controller/watch/vmi.go -@@ -1836,6 +1836,10 @@ func (c *VMIController) handleHotplugVolumes(hotplugVolumes []*virtv1.Volume, ho - readyHotplugVolumes := make([]*virtv1.Volume, 0) - // Find all ready volumes - for _, volume := range hotplugVolumes { -+ if volume.ContainerDisk != nil { -+ readyHotplugVolumes = append(readyHotplugVolumes, volume) -+ continue -+ } - var err error - ready, wffc, err := storagetypes.VolumeReadyToAttachToNode(vmi.Namespace, *volume, dataVolumes, c.dataVolumeIndexer, c.pvcIndexer) - if err != nil { -@@ -1884,7 +1888,15 @@ func (c *VMIController) handleHotplugVolumes(hotplugVolumes []*virtv1.Volume, ho - - func (c *VMIController) podVolumesMatchesReadyVolumes(attachmentPod *k8sv1.Pod, volumes []*virtv1.Volume) bool { - // -2 for empty dir and token -- if len(attachmentPod.Spec.Volumes)-2 != len(volumes) { -+ // -3 if exist container-disk -+ magicNum := len(attachmentPod.Spec.Volumes) - 2 -+ for _, volume := range volumes { -+ if volume.ContainerDisk != nil { -+ magicNum -= 1 -+ break -+ } -+ } -+ if magicNum != len(volumes) { - return false - } - podVolumeMap := make(map[string]k8sv1.Volume) -@@ -1893,10 +1905,20 @@ func (c *VMIController) podVolumesMatchesReadyVolumes(attachmentPod *k8sv1.Pod, - podVolumeMap[volume.Name] = volume - } - } -+ containerDisksNames := make(map[string]struct{}) -+ for _, ctr := range attachmentPod.Spec.Containers { -+ if strings.HasPrefix(ctr.Name, services.HotplugContainerDisk) { -+ containerDisksNames[strings.TrimPrefix(ctr.Name, services.HotplugContainerDisk)] = struct{}{} -+ } -+ } - for _, volume := range volumes { -+ if volume.ContainerDisk != nil { -+ delete(containerDisksNames, volume.Name) -+ continue -+ } - delete(podVolumeMap, volume.Name) - } -- return len(podVolumeMap) == 0 -+ return len(podVolumeMap) == 0 && len(containerDisksNames) == 0 - } - - func (c *VMIController) createAttachmentPod(vmi *virtv1.VirtualMachineInstance, virtLauncherPod *k8sv1.Pod, volumes []*virtv1.Volume) (*k8sv1.Pod, syncError) { -@@ -2007,7 +2029,17 @@ func (c *VMIController) createAttachmentPodTemplate(vmi *virtv1.VirtualMachineIn - var pod *k8sv1.Pod - var err error - -- volumeNamesPVCMap, err := storagetypes.VirtVolumesToPVCMap(volumes, c.pvcIndexer, virtlauncherPod.Namespace) -+ var hasContainerDisk bool -+ var newVolumes []*virtv1.Volume -+ for _, volume := range volumes { -+ if volume.VolumeSource.ContainerDisk != nil { -+ hasContainerDisk = true -+ continue -+ } -+ newVolumes = append(newVolumes, volume) -+ } -+ -+ volumeNamesPVCMap, err := storagetypes.VirtVolumesToPVCMap(newVolumes, c.pvcIndexer, virtlauncherPod.Namespace) - if err != nil { - return nil, fmt.Errorf("failed to get PVC map: %v", err) - } -@@ -2029,7 +2061,7 @@ func (c *VMIController) createAttachmentPodTemplate(vmi *virtv1.VirtualMachineIn - } - } - -- if len(volumeNamesPVCMap) > 0 { -+ if len(volumeNamesPVCMap) > 0 || hasContainerDisk { - pod, err = c.templateService.RenderHotplugAttachmentPodTemplate(volumes, virtlauncherPod, vmi, volumeNamesPVCMap) - } - return pod, err -@@ -2151,23 +2183,39 @@ func (c *VMIController) updateVolumeStatus(vmi *virtv1.VirtualMachineInstance, v - ClaimName: volume.Name, - } - } -+ if volume.ContainerDisk != nil && status.ContainerDiskVolume == nil { -+ status.ContainerDiskVolume = &virtv1.ContainerDiskInfo{} -+ } - if attachmentPod == nil { -- if !c.volumeReady(status.Phase) { -- status.HotplugVolume.AttachPodUID = "" -- // Volume is not hotplugged in VM and Pod is gone, or hasn't been created yet, check for the PVC associated with the volume to set phase and message -- phase, reason, message := c.getVolumePhaseMessageReason(&vmi.Spec.Volumes[i], vmi.Namespace) -- status.Phase = phase -- status.Message = message -- status.Reason = reason -+ if volume.ContainerDisk != nil { -+ if !c.volumeReady(status.Phase) { -+ status.HotplugVolume.AttachPodUID = "" -+ // Volume is not hotplugged in VM and Pod is gone, or hasn't been created yet, check for the PVC associated with the volume to set phase and message -+ phase, reason, message := c.getVolumePhaseMessageReason(&vmi.Spec.Volumes[i], vmi.Namespace) -+ status.Phase = phase -+ status.Message = message -+ status.Reason = reason -+ } - } - } else { - status.HotplugVolume.AttachPodName = attachmentPod.Name -- if len(attachmentPod.Status.ContainerStatuses) == 1 && attachmentPod.Status.ContainerStatuses[0].Ready { -+ if volume.ContainerDisk != nil { -+ uid := types.UID("") -+ for _, cs := range attachmentPod.Status.ContainerStatuses { -+ name := strings.TrimPrefix(cs.Name, "hotplug-container-disk-") -+ if volume.Name == name && cs.Ready { -+ uid = attachmentPod.UID -+ break -+ } -+ } -+ status.HotplugVolume.AttachPodUID = uid -+ } else if len(attachmentPod.Status.ContainerStatuses) == 1 && attachmentPod.Status.ContainerStatuses[0].Ready { - status.HotplugVolume.AttachPodUID = attachmentPod.UID - } else { - // Remove UID of old pod if a new one is available, but not yet ready - status.HotplugVolume.AttachPodUID = "" - } -+ - if c.canMoveToAttachedPhase(status.Phase) { - status.Phase = virtv1.HotplugVolumeAttachedToNode - status.Message = fmt.Sprintf("Created hotplug attachment pod %s, for volume %s", attachmentPod.Name, volume.Name) -@@ -2176,7 +2224,6 @@ func (c *VMIController) updateVolumeStatus(vmi *virtv1.VirtualMachineInstance, v - } - } - } -- - if volume.VolumeSource.PersistentVolumeClaim != nil || volume.VolumeSource.DataVolume != nil || volume.VolumeSource.MemoryDump != nil { - - pvcName := storagetypes.PVCNameFromVirtVolume(&volume) -diff --git a/pkg/virt-handler/container-disk/hotplug.go b/pkg/virt-handler/container-disk/hotplug.go -new file mode 100644 -index 0000000000..f0d3a0607c ---- /dev/null -+++ b/pkg/virt-handler/container-disk/hotplug.go -@@ -0,0 +1,481 @@ -+package container_disk -+ -+import ( -+ "encoding/json" -+ "errors" -+ "fmt" -+ "os" -+ "path/filepath" -+ "strings" -+ "sync" -+ "time" -+ -+ hotplugdisk "kubevirt.io/kubevirt/pkg/hotplug-disk" -+ "kubevirt.io/kubevirt/pkg/unsafepath" -+ -+ "kubevirt.io/kubevirt/pkg/safepath" -+ virtconfig "kubevirt.io/kubevirt/pkg/virt-config" -+ virt_chroot "kubevirt.io/kubevirt/pkg/virt-handler/virt-chroot" -+ -+ "kubevirt.io/client-go/log" -+ -+ containerdisk "kubevirt.io/kubevirt/pkg/container-disk" -+ diskutils "kubevirt.io/kubevirt/pkg/ephemeral-disk-utils" -+ "kubevirt.io/kubevirt/pkg/virt-handler/isolation" -+ -+ "k8s.io/apimachinery/pkg/api/equality" -+ "k8s.io/apimachinery/pkg/types" -+ -+ v1 "kubevirt.io/api/core/v1" -+) -+ -+type HotplugMounter interface { -+ ContainerDisksReady(vmi *v1.VirtualMachineInstance, notInitializedSince time.Time) (bool, error) -+ MountAndVerify(vmi *v1.VirtualMachineInstance) (map[string]*containerdisk.DiskInfo, error) -+ IsMounted(vmi *v1.VirtualMachineInstance, volumeName string) (bool, error) -+ Umount(vmi *v1.VirtualMachineInstance) error -+ UmountAll(vmi *v1.VirtualMachineInstance) error -+ ComputeChecksums(vmi *v1.VirtualMachineInstance) (*DiskChecksums, error) -+} -+ -+type hotplugMounter struct { -+ podIsolationDetector isolation.PodIsolationDetector -+ mountStateDir string -+ mountRecords map[types.UID]*vmiMountTargetRecord -+ mountRecordsLock sync.Mutex -+ suppressWarningTimeout time.Duration -+ clusterConfig *virtconfig.ClusterConfig -+ nodeIsolationResult isolation.IsolationResult -+ -+ hotplugPathGetter containerdisk.HotplugSocketPathGetter -+ hotplugManager hotplugdisk.HotplugDiskManagerInterface -+} -+ -+func (m *hotplugMounter) IsMounted(vmi *v1.VirtualMachineInstance, volumeName string) (bool, error) { -+ virtLauncherUID := m.findVirtlauncherUID(vmi) -+ if virtLauncherUID == "" { -+ return false, nil -+ } -+ target, err := m.hotplugManager.GetFileSystemDiskTargetPathFromHostView(virtLauncherUID, volumeName, false) -+ if err != nil { -+ return false, err -+ } -+ return isolation.IsMounted(target) -+} -+ -+func NewHotplugMounter(isoDetector isolation.PodIsolationDetector, -+ mountStateDir string, -+ clusterConfig *virtconfig.ClusterConfig, -+ hotplugManager hotplugdisk.HotplugDiskManagerInterface, -+) HotplugMounter { -+ return &hotplugMounter{ -+ mountRecords: make(map[types.UID]*vmiMountTargetRecord), -+ podIsolationDetector: isoDetector, -+ mountStateDir: mountStateDir, -+ suppressWarningTimeout: 1 * time.Minute, -+ clusterConfig: clusterConfig, -+ nodeIsolationResult: isolation.NodeIsolationResult(), -+ -+ hotplugPathGetter: containerdisk.NewHotplugSocketPathGetter(""), -+ hotplugManager: hotplugManager, -+ } -+} -+ -+func (m *hotplugMounter) deleteMountTargetRecord(vmi *v1.VirtualMachineInstance) error { -+ if string(vmi.UID) == "" { -+ return fmt.Errorf("unable to find container disk mounted directories for vmi without uid") -+ } -+ -+ recordFile := filepath.Join(m.mountStateDir, string(vmi.UID)) -+ -+ exists, err := diskutils.FileExists(recordFile) -+ if err != nil { -+ return err -+ } -+ -+ if exists { -+ record, err := m.getMountTargetRecord(vmi) -+ if err != nil { -+ return err -+ } -+ -+ for _, target := range record.MountTargetEntries { -+ os.Remove(target.TargetFile) -+ os.Remove(target.SocketFile) -+ } -+ -+ os.Remove(recordFile) -+ } -+ -+ m.mountRecordsLock.Lock() -+ defer m.mountRecordsLock.Unlock() -+ delete(m.mountRecords, vmi.UID) -+ -+ return nil -+} -+ -+func (m *hotplugMounter) getMountTargetRecord(vmi *v1.VirtualMachineInstance) (*vmiMountTargetRecord, error) { -+ var ok bool -+ var existingRecord *vmiMountTargetRecord -+ -+ if string(vmi.UID) == "" { -+ return nil, fmt.Errorf("unable to find container disk mounted directories for vmi without uid") -+ } -+ -+ m.mountRecordsLock.Lock() -+ defer m.mountRecordsLock.Unlock() -+ existingRecord, ok = m.mountRecords[vmi.UID] -+ -+ // first check memory cache -+ if ok { -+ return existingRecord, nil -+ } -+ -+ // if not there, see if record is on disk, this can happen if virt-handler restarts -+ recordFile := filepath.Join(m.mountStateDir, filepath.Clean(string(vmi.UID))) -+ -+ exists, err := diskutils.FileExists(recordFile) -+ if err != nil { -+ return nil, err -+ } -+ -+ if exists { -+ record := vmiMountTargetRecord{} -+ // #nosec No risk for path injection. Using static base and cleaned filename -+ bytes, err := os.ReadFile(recordFile) -+ if err != nil { -+ return nil, err -+ } -+ err = json.Unmarshal(bytes, &record) -+ if err != nil { -+ return nil, err -+ } -+ -+ if !record.UsesSafePaths { -+ record.UsesSafePaths = true -+ for i, entry := range record.MountTargetEntries { -+ safePath, err := safepath.JoinAndResolveWithRelativeRoot("/", entry.TargetFile) -+ if err != nil { -+ return nil, fmt.Errorf("failed converting legacy path to safepath: %v", err) -+ } -+ record.MountTargetEntries[i].TargetFile = unsafepath.UnsafeAbsolute(safePath.Raw()) -+ } -+ } -+ -+ m.mountRecords[vmi.UID] = &record -+ return &record, nil -+ } -+ -+ // not found -+ return nil, nil -+} -+ -+func (m *hotplugMounter) addMountTargetRecord(vmi *v1.VirtualMachineInstance, record *vmiMountTargetRecord) error { -+ return m.setAddMountTargetRecordHelper(vmi, record, true) -+} -+ -+func (m *hotplugMounter) setMountTargetRecord(vmi *v1.VirtualMachineInstance, record *vmiMountTargetRecord) error { -+ return m.setAddMountTargetRecordHelper(vmi, record, false) -+} -+ -+func (m *hotplugMounter) setAddMountTargetRecordHelper(vmi *v1.VirtualMachineInstance, record *vmiMountTargetRecord, addPreviousRules bool) error { -+ if string(vmi.UID) == "" { -+ return fmt.Errorf("unable to set container disk mounted directories for vmi without uid") -+ } -+ -+ record.UsesSafePaths = true -+ -+ recordFile := filepath.Join(m.mountStateDir, string(vmi.UID)) -+ fileExists, err := diskutils.FileExists(recordFile) -+ if err != nil { -+ return err -+ } -+ -+ m.mountRecordsLock.Lock() -+ defer m.mountRecordsLock.Unlock() -+ -+ existingRecord, ok := m.mountRecords[vmi.UID] -+ if ok && fileExists && equality.Semantic.DeepEqual(existingRecord, record) { -+ // already done -+ return nil -+ } -+ -+ if addPreviousRules && existingRecord != nil && len(existingRecord.MountTargetEntries) > 0 { -+ record.MountTargetEntries = append(record.MountTargetEntries, existingRecord.MountTargetEntries...) -+ } -+ -+ bytes, err := json.Marshal(record) -+ if err != nil { -+ return err -+ } -+ -+ err = os.MkdirAll(filepath.Dir(recordFile), 0750) -+ if err != nil { -+ return err -+ } -+ -+ err = os.WriteFile(recordFile, bytes, 0600) -+ if err != nil { -+ return err -+ } -+ -+ m.mountRecords[vmi.UID] = record -+ -+ return nil -+} -+ -+func (m *hotplugMounter) MountAndVerify(vmi *v1.VirtualMachineInstance) (map[string]*containerdisk.DiskInfo, error) { -+ virtLauncherUID := m.findVirtlauncherUID(vmi) -+ if virtLauncherUID == "" { -+ return nil, nil -+ } -+ -+ record := vmiMountTargetRecord{} -+ disksInfo := map[string]*containerdisk.DiskInfo{} -+ -+ for _, volume := range vmi.Spec.Volumes { -+ if volume.ContainerDisk != nil && volume.ContainerDisk.Hotpluggable { -+ target, err := m.hotplugManager.GetFileSystemDiskTargetPathFromHostView(virtLauncherUID, volume.Name, true) -+ if err != nil { -+ return nil, err -+ } -+ -+ sock, err := m.hotplugPathGetter(vmi, volume.Name) -+ if err != nil { -+ return nil, err -+ } -+ -+ record.MountTargetEntries = append(record.MountTargetEntries, vmiMountTargetEntry{ -+ TargetFile: unsafepath.UnsafeAbsolute(target.Raw()), -+ SocketFile: sock, -+ }) -+ } -+ } -+ -+ if len(record.MountTargetEntries) > 0 { -+ err := m.setMountTargetRecord(vmi, &record) -+ if err != nil { -+ return nil, err -+ } -+ } -+ -+ vmiRes, err := m.podIsolationDetector.Detect(vmi) -+ if err != nil { -+ return nil, fmt.Errorf("failed to detect VMI pod: %v", err) -+ } -+ -+ for _, volume := range vmi.Spec.Volumes { -+ if volume.ContainerDisk != nil && volume.ContainerDisk.Hotpluggable { -+ target, err := m.hotplugManager.GetFileSystemDiskTargetPathFromHostView(virtLauncherUID, volume.Name, false) -+ -+ if isMounted, err := isolation.IsMounted(target); err != nil { -+ return nil, fmt.Errorf("failed to determine if %s is already mounted: %v", target, err) -+ } else if !isMounted { -+ -+ sourceFile, err := m.getContainerDiskPath(vmi, &volume, volume.Name) -+ if err != nil { -+ return nil, fmt.Errorf("failed to find a sourceFile in containerDisk %v: %v", volume.Name, err) -+ } -+ -+ log.DefaultLogger().Object(vmi).Infof("Bind mounting container disk at %s to %s", sourceFile, target) -+ opts := []string{ -+ "bind", "ro", "uid=107", "gid=107", -+ } -+ err = virt_chroot.MountChrootWithOptions(sourceFile, target, opts...) -+ if err != nil { -+ return nil, fmt.Errorf("failed to bindmount containerDisk %v. err: %w", volume.Name, err) -+ } -+ } -+ -+ imageInfo, err := isolation.GetImageInfo( -+ containerdisk.GetHotplugContainerDiskTargetPathFromLauncherView(volume.Name), -+ vmiRes, -+ m.clusterConfig.GetDiskVerification(), -+ ) -+ if err != nil { -+ return nil, fmt.Errorf("failed to get image info: %v", err) -+ } -+ if err := containerdisk.VerifyImage(imageInfo); err != nil { -+ return nil, fmt.Errorf("invalid image in containerDisk %v: %v", volume.Name, err) -+ } -+ disksInfo[volume.Name] = imageInfo -+ } -+ } -+ -+ return disksInfo, nil -+} -+ -+func (m *hotplugMounter) Umount(vmi *v1.VirtualMachineInstance) error { -+ record, err := m.getMountTargetRecord(vmi) -+ if err != nil { -+ return err -+ } else if record == nil { -+ // no entries to unmount -+ -+ log.DefaultLogger().Object(vmi).Infof("No container disk mount entries found to unmount") -+ return nil -+ } -+ for _, r := range record.MountTargetEntries { -+ name, err := extractNameFromSocket(r.SocketFile) -+ if err != nil { -+ return err -+ } -+ needUmount := true -+ for _, v := range vmi.Status.VolumeStatus { -+ if v.Name == name { -+ needUmount = false -+ } -+ } -+ if needUmount { -+ file, err := safepath.NewFileNoFollow(r.TargetFile) -+ if err != nil { -+ if errors.Is(err, os.ErrNotExist) { -+ continue -+ } -+ return fmt.Errorf(failedCheckMountPointFmt, r.TargetFile, err) -+ } -+ _ = file.Close() -+ // #nosec No risk for attacket injection. Parameters are predefined strings -+ out, err := virt_chroot.UmountChroot(file.Path()).CombinedOutput() -+ if err != nil { -+ return fmt.Errorf(failedUnmountFmt, file, string(out), err) -+ } -+ } -+ } -+ return nil -+} -+ -+func extractNameFromSocket(socketFile string) (string, error) { -+ base := filepath.Base(socketFile) -+ if strings.HasPrefix(base, "hotplug-container-disk-") && strings.HasSuffix(base, ".sock") { -+ name := strings.TrimPrefix(base, "hotplug-container-disk-") -+ name = strings.TrimSuffix(name, ".sock") -+ return name, nil -+ } -+ return "", fmt.Errorf("name not found in path") -+} -+ -+func (m *hotplugMounter) UmountAll(vmi *v1.VirtualMachineInstance) error { -+ if vmi.UID == "" { -+ return nil -+ } -+ -+ record, err := m.getMountTargetRecord(vmi) -+ if err != nil { -+ return err -+ } else if record == nil { -+ // no entries to unmount -+ -+ log.DefaultLogger().Object(vmi).Infof("No container disk mount entries found to unmount") -+ return nil -+ } -+ -+ log.DefaultLogger().Object(vmi).Infof("Found container disk mount entries") -+ for _, entry := range record.MountTargetEntries { -+ log.DefaultLogger().Object(vmi).Infof("Looking to see if containerdisk is mounted at path %s", entry.TargetFile) -+ file, err := safepath.NewFileNoFollow(entry.TargetFile) -+ if err != nil { -+ if errors.Is(err, os.ErrNotExist) { -+ continue -+ } -+ return fmt.Errorf(failedCheckMountPointFmt, entry.TargetFile, err) -+ } -+ _ = file.Close() -+ if mounted, err := isolation.IsMounted(file.Path()); err != nil { -+ return fmt.Errorf(failedCheckMountPointFmt, file, err) -+ } else if mounted { -+ log.DefaultLogger().Object(vmi).Infof("unmounting container disk at path %s", file) -+ // #nosec No risk for attacket injection. Parameters are predefined strings -+ out, err := virt_chroot.UmountChroot(file.Path()).CombinedOutput() -+ if err != nil { -+ return fmt.Errorf(failedUnmountFmt, file, string(out), err) -+ } -+ } -+ } -+ err = m.deleteMountTargetRecord(vmi) -+ if err != nil { -+ return err -+ } -+ -+ return nil -+} -+ -+func (m *hotplugMounter) ContainerDisksReady(vmi *v1.VirtualMachineInstance, notInitializedSince time.Time) (bool, error) { -+ for _, volume := range vmi.Spec.Volumes { -+ if volume.ContainerDisk != nil && volume.ContainerDisk.Hotpluggable { -+ _, err := m.hotplugPathGetter(vmi, volume.Name) -+ if err != nil { -+ log.DefaultLogger().Object(vmi).Reason(err).Infof("containerdisk %s not yet ready", volume.Name) -+ if time.Now().After(notInitializedSince.Add(m.suppressWarningTimeout)) { -+ return false, fmt.Errorf("containerdisk %s still not ready after one minute", volume.Name) -+ } -+ return false, nil -+ } -+ } -+ } -+ -+ log.DefaultLogger().Object(vmi).V(4).Info("all containerdisks are ready") -+ return true, nil -+} -+ -+func (m *hotplugMounter) getContainerDiskPath(vmi *v1.VirtualMachineInstance, volume *v1.Volume, volumeName string) (*safepath.Path, error) { -+ sock, err := m.hotplugPathGetter(vmi, volumeName) -+ if err != nil { -+ return nil, ErrDiskContainerGone -+ } -+ -+ res, err := m.podIsolationDetector.DetectForSocket(vmi, sock) -+ if err != nil { -+ return nil, fmt.Errorf("failed to detect socket for containerDisk %v: %v", volume.Name, err) -+ } -+ -+ mountPoint, err := isolation.ParentPathForRootMount(m.nodeIsolationResult, res) -+ if err != nil { -+ return nil, fmt.Errorf("failed to detect root mount point of containerDisk %v on the node: %v", volume.Name, err) -+ } -+ -+ return containerdisk.GetImage(mountPoint, volume.ContainerDisk.Path) -+} -+ -+func (m *hotplugMounter) ComputeChecksums(vmi *v1.VirtualMachineInstance) (*DiskChecksums, error) { -+ -+ diskChecksums := &DiskChecksums{ -+ ContainerDiskChecksums: map[string]uint32{}, -+ } -+ -+ for _, volume := range vmi.Spec.Volumes { -+ if volume.VolumeSource.ContainerDisk == nil || !volume.VolumeSource.ContainerDisk.Hotpluggable { -+ continue -+ } -+ -+ path, err := m.getContainerDiskPath(vmi, &volume, volume.Name) -+ if err != nil { -+ return nil, err -+ } -+ -+ checksum, err := getDigest(path) -+ if err != nil { -+ return nil, err -+ } -+ -+ diskChecksums.ContainerDiskChecksums[volume.Name] = checksum -+ } -+ -+ return diskChecksums, nil -+} -+ -+func (m *hotplugMounter) findVirtlauncherUID(vmi *v1.VirtualMachineInstance) (uid types.UID) { -+ cnt := 0 -+ for podUID := range vmi.Status.ActivePods { -+ _, err := m.hotplugManager.GetHotplugTargetPodPathOnHost(podUID) -+ if err == nil { -+ uid = podUID -+ cnt++ -+ } -+ } -+ if cnt == 1 { -+ return -+ } -+ // Either no pods, or multiple pods, skip. -+ return types.UID("") -+} -diff --git a/pkg/virt-handler/container-disk/mount.go b/pkg/virt-handler/container-disk/mount.go -index 953c20f3af..d99bec3a43 100644 ---- a/pkg/virt-handler/container-disk/mount.go -+++ b/pkg/virt-handler/container-disk/mount.go -@@ -54,6 +54,8 @@ type mounter struct { - kernelBootSocketPathGetter containerdisk.KernelBootSocketPathGetter - clusterConfig *virtconfig.ClusterConfig - nodeIsolationResult isolation.IsolationResult -+ -+ hotplugPathGetter containerdisk.HotplugSocketPathGetter - } - - type Mounter interface { -@@ -98,6 +100,8 @@ func NewMounter(isoDetector isolation.PodIsolationDetector, mountStateDir string - kernelBootSocketPathGetter: containerdisk.NewKernelBootSocketPathGetter(""), - clusterConfig: clusterConfig, - nodeIsolationResult: isolation.NodeIsolationResult(), -+ -+ hotplugPathGetter: containerdisk.NewHotplugSocketPathGetter(""), - } - } - -@@ -254,7 +258,7 @@ func (m *mounter) MountAndVerify(vmi *v1.VirtualMachineInstance) (map[string]*co - disksInfo := map[string]*containerdisk.DiskInfo{} - - for i, volume := range vmi.Spec.Volumes { -- if volume.ContainerDisk != nil { -+ if volume.ContainerDisk != nil && !volume.ContainerDisk.Hotpluggable { - diskTargetDir, err := containerdisk.GetDiskTargetDirFromHostView(vmi) - if err != nil { - return nil, err -@@ -296,7 +300,7 @@ func (m *mounter) MountAndVerify(vmi *v1.VirtualMachineInstance) (map[string]*co - } - - for i, volume := range vmi.Spec.Volumes { -- if volume.ContainerDisk != nil { -+ if volume.ContainerDisk != nil && !volume.ContainerDisk.Hotpluggable { - diskTargetDir, err := containerdisk.GetDiskTargetDirFromHostView(vmi) - if err != nil { - return nil, err -@@ -394,7 +398,7 @@ func (m *mounter) Unmount(vmi *v1.VirtualMachineInstance) error { - - func (m *mounter) ContainerDisksReady(vmi *v1.VirtualMachineInstance, notInitializedSince time.Time) (bool, error) { - for i, volume := range vmi.Spec.Volumes { -- if volume.ContainerDisk != nil { -+ if volume.ContainerDisk != nil && !volume.ContainerDisk.Hotpluggable { - _, err := m.socketPathGetter(vmi, i) - if err != nil { - log.DefaultLogger().Object(vmi).Reason(err).Infof("containerdisk %s not yet ready", volume.Name) -@@ -706,7 +710,7 @@ func (m *mounter) ComputeChecksums(vmi *v1.VirtualMachineInstance) (*DiskChecksu - - // compute for containerdisks - for i, volume := range vmi.Spec.Volumes { -- if volume.VolumeSource.ContainerDisk == nil { -+ if volume.VolumeSource.ContainerDisk == nil || volume.VolumeSource.ContainerDisk.Hotpluggable { - continue - } - -diff --git a/pkg/virt-handler/hotplug-disk/mount.go b/pkg/virt-handler/hotplug-disk/mount.go -index 971c8d55fc..034c3d8526 100644 ---- a/pkg/virt-handler/hotplug-disk/mount.go -+++ b/pkg/virt-handler/hotplug-disk/mount.go -@@ -343,7 +343,7 @@ func (m *volumeMounter) mountFromPod(vmi *v1.VirtualMachineInstance, sourceUID t - return err - } - for _, volumeStatus := range vmi.Status.VolumeStatus { -- if volumeStatus.HotplugVolume == nil { -+ if volumeStatus.HotplugVolume == nil || volumeStatus.ContainerDiskVolume != nil { - // Skip non hotplug volumes - continue - } -@@ -649,7 +649,7 @@ func (m *volumeMounter) Unmount(vmi *v1.VirtualMachineInstance, cgroupManager cg - return err - } - for _, volumeStatus := range vmi.Status.VolumeStatus { -- if volumeStatus.HotplugVolume == nil { -+ if volumeStatus.HotplugVolume == nil || volumeStatus.ContainerDiskVolume != nil { - continue - } - var path *safepath.Path -diff --git a/pkg/virt-handler/isolation/detector.go b/pkg/virt-handler/isolation/detector.go -index f83f96ead4..5e38c6cedd 100644 ---- a/pkg/virt-handler/isolation/detector.go -+++ b/pkg/virt-handler/isolation/detector.go -@@ -24,6 +24,8 @@ package isolation - import ( - "fmt" - "net" -+ "os" -+ "path" - "runtime" - "syscall" - "time" -@@ -207,12 +209,45 @@ func setProcessMemoryLockRLimit(pid int, size int64) error { - return nil - } - -+type deferFunc func() -+ -+func (s *socketBasedIsolationDetector) socketHack(socket string) (sock net.Conn, deferFunc deferFunc, err error) { -+ fn := func() {} -+ if len([]rune(socket)) <= 108 { -+ sock, err = net.DialTimeout("unix", socket, time.Duration(isolationDialTimeout)*time.Second) -+ fn = func() { -+ if err == nil { -+ sock.Close() -+ } -+ } -+ return sock, fn, err -+ } -+ base := path.Base(socket) -+ newPath := fmt.Sprintf("/tmp/%s", base) -+ if err = os.Symlink(socket, newPath); err != nil { -+ return nil, fn, err -+ } -+ sock, err = net.DialTimeout("unix", newPath, time.Duration(isolationDialTimeout)*time.Second) -+ fn = func() { -+ if err == nil { -+ sock.Close() -+ } -+ os.Remove(newPath) -+ } -+ return sock, fn, err -+} -+ - func (s *socketBasedIsolationDetector) getPid(socket string) (int, error) { -- sock, err := net.DialTimeout("unix", socket, time.Duration(isolationDialTimeout)*time.Second) -+ sock, defFn, err := s.socketHack(socket) -+ defer defFn() - if err != nil { - return -1, err - } -- defer sock.Close() -+ //sock, err := net.DialTimeout("unix", socket, time.Duration(isolationDialTimeout)*time.Second) -+ //if err != nil { -+ // return -1, err -+ //} -+ //defer sock.Close() - - ufile, err := sock.(*net.UnixConn).File() - if err != nil { -diff --git a/pkg/virt-handler/virt-chroot/virt-chroot.go b/pkg/virt-handler/virt-chroot/virt-chroot.go -index 4160212b7b..580b788acc 100644 ---- a/pkg/virt-handler/virt-chroot/virt-chroot.go -+++ b/pkg/virt-handler/virt-chroot/virt-chroot.go -@@ -20,7 +20,10 @@ - package virt_chroot - - import ( -+ "bytes" -+ "fmt" - "os/exec" -+ "slices" - "strings" - - "kubevirt.io/kubevirt/pkg/safepath" -@@ -48,6 +51,49 @@ func MountChroot(sourcePath, targetPath *safepath.Path, ro bool) *exec.Cmd { - return UnsafeMountChroot(trimProcPrefix(sourcePath), trimProcPrefix(targetPath), ro) - } - -+func MountChrootWithOptions(sourcePath, targetPath *safepath.Path, mountOptions ...string) error { -+ args := append(getBaseArgs(), "mount") -+ remountArgs := slices.Clone(args) -+ -+ mountOptions = slices.DeleteFunc(mountOptions, func(s string) bool { -+ return s == "remount" -+ }) -+ if len(mountOptions) > 0 { -+ opts := strings.Join(mountOptions, ",") -+ remountOpts := "remount," + opts -+ args = append(args, "-o", opts) -+ remountArgs = append(remountArgs, "-o", remountOpts) -+ } -+ -+ sp := trimProcPrefix(sourcePath) -+ tp := trimProcPrefix(targetPath) -+ args = append(args, sp, tp) -+ remountArgs = append(remountArgs, sp, tp) -+ -+ stdout := new(bytes.Buffer) -+ stderr := new(bytes.Buffer) -+ -+ cmd := exec.Command(binaryPath, args...) -+ cmd.Stdout = stdout -+ cmd.Stderr = stderr -+ err := cmd.Run() -+ if err != nil { -+ return fmt.Errorf("mount failed: %w, stdout: %s, stderr: %s", err, stdout.String(), stderr.String()) -+ } -+ -+ stdout = new(bytes.Buffer) -+ stderr = new(bytes.Buffer) -+ -+ remountCmd := exec.Command(binaryPath, remountArgs...) -+ cmd.Stdout = stdout -+ cmd.Stderr = stderr -+ err = remountCmd.Run() -+ if err != nil { -+ return fmt.Errorf("mount failed: %w, stdout: %s, stderr: %s", err, stdout.String(), stderr.String()) -+ } -+ return nil -+} -+ - // Deprecated: UnsafeMountChroot is used to connect to code which needs to be refactored - // to handle mounts securely. - func UnsafeMountChroot(sourcePath, targetPath string, ro bool) *exec.Cmd { -diff --git a/pkg/virt-handler/vm.go b/pkg/virt-handler/vm.go -index 24352cf6e9..f1de8d1149 100644 ---- a/pkg/virt-handler/vm.go -+++ b/pkg/virt-handler/vm.go -@@ -25,6 +25,7 @@ import ( - goerror "errors" - "fmt" - "io" -+ "maps" - "net" - "os" - "path/filepath" -@@ -247,6 +248,13 @@ func NewController( - vmiExpectations: controller.NewUIDTrackingControllerExpectations(controller.NewControllerExpectations()), - sriovHotplugExecutorPool: executor.NewRateLimitedExecutorPool(executor.NewExponentialLimitedBackoffCreator()), - ioErrorRetryManager: NewFailRetryManager("io-error-retry", 10*time.Second, 3*time.Minute, 30*time.Second), -+ -+ hotplugContainerDiskMounter: container_disk.NewHotplugMounter( -+ podIsolationDetector, -+ filepath.Join(virtPrivateDir, "hotplug-container-disk-mount-state"), -+ clusterConfig, -+ hotplugdisk.NewHotplugDiskManager(kubeletPodsDir), -+ ), - } - - _, err := vmiSourceInformer.AddEventHandler(cache.ResourceEventHandlerFuncs{ -@@ -342,6 +350,8 @@ type VirtualMachineController struct { - hostCpuModel string - vmiExpectations *controller.UIDTrackingControllerExpectations - ioErrorRetryManager *FailRetryManager -+ -+ hotplugContainerDiskMounter container_disk.HotplugMounter - } - - type virtLauncherCriticalSecurebootError struct { -@@ -876,7 +886,15 @@ func (d *VirtualMachineController) updateHotplugVolumeStatus(vmi *v1.VirtualMach - needsRefresh := false - if volumeStatus.Target == "" { - needsRefresh = true -- mounted, err := d.hotplugVolumeMounter.IsMounted(vmi, volumeStatus.Name, volumeStatus.HotplugVolume.AttachPodUID) -+ var ( -+ mounted bool -+ err error -+ ) -+ if volumeStatus.ContainerDiskVolume != nil { -+ mounted, err = d.hotplugContainerDiskMounter.IsMounted(vmi, volumeStatus.Name) -+ } else { -+ mounted, err = d.hotplugVolumeMounter.IsMounted(vmi, volumeStatus.Name, volumeStatus.HotplugVolume.AttachPodUID) -+ } - if err != nil { - log.Log.Object(vmi).Errorf("error occurred while checking if volume is mounted: %v", err) - } -@@ -898,6 +916,7 @@ func (d *VirtualMachineController) updateHotplugVolumeStatus(vmi *v1.VirtualMach - volumeStatus.Reason = VolumeUnMountedFromPodReason - } - } -+ - } else { - // Successfully attached to VM. - volumeStatus.Phase = v1.VolumeReady -@@ -2178,6 +2197,11 @@ func (d *VirtualMachineController) processVmCleanup(vmi *v1.VirtualMachineInstan - return err - } - -+ err := d.hotplugContainerDiskMounter.UmountAll(vmi) -+ if err != nil { -+ return err -+ } -+ - // UnmountAll does the cleanup on the "best effort" basis: it is - // safe to pass a nil cgroupManager. - cgroupManager, _ := getCgroupManager(vmi) -@@ -2829,6 +2853,12 @@ func (d *VirtualMachineController) vmUpdateHelperMigrationTarget(origVMI *v1.Vir - return err - } - -+ hotplugDiskInfo, err := d.hotplugContainerDiskMounter.MountAndVerify(vmi) -+ if err != nil { -+ return err -+ } -+ maps.Copy(disksInfo, hotplugDiskInfo) -+ - // Mount hotplug disks - if attachmentPodUID := vmi.Status.MigrationState.TargetAttachmentPodUID; attachmentPodUID != types.UID("") { - cgroupManager, err := getCgroupManager(vmi) -@@ -3051,6 +3081,11 @@ func (d *VirtualMachineController) vmUpdateHelperDefault(origVMI *v1.VirtualMach - if err != nil { - return err - } -+ hotplugDiskInfo, err := d.hotplugContainerDiskMounter.MountAndVerify(vmi) -+ if err != nil { -+ return err -+ } -+ maps.Copy(disksInfo, hotplugDiskInfo) - - // Try to mount hotplug volume if there is any during startup. - if err := d.hotplugVolumeMounter.Mount(vmi, cgroupManager); err != nil { -@@ -3138,6 +3173,11 @@ func (d *VirtualMachineController) vmUpdateHelperDefault(origVMI *v1.VirtualMach - log.Log.Object(vmi).Error(err.Error()) - } - -+ hotplugDiskInfo, err := d.hotplugContainerDiskMounter.MountAndVerify(vmi) -+ if err != nil { -+ return err -+ } -+ maps.Copy(disksInfo, hotplugDiskInfo) - if err := d.hotplugVolumeMounter.Mount(vmi, cgroupManager); err != nil { - return err - } -@@ -3215,6 +3255,9 @@ func (d *VirtualMachineController) vmUpdateHelperDefault(origVMI *v1.VirtualMach - - if vmi.IsRunning() { - // Umount any disks no longer mounted -+ if err := d.hotplugContainerDiskMounter.Umount(vmi); err != nil { -+ return err -+ } - if err := d.hotplugVolumeMounter.Unmount(vmi, cgroupManager); err != nil { - return err - } -diff --git a/pkg/virt-launcher/virtwrap/converter/converter.go b/pkg/virt-launcher/virtwrap/converter/converter.go -index 3318c1c466..ce726edf12 100644 ---- a/pkg/virt-launcher/virtwrap/converter/converter.go -+++ b/pkg/virt-launcher/virtwrap/converter/converter.go -@@ -649,6 +649,9 @@ func Convert_v1_Hotplug_Volume_To_api_Disk(source *v1.Volume, disk *api.Disk, c - if source.DataVolume != nil { - return Convert_v1_Hotplug_DataVolume_To_api_Disk(source.Name, disk, c) - } -+ if source.ContainerDisk != nil { -+ return Convert_v1_Hotplug_ContainerDisk_To_api_Disk(source.Name, disk, c) -+ } - return fmt.Errorf("hotplug disk %s references an unsupported source", disk.Alias.GetName()) - } - -@@ -690,6 +693,10 @@ func GetHotplugBlockDeviceVolumePath(volumeName string) string { - return filepath.Join(string(filepath.Separator), "var", "run", "kubevirt", "hotplug-disks", volumeName) - } - -+func GetHotplugContainerDiskPath(volumeName string) string { -+ return filepath.Join(string(filepath.Separator), "var", "run", "kubevirt", "hotplug-disks", fmt.Sprintf("%s.img", volumeName)) -+} -+ - func Convert_v1_PersistentVolumeClaim_To_api_Disk(name string, disk *api.Disk, c *ConverterContext) error { - if c.IsBlockPVC[name] { - return Convert_v1_BlockVolumeSource_To_api_Disk(name, disk, c.VolumesDiscardIgnore) -@@ -768,6 +775,35 @@ func Convert_v1_Hotplug_BlockVolumeSource_To_api_Disk(volumeName string, disk *a - return nil - } - -+func Convert_v1_Hotplug_ContainerDisk_To_api_Disk(volumeName string, disk *api.Disk, c *ConverterContext) error { -+ if disk.Type == "lun" { -+ return fmt.Errorf(deviceTypeNotCompatibleFmt, disk.Alias.GetName()) -+ } -+ info := c.DisksInfo[volumeName] -+ if info == nil { -+ return fmt.Errorf("no disk info provided for volume %s", volumeName) -+ } -+ -+ disk.Type = "file" -+ disk.Driver.Type = info.Format -+ disk.Driver.ErrorPolicy = v1.DiskErrorPolicyStop -+ disk.ReadOnly = &api.ReadOnly{} -+ if !contains(c.VolumesDiscardIgnore, volumeName) { -+ disk.Driver.Discard = "unmap" -+ } -+ disk.Source.File = GetHotplugContainerDiskPath(volumeName) -+ disk.BackingStore = &api.BackingStore{ -+ Format: &api.BackingStoreFormat{}, -+ Source: &api.DiskSource{}, -+ } -+ -+ //disk.BackingStore.Format.Type = info.Format -+ //disk.BackingStore.Source.File = info.BackingFile -+ //disk.BackingStore.Type = "file" -+ -+ return nil -+} -+ - func Convert_v1_HostDisk_To_api_Disk(volumeName string, path string, disk *api.Disk) error { - disk.Type = "file" - disk.Driver.Type = "raw" -diff --git a/pkg/virt-operator/resource/apply/BUILD.bazel b/pkg/virt-operator/resource/apply/BUILD.bazel -index f6bd9bd4f1..fe6ab54f8c 100644 ---- a/pkg/virt-operator/resource/apply/BUILD.bazel -+++ b/pkg/virt-operator/resource/apply/BUILD.bazel -@@ -4,7 +4,6 @@ go_library( - name = "go_default_library", - srcs = [ - "admissionregistration.go", -- "apiservices.go", - "apps.go", - "certificates.go", - "core.go", -@@ -65,7 +64,6 @@ go_library( - "//vendor/k8s.io/client-go/tools/cache:go_default_library", - "//vendor/k8s.io/client-go/tools/record:go_default_library", - "//vendor/k8s.io/client-go/util/workqueue:go_default_library", -- "//vendor/k8s.io/kube-aggregator/pkg/apis/apiregistration/v1:go_default_library", - "//vendor/k8s.io/utils/pointer:go_default_library", - ], - ) -diff --git a/pkg/virt-operator/resource/generate/components/BUILD.bazel b/pkg/virt-operator/resource/generate/components/BUILD.bazel -index 70d2da0897..affcd3fecd 100644 ---- a/pkg/virt-operator/resource/generate/components/BUILD.bazel -+++ b/pkg/virt-operator/resource/generate/components/BUILD.bazel -@@ -3,7 +3,6 @@ load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test") - go_library( - name = "go_default_library", - srcs = [ -- "apiservices.go", - "crds.go", - "daemonsets.go", - "deployments.go", -@@ -62,7 +61,6 @@ go_library( - "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", - "//vendor/k8s.io/apimachinery/pkg/util/intstr:go_default_library", - "//vendor/k8s.io/apimachinery/pkg/util/yaml:go_default_library", -- "//vendor/k8s.io/kube-aggregator/pkg/apis/apiregistration/v1:go_default_library", - "//vendor/k8s.io/utils/pointer:go_default_library", - ], - ) -@@ -70,7 +68,6 @@ go_library( - go_test( - name = "go_default_test", - srcs = [ -- "apiservices_test.go", - "components_suite_test.go", - "crds_test.go", - "deployments_test.go", -@@ -85,7 +82,6 @@ go_test( - deps = [ - "//pkg/certificates/bootstrap:go_default_library", - "//pkg/certificates/triple/cert:go_default_library", -- "//staging/src/kubevirt.io/api/core/v1:go_default_library", - "//staging/src/kubevirt.io/client-go/testutils:go_default_library", - "//vendor/github.com/onsi/ginkgo/v2:go_default_library", - "//vendor/github.com/onsi/gomega:go_default_library", -diff --git a/pkg/virt-operator/resource/generate/components/validations_generated.go b/pkg/virt-operator/resource/generate/components/validations_generated.go -index 4913dbead0..42225780ba 100644 ---- a/pkg/virt-operator/resource/generate/components/validations_generated.go -+++ b/pkg/virt-operator/resource/generate/components/validations_generated.go -@@ -7723,6 +7723,8 @@ var CRDsValidation map[string]string = map[string]string{ - ContainerDisk references a docker image, embedding a qcow or raw disk. - More info: https://kubevirt.gitbooks.io/user-guide/registry-disk.html - properties: -+ hotpluggable: -+ type: boolean - image: - description: Image is the name of the image with the embedded - disk. -@@ -8355,6 +8357,35 @@ var CRDsValidation map[string]string = map[string]string{ - description: VolumeSource represents the source of the volume - to map to the disk. - properties: -+ containerDisk: -+ description: Represents a docker image with an embedded disk. -+ properties: -+ hotpluggable: -+ type: boolean -+ image: -+ description: Image is the name of the image with the embedded -+ disk. -+ type: string -+ imagePullPolicy: -+ description: |- -+ Image pull policy. -+ One of Always, Never, IfNotPresent. -+ Defaults to Always if :latest tag is specified, or IfNotPresent otherwise. -+ Cannot be updated. -+ More info: https://kubernetes.io/docs/concepts/containers/images#updating-images -+ type: string -+ imagePullSecret: -+ description: ImagePullSecret is the name of the Docker -+ registry secret required to pull the image. The secret -+ must already exist. -+ type: string -+ path: -+ description: Path defines the path to disk file in the -+ container -+ type: string -+ required: -+ - image -+ type: object - dataVolume: - description: |- - DataVolume represents the dynamic creation a PVC for this volume as well as -@@ -12768,6 +12799,8 @@ var CRDsValidation map[string]string = map[string]string{ - ContainerDisk references a docker image, embedding a qcow or raw disk. - More info: https://kubevirt.gitbooks.io/user-guide/registry-disk.html - properties: -+ hotpluggable: -+ type: boolean - image: - description: Image is the name of the image with the embedded - disk. -@@ -18328,6 +18361,8 @@ var CRDsValidation map[string]string = map[string]string{ - ContainerDisk references a docker image, embedding a qcow or raw disk. - More info: https://kubevirt.gitbooks.io/user-guide/registry-disk.html - properties: -+ hotpluggable: -+ type: boolean - image: - description: Image is the name of the image with the embedded - disk. -@@ -22835,6 +22870,8 @@ var CRDsValidation map[string]string = map[string]string{ - ContainerDisk references a docker image, embedding a qcow or raw disk. - More info: https://kubevirt.gitbooks.io/user-guide/registry-disk.html - properties: -+ hotpluggable: -+ type: boolean - image: - description: Image is the name of the image with - the embedded disk. -@@ -28015,6 +28052,8 @@ var CRDsValidation map[string]string = map[string]string{ - ContainerDisk references a docker image, embedding a qcow or raw disk. - More info: https://kubevirt.gitbooks.io/user-guide/registry-disk.html - properties: -+ hotpluggable: -+ type: boolean - image: - description: Image is the name of the image - with the embedded disk. -@@ -28673,6 +28712,36 @@ var CRDsValidation map[string]string = map[string]string{ - description: VolumeSource represents the source of - the volume to map to the disk. - properties: -+ containerDisk: -+ description: Represents a docker image with an -+ embedded disk. -+ properties: -+ hotpluggable: -+ type: boolean -+ image: -+ description: Image is the name of the image -+ with the embedded disk. -+ type: string -+ imagePullPolicy: -+ description: |- -+ Image pull policy. -+ One of Always, Never, IfNotPresent. -+ Defaults to Always if :latest tag is specified, or IfNotPresent otherwise. -+ Cannot be updated. -+ More info: https://kubernetes.io/docs/concepts/containers/images#updating-images -+ type: string -+ imagePullSecret: -+ description: ImagePullSecret is the name of -+ the Docker registry secret required to pull -+ the image. The secret must already exist. -+ type: string -+ path: -+ description: Path defines the path to disk -+ file in the container -+ type: string -+ required: -+ - image -+ type: object - dataVolume: - description: |- - DataVolume represents the dynamic creation a PVC for this volume as well as -diff --git a/pkg/virt-operator/resource/generate/install/generated_mock_strategy.go b/pkg/virt-operator/resource/generate/install/generated_mock_strategy.go -index 5f1e9a3121..1fa1416af0 100644 ---- a/pkg/virt-operator/resource/generate/install/generated_mock_strategy.go -+++ b/pkg/virt-operator/resource/generate/install/generated_mock_strategy.go -@@ -241,16 +241,6 @@ func (_mr *_MockStrategyInterfaceRecorder) MutatingWebhookConfigurations() *gomo - return _mr.mock.ctrl.RecordCall(_mr.mock, "MutatingWebhookConfigurations") - } - --func (_m *MockStrategyInterface) APIServices() []*v18.APIService { -- ret := _m.ctrl.Call(_m, "APIServices") -- ret0, _ := ret[0].([]*v18.APIService) -- return ret0 --} -- --func (_mr *_MockStrategyInterfaceRecorder) APIServices() *gomock.Call { -- return _mr.mock.ctrl.RecordCall(_mr.mock, "APIServices") --} -- - func (_m *MockStrategyInterface) CertificateSecrets() []*v14.Secret { - ret := _m.ctrl.Call(_m, "CertificateSecrets") - ret0, _ := ret[0].([]*v14.Secret) -diff --git a/pkg/virt-operator/resource/generate/rbac/exportproxy.go b/pkg/virt-operator/resource/generate/rbac/exportproxy.go -index ebc9f2adbd..a0dc0586b4 100644 ---- a/pkg/virt-operator/resource/generate/rbac/exportproxy.go -+++ b/pkg/virt-operator/resource/generate/rbac/exportproxy.go -@@ -23,6 +23,7 @@ import ( - rbacv1 "k8s.io/api/rbac/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/runtime" -+ - "kubevirt.io/kubevirt/pkg/virt-operator/resource/generate/components" - - virtv1 "kubevirt.io/api/core/v1" -diff --git a/staging/src/kubevirt.io/api/apitesting/testdata/HEAD/kubevirt.io.v1.VirtualMachine.json b/staging/src/kubevirt.io/api/apitesting/testdata/HEAD/kubevirt.io.v1.VirtualMachine.json -index b651173636..3453dfb0da 100644 ---- a/staging/src/kubevirt.io/api/apitesting/testdata/HEAD/kubevirt.io.v1.VirtualMachine.json -+++ b/staging/src/kubevirt.io/api/apitesting/testdata/HEAD/kubevirt.io.v1.VirtualMachine.json -@@ -754,7 +754,8 @@ - "image": "imageValue", - "imagePullSecret": "imagePullSecretValue", - "path": "pathValue", -- "imagePullPolicy": "imagePullPolicyValue" -+ "imagePullPolicy": "imagePullPolicyValue", -+ "hotpluggable": true - }, - "ephemeral": { - "persistentVolumeClaim": { -@@ -1209,6 +1210,13 @@ - "dataVolume": { - "name": "nameValue", - "hotpluggable": true -+ }, -+ "containerDisk": { -+ "image": "imageValue", -+ "imagePullSecret": "imagePullSecretValue", -+ "path": "pathValue", -+ "imagePullPolicy": "imagePullPolicyValue", -+ "hotpluggable": true - } - }, - "dryRun": [ -diff --git a/staging/src/kubevirt.io/api/apitesting/testdata/HEAD/kubevirt.io.v1.VirtualMachine.yaml b/staging/src/kubevirt.io/api/apitesting/testdata/HEAD/kubevirt.io.v1.VirtualMachine.yaml -index 53dfdacc3b..8b23193158 100644 ---- a/staging/src/kubevirt.io/api/apitesting/testdata/HEAD/kubevirt.io.v1.VirtualMachine.yaml -+++ b/staging/src/kubevirt.io/api/apitesting/testdata/HEAD/kubevirt.io.v1.VirtualMachine.yaml -@@ -719,6 +719,7 @@ spec: - optional: true - volumeLabel: volumeLabelValue - containerDisk: -+ hotpluggable: true - image: imageValue - imagePullPolicy: imagePullPolicyValue - imagePullSecret: imagePullSecretValue -@@ -838,6 +839,12 @@ status: - - dryRunValue - name: nameValue - volumeSource: -+ containerDisk: -+ hotpluggable: true -+ image: imageValue -+ imagePullPolicy: imagePullPolicyValue -+ imagePullSecret: imagePullSecretValue -+ path: pathValue - dataVolume: - hotpluggable: true - name: nameValue -diff --git a/staging/src/kubevirt.io/api/apitesting/testdata/HEAD/kubevirt.io.v1.VirtualMachineInstance.json b/staging/src/kubevirt.io/api/apitesting/testdata/HEAD/kubevirt.io.v1.VirtualMachineInstance.json -index 3be904512c..f595798e89 100644 ---- a/staging/src/kubevirt.io/api/apitesting/testdata/HEAD/kubevirt.io.v1.VirtualMachineInstance.json -+++ b/staging/src/kubevirt.io/api/apitesting/testdata/HEAD/kubevirt.io.v1.VirtualMachineInstance.json -@@ -694,7 +694,8 @@ - "image": "imageValue", - "imagePullSecret": "imagePullSecretValue", - "path": "pathValue", -- "imagePullPolicy": "imagePullPolicyValue" -+ "imagePullPolicy": "imagePullPolicyValue", -+ "hotpluggable": true - }, - "ephemeral": { - "persistentVolumeClaim": { -diff --git a/staging/src/kubevirt.io/api/apitesting/testdata/HEAD/kubevirt.io.v1.VirtualMachineInstance.yaml b/staging/src/kubevirt.io/api/apitesting/testdata/HEAD/kubevirt.io.v1.VirtualMachineInstance.yaml -index 6fd2ab6523..b6457ec94d 100644 ---- a/staging/src/kubevirt.io/api/apitesting/testdata/HEAD/kubevirt.io.v1.VirtualMachineInstance.yaml -+++ b/staging/src/kubevirt.io/api/apitesting/testdata/HEAD/kubevirt.io.v1.VirtualMachineInstance.yaml -@@ -524,6 +524,7 @@ spec: - optional: true - volumeLabel: volumeLabelValue - containerDisk: -+ hotpluggable: true - image: imageValue - imagePullPolicy: imagePullPolicyValue - imagePullSecret: imagePullSecretValue -diff --git a/staging/src/kubevirt.io/api/core/v1/BUILD.bazel b/staging/src/kubevirt.io/api/core/v1/BUILD.bazel -index f8615293a3..0c6c166985 100644 ---- a/staging/src/kubevirt.io/api/core/v1/BUILD.bazel -+++ b/staging/src/kubevirt.io/api/core/v1/BUILD.bazel -@@ -28,7 +28,6 @@ go_library( - "//vendor/k8s.io/apimachinery/pkg/runtime/schema:go_default_library", - "//vendor/k8s.io/apimachinery/pkg/types:go_default_library", - "//vendor/k8s.io/utils/net:go_default_library", -- "//vendor/k8s.io/utils/pointer:go_default_library", - "//vendor/kubevirt.io/containerized-data-importer-api/pkg/apis/core/v1beta1:go_default_library", - ], - ) -diff --git a/staging/src/kubevirt.io/api/core/v1/deepcopy_generated.go b/staging/src/kubevirt.io/api/core/v1/deepcopy_generated.go -index abd5a495d6..7372b22a9a 100644 ---- a/staging/src/kubevirt.io/api/core/v1/deepcopy_generated.go -+++ b/staging/src/kubevirt.io/api/core/v1/deepcopy_generated.go -@@ -1948,6 +1948,11 @@ func (in *HotplugVolumeSource) DeepCopyInto(out *HotplugVolumeSource) { - *out = new(DataVolumeSource) - **out = **in - } -+ if in.ContainerDisk != nil { -+ in, out := &in.ContainerDisk, &out.ContainerDisk -+ *out = new(ContainerDiskSource) -+ **out = **in -+ } - return - } - -diff --git a/staging/src/kubevirt.io/api/core/v1/schema.go b/staging/src/kubevirt.io/api/core/v1/schema.go -index 29aa3932d3..302ed9ffde 100644 ---- a/staging/src/kubevirt.io/api/core/v1/schema.go -+++ b/staging/src/kubevirt.io/api/core/v1/schema.go -@@ -854,6 +854,8 @@ type HotplugVolumeSource struct { - // the process of populating that PVC with a disk image. - // +optional - DataVolume *DataVolumeSource `json:"dataVolume,omitempty"` -+ -+ ContainerDisk *ContainerDiskSource `json:"containerDisk,omitempty"` - } - - type DataVolumeSource struct { -@@ -911,6 +913,8 @@ type ContainerDiskSource struct { - // More info: https://kubernetes.io/docs/concepts/containers/images#updating-images - // +optional - ImagePullPolicy v1.PullPolicy `json:"imagePullPolicy,omitempty"` -+ -+ Hotpluggable bool `json:"hotpluggable,omitempty"` - } - - // Exactly one of its members must be set. -diff --git a/staging/src/kubevirt.io/client-go/api/openapi_generated.go b/staging/src/kubevirt.io/client-go/api/openapi_generated.go -index cc2d743492..b982b1620c 100644 ---- a/staging/src/kubevirt.io/client-go/api/openapi_generated.go -+++ b/staging/src/kubevirt.io/client-go/api/openapi_generated.go -@@ -17772,6 +17772,12 @@ func schema_kubevirtio_api_core_v1_ContainerDiskSource(ref common.ReferenceCallb - Enum: []interface{}{"Always", "IfNotPresent", "Never"}, - }, - }, -+ "hotpluggable": { -+ SchemaProps: spec.SchemaProps{ -+ Type: []string{"boolean"}, -+ Format: "", -+ }, -+ }, - }, - Required: []string{"image"}, - }, -@@ -19645,11 +19651,16 @@ func schema_kubevirtio_api_core_v1_HotplugVolumeSource(ref common.ReferenceCallb - Ref: ref("kubevirt.io/api/core/v1.DataVolumeSource"), - }, - }, -+ "containerDisk": { -+ SchemaProps: spec.SchemaProps{ -+ Ref: ref("kubevirt.io/api/core/v1.ContainerDiskSource"), -+ }, -+ }, - }, - }, - }, - Dependencies: []string{ -- "kubevirt.io/api/core/v1.DataVolumeSource", "kubevirt.io/api/core/v1.PersistentVolumeClaimVolumeSource"}, -+ "kubevirt.io/api/core/v1.ContainerDiskSource", "kubevirt.io/api/core/v1.DataVolumeSource", "kubevirt.io/api/core/v1.PersistentVolumeClaimVolumeSource"}, - } - } - From 7c0e6c0f2c9557f4a0dc2ff281fa7a39b8a9652f Mon Sep 17 00:00:00 2001 From: yaroslavborbat Date: Tue, 10 Dec 2024 10:30:46 +0300 Subject: [PATCH 03/11] add Signed-off-by: yaroslavborbat --- .../patches/028-hotplug-container-disk.patch | 1867 +++++++++++++++++ 1 file changed, 1867 insertions(+) create mode 100644 images/virt-artifact/patches/028-hotplug-container-disk.patch diff --git a/images/virt-artifact/patches/028-hotplug-container-disk.patch b/images/virt-artifact/patches/028-hotplug-container-disk.patch new file mode 100644 index 000000000..852b8d687 --- /dev/null +++ b/images/virt-artifact/patches/028-hotplug-container-disk.patch @@ -0,0 +1,1867 @@ +diff --git a/api/openapi-spec/swagger.json b/api/openapi-spec/swagger.json +index c4822a0448..d6bb534249 100644 +--- a/api/openapi-spec/swagger.json ++++ b/api/openapi-spec/swagger.json +@@ -12951,6 +12951,9 @@ + "image" + ], + "properties": { ++ "hotpluggable": { ++ "type": "boolean" ++ }, + "image": { + "description": "Image is the name of the image with the embedded disk.", + "type": "string", +@@ -13973,6 +13976,9 @@ + "description": "HotplugVolumeSource Represents the source of a volume to mount which are capable of being hotplugged on a live running VMI. Only one of its members may be specified.", + "type": "object", + "properties": { ++ "containerDisk": { ++ "$ref": "#/definitions/v1.ContainerDiskSource" ++ }, + "dataVolume": { + "description": "DataVolume represents the dynamic creation a PVC for this volume as well as the process of populating that PVC with a disk image.", + "$ref": "#/definitions/v1.DataVolumeSource" +diff --git a/cmd/hp-container-disk/main.go b/cmd/hp-container-disk/main.go +new file mode 100644 +index 0000000000..e4f734a516 +--- /dev/null ++++ b/cmd/hp-container-disk/main.go +@@ -0,0 +1,16 @@ ++package main ++ ++import klog "kubevirt.io/client-go/log" ++ ++func main() { ++ klog.InitializeLogging("virt-hp-container-disk") ++} ++ ++type Config struct { ++ DstDir string `json:"dstDir"` ++ Images []Image `json:"images"` ++} ++ ++type Image struct { ++ Name string `json:"name"` ++} +diff --git a/cmd/virt-chroot/main.go b/cmd/virt-chroot/main.go +index e28daa07c7..7a69b7451b 100644 +--- a/cmd/virt-chroot/main.go ++++ b/cmd/virt-chroot/main.go +@@ -20,6 +20,7 @@ var ( + cpuTime uint64 + memoryBytes uint64 + targetUser string ++ targetUserID int + ) + + func init() { +@@ -51,7 +52,12 @@ func main() { + + // Looking up users needs resources, let's do it before we set rlimits. + var u *user.User +- if targetUser != "" { ++ if targetUserID >= 0 { ++ _, _, errno := syscall.Syscall(syscall.SYS_SETUID, uintptr(targetUserID), 0, 0) ++ if errno != 0 { ++ return fmt.Errorf("failed to switch to user: %d. errno: %d", targetUserID, errno) ++ } ++ } else if targetUser != "" { + var err error + u, err = user.Lookup(targetUser) + if err != nil { +@@ -116,6 +122,7 @@ func main() { + rootCmd.PersistentFlags().Uint64Var(&memoryBytes, "memory", 0, "memory in bytes for the process") + rootCmd.PersistentFlags().StringVar(&mntNamespace, "mount", "", "mount namespace to use") + rootCmd.PersistentFlags().StringVar(&targetUser, "user", "", "switch to this targetUser to e.g. drop privileges") ++ rootCmd.PersistentFlags().IntVar(&targetUserID, "userid", -1, "switch to this targetUser to e.g. drop privileges") + + execCmd := &cobra.Command{ + Use: "exec", +@@ -136,16 +143,39 @@ func main() { + Args: cobra.MinimumNArgs(2), + RunE: func(cmd *cobra.Command, args []string) error { + var mntOpts uint = 0 ++ var dataOpts []string + + fsType := cmd.Flag("type").Value.String() + mntOptions := cmd.Flag("options").Value.String() ++ var ( ++ uid = -1 ++ gid = -1 ++ ) + for _, opt := range strings.Split(mntOptions, ",") { + opt = strings.TrimSpace(opt) +- switch opt { +- case "ro": ++ switch { ++ case opt == "ro": + mntOpts = mntOpts | syscall.MS_RDONLY +- case "bind": ++ case opt == "bind": + mntOpts = mntOpts | syscall.MS_BIND ++ case opt == "remount": ++ mntOpts = mntOpts | syscall.MS_REMOUNT ++ case strings.HasPrefix(opt, "uid="): ++ uidS := strings.TrimPrefix(opt, "uid=") ++ uidI, err := strconv.Atoi(uidS) ++ if err != nil { ++ return fmt.Errorf("failed to parse uid: %w", err) ++ } ++ uid = uidI ++ dataOpts = append(dataOpts, opt) ++ case strings.HasPrefix(opt, "gid="): ++ gidS := strings.TrimPrefix(opt, "gid=") ++ gidI, err := strconv.Atoi(gidS) ++ if err != nil { ++ return fmt.Errorf("failed to parse gid: %w", err) ++ } ++ gid = gidI ++ dataOpts = append(dataOpts, opt) + default: + return fmt.Errorf("mount option %s is not supported", opt) + } +@@ -168,8 +198,17 @@ func main() { + return fmt.Errorf("mount target invalid: %v", err) + } + defer targetFile.Close() +- +- return syscall.Mount(sourceFile.SafePath(), targetFile.SafePath(), fsType, uintptr(mntOpts), "") ++ if uid >= 0 && gid >= 0 { ++ err = os.Chown(targetFile.SafePath(), uid, gid) ++ if err != nil { ++ return fmt.Errorf("chown target failed: %w", err) ++ } ++ } ++ var data string ++ if len(dataOpts) > 0 { ++ data = strings.Join(dataOpts, ",") ++ } ++ return syscall.Mount(sourceFile.SafePath(), targetFile.SafePath(), fsType, uintptr(mntOpts), data) + }, + } + mntCmd.Flags().StringP("options", "o", "", "comma separated list of mount options") +diff --git a/manifests/generated/kv-resource.yaml b/manifests/generated/kv-resource.yaml +index 66d1b01dbf..43e36b7195 100644 +--- a/manifests/generated/kv-resource.yaml ++++ b/manifests/generated/kv-resource.yaml +@@ -3307,9 +3307,6 @@ spec: + - jsonPath: .status.phase + name: Phase + type: string +- deprecated: true +- deprecationWarning: kubevirt.io/v1alpha3 is now deprecated and will be removed +- in a future release. + name: v1alpha3 + schema: + openAPIV3Schema: +diff --git a/manifests/generated/operator-csv.yaml.in b/manifests/generated/operator-csv.yaml.in +index 400d118024..05ee099c67 100644 +--- a/manifests/generated/operator-csv.yaml.in ++++ b/manifests/generated/operator-csv.yaml.in +@@ -605,6 +605,13 @@ spec: + - '*' + verbs: + - '*' ++ - apiGroups: ++ - subresources.virtualization.deckhouse.io ++ resources: ++ - virtualmachines/addvolume ++ - virtualmachines/removevolume ++ verbs: ++ - update + - apiGroups: + - subresources.kubevirt.io + resources: +diff --git a/manifests/generated/rbac-operator.authorization.k8s.yaml.in b/manifests/generated/rbac-operator.authorization.k8s.yaml.in +index 10dbb92269..1ccc9e9fa7 100644 +--- a/manifests/generated/rbac-operator.authorization.k8s.yaml.in ++++ b/manifests/generated/rbac-operator.authorization.k8s.yaml.in +@@ -143,7 +143,7 @@ kind: RoleBinding + metadata: + labels: + kubevirt.io: "" +- name: kubevirt-operator-rolebinding ++ name: kubevirt-operator + namespace: {{.Namespace}} + roleRef: + apiGroup: rbac.authorization.k8s.io +@@ -607,6 +607,13 @@ rules: + - '*' + verbs: + - '*' ++- apiGroups: ++ - subresources.virtualization.deckhouse.io ++ resources: ++ - virtualmachines/addvolume ++ - virtualmachines/removevolume ++ verbs: ++ - update + - apiGroups: + - subresources.kubevirt.io + resources: +diff --git a/pkg/container-disk/container-disk.go b/pkg/container-disk/container-disk.go +index 3251d04787..69454ed499 100644 +--- a/pkg/container-disk/container-disk.go ++++ b/pkg/container-disk/container-disk.go +@@ -47,8 +47,10 @@ var containerDiskOwner = "qemu" + var podsBaseDir = util.KubeletPodsDir + + var mountBaseDir = filepath.Join(util.VirtShareDir, "/container-disks") ++var hotplugBaseDir = filepath.Join(util.VirtShareDir, "/hotplug-disks") + + type SocketPathGetter func(vmi *v1.VirtualMachineInstance, volumeIndex int) (string, error) ++type HotplugSocketPathGetter func(vmi *v1.VirtualMachineInstance, volumeName string) (string, error) + type KernelBootSocketPathGetter func(vmi *v1.VirtualMachineInstance) (string, error) + + const KernelBootName = "kernel-boot" +@@ -107,6 +109,10 @@ func GetDiskTargetPathFromLauncherView(volumeIndex int) string { + return filepath.Join(mountBaseDir, GetDiskTargetName(volumeIndex)) + } + ++func GetHotplugContainerDiskTargetPathFromLauncherView(volumeName string) string { ++ return filepath.Join(hotplugBaseDir, fmt.Sprintf("%s.img", volumeName)) ++} ++ + func GetKernelBootArtifactPathFromLauncherView(artifact string) string { + artifactBase := filepath.Base(artifact) + return filepath.Join(mountBaseDir, KernelBootName, artifactBase) +@@ -170,6 +176,23 @@ func NewSocketPathGetter(baseDir string) SocketPathGetter { + } + } + ++func NewHotplugSocketPathGetter(baseDir string) HotplugSocketPathGetter { ++ return func(vmi *v1.VirtualMachineInstance, volumeName string) (string, error) { ++ for _, v := range vmi.Status.VolumeStatus { ++ if v.Name == volumeName && v.HotplugVolume != nil && v.ContainerDiskVolume != nil { ++ basePath := getHotplugContainerDiskSocketBasePath(baseDir, string(v.HotplugVolume.AttachPodUID)) ++ socketPath := filepath.Join(basePath, fmt.Sprintf("hotplug-container-disk-%s.sock", volumeName)) ++ exists, _ := diskutils.FileExists(socketPath) ++ if exists { ++ return socketPath, nil ++ } ++ } ++ } ++ ++ return "", fmt.Errorf("container disk socket path not found for vmi \"%s\"", vmi.Name) ++ } ++} ++ + // NewKernelBootSocketPathGetter get the socket pat of the kernel-boot containerDisk. For testing a baseDir + // can be provided which can for instance point to /tmp. + func NewKernelBootSocketPathGetter(baseDir string) KernelBootSocketPathGetter { +@@ -398,6 +421,10 @@ func getContainerDiskSocketBasePath(baseDir, podUID string) string { + return fmt.Sprintf("%s/pods/%s/volumes/kubernetes.io~empty-dir/container-disks", baseDir, podUID) + } + ++func getHotplugContainerDiskSocketBasePath(baseDir, podUID string) string { ++ return fmt.Sprintf("%s/pods/%s/volumes/kubernetes.io~empty-dir/hotplug-container-disks", baseDir, podUID) ++} ++ + // ExtractImageIDsFromSourcePod takes the VMI and its source pod to determine the exact image used by containerdisks and boot container images, + // which is recorded in the status section of a started pod; if the status section does not contain this info the tag is used. + // It returns a map where the key is the vlume name and the value is the imageID +diff --git a/pkg/controller/controller.go b/pkg/controller/controller.go +index 490cc445ef..4b7dbc12fe 100644 +--- a/pkg/controller/controller.go ++++ b/pkg/controller/controller.go +@@ -278,6 +278,10 @@ func ApplyVolumeRequestOnVMISpec(vmiSpec *v1.VirtualMachineInstanceSpec, request + dvSource := request.AddVolumeOptions.VolumeSource.DataVolume.DeepCopy() + dvSource.Hotpluggable = true + newVolume.VolumeSource.DataVolume = dvSource ++ } else if request.AddVolumeOptions.VolumeSource.ContainerDisk != nil { ++ containerDiskSource := request.AddVolumeOptions.VolumeSource.ContainerDisk.DeepCopy() ++ containerDiskSource.Hotpluggable = true ++ newVolume.VolumeSource.ContainerDisk = containerDiskSource + } + + vmiSpec.Volumes = append(vmiSpec.Volumes, newVolume) +@@ -444,6 +448,9 @@ func VMIHasHotplugVolumes(vmi *v1.VirtualMachineInstance) bool { + if volume.PersistentVolumeClaim != nil && volume.PersistentVolumeClaim.Hotpluggable { + return true + } ++ if volume.ContainerDisk != nil && volume.ContainerDisk.Hotpluggable { ++ return true ++ } + } + return false + } +@@ -557,7 +564,7 @@ func GetHotplugVolumes(vmi *v1.VirtualMachineInstance, virtlauncherPod *k8sv1.Po + podVolumeMap[podVolume.Name] = podVolume + } + for _, vmiVolume := range vmiVolumes { +- if _, ok := podVolumeMap[vmiVolume.Name]; !ok && (vmiVolume.DataVolume != nil || vmiVolume.PersistentVolumeClaim != nil || vmiVolume.MemoryDump != nil) { ++ if _, ok := podVolumeMap[vmiVolume.Name]; !ok && (vmiVolume.DataVolume != nil || vmiVolume.PersistentVolumeClaim != nil || vmiVolume.MemoryDump != nil || vmiVolume.ContainerDisk != nil) { + hotplugVolumes = append(hotplugVolumes, vmiVolume.DeepCopy()) + } + } +diff --git a/pkg/virt-api/rest/subresource.go b/pkg/virt-api/rest/subresource.go +index b5d62f5af5..bf561f00ae 100644 +--- a/pkg/virt-api/rest/subresource.go ++++ b/pkg/virt-api/rest/subresource.go +@@ -1023,7 +1023,8 @@ func volumeSourceName(volumeSource *v1.HotplugVolumeSource) string { + + func volumeSourceExists(volume v1.Volume, volumeName string) bool { + return (volume.DataVolume != nil && volume.DataVolume.Name == volumeName) || +- (volume.PersistentVolumeClaim != nil && volume.PersistentVolumeClaim.ClaimName == volumeName) ++ (volume.PersistentVolumeClaim != nil && volume.PersistentVolumeClaim.ClaimName == volumeName) || ++ (volume.ContainerDisk != nil && volume.ContainerDisk.Image != "") + } + + func volumeExists(volume v1.Volume, volumeName string) bool { +@@ -1125,6 +1126,8 @@ func (app *SubresourceAPIApp) addVolumeRequestHandler(request *restful.Request, + opts.VolumeSource.DataVolume.Hotpluggable = true + } else if opts.VolumeSource.PersistentVolumeClaim != nil { + opts.VolumeSource.PersistentVolumeClaim.Hotpluggable = true ++ } else if opts.VolumeSource.ContainerDisk != nil { ++ opts.VolumeSource.ContainerDisk.Hotpluggable = true + } + + // inject into VMI if ephemeral, else set as a request on the VM to both make permanent and hotplug. +diff --git a/pkg/virt-api/webhooks/validating-webhook/admitters/vmi-update-admitter.go b/pkg/virt-api/webhooks/validating-webhook/admitters/vmi-update-admitter.go +index 0af25f8074..803c0ed4cd 100644 +--- a/pkg/virt-api/webhooks/validating-webhook/admitters/vmi-update-admitter.go ++++ b/pkg/virt-api/webhooks/validating-webhook/admitters/vmi-update-admitter.go +@@ -200,11 +200,11 @@ func verifyHotplugVolumes(newHotplugVolumeMap, oldHotplugVolumeMap map[string]v1 + } + } else { + // This is a new volume, ensure that the volume is either DV, PVC or memoryDumpVolume +- if v.DataVolume == nil && v.PersistentVolumeClaim == nil && v.MemoryDump == nil { ++ if v.DataVolume == nil && v.PersistentVolumeClaim == nil && v.MemoryDump == nil && v.ContainerDisk == nil { + return webhookutils.ToAdmissionResponse([]metav1.StatusCause{ + { + Type: metav1.CauseTypeFieldValueInvalid, +- Message: fmt.Sprintf("volume %s is not a PVC or DataVolume", k), ++ Message: fmt.Sprintf("volume %s is not a PVC,DataVolume,MemoryDumpVolume or ContainerDisk", k), + }, + }) + } +diff --git a/pkg/virt-controller/services/template.go b/pkg/virt-controller/services/template.go +index 76ed7307ec..f607c24786 100644 +--- a/pkg/virt-controller/services/template.go ++++ b/pkg/virt-controller/services/template.go +@@ -64,13 +64,15 @@ import ( + ) + + const ( +- containerDisks = "container-disks" +- hotplugDisks = "hotplug-disks" +- hookSidecarSocks = "hook-sidecar-sockets" +- varRun = "/var/run" +- virtBinDir = "virt-bin-share-dir" +- hotplugDisk = "hotplug-disk" +- virtExporter = "virt-exporter" ++ containerDisks = "container-disks" ++ hotplugDisks = "hotplug-disks" ++ hookSidecarSocks = "hook-sidecar-sockets" ++ varRun = "/var/run" ++ virtBinDir = "virt-bin-share-dir" ++ hotplugDisk = "hotplug-disk" ++ virtExporter = "virt-exporter" ++ hotplugContainerDisks = "hotplug-container-disks" ++ HotplugContainerDisk = "hotplug-container-disk-" + ) + + const KvmDevice = "devices.virtualization.deckhouse.io/kvm" +@@ -846,6 +848,49 @@ func sidecarContainerName(i int) string { + return fmt.Sprintf("hook-sidecar-%d", i) + } + ++func sidecarContainerHotplugContainerdDiskName(name string) string { ++ return fmt.Sprintf("%s%s", HotplugContainerDisk, name) ++} ++ ++func (t *templateService) containerForHotplugContainerDisk(name string, cd *v1.ContainerDiskSource, vmi *v1.VirtualMachineInstance) k8sv1.Container { ++ runUser := int64(util.NonRootUID) ++ sharedMount := k8sv1.MountPropagationHostToContainer ++ path := fmt.Sprintf("/path/%s", name) ++ command := []string{"/init/usr/bin/container-disk"} ++ args := []string{"--copy-path", path} ++ ++ return k8sv1.Container{ ++ Name: name, ++ Image: cd.Image, ++ Command: command, ++ Args: args, ++ Resources: hotplugContainerResourceRequirementsForVMI(vmi, t.clusterConfig), ++ SecurityContext: &k8sv1.SecurityContext{ ++ AllowPrivilegeEscalation: pointer.Bool(false), ++ RunAsNonRoot: pointer.Bool(true), ++ RunAsUser: &runUser, ++ SeccompProfile: &k8sv1.SeccompProfile{ ++ Type: k8sv1.SeccompProfileTypeRuntimeDefault, ++ }, ++ Capabilities: &k8sv1.Capabilities{ ++ Drop: []k8sv1.Capability{"ALL"}, ++ }, ++ SELinuxOptions: &k8sv1.SELinuxOptions{ ++ Type: t.clusterConfig.GetSELinuxLauncherType(), ++ Level: "s0", ++ }, ++ }, ++ VolumeMounts: []k8sv1.VolumeMount{ ++ initContainerVolumeMount(), ++ { ++ Name: hotplugContainerDisks, ++ MountPath: "/path", ++ MountPropagation: &sharedMount, ++ }, ++ }, ++ } ++} ++ + func (t *templateService) RenderHotplugAttachmentPodTemplate(volumes []*v1.Volume, ownerPod *k8sv1.Pod, vmi *v1.VirtualMachineInstance, claimMap map[string]*k8sv1.PersistentVolumeClaim) (*k8sv1.Pod, error) { + zero := int64(0) + runUser := int64(util.NonRootUID) +@@ -924,6 +969,30 @@ func (t *templateService) RenderHotplugAttachmentPodTemplate(volumes []*v1.Volum + TerminationGracePeriodSeconds: &zero, + }, + } ++ first := true ++ for _, vol := range vmi.Spec.Volumes { ++ if vol.ContainerDisk == nil || !vol.ContainerDisk.Hotpluggable { ++ continue ++ } ++ name := sidecarContainerHotplugContainerdDiskName(vol.Name) ++ pod.Spec.Containers = append(pod.Spec.Containers, t.containerForHotplugContainerDisk(name, vol.ContainerDisk, vmi)) ++ if first { ++ first = false ++ userId := int64(util.NonRootUID) ++ initContainerCommand := []string{"/usr/bin/cp", ++ "/usr/bin/container-disk", ++ "/init/usr/bin/container-disk", ++ } ++ pod.Spec.InitContainers = append( ++ pod.Spec.InitContainers, ++ t.newInitContainerRenderer(vmi, ++ initContainerVolumeMount(), ++ initContainerResourceRequirementsForVMI(vmi, v1.ContainerDisk, t.clusterConfig), ++ userId).Render(initContainerCommand)) ++ pod.Spec.Volumes = append(pod.Spec.Volumes, emptyDirVolume(hotplugContainerDisks)) ++ pod.Spec.Volumes = append(pod.Spec.Volumes, emptyDirVolume(virtBinDir)) ++ } ++ } + + err := matchSELinuxLevelOfVMI(pod, vmi) + if err != nil { +diff --git a/pkg/virt-controller/watch/BUILD.bazel b/pkg/virt-controller/watch/BUILD.bazel +index 4fd325ba86..82fcaee0a3 100644 +--- a/pkg/virt-controller/watch/BUILD.bazel ++++ b/pkg/virt-controller/watch/BUILD.bazel +@@ -101,6 +101,7 @@ go_library( + "//vendor/k8s.io/client-go/util/flowcontrol:go_default_library", + "//vendor/k8s.io/client-go/util/workqueue:go_default_library", + "//vendor/k8s.io/utils/pointer:go_default_library", ++ "//vendor/k8s.io/utils/ptr:go_default_library", + "//vendor/k8s.io/utils/trace:go_default_library", + "//vendor/kubevirt.io/containerized-data-importer-api/pkg/apis/core/v1beta1:go_default_library", + ], +diff --git a/pkg/virt-controller/watch/vmi.go b/pkg/virt-controller/watch/vmi.go +index fa4e86ee17..ebe718f90d 100644 +--- a/pkg/virt-controller/watch/vmi.go ++++ b/pkg/virt-controller/watch/vmi.go +@@ -1836,6 +1836,10 @@ func (c *VMIController) handleHotplugVolumes(hotplugVolumes []*virtv1.Volume, ho + readyHotplugVolumes := make([]*virtv1.Volume, 0) + // Find all ready volumes + for _, volume := range hotplugVolumes { ++ if volume.ContainerDisk != nil { ++ readyHotplugVolumes = append(readyHotplugVolumes, volume) ++ continue ++ } + var err error + ready, wffc, err := storagetypes.VolumeReadyToAttachToNode(vmi.Namespace, *volume, dataVolumes, c.dataVolumeIndexer, c.pvcIndexer) + if err != nil { +@@ -1884,7 +1888,15 @@ func (c *VMIController) handleHotplugVolumes(hotplugVolumes []*virtv1.Volume, ho + + func (c *VMIController) podVolumesMatchesReadyVolumes(attachmentPod *k8sv1.Pod, volumes []*virtv1.Volume) bool { + // -2 for empty dir and token +- if len(attachmentPod.Spec.Volumes)-2 != len(volumes) { ++ // -3 if exist container-disk ++ magicNum := len(attachmentPod.Spec.Volumes) - 2 ++ for _, volume := range volumes { ++ if volume.ContainerDisk != nil { ++ magicNum -= 1 ++ break ++ } ++ } ++ if magicNum != len(volumes) { + return false + } + podVolumeMap := make(map[string]k8sv1.Volume) +@@ -1893,10 +1905,20 @@ func (c *VMIController) podVolumesMatchesReadyVolumes(attachmentPod *k8sv1.Pod, + podVolumeMap[volume.Name] = volume + } + } ++ containerDisksNames := make(map[string]struct{}) ++ for _, ctr := range attachmentPod.Spec.Containers { ++ if strings.HasPrefix(ctr.Name, services.HotplugContainerDisk) { ++ containerDisksNames[strings.TrimPrefix(ctr.Name, services.HotplugContainerDisk)] = struct{}{} ++ } ++ } + for _, volume := range volumes { ++ if volume.ContainerDisk != nil { ++ delete(containerDisksNames, volume.Name) ++ continue ++ } + delete(podVolumeMap, volume.Name) + } +- return len(podVolumeMap) == 0 ++ return len(podVolumeMap) == 0 && len(containerDisksNames) == 0 + } + + func (c *VMIController) createAttachmentPod(vmi *virtv1.VirtualMachineInstance, virtLauncherPod *k8sv1.Pod, volumes []*virtv1.Volume) (*k8sv1.Pod, syncError) { +@@ -2007,7 +2029,17 @@ func (c *VMIController) createAttachmentPodTemplate(vmi *virtv1.VirtualMachineIn + var pod *k8sv1.Pod + var err error + +- volumeNamesPVCMap, err := storagetypes.VirtVolumesToPVCMap(volumes, c.pvcIndexer, virtlauncherPod.Namespace) ++ var hasContainerDisk bool ++ var newVolumes []*virtv1.Volume ++ for _, volume := range volumes { ++ if volume.VolumeSource.ContainerDisk != nil { ++ hasContainerDisk = true ++ continue ++ } ++ newVolumes = append(newVolumes, volume) ++ } ++ ++ volumeNamesPVCMap, err := storagetypes.VirtVolumesToPVCMap(newVolumes, c.pvcIndexer, virtlauncherPod.Namespace) + if err != nil { + return nil, fmt.Errorf("failed to get PVC map: %v", err) + } +@@ -2029,7 +2061,7 @@ func (c *VMIController) createAttachmentPodTemplate(vmi *virtv1.VirtualMachineIn + } + } + +- if len(volumeNamesPVCMap) > 0 { ++ if len(volumeNamesPVCMap) > 0 || hasContainerDisk { + pod, err = c.templateService.RenderHotplugAttachmentPodTemplate(volumes, virtlauncherPod, vmi, volumeNamesPVCMap) + } + return pod, err +@@ -2151,23 +2183,39 @@ func (c *VMIController) updateVolumeStatus(vmi *virtv1.VirtualMachineInstance, v + ClaimName: volume.Name, + } + } ++ if volume.ContainerDisk != nil && status.ContainerDiskVolume == nil { ++ status.ContainerDiskVolume = &virtv1.ContainerDiskInfo{} ++ } + if attachmentPod == nil { +- if !c.volumeReady(status.Phase) { +- status.HotplugVolume.AttachPodUID = "" +- // Volume is not hotplugged in VM and Pod is gone, or hasn't been created yet, check for the PVC associated with the volume to set phase and message +- phase, reason, message := c.getVolumePhaseMessageReason(&vmi.Spec.Volumes[i], vmi.Namespace) +- status.Phase = phase +- status.Message = message +- status.Reason = reason ++ if volume.ContainerDisk != nil { ++ if !c.volumeReady(status.Phase) { ++ status.HotplugVolume.AttachPodUID = "" ++ // Volume is not hotplugged in VM and Pod is gone, or hasn't been created yet, check for the PVC associated with the volume to set phase and message ++ phase, reason, message := c.getVolumePhaseMessageReason(&vmi.Spec.Volumes[i], vmi.Namespace) ++ status.Phase = phase ++ status.Message = message ++ status.Reason = reason ++ } + } + } else { + status.HotplugVolume.AttachPodName = attachmentPod.Name +- if len(attachmentPod.Status.ContainerStatuses) == 1 && attachmentPod.Status.ContainerStatuses[0].Ready { ++ if volume.ContainerDisk != nil { ++ uid := types.UID("") ++ for _, cs := range attachmentPod.Status.ContainerStatuses { ++ name := strings.TrimPrefix(cs.Name, "hotplug-container-disk-") ++ if volume.Name == name && cs.Ready { ++ uid = attachmentPod.UID ++ break ++ } ++ } ++ status.HotplugVolume.AttachPodUID = uid ++ } else if len(attachmentPod.Status.ContainerStatuses) == 1 && attachmentPod.Status.ContainerStatuses[0].Ready { + status.HotplugVolume.AttachPodUID = attachmentPod.UID + } else { + // Remove UID of old pod if a new one is available, but not yet ready + status.HotplugVolume.AttachPodUID = "" + } ++ + if c.canMoveToAttachedPhase(status.Phase) { + status.Phase = virtv1.HotplugVolumeAttachedToNode + status.Message = fmt.Sprintf("Created hotplug attachment pod %s, for volume %s", attachmentPod.Name, volume.Name) +@@ -2176,7 +2224,6 @@ func (c *VMIController) updateVolumeStatus(vmi *virtv1.VirtualMachineInstance, v + } + } + } +- + if volume.VolumeSource.PersistentVolumeClaim != nil || volume.VolumeSource.DataVolume != nil || volume.VolumeSource.MemoryDump != nil { + + pvcName := storagetypes.PVCNameFromVirtVolume(&volume) +diff --git a/pkg/virt-handler/container-disk/hotplug.go b/pkg/virt-handler/container-disk/hotplug.go +new file mode 100644 +index 0000000000..eb0bb831fe +--- /dev/null ++++ b/pkg/virt-handler/container-disk/hotplug.go +@@ -0,0 +1,487 @@ ++package container_disk ++ ++import ( ++ "encoding/json" ++ "errors" ++ "fmt" ++ "os" ++ "path/filepath" ++ "strings" ++ "sync" ++ "time" ++ ++ hotplugdisk "kubevirt.io/kubevirt/pkg/hotplug-disk" ++ "kubevirt.io/kubevirt/pkg/unsafepath" ++ ++ "kubevirt.io/kubevirt/pkg/safepath" ++ virtconfig "kubevirt.io/kubevirt/pkg/virt-config" ++ virt_chroot "kubevirt.io/kubevirt/pkg/virt-handler/virt-chroot" ++ ++ "kubevirt.io/client-go/log" ++ ++ containerdisk "kubevirt.io/kubevirt/pkg/container-disk" ++ diskutils "kubevirt.io/kubevirt/pkg/ephemeral-disk-utils" ++ "kubevirt.io/kubevirt/pkg/virt-handler/isolation" ++ ++ "k8s.io/apimachinery/pkg/api/equality" ++ "k8s.io/apimachinery/pkg/types" ++ ++ v1 "kubevirt.io/api/core/v1" ++) ++ ++type HotplugMounter interface { ++ ContainerDisksReady(vmi *v1.VirtualMachineInstance, notInitializedSince time.Time) (bool, error) ++ MountAndVerify(vmi *v1.VirtualMachineInstance) (map[string]*containerdisk.DiskInfo, error) ++ IsMounted(vmi *v1.VirtualMachineInstance, volumeName string) (bool, error) ++ Umount(vmi *v1.VirtualMachineInstance) error ++ UmountAll(vmi *v1.VirtualMachineInstance) error ++ ComputeChecksums(vmi *v1.VirtualMachineInstance) (*DiskChecksums, error) ++} ++ ++type hotplugMounter struct { ++ podIsolationDetector isolation.PodIsolationDetector ++ mountStateDir string ++ mountRecords map[types.UID]*vmiMountTargetRecord ++ mountRecordsLock sync.Mutex ++ suppressWarningTimeout time.Duration ++ clusterConfig *virtconfig.ClusterConfig ++ nodeIsolationResult isolation.IsolationResult ++ ++ hotplugPathGetter containerdisk.HotplugSocketPathGetter ++ hotplugManager hotplugdisk.HotplugDiskManagerInterface ++} ++ ++func (m *hotplugMounter) IsMounted(vmi *v1.VirtualMachineInstance, volumeName string) (bool, error) { ++ virtLauncherUID := m.findVirtlauncherUID(vmi) ++ if virtLauncherUID == "" { ++ return false, nil ++ } ++ target, err := m.hotplugManager.GetFileSystemDiskTargetPathFromHostView(virtLauncherUID, volumeName, false) ++ if err != nil { ++ return false, err ++ } ++ return isolation.IsMounted(target) ++} ++ ++func NewHotplugMounter(isoDetector isolation.PodIsolationDetector, ++ mountStateDir string, ++ clusterConfig *virtconfig.ClusterConfig, ++ hotplugManager hotplugdisk.HotplugDiskManagerInterface, ++) HotplugMounter { ++ return &hotplugMounter{ ++ mountRecords: make(map[types.UID]*vmiMountTargetRecord), ++ podIsolationDetector: isoDetector, ++ mountStateDir: mountStateDir, ++ suppressWarningTimeout: 1 * time.Minute, ++ clusterConfig: clusterConfig, ++ nodeIsolationResult: isolation.NodeIsolationResult(), ++ ++ hotplugPathGetter: containerdisk.NewHotplugSocketPathGetter(""), ++ hotplugManager: hotplugManager, ++ } ++} ++ ++func (m *hotplugMounter) deleteMountTargetRecord(vmi *v1.VirtualMachineInstance) error { ++ if string(vmi.UID) == "" { ++ return fmt.Errorf("unable to find container disk mounted directories for vmi without uid") ++ } ++ ++ recordFile := filepath.Join(m.mountStateDir, string(vmi.UID)) ++ ++ exists, err := diskutils.FileExists(recordFile) ++ if err != nil { ++ return err ++ } ++ ++ if exists { ++ record, err := m.getMountTargetRecord(vmi) ++ if err != nil { ++ return err ++ } ++ ++ for _, target := range record.MountTargetEntries { ++ os.Remove(target.TargetFile) ++ os.Remove(target.SocketFile) ++ } ++ ++ os.Remove(recordFile) ++ } ++ ++ m.mountRecordsLock.Lock() ++ defer m.mountRecordsLock.Unlock() ++ delete(m.mountRecords, vmi.UID) ++ ++ return nil ++} ++ ++func (m *hotplugMounter) getMountTargetRecord(vmi *v1.VirtualMachineInstance) (*vmiMountTargetRecord, error) { ++ var ok bool ++ var existingRecord *vmiMountTargetRecord ++ ++ if string(vmi.UID) == "" { ++ return nil, fmt.Errorf("unable to find container disk mounted directories for vmi without uid") ++ } ++ ++ m.mountRecordsLock.Lock() ++ defer m.mountRecordsLock.Unlock() ++ existingRecord, ok = m.mountRecords[vmi.UID] ++ ++ // first check memory cache ++ if ok { ++ return existingRecord, nil ++ } ++ ++ // if not there, see if record is on disk, this can happen if virt-handler restarts ++ recordFile := filepath.Join(m.mountStateDir, filepath.Clean(string(vmi.UID))) ++ ++ exists, err := diskutils.FileExists(recordFile) ++ if err != nil { ++ return nil, err ++ } ++ ++ if exists { ++ record := vmiMountTargetRecord{} ++ // #nosec No risk for path injection. Using static base and cleaned filename ++ bytes, err := os.ReadFile(recordFile) ++ if err != nil { ++ return nil, err ++ } ++ err = json.Unmarshal(bytes, &record) ++ if err != nil { ++ return nil, err ++ } ++ ++ // XXX: backward compatibility for old unresolved paths, can be removed in July 2023 ++ // After a one-time convert and persist, old records are safe too. ++ if !record.UsesSafePaths { ++ record.UsesSafePaths = true ++ for i, entry := range record.MountTargetEntries { ++ safePath, err := safepath.JoinAndResolveWithRelativeRoot("/", entry.TargetFile) ++ if err != nil { ++ return nil, fmt.Errorf("failed converting legacy path to safepath: %v", err) ++ } ++ record.MountTargetEntries[i].TargetFile = unsafepath.UnsafeAbsolute(safePath.Raw()) ++ } ++ } ++ ++ m.mountRecords[vmi.UID] = &record ++ return &record, nil ++ } ++ ++ // not found ++ return nil, nil ++} ++ ++func (m *hotplugMounter) addMountTargetRecord(vmi *v1.VirtualMachineInstance, record *vmiMountTargetRecord) error { ++ return m.setAddMountTargetRecordHelper(vmi, record, true) ++} ++ ++func (m *hotplugMounter) setMountTargetRecord(vmi *v1.VirtualMachineInstance, record *vmiMountTargetRecord) error { ++ return m.setAddMountTargetRecordHelper(vmi, record, false) ++} ++ ++func (m *hotplugMounter) setAddMountTargetRecordHelper(vmi *v1.VirtualMachineInstance, record *vmiMountTargetRecord, addPreviousRules bool) error { ++ if string(vmi.UID) == "" { ++ return fmt.Errorf("unable to set container disk mounted directories for vmi without uid") ++ } ++ ++ record.UsesSafePaths = true ++ ++ recordFile := filepath.Join(m.mountStateDir, string(vmi.UID)) ++ fileExists, err := diskutils.FileExists(recordFile) ++ if err != nil { ++ return err ++ } ++ ++ m.mountRecordsLock.Lock() ++ defer m.mountRecordsLock.Unlock() ++ ++ existingRecord, ok := m.mountRecords[vmi.UID] ++ if ok && fileExists && equality.Semantic.DeepEqual(existingRecord, record) { ++ // already done ++ return nil ++ } ++ ++ if addPreviousRules && existingRecord != nil && len(existingRecord.MountTargetEntries) > 0 { ++ record.MountTargetEntries = append(record.MountTargetEntries, existingRecord.MountTargetEntries...) ++ } ++ ++ bytes, err := json.Marshal(record) ++ if err != nil { ++ return err ++ } ++ ++ err = os.MkdirAll(filepath.Dir(recordFile), 0750) ++ if err != nil { ++ return err ++ } ++ ++ err = os.WriteFile(recordFile, bytes, 0600) ++ if err != nil { ++ return err ++ } ++ ++ m.mountRecords[vmi.UID] = record ++ ++ return nil ++} ++ ++func (m *hotplugMounter) MountAndVerify(vmi *v1.VirtualMachineInstance) (map[string]*containerdisk.DiskInfo, error) { ++ virtLauncherUID := m.findVirtlauncherUID(vmi) ++ if virtLauncherUID == "" { ++ return nil, nil ++ } ++ ++ record := vmiMountTargetRecord{} ++ disksInfo := map[string]*containerdisk.DiskInfo{} ++ ++ for _, volume := range vmi.Spec.Volumes { ++ if volume.ContainerDisk != nil && volume.ContainerDisk.Hotpluggable { ++ target, err := m.hotplugManager.GetFileSystemDiskTargetPathFromHostView(virtLauncherUID, volume.Name, true) ++ if err != nil { ++ return nil, err ++ } ++ ++ sock, err := m.hotplugPathGetter(vmi, volume.Name) ++ if err != nil { ++ return nil, err ++ } ++ ++ record.MountTargetEntries = append(record.MountTargetEntries, vmiMountTargetEntry{ ++ TargetFile: unsafepath.UnsafeAbsolute(target.Raw()), ++ SocketFile: sock, ++ }) ++ } ++ } ++ ++ if len(record.MountTargetEntries) > 0 { ++ err := m.setMountTargetRecord(vmi, &record) ++ if err != nil { ++ return nil, err ++ } ++ } ++ ++ vmiRes, err := m.podIsolationDetector.Detect(vmi) ++ if err != nil { ++ return nil, fmt.Errorf("failed to detect VMI pod: %v", err) ++ } ++ ++ for _, volume := range vmi.Spec.Volumes { ++ if volume.ContainerDisk != nil && volume.ContainerDisk.Hotpluggable { ++ target, err := m.hotplugManager.GetFileSystemDiskTargetPathFromHostView(virtLauncherUID, volume.Name, false) ++ ++ if isMounted, err := isolation.IsMounted(target); err != nil { ++ return nil, fmt.Errorf("failed to determine if %s is already mounted: %v", target, err) ++ } else if !isMounted { ++ ++ sourceFile, err := m.getContainerDiskPath(vmi, &volume, volume.Name) ++ if err != nil { ++ return nil, fmt.Errorf("failed to find a sourceFile in containerDisk %v: %v", volume.Name, err) ++ } ++ ++ log.DefaultLogger().Object(vmi).Infof("Bind mounting container disk at %s to %s", sourceFile, target) ++ opts := []string{ ++ "bind", "ro", "uid=107", "gid=107", ++ } ++ err = virt_chroot.MountChrootWithOptions(sourceFile, target, opts...) ++ if err != nil { ++ return nil, fmt.Errorf("failed to bindmount containerDisk %v. err: %w", volume.Name, err) ++ } ++ } ++ // qemu-img: Could not open '/var/run/kubevirt/hotplug-disks/alpine.img': Could not open '/var/run/kubevirt/hotplug-disks/alpine.img': Permission denied ++ // containerdisk.GetHotplugContainerDiskTargetPathFromLauncherView(volume.Name), ++ // qemu-img: Could not open 'root: /var/lib/kubelet/pods/06b70b2f-7041-45e6-a333-9b9f009e72ef/volumes/kubernetes.io~empty-dir/hotplug-disks, relative: /alpine.im ++ // qemu-img: Could not open '/var/lib/kubelet/pods/06b70b2f-7041-45e6-a333-9b9f009e72ef/volumes/kubernetes.io~empty-dir/hotplug-disks/alpine.img': Could not open '/var/lib/kubelet/pods/06b70b2f-7041-45e6-a333-9b9f009e72ef/volumes/kubernetes.io~empty-dir/hotplug-disks/alpine.img': No such file or directory ++ // unsafepath.UnsafeAbsolute(target.Raw( ++ imageInfo, err := isolation.GetImageInfo( ++ containerdisk.GetHotplugContainerDiskTargetPathFromLauncherView(volume.Name), ++ vmiRes, ++ m.clusterConfig.GetDiskVerification(), ++ ) ++ if err != nil { ++ return nil, fmt.Errorf("failed to get image info: %v", err) ++ } ++ if err := containerdisk.VerifyImage(imageInfo); err != nil { ++ return nil, fmt.Errorf("invalid image in containerDisk %v: %v", volume.Name, err) ++ } ++ disksInfo[volume.Name] = imageInfo ++ } ++ } ++ ++ return disksInfo, nil ++} ++ ++func (m *hotplugMounter) Umount(vmi *v1.VirtualMachineInstance) error { ++ record, err := m.getMountTargetRecord(vmi) ++ if err != nil { ++ return err ++ } else if record == nil { ++ // no entries to unmount ++ ++ log.DefaultLogger().Object(vmi).Infof("No container disk mount entries found to unmount") ++ return nil ++ } ++ for _, r := range record.MountTargetEntries { ++ name, err := extractNameFromSocket(r.SocketFile) ++ if err != nil { ++ return err ++ } ++ needUmount := true ++ for _, v := range vmi.Status.VolumeStatus { ++ if v.Name == name { ++ needUmount = false ++ } ++ } ++ if needUmount { ++ file, err := safepath.NewFileNoFollow(r.TargetFile) ++ if err != nil { ++ if errors.Is(err, os.ErrNotExist) { ++ continue ++ } ++ return fmt.Errorf(failedCheckMountPointFmt, r.TargetFile, err) ++ } ++ _ = file.Close() ++ // #nosec No risk for attacket injection. Parameters are predefined strings ++ out, err := virt_chroot.UmountChroot(file.Path()).CombinedOutput() ++ if err != nil { ++ return fmt.Errorf(failedUnmountFmt, file, string(out), err) ++ } ++ } ++ } ++ return nil ++} ++ ++func extractNameFromSocket(socketFile string) (string, error) { ++ base := filepath.Base(socketFile) ++ if strings.HasPrefix(base, "hotplug-container-disk-") && strings.HasSuffix(base, ".sock") { ++ name := strings.TrimPrefix(base, "hotplug-container-disk-") ++ name = strings.TrimSuffix(name, ".sock") ++ return name, nil ++ } ++ return "", fmt.Errorf("name not found in path") ++} ++ ++func (m *hotplugMounter) UmountAll(vmi *v1.VirtualMachineInstance) error { ++ if vmi.UID == "" { ++ return nil ++ } ++ ++ record, err := m.getMountTargetRecord(vmi) ++ if err != nil { ++ return err ++ } else if record == nil { ++ // no entries to unmount ++ ++ log.DefaultLogger().Object(vmi).Infof("No container disk mount entries found to unmount") ++ return nil ++ } ++ ++ log.DefaultLogger().Object(vmi).Infof("Found container disk mount entries") ++ for _, entry := range record.MountTargetEntries { ++ log.DefaultLogger().Object(vmi).Infof("Looking to see if containerdisk is mounted at path %s", entry.TargetFile) ++ file, err := safepath.NewFileNoFollow(entry.TargetFile) ++ if err != nil { ++ if errors.Is(err, os.ErrNotExist) { ++ continue ++ } ++ return fmt.Errorf(failedCheckMountPointFmt, entry.TargetFile, err) ++ } ++ _ = file.Close() ++ if mounted, err := isolation.IsMounted(file.Path()); err != nil { ++ return fmt.Errorf(failedCheckMountPointFmt, file, err) ++ } else if mounted { ++ log.DefaultLogger().Object(vmi).Infof("unmounting container disk at path %s", file) ++ // #nosec No risk for attacket injection. Parameters are predefined strings ++ out, err := virt_chroot.UmountChroot(file.Path()).CombinedOutput() ++ if err != nil { ++ return fmt.Errorf(failedUnmountFmt, file, string(out), err) ++ } ++ } ++ } ++ err = m.deleteMountTargetRecord(vmi) ++ if err != nil { ++ return err ++ } ++ ++ return nil ++} ++ ++func (m *hotplugMounter) ContainerDisksReady(vmi *v1.VirtualMachineInstance, notInitializedSince time.Time) (bool, error) { ++ for _, volume := range vmi.Spec.Volumes { ++ if volume.ContainerDisk != nil && volume.ContainerDisk.Hotpluggable { ++ _, err := m.hotplugPathGetter(vmi, volume.Name) ++ if err != nil { ++ log.DefaultLogger().Object(vmi).Reason(err).Infof("containerdisk %s not yet ready", volume.Name) ++ if time.Now().After(notInitializedSince.Add(m.suppressWarningTimeout)) { ++ return false, fmt.Errorf("containerdisk %s still not ready after one minute", volume.Name) ++ } ++ return false, nil ++ } ++ } ++ } ++ ++ log.DefaultLogger().Object(vmi).V(4).Info("all containerdisks are ready") ++ return true, nil ++} ++ ++func (m *hotplugMounter) getContainerDiskPath(vmi *v1.VirtualMachineInstance, volume *v1.Volume, volumeName string) (*safepath.Path, error) { ++ sock, err := m.hotplugPathGetter(vmi, volumeName) ++ if err != nil { ++ return nil, ErrDiskContainerGone ++ } ++ ++ res, err := m.podIsolationDetector.DetectForSocket(vmi, sock) ++ if err != nil { ++ return nil, fmt.Errorf("failed to detect socket for containerDisk %v: %v", volume.Name, err) ++ } ++ ++ mountPoint, err := isolation.ParentPathForRootMount(m.nodeIsolationResult, res) ++ if err != nil { ++ return nil, fmt.Errorf("failed to detect root mount point of containerDisk %v on the node: %v", volume.Name, err) ++ } ++ ++ return containerdisk.GetImage(mountPoint, volume.ContainerDisk.Path) ++} ++ ++func (m *hotplugMounter) ComputeChecksums(vmi *v1.VirtualMachineInstance) (*DiskChecksums, error) { ++ ++ diskChecksums := &DiskChecksums{ ++ ContainerDiskChecksums: map[string]uint32{}, ++ } ++ ++ for _, volume := range vmi.Spec.Volumes { ++ if volume.VolumeSource.ContainerDisk == nil || !volume.VolumeSource.ContainerDisk.Hotpluggable { ++ continue ++ } ++ ++ path, err := m.getContainerDiskPath(vmi, &volume, volume.Name) ++ if err != nil { ++ return nil, err ++ } ++ ++ checksum, err := getDigest(path) ++ if err != nil { ++ return nil, err ++ } ++ ++ diskChecksums.ContainerDiskChecksums[volume.Name] = checksum ++ } ++ ++ return diskChecksums, nil ++} ++ ++func (m *hotplugMounter) findVirtlauncherUID(vmi *v1.VirtualMachineInstance) (uid types.UID) { ++ cnt := 0 ++ for podUID := range vmi.Status.ActivePods { ++ _, err := m.hotplugManager.GetHotplugTargetPodPathOnHost(podUID) ++ if err == nil { ++ uid = podUID ++ cnt++ ++ } ++ } ++ if cnt == 1 { ++ return ++ } ++ // Either no pods, or multiple pods, skip. ++ return types.UID("") ++} +diff --git a/pkg/virt-handler/container-disk/mount.go b/pkg/virt-handler/container-disk/mount.go +index 953c20f3af..d99bec3a43 100644 +--- a/pkg/virt-handler/container-disk/mount.go ++++ b/pkg/virt-handler/container-disk/mount.go +@@ -54,6 +54,8 @@ type mounter struct { + kernelBootSocketPathGetter containerdisk.KernelBootSocketPathGetter + clusterConfig *virtconfig.ClusterConfig + nodeIsolationResult isolation.IsolationResult ++ ++ hotplugPathGetter containerdisk.HotplugSocketPathGetter + } + + type Mounter interface { +@@ -98,6 +100,8 @@ func NewMounter(isoDetector isolation.PodIsolationDetector, mountStateDir string + kernelBootSocketPathGetter: containerdisk.NewKernelBootSocketPathGetter(""), + clusterConfig: clusterConfig, + nodeIsolationResult: isolation.NodeIsolationResult(), ++ ++ hotplugPathGetter: containerdisk.NewHotplugSocketPathGetter(""), + } + } + +@@ -254,7 +258,7 @@ func (m *mounter) MountAndVerify(vmi *v1.VirtualMachineInstance) (map[string]*co + disksInfo := map[string]*containerdisk.DiskInfo{} + + for i, volume := range vmi.Spec.Volumes { +- if volume.ContainerDisk != nil { ++ if volume.ContainerDisk != nil && !volume.ContainerDisk.Hotpluggable { + diskTargetDir, err := containerdisk.GetDiskTargetDirFromHostView(vmi) + if err != nil { + return nil, err +@@ -296,7 +300,7 @@ func (m *mounter) MountAndVerify(vmi *v1.VirtualMachineInstance) (map[string]*co + } + + for i, volume := range vmi.Spec.Volumes { +- if volume.ContainerDisk != nil { ++ if volume.ContainerDisk != nil && !volume.ContainerDisk.Hotpluggable { + diskTargetDir, err := containerdisk.GetDiskTargetDirFromHostView(vmi) + if err != nil { + return nil, err +@@ -394,7 +398,7 @@ func (m *mounter) Unmount(vmi *v1.VirtualMachineInstance) error { + + func (m *mounter) ContainerDisksReady(vmi *v1.VirtualMachineInstance, notInitializedSince time.Time) (bool, error) { + for i, volume := range vmi.Spec.Volumes { +- if volume.ContainerDisk != nil { ++ if volume.ContainerDisk != nil && !volume.ContainerDisk.Hotpluggable { + _, err := m.socketPathGetter(vmi, i) + if err != nil { + log.DefaultLogger().Object(vmi).Reason(err).Infof("containerdisk %s not yet ready", volume.Name) +@@ -706,7 +710,7 @@ func (m *mounter) ComputeChecksums(vmi *v1.VirtualMachineInstance) (*DiskChecksu + + // compute for containerdisks + for i, volume := range vmi.Spec.Volumes { +- if volume.VolumeSource.ContainerDisk == nil { ++ if volume.VolumeSource.ContainerDisk == nil || volume.VolumeSource.ContainerDisk.Hotpluggable { + continue + } + +diff --git a/pkg/virt-handler/hotplug-disk/mount.go b/pkg/virt-handler/hotplug-disk/mount.go +index 971c8d55fc..03fcec8c92 100644 +--- a/pkg/virt-handler/hotplug-disk/mount.go ++++ b/pkg/virt-handler/hotplug-disk/mount.go +@@ -310,14 +310,17 @@ func (m *volumeMounter) mountHotplugVolume( + logger := log.DefaultLogger() + logger.V(4).Infof("Hotplug check volume name: %s", volumeName) + if sourceUID != types.UID("") { +- if m.isBlockVolume(&vmi.Status, volumeName) { ++ switch { ++ case m.isContainerDisk(&vmi.Status, volumeName): ++ // skip ++ case m.isBlockVolume(&vmi.Status, volumeName): + logger.V(4).Infof("Mounting block volume: %s", volumeName) + if err := m.mountBlockHotplugVolume(vmi, volumeName, sourceUID, record, cgroupManager); err != nil { + if !errors.Is(err, os.ErrNotExist) { + return fmt.Errorf("failed to mount block hotplug volume %s: %v", volumeName, err) + } + } +- } else { ++ default: + logger.V(4).Infof("Mounting file system volume: %s", volumeName) + if err := m.mountFileSystemHotplugVolume(vmi, volumeName, sourceUID, record, mountDirectory); err != nil { + if !errors.Is(err, os.ErrNotExist) { +@@ -382,6 +385,15 @@ func (m *volumeMounter) isBlockVolume(vmiStatus *v1.VirtualMachineInstanceStatus + return false + } + ++func (m *volumeMounter) isContainerDisk(vmiStatus *v1.VirtualMachineInstanceStatus, volumeName string) bool { ++ for _, status := range vmiStatus.VolumeStatus { ++ if status.Name == volumeName { ++ return status.ContainerDiskVolume != nil ++ } ++ } ++ return false ++} ++ + func (m *volumeMounter) mountBlockHotplugVolume( + vmi *v1.VirtualMachineInstance, + volume string, +diff --git a/pkg/virt-handler/isolation/detector.go b/pkg/virt-handler/isolation/detector.go +index f83f96ead4..5e38c6cedd 100644 +--- a/pkg/virt-handler/isolation/detector.go ++++ b/pkg/virt-handler/isolation/detector.go +@@ -24,6 +24,8 @@ package isolation + import ( + "fmt" + "net" ++ "os" ++ "path" + "runtime" + "syscall" + "time" +@@ -207,12 +209,45 @@ func setProcessMemoryLockRLimit(pid int, size int64) error { + return nil + } + ++type deferFunc func() ++ ++func (s *socketBasedIsolationDetector) socketHack(socket string) (sock net.Conn, deferFunc deferFunc, err error) { ++ fn := func() {} ++ if len([]rune(socket)) <= 108 { ++ sock, err = net.DialTimeout("unix", socket, time.Duration(isolationDialTimeout)*time.Second) ++ fn = func() { ++ if err == nil { ++ sock.Close() ++ } ++ } ++ return sock, fn, err ++ } ++ base := path.Base(socket) ++ newPath := fmt.Sprintf("/tmp/%s", base) ++ if err = os.Symlink(socket, newPath); err != nil { ++ return nil, fn, err ++ } ++ sock, err = net.DialTimeout("unix", newPath, time.Duration(isolationDialTimeout)*time.Second) ++ fn = func() { ++ if err == nil { ++ sock.Close() ++ } ++ os.Remove(newPath) ++ } ++ return sock, fn, err ++} ++ + func (s *socketBasedIsolationDetector) getPid(socket string) (int, error) { +- sock, err := net.DialTimeout("unix", socket, time.Duration(isolationDialTimeout)*time.Second) ++ sock, defFn, err := s.socketHack(socket) ++ defer defFn() + if err != nil { + return -1, err + } +- defer sock.Close() ++ //sock, err := net.DialTimeout("unix", socket, time.Duration(isolationDialTimeout)*time.Second) ++ //if err != nil { ++ // return -1, err ++ //} ++ //defer sock.Close() + + ufile, err := sock.(*net.UnixConn).File() + if err != nil { +diff --git a/pkg/virt-handler/virt-chroot/virt-chroot.go b/pkg/virt-handler/virt-chroot/virt-chroot.go +index 4160212b7b..580b788acc 100644 +--- a/pkg/virt-handler/virt-chroot/virt-chroot.go ++++ b/pkg/virt-handler/virt-chroot/virt-chroot.go +@@ -20,7 +20,10 @@ + package virt_chroot + + import ( ++ "bytes" ++ "fmt" + "os/exec" ++ "slices" + "strings" + + "kubevirt.io/kubevirt/pkg/safepath" +@@ -48,6 +51,49 @@ func MountChroot(sourcePath, targetPath *safepath.Path, ro bool) *exec.Cmd { + return UnsafeMountChroot(trimProcPrefix(sourcePath), trimProcPrefix(targetPath), ro) + } + ++func MountChrootWithOptions(sourcePath, targetPath *safepath.Path, mountOptions ...string) error { ++ args := append(getBaseArgs(), "mount") ++ remountArgs := slices.Clone(args) ++ ++ mountOptions = slices.DeleteFunc(mountOptions, func(s string) bool { ++ return s == "remount" ++ }) ++ if len(mountOptions) > 0 { ++ opts := strings.Join(mountOptions, ",") ++ remountOpts := "remount," + opts ++ args = append(args, "-o", opts) ++ remountArgs = append(remountArgs, "-o", remountOpts) ++ } ++ ++ sp := trimProcPrefix(sourcePath) ++ tp := trimProcPrefix(targetPath) ++ args = append(args, sp, tp) ++ remountArgs = append(remountArgs, sp, tp) ++ ++ stdout := new(bytes.Buffer) ++ stderr := new(bytes.Buffer) ++ ++ cmd := exec.Command(binaryPath, args...) ++ cmd.Stdout = stdout ++ cmd.Stderr = stderr ++ err := cmd.Run() ++ if err != nil { ++ return fmt.Errorf("mount failed: %w, stdout: %s, stderr: %s", err, stdout.String(), stderr.String()) ++ } ++ ++ stdout = new(bytes.Buffer) ++ stderr = new(bytes.Buffer) ++ ++ remountCmd := exec.Command(binaryPath, remountArgs...) ++ cmd.Stdout = stdout ++ cmd.Stderr = stderr ++ err = remountCmd.Run() ++ if err != nil { ++ return fmt.Errorf("mount failed: %w, stdout: %s, stderr: %s", err, stdout.String(), stderr.String()) ++ } ++ return nil ++} ++ + // Deprecated: UnsafeMountChroot is used to connect to code which needs to be refactored + // to handle mounts securely. + func UnsafeMountChroot(sourcePath, targetPath string, ro bool) *exec.Cmd { +diff --git a/pkg/virt-handler/vm.go b/pkg/virt-handler/vm.go +index 24352cf6e9..f1de8d1149 100644 +--- a/pkg/virt-handler/vm.go ++++ b/pkg/virt-handler/vm.go +@@ -25,6 +25,7 @@ import ( + goerror "errors" + "fmt" + "io" ++ "maps" + "net" + "os" + "path/filepath" +@@ -247,6 +248,13 @@ func NewController( + vmiExpectations: controller.NewUIDTrackingControllerExpectations(controller.NewControllerExpectations()), + sriovHotplugExecutorPool: executor.NewRateLimitedExecutorPool(executor.NewExponentialLimitedBackoffCreator()), + ioErrorRetryManager: NewFailRetryManager("io-error-retry", 10*time.Second, 3*time.Minute, 30*time.Second), ++ ++ hotplugContainerDiskMounter: container_disk.NewHotplugMounter( ++ podIsolationDetector, ++ filepath.Join(virtPrivateDir, "hotplug-container-disk-mount-state"), ++ clusterConfig, ++ hotplugdisk.NewHotplugDiskManager(kubeletPodsDir), ++ ), + } + + _, err := vmiSourceInformer.AddEventHandler(cache.ResourceEventHandlerFuncs{ +@@ -342,6 +350,8 @@ type VirtualMachineController struct { + hostCpuModel string + vmiExpectations *controller.UIDTrackingControllerExpectations + ioErrorRetryManager *FailRetryManager ++ ++ hotplugContainerDiskMounter container_disk.HotplugMounter + } + + type virtLauncherCriticalSecurebootError struct { +@@ -876,7 +886,15 @@ func (d *VirtualMachineController) updateHotplugVolumeStatus(vmi *v1.VirtualMach + needsRefresh := false + if volumeStatus.Target == "" { + needsRefresh = true +- mounted, err := d.hotplugVolumeMounter.IsMounted(vmi, volumeStatus.Name, volumeStatus.HotplugVolume.AttachPodUID) ++ var ( ++ mounted bool ++ err error ++ ) ++ if volumeStatus.ContainerDiskVolume != nil { ++ mounted, err = d.hotplugContainerDiskMounter.IsMounted(vmi, volumeStatus.Name) ++ } else { ++ mounted, err = d.hotplugVolumeMounter.IsMounted(vmi, volumeStatus.Name, volumeStatus.HotplugVolume.AttachPodUID) ++ } + if err != nil { + log.Log.Object(vmi).Errorf("error occurred while checking if volume is mounted: %v", err) + } +@@ -898,6 +916,7 @@ func (d *VirtualMachineController) updateHotplugVolumeStatus(vmi *v1.VirtualMach + volumeStatus.Reason = VolumeUnMountedFromPodReason + } + } ++ + } else { + // Successfully attached to VM. + volumeStatus.Phase = v1.VolumeReady +@@ -2178,6 +2197,11 @@ func (d *VirtualMachineController) processVmCleanup(vmi *v1.VirtualMachineInstan + return err + } + ++ err := d.hotplugContainerDiskMounter.UmountAll(vmi) ++ if err != nil { ++ return err ++ } ++ + // UnmountAll does the cleanup on the "best effort" basis: it is + // safe to pass a nil cgroupManager. + cgroupManager, _ := getCgroupManager(vmi) +@@ -2829,6 +2853,12 @@ func (d *VirtualMachineController) vmUpdateHelperMigrationTarget(origVMI *v1.Vir + return err + } + ++ hotplugDiskInfo, err := d.hotplugContainerDiskMounter.MountAndVerify(vmi) ++ if err != nil { ++ return err ++ } ++ maps.Copy(disksInfo, hotplugDiskInfo) ++ + // Mount hotplug disks + if attachmentPodUID := vmi.Status.MigrationState.TargetAttachmentPodUID; attachmentPodUID != types.UID("") { + cgroupManager, err := getCgroupManager(vmi) +@@ -3051,6 +3081,11 @@ func (d *VirtualMachineController) vmUpdateHelperDefault(origVMI *v1.VirtualMach + if err != nil { + return err + } ++ hotplugDiskInfo, err := d.hotplugContainerDiskMounter.MountAndVerify(vmi) ++ if err != nil { ++ return err ++ } ++ maps.Copy(disksInfo, hotplugDiskInfo) + + // Try to mount hotplug volume if there is any during startup. + if err := d.hotplugVolumeMounter.Mount(vmi, cgroupManager); err != nil { +@@ -3138,6 +3173,11 @@ func (d *VirtualMachineController) vmUpdateHelperDefault(origVMI *v1.VirtualMach + log.Log.Object(vmi).Error(err.Error()) + } + ++ hotplugDiskInfo, err := d.hotplugContainerDiskMounter.MountAndVerify(vmi) ++ if err != nil { ++ return err ++ } ++ maps.Copy(disksInfo, hotplugDiskInfo) + if err := d.hotplugVolumeMounter.Mount(vmi, cgroupManager); err != nil { + return err + } +@@ -3215,6 +3255,9 @@ func (d *VirtualMachineController) vmUpdateHelperDefault(origVMI *v1.VirtualMach + + if vmi.IsRunning() { + // Umount any disks no longer mounted ++ if err := d.hotplugContainerDiskMounter.Umount(vmi); err != nil { ++ return err ++ } + if err := d.hotplugVolumeMounter.Unmount(vmi, cgroupManager); err != nil { + return err + } +diff --git a/pkg/virt-launcher/virtwrap/converter/converter.go b/pkg/virt-launcher/virtwrap/converter/converter.go +index 3318c1c466..1286ef4a06 100644 +--- a/pkg/virt-launcher/virtwrap/converter/converter.go ++++ b/pkg/virt-launcher/virtwrap/converter/converter.go +@@ -649,6 +649,9 @@ func Convert_v1_Hotplug_Volume_To_api_Disk(source *v1.Volume, disk *api.Disk, c + if source.DataVolume != nil { + return Convert_v1_Hotplug_DataVolume_To_api_Disk(source.Name, disk, c) + } ++ if source.ContainerDisk != nil { ++ return Convert_v1_Hotplug_ContainerDisk_To_api_Disk(source.Name, disk, c) ++ } + return fmt.Errorf("hotplug disk %s references an unsupported source", disk.Alias.GetName()) + } + +@@ -690,6 +693,10 @@ func GetHotplugBlockDeviceVolumePath(volumeName string) string { + return filepath.Join(string(filepath.Separator), "var", "run", "kubevirt", "hotplug-disks", volumeName) + } + ++func GetHotplugContainerDiskPath(volumeName string) string { ++ return filepath.Join(string(filepath.Separator), "var", "run", "kubevirt", "hotplug-disks", volumeName) ++} ++ + func Convert_v1_PersistentVolumeClaim_To_api_Disk(name string, disk *api.Disk, c *ConverterContext) error { + if c.IsBlockPVC[name] { + return Convert_v1_BlockVolumeSource_To_api_Disk(name, disk, c.VolumesDiscardIgnore) +@@ -768,6 +775,34 @@ func Convert_v1_Hotplug_BlockVolumeSource_To_api_Disk(volumeName string, disk *a + return nil + } + ++func Convert_v1_Hotplug_ContainerDisk_To_api_Disk(volumeName string, disk *api.Disk, c *ConverterContext) error { ++ if disk.Type == "lun" { ++ return fmt.Errorf(deviceTypeNotCompatibleFmt, disk.Alias.GetName()) ++ } ++ disk.Type = "file" ++ disk.Driver.Type = "qcow2" ++ disk.Driver.ErrorPolicy = v1.DiskErrorPolicyStop ++ disk.ReadOnly = &api.ReadOnly{} ++ if !contains(c.VolumesDiscardIgnore, volumeName) { ++ disk.Driver.Discard = "unmap" ++ } ++ disk.Source.File = GetHotplugContainerDiskPath(volumeName) ++ disk.BackingStore = &api.BackingStore{ ++ Format: &api.BackingStoreFormat{}, ++ Source: &api.DiskSource{}, ++ } ++ ++ if info := c.DisksInfo[volumeName]; info != nil { ++ disk.BackingStore.Format.Type = info.Format ++ disk.BackingStore.Source.File = info.BackingFile ++ } else { ++ return fmt.Errorf("no disk info provided for volume %s", volumeName) ++ } ++ disk.BackingStore.Type = "file" ++ ++ return nil ++} ++ + func Convert_v1_HostDisk_To_api_Disk(volumeName string, path string, disk *api.Disk) error { + disk.Type = "file" + disk.Driver.Type = "raw" +diff --git a/pkg/virt-operator/resource/apply/BUILD.bazel b/pkg/virt-operator/resource/apply/BUILD.bazel +index f6bd9bd4f1..fe6ab54f8c 100644 +--- a/pkg/virt-operator/resource/apply/BUILD.bazel ++++ b/pkg/virt-operator/resource/apply/BUILD.bazel +@@ -4,7 +4,6 @@ go_library( + name = "go_default_library", + srcs = [ + "admissionregistration.go", +- "apiservices.go", + "apps.go", + "certificates.go", + "core.go", +@@ -65,7 +64,6 @@ go_library( + "//vendor/k8s.io/client-go/tools/cache:go_default_library", + "//vendor/k8s.io/client-go/tools/record:go_default_library", + "//vendor/k8s.io/client-go/util/workqueue:go_default_library", +- "//vendor/k8s.io/kube-aggregator/pkg/apis/apiregistration/v1:go_default_library", + "//vendor/k8s.io/utils/pointer:go_default_library", + ], + ) +diff --git a/pkg/virt-operator/resource/generate/components/BUILD.bazel b/pkg/virt-operator/resource/generate/components/BUILD.bazel +index 70d2da0897..affcd3fecd 100644 +--- a/pkg/virt-operator/resource/generate/components/BUILD.bazel ++++ b/pkg/virt-operator/resource/generate/components/BUILD.bazel +@@ -3,7 +3,6 @@ load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test") + go_library( + name = "go_default_library", + srcs = [ +- "apiservices.go", + "crds.go", + "daemonsets.go", + "deployments.go", +@@ -62,7 +61,6 @@ go_library( + "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", + "//vendor/k8s.io/apimachinery/pkg/util/intstr:go_default_library", + "//vendor/k8s.io/apimachinery/pkg/util/yaml:go_default_library", +- "//vendor/k8s.io/kube-aggregator/pkg/apis/apiregistration/v1:go_default_library", + "//vendor/k8s.io/utils/pointer:go_default_library", + ], + ) +@@ -70,7 +68,6 @@ go_library( + go_test( + name = "go_default_test", + srcs = [ +- "apiservices_test.go", + "components_suite_test.go", + "crds_test.go", + "deployments_test.go", +@@ -85,7 +82,6 @@ go_test( + deps = [ + "//pkg/certificates/bootstrap:go_default_library", + "//pkg/certificates/triple/cert:go_default_library", +- "//staging/src/kubevirt.io/api/core/v1:go_default_library", + "//staging/src/kubevirt.io/client-go/testutils:go_default_library", + "//vendor/github.com/onsi/ginkgo/v2:go_default_library", + "//vendor/github.com/onsi/gomega:go_default_library", +diff --git a/pkg/virt-operator/resource/generate/components/validations_generated.go b/pkg/virt-operator/resource/generate/components/validations_generated.go +index 4913dbead0..42225780ba 100644 +--- a/pkg/virt-operator/resource/generate/components/validations_generated.go ++++ b/pkg/virt-operator/resource/generate/components/validations_generated.go +@@ -7723,6 +7723,8 @@ var CRDsValidation map[string]string = map[string]string{ + ContainerDisk references a docker image, embedding a qcow or raw disk. + More info: https://kubevirt.gitbooks.io/user-guide/registry-disk.html + properties: ++ hotpluggable: ++ type: boolean + image: + description: Image is the name of the image with the embedded + disk. +@@ -8355,6 +8357,35 @@ var CRDsValidation map[string]string = map[string]string{ + description: VolumeSource represents the source of the volume + to map to the disk. + properties: ++ containerDisk: ++ description: Represents a docker image with an embedded disk. ++ properties: ++ hotpluggable: ++ type: boolean ++ image: ++ description: Image is the name of the image with the embedded ++ disk. ++ type: string ++ imagePullPolicy: ++ description: |- ++ Image pull policy. ++ One of Always, Never, IfNotPresent. ++ Defaults to Always if :latest tag is specified, or IfNotPresent otherwise. ++ Cannot be updated. ++ More info: https://kubernetes.io/docs/concepts/containers/images#updating-images ++ type: string ++ imagePullSecret: ++ description: ImagePullSecret is the name of the Docker ++ registry secret required to pull the image. The secret ++ must already exist. ++ type: string ++ path: ++ description: Path defines the path to disk file in the ++ container ++ type: string ++ required: ++ - image ++ type: object + dataVolume: + description: |- + DataVolume represents the dynamic creation a PVC for this volume as well as +@@ -12768,6 +12799,8 @@ var CRDsValidation map[string]string = map[string]string{ + ContainerDisk references a docker image, embedding a qcow or raw disk. + More info: https://kubevirt.gitbooks.io/user-guide/registry-disk.html + properties: ++ hotpluggable: ++ type: boolean + image: + description: Image is the name of the image with the embedded + disk. +@@ -18328,6 +18361,8 @@ var CRDsValidation map[string]string = map[string]string{ + ContainerDisk references a docker image, embedding a qcow or raw disk. + More info: https://kubevirt.gitbooks.io/user-guide/registry-disk.html + properties: ++ hotpluggable: ++ type: boolean + image: + description: Image is the name of the image with the embedded + disk. +@@ -22835,6 +22870,8 @@ var CRDsValidation map[string]string = map[string]string{ + ContainerDisk references a docker image, embedding a qcow or raw disk. + More info: https://kubevirt.gitbooks.io/user-guide/registry-disk.html + properties: ++ hotpluggable: ++ type: boolean + image: + description: Image is the name of the image with + the embedded disk. +@@ -28015,6 +28052,8 @@ var CRDsValidation map[string]string = map[string]string{ + ContainerDisk references a docker image, embedding a qcow or raw disk. + More info: https://kubevirt.gitbooks.io/user-guide/registry-disk.html + properties: ++ hotpluggable: ++ type: boolean + image: + description: Image is the name of the image + with the embedded disk. +@@ -28673,6 +28712,36 @@ var CRDsValidation map[string]string = map[string]string{ + description: VolumeSource represents the source of + the volume to map to the disk. + properties: ++ containerDisk: ++ description: Represents a docker image with an ++ embedded disk. ++ properties: ++ hotpluggable: ++ type: boolean ++ image: ++ description: Image is the name of the image ++ with the embedded disk. ++ type: string ++ imagePullPolicy: ++ description: |- ++ Image pull policy. ++ One of Always, Never, IfNotPresent. ++ Defaults to Always if :latest tag is specified, or IfNotPresent otherwise. ++ Cannot be updated. ++ More info: https://kubernetes.io/docs/concepts/containers/images#updating-images ++ type: string ++ imagePullSecret: ++ description: ImagePullSecret is the name of ++ the Docker registry secret required to pull ++ the image. The secret must already exist. ++ type: string ++ path: ++ description: Path defines the path to disk ++ file in the container ++ type: string ++ required: ++ - image ++ type: object + dataVolume: + description: |- + DataVolume represents the dynamic creation a PVC for this volume as well as +diff --git a/pkg/virt-operator/resource/generate/install/generated_mock_strategy.go b/pkg/virt-operator/resource/generate/install/generated_mock_strategy.go +index 5f1e9a3121..1fa1416af0 100644 +--- a/pkg/virt-operator/resource/generate/install/generated_mock_strategy.go ++++ b/pkg/virt-operator/resource/generate/install/generated_mock_strategy.go +@@ -241,16 +241,6 @@ func (_mr *_MockStrategyInterfaceRecorder) MutatingWebhookConfigurations() *gomo + return _mr.mock.ctrl.RecordCall(_mr.mock, "MutatingWebhookConfigurations") + } + +-func (_m *MockStrategyInterface) APIServices() []*v18.APIService { +- ret := _m.ctrl.Call(_m, "APIServices") +- ret0, _ := ret[0].([]*v18.APIService) +- return ret0 +-} +- +-func (_mr *_MockStrategyInterfaceRecorder) APIServices() *gomock.Call { +- return _mr.mock.ctrl.RecordCall(_mr.mock, "APIServices") +-} +- + func (_m *MockStrategyInterface) CertificateSecrets() []*v14.Secret { + ret := _m.ctrl.Call(_m, "CertificateSecrets") + ret0, _ := ret[0].([]*v14.Secret) +diff --git a/pkg/virt-operator/resource/generate/rbac/exportproxy.go b/pkg/virt-operator/resource/generate/rbac/exportproxy.go +index ebc9f2adbd..a0dc0586b4 100644 +--- a/pkg/virt-operator/resource/generate/rbac/exportproxy.go ++++ b/pkg/virt-operator/resource/generate/rbac/exportproxy.go +@@ -23,6 +23,7 @@ import ( + rbacv1 "k8s.io/api/rbac/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" ++ + "kubevirt.io/kubevirt/pkg/virt-operator/resource/generate/components" + + virtv1 "kubevirt.io/api/core/v1" +diff --git a/staging/src/kubevirt.io/api/apitesting/testdata/HEAD/kubevirt.io.v1.VirtualMachine.json b/staging/src/kubevirt.io/api/apitesting/testdata/HEAD/kubevirt.io.v1.VirtualMachine.json +index b651173636..3453dfb0da 100644 +--- a/staging/src/kubevirt.io/api/apitesting/testdata/HEAD/kubevirt.io.v1.VirtualMachine.json ++++ b/staging/src/kubevirt.io/api/apitesting/testdata/HEAD/kubevirt.io.v1.VirtualMachine.json +@@ -754,7 +754,8 @@ + "image": "imageValue", + "imagePullSecret": "imagePullSecretValue", + "path": "pathValue", +- "imagePullPolicy": "imagePullPolicyValue" ++ "imagePullPolicy": "imagePullPolicyValue", ++ "hotpluggable": true + }, + "ephemeral": { + "persistentVolumeClaim": { +@@ -1209,6 +1210,13 @@ + "dataVolume": { + "name": "nameValue", + "hotpluggable": true ++ }, ++ "containerDisk": { ++ "image": "imageValue", ++ "imagePullSecret": "imagePullSecretValue", ++ "path": "pathValue", ++ "imagePullPolicy": "imagePullPolicyValue", ++ "hotpluggable": true + } + }, + "dryRun": [ +diff --git a/staging/src/kubevirt.io/api/apitesting/testdata/HEAD/kubevirt.io.v1.VirtualMachine.yaml b/staging/src/kubevirt.io/api/apitesting/testdata/HEAD/kubevirt.io.v1.VirtualMachine.yaml +index 53dfdacc3b..8b23193158 100644 +--- a/staging/src/kubevirt.io/api/apitesting/testdata/HEAD/kubevirt.io.v1.VirtualMachine.yaml ++++ b/staging/src/kubevirt.io/api/apitesting/testdata/HEAD/kubevirt.io.v1.VirtualMachine.yaml +@@ -719,6 +719,7 @@ spec: + optional: true + volumeLabel: volumeLabelValue + containerDisk: ++ hotpluggable: true + image: imageValue + imagePullPolicy: imagePullPolicyValue + imagePullSecret: imagePullSecretValue +@@ -838,6 +839,12 @@ status: + - dryRunValue + name: nameValue + volumeSource: ++ containerDisk: ++ hotpluggable: true ++ image: imageValue ++ imagePullPolicy: imagePullPolicyValue ++ imagePullSecret: imagePullSecretValue ++ path: pathValue + dataVolume: + hotpluggable: true + name: nameValue +diff --git a/staging/src/kubevirt.io/api/apitesting/testdata/HEAD/kubevirt.io.v1.VirtualMachineInstance.json b/staging/src/kubevirt.io/api/apitesting/testdata/HEAD/kubevirt.io.v1.VirtualMachineInstance.json +index 3be904512c..f595798e89 100644 +--- a/staging/src/kubevirt.io/api/apitesting/testdata/HEAD/kubevirt.io.v1.VirtualMachineInstance.json ++++ b/staging/src/kubevirt.io/api/apitesting/testdata/HEAD/kubevirt.io.v1.VirtualMachineInstance.json +@@ -694,7 +694,8 @@ + "image": "imageValue", + "imagePullSecret": "imagePullSecretValue", + "path": "pathValue", +- "imagePullPolicy": "imagePullPolicyValue" ++ "imagePullPolicy": "imagePullPolicyValue", ++ "hotpluggable": true + }, + "ephemeral": { + "persistentVolumeClaim": { +diff --git a/staging/src/kubevirt.io/api/apitesting/testdata/HEAD/kubevirt.io.v1.VirtualMachineInstance.yaml b/staging/src/kubevirt.io/api/apitesting/testdata/HEAD/kubevirt.io.v1.VirtualMachineInstance.yaml +index 6fd2ab6523..b6457ec94d 100644 +--- a/staging/src/kubevirt.io/api/apitesting/testdata/HEAD/kubevirt.io.v1.VirtualMachineInstance.yaml ++++ b/staging/src/kubevirt.io/api/apitesting/testdata/HEAD/kubevirt.io.v1.VirtualMachineInstance.yaml +@@ -524,6 +524,7 @@ spec: + optional: true + volumeLabel: volumeLabelValue + containerDisk: ++ hotpluggable: true + image: imageValue + imagePullPolicy: imagePullPolicyValue + imagePullSecret: imagePullSecretValue +diff --git a/staging/src/kubevirt.io/api/core/v1/BUILD.bazel b/staging/src/kubevirt.io/api/core/v1/BUILD.bazel +index f8615293a3..0c6c166985 100644 +--- a/staging/src/kubevirt.io/api/core/v1/BUILD.bazel ++++ b/staging/src/kubevirt.io/api/core/v1/BUILD.bazel +@@ -28,7 +28,6 @@ go_library( + "//vendor/k8s.io/apimachinery/pkg/runtime/schema:go_default_library", + "//vendor/k8s.io/apimachinery/pkg/types:go_default_library", + "//vendor/k8s.io/utils/net:go_default_library", +- "//vendor/k8s.io/utils/pointer:go_default_library", + "//vendor/kubevirt.io/containerized-data-importer-api/pkg/apis/core/v1beta1:go_default_library", + ], + ) +diff --git a/staging/src/kubevirt.io/api/core/v1/deepcopy_generated.go b/staging/src/kubevirt.io/api/core/v1/deepcopy_generated.go +index abd5a495d6..7372b22a9a 100644 +--- a/staging/src/kubevirt.io/api/core/v1/deepcopy_generated.go ++++ b/staging/src/kubevirt.io/api/core/v1/deepcopy_generated.go +@@ -1948,6 +1948,11 @@ func (in *HotplugVolumeSource) DeepCopyInto(out *HotplugVolumeSource) { + *out = new(DataVolumeSource) + **out = **in + } ++ if in.ContainerDisk != nil { ++ in, out := &in.ContainerDisk, &out.ContainerDisk ++ *out = new(ContainerDiskSource) ++ **out = **in ++ } + return + } + +diff --git a/staging/src/kubevirt.io/api/core/v1/schema.go b/staging/src/kubevirt.io/api/core/v1/schema.go +index 29aa3932d3..302ed9ffde 100644 +--- a/staging/src/kubevirt.io/api/core/v1/schema.go ++++ b/staging/src/kubevirt.io/api/core/v1/schema.go +@@ -854,6 +854,8 @@ type HotplugVolumeSource struct { + // the process of populating that PVC with a disk image. + // +optional + DataVolume *DataVolumeSource `json:"dataVolume,omitempty"` ++ ++ ContainerDisk *ContainerDiskSource `json:"containerDisk,omitempty"` + } + + type DataVolumeSource struct { +@@ -911,6 +913,8 @@ type ContainerDiskSource struct { + // More info: https://kubernetes.io/docs/concepts/containers/images#updating-images + // +optional + ImagePullPolicy v1.PullPolicy `json:"imagePullPolicy,omitempty"` ++ ++ Hotpluggable bool `json:"hotpluggable,omitempty"` + } + + // Exactly one of its members must be set. +diff --git a/staging/src/kubevirt.io/client-go/api/openapi_generated.go b/staging/src/kubevirt.io/client-go/api/openapi_generated.go +index cc2d743492..b982b1620c 100644 +--- a/staging/src/kubevirt.io/client-go/api/openapi_generated.go ++++ b/staging/src/kubevirt.io/client-go/api/openapi_generated.go +@@ -17772,6 +17772,12 @@ func schema_kubevirtio_api_core_v1_ContainerDiskSource(ref common.ReferenceCallb + Enum: []interface{}{"Always", "IfNotPresent", "Never"}, + }, + }, ++ "hotpluggable": { ++ SchemaProps: spec.SchemaProps{ ++ Type: []string{"boolean"}, ++ Format: "", ++ }, ++ }, + }, + Required: []string{"image"}, + }, +@@ -19645,11 +19651,16 @@ func schema_kubevirtio_api_core_v1_HotplugVolumeSource(ref common.ReferenceCallb + Ref: ref("kubevirt.io/api/core/v1.DataVolumeSource"), + }, + }, ++ "containerDisk": { ++ SchemaProps: spec.SchemaProps{ ++ Ref: ref("kubevirt.io/api/core/v1.ContainerDiskSource"), ++ }, ++ }, + }, + }, + }, + Dependencies: []string{ +- "kubevirt.io/api/core/v1.DataVolumeSource", "kubevirt.io/api/core/v1.PersistentVolumeClaimVolumeSource"}, ++ "kubevirt.io/api/core/v1.ContainerDiskSource", "kubevirt.io/api/core/v1.DataVolumeSource", "kubevirt.io/api/core/v1.PersistentVolumeClaimVolumeSource"}, + } + } + From 3e0bdaf15ba117b195462c154e85a6b065fd297f Mon Sep 17 00:00:00 2001 From: yaroslavborbat Date: Thu, 19 Dec 2024 17:58:35 +0300 Subject: [PATCH 04/11] fix Signed-off-by: yaroslavborbat --- .../patches/028-hotplug-container-disk.patch | 76 +++++-------------- 1 file changed, 18 insertions(+), 58 deletions(-) diff --git a/images/virt-artifact/patches/028-hotplug-container-disk.patch b/images/virt-artifact/patches/028-hotplug-container-disk.patch index 852b8d687..b1f60dfc2 100644 --- a/images/virt-artifact/patches/028-hotplug-container-disk.patch +++ b/images/virt-artifact/patches/028-hotplug-container-disk.patch @@ -22,28 +22,6 @@ index c4822a0448..d6bb534249 100644 "dataVolume": { "description": "DataVolume represents the dynamic creation a PVC for this volume as well as the process of populating that PVC with a disk image.", "$ref": "#/definitions/v1.DataVolumeSource" -diff --git a/cmd/hp-container-disk/main.go b/cmd/hp-container-disk/main.go -new file mode 100644 -index 0000000000..e4f734a516 ---- /dev/null -+++ b/cmd/hp-container-disk/main.go -@@ -0,0 +1,16 @@ -+package main -+ -+import klog "kubevirt.io/client-go/log" -+ -+func main() { -+ klog.InitializeLogging("virt-hp-container-disk") -+} -+ -+type Config struct { -+ DstDir string `json:"dstDir"` -+ Images []Image `json:"images"` -+} -+ -+type Image struct { -+ Name string `json:"name"` -+} diff --git a/cmd/virt-chroot/main.go b/cmd/virt-chroot/main.go index e28daa07c7..7a69b7451b 100644 --- a/cmd/virt-chroot/main.go @@ -1146,45 +1124,27 @@ index 953c20f3af..d99bec3a43 100644 } diff --git a/pkg/virt-handler/hotplug-disk/mount.go b/pkg/virt-handler/hotplug-disk/mount.go -index 971c8d55fc..03fcec8c92 100644 +index 971c8d55fc..034c3d8526 100644 --- a/pkg/virt-handler/hotplug-disk/mount.go +++ b/pkg/virt-handler/hotplug-disk/mount.go -@@ -310,14 +310,17 @@ func (m *volumeMounter) mountHotplugVolume( - logger := log.DefaultLogger() - logger.V(4).Infof("Hotplug check volume name: %s", volumeName) - if sourceUID != types.UID("") { -- if m.isBlockVolume(&vmi.Status, volumeName) { -+ switch { -+ case m.isContainerDisk(&vmi.Status, volumeName): -+ // skip -+ case m.isBlockVolume(&vmi.Status, volumeName): - logger.V(4).Infof("Mounting block volume: %s", volumeName) - if err := m.mountBlockHotplugVolume(vmi, volumeName, sourceUID, record, cgroupManager); err != nil { - if !errors.Is(err, os.ErrNotExist) { - return fmt.Errorf("failed to mount block hotplug volume %s: %v", volumeName, err) - } +@@ -343,7 +343,7 @@ func (m *volumeMounter) mountFromPod(vmi *v1.VirtualMachineInstance, sourceUID t + return err + } + for _, volumeStatus := range vmi.Status.VolumeStatus { +- if volumeStatus.HotplugVolume == nil { ++ if volumeStatus.HotplugVolume == nil || volumeStatus.ContainerDiskVolume != nil { + // Skip non hotplug volumes + continue + } +@@ -649,7 +649,7 @@ func (m *volumeMounter) Unmount(vmi *v1.VirtualMachineInstance, cgroupManager cg + return err + } + for _, volumeStatus := range vmi.Status.VolumeStatus { +- if volumeStatus.HotplugVolume == nil { ++ if volumeStatus.HotplugVolume == nil || volumeStatus.ContainerDiskVolume != nil { + continue } -- } else { -+ default: - logger.V(4).Infof("Mounting file system volume: %s", volumeName) - if err := m.mountFileSystemHotplugVolume(vmi, volumeName, sourceUID, record, mountDirectory); err != nil { - if !errors.Is(err, os.ErrNotExist) { -@@ -382,6 +385,15 @@ func (m *volumeMounter) isBlockVolume(vmiStatus *v1.VirtualMachineInstanceStatus - return false - } - -+func (m *volumeMounter) isContainerDisk(vmiStatus *v1.VirtualMachineInstanceStatus, volumeName string) bool { -+ for _, status := range vmiStatus.VolumeStatus { -+ if status.Name == volumeName { -+ return status.ContainerDiskVolume != nil -+ } -+ } -+ return false -+} -+ - func (m *volumeMounter) mountBlockHotplugVolume( - vmi *v1.VirtualMachineInstance, - volume string, + var path *safepath.Path diff --git a/pkg/virt-handler/isolation/detector.go b/pkg/virt-handler/isolation/detector.go index f83f96ead4..5e38c6cedd 100644 --- a/pkg/virt-handler/isolation/detector.go From 79a659f25f524acd15f2c36d2fb5f33dca9e8fb4 Mon Sep 17 00:00:00 2001 From: yaroslavborbat Date: Mon, 23 Dec 2024 11:39:57 +0300 Subject: [PATCH 05/11] fix rebase Signed-off-by: yaroslavborbat --- ...plug-container-disk.patch => 029-hotplug-container-disk.patch} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename images/virt-artifact/patches/{028-hotplug-container-disk.patch => 029-hotplug-container-disk.patch} (100%) diff --git a/images/virt-artifact/patches/028-hotplug-container-disk.patch b/images/virt-artifact/patches/029-hotplug-container-disk.patch similarity index 100% rename from images/virt-artifact/patches/028-hotplug-container-disk.patch rename to images/virt-artifact/patches/029-hotplug-container-disk.patch From 7a0e0604341c075a690cf2ed5ca90bfaaf51a335 Mon Sep 17 00:00:00 2001 From: yaroslavborbat Date: Tue, 14 Jan 2025 10:33:20 +0300 Subject: [PATCH 06/11] + Signed-off-by: yaroslavborbat --- ...patch => 030-hotplug-container-disk.patch} | 64 +++++++++++++------ 1 file changed, 43 insertions(+), 21 deletions(-) rename images/virt-artifact/patches/{029-hotplug-container-disk.patch => 030-hotplug-container-disk.patch} (98%) diff --git a/images/virt-artifact/patches/029-hotplug-container-disk.patch b/images/virt-artifact/patches/030-hotplug-container-disk.patch similarity index 98% rename from images/virt-artifact/patches/029-hotplug-container-disk.patch rename to images/virt-artifact/patches/030-hotplug-container-disk.patch index b1f60dfc2..37967a21f 100644 --- a/images/virt-artifact/patches/029-hotplug-container-disk.patch +++ b/images/virt-artifact/patches/030-hotplug-container-disk.patch @@ -179,7 +179,7 @@ index 10dbb92269..1ccc9e9fa7 100644 - subresources.kubevirt.io resources: diff --git a/pkg/container-disk/container-disk.go b/pkg/container-disk/container-disk.go -index 3251d04787..69454ed499 100644 +index 3251d04787..34affe841a 100644 --- a/pkg/container-disk/container-disk.go +++ b/pkg/container-disk/container-disk.go @@ -47,8 +47,10 @@ var containerDiskOwner = "qemu" @@ -228,7 +228,34 @@ index 3251d04787..69454ed499 100644 // NewKernelBootSocketPathGetter get the socket pat of the kernel-boot containerDisk. For testing a baseDir // can be provided which can for instance point to /tmp. func NewKernelBootSocketPathGetter(baseDir string) KernelBootSocketPathGetter { -@@ -398,6 +421,10 @@ func getContainerDiskSocketBasePath(baseDir, podUID string) string { +@@ -394,10 +417,37 @@ func CreateEphemeralImages( + return nil + } + ++func CreateEphemeralImagesForHotplug( ++ vmi *v1.VirtualMachineInstance, ++ diskCreator ephemeraldisk.EphemeralDiskCreatorInterface, ++ disksInfo map[string]*DiskInfo, ++) error { ++ for i, volume := range vmi.Spec.Volumes { ++ if volume.VolumeSource.ContainerDisk != nil && volume.VolumeSource.ContainerDisk.Hotpluggable { ++ info, _ := disksInfo[volume.Name] ++ if info == nil { ++ return fmt.Errorf("no disk info provided for volume %s", volume.Name) ++ } ++ ++ if backingFile, err := GetDiskTargetPartFromLauncherView(i); err != nil { ++ return err ++ } else if err := diskCreator.CreateBackedImageForVolume(volume, backingFile, info.Format); err != nil { ++ return err ++ } ++ } ++ } ++ ++ return nil ++} ++ + func getContainerDiskSocketBasePath(baseDir, podUID string) string { return fmt.Sprintf("%s/pods/%s/volumes/kubernetes.io~empty-dir/container-disks", baseDir, podUID) } @@ -574,10 +601,10 @@ index fa4e86ee17..ebe718f90d 100644 pvcName := storagetypes.PVCNameFromVirtVolume(&volume) diff --git a/pkg/virt-handler/container-disk/hotplug.go b/pkg/virt-handler/container-disk/hotplug.go new file mode 100644 -index 0000000000..eb0bb831fe +index 0000000000..f0d3a0607c --- /dev/null +++ b/pkg/virt-handler/container-disk/hotplug.go -@@ -0,0 +1,487 @@ +@@ -0,0 +1,481 @@ +package container_disk + +import ( @@ -731,8 +758,6 @@ index 0000000000..eb0bb831fe + return nil, err + } + -+ // XXX: backward compatibility for old unresolved paths, can be removed in July 2023 -+ // After a one-time convert and persist, old records are safe too. + if !record.UsesSafePaths { + record.UsesSafePaths = true + for i, entry := range record.MountTargetEntries { @@ -868,11 +893,7 @@ index 0000000000..eb0bb831fe + return nil, fmt.Errorf("failed to bindmount containerDisk %v. err: %w", volume.Name, err) + } + } -+ // qemu-img: Could not open '/var/run/kubevirt/hotplug-disks/alpine.img': Could not open '/var/run/kubevirt/hotplug-disks/alpine.img': Permission denied -+ // containerdisk.GetHotplugContainerDiskTargetPathFromLauncherView(volume.Name), -+ // qemu-img: Could not open 'root: /var/lib/kubelet/pods/06b70b2f-7041-45e6-a333-9b9f009e72ef/volumes/kubernetes.io~empty-dir/hotplug-disks, relative: /alpine.im -+ // qemu-img: Could not open '/var/lib/kubelet/pods/06b70b2f-7041-45e6-a333-9b9f009e72ef/volumes/kubernetes.io~empty-dir/hotplug-disks/alpine.img': Could not open '/var/lib/kubelet/pods/06b70b2f-7041-45e6-a333-9b9f009e72ef/volumes/kubernetes.io~empty-dir/hotplug-disks/alpine.img': No such file or directory -+ // unsafepath.UnsafeAbsolute(target.Raw( ++ + imageInfo, err := isolation.GetImageInfo( + containerdisk.GetHotplugContainerDiskTargetPathFromLauncherView(volume.Name), + vmiRes, @@ -1391,7 +1412,7 @@ index 24352cf6e9..f1de8d1149 100644 return err } diff --git a/pkg/virt-launcher/virtwrap/converter/converter.go b/pkg/virt-launcher/virtwrap/converter/converter.go -index 3318c1c466..1286ef4a06 100644 +index 3318c1c466..2f5a0f1215 100644 --- a/pkg/virt-launcher/virtwrap/converter/converter.go +++ b/pkg/virt-launcher/virtwrap/converter/converter.go @@ -649,6 +649,9 @@ func Convert_v1_Hotplug_Volume_To_api_Disk(source *v1.Volume, disk *api.Disk, c @@ -1415,7 +1436,7 @@ index 3318c1c466..1286ef4a06 100644 func Convert_v1_PersistentVolumeClaim_To_api_Disk(name string, disk *api.Disk, c *ConverterContext) error { if c.IsBlockPVC[name] { return Convert_v1_BlockVolumeSource_To_api_Disk(name, disk, c.VolumesDiscardIgnore) -@@ -768,6 +775,34 @@ func Convert_v1_Hotplug_BlockVolumeSource_To_api_Disk(volumeName string, disk *a +@@ -768,6 +775,35 @@ func Convert_v1_Hotplug_BlockVolumeSource_To_api_Disk(volumeName string, disk *a return nil } @@ -1423,8 +1444,13 @@ index 3318c1c466..1286ef4a06 100644 + if disk.Type == "lun" { + return fmt.Errorf(deviceTypeNotCompatibleFmt, disk.Alias.GetName()) + } ++ info := c.DisksInfo[volumeName] ++ if info == nil { ++ return fmt.Errorf("no disk info provided for volume %s", volumeName) ++ } ++ + disk.Type = "file" -+ disk.Driver.Type = "qcow2" ++ disk.Driver.Type = info.Format + disk.Driver.ErrorPolicy = v1.DiskErrorPolicyStop + disk.ReadOnly = &api.ReadOnly{} + if !contains(c.VolumesDiscardIgnore, volumeName) { @@ -1436,13 +1462,9 @@ index 3318c1c466..1286ef4a06 100644 + Source: &api.DiskSource{}, + } + -+ if info := c.DisksInfo[volumeName]; info != nil { -+ disk.BackingStore.Format.Type = info.Format -+ disk.BackingStore.Source.File = info.BackingFile -+ } else { -+ return fmt.Errorf("no disk info provided for volume %s", volumeName) -+ } -+ disk.BackingStore.Type = "file" ++ //disk.BackingStore.Format.Type = info.Format ++ //disk.BackingStore.Source.File = info.BackingFile ++ //disk.BackingStore.Type = "file" + + return nil +} From a2ae4b1d3181e6b66266f8db104c984c8588f37f Mon Sep 17 00:00:00 2001 From: yaroslavborbat Date: Mon, 25 Nov 2024 11:10:01 +0300 Subject: [PATCH 07/11] add hotplug vi Signed-off-by: yaroslavborbat --- api/Taskfile.dist.yaml | 2 +- api/client/kubeclient/client.go | 2 + api/client/kubeclient/vm.go | 26 + ...irtual_machine_block_device_attachment.go} | 8 +- .../generated/openapi/zz_generated.openapi.go | 46 +- api/subresources/types.go | 10 +- api/subresources/v1alpha2/types.go | 6 + .../v1alpha2/zz_generated.conversion.go | 54 + ...-virtualmachineblockdeviceattachments.yaml | 2 + .../virtualmachineblockdeviceattachments.yaml | 4 + .../patches/030-hotplug-container-disk.patch | 1849 ----------------- .../cmd/virtualization-controller/main.go | 2 +- .../apiserver/registry/vm/rest/add_volume.go | 135 +- .../registry/vm/rest/remove_volume.go | 44 +- .../pkg/apiserver/registry/vm/rest/stream.go | 26 + .../pkg/controller/kvapi/kvapi.go | 5 + .../pkg/controller/kvbuilder/kvvm_utils.go | 7 +- .../controller/service/attachment_service.go | 199 +- .../service/attachment_service_test.go | 12 +- .../vmbda/internal/block_device_ready.go | 125 +- .../pkg/controller/vmbda/internal/deletion.go | 73 +- .../controller/vmbda/internal/life_cycle.go | 64 +- .../vmbda/internal/watcher/cvi_watcher.go | 106 + .../vmbda/internal/watcher/vi_watcher.go | 108 + .../pkg/controller/vmbda/vmbda_controller.go | 6 +- .../pkg/controller/vmbda/vmbda_reconciler.go | 2 + .../rbac-for-us.yaml | 2 + 27 files changed, 933 insertions(+), 1992 deletions(-) rename api/core/v1alpha2/{virtual_machine_block_disk_attachment.go => virtual_machine_block_device_attachment.go} (89%) delete mode 100644 images/virt-artifact/patches/030-hotplug-container-disk.patch create mode 100644 images/virtualization-artifact/pkg/controller/vmbda/internal/watcher/cvi_watcher.go create mode 100644 images/virtualization-artifact/pkg/controller/vmbda/internal/watcher/vi_watcher.go diff --git a/api/Taskfile.dist.yaml b/api/Taskfile.dist.yaml index c5ab64b9e..606fe04e9 100644 --- a/api/Taskfile.dist.yaml +++ b/api/Taskfile.dist.yaml @@ -87,4 +87,4 @@ tasks: - go install -mod=readonly sigs.k8s.io/controller-tools/cmd/controller-gen@v{{ .CONTROLLER_GEN_VERSION }} status: - | - ls $GOPATH/bin/controller-gen + $GOPATH/bin/controller-gen --version | grep -q "v{{ .CONTROLLER_GEN_VERSION }}" diff --git a/api/client/kubeclient/client.go b/api/client/kubeclient/client.go index 98c0bdb5d..2a8ed7f57 100644 --- a/api/client/kubeclient/client.go +++ b/api/client/kubeclient/client.go @@ -83,6 +83,8 @@ type VirtualMachineInterface interface { Freeze(ctx context.Context, name string, opts v1alpha2.VirtualMachineFreeze) error Unfreeze(ctx context.Context, name string) error Migrate(ctx context.Context, name string, opts v1alpha2.VirtualMachineMigrate) error + AddVolume(ctx context.Context, name string, opts v1alpha2.VirtualMachineAddVolume) error + RemoveVolume(ctx context.Context, name string, opts v1alpha2.VirtualMachineRemoveVolume) error } type client struct { diff --git a/api/client/kubeclient/vm.go b/api/client/kubeclient/vm.go index 3d2f41eb5..ed75d7940 100644 --- a/api/client/kubeclient/vm.go +++ b/api/client/kubeclient/vm.go @@ -149,3 +149,29 @@ func (v vm) Migrate(ctx context.Context, name string, opts v1alpha2.VirtualMachi } return v.restClient.Put().AbsPath(path).Body(body).Do(ctx).Error() } + +func (v vm) AddVolume(ctx context.Context, name string, opts v1alpha2.VirtualMachineAddVolume) error { + path := fmt.Sprintf(subresourceURLTpl, v.namespace, v.resource, name, "addvolume") + + return v.restClient. + Put(). + AbsPath(path). + Param("name", opts.Name). + Param("volumeKind", opts.VolumeKind). + Param("pvcName", opts.PVCName). + Param("image", opts.Image). + Param("isCdrom", strconv.FormatBool(opts.IsCdrom)). + Do(ctx). + Error() +} + +func (v vm) RemoveVolume(ctx context.Context, name string, opts v1alpha2.VirtualMachineRemoveVolume) error { + path := fmt.Sprintf(subresourceURLTpl, v.namespace, v.resource, name, "removevolume") + + return v.restClient. + Put(). + AbsPath(path). + Param("name", opts.Name). + Do(ctx). + Error() +} diff --git a/api/core/v1alpha2/virtual_machine_block_disk_attachment.go b/api/core/v1alpha2/virtual_machine_block_device_attachment.go similarity index 89% rename from api/core/v1alpha2/virtual_machine_block_disk_attachment.go rename to api/core/v1alpha2/virtual_machine_block_device_attachment.go index 287a0bb2e..912d45395 100644 --- a/api/core/v1alpha2/virtual_machine_block_disk_attachment.go +++ b/api/core/v1alpha2/virtual_machine_block_device_attachment.go @@ -72,6 +72,8 @@ type VirtualMachineBlockDeviceAttachmentStatus struct { type VMBDAObjectRef struct { // The type of the block device. Options are: // * `VirtualDisk` — use `VirtualDisk` as the disk. This type is always mounted in RW mode. + // * `VirtualImage` — use `VirtualImage` as the disk. This type is always mounted in RO mode. + // * `ClusterVirtualImage` - use `ClusterVirtualImage` as the disk. This type is always mounted in RO mode. Kind VMBDAObjectRefKind `json:"kind,omitempty"` // The name of block device to attach. Name string `json:"name,omitempty"` @@ -79,11 +81,13 @@ type VMBDAObjectRef struct { // VMBDAObjectRefKind defines the type of the block device. // -// +kubebuilder:validation:Enum={VirtualDisk} +// +kubebuilder:validation:Enum={VirtualDisk,VirtualImage,ClusterVirtualImage} type VMBDAObjectRefKind string const ( - VMBDAObjectRefKindVirtualDisk VMBDAObjectRefKind = "VirtualDisk" + VMBDAObjectRefKindVirtualDisk VMBDAObjectRefKind = "VirtualDisk" + VMBDAObjectRefKindVirtualImage VMBDAObjectRefKind = "VirtualImage" + VMBDAObjectRefKindClusterVirtualImage VMBDAObjectRefKind = "ClusterVirtualImage" ) // BlockDeviceAttachmentPhase defines current status of resource: diff --git a/api/pkg/apiserver/api/generated/openapi/zz_generated.openapi.go b/api/pkg/apiserver/api/generated/openapi/zz_generated.openapi.go index d769f5487..546666405 100644 --- a/api/pkg/apiserver/api/generated/openapi/zz_generated.openapi.go +++ b/api/pkg/apiserver/api/generated/openapi/zz_generated.openapi.go @@ -2072,7 +2072,7 @@ func schema_virtualization_api_core_v1alpha2_VMBDAObjectRef(ref common.Reference Properties: map[string]spec.Schema{ "kind": { SchemaProps: spec.SchemaProps{ - Description: "The type of the block device. Options are: * `VirtualDisk` — use `VirtualDisk` as the disk. This type is always mounted in RW mode.", + Description: "The type of the block device. Options are: * `VirtualDisk` — use `VirtualDisk` as the disk. This type is always mounted in RW mode. * `VirtualImage` — use `VirtualImage` as the disk. This type is always mounted in RO mode. * `ClusterVirtualImage` - use `ClusterVirtualImage` as the disk. This type is always mounted in RO mode.", Type: []string{"string"}, Format: "", }, @@ -5235,7 +5235,43 @@ func schema_virtualization_api_subresources_v1alpha2_VirtualMachineAddVolume(ref Format: "", }, }, + "name": { + SchemaProps: spec.SchemaProps{ + Default: "", + Type: []string{"string"}, + Format: "", + }, + }, + "volumeKind": { + SchemaProps: spec.SchemaProps{ + Default: "", + Type: []string{"string"}, + Format: "", + }, + }, + "pvcName": { + SchemaProps: spec.SchemaProps{ + Default: "", + Type: []string{"string"}, + Format: "", + }, + }, + "image": { + SchemaProps: spec.SchemaProps{ + Default: "", + Type: []string{"string"}, + Format: "", + }, + }, + "isCdrom": { + SchemaProps: spec.SchemaProps{ + Default: false, + Type: []string{"boolean"}, + Format: "", + }, + }, }, + Required: []string{"name", "volumeKind", "pvcName", "image", "isCdrom"}, }, }, } @@ -5402,7 +5438,15 @@ func schema_virtualization_api_subresources_v1alpha2_VirtualMachineRemoveVolume( Format: "", }, }, + "name": { + SchemaProps: spec.SchemaProps{ + Default: "", + Type: []string{"string"}, + Format: "", + }, + }, }, + Required: []string{"name"}, }, }, } diff --git a/api/subresources/types.go b/api/subresources/types.go index 264029e84..c8d84e5c1 100644 --- a/api/subresources/types.go +++ b/api/subresources/types.go @@ -16,7 +16,9 @@ limitations under the License. package subresources -import metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +import ( + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) // +genclient // +genclient:readonly @@ -51,6 +53,11 @@ type VirtualMachinePortForward struct { type VirtualMachineAddVolume struct { metav1.TypeMeta + Name string + VolumeKind string + PVCName string + Image string + IsCdrom bool } // +genclient @@ -59,6 +66,7 @@ type VirtualMachineAddVolume struct { type VirtualMachineRemoveVolume struct { metav1.TypeMeta + Name string } // +genclient diff --git a/api/subresources/v1alpha2/types.go b/api/subresources/v1alpha2/types.go index 916d7cbad..a4c1a054c 100644 --- a/api/subresources/v1alpha2/types.go +++ b/api/subresources/v1alpha2/types.go @@ -55,6 +55,11 @@ type VirtualMachinePortForward struct { type VirtualMachineAddVolume struct { metav1.TypeMeta `json:",inline"` + Name string `json:"name"` + VolumeKind string `json:"volumeKind"` + PVCName string `json:"pvcName"` + Image string `json:"image"` + IsCdrom bool `json:"isCdrom"` } // +genclient @@ -64,6 +69,7 @@ type VirtualMachineAddVolume struct { type VirtualMachineRemoveVolume struct { metav1.TypeMeta `json:",inline"` + Name string `json:"name"` } // +genclient diff --git a/api/subresources/v1alpha2/zz_generated.conversion.go b/api/subresources/v1alpha2/zz_generated.conversion.go index 330a5be60..4ed39f59c 100644 --- a/api/subresources/v1alpha2/zz_generated.conversion.go +++ b/api/subresources/v1alpha2/zz_generated.conversion.go @@ -162,6 +162,11 @@ func RegisterConversions(s *runtime.Scheme) error { } func autoConvert_v1alpha2_VirtualMachineAddVolume_To_subresources_VirtualMachineAddVolume(in *VirtualMachineAddVolume, out *subresources.VirtualMachineAddVolume, s conversion.Scope) error { + out.Name = in.Name + out.VolumeKind = in.VolumeKind + out.PVCName = in.PVCName + out.Image = in.Image + out.IsCdrom = in.IsCdrom return nil } @@ -171,6 +176,11 @@ func Convert_v1alpha2_VirtualMachineAddVolume_To_subresources_VirtualMachineAddV } func autoConvert_subresources_VirtualMachineAddVolume_To_v1alpha2_VirtualMachineAddVolume(in *subresources.VirtualMachineAddVolume, out *VirtualMachineAddVolume, s conversion.Scope) error { + out.Name = in.Name + out.VolumeKind = in.VolumeKind + out.PVCName = in.PVCName + out.Image = in.Image + out.IsCdrom = in.IsCdrom return nil } @@ -182,6 +192,41 @@ func Convert_subresources_VirtualMachineAddVolume_To_v1alpha2_VirtualMachineAddV func autoConvert_url_Values_To_v1alpha2_VirtualMachineAddVolume(in *url.Values, out *VirtualMachineAddVolume, s conversion.Scope) error { // WARNING: Field TypeMeta does not have json tag, skipping. + if values, ok := map[string][]string(*in)["name"]; ok && len(values) > 0 { + if err := runtime.Convert_Slice_string_To_string(&values, &out.Name, s); err != nil { + return err + } + } else { + out.Name = "" + } + if values, ok := map[string][]string(*in)["volumeKind"]; ok && len(values) > 0 { + if err := runtime.Convert_Slice_string_To_string(&values, &out.VolumeKind, s); err != nil { + return err + } + } else { + out.VolumeKind = "" + } + if values, ok := map[string][]string(*in)["pvcName"]; ok && len(values) > 0 { + if err := runtime.Convert_Slice_string_To_string(&values, &out.PVCName, s); err != nil { + return err + } + } else { + out.PVCName = "" + } + if values, ok := map[string][]string(*in)["image"]; ok && len(values) > 0 { + if err := runtime.Convert_Slice_string_To_string(&values, &out.Image, s); err != nil { + return err + } + } else { + out.Image = "" + } + if values, ok := map[string][]string(*in)["isCdrom"]; ok && len(values) > 0 { + if err := runtime.Convert_Slice_string_To_bool(&values, &out.IsCdrom, s); err != nil { + return err + } + } else { + out.IsCdrom = false + } return nil } @@ -339,6 +384,7 @@ func Convert_url_Values_To_v1alpha2_VirtualMachinePortForward(in *url.Values, ou } func autoConvert_v1alpha2_VirtualMachineRemoveVolume_To_subresources_VirtualMachineRemoveVolume(in *VirtualMachineRemoveVolume, out *subresources.VirtualMachineRemoveVolume, s conversion.Scope) error { + out.Name = in.Name return nil } @@ -348,6 +394,7 @@ func Convert_v1alpha2_VirtualMachineRemoveVolume_To_subresources_VirtualMachineR } func autoConvert_subresources_VirtualMachineRemoveVolume_To_v1alpha2_VirtualMachineRemoveVolume(in *subresources.VirtualMachineRemoveVolume, out *VirtualMachineRemoveVolume, s conversion.Scope) error { + out.Name = in.Name return nil } @@ -359,6 +406,13 @@ func Convert_subresources_VirtualMachineRemoveVolume_To_v1alpha2_VirtualMachineR func autoConvert_url_Values_To_v1alpha2_VirtualMachineRemoveVolume(in *url.Values, out *VirtualMachineRemoveVolume, s conversion.Scope) error { // WARNING: Field TypeMeta does not have json tag, skipping. + if values, ok := map[string][]string(*in)["name"]; ok && len(values) > 0 { + if err := runtime.Convert_Slice_string_To_string(&values, &out.Name, s); err != nil { + return err + } + } else { + out.Name = "" + } return nil } diff --git a/crds/doc-ru-virtualmachineblockdeviceattachments.yaml b/crds/doc-ru-virtualmachineblockdeviceattachments.yaml index e32506de5..3ee322347 100644 --- a/crds/doc-ru-virtualmachineblockdeviceattachments.yaml +++ b/crds/doc-ru-virtualmachineblockdeviceattachments.yaml @@ -16,6 +16,8 @@ spec: description: | Тип блочного устройства. Возможны следующие варианты: * `VirtualDisk` — использовать `VirtualDisk` в качестве диска. Этот тип всегда монтируется в режиме RW. + * `VirtualImage` — использовать `VirtualImage` в качестве диска. Этот тип всегда монтируется в режиме RO. + * `ClusterVirtualImage` - использовать `ClusterVirtualImage` в качестве диска. Этот тип всегда монтируется в режиме RO. name: description: | Имя блочного устройства diff --git a/crds/virtualmachineblockdeviceattachments.yaml b/crds/virtualmachineblockdeviceattachments.yaml index 953465bc1..e140fb88d 100644 --- a/crds/virtualmachineblockdeviceattachments.yaml +++ b/crds/virtualmachineblockdeviceattachments.yaml @@ -71,8 +71,12 @@ spec: description: |- The type of the block device. Options are: * `VirtualDisk` — use `VirtualDisk` as the disk. This type is always mounted in RW mode. + * `VirtualImage` — use `VirtualImage` as the disk. This type is always mounted in RO mode. + * `ClusterVirtualImage` - use `ClusterVirtualImage` as the disk. This type is always mounted in RO mode. enum: - VirtualDisk + - VirtualImage + - ClusterVirtualImage type: string name: description: The name of block device to attach. diff --git a/images/virt-artifact/patches/030-hotplug-container-disk.patch b/images/virt-artifact/patches/030-hotplug-container-disk.patch deleted file mode 100644 index 37967a21f..000000000 --- a/images/virt-artifact/patches/030-hotplug-container-disk.patch +++ /dev/null @@ -1,1849 +0,0 @@ -diff --git a/api/openapi-spec/swagger.json b/api/openapi-spec/swagger.json -index c4822a0448..d6bb534249 100644 ---- a/api/openapi-spec/swagger.json -+++ b/api/openapi-spec/swagger.json -@@ -12951,6 +12951,9 @@ - "image" - ], - "properties": { -+ "hotpluggable": { -+ "type": "boolean" -+ }, - "image": { - "description": "Image is the name of the image with the embedded disk.", - "type": "string", -@@ -13973,6 +13976,9 @@ - "description": "HotplugVolumeSource Represents the source of a volume to mount which are capable of being hotplugged on a live running VMI. Only one of its members may be specified.", - "type": "object", - "properties": { -+ "containerDisk": { -+ "$ref": "#/definitions/v1.ContainerDiskSource" -+ }, - "dataVolume": { - "description": "DataVolume represents the dynamic creation a PVC for this volume as well as the process of populating that PVC with a disk image.", - "$ref": "#/definitions/v1.DataVolumeSource" -diff --git a/cmd/virt-chroot/main.go b/cmd/virt-chroot/main.go -index e28daa07c7..7a69b7451b 100644 ---- a/cmd/virt-chroot/main.go -+++ b/cmd/virt-chroot/main.go -@@ -20,6 +20,7 @@ var ( - cpuTime uint64 - memoryBytes uint64 - targetUser string -+ targetUserID int - ) - - func init() { -@@ -51,7 +52,12 @@ func main() { - - // Looking up users needs resources, let's do it before we set rlimits. - var u *user.User -- if targetUser != "" { -+ if targetUserID >= 0 { -+ _, _, errno := syscall.Syscall(syscall.SYS_SETUID, uintptr(targetUserID), 0, 0) -+ if errno != 0 { -+ return fmt.Errorf("failed to switch to user: %d. errno: %d", targetUserID, errno) -+ } -+ } else if targetUser != "" { - var err error - u, err = user.Lookup(targetUser) - if err != nil { -@@ -116,6 +122,7 @@ func main() { - rootCmd.PersistentFlags().Uint64Var(&memoryBytes, "memory", 0, "memory in bytes for the process") - rootCmd.PersistentFlags().StringVar(&mntNamespace, "mount", "", "mount namespace to use") - rootCmd.PersistentFlags().StringVar(&targetUser, "user", "", "switch to this targetUser to e.g. drop privileges") -+ rootCmd.PersistentFlags().IntVar(&targetUserID, "userid", -1, "switch to this targetUser to e.g. drop privileges") - - execCmd := &cobra.Command{ - Use: "exec", -@@ -136,16 +143,39 @@ func main() { - Args: cobra.MinimumNArgs(2), - RunE: func(cmd *cobra.Command, args []string) error { - var mntOpts uint = 0 -+ var dataOpts []string - - fsType := cmd.Flag("type").Value.String() - mntOptions := cmd.Flag("options").Value.String() -+ var ( -+ uid = -1 -+ gid = -1 -+ ) - for _, opt := range strings.Split(mntOptions, ",") { - opt = strings.TrimSpace(opt) -- switch opt { -- case "ro": -+ switch { -+ case opt == "ro": - mntOpts = mntOpts | syscall.MS_RDONLY -- case "bind": -+ case opt == "bind": - mntOpts = mntOpts | syscall.MS_BIND -+ case opt == "remount": -+ mntOpts = mntOpts | syscall.MS_REMOUNT -+ case strings.HasPrefix(opt, "uid="): -+ uidS := strings.TrimPrefix(opt, "uid=") -+ uidI, err := strconv.Atoi(uidS) -+ if err != nil { -+ return fmt.Errorf("failed to parse uid: %w", err) -+ } -+ uid = uidI -+ dataOpts = append(dataOpts, opt) -+ case strings.HasPrefix(opt, "gid="): -+ gidS := strings.TrimPrefix(opt, "gid=") -+ gidI, err := strconv.Atoi(gidS) -+ if err != nil { -+ return fmt.Errorf("failed to parse gid: %w", err) -+ } -+ gid = gidI -+ dataOpts = append(dataOpts, opt) - default: - return fmt.Errorf("mount option %s is not supported", opt) - } -@@ -168,8 +198,17 @@ func main() { - return fmt.Errorf("mount target invalid: %v", err) - } - defer targetFile.Close() -- -- return syscall.Mount(sourceFile.SafePath(), targetFile.SafePath(), fsType, uintptr(mntOpts), "") -+ if uid >= 0 && gid >= 0 { -+ err = os.Chown(targetFile.SafePath(), uid, gid) -+ if err != nil { -+ return fmt.Errorf("chown target failed: %w", err) -+ } -+ } -+ var data string -+ if len(dataOpts) > 0 { -+ data = strings.Join(dataOpts, ",") -+ } -+ return syscall.Mount(sourceFile.SafePath(), targetFile.SafePath(), fsType, uintptr(mntOpts), data) - }, - } - mntCmd.Flags().StringP("options", "o", "", "comma separated list of mount options") -diff --git a/manifests/generated/kv-resource.yaml b/manifests/generated/kv-resource.yaml -index 66d1b01dbf..43e36b7195 100644 ---- a/manifests/generated/kv-resource.yaml -+++ b/manifests/generated/kv-resource.yaml -@@ -3307,9 +3307,6 @@ spec: - - jsonPath: .status.phase - name: Phase - type: string -- deprecated: true -- deprecationWarning: kubevirt.io/v1alpha3 is now deprecated and will be removed -- in a future release. - name: v1alpha3 - schema: - openAPIV3Schema: -diff --git a/manifests/generated/operator-csv.yaml.in b/manifests/generated/operator-csv.yaml.in -index 400d118024..05ee099c67 100644 ---- a/manifests/generated/operator-csv.yaml.in -+++ b/manifests/generated/operator-csv.yaml.in -@@ -605,6 +605,13 @@ spec: - - '*' - verbs: - - '*' -+ - apiGroups: -+ - subresources.virtualization.deckhouse.io -+ resources: -+ - virtualmachines/addvolume -+ - virtualmachines/removevolume -+ verbs: -+ - update - - apiGroups: - - subresources.kubevirt.io - resources: -diff --git a/manifests/generated/rbac-operator.authorization.k8s.yaml.in b/manifests/generated/rbac-operator.authorization.k8s.yaml.in -index 10dbb92269..1ccc9e9fa7 100644 ---- a/manifests/generated/rbac-operator.authorization.k8s.yaml.in -+++ b/manifests/generated/rbac-operator.authorization.k8s.yaml.in -@@ -143,7 +143,7 @@ kind: RoleBinding - metadata: - labels: - kubevirt.io: "" -- name: kubevirt-operator-rolebinding -+ name: kubevirt-operator - namespace: {{.Namespace}} - roleRef: - apiGroup: rbac.authorization.k8s.io -@@ -607,6 +607,13 @@ rules: - - '*' - verbs: - - '*' -+- apiGroups: -+ - subresources.virtualization.deckhouse.io -+ resources: -+ - virtualmachines/addvolume -+ - virtualmachines/removevolume -+ verbs: -+ - update - - apiGroups: - - subresources.kubevirt.io - resources: -diff --git a/pkg/container-disk/container-disk.go b/pkg/container-disk/container-disk.go -index 3251d04787..34affe841a 100644 ---- a/pkg/container-disk/container-disk.go -+++ b/pkg/container-disk/container-disk.go -@@ -47,8 +47,10 @@ var containerDiskOwner = "qemu" - var podsBaseDir = util.KubeletPodsDir - - var mountBaseDir = filepath.Join(util.VirtShareDir, "/container-disks") -+var hotplugBaseDir = filepath.Join(util.VirtShareDir, "/hotplug-disks") - - type SocketPathGetter func(vmi *v1.VirtualMachineInstance, volumeIndex int) (string, error) -+type HotplugSocketPathGetter func(vmi *v1.VirtualMachineInstance, volumeName string) (string, error) - type KernelBootSocketPathGetter func(vmi *v1.VirtualMachineInstance) (string, error) - - const KernelBootName = "kernel-boot" -@@ -107,6 +109,10 @@ func GetDiskTargetPathFromLauncherView(volumeIndex int) string { - return filepath.Join(mountBaseDir, GetDiskTargetName(volumeIndex)) - } - -+func GetHotplugContainerDiskTargetPathFromLauncherView(volumeName string) string { -+ return filepath.Join(hotplugBaseDir, fmt.Sprintf("%s.img", volumeName)) -+} -+ - func GetKernelBootArtifactPathFromLauncherView(artifact string) string { - artifactBase := filepath.Base(artifact) - return filepath.Join(mountBaseDir, KernelBootName, artifactBase) -@@ -170,6 +176,23 @@ func NewSocketPathGetter(baseDir string) SocketPathGetter { - } - } - -+func NewHotplugSocketPathGetter(baseDir string) HotplugSocketPathGetter { -+ return func(vmi *v1.VirtualMachineInstance, volumeName string) (string, error) { -+ for _, v := range vmi.Status.VolumeStatus { -+ if v.Name == volumeName && v.HotplugVolume != nil && v.ContainerDiskVolume != nil { -+ basePath := getHotplugContainerDiskSocketBasePath(baseDir, string(v.HotplugVolume.AttachPodUID)) -+ socketPath := filepath.Join(basePath, fmt.Sprintf("hotplug-container-disk-%s.sock", volumeName)) -+ exists, _ := diskutils.FileExists(socketPath) -+ if exists { -+ return socketPath, nil -+ } -+ } -+ } -+ -+ return "", fmt.Errorf("container disk socket path not found for vmi \"%s\"", vmi.Name) -+ } -+} -+ - // NewKernelBootSocketPathGetter get the socket pat of the kernel-boot containerDisk. For testing a baseDir - // can be provided which can for instance point to /tmp. - func NewKernelBootSocketPathGetter(baseDir string) KernelBootSocketPathGetter { -@@ -394,10 +417,37 @@ func CreateEphemeralImages( - return nil - } - -+func CreateEphemeralImagesForHotplug( -+ vmi *v1.VirtualMachineInstance, -+ diskCreator ephemeraldisk.EphemeralDiskCreatorInterface, -+ disksInfo map[string]*DiskInfo, -+) error { -+ for i, volume := range vmi.Spec.Volumes { -+ if volume.VolumeSource.ContainerDisk != nil && volume.VolumeSource.ContainerDisk.Hotpluggable { -+ info, _ := disksInfo[volume.Name] -+ if info == nil { -+ return fmt.Errorf("no disk info provided for volume %s", volume.Name) -+ } -+ -+ if backingFile, err := GetDiskTargetPartFromLauncherView(i); err != nil { -+ return err -+ } else if err := diskCreator.CreateBackedImageForVolume(volume, backingFile, info.Format); err != nil { -+ return err -+ } -+ } -+ } -+ -+ return nil -+} -+ - func getContainerDiskSocketBasePath(baseDir, podUID string) string { - return fmt.Sprintf("%s/pods/%s/volumes/kubernetes.io~empty-dir/container-disks", baseDir, podUID) - } - -+func getHotplugContainerDiskSocketBasePath(baseDir, podUID string) string { -+ return fmt.Sprintf("%s/pods/%s/volumes/kubernetes.io~empty-dir/hotplug-container-disks", baseDir, podUID) -+} -+ - // ExtractImageIDsFromSourcePod takes the VMI and its source pod to determine the exact image used by containerdisks and boot container images, - // which is recorded in the status section of a started pod; if the status section does not contain this info the tag is used. - // It returns a map where the key is the vlume name and the value is the imageID -diff --git a/pkg/controller/controller.go b/pkg/controller/controller.go -index 490cc445ef..4b7dbc12fe 100644 ---- a/pkg/controller/controller.go -+++ b/pkg/controller/controller.go -@@ -278,6 +278,10 @@ func ApplyVolumeRequestOnVMISpec(vmiSpec *v1.VirtualMachineInstanceSpec, request - dvSource := request.AddVolumeOptions.VolumeSource.DataVolume.DeepCopy() - dvSource.Hotpluggable = true - newVolume.VolumeSource.DataVolume = dvSource -+ } else if request.AddVolumeOptions.VolumeSource.ContainerDisk != nil { -+ containerDiskSource := request.AddVolumeOptions.VolumeSource.ContainerDisk.DeepCopy() -+ containerDiskSource.Hotpluggable = true -+ newVolume.VolumeSource.ContainerDisk = containerDiskSource - } - - vmiSpec.Volumes = append(vmiSpec.Volumes, newVolume) -@@ -444,6 +448,9 @@ func VMIHasHotplugVolumes(vmi *v1.VirtualMachineInstance) bool { - if volume.PersistentVolumeClaim != nil && volume.PersistentVolumeClaim.Hotpluggable { - return true - } -+ if volume.ContainerDisk != nil && volume.ContainerDisk.Hotpluggable { -+ return true -+ } - } - return false - } -@@ -557,7 +564,7 @@ func GetHotplugVolumes(vmi *v1.VirtualMachineInstance, virtlauncherPod *k8sv1.Po - podVolumeMap[podVolume.Name] = podVolume - } - for _, vmiVolume := range vmiVolumes { -- if _, ok := podVolumeMap[vmiVolume.Name]; !ok && (vmiVolume.DataVolume != nil || vmiVolume.PersistentVolumeClaim != nil || vmiVolume.MemoryDump != nil) { -+ if _, ok := podVolumeMap[vmiVolume.Name]; !ok && (vmiVolume.DataVolume != nil || vmiVolume.PersistentVolumeClaim != nil || vmiVolume.MemoryDump != nil || vmiVolume.ContainerDisk != nil) { - hotplugVolumes = append(hotplugVolumes, vmiVolume.DeepCopy()) - } - } -diff --git a/pkg/virt-api/rest/subresource.go b/pkg/virt-api/rest/subresource.go -index b5d62f5af5..bf561f00ae 100644 ---- a/pkg/virt-api/rest/subresource.go -+++ b/pkg/virt-api/rest/subresource.go -@@ -1023,7 +1023,8 @@ func volumeSourceName(volumeSource *v1.HotplugVolumeSource) string { - - func volumeSourceExists(volume v1.Volume, volumeName string) bool { - return (volume.DataVolume != nil && volume.DataVolume.Name == volumeName) || -- (volume.PersistentVolumeClaim != nil && volume.PersistentVolumeClaim.ClaimName == volumeName) -+ (volume.PersistentVolumeClaim != nil && volume.PersistentVolumeClaim.ClaimName == volumeName) || -+ (volume.ContainerDisk != nil && volume.ContainerDisk.Image != "") - } - - func volumeExists(volume v1.Volume, volumeName string) bool { -@@ -1125,6 +1126,8 @@ func (app *SubresourceAPIApp) addVolumeRequestHandler(request *restful.Request, - opts.VolumeSource.DataVolume.Hotpluggable = true - } else if opts.VolumeSource.PersistentVolumeClaim != nil { - opts.VolumeSource.PersistentVolumeClaim.Hotpluggable = true -+ } else if opts.VolumeSource.ContainerDisk != nil { -+ opts.VolumeSource.ContainerDisk.Hotpluggable = true - } - - // inject into VMI if ephemeral, else set as a request on the VM to both make permanent and hotplug. -diff --git a/pkg/virt-api/webhooks/validating-webhook/admitters/vmi-update-admitter.go b/pkg/virt-api/webhooks/validating-webhook/admitters/vmi-update-admitter.go -index 0af25f8074..803c0ed4cd 100644 ---- a/pkg/virt-api/webhooks/validating-webhook/admitters/vmi-update-admitter.go -+++ b/pkg/virt-api/webhooks/validating-webhook/admitters/vmi-update-admitter.go -@@ -200,11 +200,11 @@ func verifyHotplugVolumes(newHotplugVolumeMap, oldHotplugVolumeMap map[string]v1 - } - } else { - // This is a new volume, ensure that the volume is either DV, PVC or memoryDumpVolume -- if v.DataVolume == nil && v.PersistentVolumeClaim == nil && v.MemoryDump == nil { -+ if v.DataVolume == nil && v.PersistentVolumeClaim == nil && v.MemoryDump == nil && v.ContainerDisk == nil { - return webhookutils.ToAdmissionResponse([]metav1.StatusCause{ - { - Type: metav1.CauseTypeFieldValueInvalid, -- Message: fmt.Sprintf("volume %s is not a PVC or DataVolume", k), -+ Message: fmt.Sprintf("volume %s is not a PVC,DataVolume,MemoryDumpVolume or ContainerDisk", k), - }, - }) - } -diff --git a/pkg/virt-controller/services/template.go b/pkg/virt-controller/services/template.go -index 76ed7307ec..f607c24786 100644 ---- a/pkg/virt-controller/services/template.go -+++ b/pkg/virt-controller/services/template.go -@@ -64,13 +64,15 @@ import ( - ) - - const ( -- containerDisks = "container-disks" -- hotplugDisks = "hotplug-disks" -- hookSidecarSocks = "hook-sidecar-sockets" -- varRun = "/var/run" -- virtBinDir = "virt-bin-share-dir" -- hotplugDisk = "hotplug-disk" -- virtExporter = "virt-exporter" -+ containerDisks = "container-disks" -+ hotplugDisks = "hotplug-disks" -+ hookSidecarSocks = "hook-sidecar-sockets" -+ varRun = "/var/run" -+ virtBinDir = "virt-bin-share-dir" -+ hotplugDisk = "hotplug-disk" -+ virtExporter = "virt-exporter" -+ hotplugContainerDisks = "hotplug-container-disks" -+ HotplugContainerDisk = "hotplug-container-disk-" - ) - - const KvmDevice = "devices.virtualization.deckhouse.io/kvm" -@@ -846,6 +848,49 @@ func sidecarContainerName(i int) string { - return fmt.Sprintf("hook-sidecar-%d", i) - } - -+func sidecarContainerHotplugContainerdDiskName(name string) string { -+ return fmt.Sprintf("%s%s", HotplugContainerDisk, name) -+} -+ -+func (t *templateService) containerForHotplugContainerDisk(name string, cd *v1.ContainerDiskSource, vmi *v1.VirtualMachineInstance) k8sv1.Container { -+ runUser := int64(util.NonRootUID) -+ sharedMount := k8sv1.MountPropagationHostToContainer -+ path := fmt.Sprintf("/path/%s", name) -+ command := []string{"/init/usr/bin/container-disk"} -+ args := []string{"--copy-path", path} -+ -+ return k8sv1.Container{ -+ Name: name, -+ Image: cd.Image, -+ Command: command, -+ Args: args, -+ Resources: hotplugContainerResourceRequirementsForVMI(vmi, t.clusterConfig), -+ SecurityContext: &k8sv1.SecurityContext{ -+ AllowPrivilegeEscalation: pointer.Bool(false), -+ RunAsNonRoot: pointer.Bool(true), -+ RunAsUser: &runUser, -+ SeccompProfile: &k8sv1.SeccompProfile{ -+ Type: k8sv1.SeccompProfileTypeRuntimeDefault, -+ }, -+ Capabilities: &k8sv1.Capabilities{ -+ Drop: []k8sv1.Capability{"ALL"}, -+ }, -+ SELinuxOptions: &k8sv1.SELinuxOptions{ -+ Type: t.clusterConfig.GetSELinuxLauncherType(), -+ Level: "s0", -+ }, -+ }, -+ VolumeMounts: []k8sv1.VolumeMount{ -+ initContainerVolumeMount(), -+ { -+ Name: hotplugContainerDisks, -+ MountPath: "/path", -+ MountPropagation: &sharedMount, -+ }, -+ }, -+ } -+} -+ - func (t *templateService) RenderHotplugAttachmentPodTemplate(volumes []*v1.Volume, ownerPod *k8sv1.Pod, vmi *v1.VirtualMachineInstance, claimMap map[string]*k8sv1.PersistentVolumeClaim) (*k8sv1.Pod, error) { - zero := int64(0) - runUser := int64(util.NonRootUID) -@@ -924,6 +969,30 @@ func (t *templateService) RenderHotplugAttachmentPodTemplate(volumes []*v1.Volum - TerminationGracePeriodSeconds: &zero, - }, - } -+ first := true -+ for _, vol := range vmi.Spec.Volumes { -+ if vol.ContainerDisk == nil || !vol.ContainerDisk.Hotpluggable { -+ continue -+ } -+ name := sidecarContainerHotplugContainerdDiskName(vol.Name) -+ pod.Spec.Containers = append(pod.Spec.Containers, t.containerForHotplugContainerDisk(name, vol.ContainerDisk, vmi)) -+ if first { -+ first = false -+ userId := int64(util.NonRootUID) -+ initContainerCommand := []string{"/usr/bin/cp", -+ "/usr/bin/container-disk", -+ "/init/usr/bin/container-disk", -+ } -+ pod.Spec.InitContainers = append( -+ pod.Spec.InitContainers, -+ t.newInitContainerRenderer(vmi, -+ initContainerVolumeMount(), -+ initContainerResourceRequirementsForVMI(vmi, v1.ContainerDisk, t.clusterConfig), -+ userId).Render(initContainerCommand)) -+ pod.Spec.Volumes = append(pod.Spec.Volumes, emptyDirVolume(hotplugContainerDisks)) -+ pod.Spec.Volumes = append(pod.Spec.Volumes, emptyDirVolume(virtBinDir)) -+ } -+ } - - err := matchSELinuxLevelOfVMI(pod, vmi) - if err != nil { -diff --git a/pkg/virt-controller/watch/BUILD.bazel b/pkg/virt-controller/watch/BUILD.bazel -index 4fd325ba86..82fcaee0a3 100644 ---- a/pkg/virt-controller/watch/BUILD.bazel -+++ b/pkg/virt-controller/watch/BUILD.bazel -@@ -101,6 +101,7 @@ go_library( - "//vendor/k8s.io/client-go/util/flowcontrol:go_default_library", - "//vendor/k8s.io/client-go/util/workqueue:go_default_library", - "//vendor/k8s.io/utils/pointer:go_default_library", -+ "//vendor/k8s.io/utils/ptr:go_default_library", - "//vendor/k8s.io/utils/trace:go_default_library", - "//vendor/kubevirt.io/containerized-data-importer-api/pkg/apis/core/v1beta1:go_default_library", - ], -diff --git a/pkg/virt-controller/watch/vmi.go b/pkg/virt-controller/watch/vmi.go -index fa4e86ee17..ebe718f90d 100644 ---- a/pkg/virt-controller/watch/vmi.go -+++ b/pkg/virt-controller/watch/vmi.go -@@ -1836,6 +1836,10 @@ func (c *VMIController) handleHotplugVolumes(hotplugVolumes []*virtv1.Volume, ho - readyHotplugVolumes := make([]*virtv1.Volume, 0) - // Find all ready volumes - for _, volume := range hotplugVolumes { -+ if volume.ContainerDisk != nil { -+ readyHotplugVolumes = append(readyHotplugVolumes, volume) -+ continue -+ } - var err error - ready, wffc, err := storagetypes.VolumeReadyToAttachToNode(vmi.Namespace, *volume, dataVolumes, c.dataVolumeIndexer, c.pvcIndexer) - if err != nil { -@@ -1884,7 +1888,15 @@ func (c *VMIController) handleHotplugVolumes(hotplugVolumes []*virtv1.Volume, ho - - func (c *VMIController) podVolumesMatchesReadyVolumes(attachmentPod *k8sv1.Pod, volumes []*virtv1.Volume) bool { - // -2 for empty dir and token -- if len(attachmentPod.Spec.Volumes)-2 != len(volumes) { -+ // -3 if exist container-disk -+ magicNum := len(attachmentPod.Spec.Volumes) - 2 -+ for _, volume := range volumes { -+ if volume.ContainerDisk != nil { -+ magicNum -= 1 -+ break -+ } -+ } -+ if magicNum != len(volumes) { - return false - } - podVolumeMap := make(map[string]k8sv1.Volume) -@@ -1893,10 +1905,20 @@ func (c *VMIController) podVolumesMatchesReadyVolumes(attachmentPod *k8sv1.Pod, - podVolumeMap[volume.Name] = volume - } - } -+ containerDisksNames := make(map[string]struct{}) -+ for _, ctr := range attachmentPod.Spec.Containers { -+ if strings.HasPrefix(ctr.Name, services.HotplugContainerDisk) { -+ containerDisksNames[strings.TrimPrefix(ctr.Name, services.HotplugContainerDisk)] = struct{}{} -+ } -+ } - for _, volume := range volumes { -+ if volume.ContainerDisk != nil { -+ delete(containerDisksNames, volume.Name) -+ continue -+ } - delete(podVolumeMap, volume.Name) - } -- return len(podVolumeMap) == 0 -+ return len(podVolumeMap) == 0 && len(containerDisksNames) == 0 - } - - func (c *VMIController) createAttachmentPod(vmi *virtv1.VirtualMachineInstance, virtLauncherPod *k8sv1.Pod, volumes []*virtv1.Volume) (*k8sv1.Pod, syncError) { -@@ -2007,7 +2029,17 @@ func (c *VMIController) createAttachmentPodTemplate(vmi *virtv1.VirtualMachineIn - var pod *k8sv1.Pod - var err error - -- volumeNamesPVCMap, err := storagetypes.VirtVolumesToPVCMap(volumes, c.pvcIndexer, virtlauncherPod.Namespace) -+ var hasContainerDisk bool -+ var newVolumes []*virtv1.Volume -+ for _, volume := range volumes { -+ if volume.VolumeSource.ContainerDisk != nil { -+ hasContainerDisk = true -+ continue -+ } -+ newVolumes = append(newVolumes, volume) -+ } -+ -+ volumeNamesPVCMap, err := storagetypes.VirtVolumesToPVCMap(newVolumes, c.pvcIndexer, virtlauncherPod.Namespace) - if err != nil { - return nil, fmt.Errorf("failed to get PVC map: %v", err) - } -@@ -2029,7 +2061,7 @@ func (c *VMIController) createAttachmentPodTemplate(vmi *virtv1.VirtualMachineIn - } - } - -- if len(volumeNamesPVCMap) > 0 { -+ if len(volumeNamesPVCMap) > 0 || hasContainerDisk { - pod, err = c.templateService.RenderHotplugAttachmentPodTemplate(volumes, virtlauncherPod, vmi, volumeNamesPVCMap) - } - return pod, err -@@ -2151,23 +2183,39 @@ func (c *VMIController) updateVolumeStatus(vmi *virtv1.VirtualMachineInstance, v - ClaimName: volume.Name, - } - } -+ if volume.ContainerDisk != nil && status.ContainerDiskVolume == nil { -+ status.ContainerDiskVolume = &virtv1.ContainerDiskInfo{} -+ } - if attachmentPod == nil { -- if !c.volumeReady(status.Phase) { -- status.HotplugVolume.AttachPodUID = "" -- // Volume is not hotplugged in VM and Pod is gone, or hasn't been created yet, check for the PVC associated with the volume to set phase and message -- phase, reason, message := c.getVolumePhaseMessageReason(&vmi.Spec.Volumes[i], vmi.Namespace) -- status.Phase = phase -- status.Message = message -- status.Reason = reason -+ if volume.ContainerDisk != nil { -+ if !c.volumeReady(status.Phase) { -+ status.HotplugVolume.AttachPodUID = "" -+ // Volume is not hotplugged in VM and Pod is gone, or hasn't been created yet, check for the PVC associated with the volume to set phase and message -+ phase, reason, message := c.getVolumePhaseMessageReason(&vmi.Spec.Volumes[i], vmi.Namespace) -+ status.Phase = phase -+ status.Message = message -+ status.Reason = reason -+ } - } - } else { - status.HotplugVolume.AttachPodName = attachmentPod.Name -- if len(attachmentPod.Status.ContainerStatuses) == 1 && attachmentPod.Status.ContainerStatuses[0].Ready { -+ if volume.ContainerDisk != nil { -+ uid := types.UID("") -+ for _, cs := range attachmentPod.Status.ContainerStatuses { -+ name := strings.TrimPrefix(cs.Name, "hotplug-container-disk-") -+ if volume.Name == name && cs.Ready { -+ uid = attachmentPod.UID -+ break -+ } -+ } -+ status.HotplugVolume.AttachPodUID = uid -+ } else if len(attachmentPod.Status.ContainerStatuses) == 1 && attachmentPod.Status.ContainerStatuses[0].Ready { - status.HotplugVolume.AttachPodUID = attachmentPod.UID - } else { - // Remove UID of old pod if a new one is available, but not yet ready - status.HotplugVolume.AttachPodUID = "" - } -+ - if c.canMoveToAttachedPhase(status.Phase) { - status.Phase = virtv1.HotplugVolumeAttachedToNode - status.Message = fmt.Sprintf("Created hotplug attachment pod %s, for volume %s", attachmentPod.Name, volume.Name) -@@ -2176,7 +2224,6 @@ func (c *VMIController) updateVolumeStatus(vmi *virtv1.VirtualMachineInstance, v - } - } - } -- - if volume.VolumeSource.PersistentVolumeClaim != nil || volume.VolumeSource.DataVolume != nil || volume.VolumeSource.MemoryDump != nil { - - pvcName := storagetypes.PVCNameFromVirtVolume(&volume) -diff --git a/pkg/virt-handler/container-disk/hotplug.go b/pkg/virt-handler/container-disk/hotplug.go -new file mode 100644 -index 0000000000..f0d3a0607c ---- /dev/null -+++ b/pkg/virt-handler/container-disk/hotplug.go -@@ -0,0 +1,481 @@ -+package container_disk -+ -+import ( -+ "encoding/json" -+ "errors" -+ "fmt" -+ "os" -+ "path/filepath" -+ "strings" -+ "sync" -+ "time" -+ -+ hotplugdisk "kubevirt.io/kubevirt/pkg/hotplug-disk" -+ "kubevirt.io/kubevirt/pkg/unsafepath" -+ -+ "kubevirt.io/kubevirt/pkg/safepath" -+ virtconfig "kubevirt.io/kubevirt/pkg/virt-config" -+ virt_chroot "kubevirt.io/kubevirt/pkg/virt-handler/virt-chroot" -+ -+ "kubevirt.io/client-go/log" -+ -+ containerdisk "kubevirt.io/kubevirt/pkg/container-disk" -+ diskutils "kubevirt.io/kubevirt/pkg/ephemeral-disk-utils" -+ "kubevirt.io/kubevirt/pkg/virt-handler/isolation" -+ -+ "k8s.io/apimachinery/pkg/api/equality" -+ "k8s.io/apimachinery/pkg/types" -+ -+ v1 "kubevirt.io/api/core/v1" -+) -+ -+type HotplugMounter interface { -+ ContainerDisksReady(vmi *v1.VirtualMachineInstance, notInitializedSince time.Time) (bool, error) -+ MountAndVerify(vmi *v1.VirtualMachineInstance) (map[string]*containerdisk.DiskInfo, error) -+ IsMounted(vmi *v1.VirtualMachineInstance, volumeName string) (bool, error) -+ Umount(vmi *v1.VirtualMachineInstance) error -+ UmountAll(vmi *v1.VirtualMachineInstance) error -+ ComputeChecksums(vmi *v1.VirtualMachineInstance) (*DiskChecksums, error) -+} -+ -+type hotplugMounter struct { -+ podIsolationDetector isolation.PodIsolationDetector -+ mountStateDir string -+ mountRecords map[types.UID]*vmiMountTargetRecord -+ mountRecordsLock sync.Mutex -+ suppressWarningTimeout time.Duration -+ clusterConfig *virtconfig.ClusterConfig -+ nodeIsolationResult isolation.IsolationResult -+ -+ hotplugPathGetter containerdisk.HotplugSocketPathGetter -+ hotplugManager hotplugdisk.HotplugDiskManagerInterface -+} -+ -+func (m *hotplugMounter) IsMounted(vmi *v1.VirtualMachineInstance, volumeName string) (bool, error) { -+ virtLauncherUID := m.findVirtlauncherUID(vmi) -+ if virtLauncherUID == "" { -+ return false, nil -+ } -+ target, err := m.hotplugManager.GetFileSystemDiskTargetPathFromHostView(virtLauncherUID, volumeName, false) -+ if err != nil { -+ return false, err -+ } -+ return isolation.IsMounted(target) -+} -+ -+func NewHotplugMounter(isoDetector isolation.PodIsolationDetector, -+ mountStateDir string, -+ clusterConfig *virtconfig.ClusterConfig, -+ hotplugManager hotplugdisk.HotplugDiskManagerInterface, -+) HotplugMounter { -+ return &hotplugMounter{ -+ mountRecords: make(map[types.UID]*vmiMountTargetRecord), -+ podIsolationDetector: isoDetector, -+ mountStateDir: mountStateDir, -+ suppressWarningTimeout: 1 * time.Minute, -+ clusterConfig: clusterConfig, -+ nodeIsolationResult: isolation.NodeIsolationResult(), -+ -+ hotplugPathGetter: containerdisk.NewHotplugSocketPathGetter(""), -+ hotplugManager: hotplugManager, -+ } -+} -+ -+func (m *hotplugMounter) deleteMountTargetRecord(vmi *v1.VirtualMachineInstance) error { -+ if string(vmi.UID) == "" { -+ return fmt.Errorf("unable to find container disk mounted directories for vmi without uid") -+ } -+ -+ recordFile := filepath.Join(m.mountStateDir, string(vmi.UID)) -+ -+ exists, err := diskutils.FileExists(recordFile) -+ if err != nil { -+ return err -+ } -+ -+ if exists { -+ record, err := m.getMountTargetRecord(vmi) -+ if err != nil { -+ return err -+ } -+ -+ for _, target := range record.MountTargetEntries { -+ os.Remove(target.TargetFile) -+ os.Remove(target.SocketFile) -+ } -+ -+ os.Remove(recordFile) -+ } -+ -+ m.mountRecordsLock.Lock() -+ defer m.mountRecordsLock.Unlock() -+ delete(m.mountRecords, vmi.UID) -+ -+ return nil -+} -+ -+func (m *hotplugMounter) getMountTargetRecord(vmi *v1.VirtualMachineInstance) (*vmiMountTargetRecord, error) { -+ var ok bool -+ var existingRecord *vmiMountTargetRecord -+ -+ if string(vmi.UID) == "" { -+ return nil, fmt.Errorf("unable to find container disk mounted directories for vmi without uid") -+ } -+ -+ m.mountRecordsLock.Lock() -+ defer m.mountRecordsLock.Unlock() -+ existingRecord, ok = m.mountRecords[vmi.UID] -+ -+ // first check memory cache -+ if ok { -+ return existingRecord, nil -+ } -+ -+ // if not there, see if record is on disk, this can happen if virt-handler restarts -+ recordFile := filepath.Join(m.mountStateDir, filepath.Clean(string(vmi.UID))) -+ -+ exists, err := diskutils.FileExists(recordFile) -+ if err != nil { -+ return nil, err -+ } -+ -+ if exists { -+ record := vmiMountTargetRecord{} -+ // #nosec No risk for path injection. Using static base and cleaned filename -+ bytes, err := os.ReadFile(recordFile) -+ if err != nil { -+ return nil, err -+ } -+ err = json.Unmarshal(bytes, &record) -+ if err != nil { -+ return nil, err -+ } -+ -+ if !record.UsesSafePaths { -+ record.UsesSafePaths = true -+ for i, entry := range record.MountTargetEntries { -+ safePath, err := safepath.JoinAndResolveWithRelativeRoot("/", entry.TargetFile) -+ if err != nil { -+ return nil, fmt.Errorf("failed converting legacy path to safepath: %v", err) -+ } -+ record.MountTargetEntries[i].TargetFile = unsafepath.UnsafeAbsolute(safePath.Raw()) -+ } -+ } -+ -+ m.mountRecords[vmi.UID] = &record -+ return &record, nil -+ } -+ -+ // not found -+ return nil, nil -+} -+ -+func (m *hotplugMounter) addMountTargetRecord(vmi *v1.VirtualMachineInstance, record *vmiMountTargetRecord) error { -+ return m.setAddMountTargetRecordHelper(vmi, record, true) -+} -+ -+func (m *hotplugMounter) setMountTargetRecord(vmi *v1.VirtualMachineInstance, record *vmiMountTargetRecord) error { -+ return m.setAddMountTargetRecordHelper(vmi, record, false) -+} -+ -+func (m *hotplugMounter) setAddMountTargetRecordHelper(vmi *v1.VirtualMachineInstance, record *vmiMountTargetRecord, addPreviousRules bool) error { -+ if string(vmi.UID) == "" { -+ return fmt.Errorf("unable to set container disk mounted directories for vmi without uid") -+ } -+ -+ record.UsesSafePaths = true -+ -+ recordFile := filepath.Join(m.mountStateDir, string(vmi.UID)) -+ fileExists, err := diskutils.FileExists(recordFile) -+ if err != nil { -+ return err -+ } -+ -+ m.mountRecordsLock.Lock() -+ defer m.mountRecordsLock.Unlock() -+ -+ existingRecord, ok := m.mountRecords[vmi.UID] -+ if ok && fileExists && equality.Semantic.DeepEqual(existingRecord, record) { -+ // already done -+ return nil -+ } -+ -+ if addPreviousRules && existingRecord != nil && len(existingRecord.MountTargetEntries) > 0 { -+ record.MountTargetEntries = append(record.MountTargetEntries, existingRecord.MountTargetEntries...) -+ } -+ -+ bytes, err := json.Marshal(record) -+ if err != nil { -+ return err -+ } -+ -+ err = os.MkdirAll(filepath.Dir(recordFile), 0750) -+ if err != nil { -+ return err -+ } -+ -+ err = os.WriteFile(recordFile, bytes, 0600) -+ if err != nil { -+ return err -+ } -+ -+ m.mountRecords[vmi.UID] = record -+ -+ return nil -+} -+ -+func (m *hotplugMounter) MountAndVerify(vmi *v1.VirtualMachineInstance) (map[string]*containerdisk.DiskInfo, error) { -+ virtLauncherUID := m.findVirtlauncherUID(vmi) -+ if virtLauncherUID == "" { -+ return nil, nil -+ } -+ -+ record := vmiMountTargetRecord{} -+ disksInfo := map[string]*containerdisk.DiskInfo{} -+ -+ for _, volume := range vmi.Spec.Volumes { -+ if volume.ContainerDisk != nil && volume.ContainerDisk.Hotpluggable { -+ target, err := m.hotplugManager.GetFileSystemDiskTargetPathFromHostView(virtLauncherUID, volume.Name, true) -+ if err != nil { -+ return nil, err -+ } -+ -+ sock, err := m.hotplugPathGetter(vmi, volume.Name) -+ if err != nil { -+ return nil, err -+ } -+ -+ record.MountTargetEntries = append(record.MountTargetEntries, vmiMountTargetEntry{ -+ TargetFile: unsafepath.UnsafeAbsolute(target.Raw()), -+ SocketFile: sock, -+ }) -+ } -+ } -+ -+ if len(record.MountTargetEntries) > 0 { -+ err := m.setMountTargetRecord(vmi, &record) -+ if err != nil { -+ return nil, err -+ } -+ } -+ -+ vmiRes, err := m.podIsolationDetector.Detect(vmi) -+ if err != nil { -+ return nil, fmt.Errorf("failed to detect VMI pod: %v", err) -+ } -+ -+ for _, volume := range vmi.Spec.Volumes { -+ if volume.ContainerDisk != nil && volume.ContainerDisk.Hotpluggable { -+ target, err := m.hotplugManager.GetFileSystemDiskTargetPathFromHostView(virtLauncherUID, volume.Name, false) -+ -+ if isMounted, err := isolation.IsMounted(target); err != nil { -+ return nil, fmt.Errorf("failed to determine if %s is already mounted: %v", target, err) -+ } else if !isMounted { -+ -+ sourceFile, err := m.getContainerDiskPath(vmi, &volume, volume.Name) -+ if err != nil { -+ return nil, fmt.Errorf("failed to find a sourceFile in containerDisk %v: %v", volume.Name, err) -+ } -+ -+ log.DefaultLogger().Object(vmi).Infof("Bind mounting container disk at %s to %s", sourceFile, target) -+ opts := []string{ -+ "bind", "ro", "uid=107", "gid=107", -+ } -+ err = virt_chroot.MountChrootWithOptions(sourceFile, target, opts...) -+ if err != nil { -+ return nil, fmt.Errorf("failed to bindmount containerDisk %v. err: %w", volume.Name, err) -+ } -+ } -+ -+ imageInfo, err := isolation.GetImageInfo( -+ containerdisk.GetHotplugContainerDiskTargetPathFromLauncherView(volume.Name), -+ vmiRes, -+ m.clusterConfig.GetDiskVerification(), -+ ) -+ if err != nil { -+ return nil, fmt.Errorf("failed to get image info: %v", err) -+ } -+ if err := containerdisk.VerifyImage(imageInfo); err != nil { -+ return nil, fmt.Errorf("invalid image in containerDisk %v: %v", volume.Name, err) -+ } -+ disksInfo[volume.Name] = imageInfo -+ } -+ } -+ -+ return disksInfo, nil -+} -+ -+func (m *hotplugMounter) Umount(vmi *v1.VirtualMachineInstance) error { -+ record, err := m.getMountTargetRecord(vmi) -+ if err != nil { -+ return err -+ } else if record == nil { -+ // no entries to unmount -+ -+ log.DefaultLogger().Object(vmi).Infof("No container disk mount entries found to unmount") -+ return nil -+ } -+ for _, r := range record.MountTargetEntries { -+ name, err := extractNameFromSocket(r.SocketFile) -+ if err != nil { -+ return err -+ } -+ needUmount := true -+ for _, v := range vmi.Status.VolumeStatus { -+ if v.Name == name { -+ needUmount = false -+ } -+ } -+ if needUmount { -+ file, err := safepath.NewFileNoFollow(r.TargetFile) -+ if err != nil { -+ if errors.Is(err, os.ErrNotExist) { -+ continue -+ } -+ return fmt.Errorf(failedCheckMountPointFmt, r.TargetFile, err) -+ } -+ _ = file.Close() -+ // #nosec No risk for attacket injection. Parameters are predefined strings -+ out, err := virt_chroot.UmountChroot(file.Path()).CombinedOutput() -+ if err != nil { -+ return fmt.Errorf(failedUnmountFmt, file, string(out), err) -+ } -+ } -+ } -+ return nil -+} -+ -+func extractNameFromSocket(socketFile string) (string, error) { -+ base := filepath.Base(socketFile) -+ if strings.HasPrefix(base, "hotplug-container-disk-") && strings.HasSuffix(base, ".sock") { -+ name := strings.TrimPrefix(base, "hotplug-container-disk-") -+ name = strings.TrimSuffix(name, ".sock") -+ return name, nil -+ } -+ return "", fmt.Errorf("name not found in path") -+} -+ -+func (m *hotplugMounter) UmountAll(vmi *v1.VirtualMachineInstance) error { -+ if vmi.UID == "" { -+ return nil -+ } -+ -+ record, err := m.getMountTargetRecord(vmi) -+ if err != nil { -+ return err -+ } else if record == nil { -+ // no entries to unmount -+ -+ log.DefaultLogger().Object(vmi).Infof("No container disk mount entries found to unmount") -+ return nil -+ } -+ -+ log.DefaultLogger().Object(vmi).Infof("Found container disk mount entries") -+ for _, entry := range record.MountTargetEntries { -+ log.DefaultLogger().Object(vmi).Infof("Looking to see if containerdisk is mounted at path %s", entry.TargetFile) -+ file, err := safepath.NewFileNoFollow(entry.TargetFile) -+ if err != nil { -+ if errors.Is(err, os.ErrNotExist) { -+ continue -+ } -+ return fmt.Errorf(failedCheckMountPointFmt, entry.TargetFile, err) -+ } -+ _ = file.Close() -+ if mounted, err := isolation.IsMounted(file.Path()); err != nil { -+ return fmt.Errorf(failedCheckMountPointFmt, file, err) -+ } else if mounted { -+ log.DefaultLogger().Object(vmi).Infof("unmounting container disk at path %s", file) -+ // #nosec No risk for attacket injection. Parameters are predefined strings -+ out, err := virt_chroot.UmountChroot(file.Path()).CombinedOutput() -+ if err != nil { -+ return fmt.Errorf(failedUnmountFmt, file, string(out), err) -+ } -+ } -+ } -+ err = m.deleteMountTargetRecord(vmi) -+ if err != nil { -+ return err -+ } -+ -+ return nil -+} -+ -+func (m *hotplugMounter) ContainerDisksReady(vmi *v1.VirtualMachineInstance, notInitializedSince time.Time) (bool, error) { -+ for _, volume := range vmi.Spec.Volumes { -+ if volume.ContainerDisk != nil && volume.ContainerDisk.Hotpluggable { -+ _, err := m.hotplugPathGetter(vmi, volume.Name) -+ if err != nil { -+ log.DefaultLogger().Object(vmi).Reason(err).Infof("containerdisk %s not yet ready", volume.Name) -+ if time.Now().After(notInitializedSince.Add(m.suppressWarningTimeout)) { -+ return false, fmt.Errorf("containerdisk %s still not ready after one minute", volume.Name) -+ } -+ return false, nil -+ } -+ } -+ } -+ -+ log.DefaultLogger().Object(vmi).V(4).Info("all containerdisks are ready") -+ return true, nil -+} -+ -+func (m *hotplugMounter) getContainerDiskPath(vmi *v1.VirtualMachineInstance, volume *v1.Volume, volumeName string) (*safepath.Path, error) { -+ sock, err := m.hotplugPathGetter(vmi, volumeName) -+ if err != nil { -+ return nil, ErrDiskContainerGone -+ } -+ -+ res, err := m.podIsolationDetector.DetectForSocket(vmi, sock) -+ if err != nil { -+ return nil, fmt.Errorf("failed to detect socket for containerDisk %v: %v", volume.Name, err) -+ } -+ -+ mountPoint, err := isolation.ParentPathForRootMount(m.nodeIsolationResult, res) -+ if err != nil { -+ return nil, fmt.Errorf("failed to detect root mount point of containerDisk %v on the node: %v", volume.Name, err) -+ } -+ -+ return containerdisk.GetImage(mountPoint, volume.ContainerDisk.Path) -+} -+ -+func (m *hotplugMounter) ComputeChecksums(vmi *v1.VirtualMachineInstance) (*DiskChecksums, error) { -+ -+ diskChecksums := &DiskChecksums{ -+ ContainerDiskChecksums: map[string]uint32{}, -+ } -+ -+ for _, volume := range vmi.Spec.Volumes { -+ if volume.VolumeSource.ContainerDisk == nil || !volume.VolumeSource.ContainerDisk.Hotpluggable { -+ continue -+ } -+ -+ path, err := m.getContainerDiskPath(vmi, &volume, volume.Name) -+ if err != nil { -+ return nil, err -+ } -+ -+ checksum, err := getDigest(path) -+ if err != nil { -+ return nil, err -+ } -+ -+ diskChecksums.ContainerDiskChecksums[volume.Name] = checksum -+ } -+ -+ return diskChecksums, nil -+} -+ -+func (m *hotplugMounter) findVirtlauncherUID(vmi *v1.VirtualMachineInstance) (uid types.UID) { -+ cnt := 0 -+ for podUID := range vmi.Status.ActivePods { -+ _, err := m.hotplugManager.GetHotplugTargetPodPathOnHost(podUID) -+ if err == nil { -+ uid = podUID -+ cnt++ -+ } -+ } -+ if cnt == 1 { -+ return -+ } -+ // Either no pods, or multiple pods, skip. -+ return types.UID("") -+} -diff --git a/pkg/virt-handler/container-disk/mount.go b/pkg/virt-handler/container-disk/mount.go -index 953c20f3af..d99bec3a43 100644 ---- a/pkg/virt-handler/container-disk/mount.go -+++ b/pkg/virt-handler/container-disk/mount.go -@@ -54,6 +54,8 @@ type mounter struct { - kernelBootSocketPathGetter containerdisk.KernelBootSocketPathGetter - clusterConfig *virtconfig.ClusterConfig - nodeIsolationResult isolation.IsolationResult -+ -+ hotplugPathGetter containerdisk.HotplugSocketPathGetter - } - - type Mounter interface { -@@ -98,6 +100,8 @@ func NewMounter(isoDetector isolation.PodIsolationDetector, mountStateDir string - kernelBootSocketPathGetter: containerdisk.NewKernelBootSocketPathGetter(""), - clusterConfig: clusterConfig, - nodeIsolationResult: isolation.NodeIsolationResult(), -+ -+ hotplugPathGetter: containerdisk.NewHotplugSocketPathGetter(""), - } - } - -@@ -254,7 +258,7 @@ func (m *mounter) MountAndVerify(vmi *v1.VirtualMachineInstance) (map[string]*co - disksInfo := map[string]*containerdisk.DiskInfo{} - - for i, volume := range vmi.Spec.Volumes { -- if volume.ContainerDisk != nil { -+ if volume.ContainerDisk != nil && !volume.ContainerDisk.Hotpluggable { - diskTargetDir, err := containerdisk.GetDiskTargetDirFromHostView(vmi) - if err != nil { - return nil, err -@@ -296,7 +300,7 @@ func (m *mounter) MountAndVerify(vmi *v1.VirtualMachineInstance) (map[string]*co - } - - for i, volume := range vmi.Spec.Volumes { -- if volume.ContainerDisk != nil { -+ if volume.ContainerDisk != nil && !volume.ContainerDisk.Hotpluggable { - diskTargetDir, err := containerdisk.GetDiskTargetDirFromHostView(vmi) - if err != nil { - return nil, err -@@ -394,7 +398,7 @@ func (m *mounter) Unmount(vmi *v1.VirtualMachineInstance) error { - - func (m *mounter) ContainerDisksReady(vmi *v1.VirtualMachineInstance, notInitializedSince time.Time) (bool, error) { - for i, volume := range vmi.Spec.Volumes { -- if volume.ContainerDisk != nil { -+ if volume.ContainerDisk != nil && !volume.ContainerDisk.Hotpluggable { - _, err := m.socketPathGetter(vmi, i) - if err != nil { - log.DefaultLogger().Object(vmi).Reason(err).Infof("containerdisk %s not yet ready", volume.Name) -@@ -706,7 +710,7 @@ func (m *mounter) ComputeChecksums(vmi *v1.VirtualMachineInstance) (*DiskChecksu - - // compute for containerdisks - for i, volume := range vmi.Spec.Volumes { -- if volume.VolumeSource.ContainerDisk == nil { -+ if volume.VolumeSource.ContainerDisk == nil || volume.VolumeSource.ContainerDisk.Hotpluggable { - continue - } - -diff --git a/pkg/virt-handler/hotplug-disk/mount.go b/pkg/virt-handler/hotplug-disk/mount.go -index 971c8d55fc..034c3d8526 100644 ---- a/pkg/virt-handler/hotplug-disk/mount.go -+++ b/pkg/virt-handler/hotplug-disk/mount.go -@@ -343,7 +343,7 @@ func (m *volumeMounter) mountFromPod(vmi *v1.VirtualMachineInstance, sourceUID t - return err - } - for _, volumeStatus := range vmi.Status.VolumeStatus { -- if volumeStatus.HotplugVolume == nil { -+ if volumeStatus.HotplugVolume == nil || volumeStatus.ContainerDiskVolume != nil { - // Skip non hotplug volumes - continue - } -@@ -649,7 +649,7 @@ func (m *volumeMounter) Unmount(vmi *v1.VirtualMachineInstance, cgroupManager cg - return err - } - for _, volumeStatus := range vmi.Status.VolumeStatus { -- if volumeStatus.HotplugVolume == nil { -+ if volumeStatus.HotplugVolume == nil || volumeStatus.ContainerDiskVolume != nil { - continue - } - var path *safepath.Path -diff --git a/pkg/virt-handler/isolation/detector.go b/pkg/virt-handler/isolation/detector.go -index f83f96ead4..5e38c6cedd 100644 ---- a/pkg/virt-handler/isolation/detector.go -+++ b/pkg/virt-handler/isolation/detector.go -@@ -24,6 +24,8 @@ package isolation - import ( - "fmt" - "net" -+ "os" -+ "path" - "runtime" - "syscall" - "time" -@@ -207,12 +209,45 @@ func setProcessMemoryLockRLimit(pid int, size int64) error { - return nil - } - -+type deferFunc func() -+ -+func (s *socketBasedIsolationDetector) socketHack(socket string) (sock net.Conn, deferFunc deferFunc, err error) { -+ fn := func() {} -+ if len([]rune(socket)) <= 108 { -+ sock, err = net.DialTimeout("unix", socket, time.Duration(isolationDialTimeout)*time.Second) -+ fn = func() { -+ if err == nil { -+ sock.Close() -+ } -+ } -+ return sock, fn, err -+ } -+ base := path.Base(socket) -+ newPath := fmt.Sprintf("/tmp/%s", base) -+ if err = os.Symlink(socket, newPath); err != nil { -+ return nil, fn, err -+ } -+ sock, err = net.DialTimeout("unix", newPath, time.Duration(isolationDialTimeout)*time.Second) -+ fn = func() { -+ if err == nil { -+ sock.Close() -+ } -+ os.Remove(newPath) -+ } -+ return sock, fn, err -+} -+ - func (s *socketBasedIsolationDetector) getPid(socket string) (int, error) { -- sock, err := net.DialTimeout("unix", socket, time.Duration(isolationDialTimeout)*time.Second) -+ sock, defFn, err := s.socketHack(socket) -+ defer defFn() - if err != nil { - return -1, err - } -- defer sock.Close() -+ //sock, err := net.DialTimeout("unix", socket, time.Duration(isolationDialTimeout)*time.Second) -+ //if err != nil { -+ // return -1, err -+ //} -+ //defer sock.Close() - - ufile, err := sock.(*net.UnixConn).File() - if err != nil { -diff --git a/pkg/virt-handler/virt-chroot/virt-chroot.go b/pkg/virt-handler/virt-chroot/virt-chroot.go -index 4160212b7b..580b788acc 100644 ---- a/pkg/virt-handler/virt-chroot/virt-chroot.go -+++ b/pkg/virt-handler/virt-chroot/virt-chroot.go -@@ -20,7 +20,10 @@ - package virt_chroot - - import ( -+ "bytes" -+ "fmt" - "os/exec" -+ "slices" - "strings" - - "kubevirt.io/kubevirt/pkg/safepath" -@@ -48,6 +51,49 @@ func MountChroot(sourcePath, targetPath *safepath.Path, ro bool) *exec.Cmd { - return UnsafeMountChroot(trimProcPrefix(sourcePath), trimProcPrefix(targetPath), ro) - } - -+func MountChrootWithOptions(sourcePath, targetPath *safepath.Path, mountOptions ...string) error { -+ args := append(getBaseArgs(), "mount") -+ remountArgs := slices.Clone(args) -+ -+ mountOptions = slices.DeleteFunc(mountOptions, func(s string) bool { -+ return s == "remount" -+ }) -+ if len(mountOptions) > 0 { -+ opts := strings.Join(mountOptions, ",") -+ remountOpts := "remount," + opts -+ args = append(args, "-o", opts) -+ remountArgs = append(remountArgs, "-o", remountOpts) -+ } -+ -+ sp := trimProcPrefix(sourcePath) -+ tp := trimProcPrefix(targetPath) -+ args = append(args, sp, tp) -+ remountArgs = append(remountArgs, sp, tp) -+ -+ stdout := new(bytes.Buffer) -+ stderr := new(bytes.Buffer) -+ -+ cmd := exec.Command(binaryPath, args...) -+ cmd.Stdout = stdout -+ cmd.Stderr = stderr -+ err := cmd.Run() -+ if err != nil { -+ return fmt.Errorf("mount failed: %w, stdout: %s, stderr: %s", err, stdout.String(), stderr.String()) -+ } -+ -+ stdout = new(bytes.Buffer) -+ stderr = new(bytes.Buffer) -+ -+ remountCmd := exec.Command(binaryPath, remountArgs...) -+ cmd.Stdout = stdout -+ cmd.Stderr = stderr -+ err = remountCmd.Run() -+ if err != nil { -+ return fmt.Errorf("mount failed: %w, stdout: %s, stderr: %s", err, stdout.String(), stderr.String()) -+ } -+ return nil -+} -+ - // Deprecated: UnsafeMountChroot is used to connect to code which needs to be refactored - // to handle mounts securely. - func UnsafeMountChroot(sourcePath, targetPath string, ro bool) *exec.Cmd { -diff --git a/pkg/virt-handler/vm.go b/pkg/virt-handler/vm.go -index 24352cf6e9..f1de8d1149 100644 ---- a/pkg/virt-handler/vm.go -+++ b/pkg/virt-handler/vm.go -@@ -25,6 +25,7 @@ import ( - goerror "errors" - "fmt" - "io" -+ "maps" - "net" - "os" - "path/filepath" -@@ -247,6 +248,13 @@ func NewController( - vmiExpectations: controller.NewUIDTrackingControllerExpectations(controller.NewControllerExpectations()), - sriovHotplugExecutorPool: executor.NewRateLimitedExecutorPool(executor.NewExponentialLimitedBackoffCreator()), - ioErrorRetryManager: NewFailRetryManager("io-error-retry", 10*time.Second, 3*time.Minute, 30*time.Second), -+ -+ hotplugContainerDiskMounter: container_disk.NewHotplugMounter( -+ podIsolationDetector, -+ filepath.Join(virtPrivateDir, "hotplug-container-disk-mount-state"), -+ clusterConfig, -+ hotplugdisk.NewHotplugDiskManager(kubeletPodsDir), -+ ), - } - - _, err := vmiSourceInformer.AddEventHandler(cache.ResourceEventHandlerFuncs{ -@@ -342,6 +350,8 @@ type VirtualMachineController struct { - hostCpuModel string - vmiExpectations *controller.UIDTrackingControllerExpectations - ioErrorRetryManager *FailRetryManager -+ -+ hotplugContainerDiskMounter container_disk.HotplugMounter - } - - type virtLauncherCriticalSecurebootError struct { -@@ -876,7 +886,15 @@ func (d *VirtualMachineController) updateHotplugVolumeStatus(vmi *v1.VirtualMach - needsRefresh := false - if volumeStatus.Target == "" { - needsRefresh = true -- mounted, err := d.hotplugVolumeMounter.IsMounted(vmi, volumeStatus.Name, volumeStatus.HotplugVolume.AttachPodUID) -+ var ( -+ mounted bool -+ err error -+ ) -+ if volumeStatus.ContainerDiskVolume != nil { -+ mounted, err = d.hotplugContainerDiskMounter.IsMounted(vmi, volumeStatus.Name) -+ } else { -+ mounted, err = d.hotplugVolumeMounter.IsMounted(vmi, volumeStatus.Name, volumeStatus.HotplugVolume.AttachPodUID) -+ } - if err != nil { - log.Log.Object(vmi).Errorf("error occurred while checking if volume is mounted: %v", err) - } -@@ -898,6 +916,7 @@ func (d *VirtualMachineController) updateHotplugVolumeStatus(vmi *v1.VirtualMach - volumeStatus.Reason = VolumeUnMountedFromPodReason - } - } -+ - } else { - // Successfully attached to VM. - volumeStatus.Phase = v1.VolumeReady -@@ -2178,6 +2197,11 @@ func (d *VirtualMachineController) processVmCleanup(vmi *v1.VirtualMachineInstan - return err - } - -+ err := d.hotplugContainerDiskMounter.UmountAll(vmi) -+ if err != nil { -+ return err -+ } -+ - // UnmountAll does the cleanup on the "best effort" basis: it is - // safe to pass a nil cgroupManager. - cgroupManager, _ := getCgroupManager(vmi) -@@ -2829,6 +2853,12 @@ func (d *VirtualMachineController) vmUpdateHelperMigrationTarget(origVMI *v1.Vir - return err - } - -+ hotplugDiskInfo, err := d.hotplugContainerDiskMounter.MountAndVerify(vmi) -+ if err != nil { -+ return err -+ } -+ maps.Copy(disksInfo, hotplugDiskInfo) -+ - // Mount hotplug disks - if attachmentPodUID := vmi.Status.MigrationState.TargetAttachmentPodUID; attachmentPodUID != types.UID("") { - cgroupManager, err := getCgroupManager(vmi) -@@ -3051,6 +3081,11 @@ func (d *VirtualMachineController) vmUpdateHelperDefault(origVMI *v1.VirtualMach - if err != nil { - return err - } -+ hotplugDiskInfo, err := d.hotplugContainerDiskMounter.MountAndVerify(vmi) -+ if err != nil { -+ return err -+ } -+ maps.Copy(disksInfo, hotplugDiskInfo) - - // Try to mount hotplug volume if there is any during startup. - if err := d.hotplugVolumeMounter.Mount(vmi, cgroupManager); err != nil { -@@ -3138,6 +3173,11 @@ func (d *VirtualMachineController) vmUpdateHelperDefault(origVMI *v1.VirtualMach - log.Log.Object(vmi).Error(err.Error()) - } - -+ hotplugDiskInfo, err := d.hotplugContainerDiskMounter.MountAndVerify(vmi) -+ if err != nil { -+ return err -+ } -+ maps.Copy(disksInfo, hotplugDiskInfo) - if err := d.hotplugVolumeMounter.Mount(vmi, cgroupManager); err != nil { - return err - } -@@ -3215,6 +3255,9 @@ func (d *VirtualMachineController) vmUpdateHelperDefault(origVMI *v1.VirtualMach - - if vmi.IsRunning() { - // Umount any disks no longer mounted -+ if err := d.hotplugContainerDiskMounter.Umount(vmi); err != nil { -+ return err -+ } - if err := d.hotplugVolumeMounter.Unmount(vmi, cgroupManager); err != nil { - return err - } -diff --git a/pkg/virt-launcher/virtwrap/converter/converter.go b/pkg/virt-launcher/virtwrap/converter/converter.go -index 3318c1c466..2f5a0f1215 100644 ---- a/pkg/virt-launcher/virtwrap/converter/converter.go -+++ b/pkg/virt-launcher/virtwrap/converter/converter.go -@@ -649,6 +649,9 @@ func Convert_v1_Hotplug_Volume_To_api_Disk(source *v1.Volume, disk *api.Disk, c - if source.DataVolume != nil { - return Convert_v1_Hotplug_DataVolume_To_api_Disk(source.Name, disk, c) - } -+ if source.ContainerDisk != nil { -+ return Convert_v1_Hotplug_ContainerDisk_To_api_Disk(source.Name, disk, c) -+ } - return fmt.Errorf("hotplug disk %s references an unsupported source", disk.Alias.GetName()) - } - -@@ -690,6 +693,10 @@ func GetHotplugBlockDeviceVolumePath(volumeName string) string { - return filepath.Join(string(filepath.Separator), "var", "run", "kubevirt", "hotplug-disks", volumeName) - } - -+func GetHotplugContainerDiskPath(volumeName string) string { -+ return filepath.Join(string(filepath.Separator), "var", "run", "kubevirt", "hotplug-disks", volumeName) -+} -+ - func Convert_v1_PersistentVolumeClaim_To_api_Disk(name string, disk *api.Disk, c *ConverterContext) error { - if c.IsBlockPVC[name] { - return Convert_v1_BlockVolumeSource_To_api_Disk(name, disk, c.VolumesDiscardIgnore) -@@ -768,6 +775,35 @@ func Convert_v1_Hotplug_BlockVolumeSource_To_api_Disk(volumeName string, disk *a - return nil - } - -+func Convert_v1_Hotplug_ContainerDisk_To_api_Disk(volumeName string, disk *api.Disk, c *ConverterContext) error { -+ if disk.Type == "lun" { -+ return fmt.Errorf(deviceTypeNotCompatibleFmt, disk.Alias.GetName()) -+ } -+ info := c.DisksInfo[volumeName] -+ if info == nil { -+ return fmt.Errorf("no disk info provided for volume %s", volumeName) -+ } -+ -+ disk.Type = "file" -+ disk.Driver.Type = info.Format -+ disk.Driver.ErrorPolicy = v1.DiskErrorPolicyStop -+ disk.ReadOnly = &api.ReadOnly{} -+ if !contains(c.VolumesDiscardIgnore, volumeName) { -+ disk.Driver.Discard = "unmap" -+ } -+ disk.Source.File = GetHotplugContainerDiskPath(volumeName) -+ disk.BackingStore = &api.BackingStore{ -+ Format: &api.BackingStoreFormat{}, -+ Source: &api.DiskSource{}, -+ } -+ -+ //disk.BackingStore.Format.Type = info.Format -+ //disk.BackingStore.Source.File = info.BackingFile -+ //disk.BackingStore.Type = "file" -+ -+ return nil -+} -+ - func Convert_v1_HostDisk_To_api_Disk(volumeName string, path string, disk *api.Disk) error { - disk.Type = "file" - disk.Driver.Type = "raw" -diff --git a/pkg/virt-operator/resource/apply/BUILD.bazel b/pkg/virt-operator/resource/apply/BUILD.bazel -index f6bd9bd4f1..fe6ab54f8c 100644 ---- a/pkg/virt-operator/resource/apply/BUILD.bazel -+++ b/pkg/virt-operator/resource/apply/BUILD.bazel -@@ -4,7 +4,6 @@ go_library( - name = "go_default_library", - srcs = [ - "admissionregistration.go", -- "apiservices.go", - "apps.go", - "certificates.go", - "core.go", -@@ -65,7 +64,6 @@ go_library( - "//vendor/k8s.io/client-go/tools/cache:go_default_library", - "//vendor/k8s.io/client-go/tools/record:go_default_library", - "//vendor/k8s.io/client-go/util/workqueue:go_default_library", -- "//vendor/k8s.io/kube-aggregator/pkg/apis/apiregistration/v1:go_default_library", - "//vendor/k8s.io/utils/pointer:go_default_library", - ], - ) -diff --git a/pkg/virt-operator/resource/generate/components/BUILD.bazel b/pkg/virt-operator/resource/generate/components/BUILD.bazel -index 70d2da0897..affcd3fecd 100644 ---- a/pkg/virt-operator/resource/generate/components/BUILD.bazel -+++ b/pkg/virt-operator/resource/generate/components/BUILD.bazel -@@ -3,7 +3,6 @@ load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test") - go_library( - name = "go_default_library", - srcs = [ -- "apiservices.go", - "crds.go", - "daemonsets.go", - "deployments.go", -@@ -62,7 +61,6 @@ go_library( - "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", - "//vendor/k8s.io/apimachinery/pkg/util/intstr:go_default_library", - "//vendor/k8s.io/apimachinery/pkg/util/yaml:go_default_library", -- "//vendor/k8s.io/kube-aggregator/pkg/apis/apiregistration/v1:go_default_library", - "//vendor/k8s.io/utils/pointer:go_default_library", - ], - ) -@@ -70,7 +68,6 @@ go_library( - go_test( - name = "go_default_test", - srcs = [ -- "apiservices_test.go", - "components_suite_test.go", - "crds_test.go", - "deployments_test.go", -@@ -85,7 +82,6 @@ go_test( - deps = [ - "//pkg/certificates/bootstrap:go_default_library", - "//pkg/certificates/triple/cert:go_default_library", -- "//staging/src/kubevirt.io/api/core/v1:go_default_library", - "//staging/src/kubevirt.io/client-go/testutils:go_default_library", - "//vendor/github.com/onsi/ginkgo/v2:go_default_library", - "//vendor/github.com/onsi/gomega:go_default_library", -diff --git a/pkg/virt-operator/resource/generate/components/validations_generated.go b/pkg/virt-operator/resource/generate/components/validations_generated.go -index 4913dbead0..42225780ba 100644 ---- a/pkg/virt-operator/resource/generate/components/validations_generated.go -+++ b/pkg/virt-operator/resource/generate/components/validations_generated.go -@@ -7723,6 +7723,8 @@ var CRDsValidation map[string]string = map[string]string{ - ContainerDisk references a docker image, embedding a qcow or raw disk. - More info: https://kubevirt.gitbooks.io/user-guide/registry-disk.html - properties: -+ hotpluggable: -+ type: boolean - image: - description: Image is the name of the image with the embedded - disk. -@@ -8355,6 +8357,35 @@ var CRDsValidation map[string]string = map[string]string{ - description: VolumeSource represents the source of the volume - to map to the disk. - properties: -+ containerDisk: -+ description: Represents a docker image with an embedded disk. -+ properties: -+ hotpluggable: -+ type: boolean -+ image: -+ description: Image is the name of the image with the embedded -+ disk. -+ type: string -+ imagePullPolicy: -+ description: |- -+ Image pull policy. -+ One of Always, Never, IfNotPresent. -+ Defaults to Always if :latest tag is specified, or IfNotPresent otherwise. -+ Cannot be updated. -+ More info: https://kubernetes.io/docs/concepts/containers/images#updating-images -+ type: string -+ imagePullSecret: -+ description: ImagePullSecret is the name of the Docker -+ registry secret required to pull the image. The secret -+ must already exist. -+ type: string -+ path: -+ description: Path defines the path to disk file in the -+ container -+ type: string -+ required: -+ - image -+ type: object - dataVolume: - description: |- - DataVolume represents the dynamic creation a PVC for this volume as well as -@@ -12768,6 +12799,8 @@ var CRDsValidation map[string]string = map[string]string{ - ContainerDisk references a docker image, embedding a qcow or raw disk. - More info: https://kubevirt.gitbooks.io/user-guide/registry-disk.html - properties: -+ hotpluggable: -+ type: boolean - image: - description: Image is the name of the image with the embedded - disk. -@@ -18328,6 +18361,8 @@ var CRDsValidation map[string]string = map[string]string{ - ContainerDisk references a docker image, embedding a qcow or raw disk. - More info: https://kubevirt.gitbooks.io/user-guide/registry-disk.html - properties: -+ hotpluggable: -+ type: boolean - image: - description: Image is the name of the image with the embedded - disk. -@@ -22835,6 +22870,8 @@ var CRDsValidation map[string]string = map[string]string{ - ContainerDisk references a docker image, embedding a qcow or raw disk. - More info: https://kubevirt.gitbooks.io/user-guide/registry-disk.html - properties: -+ hotpluggable: -+ type: boolean - image: - description: Image is the name of the image with - the embedded disk. -@@ -28015,6 +28052,8 @@ var CRDsValidation map[string]string = map[string]string{ - ContainerDisk references a docker image, embedding a qcow or raw disk. - More info: https://kubevirt.gitbooks.io/user-guide/registry-disk.html - properties: -+ hotpluggable: -+ type: boolean - image: - description: Image is the name of the image - with the embedded disk. -@@ -28673,6 +28712,36 @@ var CRDsValidation map[string]string = map[string]string{ - description: VolumeSource represents the source of - the volume to map to the disk. - properties: -+ containerDisk: -+ description: Represents a docker image with an -+ embedded disk. -+ properties: -+ hotpluggable: -+ type: boolean -+ image: -+ description: Image is the name of the image -+ with the embedded disk. -+ type: string -+ imagePullPolicy: -+ description: |- -+ Image pull policy. -+ One of Always, Never, IfNotPresent. -+ Defaults to Always if :latest tag is specified, or IfNotPresent otherwise. -+ Cannot be updated. -+ More info: https://kubernetes.io/docs/concepts/containers/images#updating-images -+ type: string -+ imagePullSecret: -+ description: ImagePullSecret is the name of -+ the Docker registry secret required to pull -+ the image. The secret must already exist. -+ type: string -+ path: -+ description: Path defines the path to disk -+ file in the container -+ type: string -+ required: -+ - image -+ type: object - dataVolume: - description: |- - DataVolume represents the dynamic creation a PVC for this volume as well as -diff --git a/pkg/virt-operator/resource/generate/install/generated_mock_strategy.go b/pkg/virt-operator/resource/generate/install/generated_mock_strategy.go -index 5f1e9a3121..1fa1416af0 100644 ---- a/pkg/virt-operator/resource/generate/install/generated_mock_strategy.go -+++ b/pkg/virt-operator/resource/generate/install/generated_mock_strategy.go -@@ -241,16 +241,6 @@ func (_mr *_MockStrategyInterfaceRecorder) MutatingWebhookConfigurations() *gomo - return _mr.mock.ctrl.RecordCall(_mr.mock, "MutatingWebhookConfigurations") - } - --func (_m *MockStrategyInterface) APIServices() []*v18.APIService { -- ret := _m.ctrl.Call(_m, "APIServices") -- ret0, _ := ret[0].([]*v18.APIService) -- return ret0 --} -- --func (_mr *_MockStrategyInterfaceRecorder) APIServices() *gomock.Call { -- return _mr.mock.ctrl.RecordCall(_mr.mock, "APIServices") --} -- - func (_m *MockStrategyInterface) CertificateSecrets() []*v14.Secret { - ret := _m.ctrl.Call(_m, "CertificateSecrets") - ret0, _ := ret[0].([]*v14.Secret) -diff --git a/pkg/virt-operator/resource/generate/rbac/exportproxy.go b/pkg/virt-operator/resource/generate/rbac/exportproxy.go -index ebc9f2adbd..a0dc0586b4 100644 ---- a/pkg/virt-operator/resource/generate/rbac/exportproxy.go -+++ b/pkg/virt-operator/resource/generate/rbac/exportproxy.go -@@ -23,6 +23,7 @@ import ( - rbacv1 "k8s.io/api/rbac/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/runtime" -+ - "kubevirt.io/kubevirt/pkg/virt-operator/resource/generate/components" - - virtv1 "kubevirt.io/api/core/v1" -diff --git a/staging/src/kubevirt.io/api/apitesting/testdata/HEAD/kubevirt.io.v1.VirtualMachine.json b/staging/src/kubevirt.io/api/apitesting/testdata/HEAD/kubevirt.io.v1.VirtualMachine.json -index b651173636..3453dfb0da 100644 ---- a/staging/src/kubevirt.io/api/apitesting/testdata/HEAD/kubevirt.io.v1.VirtualMachine.json -+++ b/staging/src/kubevirt.io/api/apitesting/testdata/HEAD/kubevirt.io.v1.VirtualMachine.json -@@ -754,7 +754,8 @@ - "image": "imageValue", - "imagePullSecret": "imagePullSecretValue", - "path": "pathValue", -- "imagePullPolicy": "imagePullPolicyValue" -+ "imagePullPolicy": "imagePullPolicyValue", -+ "hotpluggable": true - }, - "ephemeral": { - "persistentVolumeClaim": { -@@ -1209,6 +1210,13 @@ - "dataVolume": { - "name": "nameValue", - "hotpluggable": true -+ }, -+ "containerDisk": { -+ "image": "imageValue", -+ "imagePullSecret": "imagePullSecretValue", -+ "path": "pathValue", -+ "imagePullPolicy": "imagePullPolicyValue", -+ "hotpluggable": true - } - }, - "dryRun": [ -diff --git a/staging/src/kubevirt.io/api/apitesting/testdata/HEAD/kubevirt.io.v1.VirtualMachine.yaml b/staging/src/kubevirt.io/api/apitesting/testdata/HEAD/kubevirt.io.v1.VirtualMachine.yaml -index 53dfdacc3b..8b23193158 100644 ---- a/staging/src/kubevirt.io/api/apitesting/testdata/HEAD/kubevirt.io.v1.VirtualMachine.yaml -+++ b/staging/src/kubevirt.io/api/apitesting/testdata/HEAD/kubevirt.io.v1.VirtualMachine.yaml -@@ -719,6 +719,7 @@ spec: - optional: true - volumeLabel: volumeLabelValue - containerDisk: -+ hotpluggable: true - image: imageValue - imagePullPolicy: imagePullPolicyValue - imagePullSecret: imagePullSecretValue -@@ -838,6 +839,12 @@ status: - - dryRunValue - name: nameValue - volumeSource: -+ containerDisk: -+ hotpluggable: true -+ image: imageValue -+ imagePullPolicy: imagePullPolicyValue -+ imagePullSecret: imagePullSecretValue -+ path: pathValue - dataVolume: - hotpluggable: true - name: nameValue -diff --git a/staging/src/kubevirt.io/api/apitesting/testdata/HEAD/kubevirt.io.v1.VirtualMachineInstance.json b/staging/src/kubevirt.io/api/apitesting/testdata/HEAD/kubevirt.io.v1.VirtualMachineInstance.json -index 3be904512c..f595798e89 100644 ---- a/staging/src/kubevirt.io/api/apitesting/testdata/HEAD/kubevirt.io.v1.VirtualMachineInstance.json -+++ b/staging/src/kubevirt.io/api/apitesting/testdata/HEAD/kubevirt.io.v1.VirtualMachineInstance.json -@@ -694,7 +694,8 @@ - "image": "imageValue", - "imagePullSecret": "imagePullSecretValue", - "path": "pathValue", -- "imagePullPolicy": "imagePullPolicyValue" -+ "imagePullPolicy": "imagePullPolicyValue", -+ "hotpluggable": true - }, - "ephemeral": { - "persistentVolumeClaim": { -diff --git a/staging/src/kubevirt.io/api/apitesting/testdata/HEAD/kubevirt.io.v1.VirtualMachineInstance.yaml b/staging/src/kubevirt.io/api/apitesting/testdata/HEAD/kubevirt.io.v1.VirtualMachineInstance.yaml -index 6fd2ab6523..b6457ec94d 100644 ---- a/staging/src/kubevirt.io/api/apitesting/testdata/HEAD/kubevirt.io.v1.VirtualMachineInstance.yaml -+++ b/staging/src/kubevirt.io/api/apitesting/testdata/HEAD/kubevirt.io.v1.VirtualMachineInstance.yaml -@@ -524,6 +524,7 @@ spec: - optional: true - volumeLabel: volumeLabelValue - containerDisk: -+ hotpluggable: true - image: imageValue - imagePullPolicy: imagePullPolicyValue - imagePullSecret: imagePullSecretValue -diff --git a/staging/src/kubevirt.io/api/core/v1/BUILD.bazel b/staging/src/kubevirt.io/api/core/v1/BUILD.bazel -index f8615293a3..0c6c166985 100644 ---- a/staging/src/kubevirt.io/api/core/v1/BUILD.bazel -+++ b/staging/src/kubevirt.io/api/core/v1/BUILD.bazel -@@ -28,7 +28,6 @@ go_library( - "//vendor/k8s.io/apimachinery/pkg/runtime/schema:go_default_library", - "//vendor/k8s.io/apimachinery/pkg/types:go_default_library", - "//vendor/k8s.io/utils/net:go_default_library", -- "//vendor/k8s.io/utils/pointer:go_default_library", - "//vendor/kubevirt.io/containerized-data-importer-api/pkg/apis/core/v1beta1:go_default_library", - ], - ) -diff --git a/staging/src/kubevirt.io/api/core/v1/deepcopy_generated.go b/staging/src/kubevirt.io/api/core/v1/deepcopy_generated.go -index abd5a495d6..7372b22a9a 100644 ---- a/staging/src/kubevirt.io/api/core/v1/deepcopy_generated.go -+++ b/staging/src/kubevirt.io/api/core/v1/deepcopy_generated.go -@@ -1948,6 +1948,11 @@ func (in *HotplugVolumeSource) DeepCopyInto(out *HotplugVolumeSource) { - *out = new(DataVolumeSource) - **out = **in - } -+ if in.ContainerDisk != nil { -+ in, out := &in.ContainerDisk, &out.ContainerDisk -+ *out = new(ContainerDiskSource) -+ **out = **in -+ } - return - } - -diff --git a/staging/src/kubevirt.io/api/core/v1/schema.go b/staging/src/kubevirt.io/api/core/v1/schema.go -index 29aa3932d3..302ed9ffde 100644 ---- a/staging/src/kubevirt.io/api/core/v1/schema.go -+++ b/staging/src/kubevirt.io/api/core/v1/schema.go -@@ -854,6 +854,8 @@ type HotplugVolumeSource struct { - // the process of populating that PVC with a disk image. - // +optional - DataVolume *DataVolumeSource `json:"dataVolume,omitempty"` -+ -+ ContainerDisk *ContainerDiskSource `json:"containerDisk,omitempty"` - } - - type DataVolumeSource struct { -@@ -911,6 +913,8 @@ type ContainerDiskSource struct { - // More info: https://kubernetes.io/docs/concepts/containers/images#updating-images - // +optional - ImagePullPolicy v1.PullPolicy `json:"imagePullPolicy,omitempty"` -+ -+ Hotpluggable bool `json:"hotpluggable,omitempty"` - } - - // Exactly one of its members must be set. -diff --git a/staging/src/kubevirt.io/client-go/api/openapi_generated.go b/staging/src/kubevirt.io/client-go/api/openapi_generated.go -index cc2d743492..b982b1620c 100644 ---- a/staging/src/kubevirt.io/client-go/api/openapi_generated.go -+++ b/staging/src/kubevirt.io/client-go/api/openapi_generated.go -@@ -17772,6 +17772,12 @@ func schema_kubevirtio_api_core_v1_ContainerDiskSource(ref common.ReferenceCallb - Enum: []interface{}{"Always", "IfNotPresent", "Never"}, - }, - }, -+ "hotpluggable": { -+ SchemaProps: spec.SchemaProps{ -+ Type: []string{"boolean"}, -+ Format: "", -+ }, -+ }, - }, - Required: []string{"image"}, - }, -@@ -19645,11 +19651,16 @@ func schema_kubevirtio_api_core_v1_HotplugVolumeSource(ref common.ReferenceCallb - Ref: ref("kubevirt.io/api/core/v1.DataVolumeSource"), - }, - }, -+ "containerDisk": { -+ SchemaProps: spec.SchemaProps{ -+ Ref: ref("kubevirt.io/api/core/v1.ContainerDiskSource"), -+ }, -+ }, - }, - }, - }, - Dependencies: []string{ -- "kubevirt.io/api/core/v1.DataVolumeSource", "kubevirt.io/api/core/v1.PersistentVolumeClaimVolumeSource"}, -+ "kubevirt.io/api/core/v1.ContainerDiskSource", "kubevirt.io/api/core/v1.DataVolumeSource", "kubevirt.io/api/core/v1.PersistentVolumeClaimVolumeSource"}, - } - } - diff --git a/images/virtualization-artifact/cmd/virtualization-controller/main.go b/images/virtualization-artifact/cmd/virtualization-controller/main.go index b5cf24b01..01b15fb88 100644 --- a/images/virtualization-artifact/cmd/virtualization-controller/main.go +++ b/images/virtualization-artifact/cmd/virtualization-controller/main.go @@ -257,7 +257,7 @@ func main() { } vmbdaLogger := logger.NewControllerLogger(vmbda.ControllerName, logLevel, logOutput, logDebugVerbosity, logDebugControllerList) - if _, err = vmbda.NewController(ctx, mgr, vmbdaLogger, controllerNamespace); err != nil { + if _, err = vmbda.NewController(ctx, mgr, virtClient, vmbdaLogger, controllerNamespace); err != nil { log.Error(err.Error()) os.Exit(1) } diff --git a/images/virtualization-artifact/pkg/apiserver/registry/vm/rest/add_volume.go b/images/virtualization-artifact/pkg/apiserver/registry/vm/rest/add_volume.go index 2688425c9..4628a106b 100644 --- a/images/virtualization-artifact/pkg/apiserver/registry/vm/rest/add_volume.go +++ b/images/virtualization-artifact/pkg/apiserver/registry/vm/rest/add_volume.go @@ -18,12 +18,15 @@ package rest import ( "context" + "encoding/json" "fmt" "net/http" "net/url" + corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apiserver/pkg/registry/rest" + virtv1 "kubevirt.io/api/core/v1" "github.com/deckhouse/virtualization-controller/pkg/tls/certmanager" virtlisters "github.com/deckhouse/virtualization/api/client/generated/listers/core/v1alpha2" @@ -61,11 +64,27 @@ func (r AddVolumeREST) Connect(ctx context.Context, name string, opts runtime.Ob if !ok { return nil, fmt.Errorf("invalid options object: %#v", opts) } - location, transport, err := AddVolumeLocation(ctx, r.vmLister, name, addVolumeOpts, r.kubevirt, r.proxyCertManager) + var ( + addVolumePather pather + hooks []mutateRequestHook + ) + + if r.requestFromKubevirt(addVolumeOpts) { + addVolumePather = newKVVMIPather("addvolume") + } else { + addVolumePather = newKVVMPather("addvolume") + h, err := r.genMutateRequestHook(addVolumeOpts) + if err != nil { + return nil, err + } + hooks = append(hooks, h) + } + location, transport, err := AddVolumeLocation(ctx, r.vmLister, name, addVolumeOpts, r.kubevirt, r.proxyCertManager, addVolumePather) if err != nil { return nil, err } - handler := newThrottledUpgradeAwareProxyHandler(location, transport, false, responder, r.kubevirt.ServiceAccount) + handler := newThrottledUpgradeAwareProxyHandler(location, transport, false, responder, r.kubevirt.ServiceAccount, hooks...) + return handler, nil } @@ -79,6 +98,90 @@ func (r AddVolumeREST) ConnectMethods() []string { return []string{http.MethodPut} } +func (r AddVolumeREST) requestFromKubevirt(opts *subresources.VirtualMachineAddVolume) bool { + return opts == nil || (opts.Image == "" && opts.VolumeKind == "" && opts.PVCName == "") +} + +func (r AddVolumeREST) genMutateRequestHook(opts *subresources.VirtualMachineAddVolume) (mutateRequestHook, error) { + var dd virtv1.DiskDevice + if opts.IsCdrom { + dd.CDRom = &virtv1.CDRomTarget{ + Bus: virtv1.DiskBusSCSI, + } + } else { + dd.Disk = &virtv1.DiskTarget{ + Bus: virtv1.DiskBusSCSI, + } + } + + hotplugRequest := AddVolumeOptions{ + Name: opts.Name, + Disk: &virtv1.Disk{ + Name: opts.Name, + DiskDevice: dd, + Serial: opts.Name, + }, + } + switch opts.VolumeKind { + case "VirtualDisk": + if opts.PVCName == "" { + return nil, fmt.Errorf("must specify PVCName") + } + hotplugRequest.VolumeSource = &HotplugVolumeSource{ + PersistentVolumeClaim: &virtv1.PersistentVolumeClaimVolumeSource{ + PersistentVolumeClaimVolumeSource: corev1.PersistentVolumeClaimVolumeSource{ + ClaimName: opts.PVCName, + }, + Hotpluggable: true, + }, + } + case "VirtualImage": + switch { + case opts.PVCName != "" && opts.Image != "": + return nil, fmt.Errorf("must specify only one of PersistentVolumeClaimName or Image") + case opts.PVCName != "": + hotplugRequest.VolumeSource = &HotplugVolumeSource{ + PersistentVolumeClaim: &virtv1.PersistentVolumeClaimVolumeSource{ + PersistentVolumeClaimVolumeSource: corev1.PersistentVolumeClaimVolumeSource{ + ClaimName: opts.PVCName, + }, + Hotpluggable: true, + }, + } + case opts.Image != "": + hotplugRequest.VolumeSource = &HotplugVolumeSource{ + ContainerDisk: &ContainerDiskSource{ + Image: opts.Image, + Hotpluggable: true, + }, + } + default: + return nil, fmt.Errorf("must specify one of PersistentVolumeClaimName or Image") + } + case "ClusterVirtualImage": + if opts.Image == "" { + return nil, fmt.Errorf("must specify Image") + } + hotplugRequest.VolumeSource = &HotplugVolumeSource{ + ContainerDisk: &ContainerDiskSource{ + Image: opts.Image, + Hotpluggable: true, + }, + } + default: + return nil, fmt.Errorf("invalid volume kind: %s", opts.VolumeKind) + } + + newBody, err := json.Marshal(&hotplugRequest) + if err != nil { + return nil, err + } + + return func(req *http.Request) error { + return rewriteBody(req, newBody) + }, nil +} + func AddVolumeLocation( ctx context.Context, getter virtlisters.VirtualMachineLister, @@ -86,6 +189,32 @@ func AddVolumeLocation( opts *subresources.VirtualMachineAddVolume, kubevirt KubevirtApiServerConfig, proxyCertManager certmanager.CertificateManager, + addVolumePather pather, ) (*url.URL, *http.Transport, error) { - return streamLocation(ctx, getter, name, opts, newKVVMIPather("addvolume"), kubevirt, proxyCertManager) + return streamLocation(ctx, getter, name, opts, addVolumePather, kubevirt, proxyCertManager) +} + +type VirtualMachineVolumeRequest struct { + AddVolumeOptions *AddVolumeOptions `json:"addVolumeOptions,omitempty" optional:"true"` + RemoveVolumeOptions *virtv1.RemoveVolumeOptions `json:"removeVolumeOptions,omitempty" optional:"true"` +} +type AddVolumeOptions struct { + Name string `json:"name"` + Disk *virtv1.Disk `json:"disk"` + VolumeSource *HotplugVolumeSource `json:"volumeSource"` + DryRun []string `json:"dryRun,omitempty"` +} + +type HotplugVolumeSource struct { + PersistentVolumeClaim *virtv1.PersistentVolumeClaimVolumeSource `json:"persistentVolumeClaim,omitempty"` + DataVolume *virtv1.DataVolumeSource `json:"dataVolume,omitempty"` + ContainerDisk *ContainerDiskSource `json:"containerDisk,omitempty"` +} + +type ContainerDiskSource struct { + Image string `json:"image"` + ImagePullSecret string `json:"imagePullSecret,omitempty"` + Path string `json:"path,omitempty"` + ImagePullPolicy corev1.PullPolicy `json:"imagePullPolicy,omitempty"` + Hotpluggable bool `json:"hotpluggable,omitempty"` } diff --git a/images/virtualization-artifact/pkg/apiserver/registry/vm/rest/remove_volume.go b/images/virtualization-artifact/pkg/apiserver/registry/vm/rest/remove_volume.go index d3c665113..f279a4ecb 100644 --- a/images/virtualization-artifact/pkg/apiserver/registry/vm/rest/remove_volume.go +++ b/images/virtualization-artifact/pkg/apiserver/registry/vm/rest/remove_volume.go @@ -18,12 +18,14 @@ package rest import ( "context" + "encoding/json" "fmt" "net/http" "net/url" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apiserver/pkg/registry/rest" + virtv1 "kubevirt.io/api/core/v1" "github.com/deckhouse/virtualization-controller/pkg/tls/certmanager" virtlisters "github.com/deckhouse/virtualization/api/client/generated/listers/core/v1alpha2" @@ -61,11 +63,27 @@ func (r RemoveVolumeREST) Connect(ctx context.Context, name string, opts runtime if !ok { return nil, fmt.Errorf("invalid options object: %#v", opts) } - location, transport, err := RemoveVolumeRESTLocation(ctx, r.vmLister, name, removeVolumeOpts, r.kubevirt, r.proxyCertManager) + var ( + removeVolumePather pather + hooks []mutateRequestHook + ) + + if r.requestFromKubevirt(removeVolumeOpts) { + removeVolumePather = newKVVMIPather("removevolume") + } else { + removeVolumePather = newKVVMPather("removevolume") + h, err := r.genMutateRequestHook(removeVolumeOpts) + if err != nil { + return nil, err + } + hooks = append(hooks, h) + } + + location, transport, err := RemoveVolumeRESTLocation(ctx, r.vmLister, name, removeVolumeOpts, r.kubevirt, r.proxyCertManager, removeVolumePather) if err != nil { return nil, err } - handler := newThrottledUpgradeAwareProxyHandler(location, transport, false, responder, r.kubevirt.ServiceAccount) + handler := newThrottledUpgradeAwareProxyHandler(location, transport, false, responder, r.kubevirt.ServiceAccount, hooks...) return handler, nil } @@ -79,6 +97,25 @@ func (r RemoveVolumeREST) ConnectMethods() []string { return []string{http.MethodPut} } +func (r RemoveVolumeREST) requestFromKubevirt(opts *subresources.VirtualMachineRemoveVolume) bool { + return opts == nil || opts.Name == "" +} + +func (r RemoveVolumeREST) genMutateRequestHook(opts *subresources.VirtualMachineRemoveVolume) (mutateRequestHook, error) { + unplugRequest := virtv1.RemoveVolumeOptions{ + Name: opts.Name, + } + + newBody, err := json.Marshal(&unplugRequest) + if err != nil { + return nil, err + } + + return func(req *http.Request) error { + return rewriteBody(req, newBody) + }, nil +} + func RemoveVolumeRESTLocation( ctx context.Context, getter virtlisters.VirtualMachineLister, @@ -86,6 +123,7 @@ func RemoveVolumeRESTLocation( opts *subresources.VirtualMachineRemoveVolume, kubevirt KubevirtApiServerConfig, proxyCertManager certmanager.CertificateManager, + removeVolumePather pather, ) (*url.URL, *http.Transport, error) { - return streamLocation(ctx, getter, name, opts, newKVVMIPather("removevolume"), kubevirt, proxyCertManager) + return streamLocation(ctx, getter, name, opts, removeVolumePather, kubevirt, proxyCertManager) } diff --git a/images/virtualization-artifact/pkg/apiserver/registry/vm/rest/stream.go b/images/virtualization-artifact/pkg/apiserver/registry/vm/rest/stream.go index fe239be6f..ea294f9ae 100644 --- a/images/virtualization-artifact/pkg/apiserver/registry/vm/rest/stream.go +++ b/images/virtualization-artifact/pkg/apiserver/registry/vm/rest/stream.go @@ -17,10 +17,12 @@ limitations under the License. package rest import ( + "bytes" "context" "crypto/tls" "crypto/x509" "fmt" + "io" "net/http" "net/url" "os" @@ -143,18 +145,42 @@ func streamParams(_ url.Values, opts runtime.Object) error { } } +type mutateRequestHook func(req *http.Request) error + func newThrottledUpgradeAwareProxyHandler( location *url.URL, transport *http.Transport, upgradeRequired bool, responder rest.Responder, sa types.NamespacedName, + mutateHooks ...mutateRequestHook, ) http.Handler { var handler http.HandlerFunc = func(w http.ResponseWriter, r *http.Request) { r.Header.Add(userHeader, fmt.Sprintf("system:serviceaccount:%s:%s", sa.Namespace, sa.Name)) r.Header.Add(groupHeader, "system:serviceaccounts") + for _, hook := range mutateHooks { + if hook != nil { + if err := hook(r); err != nil { + w.WriteHeader(http.StatusInternalServerError) + _, _ = w.Write([]byte(err.Error())) + return + } + } + } proxyHandler := proxy.NewUpgradeAwareHandler(location, transport, false, upgradeRequired, proxy.NewErrorResponder(responder)) proxyHandler.ServeHTTP(w, r) } return handler } + +func rewriteBody(req *http.Request, newBody []byte) error { + if req.Body != nil { + err := req.Body.Close() + if err != nil { + return err + } + } + req.Body = io.NopCloser(bytes.NewBuffer(newBody)) + req.ContentLength = int64(len(newBody)) + return nil +} diff --git a/images/virtualization-artifact/pkg/controller/kvapi/kvapi.go b/images/virtualization-artifact/pkg/controller/kvapi/kvapi.go index 1ecdb0502..359405864 100644 --- a/images/virtualization-artifact/pkg/controller/kvapi/kvapi.go +++ b/images/virtualization-artifact/pkg/controller/kvapi/kvapi.go @@ -32,6 +32,7 @@ type Kubevirt interface { HotplugVolumesEnabled() bool } +// Deprecated: use virt client. func New(cli client.Client, kv Kubevirt) *KvApi { return &KvApi{ Client: cli, @@ -39,15 +40,18 @@ func New(cli client.Client, kv Kubevirt) *KvApi { } } +// Deprecated: use virt client. type KvApi struct { client.Client kubevirt Kubevirt } +// Deprecated: use virt client. func (api *KvApi) AddVolume(ctx context.Context, kvvm *virtv1.VirtualMachine, opts *virtv1.AddVolumeOptions) error { return api.addVolume(ctx, kvvm, opts) } +// Deprecated: use virt client. func (api *KvApi) RemoveVolume(ctx context.Context, kvvm *virtv1.VirtualMachine, opts *virtv1.RemoveVolumeOptions) error { return api.removeVolume(ctx, kvvm, opts) } @@ -187,6 +191,7 @@ func volumeNameExists(volume virtv1.Volume, volumeName string) bool { } func volumeSourceExists(volume virtv1.Volume, volumeName string) bool { + // Do not add ContainerDisk!!! return (volume.DataVolume != nil && volume.DataVolume.Name == volumeName) || (volume.PersistentVolumeClaim != nil && volume.PersistentVolumeClaim.ClaimName == volumeName) } diff --git a/images/virtualization-artifact/pkg/controller/kvbuilder/kvvm_utils.go b/images/virtualization-artifact/pkg/controller/kvbuilder/kvvm_utils.go index ee8798b32..9f2ed0aef 100644 --- a/images/virtualization-artifact/pkg/controller/kvbuilder/kvvm_utils.go +++ b/images/virtualization-artifact/pkg/controller/kvbuilder/kvvm_utils.go @@ -50,8 +50,13 @@ func GenerateCVMIDiskName(name string) string { } func GetOriginalDiskName(prefixedName string) (string, bool) { - if strings.HasPrefix(prefixedName, VMDDiskPrefix) { + switch { + case strings.HasPrefix(prefixedName, VMDDiskPrefix): return strings.TrimPrefix(prefixedName, VMDDiskPrefix), true + case strings.HasPrefix(prefixedName, VMIDiskPrefix): + return strings.TrimPrefix(prefixedName, VMIDiskPrefix), true + case strings.HasPrefix(prefixedName, CVMIDiskPrefix): + return strings.TrimPrefix(prefixedName, CVMIDiskPrefix), true } return prefixedName, false diff --git a/images/virtualization-artifact/pkg/controller/service/attachment_service.go b/images/virtualization-artifact/pkg/controller/service/attachment_service.go index b9627a273..486840be8 100644 --- a/images/virtualization-artifact/pkg/controller/service/attachment_service.go +++ b/images/virtualization-artifact/pkg/controller/service/attachment_service.go @@ -28,34 +28,36 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" "github.com/deckhouse/virtualization-controller/pkg/common/object" - "github.com/deckhouse/virtualization-controller/pkg/controller/kubevirt" "github.com/deckhouse/virtualization-controller/pkg/controller/kvapi" "github.com/deckhouse/virtualization-controller/pkg/controller/kvbuilder" + "github.com/deckhouse/virtualization/api/client/kubeclient" virtv2 "github.com/deckhouse/virtualization/api/core/v1alpha2" + "github.com/deckhouse/virtualization/api/subresources/v1alpha2" ) type AttachmentService struct { client Client + virtClient kubeclient.Client controllerNamespace string } -func NewAttachmentService(client Client, controllerNamespace string) *AttachmentService { +func NewAttachmentService(client Client, virtClient kubeclient.Client, controllerNamespace string) *AttachmentService { return &AttachmentService{ client: client, + virtClient: virtClient, controllerNamespace: controllerNamespace, } } var ( - ErrVolumeStatusNotReady = errors.New("hotplug is not ready") - ErrDiskIsSpecAttached = errors.New("virtual disk is already attached to the virtual machine spec") - ErrHotPlugRequestAlreadySent = errors.New("attachment request is already sent") - ErrVirtualMachineWaitsForRestartApproval = errors.New("virtual machine waits for restart approval") + ErrVolumeStatusNotReady = errors.New("hotplug is not ready") + ErrBlockDeviceIsSpecAttached = errors.New("block device is already attached to the virtual machine spec") + ErrHotPlugRequestAlreadySent = errors.New("attachment request is already sent") ) -func (s AttachmentService) IsHotPlugged(vd *virtv2.VirtualDisk, vm *virtv2.VirtualMachine, kvvmi *virtv1.VirtualMachineInstance) (bool, error) { - if vd == nil { - return false, errors.New("cannot check if a nil VirtualDisk is hot plugged") +func (s AttachmentService) IsHotPlugged(ad *AttachmentDisk, vm *virtv2.VirtualMachine, kvvmi *virtv1.VirtualMachineInstance) (bool, error) { + if ad == nil { + return false, errors.New("cannot check if a empty AttachmentDisk is hot plugged") } if vm == nil { @@ -67,7 +69,7 @@ func (s AttachmentService) IsHotPlugged(vd *virtv2.VirtualDisk, vm *virtv2.Virtu } for _, vs := range kvvmi.Status.VolumeStatus { - if vs.HotplugVolume != nil && vs.Name == kvbuilder.GenerateVMDDiskName(vd.Name) { + if vs.HotplugVolume != nil && vs.Name == ad.GenerateName { if vs.Phase == virtv1.VolumeReady { return true, nil } @@ -79,9 +81,9 @@ func (s AttachmentService) IsHotPlugged(vd *virtv2.VirtualDisk, vm *virtv2.Virtu return false, nil } -func (s AttachmentService) CanHotPlug(vd *virtv2.VirtualDisk, vm *virtv2.VirtualMachine, kvvm *virtv1.VirtualMachine) (bool, error) { - if vd == nil { - return false, errors.New("cannot hot plug a nil VirtualDisk") +func (s AttachmentService) CanHotPlug(ad *AttachmentDisk, vm *virtv2.VirtualMachine, kvvm *virtv1.VirtualMachine) (bool, error) { + if ad == nil { + return false, errors.New("cannot hot plug a nil AttachmentDisk") } if vm == nil { @@ -93,12 +95,12 @@ func (s AttachmentService) CanHotPlug(vd *virtv2.VirtualDisk, vm *virtv2.Virtual } for _, bdr := range vm.Spec.BlockDeviceRefs { - if bdr.Kind == virtv2.DiskDevice && bdr.Name == vd.Name { - return false, fmt.Errorf("%w: virtual machine has a virtual disk reference, but it is not a hot-plugged volume", ErrDiskIsSpecAttached) + if bdr.Kind == ad.Kind && bdr.Name == ad.Name { + return false, fmt.Errorf("%w: virtual machine has a block device reference, but it is not a hot-plugged volume", ErrBlockDeviceIsSpecAttached) } } - name := kvbuilder.GenerateVMDDiskName(vd.Name) + name := ad.GenerateName if kvvm.Spec.Template != nil { for _, vs := range kvvm.Spec.Template.Spec.Volumes { @@ -108,7 +110,7 @@ func (s AttachmentService) CanHotPlug(vd *virtv2.VirtualDisk, vm *virtv2.Virtual } if !vs.PersistentVolumeClaim.Hotpluggable { - return false, fmt.Errorf("%w: virtual machine has a virtual disk reference, but it is not a hot-plugged volume", ErrDiskIsSpecAttached) + return false, fmt.Errorf("%w: virtual machine has a block device reference, but it is not a hot-plugged volume", ErrBlockDeviceIsSpecAttached) } return false, ErrHotPlugRequestAlreadySent @@ -122,16 +124,12 @@ func (s AttachmentService) CanHotPlug(vd *virtv2.VirtualDisk, vm *virtv2.Virtual } } - if len(vm.Status.RestartAwaitingChanges) > 0 { - return false, ErrVirtualMachineWaitsForRestartApproval - } - return true, nil } -func (s AttachmentService) HotPlugDisk(ctx context.Context, vd *virtv2.VirtualDisk, vm *virtv2.VirtualMachine, kvvm *virtv1.VirtualMachine) error { - if vd == nil { - return errors.New("cannot hot plug a nil VirtualDisk") +func (s AttachmentService) HotPlugDisk(ctx context.Context, ad *AttachmentDisk, vm *virtv2.VirtualMachine, kvvm *virtv1.VirtualMachine) error { + if ad == nil { + return errors.New("cannot hot plug a nil AttachmentDisk") } if vm == nil { @@ -142,49 +140,22 @@ func (s AttachmentService) HotPlugDisk(ctx context.Context, vd *virtv2.VirtualDi return errors.New("cannot hot plug a disk into a nil KVVM") } - name := kvbuilder.GenerateVMDDiskName(vd.Name) - - hotplugRequest := virtv1.AddVolumeOptions{ - Name: name, - Disk: &virtv1.Disk{ - Name: name, - DiskDevice: virtv1.DiskDevice{ - Disk: &virtv1.DiskTarget{ - Bus: "scsi", - }, - }, - Serial: vd.Name, - }, - VolumeSource: &virtv1.HotplugVolumeSource{ - PersistentVolumeClaim: &virtv1.PersistentVolumeClaimVolumeSource{ - PersistentVolumeClaimVolumeSource: corev1.PersistentVolumeClaimVolumeSource{ - ClaimName: vd.Status.Target.PersistentVolumeClaim, - }, - Hotpluggable: true, - }, - }, - } - - kv, err := kubevirt.New(ctx, s.client, s.controllerNamespace) - if err != nil { - return err - } - - err = kvapi.New(s.client, kv).AddVolume(ctx, kvvm, &hotplugRequest) - if err != nil { - return fmt.Errorf("error adding volume, %w", err) - } - - return nil + return s.virtClient.VirtualMachines(vm.GetNamespace()).AddVolume(ctx, vm.GetName(), v1alpha2.VirtualMachineAddVolume{ + VolumeKind: string(ad.Kind), + Name: ad.GenerateName, + Image: ad.Image, + PVCName: ad.PVCName, + IsCdrom: ad.IsCdrom, + }) } -func (s AttachmentService) CanUnplug(vd *virtv2.VirtualDisk, kvvm *virtv1.VirtualMachine) bool { - if vd == nil || kvvm == nil || kvvm.Spec.Template == nil { +func (s AttachmentService) CanUnplug(kvvm *virtv1.VirtualMachine, diskName string) bool { + if diskName == "" || kvvm == nil || kvvm.Spec.Template == nil { return false } for _, volume := range kvvm.Spec.Template.Spec.Volumes { - if kvapi.VolumeExists(volume, kvbuilder.GenerateVMDDiskName(vd.Name)) { + if kvapi.VolumeExists(volume, diskName) { return true } } @@ -192,26 +163,16 @@ func (s AttachmentService) CanUnplug(vd *virtv2.VirtualDisk, kvvm *virtv1.Virtua return false } -func (s AttachmentService) UnplugDisk(ctx context.Context, vd *virtv2.VirtualDisk, kvvm *virtv1.VirtualMachine) error { - if vd == nil || kvvm == nil { - return nil - } - - unplugRequest := virtv1.RemoveVolumeOptions{ - Name: kvbuilder.GenerateVMDDiskName(vd.Name), +func (s AttachmentService) UnplugDisk(ctx context.Context, kvvm *virtv1.VirtualMachine, diskName string) error { + if kvvm == nil { + return errors.New("cannot unplug a disk from a nil KVVM") } - - kv, err := kubevirt.New(ctx, s.client, s.controllerNamespace) - if err != nil { - return err + if diskName == "" { + return errors.New("cannot unplug a disk with a empty DiskName") } - - err = kvapi.New(s.client, kv).RemoveVolume(ctx, kvvm, &unplugRequest) - if err != nil { - return fmt.Errorf("error removing volume, %w", err) - } - - return nil + return s.virtClient.VirtualMachines(kvvm.GetNamespace()).RemoveVolume(ctx, kvvm.GetName(), v1alpha2.VirtualMachineRemoveVolume{ + Name: diskName, + }) } // IsConflictedAttachment returns true if the provided VMBDA conflicts with another @@ -235,6 +196,26 @@ func (s AttachmentService) UnplugDisk(ctx context.Context, vd *virtv2.VirtualDis // T1: -->VMBDA A Should be Non-Conflicted lexicographically // T1: VMBDA B Phase: "" func (s AttachmentService) IsConflictedAttachment(ctx context.Context, vmbda *virtv2.VirtualMachineBlockDeviceAttachment) (bool, string, error) { + // CVI always has no conflicts. Skip + if vmbda.Spec.BlockDeviceRef.Kind == virtv2.ClusterVirtualImageKind { + return false, "", nil + } + // VI has conflicts only storage on PVC. Skip for ContainerRegistry + if vmbda.Spec.BlockDeviceRef.Kind == virtv2.VirtualImageKind { + vi, err := object.FetchObject(ctx, types.NamespacedName{ + Name: vmbda.Spec.BlockDeviceRef.Name, + Namespace: vmbda.Namespace, + }, + s.client, &virtv2.VirtualImage{}, + ) + if err != nil { + return false, "", err + } + if vi == nil || vi.Spec.Storage == virtv2.StorageContainerRegistry { + return false, "", nil + } + } + var vmbdas virtv2.VirtualMachineBlockDeviceAttachmentList err := s.client.List(ctx, &vmbdas, &client.ListOptions{Namespace: vmbda.Namespace}) if err != nil { @@ -272,8 +253,16 @@ func (s AttachmentService) GetVirtualDisk(ctx context.Context, name, namespace s return object.FetchObject(ctx, types.NamespacedName{Namespace: namespace, Name: name}, s.client, &virtv2.VirtualDisk{}) } -func (s AttachmentService) GetPersistentVolumeClaim(ctx context.Context, vd *virtv2.VirtualDisk) (*corev1.PersistentVolumeClaim, error) { - return object.FetchObject(ctx, types.NamespacedName{Namespace: vd.Namespace, Name: vd.Status.Target.PersistentVolumeClaim}, s.client, &corev1.PersistentVolumeClaim{}) +func (s AttachmentService) GetVirtualImage(ctx context.Context, name, namespace string) (*virtv2.VirtualImage, error) { + return object.FetchObject(ctx, types.NamespacedName{Namespace: namespace, Name: name}, s.client, &virtv2.VirtualImage{}) +} + +func (s AttachmentService) GetClusterVirtualImage(ctx context.Context, name string) (*virtv2.ClusterVirtualImage, error) { + return object.FetchObject(ctx, types.NamespacedName{Name: name}, s.client, &virtv2.ClusterVirtualImage{}) +} + +func (s AttachmentService) GetPersistentVolumeClaim(ctx context.Context, ad *AttachmentDisk) (*corev1.PersistentVolumeClaim, error) { + return object.FetchObject(ctx, types.NamespacedName{Namespace: ad.Namespace, Name: ad.PVCName}, s.client, &corev1.PersistentVolumeClaim{}) } func (s AttachmentService) GetVirtualMachine(ctx context.Context, name, namespace string) (*virtv2.VirtualMachine, error) { @@ -291,3 +280,51 @@ func (s AttachmentService) GetKVVMI(ctx context.Context, vm *virtv2.VirtualMachi func isSameBlockDeviceRefs(a, b virtv2.VMBDAObjectRef) bool { return a.Kind == b.Kind && a.Name == b.Name } + +type AttachmentDisk struct { + Kind virtv2.BlockDeviceKind + Name string + Namespace string + GenerateName string + PVCName string + Image string + IsCdrom bool +} + +func NewAttachmentDiskFromVirtualDisk(vd *virtv2.VirtualDisk) *AttachmentDisk { + return &AttachmentDisk{ + Kind: virtv2.DiskDevice, + Name: vd.GetName(), + Namespace: vd.GetNamespace(), + GenerateName: kvbuilder.GenerateVMDDiskName(vd.GetName()), + PVCName: vd.Status.Target.PersistentVolumeClaim, + } +} + +func NewAttachmentDiskFromVirtualImage(vi *virtv2.VirtualImage) *AttachmentDisk { + ad := AttachmentDisk{ + Kind: virtv2.ImageDevice, + Name: vi.GetName(), + Namespace: vi.GetNamespace(), + GenerateName: kvbuilder.GenerateVMIDiskName(vi.GetName()), + IsCdrom: vi.Status.CDROM, + } + + if vi.Spec.Storage == virtv2.StorageContainerRegistry { + ad.Image = vi.Status.Target.RegistryURL + } else { + ad.PVCName = vi.Status.Target.PersistentVolumeClaim + } + + return &ad +} + +func NewAttachmentDiskFromClusterVirtualImage(cvi *virtv2.ClusterVirtualImage) *AttachmentDisk { + return &AttachmentDisk{ + Kind: virtv2.ClusterImageDevice, + Name: cvi.GetName(), + GenerateName: kvbuilder.GenerateCVMIDiskName(cvi.GetName()), + Image: cvi.Status.Target.RegistryURL, + IsCdrom: cvi.Status.CDROM, + } +} diff --git a/images/virtualization-artifact/pkg/controller/service/attachment_service_test.go b/images/virtualization-artifact/pkg/controller/service/attachment_service_test.go index e1dc49277..8fa52a3c2 100644 --- a/images/virtualization-artifact/pkg/controller/service/attachment_service_test.go +++ b/images/virtualization-artifact/pkg/controller/service/attachment_service_test.go @@ -75,7 +75,7 @@ var _ = Describe("AttachmentService method IsConflictedAttachment", func() { return nil } - s := NewAttachmentService(clientMock, "") + s := NewAttachmentService(clientMock, nil, "") isConflicted, conflictWithName, err := s.IsConflictedAttachment(context.Background(), vmbdaAlpha) Expect(err).To(BeNil()) Expect(isConflicted).To(BeTrue()) @@ -94,7 +94,7 @@ var _ = Describe("AttachmentService method IsConflictedAttachment", func() { return nil } - s := NewAttachmentService(clientMock, "") + s := NewAttachmentService(clientMock, nil, "") isConflicted, conflictWithName, err := s.IsConflictedAttachment(context.Background(), vmbdaAlpha) Expect(err).To(BeNil()) Expect(isConflicted).To(BeFalse()) @@ -113,7 +113,7 @@ var _ = Describe("AttachmentService method IsConflictedAttachment", func() { return nil } - s := NewAttachmentService(clientMock, "") + s := NewAttachmentService(clientMock, nil, "") isConflicted, conflictWithName, err := s.IsConflictedAttachment(context.Background(), vmbdaAlpha) Expect(err).To(BeNil()) Expect(isConflicted).To(BeTrue()) @@ -132,7 +132,7 @@ var _ = Describe("AttachmentService method IsConflictedAttachment", func() { return nil } - s := NewAttachmentService(clientMock, "") + s := NewAttachmentService(clientMock, nil, "") isConflicted, conflictWithName, err := s.IsConflictedAttachment(context.Background(), vmbdaAlpha) Expect(err).To(BeNil()) Expect(isConflicted).To(BeFalse()) @@ -150,7 +150,7 @@ var _ = Describe("AttachmentService method IsConflictedAttachment", func() { return nil } - s := NewAttachmentService(clientMock, "") + s := NewAttachmentService(clientMock, nil, "") isConflicted, conflictWithName, err := s.IsConflictedAttachment(context.Background(), vmbdaAlpha) Expect(err).To(BeNil()) Expect(isConflicted).To(BeFalse()) @@ -165,7 +165,7 @@ var _ = Describe("AttachmentService method IsConflictedAttachment", func() { return nil } - s := NewAttachmentService(clientMock, "") + s := NewAttachmentService(clientMock, nil, "") isConflicted, conflictWithName, err := s.IsConflictedAttachment(context.Background(), vmbdaAlpha) Expect(err).To(BeNil()) Expect(isConflicted).To(BeFalse()) diff --git a/images/virtualization-artifact/pkg/controller/vmbda/internal/block_device_ready.go b/images/virtualization-artifact/pkg/controller/vmbda/internal/block_device_ready.go index c32aaa808..9d41c74e4 100644 --- a/images/virtualization-artifact/pkg/controller/vmbda/internal/block_device_ready.go +++ b/images/virtualization-artifact/pkg/controller/vmbda/internal/block_device_ready.go @@ -110,7 +110,8 @@ func (h BlockDeviceReadyHandler) Handle(ctx context.Context, vmbda *virtv2.Virtu return reconcile.Result{}, nil } - pvc, err := h.attachment.GetPersistentVolumeClaim(ctx, vd) + ad := service.NewAttachmentDiskFromVirtualDisk(vd) + pvc, err := h.attachment.GetPersistentVolumeClaim(ctx, ad) if err != nil { return reconcile.Result{}, err } @@ -131,6 +132,128 @@ func (h BlockDeviceReadyHandler) Handle(ctx context.Context, vmbda *virtv2.Virtu return reconcile.Result{}, nil } + cb.Status(metav1.ConditionTrue).Reason(vmbdacondition.BlockDeviceReady) + return reconcile.Result{}, nil + case virtv2.VMBDAObjectRefKindVirtualImage: + viKey := types.NamespacedName{ + Name: vmbda.Spec.BlockDeviceRef.Name, + Namespace: vmbda.Namespace, + } + + vi, err := h.attachment.GetVirtualImage(ctx, viKey.Name, viKey.Namespace) + if err != nil { + return reconcile.Result{}, err + } + + if vi == nil { + cb. + Status(metav1.ConditionFalse). + Reason(vmbdacondition.BlockDeviceNotReady). + Message(fmt.Sprintf("VirtualImage %q not found.", viKey.String())) + return reconcile.Result{}, nil + } + + if vi.Generation != vi.Status.ObservedGeneration { + cb. + Status(metav1.ConditionFalse). + Reason(vmbdacondition.BlockDeviceNotReady). + Message(fmt.Sprintf("Waiting for the VirtualImage %q to be observed in its latest state generation.", viKey.String())) + return reconcile.Result{}, nil + } + + if vi.Status.Phase != virtv2.ImageReady { + cb. + Status(metav1.ConditionFalse). + Reason(vmbdacondition.BlockDeviceNotReady). + Message(fmt.Sprintf("VirtualImage %q is not ready to be attached to the virtual machine: waiting for the VirtualImage to be ready for attachment.", viKey.String())) + return reconcile.Result{}, nil + } + switch vi.Spec.Storage { + case virtv2.StorageKubernetes, virtv2.StoragePersistentVolumeClaim: + if vi.Status.Target.PersistentVolumeClaim == "" { + cb. + Status(metav1.ConditionFalse). + Reason(vmbdacondition.BlockDeviceNotReady). + Message("Waiting until VirtualImage has associated PersistentVolumeClaim name.") + return reconcile.Result{}, nil + } + ad := service.NewAttachmentDiskFromVirtualImage(vi) + pvc, err := h.attachment.GetPersistentVolumeClaim(ctx, ad) + if err != nil { + return reconcile.Result{}, err + } + + if pvc == nil { + cb. + Status(metav1.ConditionFalse). + Reason(vmbdacondition.BlockDeviceNotReady). + Message(fmt.Sprintf("Underlying PersistentVolumeClaim %q not found.", vi.Status.Target.PersistentVolumeClaim)) + return reconcile.Result{}, nil + } + + if vi.Status.Phase == virtv2.ImageReady && pvc.Status.Phase != corev1.ClaimBound { + cb. + Status(metav1.ConditionFalse). + Reason(vmbdacondition.BlockDeviceNotReady). + Message(fmt.Sprintf("Underlying PersistentVolumeClaim %q not bound.", vi.Status.Target.PersistentVolumeClaim)) + return reconcile.Result{}, nil + } + + cb.Status(metav1.ConditionTrue).Reason(vmbdacondition.BlockDeviceReady) + + case virtv2.StorageContainerRegistry: + if vi.Status.Target.RegistryURL == "" { + cb. + Status(metav1.ConditionFalse). + Reason(vmbdacondition.BlockDeviceNotReady). + Message("Waiting until VirtualImage has associated RegistryUrl.") + return reconcile.Result{}, nil + } + } + + cb.Status(metav1.ConditionTrue).Reason(vmbdacondition.BlockDeviceReady) + return reconcile.Result{}, nil + case virtv2.VMBDAObjectRefKindClusterVirtualImage: + cviKey := types.NamespacedName{ + Name: vmbda.Spec.BlockDeviceRef.Name, + } + + cvi, err := h.attachment.GetClusterVirtualImage(ctx, cviKey.Name) + if err != nil { + return reconcile.Result{}, err + } + + if cvi == nil { + cb. + Status(metav1.ConditionFalse). + Reason(vmbdacondition.BlockDeviceNotReady). + Message(fmt.Sprintf("ClusterVirtualImage %q not found.", cviKey.String())) + return reconcile.Result{}, nil + } + if cvi.Generation != cvi.Status.ObservedGeneration { + cb. + Status(metav1.ConditionFalse). + Reason(vmbdacondition.BlockDeviceNotReady). + Message(fmt.Sprintf("Waiting for the ClusterVirtualImage %q to be observed in its latest state generation.", cviKey.String())) + return reconcile.Result{}, nil + } + + if cvi.Status.Phase != virtv2.ImageReady { + cb. + Status(metav1.ConditionFalse). + Reason(vmbdacondition.BlockDeviceNotReady). + Message(fmt.Sprintf("ClusterVirtualImage %q is not ready to be attached to the virtual machine: waiting for the ClusterVirtualImage to be ready for attachment.", cviKey.String())) + return reconcile.Result{}, nil + } + + if cvi.Status.Target.RegistryURL == "" { + cb. + Status(metav1.ConditionFalse). + Reason(vmbdacondition.BlockDeviceNotReady). + Message("Waiting until VirtualImage has associated RegistryUrl.") + return reconcile.Result{}, nil + } + cb.Status(metav1.ConditionTrue).Reason(vmbdacondition.BlockDeviceReady) return reconcile.Result{}, nil default: diff --git a/images/virtualization-artifact/pkg/controller/vmbda/internal/deletion.go b/images/virtualization-artifact/pkg/controller/vmbda/internal/deletion.go index fec026765..7cd3eec2d 100644 --- a/images/virtualization-artifact/pkg/controller/vmbda/internal/deletion.go +++ b/images/virtualization-artifact/pkg/controller/vmbda/internal/deletion.go @@ -18,30 +18,85 @@ package internal import ( "context" + "log/slog" + "strings" + "k8s.io/apimachinery/pkg/types" + virtv1 "kubevirt.io/api/core/v1" + "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" "sigs.k8s.io/controller-runtime/pkg/reconcile" + "github.com/deckhouse/virtualization-controller/pkg/common/object" + "github.com/deckhouse/virtualization-controller/pkg/controller/kvbuilder" "github.com/deckhouse/virtualization-controller/pkg/logger" virtv2 "github.com/deckhouse/virtualization/api/core/v1alpha2" ) const deletionHandlerName = "DeletionHandler" -type DeletionHandler struct{} +type UnplugInterface interface { + CanUnplug(kvvm *virtv1.VirtualMachine, diskName string) bool + UnplugDisk(ctx context.Context, kvvm *virtv1.VirtualMachine, diskName string) error +} +type DeletionHandler struct { + unplug UnplugInterface + client client.Client + + log *slog.Logger +} -func NewDeletionHandler() *DeletionHandler { - return &DeletionHandler{} +func NewDeletionHandler(unplug UnplugInterface, client client.Client) *DeletionHandler { + return &DeletionHandler{ + unplug: unplug, + client: client, + } } -func (h DeletionHandler) Handle(ctx context.Context, vd *virtv2.VirtualMachineBlockDeviceAttachment) (reconcile.Result, error) { - log := logger.FromContext(ctx).With(logger.SlogHandler(deletionHandlerName)) - if vd.DeletionTimestamp != nil { - log.Info("Deletion observed: remove cleanup finalizer from VirtualMachineBlockDeviceAttachment") - controllerutil.RemoveFinalizer(vd, virtv2.FinalizerVMBDACleanup) +func (h *DeletionHandler) Handle(ctx context.Context, vmbda *virtv2.VirtualMachineBlockDeviceAttachment) (reconcile.Result, error) { + h.log = logger.FromContext(ctx).With(logger.SlogHandler(deletionHandlerName)) + + if vmbda.DeletionTimestamp != nil { + if err := h.cleanUp(ctx, vmbda); err != nil { + return reconcile.Result{}, err + } + h.log.Info("Deletion observed: remove cleanup finalizer from VirtualMachineBlockDeviceAttachment") + controllerutil.RemoveFinalizer(vmbda, virtv2.FinalizerVMBDACleanup) return reconcile.Result{}, nil } - controllerutil.AddFinalizer(vd, virtv2.FinalizerVMBDACleanup) + controllerutil.AddFinalizer(vmbda, virtv2.FinalizerVMBDACleanup) return reconcile.Result{}, nil } + +func (h *DeletionHandler) cleanUp(ctx context.Context, vmbda *virtv2.VirtualMachineBlockDeviceAttachment) error { + if vmbda == nil { + return nil + } + + var diskName string + switch vmbda.Spec.BlockDeviceRef.Kind { + case virtv2.VMBDAObjectRefKindVirtualDisk: + diskName = kvbuilder.GenerateVMDDiskName(vmbda.Spec.BlockDeviceRef.Name) + case virtv2.VMBDAObjectRefKindVirtualImage: + diskName = kvbuilder.GenerateVMIDiskName(vmbda.Spec.BlockDeviceRef.Name) + case virtv2.VMBDAObjectRefKindClusterVirtualImage: + diskName = kvbuilder.GenerateCVMIDiskName(vmbda.Spec.BlockDeviceRef.Name) + } + + kvvm, err := object.FetchObject(ctx, types.NamespacedName{Namespace: vmbda.GetNamespace(), Name: vmbda.Spec.VirtualMachineName}, h.client, &virtv1.VirtualMachine{}) + if err != nil { + return err + } + + if h.unplug.CanUnplug(kvvm, diskName) { + h.log.Info("Unplug Virtual Disk", slog.String("diskName", diskName), slog.String("vm", kvvm.Name)) + if err = h.unplug.UnplugDisk(ctx, kvvm, diskName); err != nil { + if strings.Contains(err.Error(), "does not exist") { + return nil + } + return err + } + } + return nil +} diff --git a/images/virtualization-artifact/pkg/controller/vmbda/internal/life_cycle.go b/images/virtualization-artifact/pkg/controller/vmbda/internal/life_cycle.go index 7656b090b..1776b408d 100644 --- a/images/virtualization-artifact/pkg/controller/vmbda/internal/life_cycle.go +++ b/images/virtualization-artifact/pkg/controller/vmbda/internal/life_cycle.go @@ -54,9 +54,32 @@ func (h LifeCycleHandler) Handle(ctx context.Context, vmbda *virtv2.VirtualMachi cb.Status(metav1.ConditionUnknown).Reason(conditions.ReasonUnknown) } - vd, err := h.attacher.GetVirtualDisk(ctx, vmbda.Spec.BlockDeviceRef.Name, vmbda.Namespace) - if err != nil { - return reconcile.Result{}, err + var ad *service.AttachmentDisk + switch vmbda.Spec.BlockDeviceRef.Kind { + case virtv2.VMBDAObjectRefKindVirtualDisk: + vd, err := h.attacher.GetVirtualDisk(ctx, vmbda.Spec.BlockDeviceRef.Name, vmbda.Namespace) + if err != nil { + return reconcile.Result{}, err + } + if vd != nil { + ad = service.NewAttachmentDiskFromVirtualDisk(vd) + } + case virtv2.VMBDAObjectRefKindVirtualImage: + vi, err := h.attacher.GetVirtualImage(ctx, vmbda.Spec.BlockDeviceRef.Name, vmbda.Namespace) + if err != nil { + return reconcile.Result{}, err + } + if vi != nil { + ad = service.NewAttachmentDiskFromVirtualImage(vi) + } + case virtv2.VMBDAObjectRefKindClusterVirtualImage: + cvi, err := h.attacher.GetClusterVirtualImage(ctx, vmbda.Spec.BlockDeviceRef.Name) + if err != nil { + return reconcile.Result{}, err + } + if cvi != nil { + ad = service.NewAttachmentDiskFromClusterVirtualImage(cvi) + } } vm, err := h.attacher.GetVirtualMachine(ctx, vmbda.Spec.VirtualMachineName, vmbda.Namespace) @@ -73,18 +96,6 @@ func (h LifeCycleHandler) Handle(ctx context.Context, vmbda *virtv2.VirtualMachi } if vmbda.DeletionTimestamp != nil { - switch vmbda.Status.Phase { - case virtv2.BlockDeviceAttachmentPhasePending, - virtv2.BlockDeviceAttachmentPhaseInProgress, - virtv2.BlockDeviceAttachmentPhaseAttached: - if h.attacher.CanUnplug(vd, kvvm) { - err = h.attacher.UnplugDisk(ctx, vd, kvvm) - if err != nil { - return reconcile.Result{}, err - } - } - } - vmbda.Status.Phase = virtv2.BlockDeviceAttachmentPhaseTerminating cb.Status(metav1.ConditionUnknown).Reason(conditions.ReasonUnknown) @@ -138,12 +149,12 @@ func (h LifeCycleHandler) Handle(ctx context.Context, vmbda *virtv2.VirtualMachi return reconcile.Result{}, nil } - if vd == nil { + if ad == nil { vmbda.Status.Phase = virtv2.BlockDeviceAttachmentPhasePending cb. Status(metav1.ConditionFalse). Reason(vmbdacondition.NotAttached). - Message(fmt.Sprintf("VirtualDisk %q not found.", vmbda.Spec.BlockDeviceRef.Name)) + Message(fmt.Sprintf("AttachmentDisk %q not found.", vmbda.Spec.BlockDeviceRef.Name)) return reconcile.Result{}, nil } @@ -179,10 +190,10 @@ func (h LifeCycleHandler) Handle(ctx context.Context, vmbda *virtv2.VirtualMachi return reconcile.Result{}, nil } - log = log.With("vmName", vm.Name, "vdName", vd.Name) + log = log.With("vmName", vm.Name, "attachmentDiskName", ad.Name) log.Info("Check if hot plug is completed and disk is attached") - isHotPlugged, err := h.attacher.IsHotPlugged(vd, vm, kvvmi) + isHotPlugged, err := h.attacher.IsHotPlugged(ad, vm, kvvmi) if err != nil { if errors.Is(err, service.ErrVolumeStatusNotReady) { vmbda.Status.Phase = virtv2.BlockDeviceAttachmentPhaseInProgress @@ -207,14 +218,14 @@ func (h LifeCycleHandler) Handle(ctx context.Context, vmbda *virtv2.VirtualMachi return reconcile.Result{}, nil } - _, err = h.attacher.CanHotPlug(vd, vm, kvvm) + _, err = h.attacher.CanHotPlug(ad, vm, kvvm) blockDeviceLimitCondition, _ := conditions.GetCondition(vmbdacondition.DiskAttachmentCapacityAvailableType, vmbda.Status.Conditions) switch { case err == nil && blockDeviceLimitCondition.Status == metav1.ConditionTrue: log.Info("Send attachment request") - err = h.attacher.HotPlugDisk(ctx, vd, vm, kvvm) + err = h.attacher.HotPlugDisk(ctx, ad, vm, kvvm) if err != nil { return reconcile.Result{}, err } @@ -225,7 +236,7 @@ func (h LifeCycleHandler) Handle(ctx context.Context, vmbda *virtv2.VirtualMachi Reason(vmbdacondition.AttachmentRequestSent). Message("Attachment request has sent: attachment is in progress.") return reconcile.Result{}, nil - case errors.Is(err, service.ErrDiskIsSpecAttached): + case errors.Is(err, service.ErrBlockDeviceIsSpecAttached): log.Info("VirtualDisk is already attached to the virtual machine spec") vmbda.Status.Phase = virtv2.BlockDeviceAttachmentPhaseFailed @@ -243,15 +254,6 @@ func (h LifeCycleHandler) Handle(ctx context.Context, vmbda *virtv2.VirtualMachi Reason(vmbdacondition.AttachmentRequestSent). Message("Attachment request sent: attachment is in progress.") return reconcile.Result{}, nil - case errors.Is(err, service.ErrVirtualMachineWaitsForRestartApproval): - log.Info("Virtual machine waits for restart approval") - - vmbda.Status.Phase = virtv2.BlockDeviceAttachmentPhasePending - cb. - Status(metav1.ConditionFalse). - Reason(vmbdacondition.NotAttached). - Message(service.CapitalizeFirstLetter(err.Error())) - return reconcile.Result{}, nil case blockDeviceLimitCondition.Status != metav1.ConditionTrue: log.Info("Virtual machine block device capacity reached") diff --git a/images/virtualization-artifact/pkg/controller/vmbda/internal/watcher/cvi_watcher.go b/images/virtualization-artifact/pkg/controller/vmbda/internal/watcher/cvi_watcher.go new file mode 100644 index 000000000..aec0a3404 --- /dev/null +++ b/images/virtualization-artifact/pkg/controller/vmbda/internal/watcher/cvi_watcher.go @@ -0,0 +1,106 @@ +/* +Copyright 2025 Flant JSC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package watcher + +import ( + "context" + "fmt" + "log/slog" + + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller" + "sigs.k8s.io/controller-runtime/pkg/event" + "sigs.k8s.io/controller-runtime/pkg/handler" + "sigs.k8s.io/controller-runtime/pkg/manager" + "sigs.k8s.io/controller-runtime/pkg/predicate" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + "sigs.k8s.io/controller-runtime/pkg/source" + + "github.com/deckhouse/virtualization-controller/pkg/controller/conditions" + virtv2 "github.com/deckhouse/virtualization/api/core/v1alpha2" + "github.com/deckhouse/virtualization/api/core/v1alpha2/cvicondition" +) + +type ClusterVirtualImageWatcher struct { + client client.Client +} + +func NewClusterVirtualImageWatcher(client client.Client) *ClusterVirtualImageWatcher { + return &ClusterVirtualImageWatcher{ + client: client, + } +} + +func (w ClusterVirtualImageWatcher) Watch(mgr manager.Manager, ctr controller.Controller) error { + return ctr.Watch( + source.Kind(mgr.GetCache(), &virtv2.ClusterVirtualImage{}), + handler.EnqueueRequestsFromMapFunc(w.enqueueRequests), + predicate.Funcs{ + CreateFunc: func(e event.CreateEvent) bool { return false }, + DeleteFunc: func(e event.DeleteEvent) bool { return true }, + UpdateFunc: w.filterUpdateEvents, + }, + ) +} + +func (w ClusterVirtualImageWatcher) enqueueRequests(ctx context.Context, obj client.Object) (requests []reconcile.Request) { + var vmbdas virtv2.VirtualMachineBlockDeviceAttachmentList + err := w.client.List(ctx, &vmbdas) + if err != nil { + slog.Default().Error(fmt.Sprintf("failed to list vmbdas: %s", err)) + return + } + + for _, vmbda := range vmbdas.Items { + if vmbda.Spec.BlockDeviceRef.Kind != virtv2.VMBDAObjectRefKindClusterVirtualImage && vmbda.Spec.BlockDeviceRef.Name != obj.GetName() { + continue + } + + requests = append(requests, reconcile.Request{ + NamespacedName: types.NamespacedName{ + Name: vmbda.Name, + Namespace: vmbda.Namespace, + }, + }) + } + + return +} + +func (w ClusterVirtualImageWatcher) filterUpdateEvents(e event.UpdateEvent) bool { + oldCVI, ok := e.ObjectOld.(*virtv2.ClusterVirtualImage) + if !ok { + slog.Default().Error(fmt.Sprintf("expected an old ClusterVirtualImage but got a %T", e.ObjectOld)) + return false + } + + newCVI, ok := e.ObjectNew.(*virtv2.ClusterVirtualImage) + if !ok { + slog.Default().Error(fmt.Sprintf("expected a new ClusterVirtualImage but got a %T", e.ObjectNew)) + return false + } + + if oldCVI.Status.Phase != newCVI.Status.Phase { + return true + } + + oldReadyCondition, _ := conditions.GetCondition(cvicondition.ReadyType, oldCVI.Status.Conditions) + newReadyCondition, _ := conditions.GetCondition(cvicondition.ReadyType, newCVI.Status.Conditions) + + return oldReadyCondition.Status != newReadyCondition.Status +} diff --git a/images/virtualization-artifact/pkg/controller/vmbda/internal/watcher/vi_watcher.go b/images/virtualization-artifact/pkg/controller/vmbda/internal/watcher/vi_watcher.go new file mode 100644 index 000000000..1e7e83054 --- /dev/null +++ b/images/virtualization-artifact/pkg/controller/vmbda/internal/watcher/vi_watcher.go @@ -0,0 +1,108 @@ +/* +Copyright 2025 Flant JSC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package watcher + +import ( + "context" + "fmt" + "log/slog" + + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller" + "sigs.k8s.io/controller-runtime/pkg/event" + "sigs.k8s.io/controller-runtime/pkg/handler" + "sigs.k8s.io/controller-runtime/pkg/manager" + "sigs.k8s.io/controller-runtime/pkg/predicate" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + "sigs.k8s.io/controller-runtime/pkg/source" + + "github.com/deckhouse/virtualization-controller/pkg/controller/conditions" + virtv2 "github.com/deckhouse/virtualization/api/core/v1alpha2" + "github.com/deckhouse/virtualization/api/core/v1alpha2/vicondition" +) + +type VirtualImageWatcher struct { + client client.Client +} + +func NewVirtualImageWatcherr(client client.Client) *VirtualImageWatcher { + return &VirtualImageWatcher{ + client: client, + } +} + +func (w VirtualImageWatcher) Watch(mgr manager.Manager, ctr controller.Controller) error { + return ctr.Watch( + source.Kind(mgr.GetCache(), &virtv2.VirtualDisk{}), + handler.EnqueueRequestsFromMapFunc(w.enqueueRequests), + predicate.Funcs{ + CreateFunc: func(e event.CreateEvent) bool { return false }, + DeleteFunc: func(e event.DeleteEvent) bool { return true }, + UpdateFunc: w.filterUpdateEvents, + }, + ) +} + +func (w VirtualImageWatcher) enqueueRequests(ctx context.Context, obj client.Object) (requests []reconcile.Request) { + var vmbdas virtv2.VirtualMachineBlockDeviceAttachmentList + err := w.client.List(ctx, &vmbdas, &client.ListOptions{ + Namespace: obj.GetNamespace(), + }) + if err != nil { + slog.Default().Error(fmt.Sprintf("failed to list vmbdas: %s", err)) + return + } + + for _, vmbda := range vmbdas.Items { + if vmbda.Spec.BlockDeviceRef.Kind != virtv2.VMBDAObjectRefKindVirtualImage && vmbda.Spec.BlockDeviceRef.Name != obj.GetName() { + continue + } + + requests = append(requests, reconcile.Request{ + NamespacedName: types.NamespacedName{ + Name: vmbda.Name, + Namespace: vmbda.Namespace, + }, + }) + } + + return +} + +func (w VirtualImageWatcher) filterUpdateEvents(e event.UpdateEvent) bool { + oldVI, ok := e.ObjectOld.(*virtv2.VirtualDisk) + if !ok { + slog.Default().Error(fmt.Sprintf("expected an old VirtualImage but got a %T", e.ObjectOld)) + return false + } + + newVI, ok := e.ObjectNew.(*virtv2.VirtualDisk) + if !ok { + slog.Default().Error(fmt.Sprintf("expected a new VirtualImage but got a %T", e.ObjectNew)) + return false + } + + if oldVI.Status.Phase != newVI.Status.Phase { + return true + } + + oldReadyCondition, _ := conditions.GetCondition(vicondition.ReadyType, oldVI.Status.Conditions) + newReadyCondition, _ := conditions.GetCondition(vicondition.ReadyType, newVI.Status.Conditions) + + return oldReadyCondition.Status != newReadyCondition.Status +} diff --git a/images/virtualization-artifact/pkg/controller/vmbda/vmbda_controller.go b/images/virtualization-artifact/pkg/controller/vmbda/vmbda_controller.go index 1f5ff0cfa..9ef2b135b 100644 --- a/images/virtualization-artifact/pkg/controller/vmbda/vmbda_controller.go +++ b/images/virtualization-artifact/pkg/controller/vmbda/vmbda_controller.go @@ -31,6 +31,7 @@ import ( "github.com/deckhouse/virtualization-controller/pkg/controller/vmbda/internal" "github.com/deckhouse/virtualization-controller/pkg/logger" vmbdametrics "github.com/deckhouse/virtualization-controller/pkg/monitoring/metrics/vmbda" + "github.com/deckhouse/virtualization/api/client/kubeclient" virtv2 "github.com/deckhouse/virtualization/api/core/v1alpha2" ) @@ -39,10 +40,11 @@ const ControllerName = "vmbda-controller" func NewController( ctx context.Context, mgr manager.Manager, + virtClient kubeclient.Client, lg *log.Logger, ns string, ) (controller.Controller, error) { - attacher := service.NewAttachmentService(mgr.GetClient(), ns) + attacher := service.NewAttachmentService(mgr.GetClient(), virtClient, ns) blockDeviceService := service.NewBlockDeviceService(mgr.GetClient()) reconciler := NewReconciler( @@ -51,7 +53,7 @@ func NewController( internal.NewBlockDeviceReadyHandler(attacher), internal.NewVirtualMachineReadyHandler(attacher), internal.NewLifeCycleHandler(attacher), - internal.NewDeletionHandler(), + internal.NewDeletionHandler(attacher, mgr.GetClient()), ) vmbdaController, err := controller.New(ControllerName, mgr, controller.Options{ diff --git a/images/virtualization-artifact/pkg/controller/vmbda/vmbda_reconciler.go b/images/virtualization-artifact/pkg/controller/vmbda/vmbda_reconciler.go index fc4e78c94..d0a63d923 100644 --- a/images/virtualization-artifact/pkg/controller/vmbda/vmbda_reconciler.go +++ b/images/virtualization-artifact/pkg/controller/vmbda/vmbda_reconciler.go @@ -101,6 +101,8 @@ func (r *Reconciler) SetupController(_ context.Context, mgr manager.Manager, ctr watcher.NewVirtualMachineBlockDeviceAttachmentWatcher(mgr.GetClient()), watcher.NewVirtualMachineWatcher(mgr.GetClient()), watcher.NewVirtualDiskWatcher(mgr.GetClient()), + watcher.NewClusterVirtualImageWatcher(mgr.GetClient()), + watcher.NewVirtualImageWatcherr(mgr.GetClient()), watcher.NewKVVMIWatcher(mgr.GetClient()), } { err := w.Watch(mgr, ctr) diff --git a/templates/virtualization-controller/rbac-for-us.yaml b/templates/virtualization-controller/rbac-for-us.yaml index c873f497f..652543356 100644 --- a/templates/virtualization-controller/rbac-for-us.yaml +++ b/templates/virtualization-controller/rbac-for-us.yaml @@ -158,6 +158,8 @@ rules: - virtualmachines/freeze - virtualmachines/unfreeze - virtualmachines/migrate + - virtualmachines/addvolume + - virtualmachines/removevolume verbs: - update - apiGroups: From 6867ac41145b24f80c57eb94e49c45cb2a6c4c67 Mon Sep 17 00:00:00 2001 From: yaroslavborbat Date: Sun, 19 Jan 2025 01:00:31 +0300 Subject: [PATCH 08/11] fix Signed-off-by: yaroslavborbat --- api/client/kubeclient/client.go | 5 +++-- api/client/kubeclient/vm.go | 18 ++++++------------ 2 files changed, 9 insertions(+), 14 deletions(-) diff --git a/api/client/kubeclient/client.go b/api/client/kubeclient/client.go index 2a8ed7f57..5e9d8e603 100644 --- a/api/client/kubeclient/client.go +++ b/api/client/kubeclient/client.go @@ -35,8 +35,9 @@ import ( ) var ( - Scheme = runtime.NewScheme() - Codecs = serializer.NewCodecFactory(Scheme) + Scheme = runtime.NewScheme() + Codecs = serializer.NewCodecFactory(Scheme) + ParameterCodec = runtime.NewParameterCodec(Scheme) ) func init() { diff --git a/api/client/kubeclient/vm.go b/api/client/kubeclient/vm.go index ed75d7940..b1b81eda9 100644 --- a/api/client/kubeclient/vm.go +++ b/api/client/kubeclient/vm.go @@ -30,6 +30,7 @@ import ( "time" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" "k8s.io/client-go/rest" virtv1 "kubevirt.io/api/core/v1" @@ -152,26 +153,19 @@ func (v vm) Migrate(ctx context.Context, name string, opts v1alpha2.VirtualMachi func (v vm) AddVolume(ctx context.Context, name string, opts v1alpha2.VirtualMachineAddVolume) error { path := fmt.Sprintf(subresourceURLTpl, v.namespace, v.resource, name, "addvolume") - - return v.restClient. - Put(). - AbsPath(path). - Param("name", opts.Name). - Param("volumeKind", opts.VolumeKind). - Param("pvcName", opts.PVCName). - Param("image", opts.Image). - Param("isCdrom", strconv.FormatBool(opts.IsCdrom)). - Do(ctx). - Error() + return v.doRequest(ctx, path, &opts) } func (v vm) RemoveVolume(ctx context.Context, name string, opts v1alpha2.VirtualMachineRemoveVolume) error { path := fmt.Sprintf(subresourceURLTpl, v.namespace, v.resource, name, "removevolume") + return v.doRequest(ctx, path, &opts) +} +func (v vm) doRequest(ctx context.Context, path string, obj runtime.Object) error { return v.restClient. Put(). AbsPath(path). - Param("name", opts.Name). + VersionedParams(obj, ParameterCodec). Do(ctx). Error() } From 8e3881f6057c6f995ab6efde71005c956450f526 Mon Sep 17 00:00:00 2001 From: yaroslavborbat Date: Sun, 19 Jan 2025 01:32:09 +0300 Subject: [PATCH 09/11] fix Signed-off-by: yaroslavborbat --- api/client/kubeclient/vm.go | 8 ++++++++ api/subresources/v1alpha2/types.go | 5 +++++ .../pkg/controller/vmbda/internal/watcher/vi_watcher.go | 6 +++--- 3 files changed, 16 insertions(+), 3 deletions(-) diff --git a/api/client/kubeclient/vm.go b/api/client/kubeclient/vm.go index b1b81eda9..346d17678 100644 --- a/api/client/kubeclient/vm.go +++ b/api/client/kubeclient/vm.go @@ -153,11 +153,19 @@ func (v vm) Migrate(ctx context.Context, name string, opts v1alpha2.VirtualMachi func (v vm) AddVolume(ctx context.Context, name string, opts v1alpha2.VirtualMachineAddVolume) error { path := fmt.Sprintf(subresourceURLTpl, v.namespace, v.resource, name, "addvolume") + opts.TypeMeta = metav1.TypeMeta{ + Kind: v1alpha2.VirtualMachineAddVolumeKind, + APIVersion: v1alpha2.SchemeGroupVersion.String(), + } return v.doRequest(ctx, path, &opts) } func (v vm) RemoveVolume(ctx context.Context, name string, opts v1alpha2.VirtualMachineRemoveVolume) error { path := fmt.Sprintf(subresourceURLTpl, v.namespace, v.resource, name, "removevolume") + opts.TypeMeta = metav1.TypeMeta{ + Kind: v1alpha2.VirtualMachineRemoveVolumeKind, + APIVersion: v1alpha2.SchemeGroupVersion.String(), + } return v.doRequest(ctx, path, &opts) } diff --git a/api/subresources/v1alpha2/types.go b/api/subresources/v1alpha2/types.go index a4c1a054c..b414d3d51 100644 --- a/api/subresources/v1alpha2/types.go +++ b/api/subresources/v1alpha2/types.go @@ -18,6 +18,11 @@ package v1alpha2 import metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +const ( + VirtualMachineAddVolumeKind = "VirtualMachineAddVolume" + VirtualMachineRemoveVolumeKind = "VirtualMachineRemoveVolume" +) + // +genclient // +genclient:readonly // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object diff --git a/images/virtualization-artifact/pkg/controller/vmbda/internal/watcher/vi_watcher.go b/images/virtualization-artifact/pkg/controller/vmbda/internal/watcher/vi_watcher.go index 1e7e83054..4bb3c8b69 100644 --- a/images/virtualization-artifact/pkg/controller/vmbda/internal/watcher/vi_watcher.go +++ b/images/virtualization-artifact/pkg/controller/vmbda/internal/watcher/vi_watcher.go @@ -48,7 +48,7 @@ func NewVirtualImageWatcherr(client client.Client) *VirtualImageWatcher { func (w VirtualImageWatcher) Watch(mgr manager.Manager, ctr controller.Controller) error { return ctr.Watch( - source.Kind(mgr.GetCache(), &virtv2.VirtualDisk{}), + source.Kind(mgr.GetCache(), &virtv2.VirtualImage{}), handler.EnqueueRequestsFromMapFunc(w.enqueueRequests), predicate.Funcs{ CreateFunc: func(e event.CreateEvent) bool { return false }, @@ -85,13 +85,13 @@ func (w VirtualImageWatcher) enqueueRequests(ctx context.Context, obj client.Obj } func (w VirtualImageWatcher) filterUpdateEvents(e event.UpdateEvent) bool { - oldVI, ok := e.ObjectOld.(*virtv2.VirtualDisk) + oldVI, ok := e.ObjectOld.(*virtv2.VirtualImage) if !ok { slog.Default().Error(fmt.Sprintf("expected an old VirtualImage but got a %T", e.ObjectOld)) return false } - newVI, ok := e.ObjectNew.(*virtv2.VirtualDisk) + newVI, ok := e.ObjectNew.(*virtv2.VirtualImage) if !ok { slog.Default().Error(fmt.Sprintf("expected a new VirtualImage but got a %T", e.ObjectNew)) return false From 21ef5e553fe6bcb895dda08b30b617f8e23d307b Mon Sep 17 00:00:00 2001 From: yaroslavborbat Date: Sun, 19 Jan 2025 01:53:04 +0300 Subject: [PATCH 10/11] fix Signed-off-by: yaroslavborbat --- api/client/kubeclient/client.go | 5 ++--- api/client/kubeclient/vm.go | 26 +++++++++++--------------- api/subresources/v1alpha2/types.go | 5 ----- 3 files changed, 13 insertions(+), 23 deletions(-) diff --git a/api/client/kubeclient/client.go b/api/client/kubeclient/client.go index 5e9d8e603..2a8ed7f57 100644 --- a/api/client/kubeclient/client.go +++ b/api/client/kubeclient/client.go @@ -35,9 +35,8 @@ import ( ) var ( - Scheme = runtime.NewScheme() - Codecs = serializer.NewCodecFactory(Scheme) - ParameterCodec = runtime.NewParameterCodec(Scheme) + Scheme = runtime.NewScheme() + Codecs = serializer.NewCodecFactory(Scheme) ) func init() { diff --git a/api/client/kubeclient/vm.go b/api/client/kubeclient/vm.go index 346d17678..e30bc4e6f 100644 --- a/api/client/kubeclient/vm.go +++ b/api/client/kubeclient/vm.go @@ -30,7 +30,6 @@ import ( "time" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/runtime" "k8s.io/client-go/rest" virtv1 "kubevirt.io/api/core/v1" @@ -153,27 +152,24 @@ func (v vm) Migrate(ctx context.Context, name string, opts v1alpha2.VirtualMachi func (v vm) AddVolume(ctx context.Context, name string, opts v1alpha2.VirtualMachineAddVolume) error { path := fmt.Sprintf(subresourceURLTpl, v.namespace, v.resource, name, "addvolume") - opts.TypeMeta = metav1.TypeMeta{ - Kind: v1alpha2.VirtualMachineAddVolumeKind, - APIVersion: v1alpha2.SchemeGroupVersion.String(), - } - return v.doRequest(ctx, path, &opts) + return v.restClient. + Put(). + AbsPath(path). + Param("name", opts.Name). + Param("volumeKind", opts.VolumeKind). + Param("pvcName", opts.PVCName). + Param("image", opts.Image). + Param("isCdrom", strconv.FormatBool(opts.IsCdrom)). + Do(ctx). + Error() } func (v vm) RemoveVolume(ctx context.Context, name string, opts v1alpha2.VirtualMachineRemoveVolume) error { path := fmt.Sprintf(subresourceURLTpl, v.namespace, v.resource, name, "removevolume") - opts.TypeMeta = metav1.TypeMeta{ - Kind: v1alpha2.VirtualMachineRemoveVolumeKind, - APIVersion: v1alpha2.SchemeGroupVersion.String(), - } - return v.doRequest(ctx, path, &opts) -} - -func (v vm) doRequest(ctx context.Context, path string, obj runtime.Object) error { return v.restClient. Put(). AbsPath(path). - VersionedParams(obj, ParameterCodec). + Param("name", opts.Name). Do(ctx). Error() } diff --git a/api/subresources/v1alpha2/types.go b/api/subresources/v1alpha2/types.go index b414d3d51..a4c1a054c 100644 --- a/api/subresources/v1alpha2/types.go +++ b/api/subresources/v1alpha2/types.go @@ -18,11 +18,6 @@ package v1alpha2 import metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" -const ( - VirtualMachineAddVolumeKind = "VirtualMachineAddVolume" - VirtualMachineRemoveVolumeKind = "VirtualMachineRemoveVolume" -) - // +genclient // +genclient:readonly // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object From 5c5149712ffed48a29eb3f58ebf559a43d11111a Mon Sep 17 00:00:00 2001 From: yaroslavborbat Date: Mon, 20 Jan 2025 13:47:46 +0300 Subject: [PATCH 11/11] fix Signed-off-by: yaroslavborbat --- .../controller/service/attachment_service.go | 19 ++----------------- 1 file changed, 2 insertions(+), 17 deletions(-) diff --git a/images/virtualization-artifact/pkg/controller/service/attachment_service.go b/images/virtualization-artifact/pkg/controller/service/attachment_service.go index 486840be8..173b467e6 100644 --- a/images/virtualization-artifact/pkg/controller/service/attachment_service.go +++ b/images/virtualization-artifact/pkg/controller/service/attachment_service.go @@ -196,25 +196,10 @@ func (s AttachmentService) UnplugDisk(ctx context.Context, kvvm *virtv1.VirtualM // T1: -->VMBDA A Should be Non-Conflicted lexicographically // T1: VMBDA B Phase: "" func (s AttachmentService) IsConflictedAttachment(ctx context.Context, vmbda *virtv2.VirtualMachineBlockDeviceAttachment) (bool, string, error) { - // CVI always has no conflicts. Skip - if vmbda.Spec.BlockDeviceRef.Kind == virtv2.ClusterVirtualImageKind { + // CVI and VI always has no conflicts. Skip + if vmbda.Spec.BlockDeviceRef.Kind == virtv2.ClusterVirtualImageKind || vmbda.Spec.BlockDeviceRef.Kind == virtv2.VirtualImageKind { return false, "", nil } - // VI has conflicts only storage on PVC. Skip for ContainerRegistry - if vmbda.Spec.BlockDeviceRef.Kind == virtv2.VirtualImageKind { - vi, err := object.FetchObject(ctx, types.NamespacedName{ - Name: vmbda.Spec.BlockDeviceRef.Name, - Namespace: vmbda.Namespace, - }, - s.client, &virtv2.VirtualImage{}, - ) - if err != nil { - return false, "", err - } - if vi == nil || vi.Spec.Storage == virtv2.StorageContainerRegistry { - return false, "", nil - } - } var vmbdas virtv2.VirtualMachineBlockDeviceAttachmentList err := s.client.List(ctx, &vmbdas, &client.ListOptions{Namespace: vmbda.Namespace})