From c1d79d59b04b62a3fc4de9d0bbe4e3ada5e5fe6c Mon Sep 17 00:00:00 2001 From: Dmitry Rakitin Date: Tue, 3 Dec 2024 12:18:27 +0100 Subject: [PATCH] fix(vd): patch cdi to convert image format When cloning via DataVolume from PVC to PVC, there was no format conversion. The target PVC should have a raw type for Block and a qcow2 type for Filesystem. The patch adds this conversion. Signed-off-by: Isteb4k --- ...dd-format-conversion-for-pvc-cloning.patch | 214 ++++++++++++++++++ images/cdi-artifact/patches/README.md | 7 + .../vmiplease/internal/retention_handler.go | 3 +- 3 files changed, 222 insertions(+), 2 deletions(-) create mode 100644 images/cdi-artifact/patches/017-add-format-conversion-for-pvc-cloning.patch diff --git a/images/cdi-artifact/patches/017-add-format-conversion-for-pvc-cloning.patch b/images/cdi-artifact/patches/017-add-format-conversion-for-pvc-cloning.patch new file mode 100644 index 000000000..7f7d65644 --- /dev/null +++ b/images/cdi-artifact/patches/017-add-format-conversion-for-pvc-cloning.patch @@ -0,0 +1,214 @@ +diff --git a/pkg/controller/upload-controller.go b/pkg/controller/upload-controller.go +index f251cae5d..99f5494dc 100644 +--- a/pkg/controller/upload-controller.go ++++ b/pkg/controller/upload-controller.go +@@ -45,6 +45,8 @@ import ( + "sigs.k8s.io/controller-runtime/pkg/source" + + cdiv1 "kubevirt.io/containerized-data-importer-api/pkg/apis/core/v1beta1" ++ sdkapi "kubevirt.io/controller-lifecycle-operator-sdk/api" ++ + "kubevirt.io/containerized-data-importer/pkg/common" + cc "kubevirt.io/containerized-data-importer/pkg/controller/common" + featuregates "kubevirt.io/containerized-data-importer/pkg/feature-gates" +@@ -54,7 +56,6 @@ import ( + "kubevirt.io/containerized-data-importer/pkg/util/cert/generator" + "kubevirt.io/containerized-data-importer/pkg/util/naming" + cryptowatch "kubevirt.io/containerized-data-importer/pkg/util/tls-crypto-watch" +- sdkapi "kubevirt.io/controller-lifecycle-operator-sdk/api" + ) + + const ( +@@ -430,7 +431,7 @@ func (r *UploadReconciler) createUploadPodForPvc(pvc *corev1.PersistentVolumeCla + args := UploadPodArgs{ + Name: podName, + PVC: pvc, +- ScratchPVCName: createScratchPvcNameFromPvc(pvc, isCloneTarget), ++ ScratchPVCName: createScratchPvcNameFromPvc(pvc), + ClientName: clientName, + FilesystemOverhead: string(fsOverhead), + ServerCert: serverCert, +@@ -723,11 +724,7 @@ func addUploadControllerWatches(mgr manager.Manager, uploadController controller + return nil + } + +-func createScratchPvcNameFromPvc(pvc *corev1.PersistentVolumeClaim, isCloneTarget bool) string { +- if isCloneTarget { +- return "" +- } +- ++func createScratchPvcNameFromPvc(pvc *corev1.PersistentVolumeClaim) string { + return naming.GetResourceName(pvc.Name, common.ScratchNameSuffix) + } + +@@ -801,6 +798,8 @@ func (r *UploadReconciler) makeUploadPodSpec(args UploadPodArgs, resourceRequire + cc.SetNodeNameIfPopulator(args.PVC, &pod.Spec) + cc.SetRestrictedSecurityContext(&pod.Spec) + ++ pod.Spec.InitContainers = r.makeUploadPodInitContainers(args) ++ + return pod + } + +@@ -904,6 +903,33 @@ func (r *UploadReconciler) makeUploadPodContainers(args UploadPodArgs, resourceR + return containers + } + ++func (r *UploadReconciler) makeUploadPodInitContainers(args UploadPodArgs) []corev1.Container { ++ if args.PVC == nil || len(args.PVC.Spec.AccessModes) == 0 || args.PVC.Spec.AccessModes[0] != corev1.ReadWriteMany { ++ return nil ++ } ++ ++ if cc.GetVolumeMode(args.PVC) == corev1.PersistentVolumeBlock { ++ return nil ++ } ++ ++ containers := []corev1.Container{ ++ { ++ Name: "chmod-" + common.UploadServerPodname, ++ Image: r.image, ++ ImagePullPolicy: corev1.PullPolicy(r.pullPolicy), ++ Command: []string{"sh", "-c", "chmod 775 " + common.UploadServerDataDir}, ++ VolumeMounts: []corev1.VolumeMount{ ++ { ++ Name: cc.DataVolName, ++ MountPath: common.UploadServerDataDir, ++ }, ++ }, ++ }, ++ } ++ ++ return containers ++} ++ + func (r *UploadReconciler) makeUploadPodVolumes(args UploadPodArgs) []corev1.Volume { + volumes := []corev1.Volume{ + { +diff --git a/pkg/uploadserver/uploadserver.go b/pkg/uploadserver/uploadserver.go +index aa9e5ab68..845981a1a 100644 +--- a/pkg/uploadserver/uploadserver.go ++++ b/pkg/uploadserver/uploadserver.go +@@ -29,6 +29,7 @@ import ( + "mime/multipart" + "net" + "net/http" ++ "net/url" + "os" + "strings" + "sync" +@@ -36,11 +37,14 @@ import ( + + "github.com/golang/snappy" + "github.com/pkg/errors" ++ "k8s.io/apimachinery/pkg/api/resource" + + "k8s.io/klog/v2" + + cdiv1 "kubevirt.io/containerized-data-importer-api/pkg/apis/core/v1beta1" ++ + "kubevirt.io/containerized-data-importer/pkg/common" ++ "kubevirt.io/containerized-data-importer/pkg/image" + "kubevirt.io/containerized-data-importer/pkg/importer" + "kubevirt.io/containerized-data-importer/pkg/util" + cryptowatch "kubevirt.io/containerized-data-importer/pkg/util/tls-crypto-watch" +@@ -491,7 +495,7 @@ func newAsyncUploadStreamProcessor(stream io.ReadCloser, dest, imageSize string, + func newUploadStreamProcessor(stream io.ReadCloser, dest, imageSize string, filesystemOverhead float64, preallocation bool, sourceContentType string, dvContentType cdiv1.DataVolumeContentType) (bool, error) { + stream = newContentReader(stream, sourceContentType) + if isCloneTarget(sourceContentType) { +- return cloneProcessor(stream, sourceContentType, dest, preallocation) ++ return cloneProcessor(stream, sourceContentType, dest, imageSize, preallocation) + } + + // Clone block device to block device or file system +@@ -501,7 +505,7 @@ func newUploadStreamProcessor(stream io.ReadCloser, dest, imageSize string, file + return processor.PreallocationApplied(), err + } + +-func cloneProcessor(stream io.ReadCloser, contentType, dest string, preallocate bool) (bool, error) { ++func cloneProcessor(stream io.ReadCloser, contentType, dest, imageSize string, preallocate bool) (bool, error) { + if contentType == common.FilesystemCloneContentType { + if dest != common.WriteBlockPath { + return fileToFileCloneProcessor(stream) +@@ -516,16 +520,79 @@ func cloneProcessor(stream io.ReadCloser, contentType, dest string, preallocate + } + + defer stream.Close() +- bytesRead, bytesWrittenn, err := util.StreamDataToFile(stream, dest, preallocate) ++ ++ scratchDisk := common.ScratchDataDir + "/" + common.DiskImageName ++ ++ bytesRead, bytesWritten, err := util.StreamDataToFile(stream, scratchDisk, preallocate) ++ if err != nil { ++ return false, fmt.Errorf("failed to stream data to file: %w", err) ++ } ++ ++ parsedScratchPath, err := url.Parse(scratchDisk) ++ if err != nil { ++ return false, fmt.Errorf("failed to parse url: %w", err) ++ } ++ ++ err = image.NewQEMUOperations().Validate(parsedScratchPath, calculateTargetSize(dest, imageSize)) ++ if err != nil { ++ return false, fmt.Errorf("failed to validate parsed scratch path: %w", err) ++ } ++ ++ err = importer.CleanAll(dest) + if err != nil { +- return false, err ++ return false, fmt.Errorf("failed to clean all: %w", err) + } + +- klog.Infof("Read %d bytes, wrote %d bytes to %s", bytesRead, bytesWrittenn, dest) ++ format, err := util.GetFormat(dest) ++ if err != nil { ++ return false, fmt.Errorf("failed to get format: %w", err) ++ } ++ ++ err = image.NewQEMUOperations().ConvertToFormatStream(parsedScratchPath, format, dest, false) ++ if err != nil { ++ return false, fmt.Errorf("failed to convert: %w", err) ++ } ++ ++ klog.Infof("Read %d bytes, wrote %d bytes to %s", bytesRead, bytesWritten, dest) + + return false, nil + } + ++func calculateTargetSize(dest, imageSize string) int64 { ++ klog.Infof("Calculating available size") ++ ++ var targetQuantity *resource.Quantity ++ size, err := util.GetAvailableSpaceBlock(dest) ++ if err != nil { ++ klog.Error(err) ++ } ++ ++ if size >= int64(0) { ++ // Block volume. ++ klog.Infof("Checking out block volume size") ++ targetQuantity = resource.NewScaledQuantity(size, 0) ++ } else { ++ // File system volume. ++ klog.Infof("Checking out file system volume size") ++ size, err := util.GetAvailableSpace(common.ImporterDataDir) ++ if err != nil { ++ klog.Error(err) ++ } ++ targetQuantity = resource.NewScaledQuantity(size, 0) ++ } ++ ++ if imageSize != "" { ++ klog.Infof("Request image size not empty") ++ newImageSizeQuantity := resource.MustParse(imageSize) ++ minQuantity := util.MinQuantity(targetQuantity, &newImageSizeQuantity) ++ targetQuantity = &minQuantity ++ } ++ ++ klog.Infof("Target size %s", targetQuantity.String()) ++ targetSize := targetQuantity.Value() ++ return targetSize ++} ++ + func fileToFileCloneProcessor(stream io.ReadCloser) (bool, error) { + defer stream.Close() + if err := util.UnArchiveTar(stream, common.ImporterVolumePath); err != nil { diff --git a/images/cdi-artifact/patches/README.md b/images/cdi-artifact/patches/README.md index bf4837721..0c6b3e70e 100644 --- a/images/cdi-artifact/patches/README.md +++ b/images/cdi-artifact/patches/README.md @@ -61,3 +61,10 @@ options to set overhead percent globally or for the particular storage class. Ho is not the same and depends on the size the virtual VM image. Previously we adjusted the whole PVC size to get bigger scratch PVC. This patch adds overhead for the scratch PVC only, leaving the size of the target PVC intact. + +#### `017-add-format-conversion-for-pvc-cloning.patch` + +When cloning via DataVolume from PVC to PVC, there was no format conversion. +The target PVC should have a raw type for volume mode Block and a qcow2 type for Filesystem. +The patch adds this conversion. + diff --git a/images/virtualization-artifact/pkg/controller/vmiplease/internal/retention_handler.go b/images/virtualization-artifact/pkg/controller/vmiplease/internal/retention_handler.go index b7c93c297..cbdf59157 100644 --- a/images/virtualization-artifact/pkg/controller/vmiplease/internal/retention_handler.go +++ b/images/virtualization-artifact/pkg/controller/vmiplease/internal/retention_handler.go @@ -36,8 +36,7 @@ type RetentionHandler struct { retentionDuration time.Duration } -func NewRetentionHandler(retentionDuration time.Duration, -) *RetentionHandler { +func NewRetentionHandler(retentionDuration time.Duration) *RetentionHandler { return &RetentionHandler{ retentionDuration: retentionDuration, }