new changes

tests/e2e/csi_snapshot_utils.go

@@ -301,7 +301,7 @@ func getRestConfigClientForGuestCluster(guestClusterRestConfig *rest.Config) *re
 // deleteVolumeSnapshot deletes volume snapshot from K8s side and CNS side
 func deleteVolumeSnapshot(ctx context.Context, snapc *snapclient.Clientset, namespace string,
  volumeSnapshot *snapV1.VolumeSnapshot, pandoraSyncWaitTime int,
- volHandle string, snapshotID string) (bool, bool, error) {
+ volHandle string, snapshotID string, performCnsQueryVolumeSnapshot bool) (bool, bool, error) {
  var err error
 
  framework.Logf("Delete volume snapshot and verify the snapshot content is deleted")

tests/e2e/raw_block_volume.go

@@ -1096,7 +1096,7 @@ var _ = ginkgo.Describe("raw block volume support", func() {
  15. Query the snapshot from CNS side - should return 0 entries
  16. Cleanup: Delete PVC, SC (validate they are removed)
  */
- ginkgo.It("[block-vanilla-snapshot] [tkg-snapshot][supervisor-snapshot] Verify snapshot dynamic provisioning workflow with "+
+ ginkgo.It("[block-vanilla-snapshot] [tkg-snapshot][supervisor-snapshot] TC17Verify snapshot dynamic provisioning workflow with "+
  "raw block volume", func() {
  ctx, cancel := context.WithCancel(context.Background())
  defer cancel()
@@ -1315,7 +1315,7 @@ var _ = ginkgo.Describe("raw block volume support", func() {
 
  ginkgo.By("Delete dyanmic volume snapshot")
  snapshotCreated, snapshotContentCreated, err = deleteVolumeSnapshot(ctx, snapc, namespace,
- volumeSnapshot, pandoraSyncWaitTime, volumeID, snapshotId)
+ volumeSnapshot, pandoraSyncWaitTime, volumeID, snapshotId, true)
  gomega.Expect(err).NotTo(gomega.HaveOccurred())
  })
 })

tests/e2e/snapshot_vmservice_vm.go

@@ -27,10 +27,12 @@ import (
  snapclient "github.com/kubernetes-csi/external-snapshotter/client/v6/clientset/versioned"
  "github.com/onsi/ginkgo/v2"
  "github.com/onsi/gomega"
+ cnstypes "github.com/vmware/govmomi/cns/types"
  "github.com/vmware/govmomi/vim25/types"
 
  vmopv1 "github.com/vmware-tanzu/vm-operator/api/v1alpha1"
  v1 "k8s.io/api/core/v1"
+ "k8s.io/apimachinery/pkg/api/resource"
  metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
  "k8s.io/apimachinery/pkg/runtime"
  clientset "k8s.io/client-go/kubernetes"
@@ -313,7 +315,7 @@ var _ bool = ginkgo.Describe("[vmsvc] vm service with csi vol tests", func() {
 
  ginkgo.By("Delete dynamic volume snapshot")
  snapshotCreated, snapshotContentCreated, err = deleteVolumeSnapshot(ctx, snapc, namespace,
- volumeSnapshot, pandoraSyncWaitTime, volHandle, snapshotId)
+ volumeSnapshot, pandoraSyncWaitTime, volHandle, snapshotId, true)
  gomega.Expect(err).NotTo(gomega.HaveOccurred())
  })
 
@@ -446,7 +448,7 @@ var _ bool = ginkgo.Describe("[vmsvc] vm service with csi vol tests", func() {
 
  ginkgo.By("Delete dynamic volume snapshot")
  snapshotCreated, snapshotContentCreated, err = deleteVolumeSnapshot(ctx, snapc, namespace,
- volumeSnapshot, pandoraSyncWaitTime, volHandle, snapshotId)
+ volumeSnapshot, pandoraSyncWaitTime, volHandle, snapshotId, true)
  gomega.Expect(err).NotTo(gomega.HaveOccurred())
  })
 
@@ -618,7 +620,7 @@ var _ bool = ginkgo.Describe("[vmsvc] vm service with csi vol tests", func() {
 
  ginkgo.By("Delete dynamic volume snapshot")
  snapshotCreated, snapshotContentCreated, err = deleteVolumeSnapshot(ctx, snapc, namespace,
- volumeSnapshot, pandoraSyncWaitTime, volHandle, snapshotId)
+ volumeSnapshot, pandoraSyncWaitTime, volHandle, snapshotId, true)
  gomega.Expect(err).NotTo(gomega.HaveOccurred())
  })
 
@@ -635,18 +637,17 @@ var _ bool = ginkgo.Describe("[vmsvc] vm service with csi vol tests", func() {
  7. Get VolumeSnapshotClass "volumesnapshotclass-delete" from supervisor cluster
  8. Take 2 snapshots (snapshot-1, snapshot-2) for the PVC created in step #1.
  9. Snapshot Verification: Execute and verify the steps mentioned in the Create snapshot mandatory checks
-
- Verify CNS metadata for a PVC created in step #1
- Create PVC-1 from Snapshot-1, PVC-2 from Snapshot-2
- Wait for PVCs to reach the Bound state.
- Create a VM service VM using pvc-1 and pvc-2 created in step #11
- Wait for the VM service to be up and in the powered-on state.
- Once the VM is up, verify that the volume is accessible inside the VM
- Write some IO to the CSI volumes, read it back from them and verify the data integrity
- Cleanup: Execute and verify the steps mentioned in the Delete snapshot mandatory checks
+ 10. Verify CNS metadata for a PVC created in step #1
+ 11. Create PVC-1 from Snapshot-1, PVC-2 from Snapshot-2
+ 12. Wait for PVCs to reach the Bound state.
+ 13. Create a VM service VM using pvc-1 and pvc-2 created in step #11
+ 14. Wait for the VM service to be up and in the powered-on state.
+ 15. Once the VM is up, verify that the volume is accessible inside the VM
+ 16. Write some IO to the CSI volumes, read it back from them and verify the data integrity
+ 17. Cleanup: Execute and verify the steps mentioned in the Delete snapshot mandatory checks
  */
 
- ginkgo.It("VM3", func() {
+ ginkgo.It("VM4", func() {
  ctx, cancel := context.WithCancel(context.Background())
  defer cancel()
 
@@ -805,7 +806,7 @@ var _ bool = ginkgo.Describe("[vmsvc] vm service with csi vol tests", func() {
 
  ginkgo.By("Wait and verify PVCs are attached to the VM")
  gomega.Expect(waitNverifyPvcsAreAttachedToVmsvcVm(ctx, vmopC, cnsopC, vm2,
- []*v1.PersistentVolumeClaim{pvc2})).NotTo(gomega.HaveOccurred())
+ []*v1.PersistentVolumeClaim{pvc2, pvc3})).NotTo(gomega.HaveOccurred())
 
  ginkgo.By("Verify PVCs are accessible to the VM")
  ginkgo.By("Write some IO to the CSI volumes and read it back from them and verify the data integrity")
@@ -818,12 +819,288 @@ var _ bool = ginkgo.Describe("[vmsvc] vm service with csi vol tests", func() {
 
  ginkgo.By("Delete dynamic volume snapshot")
  snapshotCreated1, snapshotContentCreated1, err = deleteVolumeSnapshot(ctx, snapc, namespace,
- volumeSnapshot1, pandoraSyncWaitTime, volHandle, snapshotId1)
+ volumeSnapshot1, pandoraSyncWaitTime, volHandle, snapshotId1, true)
+ gomega.Expect(err).NotTo(gomega.HaveOccurred())
+
+ ginkgo.By("Delete dynamic volume snapshot")
+ snapshotCreated2, snapshotContentCreated2, err = deleteVolumeSnapshot(ctx, snapc, namespace,
+ volumeSnapshot2, pandoraSyncWaitTime, volHandle, snapshotId2, true)
+ gomega.Expect(err).NotTo(gomega.HaveOccurred())
+ })
+
+ /*
+ Testcase-5
+ Offline and Online resize
+ PVC → VM → Snapshot → Delete VM → Restore Snapshot → Offline PVC resize → Create new VM → Online Resize → Snapshot
+
+ Steps:
+ 1. Create a PVC using the storage class (storage policy) tagged to the supervisor namespace
+ 2. Wait for PVC to reach the Bound state.
+ 3. Create a VM service VM using the PVC created in step #1
+ 4. Wait for the VM service to be up and in the powered-on state.
+ 5. Once the VM is up, verify that the volume is accessible inside the VM
+ 6. Write some data into the volume.
+ 7. Get VolumeSnapshotClass "volumesnapshotclass-delete" from supervisor cluster
+ 8. Take snapshots of the PVC created in step #1
+ 9. Snapshot Verification: Execute and verify the steps mentioned in the Create snapshot mandatory checks
+ 10. Verify CNS metadata for a PVC created in step #1
+ 11. Delete VM service VM created in step #3.
+ 12. Delete volume snapshot
+ 13. Perform offline volume resizing and verify that the operation went successfully.
+ 14. Create a new VM service VM with PVC created in step #1
+ 15. Once the VM is up, verify that the volume is accessible and the filesystem on the volume has expanded.
+ 16. Perform online volume resizing and verify that the operation went successfully.
+ 17. Take a snapshot of the PVC created in step #1.
+ 18. Snapshot Verification: Execute and verify the steps mentioned in the Create snapshot mandatory checks
+ 19. Cleanup: Execute and verify the steps mentioned in the Delete snapshot mandatory checks
+ */
+
+ ginkgo.It("VM5", func() {
+ ctx, cancel := context.WithCancel(context.Background())
+ defer cancel()
+
+ ginkgo.By("Create a storageclass")
+ storageclass, err := client.StorageV1().StorageClasses().Get(ctx, storageClassName, metav1.GetOptions{})
+ gomega.Expect(err).NotTo(gomega.HaveOccurred())
+
+ ginkgo.By("Create a PVC")
+ pvc, err := createPVC(ctx, client, namespace, labelsMap, "", storageclass, "")
+ gomega.Expect(err).NotTo(gomega.HaveOccurred())
+ ginkgo.By("Waiting for all claims to be in bound state")
+ pvs, err := fpv.WaitForPVClaimBoundPhase(ctx, client, []*v1.PersistentVolumeClaim{pvc}, pollTimeout)
+ gomega.Expect(err).NotTo(gomega.HaveOccurred())
+ pv := pvs[0]
+ volHandle := pv.Spec.CSI.VolumeHandle
+ gomega.Expect(volHandle).NotTo(gomega.BeEmpty())
+ defer func() {
+ ginkgo.By("Delete PVCs")
+ err = fpv.DeletePersistentVolumeClaim(ctx, client, pvc.Name, namespace)
+ gomega.Expect(err).NotTo(gomega.HaveOccurred())
+
+ ginkgo.By("Waiting for CNS volumes to be deleted")
+ err = e2eVSphere.waitForCNSVolumeToBeDeleted(volHandle)
+ gomega.Expect(err).NotTo(gomega.HaveOccurred())
+ }()
+
+ ginkgo.By("Creating VM bootstrap data")
+ // creating a secret for vm credentials
+ secretName := createBootstrapSecretForVmsvcVms(ctx, client, namespace)
+ defer func() {
+ ginkgo.By("Deleting VM bootstrap data")
+ err := client.CoreV1().Secrets(namespace).Delete(ctx, secretName, *metav1.NewDeleteOptions(0))
+ gomega.Expect(err).NotTo(gomega.HaveOccurred())
+ }()
+
+ ginkgo.By("Creating VM")
+ vm := createVmServiceVmWithPvcs(
+ ctx, vmopC, namespace, vmClass, []*v1.PersistentVolumeClaim{pvc}, vmi, storageClassName, secretName)
+ defer func() {
+ ginkgo.By("Deleting VM")
+ err = vmopC.Delete(ctx, &vmopv1.VirtualMachine{ObjectMeta: metav1.ObjectMeta{
+ Name: vm.Name,
+ Namespace: namespace,
+ }})
+ gomega.Expect(err).NotTo(gomega.HaveOccurred())
+ }()
+
+ ginkgo.By("Creating loadbalancing service for ssh with the VM")
+ vmlbsvc := createService4Vm(ctx, vmopC, namespace, vm.Name)
+ defer func() {
+ ginkgo.By("Deleting loadbalancing service for ssh with the VM")
+ err = vmopC.Delete(ctx, &vmopv1.VirtualMachineService{ObjectMeta: metav1.ObjectMeta{
+ Name: vmlbsvc.Name,
+ Namespace: namespace,
+ }})
+ gomega.Expect(err).NotTo(gomega.HaveOccurred())
+ }()
+
+ ginkgo.By("Wait for VM to come up and get an IP")
+ vmIp, err := waitNgetVmsvcVmIp(ctx, vmopC, namespace, vm.Name)
+ gomega.Expect(err).NotTo(gomega.HaveOccurred())
+
+ ginkgo.By("Wait and verify PVCs are attached to the VM")
+ gomega.Expect(waitNverifyPvcsAreAttachedToVmsvcVm(ctx, vmopC, cnsopC, vm,
+ []*v1.PersistentVolumeClaim{pvc})).NotTo(gomega.HaveOccurred())
+
+ ginkgo.By("Verify PVCs are accessible to the VM")
+ ginkgo.By("Write some IO to the CSI volumes and read it back from them and verify the data integrity")
+ vm, err = getVmsvcVM(ctx, vmopC, vm.Namespace, vm.Name) // refresh vm info
+ gomega.Expect(err).NotTo(gomega.HaveOccurred())
+ for i, vol := range vm.Status.Volumes {
+ volFolder := formatNVerifyPvcIsAccessible(vol.DiskUuid, i+1, vmIp)
+ verifyDataIntegrityOnVmDisk(vmIp, volFolder)
+ }
+
+ ginkgo.By("Create volume snapshot class")
+ volumeSnapshotClass, err := createVolumeSnapshotClass(ctx, snapc, deletionPolicy)
+ gomega.Expect(err).NotTo(gomega.HaveOccurred())
+ defer func() {
+ if vanillaCluster {
+ err = snapc.SnapshotV1().VolumeSnapshotClasses().Delete(ctx, volumeSnapshotClass.Name,
+ metav1.DeleteOptions{})
+ gomega.Expect(err).NotTo(gomega.HaveOccurred())
+ }
+ }()
+
+ ginkgo.By("Create a dynamic volume snapshot-1 for the volume")
+ volumeSnapshot1, snapshotContent1, snapshotCreated1,
+ snapshotContentCreated1, snapshotId1, _, err := createDynamicVolumeSnapshot(ctx, namespace, snapc, volumeSnapshotClass,
+ pvc, volHandle, diskSize, true)
+ gomega.Expect(err).NotTo(gomega.HaveOccurred())
+ defer func() {
+ if snapshotContentCreated1 {
+ err = deleteVolumeSnapshotContent(ctx, snapshotContent1, snapc, pandoraSyncWaitTime)
+ gomega.Expect(err).NotTo(gomega.HaveOccurred())
+ }
+
+ if snapshotCreated1 {
+ framework.Logf("Deleting volume snapshot")
+ deleteVolumeSnapshotWithPandoraWait(ctx, snapc, namespace, volumeSnapshot1.Name, pandoraSyncWaitTime)
+
+ framework.Logf("Wait till the volume snapshot is deleted")
+ err = waitForVolumeSnapshotContentToBeDeletedWithPandoraWait(ctx, snapc,
+ *volumeSnapshot1.Status.BoundVolumeSnapshotContentName, pandoraSyncWaitTime)
+ gomega.Expect(err).NotTo(gomega.HaveOccurred())
+ }
+ }()
+
+ ginkgo.By("Deleting VM")
+ err = vmopC.Delete(ctx, &vmopv1.VirtualMachine{ObjectMeta: metav1.ObjectMeta{
+ Name: vm.Name,
+ Namespace: namespace,
+ }})
+ gomega.Expect(err).NotTo(gomega.HaveOccurred())
+
+ ginkgo.By("Delete dynamic volume snapshot-1")
+ snapshotCreated1, snapshotContentCreated1, err = deleteVolumeSnapshot(ctx, snapc, namespace,
+ volumeSnapshot1, pandoraSyncWaitTime, volHandle, snapshotId1, true)
  gomega.Expect(err).NotTo(gomega.HaveOccurred())
 
+ ginkgo.By("Expanding the current pvc")
+ currentPvcSize := pvc.Spec.Resources.Requests[v1.ResourceStorage]
+ newSize := currentPvcSize.DeepCopy()
+ newSize.Add(resource.MustParse("4Gi"))
+ framework.Logf("currentPvcSize %v, newSize %v", currentPvcSize, newSize)
+ pvc, err = expandPVCSize(pvc, newSize, client)
+ gomega.Expect(err).NotTo(gomega.HaveOccurred())
+ gomega.Expect(pvc).NotTo(gomega.BeNil())
+ pvcSize := pvc.Spec.Resources.Requests[v1.ResourceStorage]
+ if pvcSize.Cmp(newSize) != 0 {
+ framework.Failf("error updating pvc size %q", pvc.Name)
+ }
+
+ ginkgo.By("Waiting for controller volume resize to finish")
+ err = waitForPvResizeForGivenPvc(pvc, client, totalResizeWaitPeriod)
+ gomega.Expect(err).NotTo(gomega.HaveOccurred())
+
+ ginkgo.By("Checking for conditions on pvc")
+ pvc, err = waitForPVCToReachFileSystemResizePendingCondition(client,
+ namespace, pvc.Name, pollTimeout)
+ gomega.Expect(err).NotTo(gomega.HaveOccurred())
+
+ ginkgo.By(fmt.Sprintf("Invoking QueryCNSVolumeWithResult with VolumeID: %s", volHandle))
+ queryResult, err := e2eVSphere.queryCNSVolumeWithResult(volHandle)
+ gomega.Expect(err).NotTo(gomega.HaveOccurred())
+ if len(queryResult.Volumes) == 0 {
+ err = fmt.Errorf("queryCNSVolumeWithResult returned no volume")
+ }
+ gomega.Expect(err).NotTo(gomega.HaveOccurred())
+ ginkgo.By("Verifying disk size requested in volume expansion is honored")
+ newSizeInMb := int64(6144)
+ if queryResult.Volumes[0].BackingObjectDetails.(*cnstypes.CnsBlockBackingDetails).CapacityInMb !=
+ newSizeInMb {
+ err = fmt.Errorf("got wrong disk size after volume expansion +%v ",
+ queryResult.Volumes[0].BackingObjectDetails.(*cnstypes.CnsBlockBackingDetails).CapacityInMb)
+ }
+ gomega.Expect(err).NotTo(gomega.HaveOccurred())
+
+ ginkgo.By("Creating VM")
+ vm2 := createVmServiceVmWithPvcs(
+ ctx, vmopC, namespace, vmClass, []*v1.PersistentVolumeClaim{pvc}, vmi, storageClassName, secretName)
+ defer func() {
+ ginkgo.By("Deleting VM")
+ err = vmopC.Delete(ctx, &vmopv1.VirtualMachine{ObjectMeta: metav1.ObjectMeta{
+ Name: vm2.Name,
+ Namespace: namespace,
+ }})
+ gomega.Expect(err).NotTo(gomega.HaveOccurred())
+ }()
+
+ ginkgo.By("Wait for VM to come up and get an IP")
+ vmIp2, err := waitNgetVmsvcVmIp(ctx, vmopC, namespace, vm2.Name)
+ gomega.Expect(err).NotTo(gomega.HaveOccurred())
+
+ ginkgo.By("Wait and verify PVCs are attached to the VM")
+ gomega.Expect(waitNverifyPvcsAreAttachedToVmsvcVm(ctx, vmopC, cnsopC, vm2,
+ []*v1.PersistentVolumeClaim{pvc})).NotTo(gomega.HaveOccurred())
+
+ ginkgo.By("Verify PVCs are accessible to the VM")
+ ginkgo.By("Write some IO to the CSI volumes and read it back from them and verify the data integrity")
+ vm, err = getVmsvcVM(ctx, vmopC, vm2.Namespace, vm2.Name) // refresh vm info
+ gomega.Expect(err).NotTo(gomega.HaveOccurred())
+ for i, vol := range vm.Status.Volumes {
+ volFolder := formatNVerifyPvcIsAccessible(vol.DiskUuid, i+1, vmIp2)
+ verifyDataIntegrityOnVmDisk(vmIp2, volFolder)
+ }
+
+ ginkgo.By("Modify the PVC spec to enable online volume expansion when no snapshot exists for this PVC")
+ currentPvcSize = pvc.Spec.Resources.Requests[v1.ResourceStorage]
+ newSize = currentPvcSize.DeepCopy()
+ newSize.Add(resource.MustParse("4Gi"))
+ newDiskSize := "6Gi"
+ framework.Logf("currentPvcSize %v, newSize %v", currentPvcSize, newSize)
+ claims, err := expandPVCSize(pvc, newSize, client)
+ gomega.Expect(err).NotTo(gomega.HaveOccurred())
+ gomega.Expect(claims).NotTo(gomega.BeNil())
+
+ ginkgo.By("Waiting for file system resize to finish")
+ claims, err = waitForFSResize(pvc, client)
+ gomega.Expect(err).NotTo(gomega.HaveOccurred())
+
+ pvcConditions := claims.Status.Conditions
+ expectEqual(len(pvcConditions), 0, "pvc should not have conditions")
+
+ ginkgo.By(fmt.Sprintf("Invoking QueryCNSVolumeWithResult with VolumeID: %s", volHandle))
+ queryResult, err = e2eVSphere.queryCNSVolumeWithResult(volHandle)
+ gomega.Expect(err).NotTo(gomega.HaveOccurred())
+ if len(queryResult.Volumes) == 0 {
+ err = fmt.Errorf("queryCNSVolumeWithResult returned no volume")
+ }
+ gomega.Expect(err).NotTo(gomega.HaveOccurred())
+ ginkgo.By("Verifying disk size requested in volume expansion is honored")
+ newSizeInMb = int64(6144)
+ if queryResult.Volumes[0].BackingObjectDetails.(*cnstypes.CnsBlockBackingDetails).CapacityInMb !=
+ newSizeInMb {
+ err = fmt.Errorf("got wrong disk size after volume expansion +%v ",
+ queryResult.Volumes[0].BackingObjectDetails.(*cnstypes.CnsBlockBackingDetails).CapacityInMb)
+ }
+ gomega.Expect(err).NotTo(gomega.HaveOccurred())
+
+ ginkgo.By("Create a dynamic volume snapshot-2 for the volume")
+ volumeSnapshot2, snapshotContent2, snapshotCreated2,
+ snapshotContentCreated2, snapshotId2, _, err := createDynamicVolumeSnapshot(ctx, namespace, snapc, volumeSnapshotClass,
+ pvc, volHandle, newDiskSize, true)
+ gomega.Expect(err).NotTo(gomega.HaveOccurred())
+ defer func() {
+ if snapshotContentCreated2 {
+ err = deleteVolumeSnapshotContent(ctx, snapshotContent2, snapc, pandoraSyncWaitTime)
+ gomega.Expect(err).NotTo(gomega.HaveOccurred())
+ }
+
+ if snapshotCreated2 {
+ framework.Logf("Deleting volume snapshot")
+ deleteVolumeSnapshotWithPandoraWait(ctx, snapc, namespace, volumeSnapshot2.Name, pandoraSyncWaitTime)
+
+ framework.Logf("Wait till the volume snapshot is deleted")
+ err = waitForVolumeSnapshotContentToBeDeletedWithPandoraWait(ctx, snapc,
+ *volumeSnapshot2.Status.BoundVolumeSnapshotContentName, pandoraSyncWaitTime)
+ gomega.Expect(err).NotTo(gomega.HaveOccurred())
+ }
+ }()
+
  ginkgo.By("Delete dynamic volume snapshot")
  snapshotCreated2, snapshotContentCreated2, err = deleteVolumeSnapshot(ctx, snapc, namespace,
- volumeSnapshot2, pandoraSyncWaitTime, volHandle, snapshotId2)
+ volumeSnapshot2, pandoraSyncWaitTime, volHandle, snapshotId2, true)
  gomega.Expect(err).NotTo(gomega.HaveOccurred())
  })
 })