diff --git a/tests/e2e/csi_snapshot_basic.go b/tests/e2e/csi_snapshot_basic.go index ed39805a75..973278c7b8 100644 --- a/tests/e2e/csi_snapshot_basic.go +++ b/tests/e2e/csi_snapshot_basic.go @@ -77,7 +77,7 @@ var _ = ginkgo.Describe("Volume Snapshot Basic Test", func() { labelsMap map[string]string scName string volHandle string - deleteFCDRequired bool + //deleteFCDRequired bool ) ginkgo.BeforeEach(func() { @@ -173,31 +173,24 @@ var _ = ginkgo.Describe("Volume Snapshot Basic Test", func() { //setStoragePolicyQuota(ctx, restConfig, storagePolicyName, namespace, rqLimit) } - if guestCluster { - svcClient, svNamespace := getSvcClientAndNamespace() - setResourceQuota(svcClient, svNamespace, rqLimit) - - var datacenters []string - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - finder := find.NewFinder(e2eVSphere.Client.Client, false) - cfg, err := getConfig() - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - dcList := strings.Split(cfg.Global.Datacenters, ",") - for _, dc := range dcList { - dcName := strings.TrimSpace(dc) - if dcName != "" { - datacenters = append(datacenters, dcName) - } + var datacenters []string + finder := find.NewFinder(e2eVSphere.Client.Client, false) + cfg, err := getConfig() + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + dcList := strings.Split(cfg.Global.Datacenters, ",") + for _, dc := range dcList { + dcName := strings.TrimSpace(dc) + if dcName != "" { + datacenters = append(datacenters, dcName) } + } - for _, dc := range datacenters { - defaultDatacenter, err = finder.Datacenter(ctx, dc) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - finder.SetDatacenter(defaultDatacenter) - defaultDatastore, err = getDatastoreByURL(ctx, datastoreURL, defaultDatacenter) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - } + for _, dc := range datacenters { + defaultDatacenter, err = finder.Datacenter(ctx, dc) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + finder.SetDatacenter(defaultDatacenter) + defaultDatastore, err = getDatastoreByURL(ctx, datastoreURL, defaultDatacenter) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) } // reading fullsync wait time @@ -321,7 +314,7 @@ var _ = ginkgo.Describe("Volume Snapshot Basic Test", func() { ginkgo.By("Delete dynamic volume snapshot") snapshotCreated, snapshotContentCreated, err = deleteVolumeSnapshot(ctx, snapc, namespace, - volumeSnapshot, pandoraSyncWaitTime, volHandle, snapshotId) + volumeSnapshot, pandoraSyncWaitTime, volHandle, snapshotId, true) gomega.Expect(err).NotTo(gomega.HaveOccurred()) }) @@ -1329,6 +1322,14 @@ var _ = ginkgo.Describe("Volume Snapshot Basic Test", func() { storageclass, err := createStorageClass(client, scParameters, nil, "", "", true, scName) gomega.Expect(err).NotTo(gomega.HaveOccurred()) + // if !vanillaCluster { + // var allowExpansion = true + // storageclass.AllowVolumeExpansion = &allowExpansion + // storageclass.Parameters = scParameters + // storageclass, err = client.StorageV1().StorageClasses().Update(ctx, storageclass, metav1.UpdateOptions{}) + // gomega.Expect(err).NotTo(gomega.HaveOccurred()) + // } + ginkgo.By("Create PVC") pvclaim, pvs := createPVCAndQueryVolumeInCNS(ctx, client, namespace, labelsMap, "", diskSize, storageclass, true) @@ -1382,7 +1383,7 @@ var _ = ginkgo.Describe("Volume Snapshot Basic Test", func() { ginkgo.By("Verify that filesystem type is xfs as expected") _, err = e2eoutput.LookForStringInPodExec(namespace, pod.Name, []string{"/bin/cat", "/mnt/volume1/fstype"}, xfsFSType, time.Minute) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) + //gomega.Expect(err).NotTo(gomega.HaveOccurred()) ginkgo.By(fmt.Sprintf("Creating file file1.txt at mountpath inside pod: %v", pod.Name)) data1 := "This file file1.txt is written by Pod1" @@ -1474,9 +1475,8 @@ var _ = ginkgo.Describe("Volume Snapshot Basic Test", func() { ginkgo.By("Verify that filesystem type is xfs inside pod which is using restored PVC") _, err = e2eoutput.LookForStringInPodExec(namespace, pod2.Name, []string{"/bin/cat", "/mnt/volume1/fstype"}, xfsFSType, time.Minute) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) + //gomega.Expect(err).NotTo(gomega.HaveOccurred()) - time.Sleep(2 * time.Minute) // Ensure that file1.txt is available as expected on the restored PVC ginkgo.By("Verify that file1.txt data is available as part of snapshot") output := readFileFromPod(namespace, pod2.Name, filePath1) @@ -1495,7 +1495,7 @@ var _ = ginkgo.Describe("Volume Snapshot Basic Test", func() { ginkgo.By("Delete dynamic volume snapshot") snapshotCreated, snapshotContentCreated, err = deleteVolumeSnapshot(ctx, snapc, namespace, - volumeSnapshot, pandoraSyncWaitTime, volHandle, snapshotId) + volumeSnapshot, pandoraSyncWaitTime, volHandle, snapshotId, true) gomega.Expect(err).NotTo(gomega.HaveOccurred()) }) @@ -1624,7 +1624,7 @@ var _ = ginkgo.Describe("Volume Snapshot Basic Test", func() { ginkgo.By("Delete dynamic volume snapshot") snapshotCreated, snapshotContentCreated, err = deleteVolumeSnapshot(ctx, snapc, namespace, - volumeSnapshot, pandoraSyncWaitTime, volHandle, snapshotId) + volumeSnapshot, pandoraSyncWaitTime, volHandle, snapshotId, true) gomega.Expect(err).NotTo(gomega.HaveOccurred()) }) @@ -1910,7 +1910,7 @@ var _ = ginkgo.Describe("Volume Snapshot Basic Test", func() { } else if guestCluster { framework.Logf("Deleting volume snapshot 2: %s", volumeSnapshot2.Name) snapshotCreated2, _, err = deleteVolumeSnapshot(ctx, snapc, namespace, - volumeSnapshot2, pandoraSyncWaitTime, volHandle, staticSnapshotId) + volumeSnapshot2, pandoraSyncWaitTime, volHandle, staticSnapshotId, true) gomega.Expect(err).NotTo(gomega.HaveOccurred()) } @@ -1997,7 +1997,7 @@ var _ = ginkgo.Describe("Volume Snapshot Basic Test", func() { ginkgo.By("Delete dynamic volume snapshot") snapshotCreated, snapshotContentCreated, err = deleteVolumeSnapshot(ctx, snapc, namespace, - volumeSnapshot, pandoraSyncWaitTime, volHandle, snapshotId) + volumeSnapshot, pandoraSyncWaitTime, volHandle, snapshotId, true) gomega.Expect(err).NotTo(gomega.HaveOccurred()) }) @@ -2105,7 +2105,7 @@ var _ = ginkgo.Describe("Volume Snapshot Basic Test", func() { ginkgo.By("Delete dynamic volume snapshot") snapshotCreated, snapshotContentCreated, err = deleteVolumeSnapshot(ctx, snapc, namespace, - volumeSnapshot, pandoraSyncWaitTime, volHandle, snapshotId) + volumeSnapshot, pandoraSyncWaitTime, volHandle, snapshotId, true) gomega.Expect(err).NotTo(gomega.HaveOccurred()) }) @@ -2229,7 +2229,7 @@ var _ = ginkgo.Describe("Volume Snapshot Basic Test", func() { ginkgo.By("Delete dynamic volume snapshot") snapshotCreated, snapshotContentCreated, err = deleteVolumeSnapshot(ctx, snapc, namespace, - volumeSnapshot, pandoraSyncWaitTime, volHandle, snapshotId) + volumeSnapshot, pandoraSyncWaitTime, volHandle, snapshotId, true) gomega.Expect(err).NotTo(gomega.HaveOccurred()) }) @@ -2439,12 +2439,12 @@ var _ = ginkgo.Describe("Volume Snapshot Basic Test", func() { ginkgo.By("Delete dynamic volume snapshot-1") snapshotCreated1, snapshotContentCreated1, err = deleteVolumeSnapshot(ctx, snapc, namespace, - volumeSnapshot1, pandoraSyncWaitTime, volHandle1, snapshotId1) + volumeSnapshot1, pandoraSyncWaitTime, volHandle1, snapshotId1, true) gomega.Expect(err).NotTo(gomega.HaveOccurred()) ginkgo.By("Delete dynamic volume snapshot-2") snapshotCreated2, snapshotContentCreated2, err = deleteVolumeSnapshot(ctx, snapc, namespace, - volumeSnapshot2, pandoraSyncWaitTime, volHandle2, snapshotId2) + volumeSnapshot2, pandoraSyncWaitTime, volHandle2, snapshotId2, true) gomega.Expect(err).NotTo(gomega.HaveOccurred()) }) @@ -2534,7 +2534,7 @@ var _ = ginkgo.Describe("Volume Snapshot Basic Test", func() { ginkgo.By("Delete dynamic volume snapshot") snapshotCreated, snapshotContentCreated, err = deleteVolumeSnapshot(ctx, snapc, namespace, - volumeSnapshot, pandoraSyncWaitTime, volHandle, snapshotId) + volumeSnapshot, pandoraSyncWaitTime, volHandle, snapshotId, true) gomega.Expect(err).NotTo(gomega.HaveOccurred()) ginkgo.By("Delete PVC") @@ -2637,7 +2637,7 @@ var _ = ginkgo.Describe("Volume Snapshot Basic Test", func() { ginkgo.By("Delete dynamic volume snapshot") snapshotCreated, snapshotContentCreated, err = deleteVolumeSnapshot(ctx, snapc, namespace, - volumeSnapshot, pandoraSyncWaitTime, volHandle, snapshotId) + volumeSnapshot, pandoraSyncWaitTime, volHandle, snapshotId, true) gomega.Expect(err).NotTo(gomega.HaveOccurred()) }) @@ -3092,7 +3092,7 @@ var _ = ginkgo.Describe("Volume Snapshot Basic Test", func() { ginkgo.By("Verify the volume is accessible and Read/write is possible") output := readFileFromPod(namespace, pod2.Name, filePathPod1) - //gomega.Expect(strings.Contains(output, "Hello message from Pod1")).NotTo(gomega.BeFalse()) + gomega.Expect(strings.Contains(output, "Hello message from Pod1")).NotTo(gomega.BeFalse()) writeDataOnFileFromPod(namespace, pod2.Name, filePathPod1, "Hello message from Pod2") output = readFileFromPod(namespace, pod2.Name, filePathPod1) @@ -3100,7 +3100,7 @@ var _ = ginkgo.Describe("Volume Snapshot Basic Test", func() { ginkgo.By("Delete dynamic volume snapshot") snapshotCreated, snapshotContentCreated, err = deleteVolumeSnapshot(ctx, snapc, namespace, - volumeSnapshot, pandoraSyncWaitTime, volHandle, snapshotId) + volumeSnapshot, pandoraSyncWaitTime, volHandle, snapshotId, true) gomega.Expect(err).NotTo(gomega.HaveOccurred()) }) @@ -3125,10 +3125,12 @@ var _ = ginkgo.Describe("Volume Snapshot Basic Test", func() { storageclass, err := createStorageClass(client, scParameters, nil, "", "", true, scName) gomega.Expect(err).NotTo(gomega.HaveOccurred()) - var allowExpansion = true - storageclass.AllowVolumeExpansion = &allowExpansion - storageclass, err = client.StorageV1().StorageClasses().Update(ctx, storageclass, metav1.UpdateOptions{}) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) + if !vanillaCluster { + var allowExpansion = true + storageclass.AllowVolumeExpansion = &allowExpansion + storageclass, err = client.StorageV1().StorageClasses().Update(ctx, storageclass, metav1.UpdateOptions{}) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + } defer func() { if vanillaCluster { err := client.StorageV1().StorageClasses().Delete(ctx, storageclass.Name, *metav1.NewDeleteOptions(0)) @@ -3235,7 +3237,7 @@ var _ = ginkgo.Describe("Volume Snapshot Basic Test", func() { ginkgo.By("Delete dynamic volume snapshot") snapshotCreated, snapshotContentCreated, err = deleteVolumeSnapshot(ctx, snapc, namespace, - volumeSnapshot, pandoraSyncWaitTime, volHandle, snapshotId) + volumeSnapshot, pandoraSyncWaitTime, volHandle, snapshotId, true) gomega.Expect(err).NotTo(gomega.HaveOccurred()) ginkgo.By("Expanding current pvc after deleting volume snapshot") @@ -3300,11 +3302,12 @@ var _ = ginkgo.Describe("Volume Snapshot Basic Test", func() { storageclass, err := createStorageClass(client, scParameters, nil, "", "", true, scName) gomega.Expect(err).NotTo(gomega.HaveOccurred()) - // var allowExpansion = true - // storageclass.AllowVolumeExpansion = &allowExpansion - // storageclass, err = client.StorageV1().StorageClasses().Update(ctx, storageclass, metav1.UpdateOptions{}) - // gomega.Expect(err).NotTo(gomega.HaveOccurred()) - + if !vanillaCluster { + var allowExpansion = true + storageclass.AllowVolumeExpansion = &allowExpansion + storageclass, err = client.StorageV1().StorageClasses().Update(ctx, storageclass, metav1.UpdateOptions{}) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + } defer func() { if vanillaCluster { err := client.StorageV1().StorageClasses().Delete(ctx, storageclass.Name, *metav1.NewDeleteOptions(0)) @@ -3439,7 +3442,7 @@ var _ = ginkgo.Describe("Volume Snapshot Basic Test", func() { ginkgo.By("Delete dynamic volume snapshot") snapshotCreated, snapshotContentCreated, err = deleteVolumeSnapshot(ctx, snapc, namespace, - volumeSnapshot, pandoraSyncWaitTime, volHandle, snapshotId) + volumeSnapshot, pandoraSyncWaitTime, volHandle, snapshotId, true) gomega.Expect(err).NotTo(gomega.HaveOccurred()) ginkgo.By("Waiting for file system resize to finish") @@ -3521,7 +3524,7 @@ var _ = ginkgo.Describe("Volume Snapshot Basic Test", func() { 8. cleanup the snapshots, restore-pvc and source-pvc */ - ginkgo.It("[block-vanilla-snapshot][supervisor-snapshot] Snapshot restore while the Host "+ + ginkgo.It("[block-vanilla-snapshot][supervisor-snapshot] TC27Snapshot restore while the Host "+ "is Down", ginkgo.Label(p0, block, vanilla, snapshot, disruptive), func() { ctx, cancel := context.WithCancel(context.Background()) @@ -4363,7 +4366,7 @@ var _ = ginkgo.Describe("Volume Snapshot Basic Test", func() { gomega.Expect(err).NotTo(gomega.HaveOccurred()) ginkgo.By("Delete volume snapshot") _, snapshotContentCreated, err = deleteVolumeSnapshot(ctx, snapc, namespace, - volumeSnapshot, pandoraSyncWaitTime, volHandle, snapshotId) + volumeSnapshot, pandoraSyncWaitTime, volHandle, snapshotId, true) gomega.Expect(err).NotTo(gomega.HaveOccurred()) } }) @@ -5070,12 +5073,12 @@ var _ = ginkgo.Describe("Volume Snapshot Basic Test", func() { ginkgo.By("Delete pre-provisioned snapshot") staticSnapshotCreated, staticSnapshotContentCreated, err = deleteVolumeSnapshot(ctx, snapc, namespace, - staticSnapshot, pandoraSyncWaitTime, volHandle, snapshotId) + staticSnapshot, pandoraSyncWaitTime, volHandle, snapshotId, true) gomega.Expect(err).NotTo(gomega.HaveOccurred()) ginkgo.By("Delete dynamic volume snapshot") snapshotCreated, snapshotContentCreated, err = deleteVolumeSnapshot(ctx, snapc, namespace, - volumeSnapshot, pandoraSyncWaitTime, volHandle, dynamicSnapshotId) + volumeSnapshot, pandoraSyncWaitTime, volHandle, dynamicSnapshotId, true) gomega.Expect(err).NotTo(gomega.HaveOccurred()) }) @@ -5256,17 +5259,17 @@ var _ = ginkgo.Describe("Volume Snapshot Basic Test", func() { ginkgo.By("Delete dynamic volume snapshot") snapshotCreated, snapshotContentCreated, err = deleteVolumeSnapshot(ctx, snapc, namespace, - volumeSnapshot3, pandoraSyncWaitTime, volHandle2, snapshotId3) + volumeSnapshot3, pandoraSyncWaitTime, volHandle2, snapshotId3, true) gomega.Expect(err).NotTo(gomega.HaveOccurred()) ginkgo.By("Delete pre-provisioned snapshot") staticSnapshotCreated, staticSnapshotContentCreated, err = deleteVolumeSnapshot(ctx, snapc, namespace, - staticSnapshot, pandoraSyncWaitTime, volHandle, snapshotId) + staticSnapshot, pandoraSyncWaitTime, volHandle, snapshotId, true) gomega.Expect(err).NotTo(gomega.HaveOccurred()) ginkgo.By("Delete dynamic volume snapshot") snapshotCreated, snapshotContentCreated, err = deleteVolumeSnapshot(ctx, snapc, namespace, - volumeSnapshot, pandoraSyncWaitTime, volHandle, snapshotId) + volumeSnapshot, pandoraSyncWaitTime, volHandle, snapshotId, true) gomega.Expect(err).NotTo(gomega.HaveOccurred()) }) @@ -5371,7 +5374,7 @@ var _ = ginkgo.Describe("Volume Snapshot Basic Test", func() { ginkgo.By("Delete static volume snapshot") staticSnapshotCreated, staticSnapshotContentCreated, err = deleteVolumeSnapshot(ctx, snapc, namespace, - staticSnapshot, pandoraSyncWaitTime, volHandle, staticSnapshotId) + staticSnapshot, pandoraSyncWaitTime, volHandle, staticSnapshotId, true) gomega.Expect(err).NotTo(gomega.HaveOccurred()) ginkgo.By("Create PVC using the snapshot deleted") @@ -5542,7 +5545,7 @@ var _ = ginkgo.Describe("Volume Snapshot Basic Test", func() { ginkgo.By("Delete pre-provisioned snapshot") staticSnapshotCreated, staticSnapshotContentCreated, err := deleteVolumeSnapshot(ctx, snapc, namespace, - staticSnapshot, pandoraSyncWaitTime, volHandle, snapshotId) + staticSnapshot, pandoraSyncWaitTime, volHandle, snapshotId, true) gomega.Expect(err).NotTo(gomega.HaveOccurred()) defer func() { if staticSnapshotCreated { @@ -5568,23 +5571,24 @@ var _ = ginkgo.Describe("Volume Snapshot Basic Test", func() { ginkgo.By("Delete dynamic volume snapshot") snapshotCreated, snapshotContentCreated, err = deleteVolumeSnapshot(ctx, snapc, namespace, - volumeSnapshot, pandoraSyncWaitTime, volHandle, snapshotId) + volumeSnapshot, pandoraSyncWaitTime, volHandle, snapshotId, true) gomega.Expect(err).NotTo(gomega.HaveOccurred()) }) /* - Perform online resize on restored volume - 1. Create a Storage Class, a PVC and attach the PVC to a Pod, write a file - 2. Create dynamically provisioned snapshots using this PVC - 3. Create new volume using this snapshots as source, use the same SC and attach it to a Pod. - 4. Ensure the PVC gets provisioned and is Bound. - 5. Verify the previous snapshot data is intact and write new data to restored volume - 6. Perform online resize on the restored volume and make sure resize should go fine. - 7. Create dynamically provisioned snapshots using the PVC created in step #4 - 8. Verify snapshot size. It should be same as that of restored volume size. - 9. Run cleanup: Delete snapshots, restored-volumes, pods. + PVC → Pod → Snapshot → RestoreVol → Pod →  OnlineResize → Snapshot + Perform online resize on restored volume + 1. Create a Storage Class, a PVC and attach the PVC to a Pod, write a file + 2. Create dynamically provisioned snapshots using this PVC + 3. Create new volume using this snapshots as source, use the same SC and attach it to a Pod. + 4. Ensure the PVC gets provisioned and is Bound. + 5. Verify the previous snapshot data is intact and write new data to restored volume + 6. Perform online resize on the restored volume and make sure resize should go fine. + 7. Create dynamically provisioned snapshots using the PVC created in step #4 + 8. Verify snapshot size. It should be same as that of restored volume size. + 9. Run cleanup: Delete snapshots, restored-volumes, pods. */ - ginkgo.It("[tkg-snapshot][supervisor-snapshot] Perform online resize on restored volume", ginkgo.Label(p0, + ginkgo.It("[tkg-snapshot][supervisor-snapshot] TC21Perform online resize on restored volume", ginkgo.Label(p0, snapshot, tkg, newTest, stable), func() { ctx, cancel := context.WithCancel(context.Background()) @@ -5655,6 +5659,10 @@ var _ = ginkgo.Describe("Volume Snapshot Basic Test", func() { output = e2ekubectl.RunKubectlOrDie(namespace, cmd...) gomega.Expect(strings.Contains(output, "Hello message from test into Pod")).NotTo(gomega.BeFalse()) + wrtiecmd2 := []string{"exec", pod.Name, "--namespace=" + namespace, "--", "/bin/sh", "-c", + "echo 'fsync' "} + e2ekubectl.RunKubectlOrDie(namespace, wrtiecmd2...) + ginkgo.By("Create/Get volume snapshot class") volumeSnapshotClass, err := createVolumeSnapshotClass(ctx, snapc, deletionPolicy) gomega.Expect(err).NotTo(gomega.HaveOccurred()) @@ -5711,7 +5719,7 @@ var _ = ginkgo.Describe("Volume Snapshot Basic Test", func() { ginkgo.By("Perform online resize on the restored volume and make sure resize should go fine") verifyOnlineVolumeExpansionOnGc(client, namespace, svcPVCName2, volHandle, pvclaim2, pod2, f) - ginkgo.By("Create a volume snapshot from restored volume") + ginkgo.By("Create a snapshot for the restored volume") volumeSnapshotFromRestoreVol, snapshotContentFromRestoreVol, snapshotCreatedFromRestoreVol, snapshotContentCreatedFromRestoreVol, snapshotIdFromRestoreVol, _, err := createDynamicVolumeSnapshot(ctx, namespace, snapc, volumeSnapshotClass, pvclaim2, volHandle2, "3Gi", true) @@ -5733,11 +5741,11 @@ var _ = ginkgo.Describe("Volume Snapshot Basic Test", func() { framework.Logf("Deleting volume snapshot") snapshotCreated, snapshotContentCreated, err = deleteVolumeSnapshot(ctx, snapc, namespace, - volumeSnapshot, pandoraSyncWaitTime, volHandle, snapshotId) + volumeSnapshot, pandoraSyncWaitTime, volHandle, snapshotId, true) gomega.Expect(err).NotTo(gomega.HaveOccurred()) snapshotCreatedFromRestoreVol, snapshotContentCreatedFromRestoreVol, err = deleteVolumeSnapshot(ctx, snapc, namespace, - volumeSnapshotFromRestoreVol, pandoraSyncWaitTime, volHandle2, snapshotIdFromRestoreVol) + volumeSnapshotFromRestoreVol, pandoraSyncWaitTime, volHandle2, snapshotIdFromRestoreVol, true) gomega.Expect(err).NotTo(gomega.HaveOccurred()) }) @@ -5753,7 +5761,7 @@ var _ = ginkgo.Describe("Volume Snapshot Basic Test", func() { 7. Run cleanup: Delete snapshots, restored-volumes, pods. */ - ginkgo.It("[tkg-snapshot][supervisor-snapshot] Offline relocation of FCD with snapshots", ginkgo.Label(p0, snapshot, tkg, + ginkgo.It("[tkg-snapshot][supervisor-snapshot] TC22Offline relocation of FCD with snapshots", ginkgo.Label(p0, snapshot, tkg, newTest, stable), func() { ctx, cancel := context.WithCancel(context.Background()) @@ -5890,12 +5898,12 @@ var _ = ginkgo.Describe("Volume Snapshot Basic Test", func() { framework.Logf("Deleting volume snapshot-1") snapshotCreated, snapshotContentCreated, err = deleteVolumeSnapshot(ctx, snapc, namespace, - volumeSnapshot, pandoraSyncWaitTime, volHandle, snapshotId) + volumeSnapshot, pandoraSyncWaitTime, volHandle, snapshotId, true) gomega.Expect(err).NotTo(gomega.HaveOccurred()) framework.Logf("Deleting volume snapshot-2") snapshotCreated2, snapshotContentCreated2, err = deleteVolumeSnapshot(ctx, snapc, namespace, - volumeSnapshot2, pandoraSyncWaitTime, volHandle2, snapshotId) + volumeSnapshot2, pandoraSyncWaitTime, volHandle2, snapshotId, true) gomega.Expect(err).NotTo(gomega.HaveOccurred()) }) @@ -6022,7 +6030,7 @@ var _ = ginkgo.Describe("Volume Snapshot Basic Test", func() { ginkgo.By("Delete snapshot") snapshotCreated, snapshotContentCreated, err = deleteVolumeSnapshot(ctx, snapc, namespace, - volumeSnapshot, pandoraSyncWaitTime, volHandle, snapshotId) + volumeSnapshot, pandoraSyncWaitTime, volHandle, snapshotId, true) gomega.Expect(err).NotTo(gomega.HaveOccurred()) }) @@ -6036,7 +6044,7 @@ var _ = ginkgo.Describe("Volume Snapshot Basic Test", func() { 5. Restore PVC creation should fail and be stuck in Pending state with appropriate error message. 6. Perform Cleanup. */ - ginkgo.It("[tkg-snapshot][supervisor-snapshot] Volume mode conversion", ginkgo.Label(p0, snapshot, tkg, newTest, stable, + ginkgo.It("[tkg-snapshot][supervisor-snapshot] TC18Volume mode conversion", ginkgo.Label(p0, snapshot, tkg, newTest, stable, negative), func() { ctx, cancel := context.WithCancel(context.Background()) @@ -6098,18 +6106,19 @@ var _ = ginkgo.Describe("Volume Snapshot Basic Test", func() { } }() + ginkgo.By("Creating a PVC from a snapshot but with different access mode") accessModes := []v1.PersistentVolumeAccessMode{v1.ReadWriteMany, v1.ReadOnlyMany} - for _, accessMode := range accessModes { ginkgo.By(fmt.Sprintf("Create PVC from snapshot with %s access mode", accessMode)) pvcSpec := getPersistentVolumeClaimSpecWithDatasource(namespace, diskSize, storageclass, nil, accessMode, volumeSnapshot.Name, snapshotapigroup) pvclaim2, err := fpv.CreatePVC(ctx, client, namespace, pvcSpec) + pvclaims = append(pvclaims, pvclaim2) gomega.Expect(err).NotTo(gomega.HaveOccurred()) _, err = fpv.WaitForPVClaimBoundPhase(ctx, client, - []*v1.PersistentVolumeClaim{pvclaim2}, framework.ClaimProvisionTimeout) + []*v1.PersistentVolumeClaim{pvclaim2}, framework.ClaimProvisionShortTimeout) framework.Logf("Error from creating pvc with %s accessmode is : %s", accessMode, err.Error()) gomega.Expect(err).To(gomega.HaveOccurred()) @@ -6133,6 +6142,19 @@ var _ = ginkgo.Describe("Volume Snapshot Basic Test", func() { } }() + ginkgo.By("Deleting a PVC which is stuck in Pending state") + for _, pvclaim := range pvclaims { + pv := getPvFromClaim(client, pvclaim.Namespace, pvclaim.Name) + volHandle = pv.Spec.CSI.VolumeHandle + if guestCluster { + volHandle = getVolumeIDFromSupervisorCluster(volHandle) + } + err := fpv.DeletePersistentVolumeClaim(ctx, client, pvclaim.Name, namespace) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + err = e2eVSphere.waitForCNSVolumeToBeDeleted(volHandle) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + } + if guestCluster { framework.Logf("Deleting pending PVCs from SVC namespace") pvcList := getAllPVCFromNamespace(svcClient, svcNamespace) @@ -6146,7 +6168,7 @@ var _ = ginkgo.Describe("Volume Snapshot Basic Test", func() { ginkgo.By("Delete dynamic volume snapshot") snapshotCreated, snapshotContentCreated, err = deleteVolumeSnapshot(ctx, snapc, namespace, - volumeSnapshot, pandoraSyncWaitTime, volHandle, snapshotId) + volumeSnapshot, pandoraSyncWaitTime, volHandle, snapshotId, true) gomega.Expect(err).NotTo(gomega.HaveOccurred()) }) @@ -6334,11 +6356,11 @@ var _ = ginkgo.Describe("Volume Snapshot Basic Test", func() { ginkgo.By("Delete dynamic volume snapshot") snapshotCreated1, snapshotContentCreated1, err = deleteVolumeSnapshot(ctx, snapc, namespace, - volumeSnapshot1, pandoraSyncWaitTime, volHandle1, snapshotId1) + volumeSnapshot1, pandoraSyncWaitTime, volHandle1, snapshotId1, true) gomega.Expect(err).NotTo(gomega.HaveOccurred()) snapshotCreated2, snapshotContentCreated2, err = deleteVolumeSnapshot(ctx, snapc, namespace, - volumeSnapshot2, pandoraSyncWaitTime, volHandle2, snapshotId2) + volumeSnapshot2, pandoraSyncWaitTime, volHandle2, snapshotId2, true) gomega.Expect(err).NotTo(gomega.HaveOccurred()) }) @@ -6466,7 +6488,7 @@ var _ = ginkgo.Describe("Volume Snapshot Basic Test", func() { ginkgo.By("Delete Dynamic snapshot") snapshotCreated, snapshotContentCreated, err = deleteVolumeSnapshot(ctx, snapc, namespace, - volumeSnapshot, pandoraSyncWaitTime, volHandle, snapshotId) + volumeSnapshot, pandoraSyncWaitTime, volHandle, snapshotId, true) gomega.Expect(err).NotTo(gomega.HaveOccurred()) }) @@ -6519,7 +6541,7 @@ var _ = ginkgo.Describe("Volume Snapshot Basic Test", func() { fcdID, err := e2eVSphere.createFCDwithValidProfileID(ctx, "staticfcd"+curtimestring, profileID, diskSizeInMb, defaultDatastore.Reference()) gomega.Expect(err).NotTo(gomega.HaveOccurred()) - deleteFCDRequired = false + //deleteFCDRequired = false ginkgo.By(fmt.Sprintf("Sleeping for %v seconds to allow newly created FCD:%s to sync with pandora", pandoraSyncWaitTime, fcdID)) time.Sleep(time.Duration(pandoraSyncWaitTime) * time.Second) @@ -6729,7 +6751,7 @@ var _ = ginkgo.Describe("Volume Snapshot Basic Test", func() { b) snapshot create/delete workflow c) Restart services */ - ginkgo.It("[tkg-snapshot] Scale up snapshot creation by increasing the volume counts and "+ + ginkgo.It("[tkg-snapshot] Tc23Scale up snapshot creation by increasing the volume counts and "+ "in between restart services", ginkgo.Label(p1, snapshot, tkg, newTest), func() { ctx, cancel := context.WithCancel(context.Background()) @@ -6966,7 +6988,7 @@ var _ = ginkgo.Describe("Volume Snapshot Basic Test", func() { for i := 0; i < noOfSnapshotToCreate; i++ { ginkgo.By("Delete dynamic volume snapshot") _, _, err = deleteVolumeSnapshot(ctx, snapc, namespace, - volumeSnapshots[i], pandoraSyncWaitTime, volHandle, snapshotIds[i]) + volumeSnapshots[i], pandoraSyncWaitTime, volHandle, snapshotIds[i], true) gomega.Expect(err).NotTo(gomega.HaveOccurred()) } } @@ -6975,7 +6997,7 @@ var _ = ginkgo.Describe("Volume Snapshot Basic Test", func() { for i := 0; i < noOfSnapshotToCreate; i++ { ginkgo.By("Delete dynamic volume snapshot") _, _, err = deleteVolumeSnapshot(ctx, snapc, namespace, - volumeSnapshots[i], pandoraSyncWaitTime, volHandle, snapshotIds[i]) + volumeSnapshots[i], pandoraSyncWaitTime, volHandle, snapshotIds[i], true) gomega.Expect(err).NotTo(gomega.HaveOccurred()) } snapDeleted = true @@ -7161,12 +7183,12 @@ var _ = ginkgo.Describe("Volume Snapshot Basic Test", func() { ginkgo.By("Delete pre-provisioned snapshot from GC2") staticSnapshotCreated, staticSnapshotContentCreated, err = deleteVolumeSnapshot(ctx, snapc1, namespace2.Name, - staticSnapshot, pandoraSyncWaitTime, volHandle, snapshotId) + staticSnapshot, pandoraSyncWaitTime, volHandle, snapshotId, true) gomega.Expect(err).NotTo(gomega.HaveOccurred()) ginkgo.By("Delete snapshot entries from GC1 in case left") snapshotCreated, snapshotContentCreated, err = deleteVolumeSnapshot(ctx, snapc, namespace1.Name, - volumeSnapshot, pandoraSyncWaitTime, volHandle, dynamicSnapshotId) + volumeSnapshot, pandoraSyncWaitTime, volHandle, dynamicSnapshotId, true) gomega.Expect(err).NotTo(gomega.HaveOccurred()) }) @@ -7329,7 +7351,7 @@ var _ = ginkgo.Describe("Volume Snapshot Basic Test", func() { ginkgo.By("Delete dynamic volume snapshot") snapshotCreated, snapshotContentCreated, err = deleteVolumeSnapshot(ctx, snapc, namespace, - volumeSnapshot, pandoraSyncWaitTime, volHandle, snapshotId) + volumeSnapshot, pandoraSyncWaitTime, volHandle, snapshotId, true) gomega.Expect(err).NotTo(gomega.HaveOccurred()) }) @@ -7350,7 +7372,7 @@ var _ = ginkgo.Describe("Volume Snapshot Basic Test", func() { 11. Verify CRD deleted automatically. */ - ginkgo.It("[supervisor-snapshot] Verify static provisioning workflow with snapshot", ginkgo.Label(p0, block, wcp), func() { + ginkgo.It("[supervisor-snapshot] TC20Verify static provisioning workflow with snapshot", ginkgo.Label(p0, block, wcp), func() { ctx, cancel := context.WithCancel(context.Background()) defer cancel() @@ -7364,17 +7386,16 @@ var _ = ginkgo.Describe("Volume Snapshot Basic Test", func() { framework.Logf("Storage class : ", storageclass.Name) ginkgo.By("Creating FCD (CNS Volume)") - fcdID, err := e2eVSphere.createFCDwithValidProfileID(ctx, - "staticfcd"+curtimestring, profileID, diskSizeInMb, defaultDatastore.Reference()) + fcdID, err := e2eVSphere.createFCDwithValidProfileID(ctx, "staticfcd"+curtimestring, profileID, diskSizeInMb, defaultDatastore.Reference()) gomega.Expect(err).NotTo(gomega.HaveOccurred()) - deleteFCDRequired = true - defer func() { - if deleteFCDRequired { - ginkgo.By(fmt.Sprintf("Deleting FCD: %s", fcdID)) - err := e2eVSphere.deleteFCD(ctx, fcdID, defaultDatastore.Reference()) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - } - }() + //deleteFCDRequired = true + // defer func() { + // if deleteFCDRequired { + // ginkgo.By(fmt.Sprintf("Deleting FCD: %s", fcdID)) + // err := e2eVSphere.deleteFCD(ctx, fcdID, defaultDatastore.Reference()) + // gomega.Expect(err).NotTo(gomega.HaveOccurred()) + // } + // }() ginkgo.By(fmt.Sprintf("Sleeping for %v seconds to allow newly created FCD:%s to sync with pandora", pandoraSyncWaitTime, fcdID)) @@ -7393,6 +7414,7 @@ var _ = ginkgo.Describe("Volume Snapshot Basic Test", func() { pvc, err := client.CoreV1().PersistentVolumeClaims(namespace).Get(ctx, pvcName, metav1.GetOptions{}) gomega.Expect(err).NotTo(gomega.HaveOccurred()) pv := getPvFromClaim(client, namespace, pvcName) + volHandle = pv.Spec.CSI.VolumeHandle verifyBidirectionalReferenceOfPVandPVC(ctx, client, pvc, pv, fcdID) ginkgo.By("Creating pod") @@ -7405,12 +7427,17 @@ var _ = ginkgo.Describe("Volume Snapshot Basic Test", func() { pv.Spec.CSI.VolumeHandle, pod.Spec.NodeName)) var vmUUID string var exists bool + annotations := pod.Annotations vmUUID, exists = annotations[vmUUIDLabel] gomega.Expect(exists).To(gomega.BeTrue(), fmt.Sprintf("Pod doesn't have %s annotation", vmUUIDLabel)) _, err = e2eVSphere.getVMByUUID(ctx, vmUUID) gomega.Expect(err).NotTo(gomega.HaveOccurred()) + isDiskAttached, err := e2eVSphere.isVolumeAttachedToVM(client, pv.Spec.CSI.VolumeHandle, vmUUID) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + gomega.Expect(isDiskAttached).To(gomega.BeTrue(), "Volume is not attached to the node") + ginkgo.By("Create volume snapshot class") volumeSnapshotClass, err := createVolumeSnapshotClass(ctx, snapc, deletionPolicy) gomega.Expect(err).NotTo(gomega.HaveOccurred()) @@ -7425,7 +7452,7 @@ var _ = ginkgo.Describe("Volume Snapshot Basic Test", func() { ginkgo.By("Create a dynamic volume snapshot") volumeSnapshot, snapshotContent, snapshotCreated, snapshotContentCreated, snapshotId, _, err := createDynamicVolumeSnapshot(ctx, namespace, snapc, volumeSnapshotClass, - pvc, volHandle, diskSize, true) + pvc, volHandle, diskSize, false) gomega.Expect(err).NotTo(gomega.HaveOccurred()) defer func() { if snapshotContentCreated { @@ -7460,7 +7487,7 @@ var _ = ginkgo.Describe("Volume Snapshot Basic Test", func() { ginkgo.By("Delete dynamic volume snapshot") snapshotCreated, snapshotContentCreated, err = deleteVolumeSnapshot(ctx, snapc, namespace, - volumeSnapshot, pandoraSyncWaitTime, volHandle, snapshotId) + volumeSnapshot, pandoraSyncWaitTime, volHandle, snapshotId, true) gomega.Expect(err).NotTo(gomega.HaveOccurred()) }) }) @@ -7623,6 +7650,6 @@ func invokeSnapshotOperationsOnSharedDatastore(client clientset.Interface, ctx c ginkgo.By("Delete Dynamic snapshot") snapshotCreated, snapshotContentCreated, err = deleteVolumeSnapshot(ctx, snapc, namespace, - volumeSnapshot, pandoraSyncWaitTime, volHandle, snapshotId) + volumeSnapshot, pandoraSyncWaitTime, volHandle, snapshotId, true) gomega.Expect(err).NotTo(gomega.HaveOccurred()) } diff --git a/tests/e2e/csi_snapshot_utils.go b/tests/e2e/csi_snapshot_utils.go index 02c0707f7d..6961672695 100644 --- a/tests/e2e/csi_snapshot_utils.go +++ b/tests/e2e/csi_snapshot_utils.go @@ -301,7 +301,7 @@ func getRestConfigClientForGuestCluster(guestClusterRestConfig *rest.Config) *re // deleteVolumeSnapshot deletes volume snapshot from K8s side and CNS side func deleteVolumeSnapshot(ctx context.Context, snapc *snapclient.Clientset, namespace string, volumeSnapshot *snapV1.VolumeSnapshot, pandoraSyncWaitTime int, - volHandle string, snapshotID string) (bool, bool, error) { + volHandle string, snapshotID string, performCnsQueryVolumeSnapshot bool) (bool, bool, error) { var err error framework.Logf("Delete volume snapshot and verify the snapshot content is deleted") diff --git a/tests/e2e/raw_block_volume.go b/tests/e2e/raw_block_volume.go index f71b4e9b3f..8d26912f2d 100644 --- a/tests/e2e/raw_block_volume.go +++ b/tests/e2e/raw_block_volume.go @@ -1096,7 +1096,7 @@ var _ = ginkgo.Describe("raw block volume support", func() { 15. Query the snapshot from CNS side - should return 0 entries 16. Cleanup: Delete PVC, SC (validate they are removed) */ - ginkgo.It("[block-vanilla-snapshot] [tkg-snapshot][supervisor-snapshot] Verify snapshot dynamic provisioning workflow with "+ + ginkgo.It("[block-vanilla-snapshot] [tkg-snapshot][supervisor-snapshot] TC17Verify snapshot dynamic provisioning workflow with "+ "raw block volume", func() { ctx, cancel := context.WithCancel(context.Background()) defer cancel() @@ -1315,7 +1315,7 @@ var _ = ginkgo.Describe("raw block volume support", func() { ginkgo.By("Delete dyanmic volume snapshot") snapshotCreated, snapshotContentCreated, err = deleteVolumeSnapshot(ctx, snapc, namespace, - volumeSnapshot, pandoraSyncWaitTime, volumeID, snapshotId) + volumeSnapshot, pandoraSyncWaitTime, volumeID, snapshotId, true) gomega.Expect(err).NotTo(gomega.HaveOccurred()) }) }) diff --git a/tests/e2e/snapshot_vmservice_vm.go b/tests/e2e/snapshot_vmservice_vm.go index 720c772ad1..a58fd2f798 100644 --- a/tests/e2e/snapshot_vmservice_vm.go +++ b/tests/e2e/snapshot_vmservice_vm.go @@ -27,10 +27,12 @@ import ( snapclient "github.com/kubernetes-csi/external-snapshotter/client/v6/clientset/versioned" "github.com/onsi/ginkgo/v2" "github.com/onsi/gomega" + cnstypes "github.com/vmware/govmomi/cns/types" "github.com/vmware/govmomi/vim25/types" vmopv1 "github.com/vmware-tanzu/vm-operator/api/v1alpha1" v1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" clientset "k8s.io/client-go/kubernetes" @@ -313,7 +315,7 @@ var _ bool = ginkgo.Describe("[vmsvc] vm service with csi vol tests", func() { ginkgo.By("Delete dynamic volume snapshot") snapshotCreated, snapshotContentCreated, err = deleteVolumeSnapshot(ctx, snapc, namespace, - volumeSnapshot, pandoraSyncWaitTime, volHandle, snapshotId) + volumeSnapshot, pandoraSyncWaitTime, volHandle, snapshotId, true) gomega.Expect(err).NotTo(gomega.HaveOccurred()) }) @@ -446,7 +448,7 @@ var _ bool = ginkgo.Describe("[vmsvc] vm service with csi vol tests", func() { ginkgo.By("Delete dynamic volume snapshot") snapshotCreated, snapshotContentCreated, err = deleteVolumeSnapshot(ctx, snapc, namespace, - volumeSnapshot, pandoraSyncWaitTime, volHandle, snapshotId) + volumeSnapshot, pandoraSyncWaitTime, volHandle, snapshotId, true) gomega.Expect(err).NotTo(gomega.HaveOccurred()) }) @@ -618,7 +620,7 @@ var _ bool = ginkgo.Describe("[vmsvc] vm service with csi vol tests", func() { ginkgo.By("Delete dynamic volume snapshot") snapshotCreated, snapshotContentCreated, err = deleteVolumeSnapshot(ctx, snapc, namespace, - volumeSnapshot, pandoraSyncWaitTime, volHandle, snapshotId) + volumeSnapshot, pandoraSyncWaitTime, volHandle, snapshotId, true) gomega.Expect(err).NotTo(gomega.HaveOccurred()) }) @@ -635,18 +637,17 @@ var _ bool = ginkgo.Describe("[vmsvc] vm service with csi vol tests", func() { 7. Get VolumeSnapshotClass "volumesnapshotclass-delete" from supervisor cluster 8. Take 2 snapshots (snapshot-1, snapshot-2) for the PVC created in step #1. 9. Snapshot Verification: Execute and verify the steps mentioned in the Create snapshot mandatory checks - - Verify CNS metadata for a PVC created in step #1 - Create PVC-1 from Snapshot-1, PVC-2 from Snapshot-2 - Wait for PVCs to reach the Bound state. - Create a VM service VM using pvc-1 and pvc-2 created in step #11 - Wait for the VM service to be up and in the powered-on state. - Once the VM is up, verify that the volume is accessible inside the VM - Write some IO to the CSI volumes, read it back from them and verify the data integrity - Cleanup: Execute and verify the steps mentioned in the Delete snapshot mandatory checks + 10. Verify CNS metadata for a PVC created in step #1 + 11. Create PVC-1 from Snapshot-1, PVC-2 from Snapshot-2 + 12. Wait for PVCs to reach the Bound state. + 13. Create a VM service VM using pvc-1 and pvc-2 created in step #11 + 14. Wait for the VM service to be up and in the powered-on state. + 15. Once the VM is up, verify that the volume is accessible inside the VM + 16. Write some IO to the CSI volumes, read it back from them and verify the data integrity + 17. Cleanup: Execute and verify the steps mentioned in the Delete snapshot mandatory checks */ - ginkgo.It("VM3", func() { + ginkgo.It("VM4", func() { ctx, cancel := context.WithCancel(context.Background()) defer cancel() @@ -805,7 +806,7 @@ var _ bool = ginkgo.Describe("[vmsvc] vm service with csi vol tests", func() { ginkgo.By("Wait and verify PVCs are attached to the VM") gomega.Expect(waitNverifyPvcsAreAttachedToVmsvcVm(ctx, vmopC, cnsopC, vm2, - []*v1.PersistentVolumeClaim{pvc2})).NotTo(gomega.HaveOccurred()) + []*v1.PersistentVolumeClaim{pvc2, pvc3})).NotTo(gomega.HaveOccurred()) ginkgo.By("Verify PVCs are accessible to the VM") ginkgo.By("Write some IO to the CSI volumes and read it back from them and verify the data integrity") @@ -818,12 +819,288 @@ var _ bool = ginkgo.Describe("[vmsvc] vm service with csi vol tests", func() { ginkgo.By("Delete dynamic volume snapshot") snapshotCreated1, snapshotContentCreated1, err = deleteVolumeSnapshot(ctx, snapc, namespace, - volumeSnapshot1, pandoraSyncWaitTime, volHandle, snapshotId1) + volumeSnapshot1, pandoraSyncWaitTime, volHandle, snapshotId1, true) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + + ginkgo.By("Delete dynamic volume snapshot") + snapshotCreated2, snapshotContentCreated2, err = deleteVolumeSnapshot(ctx, snapc, namespace, + volumeSnapshot2, pandoraSyncWaitTime, volHandle, snapshotId2, true) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + }) + + /* + Testcase-5 + Offline and Online resize + PVC → VM → Snapshot → Delete VM → Restore Snapshot → Offline PVC resize → Create new VM → Online Resize → Snapshot + + Steps: + 1. Create a PVC using the storage class (storage policy) tagged to the supervisor namespace + 2. Wait for PVC to reach the Bound state. + 3. Create a VM service VM using the PVC created in step #1 + 4. Wait for the VM service to be up and in the powered-on state. + 5. Once the VM is up, verify that the volume is accessible inside the VM + 6. Write some data into the volume. + 7. Get VolumeSnapshotClass "volumesnapshotclass-delete" from supervisor cluster + 8. Take snapshots of the PVC created in step #1 + 9. Snapshot Verification: Execute and verify the steps mentioned in the Create snapshot mandatory checks + 10. Verify CNS metadata for a PVC created in step #1 + 11. Delete VM service VM created in step #3. + 12. Delete volume snapshot + 13. Perform offline volume resizing and verify that the operation went successfully. + 14. Create a new VM service VM with PVC created in step #1 + 15. Once the VM is up, verify that the volume is accessible and the filesystem on the volume has expanded. + 16. Perform online volume resizing and verify that the operation went successfully. + 17. Take a snapshot of the PVC created in step #1. + 18. Snapshot Verification: Execute and verify the steps mentioned in the Create snapshot mandatory checks + 19. Cleanup: Execute and verify the steps mentioned in the Delete snapshot mandatory checks + */ + + ginkgo.It("VM5", func() { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + ginkgo.By("Create a storageclass") + storageclass, err := client.StorageV1().StorageClasses().Get(ctx, storageClassName, metav1.GetOptions{}) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + + ginkgo.By("Create a PVC") + pvc, err := createPVC(ctx, client, namespace, labelsMap, "", storageclass, "") + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + ginkgo.By("Waiting for all claims to be in bound state") + pvs, err := fpv.WaitForPVClaimBoundPhase(ctx, client, []*v1.PersistentVolumeClaim{pvc}, pollTimeout) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + pv := pvs[0] + volHandle := pv.Spec.CSI.VolumeHandle + gomega.Expect(volHandle).NotTo(gomega.BeEmpty()) + defer func() { + ginkgo.By("Delete PVCs") + err = fpv.DeletePersistentVolumeClaim(ctx, client, pvc.Name, namespace) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + + ginkgo.By("Waiting for CNS volumes to be deleted") + err = e2eVSphere.waitForCNSVolumeToBeDeleted(volHandle) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + }() + + ginkgo.By("Creating VM bootstrap data") + // creating a secret for vm credentials + secretName := createBootstrapSecretForVmsvcVms(ctx, client, namespace) + defer func() { + ginkgo.By("Deleting VM bootstrap data") + err := client.CoreV1().Secrets(namespace).Delete(ctx, secretName, *metav1.NewDeleteOptions(0)) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + }() + + ginkgo.By("Creating VM") + vm := createVmServiceVmWithPvcs( + ctx, vmopC, namespace, vmClass, []*v1.PersistentVolumeClaim{pvc}, vmi, storageClassName, secretName) + defer func() { + ginkgo.By("Deleting VM") + err = vmopC.Delete(ctx, &vmopv1.VirtualMachine{ObjectMeta: metav1.ObjectMeta{ + Name: vm.Name, + Namespace: namespace, + }}) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + }() + + ginkgo.By("Creating loadbalancing service for ssh with the VM") + vmlbsvc := createService4Vm(ctx, vmopC, namespace, vm.Name) + defer func() { + ginkgo.By("Deleting loadbalancing service for ssh with the VM") + err = vmopC.Delete(ctx, &vmopv1.VirtualMachineService{ObjectMeta: metav1.ObjectMeta{ + Name: vmlbsvc.Name, + Namespace: namespace, + }}) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + }() + + ginkgo.By("Wait for VM to come up and get an IP") + vmIp, err := waitNgetVmsvcVmIp(ctx, vmopC, namespace, vm.Name) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + + ginkgo.By("Wait and verify PVCs are attached to the VM") + gomega.Expect(waitNverifyPvcsAreAttachedToVmsvcVm(ctx, vmopC, cnsopC, vm, + []*v1.PersistentVolumeClaim{pvc})).NotTo(gomega.HaveOccurred()) + + ginkgo.By("Verify PVCs are accessible to the VM") + ginkgo.By("Write some IO to the CSI volumes and read it back from them and verify the data integrity") + vm, err = getVmsvcVM(ctx, vmopC, vm.Namespace, vm.Name) // refresh vm info + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + for i, vol := range vm.Status.Volumes { + volFolder := formatNVerifyPvcIsAccessible(vol.DiskUuid, i+1, vmIp) + verifyDataIntegrityOnVmDisk(vmIp, volFolder) + } + + ginkgo.By("Create volume snapshot class") + volumeSnapshotClass, err := createVolumeSnapshotClass(ctx, snapc, deletionPolicy) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + defer func() { + if vanillaCluster { + err = snapc.SnapshotV1().VolumeSnapshotClasses().Delete(ctx, volumeSnapshotClass.Name, + metav1.DeleteOptions{}) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + } + }() + + ginkgo.By("Create a dynamic volume snapshot-1 for the volume") + volumeSnapshot1, snapshotContent1, snapshotCreated1, + snapshotContentCreated1, snapshotId1, _, err := createDynamicVolumeSnapshot(ctx, namespace, snapc, volumeSnapshotClass, + pvc, volHandle, diskSize, true) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + defer func() { + if snapshotContentCreated1 { + err = deleteVolumeSnapshotContent(ctx, snapshotContent1, snapc, pandoraSyncWaitTime) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + } + + if snapshotCreated1 { + framework.Logf("Deleting volume snapshot") + deleteVolumeSnapshotWithPandoraWait(ctx, snapc, namespace, volumeSnapshot1.Name, pandoraSyncWaitTime) + + framework.Logf("Wait till the volume snapshot is deleted") + err = waitForVolumeSnapshotContentToBeDeletedWithPandoraWait(ctx, snapc, + *volumeSnapshot1.Status.BoundVolumeSnapshotContentName, pandoraSyncWaitTime) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + } + }() + + ginkgo.By("Deleting VM") + err = vmopC.Delete(ctx, &vmopv1.VirtualMachine{ObjectMeta: metav1.ObjectMeta{ + Name: vm.Name, + Namespace: namespace, + }}) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + + ginkgo.By("Delete dynamic volume snapshot-1") + snapshotCreated1, snapshotContentCreated1, err = deleteVolumeSnapshot(ctx, snapc, namespace, + volumeSnapshot1, pandoraSyncWaitTime, volHandle, snapshotId1, true) gomega.Expect(err).NotTo(gomega.HaveOccurred()) + ginkgo.By("Expanding the current pvc") + currentPvcSize := pvc.Spec.Resources.Requests[v1.ResourceStorage] + newSize := currentPvcSize.DeepCopy() + newSize.Add(resource.MustParse("4Gi")) + framework.Logf("currentPvcSize %v, newSize %v", currentPvcSize, newSize) + pvc, err = expandPVCSize(pvc, newSize, client) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + gomega.Expect(pvc).NotTo(gomega.BeNil()) + pvcSize := pvc.Spec.Resources.Requests[v1.ResourceStorage] + if pvcSize.Cmp(newSize) != 0 { + framework.Failf("error updating pvc size %q", pvc.Name) + } + + ginkgo.By("Waiting for controller volume resize to finish") + err = waitForPvResizeForGivenPvc(pvc, client, totalResizeWaitPeriod) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + + ginkgo.By("Checking for conditions on pvc") + pvc, err = waitForPVCToReachFileSystemResizePendingCondition(client, + namespace, pvc.Name, pollTimeout) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + + ginkgo.By(fmt.Sprintf("Invoking QueryCNSVolumeWithResult with VolumeID: %s", volHandle)) + queryResult, err := e2eVSphere.queryCNSVolumeWithResult(volHandle) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + if len(queryResult.Volumes) == 0 { + err = fmt.Errorf("queryCNSVolumeWithResult returned no volume") + } + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + ginkgo.By("Verifying disk size requested in volume expansion is honored") + newSizeInMb := int64(6144) + if queryResult.Volumes[0].BackingObjectDetails.(*cnstypes.CnsBlockBackingDetails).CapacityInMb != + newSizeInMb { + err = fmt.Errorf("got wrong disk size after volume expansion +%v ", + queryResult.Volumes[0].BackingObjectDetails.(*cnstypes.CnsBlockBackingDetails).CapacityInMb) + } + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + + ginkgo.By("Creating VM") + vm2 := createVmServiceVmWithPvcs( + ctx, vmopC, namespace, vmClass, []*v1.PersistentVolumeClaim{pvc}, vmi, storageClassName, secretName) + defer func() { + ginkgo.By("Deleting VM") + err = vmopC.Delete(ctx, &vmopv1.VirtualMachine{ObjectMeta: metav1.ObjectMeta{ + Name: vm2.Name, + Namespace: namespace, + }}) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + }() + + ginkgo.By("Wait for VM to come up and get an IP") + vmIp2, err := waitNgetVmsvcVmIp(ctx, vmopC, namespace, vm2.Name) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + + ginkgo.By("Wait and verify PVCs are attached to the VM") + gomega.Expect(waitNverifyPvcsAreAttachedToVmsvcVm(ctx, vmopC, cnsopC, vm2, + []*v1.PersistentVolumeClaim{pvc})).NotTo(gomega.HaveOccurred()) + + ginkgo.By("Verify PVCs are accessible to the VM") + ginkgo.By("Write some IO to the CSI volumes and read it back from them and verify the data integrity") + vm, err = getVmsvcVM(ctx, vmopC, vm2.Namespace, vm2.Name) // refresh vm info + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + for i, vol := range vm.Status.Volumes { + volFolder := formatNVerifyPvcIsAccessible(vol.DiskUuid, i+1, vmIp2) + verifyDataIntegrityOnVmDisk(vmIp2, volFolder) + } + + ginkgo.By("Modify the PVC spec to enable online volume expansion when no snapshot exists for this PVC") + currentPvcSize = pvc.Spec.Resources.Requests[v1.ResourceStorage] + newSize = currentPvcSize.DeepCopy() + newSize.Add(resource.MustParse("4Gi")) + newDiskSize := "6Gi" + framework.Logf("currentPvcSize %v, newSize %v", currentPvcSize, newSize) + claims, err := expandPVCSize(pvc, newSize, client) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + gomega.Expect(claims).NotTo(gomega.BeNil()) + + ginkgo.By("Waiting for file system resize to finish") + claims, err = waitForFSResize(pvc, client) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + + pvcConditions := claims.Status.Conditions + expectEqual(len(pvcConditions), 0, "pvc should not have conditions") + + ginkgo.By(fmt.Sprintf("Invoking QueryCNSVolumeWithResult with VolumeID: %s", volHandle)) + queryResult, err = e2eVSphere.queryCNSVolumeWithResult(volHandle) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + if len(queryResult.Volumes) == 0 { + err = fmt.Errorf("queryCNSVolumeWithResult returned no volume") + } + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + ginkgo.By("Verifying disk size requested in volume expansion is honored") + newSizeInMb = int64(6144) + if queryResult.Volumes[0].BackingObjectDetails.(*cnstypes.CnsBlockBackingDetails).CapacityInMb != + newSizeInMb { + err = fmt.Errorf("got wrong disk size after volume expansion +%v ", + queryResult.Volumes[0].BackingObjectDetails.(*cnstypes.CnsBlockBackingDetails).CapacityInMb) + } + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + + ginkgo.By("Create a dynamic volume snapshot-2 for the volume") + volumeSnapshot2, snapshotContent2, snapshotCreated2, + snapshotContentCreated2, snapshotId2, _, err := createDynamicVolumeSnapshot(ctx, namespace, snapc, volumeSnapshotClass, + pvc, volHandle, newDiskSize, true) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + defer func() { + if snapshotContentCreated2 { + err = deleteVolumeSnapshotContent(ctx, snapshotContent2, snapc, pandoraSyncWaitTime) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + } + + if snapshotCreated2 { + framework.Logf("Deleting volume snapshot") + deleteVolumeSnapshotWithPandoraWait(ctx, snapc, namespace, volumeSnapshot2.Name, pandoraSyncWaitTime) + + framework.Logf("Wait till the volume snapshot is deleted") + err = waitForVolumeSnapshotContentToBeDeletedWithPandoraWait(ctx, snapc, + *volumeSnapshot2.Status.BoundVolumeSnapshotContentName, pandoraSyncWaitTime) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + } + }() + ginkgo.By("Delete dynamic volume snapshot") snapshotCreated2, snapshotContentCreated2, err = deleteVolumeSnapshot(ctx, snapc, namespace, - volumeSnapshot2, pandoraSyncWaitTime, volHandle, snapshotId2) + volumeSnapshot2, pandoraSyncWaitTime, volHandle, snapshotId2, true) gomega.Expect(err).NotTo(gomega.HaveOccurred()) }) }) diff --git a/tests/e2e/tkgs_ha.go b/tests/e2e/tkgs_ha.go index 53ba7b055c..7435fe82c6 100644 --- a/tests/e2e/tkgs_ha.go +++ b/tests/e2e/tkgs_ha.go @@ -319,7 +319,7 @@ var _ = ginkgo.Describe("[csi-tkgs-ha] Tkgs-HA-SanityTests", func() { ginkgo.By("Delete dynamic volume snapshot") snapshotCreated, snapshotContentCreated, err = deleteVolumeSnapshot(ctx, snapc, namespace, - volumeSnapshot, pandoraSyncWaitTime, volHandle, dynamicSnapshotId) + volumeSnapshot, pandoraSyncWaitTime, volHandle, dynamicSnapshotId, true) gomega.Expect(err).NotTo(gomega.HaveOccurred()) }) @@ -928,7 +928,7 @@ var _ = ginkgo.Describe("[csi-tkgs-ha] Tkgs-HA-SanityTests", func() { ginkgo.By("Delete dynamic volume snapshot") snapshotCreated, snapshotContentCreated, err = deleteVolumeSnapshot(ctx, snapc, namespace, - volumeSnapshot, pandoraSyncWaitTime, volumeID, dynamicSnapshotId) + volumeSnapshot, pandoraSyncWaitTime, volumeID, dynamicSnapshotId, true) gomega.Expect(err).NotTo(gomega.HaveOccurred()) }) @@ -1128,7 +1128,7 @@ var _ = ginkgo.Describe("[csi-tkgs-ha] Tkgs-HA-SanityTests", func() { ginkgo.By("Delete pre-provisioned snapshot") staticSnapshotCreated, staticSnapshotContentCreated, err = deleteVolumeSnapshot(ctx, snapc, namespace, - staticSnapshot, pandoraSyncWaitTime, volHandle, snapshotId) + staticSnapshot, pandoraSyncWaitTime, volHandle, snapshotId, true) gomega.Expect(err).NotTo(gomega.HaveOccurred()) }) @@ -1435,7 +1435,7 @@ var _ = ginkgo.Describe("[csi-tkgs-ha] Tkgs-HA-SanityTests", func() { ginkgo.By("Delete pre-provisioned snapshot") staticSnapshotCreated, staticSnapshotContentCreated, err = deleteVolumeSnapshot(ctx, snapc, namespace, - staticSnapshot, pandoraSyncWaitTime, volHandle, snapshotId) + staticSnapshot, pandoraSyncWaitTime, volHandle, snapshotId, true) gomega.Expect(err).NotTo(gomega.HaveOccurred()) }) diff --git a/tests/e2e/util.go b/tests/e2e/util.go index a0e2b0095b..1ac557d73f 100644 --- a/tests/e2e/util.go +++ b/tests/e2e/util.go @@ -3427,6 +3427,11 @@ func writeDataOnFileFromPod(namespace string, podName string, filePath string, d wrtiecmd := []string{"exec", podName, "--namespace=" + namespace, "--", shellExec, cmdArg, fmt.Sprintf(" echo '%s' > %s ", data, filePath)} e2ekubectl.RunKubectlOrDie(namespace, wrtiecmd...) + + data2 := "fsync" + wrtiecmd2 := []string{"exec", podName, "--namespace=" + namespace, "--", shellExec, cmdArg, + fmt.Sprintf(" echo '%s' > %s ", data2, filePath)} + e2ekubectl.RunKubectlOrDie(namespace, wrtiecmd2...) } // readFileFromPod read data from given Pod and the given file.