e2e: update snapshot restore e2e to check data consistency

Currently, in rbd snapshot restore and volume clone E2E we
are not checking any data consistency after doing snapshot
restore or volume clone. Hence, this PR writes the data in
the PVC and checks the checksum of the file and verify it with
the snapshot or cloned PVC.

Signed-off-by: Yati Padia <ypadia@redhat.com>

e2e/cephfs.go

@@ -682,7 +682,13 @@ var _ = Describe("cephfs", func() {
 
 					app.Namespace = f.UniqueName
 					app.Spec.Volumes[0].PersistentVolumeClaim.ClaimName = pvc.Name
-					wErr := writeDataInPod(app, f)
+					label := make(map[string]string)
+					label[appKey] = appLabel
+					app.Labels = label
+					opt := metav1.ListOptions{
+						LabelSelector: fmt.Sprintf("%s=%s", appKey, label[appKey]),
+					}
+					wErr := writeDataInPod(app, &opt, f)
 					if wErr != nil {
 						e2elog.Failf("failed to  write data  with error %v", wErr)
 					}
@@ -893,7 +899,13 @@ var _ = Describe("cephfs", func() {
 					}
 					app.Namespace = f.UniqueName
 					app.Spec.Volumes[0].PersistentVolumeClaim.ClaimName = pvc.Name
-					wErr := writeDataInPod(app, f)
+					label := make(map[string]string)
+					label[appKey] = appLabel
+					app.Labels = label
+					opt := metav1.ListOptions{
+						LabelSelector: fmt.Sprintf("%s=%s", appKey, label[appKey]),
+					}
+					wErr := writeDataInPod(app, &opt, f)
 					if wErr != nil {
 						e2elog.Failf("failed to write data from application %v", wErr)
 					}

e2e/rbd.go

@@ -473,6 +473,7 @@ var _ = Describe("RBD", func() {
 					var wg sync.WaitGroup
 					totalCount := 10
 					wgErrs := make([]error, totalCount)
+					chErrs := make([]error, totalCount)
 					wg.Add(totalCount)
 					err := createRBDSnapshotClass(f)
 					if err != nil {
@@ -482,12 +483,28 @@ var _ = Describe("RBD", func() {
 					if err != nil {
 						e2elog.Failf("failed to load PVC with error %v", err)
 					}
-
+					label := make(map[string]string)
 					pvc.Namespace = f.UniqueName
 					err = createPVCAndvalidatePV(f.ClientSet, pvc, deployTimeout)
 					if err != nil {
 						e2elog.Failf("failed to create PVC with error %v", err)
 					}
+					app, err := loadApp(appPath)
+					if err != nil {
+						e2elog.Failf("failed to load app with error %v", err)
+					}
+					// write data in PVC
+					label[appKey] = appLabel
+					app.Namespace = f.UniqueName
+					app.Labels = label
+					opt := metav1.ListOptions{
+						LabelSelector: fmt.Sprintf("%s=%s", appKey, label[appKey]),
+					}
+					app.Spec.Volumes[0].PersistentVolumeClaim.ClaimName = pvc.Name
+					checkSum, err := writeDataAndCalChecksum(app, &opt, f)
+					if err != nil {
+						e2elog.Failf("failed to calculate checksum with error %v", err)
+					}
 					validateRBDImageCount(f, 1)
 					snap := getSnapshot(snapshotPath)
 					snap.Namespace = f.UniqueName
@@ -533,7 +550,26 @@ var _ = Describe("RBD", func() {
 					for i := 0; i < totalCount; i++ {
 						go func(w *sync.WaitGroup, n int, p v1.PersistentVolumeClaim, a v1.Pod) {
 							name := fmt.Sprintf("%s%d", f.UniqueName, n)
+							label := make(map[string]string)
+							label[appKey] = name
+							a.Labels = label
+							opt := metav1.ListOptions{
+								LabelSelector: fmt.Sprintf("%s=%s", appKey, label[appKey]),
+							}
 							wgErrs[n] = createPVCAndApp(name, f, &p, &a, deployTimeout)
+							if wgErrs[n] == nil {
+								filePath := a.Spec.Containers[0].VolumeMounts[0].MountPath + "/test"
+								checkSumClone := ""
+								e2elog.Logf("calculating checksum clone for filepath %s", filePath)
+								checkSumClone, chErrs[n] = calculateSHA512sum(f, &a, filePath, &opt)
+								e2elog.Logf("checksum value for the clone is %s with pod name %s", checkSumClone, name)
+								if chErrs[n] != nil {
+									e2elog.Logf("failed to calculte checksum for clone with error %s", chErrs[n])
+								}
+								if checkSumClone != checkSum {
+									e2elog.Logf("checksum value didn't match. checksum=%s and checksumclone=%s", checkSum, checkSumClone)
+								}
+							}
 							w.Done()
 						}(&wg, i, *pvcClone, *appClone)
 					}
@@ -550,6 +586,16 @@ var _ = Describe("RBD", func() {
 						e2elog.Failf("creating PVCs and applications failed, %d errors were logged", failed)
 					}
 
+					for i, err := range chErrs {
+						if err != nil {
+							// not using Failf() as it aborts the test and does not log other errors
+							e2elog.Logf("failed to calculate checksum (%s%d): %v", f.UniqueName, i, err)
+							failed++
+						}
+					}
+					if failed != 0 {
+						e2elog.Failf("calculating checksum failed, %d errors were logged", failed)
+					}
 					// total images in cluster is 1 parent rbd image+ total
 					// snaps+ total clones
 					totalCloneCount := totalCount + totalCount + 1
@@ -671,9 +717,8 @@ var _ = Describe("RBD", func() {
 			By("create a PVC-PVC clone and bind it to an app", func() {
 				// pvc clone is only supported from v1.16+
 				if k8sVersionGreaterEquals(f.ClientSet, 1, 16) {
-					validatePVCClone(pvcPath, pvcSmartClonePath, appSmartClonePath, f)
+					validatePVCClone(pvcPath, appPath, pvcSmartClonePath, appSmartClonePath, f)
 				}
-
 			})
 
 			By("create a block type PVC and bind it to an app", func() {
@@ -689,7 +734,7 @@ var _ = Describe("RBD", func() {
 				}
 				// pvc clone is only supported from v1.16+
 				if v.Major > "1" || (v.Major == "1" && v.Minor >= "16") {
-					validatePVCClone(rawPvcPath, pvcBlockSmartClonePath, appBlockSmartClonePath, f)
+					validatePVCClone(rawPvcPath, rawAppPath, pvcBlockSmartClonePath, appBlockSmartClonePath, f)
 				}
 			})
 			By("create/delete multiple PVCs and Apps", func() {

e2e/upgrade-cephfs.go

@@ -205,7 +205,7 @@ var _ = Describe("CephFS Upgrade Testing", func() {
 				e2elog.Logf("Calculating checksum of %s", filePath)
 				checkSum, err = calculateSHA512sum(f, app, filePath, &opt)
 				if err != nil {
-					e2elog.Failf("failed to calculate checksum of %s", filePath)
+					e2elog.Failf("failed to calculate checksum with error %v", err)
 				}
 
 				// pvc clone is only supported from v1.16+
@@ -273,10 +273,9 @@ var _ = Describe("CephFS Upgrade Testing", func() {
 					}
 					mountPath := appClone.Spec.Containers[0].VolumeMounts[0].MountPath
 					testFilePath := filepath.Join(mountPath, "testClone")
-
 					newCheckSum, err = calculateSHA512sum(f, appClone, testFilePath, &opt)
 					if err != nil {
-						e2elog.Failf("failed to calculate checksum of %s", testFilePath)
+						e2elog.Failf("failed to calculate checksum with error %v", err)
 					}
 
 					if strings.Compare(newCheckSum, checkSum) != 0 {
@@ -337,7 +336,7 @@ var _ = Describe("CephFS Upgrade Testing", func() {
 					testFilePath := filepath.Join(mountPath, "testClone")
 					newCheckSum, err = calculateSHA512sum(f, appClone, testFilePath, &opt)
 					if err != nil {
-						e2elog.Failf("failed to calculate checksum of %s", testFilePath)
+						e2elog.Failf("failed to calculate checksum with error %v", err)
 					}
 
 					if strings.Compare(newCheckSum, checkSum) != 0 {

e2e/upgrade-rbd.go

@@ -223,7 +223,7 @@ var _ = Describe("RBD Upgrade Testing", func() {
 				e2elog.Logf("Calculating checksum of %s", filePath)
 				checkSum, err = calculateSHA512sum(f, app, filePath, &opt)
 				if err != nil {
-					e2elog.Failf("failed to calculate checksum of %s", filePath)
+					e2elog.Failf("failed to calculate checksum with error %v", err)
 				}
 
 				// pvc clone is only supported from v1.16+
@@ -293,7 +293,7 @@ var _ = Describe("RBD Upgrade Testing", func() {
 					testFilePath := filepath.Join(mountPath, "testClone")
 					newCheckSum, err := calculateSHA512sum(f, appClone, testFilePath, &opt)
 					if err != nil {
-						e2elog.Failf("failed to calculate checksum of %s", testFilePath)
+						e2elog.Failf("failed to calculate checksum with error %v", err)
 					}
 					if strings.Compare(newCheckSum, checkSum) != 0 {
 						e2elog.Failf("The checksum of files did not match, expected %s received %s  ", checkSum, newCheckSum)
@@ -342,7 +342,7 @@ var _ = Describe("RBD Upgrade Testing", func() {
 					testFilePath := filepath.Join(mountPath, "testClone")
 					newCheckSum, err := calculateSHA512sum(f, appClone, testFilePath, &opt)
 					if err != nil {
-						e2elog.Failf("failed to calculate checksum of %s", testFilePath)
+						e2elog.Failf("failed to calculate checksum with error %v", err)
 					}
 					if strings.Compare(newCheckSum, checkSum) != 0 {
 						e2elog.Failf("The checksum of files did not match, expected %s received %s  ", checkSum, newCheckSum)

e2e/utils.go

@@ -34,6 +34,9 @@ const (
 	retainPolicy = v1.PersistentVolumeReclaimRetain
 	// deletePolicy is the default policy in E2E.
 	deletePolicy = v1.PersistentVolumeReclaimDelete
+	// Default key and label for Listoptions
+	appKey   = "app"
+	appLabel = "write-data-in-pod"
 )
 
 var (
@@ -294,22 +297,19 @@ func validateNormalUserPVCAccess(pvcPath string, f *framework.Framework) error {
 }
 
 // writeDataInPod fill zero content to a file in the provided POD volume.
-func writeDataInPod(app *v1.Pod, f *framework.Framework) error {
+func writeDataInPod(app *v1.Pod, opt *metav1.ListOptions, f *framework.Framework) error {
-	app.Labels = map[string]string{"app": "write-data-in-pod"}
 	app.Namespace = f.UniqueName
 
 	err := createApp(f.ClientSet, app, deployTimeout)
 	if err != nil {
 		return err
 	}
-	opt := metav1.ListOptions{
+
-		LabelSelector: "app=write-data-in-pod",
-	}
 	// write data to PVC. The idea here is to fill some content in the file
 	// instead of filling and reverifying the md5sum/data integrity
 	filePath := app.Spec.Containers[0].VolumeMounts[0].MountPath + "/test"
 	// While writing more data we are encountering issues in E2E timeout, so keeping it low for now
-	_, writeErr, err := execCommandInPod(f, fmt.Sprintf("dd if=/dev/zero of=%s bs=1M count=10 status=none", filePath), app.Namespace, &opt)
+	_, writeErr, err := execCommandInPod(f, fmt.Sprintf("dd if=/dev/zero of=%s bs=1M count=10 status=none", filePath), app.Namespace, opt)
 	if err != nil {
 		return err
 	}
@@ -471,20 +471,68 @@ func enableTopologyInTemplate(data string) string {
 	return strings.ReplaceAll(data, "--feature-gates=Topology=false", "--feature-gates=Topology=true")
 }
 
-func validatePVCClone(sourcePvcPath, clonePvcPath, clonePvcAppPath string, f *framework.Framework) {
+func writeDataAndCalChecksum(app *v1.Pod, opt *metav1.ListOptions, f *framework.Framework) (string, error) {
+	filePath := app.Spec.Containers[0].VolumeMounts[0].MountPath + "/test"
+	// write data in PVC
+	err := writeDataInPod(app, opt, f)
+	if err != nil {
+		e2elog.Logf("failed to write data in the pod with error %v", err)
+		return "", err
+	}
+
+	checkSum, err := calculateSHA512sum(f, app, filePath, opt)
+	if err != nil {
+		e2elog.Logf("failed to calculate checksum with error %v", err)
+		return checkSum, err
+	}
+
+	err = deletePod(app.Name, app.Namespace, f.ClientSet, deployTimeout)
+	if err != nil {
+		e2elog.Failf("failed to delete pod with error %v", err)
+	}
+	return checkSum, nil
+}
+
+// nolint:gocyclo // reduce complexity
+func validatePVCClone(sourcePvcPath, sourceAppPath, clonePvcPath, clonePvcAppPath string, f *framework.Framework) {
 	var wg sync.WaitGroup
 	totalCount := 10
 	wgErrs := make([]error, totalCount)
+	chErrs := make([]error, totalCount)
 	pvc, err := loadPVC(sourcePvcPath)
 	if err != nil {
 		e2elog.Failf("failed to load PVC with error %v", err)
 	}
 
+	label := make(map[string]string)
 	pvc.Namespace = f.UniqueName
 	err = createPVCAndvalidatePV(f.ClientSet, pvc, deployTimeout)
 	if err != nil {
 		e2elog.Failf("failed to create PVC with error %v", err)
 	}
+	app, err := loadApp(sourceAppPath)
+	if err != nil {
+		e2elog.Failf("failed to load app with error %v", err)
+	}
+	label[appKey] = appLabel
+	app.Namespace = f.UniqueName
+	app.Spec.Volumes[0].PersistentVolumeClaim.ClaimName = pvc.Name
+	app.Labels = label
+	opt := metav1.ListOptions{
+		LabelSelector: fmt.Sprintf("%s=%s", appKey, label[appKey]),
+	}
+
+	checkSum := ""
+	pvc, err = f.ClientSet.CoreV1().PersistentVolumeClaims(pvc.Namespace).Get(context.TODO(), pvc.Name, metav1.GetOptions{})
+	if err != nil {
+		e2elog.Failf("failed to get pvc %v", err)
+	}
+	if *pvc.Spec.VolumeMode == v1.PersistentVolumeFilesystem {
+		checkSum, err = writeDataAndCalChecksum(app, &opt, f)
+		if err != nil {
+			e2elog.Failf("failed to calculate checksum with error %v", err)
+		}
+	}
 	// validate created backend rbd images
 	validateRBDImageCount(f, 1)
 	pvcClone, err := loadPVC(clonePvcPath)
@@ -503,7 +551,26 @@ func validatePVCClone(sourcePvcPath, clonePvcPath, clonePvcAppPath string, f *fr
 	for i := 0; i < totalCount; i++ {
 		go func(w *sync.WaitGroup, n int, p v1.PersistentVolumeClaim, a v1.Pod) {
 			name := fmt.Sprintf("%s%d", f.UniqueName, n)
+			label := make(map[string]string)
+			label[appKey] = name
+			a.Labels = label
+			opt := metav1.ListOptions{
+				LabelSelector: fmt.Sprintf("%s=%s", appKey, label[appKey]),
+			}
 			wgErrs[n] = createPVCAndApp(name, f, &p, &a, deployTimeout)
+			if *pvc.Spec.VolumeMode == v1.PersistentVolumeFilesystem && wgErrs[n] == nil {
+				filePath := a.Spec.Containers[0].VolumeMounts[0].MountPath + "/test"
+				checkSumClone := ""
+				e2elog.Logf("Calculating checksum clone for filepath %s", filePath)
+				checkSumClone, chErrs[n] = calculateSHA512sum(f, &a, filePath, &opt)
+				e2elog.Logf("checksum for clone is %s", checkSumClone)
+				if chErrs[n] != nil {
+					e2elog.Logf("Failed calculating checksum clone %s", chErrs[n])
+				}
+				if checkSumClone != checkSum {
+					e2elog.Logf("checksum didn't match. checksum=%s and checksumclone=%s", checkSum, checkSumClone)
+				}
+			}
 			w.Done()
 		}(&wg, i, *pvcClone, *appClone)
 	}
@@ -521,6 +588,17 @@ func validatePVCClone(sourcePvcPath, clonePvcPath, clonePvcAppPath string, f *fr
 		e2elog.Failf("creating PVCs failed, %d errors were logged", failed)
 	}
 
+	for i, err := range chErrs {
+		if err != nil {
+			// not using Failf() as it aborts the test and does not log other errors
+			e2elog.Logf("failed to calculate checksum (%s%d): %v", f.UniqueName, i, err)
+			failed++
+		}
+	}
+	if failed != 0 {
+		e2elog.Failf("calculating checksum failed, %d errors were logged", failed)
+	}
+
 	// total images in cluster is 1 parent rbd image+ total
 	// temporary clone+ total clones
 	totalCloneCount := totalCount + totalCount + 1