Add validation of backend rbd image and snapshots

Signed-off-by: Madhu Rajanna <madhupr007@gmail.com>

e2e/cephfs.go

@@ -1,6 +1,7 @@
 package e2e
 
 import (
+	"fmt"
 	"time"
 
 	. "github.com/onsi/ginkgo" // nolint
@@ -43,8 +44,10 @@ var _ = Describe("cephfs", func() {
 		cephfsFiles := getFilesinDirectory(cephfsDirPath)
 		for _, file := range cephfsFiles {
 			res, err := framework.RunKubectl("delete", "-f", cephfsDirPath+file.Name())
+			if err != nil {
 				framework.Logf("failed to delete resource in %s with err %v", res, err)
 			}
+		}
 		deleteResource(cephfsExamplePath + "secret.yaml")
 		deleteResource(cephfsExamplePath + "storageclass.yaml")
 		deleteFileSystem()
@@ -52,6 +55,8 @@ var _ = Describe("cephfs", func() {
 
 	Context("Test cephfs CSI", func() {
 		It("Test cephfs CSI", func() {
+			pvcPath := cephfsExamplePath + "pvc.yaml"
+			appPath := cephfsExamplePath + "pod.yaml"
 			By("checking provisioner statefulset is running")
 			timeout := time.Duration(deployTimeout) * time.Minute
 			err := framework.WaitForStatefulSetReplicasReady(cephfsDeploymentName, namespace, f.ClientSet, 1*time.Second, timeout)
@@ -67,16 +72,49 @@ var _ = Describe("cephfs", func() {
 
 			By("create and delete a PVC", func() {
 				By("create a PVC and Bind it to an app", func() {
-					pvcPath := cephfsExamplePath + "pvc.yaml"
-					appPath := cephfsExamplePath + "pod.yaml"
 					validatePVCAndAppBinding(pvcPath, appPath, f)
 
 				})
 
 				By("create a PVC and Bind it to an app with normal user", func() {
-					pvcPath := cephfsExamplePath + "pvc.yaml"
 					validateNormalUserPVCAccess(pvcPath, f)
 				})
+
+				By("create/delete multiple PVCs and Apps", func() {
+					totalCount := 2
+					pvc, err := loadPVC(pvcPath)
+					if err != nil {
+						Fail(err.Error())
+					}
+					pvc.Namespace = f.UniqueName
+
+					app, err := loadApp(appPath)
+					if err != nil {
+						Fail(err.Error())
+					}
+					app.Namespace = f.UniqueName
+					// create pvc and app
+					for i := 0; i < totalCount; i++ {
+						name := fmt.Sprintf("%s%d", f.UniqueName, i)
+						err := createPVCAndApp(name, f, pvc, app)
+						if err != nil {
+							Fail(err.Error())
+						}
+
+					}
+					// TODO add cephfs backend validation
+
+					// delete pvc and app
+					for i := 0; i < totalCount; i++ {
+						name := fmt.Sprintf("%s%d", f.UniqueName, i)
+						err := deletePVCAndApp(name, f, pvc, app)
+						if err != nil {
+							Fail(err.Error())
+						}
+
+					}
+				})
+
 			})
 
 		})

e2e/deploy-rook.go

@@ -49,19 +49,26 @@ func createRBDPool() {
 }
 func deleteFileSystem() {
 	commonPath := fmt.Sprintf("%s/%s", rookURL, "filesystem-test.yaml")
-	framework.RunKubectlOrDie("delete", "-f", commonPath)
+	_, err := framework.RunKubectl("delete", "-f", commonPath)
+	if err != nil {
+		framework.Logf("failed to delete file-system %v", err)
+	}
 }
 
 func deleteRBDPool() {
 	commonPath := fmt.Sprintf("%s/%s", rookURL, "pool-test.yaml")
-	framework.RunKubectlOrDie("delete", "-f", commonPath)
+	_, err := framework.RunKubectl("delete", "-f", commonPath)
+	if err != nil {
+		framework.Logf("failed to delete pool %v", err)
+	}
 }
 
 func deployOperator(c kubernetes.Interface) {
 	opPath := fmt.Sprintf("%s/%s", rookURL, "operator.yaml")
 
-	framework.RunKubectlOrDie("create", "-f", opPath)
+	_, err := framework.RunKubectl("create", "-f", opPath)
-	err := waitForDaemonSets("rook-ceph-agent", rookNS, c, deployTimeout)
+	Expect(err).Should(BeNil())
+	err = waitForDaemonSets("rook-ceph-agent", rookNS, c, deployTimeout)
 	Expect(err).Should(BeNil())
 	err = waitForDaemonSets("rook-discover", rookNS, c, deployTimeout)
 	Expect(err).Should(BeNil())
@@ -109,5 +116,8 @@ func tearDownRook() {
 	// TODO need to add selector for cleanup validation
 	framework.Cleanup(opPath, rookNS)
 	commonPath := fmt.Sprintf("%s/%s", rookURL, "common.yaml")
-	framework.RunKubectlOrDie("delete", "-f", commonPath)
+	_, err := framework.RunKubectl("delete", "-f", commonPath)
+	if err != nil {
+		framework.Logf("failed to delete rook common %v", err)
+	}
 }

e2e/rbd.go

@@ -1,6 +1,7 @@
 package e2e
 
 import (
+	"fmt"
 	"time"
 
 	. "github.com/onsi/ginkgo" // nolint
@@ -46,8 +47,10 @@ var _ = Describe("RBD", func() {
 		rbdFiles := getFilesinDirectory(rbdDirPath)
 		for _, file := range rbdFiles {
 			res, err := framework.RunKubectl("delete", "-f", rbdDirPath+file.Name())
+			if err != nil {
 				framework.Logf("failed to delete resource in %s with err %v", res, err)
 			}
+		}
 		deleteRBDPool()
 		deleteResource(rbdExamplePath + "secret.yaml")
 		deleteResource(rbdExamplePath + "storageclass.yaml")
@@ -56,6 +59,14 @@ var _ = Describe("RBD", func() {
 
 	Context("Test RBD CSI", func() {
 		It("Test RBD CSI", func() {
+			pvcPath := rbdExamplePath + "raw-block-pvc.yaml"
+			appPath := rbdExamplePath + "raw-block-pod.yaml"
+			rawPvcPath := rbdExamplePath + "raw-block-pvc.yaml"
+			rawAppPath := rbdExamplePath + "raw-block-pod.yaml"
+			pvcClonePath := rbdExamplePath + "pvc-restore.yaml"
+			appClonePath := rbdExamplePath + "pod-restore.yaml"
+			snapshotPath := rbdExamplePath + "snapshot.yaml"
+
 			By("checking provisioner statefulset is running")
 			timeout := time.Duration(deployTimeout) * time.Minute
 			err := framework.WaitForStatefulSetReplicasReady(rbdDeploymentName, namespace, f.ClientSet, 1*time.Second, timeout)
@@ -70,22 +81,15 @@ var _ = Describe("RBD", func() {
 			}
 
 			By("create a PVC and Bind it to an app", func() {
-				pvcPath := rbdExamplePath + "pvc.yaml"
-				appPath := rbdExamplePath + "pod.yaml"
 				validatePVCAndAppBinding(pvcPath, appPath, f)
 			})
 
 			By("create a PVC and Bind it to an app with normal user", func() {
-				pvcPath := rbdExamplePath + "pvc.yaml"
 				validateNormalUserPVCAccess(pvcPath, f)
 			})
 
 			By("create a PVC clone and Bind it to an app", func() {
 				createRBDSnapshotClass(f)
-				pvcPath := rbdExamplePath + "pvc.yaml"
-				pvcClonePath := rbdExamplePath + "pvc-restore.yaml"
-				appClonePath := rbdExamplePath + "pod-restore.yaml"
-				snapshotPath := rbdExamplePath + "snapshot.yaml"
 				pvc, err := loadPVC(pvcPath)
 				if err != nil {
 					Fail(err.Error())
@@ -97,6 +101,12 @@ var _ = Describe("RBD", func() {
 				if err != nil {
 					Fail(err.Error())
 				}
+				// validate created backend rbd images
+				images := listRBDImages(f)
+				if len(images) != 1 {
+					framework.Logf("backend image count %d expected image count %d", len(images), 1)
+					Fail("validate backend image failed")
+				}
 				snap := getSnapshot(snapshotPath)
 				snap.Namespace = f.UniqueName
 				snap.Spec.Source.Name = pvc.Name
@@ -105,6 +115,15 @@ var _ = Describe("RBD", func() {
 				if err != nil {
 					Fail(err.Error())
 				}
+				pool := "replicapool"
+				snapList, err := listSnapshots(f, pool, images[0])
+				if err != nil {
+					Fail(err.Error())
+				}
+				if len(snapList) != 1 {
+					framework.Logf("backend snapshot not matching kube snap count,snap count = % kube snap count %d", len(snapList), 1)
+					Fail("validate backend snapshot failed")
+				}
 
 				validatePVCAndAppBinding(pvcClonePath, appClonePath, f)
 
@@ -119,9 +138,54 @@ var _ = Describe("RBD", func() {
 			})
 
 			By("create a block type PVC and Bind it to an app", func() {
-				pvcPath := rbdExamplePath + "raw-block-pvc.yaml"
+				validatePVCAndAppBinding(rawPvcPath, rawAppPath, f)
-				appPath := rbdExamplePath + "raw-block-pod.yaml"
+			})
-				validatePVCAndAppBinding(pvcPath, appPath, f)
+
+			By("create/delete multiple PVCs and Apps", func() {
+				totalCount := 2
+				pvc, err := loadPVC(pvcPath)
+				if err != nil {
+					Fail(err.Error())
+				}
+				pvc.Namespace = f.UniqueName
+
+				app, err := loadApp(appPath)
+				if err != nil {
+					Fail(err.Error())
+				}
+				app.Namespace = f.UniqueName
+				// create pvc and app
+				for i := 0; i < totalCount; i++ {
+					name := fmt.Sprintf("%s%d", f.UniqueName, i)
+					err := createPVCAndApp(name, f, pvc, app)
+					if err != nil {
+						Fail(err.Error())
+					}
+
+				}
+				// validate created backend rbd images
+				images := listRBDImages(f)
+				if len(images) != totalCount {
+					framework.Logf("backend image creation not matching pvc count, image count = % pvc count %d", len(images), totalCount)
+					Fail("validate multiple pvc failed")
+				}
+
+				// delete pvc and app
+				for i := 0; i < totalCount; i++ {
+					name := fmt.Sprintf("%s%d", f.UniqueName, i)
+					err := deletePVCAndApp(name, f, pvc, app)
+					if err != nil {
+						Fail(err.Error())
+					}
+
+				}
+
+				// validate created backend rbd images
+				images = listRBDImages(f)
+				if len(images) > 0 {
+					framework.Logf("left out rbd backend images count %d", len(images))
+					Fail("validate multiple pvc failed")
+				}
 			})
 		})
 	})

e2e/utils.go

@@ -37,6 +37,13 @@ func getFilesinDirectory(path string) []os.FileInfo {
 
 var poll = 2 * time.Second
 
+type snapInfo struct {
+	ID        int64  `json:"id"`
+	Name      string `json:"name"`
+	Size      int64  `json:"size"`
+	Timestamp string `json:"timestamp"`
+}
+
 func waitForDaemonSets(name, ns string, c clientset.Interface, t int) error {
 	timeout := time.Duration(t) * time.Minute
 	start := time.Now()
@@ -507,6 +514,41 @@ func checkCephPods(ns string, c kubernetes.Interface, count, t int, opt *metav1.
 
 }
 
+// createPVCAndApp creates pvc and pod
+// if name is not empty same will be set as pvc and app name
+func createPVCAndApp(name string, f *framework.Framework, pvc *v1.PersistentVolumeClaim, app *v1.Pod) error {
+
+	if name != "" {
+		pvc.Name = name
+		app.Name = name
+		app.Spec.Volumes[0].PersistentVolumeClaim.ClaimName = name
+	}
+	err := createPVCAndvalidatePV(f.ClientSet, pvc, deployTimeout)
+	if err != nil {
+		return err
+	}
+	err = createApp(f.ClientSet, app, deployTimeout)
+	return err
+}
+
+// deletePVCAndApp delete pvc and pod
+// if name is not empty same will be set as pvc and app name
+func deletePVCAndApp(name string, f *framework.Framework, pvc *v1.PersistentVolumeClaim, app *v1.Pod) error {
+
+	if name != "" {
+		pvc.Name = name
+		app.Name = name
+		app.Spec.Volumes[0].PersistentVolumeClaim.ClaimName = name
+	}
+
+	err := deletePod(app.Name, app.Namespace, f.ClientSet, deployTimeout)
+	if err != nil {
+		return err
+	}
+	err = deletePVCAndValidatePV(f.ClientSet, pvc, deployTimeout)
+	return err
+}
+
 func validatePVCAndAppBinding(pvcPath, appPath string, f *framework.Framework) {
 	pvc, err := loadPVC(pvcPath)
 	if pvc == nil {
@@ -514,27 +556,19 @@ func validatePVCAndAppBinding(pvcPath, appPath string, f *framework.Framework) {
 	}
 	pvc.Namespace = f.UniqueName
 	framework.Logf("The PVC  template %+v", pvc)
-	err = createPVCAndvalidatePV(f.ClientSet, pvc, deployTimeout)
-	if err != nil {
-		Fail(err.Error())
-	}
 
 	app, err := loadApp(appPath)
 	if err != nil {
 		Fail(err.Error())
 	}
 	app.Namespace = f.UniqueName
-	err = createApp(f.ClientSet, app, deployTimeout)
+
+	err = createPVCAndApp("", f, pvc, app)
 	if err != nil {
 		Fail(err.Error())
 	}
 
-	err = deletePod(app.Name, app.Namespace, f.ClientSet, deployTimeout)
+	err = deletePVCAndApp("", f, pvc, app)
-	if err != nil {
-		Fail(err.Error())
-	}
-
-	err = deletePVCAndValidatePV(f.ClientSet, pvc, deployTimeout)
 	if err != nil {
 		Fail(err.Error())
 	}
@@ -626,6 +660,7 @@ func createSnapshot(snap *v1alpha1.VolumeSnapshot, t int) error {
 	if err != nil {
 		return err
 	}
+	framework.Logf("snapshot with name %v created in %v namespace", snap.Name, snap.Namespace)
 
 	timeout := time.Duration(t) * time.Minute
 	name := snap.Name
@@ -665,12 +700,11 @@ func deleteSnapshot(snap *v1alpha1.VolumeSnapshot, t int) error {
 	timeout := time.Duration(t) * time.Minute
 	name := snap.Name
 	start := time.Now()
-	framework.Logf("Waiting up to %v to be in Ready state", snap)
+	framework.Logf("Waiting up to %v to be deleted", snap)
 
 	return wait.PollImmediate(poll, timeout, func() (bool, error) {
-		framework.Logf("waiting for snapshot %s (%d seconds elapsed)", name, int(time.Since(start).Seconds()))
+		framework.Logf("deleting snapshot %s (%d seconds elapsed)", name, int(time.Since(start).Seconds()))
 		_, err := sclient.VolumeSnapshots(snap.Namespace).Get(name, metav1.GetOptions{})
-
 		if err == nil {
 			return false, nil
 		}
@@ -679,6 +713,35 @@ func deleteSnapshot(snap *v1alpha1.VolumeSnapshot, t int) error {
 			return false, fmt.Errorf("get on deleted snapshot %v failed with error other than \"not found\": %v", name, err)
 		}
 
-		return false, nil
+		return true, nil
 	})
 }
+
+func listRBDImages(f *framework.Framework) []string {
+
+	opt := metav1.ListOptions{
+		LabelSelector: "app=rook-ceph-tools",
+	}
+	stdout := execCommandInPod(f, "rbd ls --pool=replicapool --format=json", rookNS, &opt)
+
+	var imgInfos []string
+
+	err := json.Unmarshal([]byte(stdout), &imgInfos)
+	if err != nil {
+		Fail(err.Error())
+	}
+	return imgInfos
+}
+
+func listSnapshots(f *framework.Framework, pool, imageName string) ([]snapInfo, error) {
+	opt := metav1.ListOptions{
+		LabelSelector: "app=rook-ceph-tools",
+	}
+	command := fmt.Sprintf("rbd snap ls %s/%s --format=json", pool, imageName)
+	stdout := execCommandInPod(f, command, rookNS, &opt)
+
+	var snapInfos []snapInfo
+
+	err := json.Unmarshal([]byte(stdout), &snapInfos)
+	return snapInfos, err
+}