e2e: Enable E2E for rbd snapshot

Updated E2E to use beta client and beta
Snapshot resources.

Signed-off-by: Madhu Rajanna <madhupr007@gmail.com>

e2e/rbd.go

@@ -180,9 +180,9 @@ var _ = Describe("RBD", func() {
 			appPath := rbdExamplePath + "pod.yaml"
 			rawPvcPath := rbdExamplePath + "raw-block-pvc.yaml"
 			rawAppPath := rbdExamplePath + "raw-block-pod.yaml"
-			// pvcClonePath := rbdExamplePath + "pvc-restore.yaml"
+			pvcClonePath := rbdExamplePath + "pvc-restore.yaml"
-			// appClonePath := rbdExamplePath + "pod-restore.yaml"
+			appClonePath := rbdExamplePath + "pod-restore.yaml"
-			// snapshotPath := rbdExamplePath + "snapshot.yaml"
+			snapshotPath := rbdExamplePath + "snapshot.yaml"
 
 			By("checking provisioner deployment is running", func() {
 				err := waitForDeploymentComplete(rbdDeploymentName, cephCSINamespace, f.ClientSet, deployTimeout)
@@ -235,55 +235,53 @@ var _ = Describe("RBD", func() {
 			})
 
 			// skipping snapshot testing
+			By("create a PVC clone and Bind it to an app", func() {
+				createRBDSnapshotClass(f)
+				pvc, err := loadPVC(pvcPath)
+				if err != nil {
+					Fail(err.Error())
+				}
 
-			// By("create a PVC clone and Bind it to an app", func() {
+				pvc.Namespace = f.UniqueName
-			// 	createRBDSnapshotClass(f)
+				e2elog.Logf("The PVC  template %+v", pvc)
-			// 	pvc, err := loadPVC(pvcPath)
+				err = createPVCAndvalidatePV(f.ClientSet, pvc, deployTimeout)
-			// 	if err != nil {
+				if err != nil {
-			// 		Fail(err.Error())
+					Fail(err.Error())
-			// 	}
+				}
+				// validate created backend rbd images
+				images := listRBDImages(f)
+				if len(images) != 1 {
+					e2elog.Logf("backend image count %d expected image count %d", len(images), 1)
+					Fail("validate backend image failed")
+				}
+				snap := getSnapshot(snapshotPath)
+				snap.Namespace = f.UniqueName
+				snap.Spec.Source.PersistentVolumeClaimName = &pvc.Name
+				err = createSnapshot(&snap, deployTimeout)
+				if err != nil {
+					Fail(err.Error())
+				}
+				pool := "replicapool"
+				snapList, err := listSnapshots(f, pool, images[0])
+				if err != nil {
+					Fail(err.Error())
+				}
+				if len(snapList) != 1 {
+					e2elog.Logf("backend snapshot not matching kube snap count,snap count = % kube snap count %d", len(snapList), 1)
+					Fail("validate backend snapshot failed")
+				}
 
-			// 	pvc.Namespace = f.UniqueName
+				validatePVCAndAppBinding(pvcClonePath, appClonePath, f)
-			// 	e2elog.Logf("The PVC  template %+v", pvc)
-			// 	err = createPVCAndvalidatePV(f.ClientSet, pvc, deployTimeout)
-			// 	if err != nil {
-			// 		Fail(err.Error())
-			// 	}
-			// 	// validate created backend rbd images
-			// 	images := listRBDImages(f)
-			// 	if len(images) != 1 {
-			// 		e2elog.Logf("backend image count %d expected image count %d", len(images), 1)
-			// 		Fail("validate backend image failed")
-			// 	}
-			// 	snap := getSnapshot(snapshotPath)
-			// 	snap.Namespace = f.UniqueName
-			// 	snap.Spec.Source.Name = pvc.Name
-			// 	snap.Spec.Source.Kind = "PersistentVolumeClaim"
-			// 	err = createSnapshot(&snap, deployTimeout)
-			// 	if err != nil {
-			// 		Fail(err.Error())
-			// 	}
-			// 	pool := "replicapool"
-			// 	snapList, err := listSnapshots(f, pool, images[0])
-			// 	if err != nil {
-			// 		Fail(err.Error())
-			// 	}
-			// 	if len(snapList) != 1 {
-			// 		e2elog.Logf("backend snapshot not matching kube snap count,snap count = % kube snap count %d", len(snapList), 1)
-			// 		Fail("validate backend snapshot failed")
-			// 	}
 
-			// 	validatePVCAndAppBinding(pvcClonePath, appClonePath, f)
+				err = deleteSnapshot(&snap, deployTimeout)
-
+				if err != nil {
-			// 	err = deleteSnapshot(&snap, deployTimeout)
+					Fail(err.Error())
-			// 	if err != nil {
+				}
-			// 		Fail(err.Error())
+				err = deletePVCAndValidatePV(f.ClientSet, pvc, deployTimeout)
-			// 	}
+				if err != nil {
-			// 	err = deletePVCAndValidatePV(f.ClientSet, pvc, deployTimeout)
+					Fail(err.Error())
-			// 	if err != nil {
+				}
-			// 		Fail(err.Error())
+			})
-			// 	}
-			// })
 
 			By("create a block type PVC and Bind it to an app", func() {
 				validatePVCAndAppBinding(rawPvcPath, rawAppPath, f)

e2e/snapshot.go

@@ -0,0 +1,156 @@
+package e2e
+
+import (
+	"encoding/json"
+	"fmt"
+	"strings"
+	"time"
+
+	snapapi "github.com/kubernetes-csi/external-snapshotter/v2/pkg/apis/volumesnapshot/v1beta1"
+	snapclient "github.com/kubernetes-csi/external-snapshotter/v2/pkg/client/clientset/versioned"
+	. "github.com/onsi/gomega" // nolint
+	apierrs "k8s.io/apimachinery/pkg/api/errors"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/util/wait"
+	"k8s.io/kubernetes/test/e2e/framework"
+	e2elog "k8s.io/kubernetes/test/e2e/framework/log"
+	testutils "k8s.io/kubernetes/test/utils"
+)
+
+type snapInfo struct {
+	ID        int64  `json:"id"`
+	Name      string `json:"name"`
+	Size      int64  `json:"size"`
+	Timestamp string `json:"timestamp"`
+}
+
+func getSnapshotClass(path string) snapapi.VolumeSnapshotClass {
+	sc := snapapi.VolumeSnapshotClass{}
+	sc.Kind = "VolumeSnapshotClass"
+	sc.APIVersion = "snapshot.storage.k8s.io/v1beta1"
+	err := unmarshal(path, &sc)
+	Expect(err).Should(BeNil())
+	return sc
+}
+
+func getSnapshot(path string) snapapi.VolumeSnapshot {
+	sc := snapapi.VolumeSnapshot{}
+	err := unmarshal(path, &sc)
+	Expect(err).Should(BeNil())
+	return sc
+}
+
+func newSnapshotClient() (*snapclient.Clientset, error) {
+	config, err := framework.LoadConfig()
+	if err != nil {
+		return nil, fmt.Errorf("error creating client: %v", err.Error())
+	}
+	c, err := snapclient.NewForConfig(config)
+	if err != nil {
+		return nil, fmt.Errorf("error creating snapshot client: %v", err.Error())
+	}
+	return c, err
+}
+
+func createSnapshot(snap *snapapi.VolumeSnapshot, t int) error {
+	sclient, err := newSnapshotClient()
+	if err != nil {
+		return err
+	}
+	_, err = sclient.SnapshotV1beta1().VolumeSnapshots(snap.Namespace).Create(snap)
+	if err != nil {
+		return err
+	}
+	e2elog.Logf("snapshot with name %v created in %v namespace", snap.Name, snap.Namespace)
+
+	timeout := time.Duration(t) * time.Minute
+	name := snap.Name
+	start := time.Now()
+	e2elog.Logf("Waiting up to %v to be in Ready state", snap)
+
+	return wait.PollImmediate(poll, timeout, func() (bool, error) {
+		e2elog.Logf("waiting for snapshot %s (%d seconds elapsed)", snap.Name, int(time.Since(start).Seconds()))
+		snaps, err := sclient.SnapshotV1beta1().VolumeSnapshots(snap.Namespace).Get(name, metav1.GetOptions{})
+		if err != nil {
+			e2elog.Logf("Error getting snapshot in namespace: '%s': %v", snap.Namespace, err)
+			if testutils.IsRetryableAPIError(err) {
+				return false, nil
+			}
+			if apierrs.IsNotFound(err) {
+				return false, nil
+			}
+			return false, err
+		}
+		if snaps.Status == nil || snaps.Status.ReadyToUse == nil {
+			return false, nil
+		}
+		if *snaps.Status.ReadyToUse {
+			return true, nil
+		}
+		e2elog.Logf("snapshot %s in %v state", snap.Name, *snaps.Status.ReadyToUse)
+		return false, nil
+	})
+}
+
+func deleteSnapshot(snap *snapapi.VolumeSnapshot, t int) error {
+	sclient, err := newSnapshotClient()
+	if err != nil {
+		return err
+	}
+	err = sclient.SnapshotV1beta1().VolumeSnapshots(snap.Namespace).Delete(snap.Name, &metav1.DeleteOptions{})
+	if err != nil {
+		return err
+	}
+
+	timeout := time.Duration(t) * time.Minute
+	name := snap.Name
+	start := time.Now()
+	e2elog.Logf("Waiting up to %v to be deleted", snap)
+
+	return wait.PollImmediate(poll, timeout, func() (bool, error) {
+		e2elog.Logf("deleting snapshot %s (%d seconds elapsed)", name, int(time.Since(start).Seconds()))
+		_, err := sclient.SnapshotV1beta1().VolumeSnapshots(snap.Namespace).Get(name, metav1.GetOptions{})
+		if err == nil {
+			return false, nil
+		}
+
+		if !apierrs.IsNotFound(err) {
+			return false, fmt.Errorf("get on deleted snapshot %v failed with error other than \"not found\": %v", name, err)
+		}
+
+		return true, nil
+	})
+}
+
+func listSnapshots(f *framework.Framework, pool, imageName string) ([]snapInfo, error) {
+	opt := metav1.ListOptions{
+		LabelSelector: "app=rook-ceph-tools",
+	}
+	command := fmt.Sprintf("rbd snap ls %s/%s --format=json", pool, imageName)
+	stdout, stdErr := execCommandInPod(f, command, rookNamespace, &opt)
+	Expect(stdErr).Should(BeEmpty())
+
+	var snapInfos []snapInfo
+
+	err := json.Unmarshal([]byte(stdout), &snapInfos)
+	return snapInfos, err
+}
+
+func createRBDSnapshotClass(f *framework.Framework) {
+	scPath := fmt.Sprintf("%s/%s", rbdExamplePath, "snapshotclass.yaml")
+	sc := getSnapshotClass(scPath)
+
+	sc.Parameters["csi.storage.k8s.io/snapshotter-secret-namespace"] = cephCSINamespace
+
+	opt := metav1.ListOptions{
+		LabelSelector: "app=rook-ceph-tools",
+	}
+	fsID, stdErr := execCommandInPod(f, "ceph fsid", rookNamespace, &opt)
+	Expect(stdErr).Should(BeEmpty())
+	fsID = strings.Trim(fsID, "\n")
+	sc.Parameters["clusterID"] = fsID
+	sclient, err := newSnapshotClient()
+	Expect(err).Should(BeNil())
+	_, err = sclient.SnapshotV1beta1().VolumeSnapshotClasses().Create(&sc)
+	Expect(err).Should(BeNil())
+}

e2e/utils.go

@@ -10,8 +10,6 @@ import (
 	"strings"
 	"time"
 
-	// _ "github.com/kubernetes-csi/external-snapshotter/pkg/apis/volumesnapshot/v1alpha1"                             // nolint
-	// _ "github.com/kubernetes-csi/external-snapshotter/pkg/client/clientset/versioned/typed/volumesnapshot/v1alpha1" // nolint
 	"github.com/ceph/ceph-csi/internal/util"
 
 	. "github.com/onsi/ginkgo" // nolint
@@ -63,13 +61,6 @@ func initResouces() {
 	vaultAddr = fmt.Sprintf("http://vault.%s.svc.cluster.local:8200", cephCSINamespace)
 }
 
-// type snapInfo struct {
-// 	ID        int64  `json:"id"`
-// 	Name      string `json:"name"`
-// 	Size      int64  `json:"size"`
-// 	Timestamp string `json:"timestamp"`
-// }
-
 func createNamespace(c clientset.Interface, name string) error {
 	timeout := time.Duration(deployTimeout) * time.Minute
 	ns := &v1.Namespace{
@@ -251,22 +242,6 @@ func getStorageClass(path string) scv1.StorageClass {
 	return sc
 }
 
-// func getSnapshotClass(path string) v1alpha1.VolumeSnapshotClass {
-// 	sc := v1alpha1.VolumeSnapshotClass{}
-// 	sc.Kind = "VolumeSnapshotClass"
-// 	sc.APIVersion = "snapshot.storage.k8s.io/v1alpha1"
-// 	err := unmarshal(path, &sc)
-// 	Expect(err).Should(BeNil())
-// 	return sc
-// }
-
-// func getSnapshot(path string) v1alpha1.VolumeSnapshot {
-// 	sc := v1alpha1.VolumeSnapshot{}
-// 	err := unmarshal(path, &sc)
-// 	Expect(err).Should(BeNil())
-// 	return sc
-// }
-
 func createCephfsStorageClass(c kubernetes.Interface, f *framework.Framework, enablePool bool) {
 	scPath := fmt.Sprintf("%s/%s", cephfsExamplePath, "storageclass.yaml")
 	sc := getStorageClass(scPath)
@@ -331,34 +306,6 @@ func createRBDStorageClass(c kubernetes.Interface, f *framework.Framework, scOpt
 	Expect(err).Should(BeNil())
 }
 
-// func newSnapshotClient() (*snapClient.SnapshotV1alpha1Client, error) {
-// 	config, err := framework.LoadConfig()
-// 	if err != nil {
-// 		return nil, fmt.Errorf("error creating client: %v", err.Error())
-// 	}
-// 	c, err := snapClient.NewForConfig(config)
-// 	if err != nil {
-// 		return nil, fmt.Errorf("error creating snapshot client: %v", err.Error())
-// 	}
-// 	return c, err
-// }
-// func createRBDSnapshotClass(f *framework.Framework) {
-// 	scPath := fmt.Sprintf("%s/%s", rbdExamplePath, "snapshotclass.yaml")
-// 	sc := getSnapshotClass(scPath)
-
-// 	opt := metav1.ListOptions{
-// 		LabelSelector: "app=rook-ceph-tools",
-// 	}
-// 	fsID := execCommandInPod(f, "ceph fsid", rookNS, &opt)
-// 	// remove new line present in fsID
-// 	fsID = strings.Trim(fsID, "\n")
-// 	sc.Parameters["clusterID"] = fsID
-// 	sclient, err := newSnapshotClient()
-// 	Expect(err).Should(BeNil())
-// 	_, err = sclient.VolumeSnapshotClasses().Create(&sc)
-// 	Expect(err).Should(BeNil())
-// }
-
 func deleteConfigMap(pluginPath string) {
 	path := pluginPath + configMap
 	_, err := framework.RunKubectl(cephCSINamespace, "delete", "-f", path, ns)
@@ -934,73 +881,6 @@ func validateNormalUserPVCAccess(pvcPath string, f *framework.Framework) {
 	}
 }
 
-// func createSnapshot(snap *v1alpha1.VolumeSnapshot, t int) error {
-
-// 	sclient, err := newSnapshotClient()
-// 	if err != nil {
-// 		return err
-// 	}
-// 	_, err = sclient.VolumeSnapshots(snap.Namespace).Create(snap)
-// 	if err != nil {
-// 		return err
-// 	}
-// 	e2elog.Logf("snapshot with name %v created in %v namespace", snap.Name, snap.Namespace)
-
-// 	timeout := time.Duration(t) * time.Minute
-// 	name := snap.Name
-// 	start := time.Now()
-// 	e2elog.Logf("Waiting up to %v to be in Ready state", snap)
-
-// 	return wait.PollImmediate(poll, timeout, func() (bool, error) {
-// 		e2elog.Logf("waiting for snapshot %s (%d seconds elapsed)", snap.Name, int(time.Since(start).Seconds()))
-// 		snaps, err := sclient.VolumeSnapshots(snap.Namespace).Get(name, metav1.GetOptions{})
-// 		if err != nil {
-// 			e2elog.Logf("Error getting snapshot in namespace: '%s': %v", snap.Namespace, err)
-// 			if testutils.IsRetryableAPIError(err) {
-// 				return false, nil
-// 			}
-// 			if apierrs.IsNotFound(err) {
-// 				return false, nil
-// 			}
-// 			return false, err
-// 		}
-// 		if snaps.Status.ReadyToUse {
-// 			return true, nil
-// 		}
-// 		return false, nil
-// 	})
-// }
-
-// func deleteSnapshot(snap *v1alpha1.VolumeSnapshot, t int) error {
-// 	sclient, err := newSnapshotClient()
-// 	if err != nil {
-// 		return err
-// 	}
-// 	err = sclient.VolumeSnapshots(snap.Namespace).Delete(snap.Name, &metav1.DeleteOptions{})
-// 	if err != nil {
-// 		return err
-// 	}
-
-// 	timeout := time.Duration(t) * time.Minute
-// 	name := snap.Name
-// 	start := time.Now()
-// 	e2elog.Logf("Waiting up to %v to be deleted", snap)
-
-// 	return wait.PollImmediate(poll, timeout, func() (bool, error) {
-// 		e2elog.Logf("deleting snapshot %s (%d seconds elapsed)", name, int(time.Since(start).Seconds()))
-// 		_, err := sclient.VolumeSnapshots(snap.Namespace).Get(name, metav1.GetOptions{})
-// 		if err == nil {
-// 			return false, nil
-// 		}
-
-// 		if !apierrs.IsNotFound(err) {
-// 			return false, fmt.Errorf("get on deleted snapshot %v failed with error other than \"not found\": %v", name, err)
-// 		}
-
-// 		return true, nil
-// 	})
-// }
-
 func deleteBackingCephFSVolume(f *framework.Framework, pvc *v1.PersistentVolumeClaim) error {
 	imageData, err := getImageInfoFromPVC(pvc.Namespace, pvc.Name, f)
 	if err != nil {
@@ -1034,19 +914,6 @@ func listRBDImages(f *framework.Framework) []string {
 	return imgInfos
 }
 
-// func listSnapshots(f *framework.Framework, pool, imageName string) ([]snapInfo, error) {
-// 	opt := metav1.ListOptions{
-// 		LabelSelector: "app=rook-ceph-tools",
-// 	}
-// 	command := fmt.Sprintf("rbd snap ls %s/%s --format=json", pool, imageName)
-// 	stdout := execCommandInPod(f, command, rookNS, &opt)
-
-// 	var snapInfos []snapInfo
-
-// 	err := json.Unmarshal([]byte(stdout), &snapInfos)
-// 	return snapInfos, err
-// }
-
 func checkDataPersist(pvcPath, appPath string, f *framework.Framework) error {
 	data := "checking data persist"
 	pvc, err := loadPVC(pvcPath)