e2e: adjust migration e2e tests and introduce helper functions

This commit adjust existing migration e2e tests to a couple of tests
to cover the scenarios. The seperate filesystem and block tests have
been shrinked to single one and also introduced a couple of helper
functions to setup and teardown migraition specific secret,configmap
and sc. The static pv function has been renamed to a general name
while the tests were adjusted.

Signed-off-by: Humble Chirammal <hchiramm@redhat.com>

e2e/migration.go

@@ -4,7 +4,6 @@ import (
 	"context"
 	"encoding/hex"
 	"fmt"
-	"strconv"
 	"strings"
 
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -12,96 +11,6 @@ import (
 	"k8s.io/kubernetes/test/e2e/framework"
 )
 
-func validateRBDStaticMigrationPVDeletion(f *framework.Framework, appPath, scName string, isBlock bool) error {
-	opt := make(map[string]string)
-	var (
-		rbdImageName        = "kubernetes-dynamic-pvc-e0b45b52-7e09-47d3-8f1b-806995fa4412"
-		pvName              = "pv-name"
-		pvcName             = "pvc-name"
-		namespace           = f.UniqueName
-		sc                  = scName
-		provisionerAnnKey   = "pv.kubernetes.io/provisioned-by"
-		provisionerAnnValue = "rbd.csi.ceph.com"
-	)
-
-	c := f.ClientSet
-	PVAnnMap := make(map[string]string)
-	PVAnnMap[provisionerAnnKey] = provisionerAnnValue
-	mons, err := getMons(rookNamespace, c)
-	if err != nil {
-		return fmt.Errorf("failed to get mons: %w", err)
-	}
-	mon := strings.Join(mons, ",")
-	size := staticPVSize
-	// create rbd image
-	cmd := fmt.Sprintf(
-		"rbd create %s --size=%s --image-feature=layering %s",
-		rbdImageName,
-		staticPVSize,
-		rbdOptions(defaultRBDPool))
-
-	_, stdErr, err := execCommandInToolBoxPod(f, cmd, rookNamespace)
-	if err != nil {
-		return err
-	}
-	if stdErr != "" {
-		return fmt.Errorf("failed to create rbd image %s", stdErr)
-	}
-
-	opt["migration"] = "true"
-	opt["clusterID"] = getMonsHash(mon)
-	opt["imageFeatures"] = staticPVImageFeature
-	opt["pool"] = defaultRBDPool
-	opt["staticVolume"] = strconv.FormatBool(true)
-	opt["imageName"] = rbdImageName
-
-	// Make volumeID similar to the migration volumeID
-	volID := composeIntreeMigVolID(mon, rbdImageName)
-	pv := getStaticPV(
-		pvName,
-		volID,
-		size,
-		rbdNodePluginSecretName,
-		cephCSINamespace,
-		sc,
-		provisionerAnnValue,
-		isBlock,
-		opt,
-		PVAnnMap,
-		deletePolicy)
-
-	_, err = c.CoreV1().PersistentVolumes().Create(context.TODO(), pv, metav1.CreateOptions{})
-	if err != nil {
-		return fmt.Errorf("PV Create API error: %w", err)
-	}
-
-	pvc := getStaticPVC(pvcName, pvName, size, namespace, sc, isBlock)
-
-	_, err = c.CoreV1().PersistentVolumeClaims(pvc.Namespace).Create(context.TODO(), pvc, metav1.CreateOptions{})
-	if err != nil {
-		return fmt.Errorf("PVC Create API error: %w", err)
-	}
-	// bind pvc to app
-	app, err := loadApp(appPath)
-	if err != nil {
-		return err
-	}
-
-	app.Namespace = namespace
-	app.Spec.Volumes[0].PersistentVolumeClaim.ClaimName = pvcName
-	err = createApp(f.ClientSet, app, deployTimeout)
-	if err != nil {
-		return err
-	}
-
-	err = deletePVCAndApp("", f, pvc, app)
-	if err != nil {
-		return fmt.Errorf("failed to delete PVC and application: %w", err)
-	}
-
-	return err
-}
-
 // composeIntreeMigVolID create a volID similar to intree migration volID
 // the migration volID format looks like below
 // mig-mons-<hash>-image-<UUID_<poolhash>
@@ -190,6 +99,7 @@ func createMigrationUserSecretAndSC(f *framework.Framework, scName string) error
 	return nil
 }
 
+// createMigrationSC create a SC with migration specific secrets and clusterid.
 func createMigrationSC(f *framework.Framework, scName string) error {
 	err := deleteResource(rbdExamplePath + "storageclass.yaml")
 	if err != nil {
@@ -203,6 +113,13 @@ func createMigrationSC(f *framework.Framework, scName string) error {
 	param["csi.storage.k8s.io/controller-expand-secret-name"] = rbdMigrationProvisionerSecretName
 	param["csi.storage.k8s.io/node-stage-secret-namespace"] = cephCSINamespace
 	param["csi.storage.k8s.io/node-stage-secret-name"] = rbdMigrationNodePluginSecretName
+	mons, err := getMons(rookNamespace, f.ClientSet)
+	if err != nil {
+		return fmt.Errorf("failed to get mons: %w", err)
+	}
+	mon := strings.Join(mons, ",")
+	param["migration"] = "true"
+	param["clusterID"] = getMonsHash(mon)
 	err = createRBDStorageClass(f.ClientSet, f, scName, nil, param, deletePolicy)
 	if err != nil {
 		return fmt.Errorf("failed to create storageclass: %w", err)
@@ -274,3 +191,40 @@ func deleteProvNodeMigrationSecret(f *framework.Framework, provisionerSecret, no
 
 	return nil
 }
+
+// setupMigrationCMSecretAndSC create custom configmap, secret and SC for migration tests.
+func setupMigrationCMSecretAndSC(f *framework.Framework, scName string) error {
+	c := f.ClientSet
+	if scName == "" {
+		scName = defaultSCName
+	}
+	err := generateClusterIDConfigMapForMigration(f, c)
+	if err != nil {
+		return fmt.Errorf("failed to generate clusterID configmap: %w", err)
+	}
+
+	err = createMigrationUserSecretAndSC(f, scName)
+	if err != nil {
+		return fmt.Errorf("failed to create storageclass: %w", err)
+	}
+
+	return nil
+}
+
+// tearDownMigrationSetup deletes custom configmap and secret.
+func tearDownMigrationSetup(f *framework.Framework) error {
+	err := deleteConfigMap(rbdDirPath)
+	if err != nil {
+		return fmt.Errorf("failed to delete configmap: %w", err)
+	}
+	err = createConfigMap(rbdDirPath, f.ClientSet, f)
+	if err != nil {
+		return fmt.Errorf("failed to create configmap: %w", err)
+	}
+	err = deleteProvNodeMigrationSecret(f, true, true)
+	if err != nil {
+		return fmt.Errorf("failed to delete migration users and Secrets associated: %w", err)
+	}
+
+	return nil
+}

e2e/rbd.go

@@ -402,37 +402,58 @@ var _ = Describe("RBD", func() {
 				}
 			})
 
-			// todo: may be remove the below deletion test later once the migration nodestage tests are adjusted
+			By("validate RBD migration PVC", func() {
-			// also to have deletion validation through the same.
+				err := setupMigrationCMSecretAndSC(f, "")
-			By("validate RBD migration+static Block PVC Deletion", func() {
-				err := generateClusterIDConfigMapForMigration(f, c)
 				if err != nil {
-					e2elog.Failf("failed to generate clusterID configmap: %v", err)
+					e2elog.Failf("failed to setup migration prerequisites: %v", err)
 				}
-
+				err = validatePVCAndAppBinding(pvcPath, appPath, f)
-				// create a sc with different migration secret
-				err = createMigrationUserSecretAndSC(f, "migrationsc")
 				if err != nil {
-					e2elog.Failf("failed to create storageclass: %v", err)
+					e2elog.Failf("failed to validate pvc and application binding: %v", err)
-				}
-				err = validateRBDStaticMigrationPVDeletion(f, rawAppPath, "migrationsc", true)
-				if err != nil {
-					e2elog.Failf("failed to validate rbd migrated static block pv: %v", err)
 				}
 				// validate created backend rbd images
 				validateRBDImageCount(f, 0, defaultRBDPool)
-				err = deleteConfigMap(rbdDirPath)
+
+				// Block PVC resize
+				err = resizePVCAndValidateSize(rawPvcPath, rawAppPath, f)
 				if err != nil {
-					e2elog.Failf("failed to delete configmap: %v", err)
+					e2elog.Failf("failed to resize block PVC: %v", err)
-				}
-				err = createConfigMap(rbdDirPath, f.ClientSet, f)
-				if err != nil {
-					e2elog.Failf("failed to create configmap: %v", err)
 				}
 
-				err = deleteProvNodeMigrationSecret(f, true, true)
+				// FileSystem PVC resize
+				err = resizePVCAndValidateSize(pvcPath, appPath, f)
 				if err != nil {
-					e2elog.Failf("failed to delete migration users and Secrets associated: %v", err)
+					e2elog.Failf("failed to resize filesystem PVC: %v", err)
+				}
+				err = deleteResource(rbdExamplePath + "storageclass.yaml")
+				if err != nil {
+					e2elog.Failf("failed to delete storageclass: %v", err)
+				}
+				err = createRBDStorageClass(f.ClientSet, f, defaultSCName, nil, nil, deletePolicy)
+				if err != nil {
+					e2elog.Failf("failed to create storageclass: %v", err)
+				}
+				err = tearDownMigrationSetup(f)
+				if err != nil {
+					e2elog.Failf("failed to tear down migration setup: %v", err)
+				}
+			})
+
+			By("validate RBD migration+static FileSystem", func() {
+				err := setupMigrationCMSecretAndSC(f, "migrationsc")
+				if err != nil {
+					e2elog.Failf("failed to setup migration prerequisites: %v", err)
+				}
+				// validate filesystem pvc mount
+				err = validateRBDStaticMigrationPVC(f, appPath, "migrationsc", false)
+				if err != nil {
+					e2elog.Failf("failed to validate rbd migrated static file mode pvc: %v", err)
+				}
+				// validate created backend rbd images
+				validateRBDImageCount(f, 0, defaultRBDPool)
+				err = tearDownMigrationSetup(f)
+				if err != nil {
+					e2elog.Failf("failed to tear down migration setup: %v", err)
 				}
 				err = createRBDStorageClass(f.ClientSet, f, defaultSCName, nil, nil, deletePolicy)
 				if err != nil {
@@ -2117,72 +2138,6 @@ var _ = Describe("RBD", func() {
 				validateRBDImageCount(f, 0, defaultRBDPool)
 			})
 
-			By("validate RBD migration+static FileSystem PVC", func() {
-				err := generateClusterIDConfigMapForMigration(f, c)
-				if err != nil {
-					e2elog.Failf("failed to generate clusterID configmap: %v", err)
-				}
-				// create node user and migration secret.
-				err = createProvNodeCephUserAndSecret(f, false, true)
-				if err != nil {
-					e2elog.Failf("failed to create users and secret: %v", err)
-				}
-
-				err = validateRBDStaticMigrationPV(f, appPath, rbdMigrationNodePluginSecretName, false)
-				if err != nil {
-					e2elog.Failf("failed to validate rbd migrated static pv: %v", err)
-				}
-				// validate created backend rbd images
-				validateRBDImageCount(f, 0, defaultRBDPool)
-
-				err = deleteProvNodeMigrationSecret(f, false, true)
-				if err != nil {
-					e2elog.Failf("failed to delete users and secret: %v", err)
-				}
-
-				err = deleteConfigMap(rbdDirPath)
-				if err != nil {
-					e2elog.Failf("failed to delete configmap: %v", err)
-				}
-				err = createConfigMap(rbdDirPath, f.ClientSet, f)
-				if err != nil {
-					e2elog.Failf("failed to create configmap: %v", err)
-				}
-			})
-
-			By("validate RBD migration+static Block PVC", func() {
-				err := generateClusterIDConfigMapForMigration(f, c)
-				if err != nil {
-					e2elog.Failf("failed to generate clusterID configmap: %v", err)
-				}
-				// create node user and migration secret.
-				err = createProvNodeCephUserAndSecret(f, false, true)
-				if err != nil {
-					e2elog.Failf("failed to create users and secret: %v", err)
-				}
-
-				err = validateRBDStaticMigrationPV(f, rawAppPath, rbdMigrationNodePluginSecretName, true)
-				if err != nil {
-					e2elog.Failf("failed to validate rbd migrated static block pv: %v", err)
-				}
-				// validate created backend rbd images
-				validateRBDImageCount(f, 0, defaultRBDPool)
-
-				err = deleteProvNodeMigrationSecret(f, false, true)
-				if err != nil {
-					e2elog.Failf("failed to delete users and secret: %v", err)
-				}
-
-				err = deleteConfigMap(rbdDirPath)
-				if err != nil {
-					e2elog.Failf("failed to delete configmap: %v", err)
-				}
-				err = createConfigMap(rbdDirPath, f.ClientSet, f)
-				if err != nil {
-					e2elog.Failf("failed to create configmap: %v", err)
-				}
-			})
-
 			By("validate failure of RBD static PVC without imageFeatures parameter", func() {
 				err := validateRBDStaticPV(f, rawAppPath, true, true)
 				if err != nil {

e2e/staticpvc.go

@@ -219,19 +219,21 @@ func validateRBDStaticPV(f *framework.Framework, appPath string, isBlock, checkI
 	return err
 }
 
-func validateRBDStaticMigrationPV(f *framework.Framework, appPath, nodeSecretName string, isBlock bool) error {
+func validateRBDStaticMigrationPVC(f *framework.Framework, appPath, scName string, isBlock bool) error {
 	opt := make(map[string]string)
 	var (
-		rbdImageName = "test-static-pv"
+		rbdImageName        = "kubernetes-dynamic-pvc-e0b45b52-7e09-47d3-8f1b-806995fa4412"
-		pvName       = "pv-name"
+		pvName              = "pv-name"
-		pvcName      = "pvc-name"
+		pvcName             = "pvc-name"
-		namespace    = f.UniqueName
+		namespace           = f.UniqueName
-		// minikube creates default class in cluster, we need to set dummy
+		sc                  = scName
-		// storageclass on PV and PVC to avoid storageclass name mismatch
+		provisionerAnnKey   = "pv.kubernetes.io/provisioned-by"
-		sc = "storage-class"
+		provisionerAnnValue = "rbd.csi.ceph.com"
 	)
 
 	c := f.ClientSet
+	PVAnnMap := make(map[string]string)
+	PVAnnMap[provisionerAnnKey] = provisionerAnnValue
 	mons, err := getMons(rookNamespace, c)
 	if err != nil {
 		return fmt.Errorf("failed to get mons: %w", err)
@@ -240,20 +242,17 @@ func validateRBDStaticMigrationPV(f *framework.Framework, appPath, nodeSecretNam
 	size := staticPVSize
 	// create rbd image
 	cmd := fmt.Sprintf(
-		"rbd create %s --size=%d --image-feature=layering %s",
+		"rbd create %s --size=%s --image-feature=layering %s",
 		rbdImageName,
-		4096,
+		staticPVSize,
 		rbdOptions(defaultRBDPool))
 
-	_, e, err := execCommandInToolBoxPod(f, cmd, rookNamespace)
+	_, stdErr, err := execCommandInToolBoxPod(f, cmd, rookNamespace)
 	if err != nil {
 		return err
 	}
-	if e != "" {
+	if stdErr != "" {
-		return fmt.Errorf("failed to create rbd image %s", e)
+		return fmt.Errorf("failed to create rbd image %s", stdErr)
-	}
-	if nodeSecretName == "" {
-		nodeSecretName = rbdNodePluginSecretName
 	}
 
 	opt["migration"] = "true"
@@ -262,16 +261,21 @@ func validateRBDStaticMigrationPV(f *framework.Framework, appPath, nodeSecretNam
 	opt["pool"] = defaultRBDPool
 	opt["staticVolume"] = strconv.FormatBool(true)
 	opt["imageName"] = rbdImageName
+
+	// Make volumeID similar to the migration volumeID
+	volID := composeIntreeMigVolID(mon, rbdImageName)
 	pv := getStaticPV(
 		pvName,
-		rbdImageName,
+		volID,
 		size,
-		nodeSecretName,
+		rbdNodePluginSecretName,
 		cephCSINamespace,
 		sc,
-		"rbd.csi.ceph.com",
+		provisionerAnnValue,
 		isBlock,
-		opt, nil, retainPolicy)
+		opt,
+		PVAnnMap,
+		deletePolicy)
 
 	_, err = c.CoreV1().PersistentVolumes().Create(context.TODO(), pv, metav1.CreateOptions{})
 	if err != nil {
@@ -297,24 +301,11 @@ func validateRBDStaticMigrationPV(f *framework.Framework, appPath, nodeSecretNam
 		return err
 	}
 
-	err = deletePod(app.Name, app.Namespace, f.ClientSet, deployTimeout)
+	err = deletePVCAndApp("", f, pvc, app)
 	if err != nil {
-		return err
+		return fmt.Errorf("failed to delete PVC and application: %w", err)
 	}
 
-	err = c.CoreV1().PersistentVolumeClaims(pvc.Namespace).Delete(context.TODO(), pvc.Name, metav1.DeleteOptions{})
-	if err != nil {
-		return fmt.Errorf("failed to delete pvc: %w", err)
-	}
-
-	err = c.CoreV1().PersistentVolumes().Delete(context.TODO(), pv.Name, metav1.DeleteOptions{})
-	if err != nil {
-		return fmt.Errorf("failed to delete pv: %w", err)
-	}
-
-	cmd = fmt.Sprintf("rbd rm %s %s", rbdImageName, rbdOptions(defaultRBDPool))
-	_, _, err = execCommandInToolBoxPod(f, cmd, rookNamespace)
-
 	return err
 }