e2e: adjust deletion, filesystem and block tests for migration volume

this commit create and make use of migration secret in the requests and
validate various csi operations

Signed-off-by: Humble Chirammal <hchiramm@redhat.com>

e2e/ceph_user.go

@@ -21,6 +21,8 @@ const (
 	rbdProvisionerSecretName          = "cephcsi-rbd-provisioner"
 	rbdNamespaceNodePluginSecretName  = "cephcsi-rbd-ns-node"
 	rbdNamespaceProvisionerSecretName = "cephcsi-rbd-ns-provisioner"
+	rbdMigrationNodePluginSecretName  = "cephcsi-rbd-mig-node"
+	rbdMigrationProvisionerSecretName = "cephcsi-rbd-mig-provisioner"
 	cephFSNodePluginSecretName        = "cephcsi-cephfs-node"
 	cephFSProvisionerSecretName       = "cephcsi-cephfs-provisioner"
 )

e2e/migration.go

@@ -96,7 +96,7 @@ func validateRBDStaticMigrationPVDeletion(f *framework.Framework, appPath, scNam
 
 	err = deletePVCAndApp("", f, pvc, app)
 	if err != nil {
-		return fmt.Errorf("failed to delete PVC and application with error %w", err)
+		return fmt.Errorf("failed to delete PVC and application: %w", err)
 	}
 
 	return err
@@ -134,12 +134,142 @@ func generateClusterIDConfigMapForMigration(f *framework.Framework, c kubernetes
 	// create custom configmap
 	err = createCustomConfigMap(f.ClientSet, rbdDirPath, clusterInfo)
 	if err != nil {
-		return fmt.Errorf("failed to create configmap with error %w", err)
+		return fmt.Errorf("failed to create configmap: %w", err)
 	}
 	// restart csi pods for the configmap to take effect.
 	err = recreateCSIRBDPods(f)
 	if err != nil {
-		return fmt.Errorf("failed to recreate rbd csi pods with error %w", err)
+		return fmt.Errorf("failed to recreate rbd csi pods: %w", err)
+	}
+
+	return nil
+}
+
+// createRBDMigrationSecret creates a migration secret with the passed in user name.
+// this secret differs from csi secret data on below aspects.
+// equivalent to the `UserKey` field, migration secret has `key` field.
+// if 'userName' has passed and if it is not admin, the passed in userName will be
+// set as the `adminId` field in the secret.
+func createRBDMigrationSecret(f *framework.Framework, secretName, userName, userKey string) error {
+	secPath := fmt.Sprintf("%s/%s", rbdExamplePath, "secret.yaml")
+	sec, err := getSecret(secPath)
+	if err != nil {
+		return err
+	}
+	if secretName != "" {
+		sec.Name = secretName
+	}
+	// if its admin, we dont need to change anything in the migration secret, the CSI driver
+	// will use the key from existing secret and continue.
+	if userName != "admin" {
+		sec.StringData["adminId"] = userName
+	}
+	sec.StringData["key"] = userKey
+	sec.Namespace = cephCSINamespace
+	_, err = f.ClientSet.CoreV1().Secrets(cephCSINamespace).Create(context.TODO(), &sec, metav1.CreateOptions{})
+
+	return err
+}
+
+// createMigrationUserSecretAndSC creates migration user and a secret associated with this user first,
+// then create SC based on the same.
+func createMigrationUserSecretAndSC(f *framework.Framework, scName string) error {
+	if scName == "" {
+		scName = defaultSCName
+	}
+	err := createProvNodeCephUserAndSecret(f, true, true)
+	if err != nil {
+		return err
+	}
+
+	err = createMigrationSC(f, scName)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func createMigrationSC(f *framework.Framework, scName string) error {
+	err := deleteResource(rbdExamplePath + "storageclass.yaml")
+	if err != nil {
+		return fmt.Errorf("failed to delete storageclass: %w", err)
+	}
+	param := make(map[string]string)
+	// add new secrets to the SC parameters
+	param["csi.storage.k8s.io/provisioner-secret-namespace"] = cephCSINamespace
+	param["csi.storage.k8s.io/provisioner-secret-name"] = rbdMigrationProvisionerSecretName
+	param["csi.storage.k8s.io/controller-expand-secret-namespace"] = cephCSINamespace
+	param["csi.storage.k8s.io/controller-expand-secret-name"] = rbdMigrationProvisionerSecretName
+	param["csi.storage.k8s.io/node-stage-secret-namespace"] = cephCSINamespace
+	param["csi.storage.k8s.io/node-stage-secret-name"] = rbdMigrationNodePluginSecretName
+	err = createRBDStorageClass(f.ClientSet, f, scName, nil, param, deletePolicy)
+	if err != nil {
+		return fmt.Errorf("failed to create storageclass: %w", err)
+	}
+
+	return nil
+}
+
+// createProvNodeCephUserAndSecret fetches the ceph migration user's key and create migration secret
+// with it based on the arg values of 'provSecret' and 'nodeSecret'.
+func createProvNodeCephUserAndSecret(f *framework.Framework, provisionerSecret, nodeSecret bool) error {
+	if provisionerSecret {
+		// Fetch the key.
+		key, err := createCephUser(
+			f,
+			keyringRBDProvisionerUsername,
+			rbdProvisionerCaps(defaultRBDPool, radosNamespace),
+		)
+		if err != nil {
+			return fmt.Errorf("failed to create user %q: %w", keyringRBDProvisionerUsername, err)
+		}
+		err = createRBDMigrationSecret(f, rbdMigrationProvisionerSecretName, keyringRBDProvisionerUsername, key)
+		if err != nil {
+			return fmt.Errorf("failed to create provisioner secret: %w", err)
+		}
+	}
+
+	if nodeSecret {
+		// Fetch the key.
+		key, err := createCephUser(
+			f,
+			keyringRBDNodePluginUsername,
+			rbdNodePluginCaps(defaultRBDPool, radosNamespace))
+		if err != nil {
+			return fmt.Errorf("failed to create user %q: %w", keyringRBDNodePluginUsername, err)
+		}
+		err = createRBDMigrationSecret(f, rbdMigrationNodePluginSecretName, keyringRBDNodePluginUsername, key)
+		if err != nil {
+			return fmt.Errorf("failed to create node secret: %w", err)
+		}
+	}
+
+	return nil
+}
+
+// deleteProvNodeMigrationSecret deletes ceph migration secrets based on the
+// arg values of 'provisionerSecret' and 'nodeSecret'.
+func deleteProvNodeMigrationSecret(f *framework.Framework, provisionerSecret, nodeSecret bool) error {
+	c := f.ClientSet
+	if provisionerSecret {
+		// delete RBD provisioner secret.
+		err := c.CoreV1().
+			Secrets(cephCSINamespace).
+			Delete(context.TODO(), rbdMigrationProvisionerSecretName, metav1.DeleteOptions{})
+		if err != nil {
+			return fmt.Errorf("failed to delete provisioner secret: %w", err)
+		}
+	}
+
+	if nodeSecret {
+		// delete RBD node secret.
+		err := c.CoreV1().
+			Secrets(cephCSINamespace).
+			Delete(context.TODO(), rbdMigrationNodePluginSecretName, metav1.DeleteOptions{})
+		if err != nil {
+			return fmt.Errorf("failed to delete node secret: %w", err)
+		}
 	}
 
 	return nil

e2e/rbd.go

@@ -374,7 +374,9 @@ var _ = Describe("RBD", func() {
 				if err != nil {
 					e2elog.Failf("failed to generate clusterID configmap with error %v", err)
 				}
-				err = createRBDStorageClass(f.ClientSet, f, "migrationsc", nil, nil, deletePolicy)
+
+				// create a sc with different migration secret
+				err = createMigrationUserSecretAndSC(f, "migrationsc")
 				if err != nil {
 					e2elog.Failf("failed to create storageclass with error %v", err)
 				}
@@ -392,6 +394,15 @@ var _ = Describe("RBD", func() {
 				if err != nil {
 					e2elog.Failf("failed to create configmap with error %v", err)
 				}
+
+				err = deleteProvNodeMigrationSecret(f, true, true)
+				if err != nil {
+					e2elog.Failf("failed to delete migration users and Secrets associated with error %v", err)
+				}
+				err = createRBDStorageClass(f.ClientSet, f, defaultSCName, nil, nil, deletePolicy)
+				if err != nil {
+					e2elog.Failf("failed to create storageclass with error %v", err)
+				}
 			})
 
 			By("create a PVC and validate owner", func() {
@@ -1606,12 +1617,24 @@ var _ = Describe("RBD", func() {
 				if err != nil {
 					e2elog.Failf("failed to generate clusterID configmap with error %v", err)
 				}
-				err = validateRBDStaticMigrationPV(f, appPath, false)
+				// create node user and migration secret.
+				err = createProvNodeCephUserAndSecret(f, false, true)
+				if err != nil {
+					e2elog.Failf("failed to create users and secret with error %v", err)
+				}
+
+				err = validateRBDStaticMigrationPV(f, appPath, rbdMigrationNodePluginSecretName, false)
 				if err != nil {
 					e2elog.Failf("failed to validate rbd migrated static pv with error %v", err)
 				}
 				// validate created backend rbd images
 				validateRBDImageCount(f, 0, defaultRBDPool)
+
+				err = deleteProvNodeMigrationSecret(f, false, true)
+				if err != nil {
+					e2elog.Failf("failed to delete users and secret with error %v", err)
+				}
+
 				err = deleteConfigMap(rbdDirPath)
 				if err != nil {
 					e2elog.Failf("failed to delete configmap with error %v", err)
@@ -1627,12 +1650,24 @@ var _ = Describe("RBD", func() {
 				if err != nil {
 					e2elog.Failf("failed to generate clusterID configmap with error %v", err)
 				}
-				err = validateRBDStaticMigrationPV(f, rawAppPath, true)
+				// create node user and migration secret.
+				err = createProvNodeCephUserAndSecret(f, false, true)
+				if err != nil {
+					e2elog.Failf("failed to create users and secret with error %v", err)
+				}
+
+				err = validateRBDStaticMigrationPV(f, rawAppPath, rbdMigrationNodePluginSecretName, true)
 				if err != nil {
 					e2elog.Failf("failed to validate rbd migrated static block pv with error %v", err)
 				}
 				// validate created backend rbd images
 				validateRBDImageCount(f, 0, defaultRBDPool)
+
+				err = deleteProvNodeMigrationSecret(f, false, true)
+				if err != nil {
+					e2elog.Failf("failed to delete users and secret with error %v", err)
+				}
+
 				err = deleteConfigMap(rbdDirPath)
 				if err != nil {
 					e2elog.Failf("failed to delete configmap with error %v", err)

e2e/staticpvc.go

@@ -221,7 +221,7 @@ func validateRBDStaticPV(f *framework.Framework, appPath string, isBlock, checkI
 	return err
 }
 
-func validateRBDStaticMigrationPV(f *framework.Framework, appPath string, isBlock bool) error {
+func validateRBDStaticMigrationPV(f *framework.Framework, appPath, nodeSecretName string, isBlock bool) error {
 	opt := make(map[string]string)
 	var (
 		rbdImageName = "test-static-pv"
@@ -254,6 +254,9 @@ func validateRBDStaticMigrationPV(f *framework.Framework, appPath string, isBloc
 	if e != "" {
 		return fmt.Errorf("failed to create rbd image %s", e)
 	}
+	if nodeSecretName == "" {
+		nodeSecretName = rbdNodePluginSecretName
+	}
 
 	opt["migration"] = "true"
 	opt["clusterID"] = getMonsHash(mon)
@@ -265,7 +268,7 @@ func validateRBDStaticMigrationPV(f *framework.Framework, appPath string, isBloc
 		pvName,
 		rbdImageName,
 		size,
-		rbdNodePluginSecretName,
+		nodeSecretName,
 		cephCSINamespace,
 		sc,
 		"rbd.csi.ceph.com",