e2e: add test for migration volID detection and delete of image

This commit add test for migration delete volID detection scenario
by passing a custom volID and with the entries in configmap changed
to simulate the situation. The staticPV function also changed its
accept the annotation map which make it more general usage.

Signed-off-by: Humble Chirammal <hchiramm@redhat.com>

e2e/configmap.go

@@ -96,15 +96,11 @@ func createCustomConfigMap(
 	for key := range clusterInfo {
 		clusterID = append(clusterID, key)
 	}
-	conmap := []util.ClusterInfo{
+	conmap := make([]util.ClusterInfo, len(clusterID))
-		{
+
-			ClusterID: clusterID[0],
+	for i, j := range clusterID {
-			Monitors:  mons,
+		conmap[i].ClusterID = j
-		},
+		conmap[i].Monitors = mons
-		{
-			ClusterID: clusterID[1],
-			Monitors:  mons,
-		},
 	}
 
 	// fill radosNamespace and subvolgroups

e2e/migration.go

@@ -0,0 +1,116 @@
+package e2e
+
+import (
+	"context"
+	"encoding/hex"
+	"fmt"
+	"strconv"
+	"strings"
+
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/kubernetes/test/e2e/framework"
+)
+
+func validateRBDStaticMigrationPVDeletion(f *framework.Framework, appPath, scName string, isBlock bool) error {
+	opt := make(map[string]string)
+	var (
+		rbdImageName        = "kubernetes-dynamic-pvc-e0b45b52-7e09-47d3-8f1b-806995fa4412"
+		pvName              = "pv-name"
+		pvcName             = "pvc-name"
+		namespace           = f.UniqueName
+		sc                  = scName
+		provisionerAnnKey   = "pv.kubernetes.io/provisioned-by"
+		provisionerAnnValue = "rbd.csi.ceph.com"
+	)
+
+	c := f.ClientSet
+	PVAnnMap := make(map[string]string)
+	PVAnnMap[provisionerAnnKey] = provisionerAnnValue
+	mons, err := getMons(rookNamespace, c)
+	if err != nil {
+		return fmt.Errorf("failed to get mons: %w", err)
+	}
+	mon := strings.Join(mons, ",")
+	size := staticPVSize
+	// create rbd image
+	cmd := fmt.Sprintf(
+		"rbd create %s --size=%s --image-feature=layering %s",
+		rbdImageName,
+		staticPVSize,
+		rbdOptions(defaultRBDPool))
+
+	_, stdErr, err := execCommandInToolBoxPod(f, cmd, rookNamespace)
+	if err != nil {
+		return err
+	}
+	if stdErr != "" {
+		return fmt.Errorf("failed to create rbd image %s", stdErr)
+	}
+
+	opt["migration"] = "true"
+	opt["monitors"] = mon
+	opt["imageFeatures"] = staticPVImageFeature
+	opt["pool"] = defaultRBDPool
+	opt["staticVolume"] = strconv.FormatBool(true)
+	opt["imageName"] = rbdImageName
+
+	// Make volumeID similar to the migration volumeID
+	volID := composeIntreeMigVolID(mon, rbdImageName)
+	pv := getStaticPV(
+		pvName,
+		volID,
+		size,
+		rbdNodePluginSecretName,
+		cephCSINamespace,
+		sc,
+		provisionerAnnValue,
+		isBlock,
+		opt,
+		PVAnnMap,
+		deletePolicy)
+
+	_, err = c.CoreV1().PersistentVolumes().Create(context.TODO(), pv, metav1.CreateOptions{})
+	if err != nil {
+		return fmt.Errorf("PV Create API error: %w", err)
+	}
+
+	pvc := getStaticPVC(pvcName, pvName, size, namespace, sc, isBlock)
+
+	_, err = c.CoreV1().PersistentVolumeClaims(pvc.Namespace).Create(context.TODO(), pvc, metav1.CreateOptions{})
+	if err != nil {
+		return fmt.Errorf("PVC Create API error: %w", err)
+	}
+	// bind pvc to app
+	app, err := loadApp(appPath)
+	if err != nil {
+		return err
+	}
+
+	app.Namespace = namespace
+	app.Spec.Volumes[0].PersistentVolumeClaim.ClaimName = pvcName
+	err = createApp(f.ClientSet, app, deployTimeout)
+	if err != nil {
+		return err
+	}
+
+	err = deletePVCAndApp("", f, pvc, app)
+	if err != nil {
+		return fmt.Errorf("failed to delete PVC and application with error %w", err)
+	}
+
+	return err
+}
+
+// composeIntreeMigVolID create a volID similar to intree migration volID
+// the migration volID format looks like below
+// mig-mons-<hash>-image-<UUID_<poolhash>
+// nolint:lll    // ex: "mig_mons-b7f67366bb43f32e07d8a261a7840da9_image-e0b45b52-7e09-47d3-8f1b-806995fa4412_706f6f6c5f7265706c6963615f706f6f6c
+func composeIntreeMigVolID(mons, rbdImageName string) string {
+	poolField := hex.EncodeToString([]byte(defaultRBDPool))
+	monsField := monsPrefix + getMonsHash(mons)
+	imageUID := strings.Split(rbdImageName, intreeVolPrefix)[1:]
+	imageField := imagePrefix + imageUID[0]
+	vhSlice := []string{migIdentifier, monsField, imageField, poolField}
+
+	return strings.Join(vhSlice, "_")
+}

e2e/rbd.go

@@ -366,6 +366,47 @@ var _ = Describe("RBD", func() {
 					}
 				})
 			}
+			By("validate RBD migration+static Block PVC Deletion", func() {
+				// create monitors hash by fetching monitors from the cluster.
+				mons, err := getMons(rookNamespace, c)
+				if err != nil {
+					e2elog.Failf("failed to get monitors %v", err)
+				}
+				mon := strings.Join(mons, ",")
+				inClusterID := getMonsHash(mon)
+
+				clusterInfo := map[string]map[string]string{}
+				clusterInfo[inClusterID] = map[string]string{}
+
+				// create custom configmap
+				err = createCustomConfigMap(f.ClientSet, rbdDirPath, clusterInfo)
+				if err != nil {
+					e2elog.Failf("failed to create configmap with error %v", err)
+				}
+				err = createRBDStorageClass(f.ClientSet, f, "migrationsc", nil, nil, deletePolicy)
+				if err != nil {
+					e2elog.Failf("failed to create storageclass with error %v", err)
+				}
+				// restart csi pods for the configmap to take effect.
+				err = recreateCSIRBDPods(f)
+				if err != nil {
+					e2elog.Failf("failed to recreate rbd csi pods with error %v", err)
+				}
+				err = validateRBDStaticMigrationPVDeletion(f, rawAppPath, "migrationsc", true)
+				if err != nil {
+					e2elog.Failf("failed to validate rbd migrated static block pv with error %v", err)
+				}
+				// validate created backend rbd images
+				validateRBDImageCount(f, 0, defaultRBDPool)
+				err = deleteConfigMap(rbdDirPath)
+				if err != nil {
+					e2elog.Failf("failed to delete configmap with error %v", err)
+				}
+				err = createConfigMap(rbdDirPath, f.ClientSet, f)
+				if err != nil {
+					e2elog.Failf("failed to create configmap with error %v", err)
+				}
+			})
 
 			By("create a PVC and validate owner", func() {
 				err := validateImageOwner(pvcPath, f)

e2e/rbd_helper.go

@@ -974,3 +974,22 @@ func waitToRemoveImagesFromTrash(f *framework.Framework, poolName string, t int)
 
 	return err
 }
+
+func recreateCSIRBDPods(f *framework.Framework) error {
+	err := deletePodWithLabel("app in (ceph-csi-rbd, csi-rbdplugin, csi-rbdplugin-provisioner)",
+		cephCSINamespace, false)
+	if err != nil {
+		return fmt.Errorf("failed to delete pods with labels with error %w", err)
+	}
+	// wait for csi pods to come up
+	err = waitForDaemonSets(rbdDaemonsetName, cephCSINamespace, f.ClientSet, deployTimeout)
+	if err != nil {
+		return fmt.Errorf("timeout waiting for daemonset pods with error %w", err)
+	}
+	err = waitForDeploymentComplete(rbdDeploymentName, cephCSINamespace, f.ClientSet, deployTimeout)
+	if err != nil {
+		return fmt.Errorf("timeout waiting for deployment to be in running state with error %w", err)
+	}
+
+	return nil
+}

e2e/staticpvc.go

@@ -12,16 +12,26 @@ import (
 	"k8s.io/kubernetes/test/e2e/framework"
 )
 
+const (
+	staticPVSize         = "4Gi"
+	staticPVImageFeature = "layering"
+	monsPrefix           = "mons-"
+	imagePrefix          = "image-"
+	migIdentifier        = "mig"
+	intreeVolPrefix      = "kubernetes-dynamic-pvc-"
+)
+
+// nolint:unparam // currently name receive pvName, this can change in the future
 func getStaticPV(
 	name, volName, size, secretName, secretNS, sc, driverName string,
 	blockPV bool,
-	options map[string]string) *v1.PersistentVolume {
+	options, annotations map[string]string, policy v1.PersistentVolumeReclaimPolicy) *v1.PersistentVolume {
 	pv := &v1.PersistentVolume{
 		ObjectMeta: metav1.ObjectMeta{
 			Name: name,
 		},
 		Spec: v1.PersistentVolumeSpec{
-			PersistentVolumeReclaimPolicy: v1.PersistentVolumeReclaimRetain,
+			PersistentVolumeReclaimPolicy: policy,
 			Capacity: v1.ResourceList{
 				v1.ResourceStorage: resource.MustParse(size),
 			},
@@ -49,10 +59,17 @@ func getStaticPV(
 		volumeMode := v1.PersistentVolumeFilesystem
 		pv.Spec.VolumeMode = &volumeMode
 	}
+	if len(annotations) > 0 {
+		pv.Annotations = make(map[string]string)
+		for k, v := range annotations {
+			pv.Annotations[k] = v
+		}
+	}
 
 	return pv
 }
 
+// nolint:unparam // currently name receive same name, this can change in the future
 func getStaticPVC(name, pvName, size, ns, sc string, blockPVC bool) *v1.PersistentVolumeClaim {
 	pvc := &v1.PersistentVolumeClaim{
 		ObjectMeta: metav1.ObjectMeta{
@@ -104,12 +121,12 @@ func validateRBDStaticPV(f *framework.Framework, appPath string, isBlock, checkI
 	}
 	// remove new line present in fsID
 	fsID = strings.Trim(fsID, "\n")
-	size := "4Gi"
+	size := staticPVSize
 	// create rbd image
 	cmd := fmt.Sprintf(
-		"rbd create %s --size=%d --image-feature=layering %s",
+		"rbd create %s --size=%s --image-feature=layering %s",
 		rbdImageName,
-		4096,
+		staticPVSize,
 		rbdOptions(defaultRBDPool))
 
 	_, e, err = execCommandInToolBoxPod(f, cmd, rookNamespace)
@@ -121,7 +138,7 @@ func validateRBDStaticPV(f *framework.Framework, appPath string, isBlock, checkI
 	}
 	opt["clusterID"] = fsID
 	if !checkImgFeat {
-		opt["imageFeatures"] = "layering"
+		opt["imageFeatures"] = staticPVImageFeature
 	}
 	opt["pool"] = defaultRBDPool
 	opt["staticVolume"] = strconv.FormatBool(true)
@@ -138,7 +155,8 @@ func validateRBDStaticPV(f *framework.Framework, appPath string, isBlock, checkI
 		sc,
 		"rbd.csi.ceph.com",
 		isBlock,
-		opt)
+		opt,
+		nil, retainPolicy)
 
 	_, err = c.CoreV1().PersistentVolumes().Create(context.TODO(), pv, metav1.CreateOptions{})
 	if err != nil {
@@ -207,7 +225,7 @@ func validateRBDStaticMigrationPV(f *framework.Framework, appPath string, isBloc
 		return fmt.Errorf("failed to get mons: %w", err)
 	}
 	mon := strings.Join(mons, ",")
-	size := "4Gi"
+	size := staticPVSize
 	// create rbd image
 	cmd := fmt.Sprintf(
 		"rbd create %s --size=%d --image-feature=layering %s",
@@ -225,7 +243,7 @@ func validateRBDStaticMigrationPV(f *framework.Framework, appPath string, isBloc
 
 	opt["migration"] = "true"
 	opt["monitors"] = mon
-	opt["imageFeatures"] = "layering"
+	opt["imageFeatures"] = staticPVImageFeature
 	opt["pool"] = defaultRBDPool
 	opt["staticVolume"] = strconv.FormatBool(true)
 	opt["imageName"] = rbdImageName
@@ -238,7 +256,7 @@ func validateRBDStaticMigrationPV(f *framework.Framework, appPath string, isBloc
 		sc,
 		"rbd.csi.ceph.com",
 		isBlock,
-		opt)
+		opt, nil, retainPolicy)
 
 	_, err = c.CoreV1().PersistentVolumes().Create(context.TODO(), pv, metav1.CreateOptions{})
 	if err != nil {
@@ -378,7 +396,18 @@ func validateCephFsStaticPV(f *framework.Framework, appPath, scPath string) erro
 	opt["fsName"] = fsName
 	opt["staticVolume"] = strconv.FormatBool(true)
 	opt["rootPath"] = rootPath
-	pv := getStaticPV(pvName, pvName, "4Gi", secretName, cephCSINamespace, sc, "cephfs.csi.ceph.com", false, opt)
+	pv := getStaticPV(
+		pvName,
+		pvName,
+		staticPVSize,
+		secretName,
+		cephCSINamespace,
+		sc,
+		"cephfs.csi.ceph.com",
+		false,
+		opt,
+		nil,
+		retainPolicy)
 	_, err = c.CoreV1().PersistentVolumes().Create(context.TODO(), pv, metav1.CreateOptions{})
 	if err != nil {
 		return fmt.Errorf("failed to create PV: %w", err)

e2e/utils.go

@@ -2,6 +2,7 @@ package e2e
 
 import (
 	"context"
+	"crypto/md5" //nolint:gosec // hash generation
 	"encoding/base64"
 	"encoding/json"
 	"errors"
@@ -102,6 +103,10 @@ func getMons(ns string, c kubernetes.Interface) ([]string, error) {
 	return services, nil
 }
 
+func getMonsHash(mons string) string {
+	return fmt.Sprintf("%x", md5.Sum([]byte(mons))) //nolint:gosec // hash generation
+}
+
 func getStorageClass(path string) (scv1.StorageClass, error) {
 	sc := scv1.StorageClass{}
 	err := unmarshal(path, &sc)