diff --git a/e2e/configmap.go b/e2e/configmap.go index 4283cf6c4..b02609b24 100644 --- a/e2e/configmap.go +++ b/e2e/configmap.go @@ -96,15 +96,11 @@ func createCustomConfigMap( for key := range clusterInfo { clusterID = append(clusterID, key) } - conmap := []util.ClusterInfo{ - { - ClusterID: clusterID[0], - Monitors: mons, - }, - { - ClusterID: clusterID[1], - Monitors: mons, - }, + conmap := make([]util.ClusterInfo, len(clusterID)) + + for i, j := range clusterID { + conmap[i].ClusterID = j + conmap[i].Monitors = mons } // fill radosNamespace and subvolgroups diff --git a/e2e/migration.go b/e2e/migration.go new file mode 100644 index 000000000..c2bad637e --- /dev/null +++ b/e2e/migration.go @@ -0,0 +1,116 @@ +package e2e + +import ( + "context" + "encoding/hex" + "fmt" + "strconv" + "strings" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/kubernetes/test/e2e/framework" +) + +func validateRBDStaticMigrationPVDeletion(f *framework.Framework, appPath, scName string, isBlock bool) error { + opt := make(map[string]string) + var ( + rbdImageName = "kubernetes-dynamic-pvc-e0b45b52-7e09-47d3-8f1b-806995fa4412" + pvName = "pv-name" + pvcName = "pvc-name" + namespace = f.UniqueName + sc = scName + provisionerAnnKey = "pv.kubernetes.io/provisioned-by" + provisionerAnnValue = "rbd.csi.ceph.com" + ) + + c := f.ClientSet + PVAnnMap := make(map[string]string) + PVAnnMap[provisionerAnnKey] = provisionerAnnValue + mons, err := getMons(rookNamespace, c) + if err != nil { + return fmt.Errorf("failed to get mons: %w", err) + } + mon := strings.Join(mons, ",") + size := staticPVSize + // create rbd image + cmd := fmt.Sprintf( + "rbd create %s --size=%s --image-feature=layering %s", + rbdImageName, + staticPVSize, + rbdOptions(defaultRBDPool)) + + _, stdErr, err := execCommandInToolBoxPod(f, cmd, rookNamespace) + if err != nil { + return err + } + if stdErr != "" { + return fmt.Errorf("failed to create rbd image %s", stdErr) + } + + opt["migration"] = "true" + opt["monitors"] = mon + opt["imageFeatures"] = staticPVImageFeature + opt["pool"] = defaultRBDPool + opt["staticVolume"] = strconv.FormatBool(true) + opt["imageName"] = rbdImageName + + // Make volumeID similar to the migration volumeID + volID := composeIntreeMigVolID(mon, rbdImageName) + pv := getStaticPV( + pvName, + volID, + size, + rbdNodePluginSecretName, + cephCSINamespace, + sc, + provisionerAnnValue, + isBlock, + opt, + PVAnnMap, + deletePolicy) + + _, err = c.CoreV1().PersistentVolumes().Create(context.TODO(), pv, metav1.CreateOptions{}) + if err != nil { + return fmt.Errorf("PV Create API error: %w", err) + } + + pvc := getStaticPVC(pvcName, pvName, size, namespace, sc, isBlock) + + _, err = c.CoreV1().PersistentVolumeClaims(pvc.Namespace).Create(context.TODO(), pvc, metav1.CreateOptions{}) + if err != nil { + return fmt.Errorf("PVC Create API error: %w", err) + } + // bind pvc to app + app, err := loadApp(appPath) + if err != nil { + return err + } + + app.Namespace = namespace + app.Spec.Volumes[0].PersistentVolumeClaim.ClaimName = pvcName + err = createApp(f.ClientSet, app, deployTimeout) + if err != nil { + return err + } + + err = deletePVCAndApp("", f, pvc, app) + if err != nil { + return fmt.Errorf("failed to delete PVC and application with error %w", err) + } + + return err +} + +// composeIntreeMigVolID create a volID similar to intree migration volID +// the migration volID format looks like below +// mig-mons--image- +// nolint:lll // ex: "mig_mons-b7f67366bb43f32e07d8a261a7840da9_image-e0b45b52-7e09-47d3-8f1b-806995fa4412_706f6f6c5f7265706c6963615f706f6f6c +func composeIntreeMigVolID(mons, rbdImageName string) string { + poolField := hex.EncodeToString([]byte(defaultRBDPool)) + monsField := monsPrefix + getMonsHash(mons) + imageUID := strings.Split(rbdImageName, intreeVolPrefix)[1:] + imageField := imagePrefix + imageUID[0] + vhSlice := []string{migIdentifier, monsField, imageField, poolField} + + return strings.Join(vhSlice, "_") +} diff --git a/e2e/rbd.go b/e2e/rbd.go index ba02a72fc..781dc29af 100644 --- a/e2e/rbd.go +++ b/e2e/rbd.go @@ -366,6 +366,47 @@ var _ = Describe("RBD", func() { } }) } + By("validate RBD migration+static Block PVC Deletion", func() { + // create monitors hash by fetching monitors from the cluster. + mons, err := getMons(rookNamespace, c) + if err != nil { + e2elog.Failf("failed to get monitors %v", err) + } + mon := strings.Join(mons, ",") + inClusterID := getMonsHash(mon) + + clusterInfo := map[string]map[string]string{} + clusterInfo[inClusterID] = map[string]string{} + + // create custom configmap + err = createCustomConfigMap(f.ClientSet, rbdDirPath, clusterInfo) + if err != nil { + e2elog.Failf("failed to create configmap with error %v", err) + } + err = createRBDStorageClass(f.ClientSet, f, "migrationsc", nil, nil, deletePolicy) + if err != nil { + e2elog.Failf("failed to create storageclass with error %v", err) + } + // restart csi pods for the configmap to take effect. + err = recreateCSIRBDPods(f) + if err != nil { + e2elog.Failf("failed to recreate rbd csi pods with error %v", err) + } + err = validateRBDStaticMigrationPVDeletion(f, rawAppPath, "migrationsc", true) + if err != nil { + e2elog.Failf("failed to validate rbd migrated static block pv with error %v", err) + } + // validate created backend rbd images + validateRBDImageCount(f, 0, defaultRBDPool) + err = deleteConfigMap(rbdDirPath) + if err != nil { + e2elog.Failf("failed to delete configmap with error %v", err) + } + err = createConfigMap(rbdDirPath, f.ClientSet, f) + if err != nil { + e2elog.Failf("failed to create configmap with error %v", err) + } + }) By("create a PVC and validate owner", func() { err := validateImageOwner(pvcPath, f) diff --git a/e2e/rbd_helper.go b/e2e/rbd_helper.go index 1ab6eaeec..9a9d1a115 100644 --- a/e2e/rbd_helper.go +++ b/e2e/rbd_helper.go @@ -974,3 +974,22 @@ func waitToRemoveImagesFromTrash(f *framework.Framework, poolName string, t int) return err } + +func recreateCSIRBDPods(f *framework.Framework) error { + err := deletePodWithLabel("app in (ceph-csi-rbd, csi-rbdplugin, csi-rbdplugin-provisioner)", + cephCSINamespace, false) + if err != nil { + return fmt.Errorf("failed to delete pods with labels with error %w", err) + } + // wait for csi pods to come up + err = waitForDaemonSets(rbdDaemonsetName, cephCSINamespace, f.ClientSet, deployTimeout) + if err != nil { + return fmt.Errorf("timeout waiting for daemonset pods with error %w", err) + } + err = waitForDeploymentComplete(rbdDeploymentName, cephCSINamespace, f.ClientSet, deployTimeout) + if err != nil { + return fmt.Errorf("timeout waiting for deployment to be in running state with error %w", err) + } + + return nil +} diff --git a/e2e/staticpvc.go b/e2e/staticpvc.go index a78893a34..0bccae778 100644 --- a/e2e/staticpvc.go +++ b/e2e/staticpvc.go @@ -12,16 +12,26 @@ import ( "k8s.io/kubernetes/test/e2e/framework" ) +const ( + staticPVSize = "4Gi" + staticPVImageFeature = "layering" + monsPrefix = "mons-" + imagePrefix = "image-" + migIdentifier = "mig" + intreeVolPrefix = "kubernetes-dynamic-pvc-" +) + +// nolint:unparam // currently name receive pvName, this can change in the future func getStaticPV( name, volName, size, secretName, secretNS, sc, driverName string, blockPV bool, - options map[string]string) *v1.PersistentVolume { + options, annotations map[string]string, policy v1.PersistentVolumeReclaimPolicy) *v1.PersistentVolume { pv := &v1.PersistentVolume{ ObjectMeta: metav1.ObjectMeta{ Name: name, }, Spec: v1.PersistentVolumeSpec{ - PersistentVolumeReclaimPolicy: v1.PersistentVolumeReclaimRetain, + PersistentVolumeReclaimPolicy: policy, Capacity: v1.ResourceList{ v1.ResourceStorage: resource.MustParse(size), }, @@ -49,10 +59,17 @@ func getStaticPV( volumeMode := v1.PersistentVolumeFilesystem pv.Spec.VolumeMode = &volumeMode } + if len(annotations) > 0 { + pv.Annotations = make(map[string]string) + for k, v := range annotations { + pv.Annotations[k] = v + } + } return pv } +// nolint:unparam // currently name receive same name, this can change in the future func getStaticPVC(name, pvName, size, ns, sc string, blockPVC bool) *v1.PersistentVolumeClaim { pvc := &v1.PersistentVolumeClaim{ ObjectMeta: metav1.ObjectMeta{ @@ -104,12 +121,12 @@ func validateRBDStaticPV(f *framework.Framework, appPath string, isBlock, checkI } // remove new line present in fsID fsID = strings.Trim(fsID, "\n") - size := "4Gi" + size := staticPVSize // create rbd image cmd := fmt.Sprintf( - "rbd create %s --size=%d --image-feature=layering %s", + "rbd create %s --size=%s --image-feature=layering %s", rbdImageName, - 4096, + staticPVSize, rbdOptions(defaultRBDPool)) _, e, err = execCommandInToolBoxPod(f, cmd, rookNamespace) @@ -121,7 +138,7 @@ func validateRBDStaticPV(f *framework.Framework, appPath string, isBlock, checkI } opt["clusterID"] = fsID if !checkImgFeat { - opt["imageFeatures"] = "layering" + opt["imageFeatures"] = staticPVImageFeature } opt["pool"] = defaultRBDPool opt["staticVolume"] = strconv.FormatBool(true) @@ -138,7 +155,8 @@ func validateRBDStaticPV(f *framework.Framework, appPath string, isBlock, checkI sc, "rbd.csi.ceph.com", isBlock, - opt) + opt, + nil, retainPolicy) _, err = c.CoreV1().PersistentVolumes().Create(context.TODO(), pv, metav1.CreateOptions{}) if err != nil { @@ -207,7 +225,7 @@ func validateRBDStaticMigrationPV(f *framework.Framework, appPath string, isBloc return fmt.Errorf("failed to get mons: %w", err) } mon := strings.Join(mons, ",") - size := "4Gi" + size := staticPVSize // create rbd image cmd := fmt.Sprintf( "rbd create %s --size=%d --image-feature=layering %s", @@ -225,7 +243,7 @@ func validateRBDStaticMigrationPV(f *framework.Framework, appPath string, isBloc opt["migration"] = "true" opt["monitors"] = mon - opt["imageFeatures"] = "layering" + opt["imageFeatures"] = staticPVImageFeature opt["pool"] = defaultRBDPool opt["staticVolume"] = strconv.FormatBool(true) opt["imageName"] = rbdImageName @@ -238,7 +256,7 @@ func validateRBDStaticMigrationPV(f *framework.Framework, appPath string, isBloc sc, "rbd.csi.ceph.com", isBlock, - opt) + opt, nil, retainPolicy) _, err = c.CoreV1().PersistentVolumes().Create(context.TODO(), pv, metav1.CreateOptions{}) if err != nil { @@ -378,7 +396,18 @@ func validateCephFsStaticPV(f *framework.Framework, appPath, scPath string) erro opt["fsName"] = fsName opt["staticVolume"] = strconv.FormatBool(true) opt["rootPath"] = rootPath - pv := getStaticPV(pvName, pvName, "4Gi", secretName, cephCSINamespace, sc, "cephfs.csi.ceph.com", false, opt) + pv := getStaticPV( + pvName, + pvName, + staticPVSize, + secretName, + cephCSINamespace, + sc, + "cephfs.csi.ceph.com", + false, + opt, + nil, + retainPolicy) _, err = c.CoreV1().PersistentVolumes().Create(context.TODO(), pv, metav1.CreateOptions{}) if err != nil { return fmt.Errorf("failed to create PV: %w", err) diff --git a/e2e/utils.go b/e2e/utils.go index 8989ab7dc..b691c1809 100644 --- a/e2e/utils.go +++ b/e2e/utils.go @@ -2,6 +2,7 @@ package e2e import ( "context" + "crypto/md5" //nolint:gosec // hash generation "encoding/base64" "encoding/json" "errors" @@ -102,6 +103,10 @@ func getMons(ns string, c kubernetes.Interface) ([]string, error) { return services, nil } +func getMonsHash(mons string) string { + return fmt.Sprintf("%x", md5.Sum([]byte(mons))) //nolint:gosec // hash generation +} + func getStorageClass(path string) (scv1.StorageClass, error) { sc := scv1.StorageClass{} err := unmarshal(path, &sc)