From cf6fdcb796471603cd87382842a652bcd81f98a9 Mon Sep 17 00:00:00 2001 From: Humble Chirammal Date: Fri, 19 Nov 2021 10:20:27 +0530 Subject: [PATCH] e2e: adjust migration e2e tests and introduce helper functions This commit adjust existing migration e2e tests to a couple of tests to cover the scenarios. The seperate filesystem and block tests have been shrinked to single one and also introduced a couple of helper functions to setup and teardown migraition specific secret,configmap and sc. The static pv function has been renamed to a general name while the tests were adjusted. Signed-off-by: Humble Chirammal --- e2e/migration.go | 136 ++++++++++++++++------------------------------- e2e/rbd.go | 129 +++++++++++++++----------------------------- e2e/staticpvc.go | 61 +++++++++------------ 3 files changed, 113 insertions(+), 213 deletions(-) diff --git a/e2e/migration.go b/e2e/migration.go index 04390fee3..2173f1202 100644 --- a/e2e/migration.go +++ b/e2e/migration.go @@ -4,7 +4,6 @@ import ( "context" "encoding/hex" "fmt" - "strconv" "strings" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -12,96 +11,6 @@ import ( "k8s.io/kubernetes/test/e2e/framework" ) -func validateRBDStaticMigrationPVDeletion(f *framework.Framework, appPath, scName string, isBlock bool) error { - opt := make(map[string]string) - var ( - rbdImageName = "kubernetes-dynamic-pvc-e0b45b52-7e09-47d3-8f1b-806995fa4412" - pvName = "pv-name" - pvcName = "pvc-name" - namespace = f.UniqueName - sc = scName - provisionerAnnKey = "pv.kubernetes.io/provisioned-by" - provisionerAnnValue = "rbd.csi.ceph.com" - ) - - c := f.ClientSet - PVAnnMap := make(map[string]string) - PVAnnMap[provisionerAnnKey] = provisionerAnnValue - mons, err := getMons(rookNamespace, c) - if err != nil { - return fmt.Errorf("failed to get mons: %w", err) - } - mon := strings.Join(mons, ",") - size := staticPVSize - // create rbd image - cmd := fmt.Sprintf( - "rbd create %s --size=%s --image-feature=layering %s", - rbdImageName, - staticPVSize, - rbdOptions(defaultRBDPool)) - - _, stdErr, err := execCommandInToolBoxPod(f, cmd, rookNamespace) - if err != nil { - return err - } - if stdErr != "" { - return fmt.Errorf("failed to create rbd image %s", stdErr) - } - - opt["migration"] = "true" - opt["clusterID"] = getMonsHash(mon) - opt["imageFeatures"] = staticPVImageFeature - opt["pool"] = defaultRBDPool - opt["staticVolume"] = strconv.FormatBool(true) - opt["imageName"] = rbdImageName - - // Make volumeID similar to the migration volumeID - volID := composeIntreeMigVolID(mon, rbdImageName) - pv := getStaticPV( - pvName, - volID, - size, - rbdNodePluginSecretName, - cephCSINamespace, - sc, - provisionerAnnValue, - isBlock, - opt, - PVAnnMap, - deletePolicy) - - _, err = c.CoreV1().PersistentVolumes().Create(context.TODO(), pv, metav1.CreateOptions{}) - if err != nil { - return fmt.Errorf("PV Create API error: %w", err) - } - - pvc := getStaticPVC(pvcName, pvName, size, namespace, sc, isBlock) - - _, err = c.CoreV1().PersistentVolumeClaims(pvc.Namespace).Create(context.TODO(), pvc, metav1.CreateOptions{}) - if err != nil { - return fmt.Errorf("PVC Create API error: %w", err) - } - // bind pvc to app - app, err := loadApp(appPath) - if err != nil { - return err - } - - app.Namespace = namespace - app.Spec.Volumes[0].PersistentVolumeClaim.ClaimName = pvcName - err = createApp(f.ClientSet, app, deployTimeout) - if err != nil { - return err - } - - err = deletePVCAndApp("", f, pvc, app) - if err != nil { - return fmt.Errorf("failed to delete PVC and application: %w", err) - } - - return err -} - // composeIntreeMigVolID create a volID similar to intree migration volID // the migration volID format looks like below // mig-mons--image- @@ -190,6 +99,7 @@ func createMigrationUserSecretAndSC(f *framework.Framework, scName string) error return nil } +// createMigrationSC create a SC with migration specific secrets and clusterid. func createMigrationSC(f *framework.Framework, scName string) error { err := deleteResource(rbdExamplePath + "storageclass.yaml") if err != nil { @@ -203,6 +113,13 @@ func createMigrationSC(f *framework.Framework, scName string) error { param["csi.storage.k8s.io/controller-expand-secret-name"] = rbdMigrationProvisionerSecretName param["csi.storage.k8s.io/node-stage-secret-namespace"] = cephCSINamespace param["csi.storage.k8s.io/node-stage-secret-name"] = rbdMigrationNodePluginSecretName + mons, err := getMons(rookNamespace, f.ClientSet) + if err != nil { + return fmt.Errorf("failed to get mons: %w", err) + } + mon := strings.Join(mons, ",") + param["migration"] = "true" + param["clusterID"] = getMonsHash(mon) err = createRBDStorageClass(f.ClientSet, f, scName, nil, param, deletePolicy) if err != nil { return fmt.Errorf("failed to create storageclass: %w", err) @@ -274,3 +191,40 @@ func deleteProvNodeMigrationSecret(f *framework.Framework, provisionerSecret, no return nil } + +// setupMigrationCMSecretAndSC create custom configmap, secret and SC for migration tests. +func setupMigrationCMSecretAndSC(f *framework.Framework, scName string) error { + c := f.ClientSet + if scName == "" { + scName = defaultSCName + } + err := generateClusterIDConfigMapForMigration(f, c) + if err != nil { + return fmt.Errorf("failed to generate clusterID configmap: %w", err) + } + + err = createMigrationUserSecretAndSC(f, scName) + if err != nil { + return fmt.Errorf("failed to create storageclass: %w", err) + } + + return nil +} + +// tearDownMigrationSetup deletes custom configmap and secret. +func tearDownMigrationSetup(f *framework.Framework) error { + err := deleteConfigMap(rbdDirPath) + if err != nil { + return fmt.Errorf("failed to delete configmap: %w", err) + } + err = createConfigMap(rbdDirPath, f.ClientSet, f) + if err != nil { + return fmt.Errorf("failed to create configmap: %w", err) + } + err = deleteProvNodeMigrationSecret(f, true, true) + if err != nil { + return fmt.Errorf("failed to delete migration users and Secrets associated: %w", err) + } + + return nil +} diff --git a/e2e/rbd.go b/e2e/rbd.go index 42fd288f6..124d9b988 100644 --- a/e2e/rbd.go +++ b/e2e/rbd.go @@ -402,37 +402,58 @@ var _ = Describe("RBD", func() { } }) - // todo: may be remove the below deletion test later once the migration nodestage tests are adjusted - // also to have deletion validation through the same. - By("validate RBD migration+static Block PVC Deletion", func() { - err := generateClusterIDConfigMapForMigration(f, c) + By("validate RBD migration PVC", func() { + err := setupMigrationCMSecretAndSC(f, "") if err != nil { - e2elog.Failf("failed to generate clusterID configmap: %v", err) + e2elog.Failf("failed to setup migration prerequisites: %v", err) } - - // create a sc with different migration secret - err = createMigrationUserSecretAndSC(f, "migrationsc") + err = validatePVCAndAppBinding(pvcPath, appPath, f) if err != nil { - e2elog.Failf("failed to create storageclass: %v", err) - } - err = validateRBDStaticMigrationPVDeletion(f, rawAppPath, "migrationsc", true) - if err != nil { - e2elog.Failf("failed to validate rbd migrated static block pv: %v", err) + e2elog.Failf("failed to validate pvc and application binding: %v", err) } // validate created backend rbd images validateRBDImageCount(f, 0, defaultRBDPool) - err = deleteConfigMap(rbdDirPath) + + // Block PVC resize + err = resizePVCAndValidateSize(rawPvcPath, rawAppPath, f) if err != nil { - e2elog.Failf("failed to delete configmap: %v", err) - } - err = createConfigMap(rbdDirPath, f.ClientSet, f) - if err != nil { - e2elog.Failf("failed to create configmap: %v", err) + e2elog.Failf("failed to resize block PVC: %v", err) } - err = deleteProvNodeMigrationSecret(f, true, true) + // FileSystem PVC resize + err = resizePVCAndValidateSize(pvcPath, appPath, f) if err != nil { - e2elog.Failf("failed to delete migration users and Secrets associated: %v", err) + e2elog.Failf("failed to resize filesystem PVC: %v", err) + } + err = deleteResource(rbdExamplePath + "storageclass.yaml") + if err != nil { + e2elog.Failf("failed to delete storageclass: %v", err) + } + err = createRBDStorageClass(f.ClientSet, f, defaultSCName, nil, nil, deletePolicy) + if err != nil { + e2elog.Failf("failed to create storageclass: %v", err) + } + err = tearDownMigrationSetup(f) + if err != nil { + e2elog.Failf("failed to tear down migration setup: %v", err) + } + }) + + By("validate RBD migration+static FileSystem", func() { + err := setupMigrationCMSecretAndSC(f, "migrationsc") + if err != nil { + e2elog.Failf("failed to setup migration prerequisites: %v", err) + } + // validate filesystem pvc mount + err = validateRBDStaticMigrationPVC(f, appPath, "migrationsc", false) + if err != nil { + e2elog.Failf("failed to validate rbd migrated static file mode pvc: %v", err) + } + // validate created backend rbd images + validateRBDImageCount(f, 0, defaultRBDPool) + err = tearDownMigrationSetup(f) + if err != nil { + e2elog.Failf("failed to tear down migration setup: %v", err) } err = createRBDStorageClass(f.ClientSet, f, defaultSCName, nil, nil, deletePolicy) if err != nil { @@ -2117,72 +2138,6 @@ var _ = Describe("RBD", func() { validateRBDImageCount(f, 0, defaultRBDPool) }) - By("validate RBD migration+static FileSystem PVC", func() { - err := generateClusterIDConfigMapForMigration(f, c) - if err != nil { - e2elog.Failf("failed to generate clusterID configmap: %v", err) - } - // create node user and migration secret. - err = createProvNodeCephUserAndSecret(f, false, true) - if err != nil { - e2elog.Failf("failed to create users and secret: %v", err) - } - - err = validateRBDStaticMigrationPV(f, appPath, rbdMigrationNodePluginSecretName, false) - if err != nil { - e2elog.Failf("failed to validate rbd migrated static pv: %v", err) - } - // validate created backend rbd images - validateRBDImageCount(f, 0, defaultRBDPool) - - err = deleteProvNodeMigrationSecret(f, false, true) - if err != nil { - e2elog.Failf("failed to delete users and secret: %v", err) - } - - err = deleteConfigMap(rbdDirPath) - if err != nil { - e2elog.Failf("failed to delete configmap: %v", err) - } - err = createConfigMap(rbdDirPath, f.ClientSet, f) - if err != nil { - e2elog.Failf("failed to create configmap: %v", err) - } - }) - - By("validate RBD migration+static Block PVC", func() { - err := generateClusterIDConfigMapForMigration(f, c) - if err != nil { - e2elog.Failf("failed to generate clusterID configmap: %v", err) - } - // create node user and migration secret. - err = createProvNodeCephUserAndSecret(f, false, true) - if err != nil { - e2elog.Failf("failed to create users and secret: %v", err) - } - - err = validateRBDStaticMigrationPV(f, rawAppPath, rbdMigrationNodePluginSecretName, true) - if err != nil { - e2elog.Failf("failed to validate rbd migrated static block pv: %v", err) - } - // validate created backend rbd images - validateRBDImageCount(f, 0, defaultRBDPool) - - err = deleteProvNodeMigrationSecret(f, false, true) - if err != nil { - e2elog.Failf("failed to delete users and secret: %v", err) - } - - err = deleteConfigMap(rbdDirPath) - if err != nil { - e2elog.Failf("failed to delete configmap: %v", err) - } - err = createConfigMap(rbdDirPath, f.ClientSet, f) - if err != nil { - e2elog.Failf("failed to create configmap: %v", err) - } - }) - By("validate failure of RBD static PVC without imageFeatures parameter", func() { err := validateRBDStaticPV(f, rawAppPath, true, true) if err != nil { diff --git a/e2e/staticpvc.go b/e2e/staticpvc.go index bcc5933af..28cd945a8 100644 --- a/e2e/staticpvc.go +++ b/e2e/staticpvc.go @@ -219,19 +219,21 @@ func validateRBDStaticPV(f *framework.Framework, appPath string, isBlock, checkI return err } -func validateRBDStaticMigrationPV(f *framework.Framework, appPath, nodeSecretName string, isBlock bool) error { +func validateRBDStaticMigrationPVC(f *framework.Framework, appPath, scName string, isBlock bool) error { opt := make(map[string]string) var ( - rbdImageName = "test-static-pv" - pvName = "pv-name" - pvcName = "pvc-name" - namespace = f.UniqueName - // minikube creates default class in cluster, we need to set dummy - // storageclass on PV and PVC to avoid storageclass name mismatch - sc = "storage-class" + rbdImageName = "kubernetes-dynamic-pvc-e0b45b52-7e09-47d3-8f1b-806995fa4412" + pvName = "pv-name" + pvcName = "pvc-name" + namespace = f.UniqueName + sc = scName + provisionerAnnKey = "pv.kubernetes.io/provisioned-by" + provisionerAnnValue = "rbd.csi.ceph.com" ) c := f.ClientSet + PVAnnMap := make(map[string]string) + PVAnnMap[provisionerAnnKey] = provisionerAnnValue mons, err := getMons(rookNamespace, c) if err != nil { return fmt.Errorf("failed to get mons: %w", err) @@ -240,20 +242,17 @@ func validateRBDStaticMigrationPV(f *framework.Framework, appPath, nodeSecretNam size := staticPVSize // create rbd image cmd := fmt.Sprintf( - "rbd create %s --size=%d --image-feature=layering %s", + "rbd create %s --size=%s --image-feature=layering %s", rbdImageName, - 4096, + staticPVSize, rbdOptions(defaultRBDPool)) - _, e, err := execCommandInToolBoxPod(f, cmd, rookNamespace) + _, stdErr, err := execCommandInToolBoxPod(f, cmd, rookNamespace) if err != nil { return err } - if e != "" { - return fmt.Errorf("failed to create rbd image %s", e) - } - if nodeSecretName == "" { - nodeSecretName = rbdNodePluginSecretName + if stdErr != "" { + return fmt.Errorf("failed to create rbd image %s", stdErr) } opt["migration"] = "true" @@ -262,16 +261,21 @@ func validateRBDStaticMigrationPV(f *framework.Framework, appPath, nodeSecretNam opt["pool"] = defaultRBDPool opt["staticVolume"] = strconv.FormatBool(true) opt["imageName"] = rbdImageName + + // Make volumeID similar to the migration volumeID + volID := composeIntreeMigVolID(mon, rbdImageName) pv := getStaticPV( pvName, - rbdImageName, + volID, size, - nodeSecretName, + rbdNodePluginSecretName, cephCSINamespace, sc, - "rbd.csi.ceph.com", + provisionerAnnValue, isBlock, - opt, nil, retainPolicy) + opt, + PVAnnMap, + deletePolicy) _, err = c.CoreV1().PersistentVolumes().Create(context.TODO(), pv, metav1.CreateOptions{}) if err != nil { @@ -297,24 +301,11 @@ func validateRBDStaticMigrationPV(f *framework.Framework, appPath, nodeSecretNam return err } - err = deletePod(app.Name, app.Namespace, f.ClientSet, deployTimeout) + err = deletePVCAndApp("", f, pvc, app) if err != nil { - return err + return fmt.Errorf("failed to delete PVC and application: %w", err) } - err = c.CoreV1().PersistentVolumeClaims(pvc.Namespace).Delete(context.TODO(), pvc.Name, metav1.DeleteOptions{}) - if err != nil { - return fmt.Errorf("failed to delete pvc: %w", err) - } - - err = c.CoreV1().PersistentVolumes().Delete(context.TODO(), pv.Name, metav1.DeleteOptions{}) - if err != nil { - return fmt.Errorf("failed to delete pv: %w", err) - } - - cmd = fmt.Sprintf("rbd rm %s %s", rbdImageName, rbdOptions(defaultRBDPool)) - _, _, err = execCommandInToolBoxPod(f, cmd, rookNamespace) - return err }