e2e: adjust migration e2e tests and introduce helper functions

This commit adjust existing migration e2e tests to a couple of tests
to cover the scenarios. The seperate filesystem and block tests have
been shrinked to single one and also introduced a couple of helper
functions to setup and teardown migraition specific secret,configmap
and sc. The static pv function has been renamed to a general name
while the tests were adjusted.

Signed-off-by: Humble Chirammal <hchiramm@redhat.com>
This commit is contained in:
Humble Chirammal 2021-11-19 10:20:27 +05:30 committed by mergify[bot]
parent 784c3ef06d
commit cf6fdcb796
3 changed files with 113 additions and 213 deletions

View File

@ -4,7 +4,6 @@ import (
"context"
"encoding/hex"
"fmt"
"strconv"
"strings"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@ -12,96 +11,6 @@ import (
"k8s.io/kubernetes/test/e2e/framework"
)
func validateRBDStaticMigrationPVDeletion(f *framework.Framework, appPath, scName string, isBlock bool) error {
opt := make(map[string]string)
var (
rbdImageName = "kubernetes-dynamic-pvc-e0b45b52-7e09-47d3-8f1b-806995fa4412"
pvName = "pv-name"
pvcName = "pvc-name"
namespace = f.UniqueName
sc = scName
provisionerAnnKey = "pv.kubernetes.io/provisioned-by"
provisionerAnnValue = "rbd.csi.ceph.com"
)
c := f.ClientSet
PVAnnMap := make(map[string]string)
PVAnnMap[provisionerAnnKey] = provisionerAnnValue
mons, err := getMons(rookNamespace, c)
if err != nil {
return fmt.Errorf("failed to get mons: %w", err)
}
mon := strings.Join(mons, ",")
size := staticPVSize
// create rbd image
cmd := fmt.Sprintf(
"rbd create %s --size=%s --image-feature=layering %s",
rbdImageName,
staticPVSize,
rbdOptions(defaultRBDPool))
_, stdErr, err := execCommandInToolBoxPod(f, cmd, rookNamespace)
if err != nil {
return err
}
if stdErr != "" {
return fmt.Errorf("failed to create rbd image %s", stdErr)
}
opt["migration"] = "true"
opt["clusterID"] = getMonsHash(mon)
opt["imageFeatures"] = staticPVImageFeature
opt["pool"] = defaultRBDPool
opt["staticVolume"] = strconv.FormatBool(true)
opt["imageName"] = rbdImageName
// Make volumeID similar to the migration volumeID
volID := composeIntreeMigVolID(mon, rbdImageName)
pv := getStaticPV(
pvName,
volID,
size,
rbdNodePluginSecretName,
cephCSINamespace,
sc,
provisionerAnnValue,
isBlock,
opt,
PVAnnMap,
deletePolicy)
_, err = c.CoreV1().PersistentVolumes().Create(context.TODO(), pv, metav1.CreateOptions{})
if err != nil {
return fmt.Errorf("PV Create API error: %w", err)
}
pvc := getStaticPVC(pvcName, pvName, size, namespace, sc, isBlock)
_, err = c.CoreV1().PersistentVolumeClaims(pvc.Namespace).Create(context.TODO(), pvc, metav1.CreateOptions{})
if err != nil {
return fmt.Errorf("PVC Create API error: %w", err)
}
// bind pvc to app
app, err := loadApp(appPath)
if err != nil {
return err
}
app.Namespace = namespace
app.Spec.Volumes[0].PersistentVolumeClaim.ClaimName = pvcName
err = createApp(f.ClientSet, app, deployTimeout)
if err != nil {
return err
}
err = deletePVCAndApp("", f, pvc, app)
if err != nil {
return fmt.Errorf("failed to delete PVC and application: %w", err)
}
return err
}
// composeIntreeMigVolID create a volID similar to intree migration volID
// the migration volID format looks like below
// mig-mons-<hash>-image-<UUID_<poolhash>
@ -190,6 +99,7 @@ func createMigrationUserSecretAndSC(f *framework.Framework, scName string) error
return nil
}
// createMigrationSC create a SC with migration specific secrets and clusterid.
func createMigrationSC(f *framework.Framework, scName string) error {
err := deleteResource(rbdExamplePath + "storageclass.yaml")
if err != nil {
@ -203,6 +113,13 @@ func createMigrationSC(f *framework.Framework, scName string) error {
param["csi.storage.k8s.io/controller-expand-secret-name"] = rbdMigrationProvisionerSecretName
param["csi.storage.k8s.io/node-stage-secret-namespace"] = cephCSINamespace
param["csi.storage.k8s.io/node-stage-secret-name"] = rbdMigrationNodePluginSecretName
mons, err := getMons(rookNamespace, f.ClientSet)
if err != nil {
return fmt.Errorf("failed to get mons: %w", err)
}
mon := strings.Join(mons, ",")
param["migration"] = "true"
param["clusterID"] = getMonsHash(mon)
err = createRBDStorageClass(f.ClientSet, f, scName, nil, param, deletePolicy)
if err != nil {
return fmt.Errorf("failed to create storageclass: %w", err)
@ -274,3 +191,40 @@ func deleteProvNodeMigrationSecret(f *framework.Framework, provisionerSecret, no
return nil
}
// setupMigrationCMSecretAndSC create custom configmap, secret and SC for migration tests.
func setupMigrationCMSecretAndSC(f *framework.Framework, scName string) error {
c := f.ClientSet
if scName == "" {
scName = defaultSCName
}
err := generateClusterIDConfigMapForMigration(f, c)
if err != nil {
return fmt.Errorf("failed to generate clusterID configmap: %w", err)
}
err = createMigrationUserSecretAndSC(f, scName)
if err != nil {
return fmt.Errorf("failed to create storageclass: %w", err)
}
return nil
}
// tearDownMigrationSetup deletes custom configmap and secret.
func tearDownMigrationSetup(f *framework.Framework) error {
err := deleteConfigMap(rbdDirPath)
if err != nil {
return fmt.Errorf("failed to delete configmap: %w", err)
}
err = createConfigMap(rbdDirPath, f.ClientSet, f)
if err != nil {
return fmt.Errorf("failed to create configmap: %w", err)
}
err = deleteProvNodeMigrationSecret(f, true, true)
if err != nil {
return fmt.Errorf("failed to delete migration users and Secrets associated: %w", err)
}
return nil
}

View File

@ -402,37 +402,58 @@ var _ = Describe("RBD", func() {
}
})
// todo: may be remove the below deletion test later once the migration nodestage tests are adjusted
// also to have deletion validation through the same.
By("validate RBD migration+static Block PVC Deletion", func() {
err := generateClusterIDConfigMapForMigration(f, c)
By("validate RBD migration PVC", func() {
err := setupMigrationCMSecretAndSC(f, "")
if err != nil {
e2elog.Failf("failed to generate clusterID configmap: %v", err)
e2elog.Failf("failed to setup migration prerequisites: %v", err)
}
// create a sc with different migration secret
err = createMigrationUserSecretAndSC(f, "migrationsc")
err = validatePVCAndAppBinding(pvcPath, appPath, f)
if err != nil {
e2elog.Failf("failed to create storageclass: %v", err)
}
err = validateRBDStaticMigrationPVDeletion(f, rawAppPath, "migrationsc", true)
if err != nil {
e2elog.Failf("failed to validate rbd migrated static block pv: %v", err)
e2elog.Failf("failed to validate pvc and application binding: %v", err)
}
// validate created backend rbd images
validateRBDImageCount(f, 0, defaultRBDPool)
err = deleteConfigMap(rbdDirPath)
// Block PVC resize
err = resizePVCAndValidateSize(rawPvcPath, rawAppPath, f)
if err != nil {
e2elog.Failf("failed to delete configmap: %v", err)
}
err = createConfigMap(rbdDirPath, f.ClientSet, f)
if err != nil {
e2elog.Failf("failed to create configmap: %v", err)
e2elog.Failf("failed to resize block PVC: %v", err)
}
err = deleteProvNodeMigrationSecret(f, true, true)
// FileSystem PVC resize
err = resizePVCAndValidateSize(pvcPath, appPath, f)
if err != nil {
e2elog.Failf("failed to delete migration users and Secrets associated: %v", err)
e2elog.Failf("failed to resize filesystem PVC: %v", err)
}
err = deleteResource(rbdExamplePath + "storageclass.yaml")
if err != nil {
e2elog.Failf("failed to delete storageclass: %v", err)
}
err = createRBDStorageClass(f.ClientSet, f, defaultSCName, nil, nil, deletePolicy)
if err != nil {
e2elog.Failf("failed to create storageclass: %v", err)
}
err = tearDownMigrationSetup(f)
if err != nil {
e2elog.Failf("failed to tear down migration setup: %v", err)
}
})
By("validate RBD migration+static FileSystem", func() {
err := setupMigrationCMSecretAndSC(f, "migrationsc")
if err != nil {
e2elog.Failf("failed to setup migration prerequisites: %v", err)
}
// validate filesystem pvc mount
err = validateRBDStaticMigrationPVC(f, appPath, "migrationsc", false)
if err != nil {
e2elog.Failf("failed to validate rbd migrated static file mode pvc: %v", err)
}
// validate created backend rbd images
validateRBDImageCount(f, 0, defaultRBDPool)
err = tearDownMigrationSetup(f)
if err != nil {
e2elog.Failf("failed to tear down migration setup: %v", err)
}
err = createRBDStorageClass(f.ClientSet, f, defaultSCName, nil, nil, deletePolicy)
if err != nil {
@ -2117,72 +2138,6 @@ var _ = Describe("RBD", func() {
validateRBDImageCount(f, 0, defaultRBDPool)
})
By("validate RBD migration+static FileSystem PVC", func() {
err := generateClusterIDConfigMapForMigration(f, c)
if err != nil {
e2elog.Failf("failed to generate clusterID configmap: %v", err)
}
// create node user and migration secret.
err = createProvNodeCephUserAndSecret(f, false, true)
if err != nil {
e2elog.Failf("failed to create users and secret: %v", err)
}
err = validateRBDStaticMigrationPV(f, appPath, rbdMigrationNodePluginSecretName, false)
if err != nil {
e2elog.Failf("failed to validate rbd migrated static pv: %v", err)
}
// validate created backend rbd images
validateRBDImageCount(f, 0, defaultRBDPool)
err = deleteProvNodeMigrationSecret(f, false, true)
if err != nil {
e2elog.Failf("failed to delete users and secret: %v", err)
}
err = deleteConfigMap(rbdDirPath)
if err != nil {
e2elog.Failf("failed to delete configmap: %v", err)
}
err = createConfigMap(rbdDirPath, f.ClientSet, f)
if err != nil {
e2elog.Failf("failed to create configmap: %v", err)
}
})
By("validate RBD migration+static Block PVC", func() {
err := generateClusterIDConfigMapForMigration(f, c)
if err != nil {
e2elog.Failf("failed to generate clusterID configmap: %v", err)
}
// create node user and migration secret.
err = createProvNodeCephUserAndSecret(f, false, true)
if err != nil {
e2elog.Failf("failed to create users and secret: %v", err)
}
err = validateRBDStaticMigrationPV(f, rawAppPath, rbdMigrationNodePluginSecretName, true)
if err != nil {
e2elog.Failf("failed to validate rbd migrated static block pv: %v", err)
}
// validate created backend rbd images
validateRBDImageCount(f, 0, defaultRBDPool)
err = deleteProvNodeMigrationSecret(f, false, true)
if err != nil {
e2elog.Failf("failed to delete users and secret: %v", err)
}
err = deleteConfigMap(rbdDirPath)
if err != nil {
e2elog.Failf("failed to delete configmap: %v", err)
}
err = createConfigMap(rbdDirPath, f.ClientSet, f)
if err != nil {
e2elog.Failf("failed to create configmap: %v", err)
}
})
By("validate failure of RBD static PVC without imageFeatures parameter", func() {
err := validateRBDStaticPV(f, rawAppPath, true, true)
if err != nil {

View File

@ -219,19 +219,21 @@ func validateRBDStaticPV(f *framework.Framework, appPath string, isBlock, checkI
return err
}
func validateRBDStaticMigrationPV(f *framework.Framework, appPath, nodeSecretName string, isBlock bool) error {
func validateRBDStaticMigrationPVC(f *framework.Framework, appPath, scName string, isBlock bool) error {
opt := make(map[string]string)
var (
rbdImageName = "test-static-pv"
pvName = "pv-name"
pvcName = "pvc-name"
namespace = f.UniqueName
// minikube creates default class in cluster, we need to set dummy
// storageclass on PV and PVC to avoid storageclass name mismatch
sc = "storage-class"
rbdImageName = "kubernetes-dynamic-pvc-e0b45b52-7e09-47d3-8f1b-806995fa4412"
pvName = "pv-name"
pvcName = "pvc-name"
namespace = f.UniqueName
sc = scName
provisionerAnnKey = "pv.kubernetes.io/provisioned-by"
provisionerAnnValue = "rbd.csi.ceph.com"
)
c := f.ClientSet
PVAnnMap := make(map[string]string)
PVAnnMap[provisionerAnnKey] = provisionerAnnValue
mons, err := getMons(rookNamespace, c)
if err != nil {
return fmt.Errorf("failed to get mons: %w", err)
@ -240,20 +242,17 @@ func validateRBDStaticMigrationPV(f *framework.Framework, appPath, nodeSecretNam
size := staticPVSize
// create rbd image
cmd := fmt.Sprintf(
"rbd create %s --size=%d --image-feature=layering %s",
"rbd create %s --size=%s --image-feature=layering %s",
rbdImageName,
4096,
staticPVSize,
rbdOptions(defaultRBDPool))
_, e, err := execCommandInToolBoxPod(f, cmd, rookNamespace)
_, stdErr, err := execCommandInToolBoxPod(f, cmd, rookNamespace)
if err != nil {
return err
}
if e != "" {
return fmt.Errorf("failed to create rbd image %s", e)
}
if nodeSecretName == "" {
nodeSecretName = rbdNodePluginSecretName
if stdErr != "" {
return fmt.Errorf("failed to create rbd image %s", stdErr)
}
opt["migration"] = "true"
@ -262,16 +261,21 @@ func validateRBDStaticMigrationPV(f *framework.Framework, appPath, nodeSecretNam
opt["pool"] = defaultRBDPool
opt["staticVolume"] = strconv.FormatBool(true)
opt["imageName"] = rbdImageName
// Make volumeID similar to the migration volumeID
volID := composeIntreeMigVolID(mon, rbdImageName)
pv := getStaticPV(
pvName,
rbdImageName,
volID,
size,
nodeSecretName,
rbdNodePluginSecretName,
cephCSINamespace,
sc,
"rbd.csi.ceph.com",
provisionerAnnValue,
isBlock,
opt, nil, retainPolicy)
opt,
PVAnnMap,
deletePolicy)
_, err = c.CoreV1().PersistentVolumes().Create(context.TODO(), pv, metav1.CreateOptions{})
if err != nil {
@ -297,24 +301,11 @@ func validateRBDStaticMigrationPV(f *framework.Framework, appPath, nodeSecretNam
return err
}
err = deletePod(app.Name, app.Namespace, f.ClientSet, deployTimeout)
err = deletePVCAndApp("", f, pvc, app)
if err != nil {
return err
return fmt.Errorf("failed to delete PVC and application: %w", err)
}
err = c.CoreV1().PersistentVolumeClaims(pvc.Namespace).Delete(context.TODO(), pvc.Name, metav1.DeleteOptions{})
if err != nil {
return fmt.Errorf("failed to delete pvc: %w", err)
}
err = c.CoreV1().PersistentVolumes().Delete(context.TODO(), pv.Name, metav1.DeleteOptions{})
if err != nil {
return fmt.Errorf("failed to delete pv: %w", err)
}
cmd = fmt.Sprintf("rbd rm %s %s", rbdImageName, rbdOptions(defaultRBDPool))
_, _, err = execCommandInToolBoxPod(f, cmd, rookNamespace)
return err
}