mirrors/ceph-csi
1
0
Fork 0
mirror of https://github.com/ceph/ceph-csi.git synced 2024-12-18 02:50:30 +00:00
Code Issues Packages Projects Releases Wiki Activity

e2e: add test for migration volID detection and delete of image

Browse Source 
This commit add test for migration delete volID detection scenario
by passing a custom volID and with the entries in configmap changed
to simulate the situation. The staticPV function also changed its
accept the annotation map which make it more general usage.

Signed-off-by: Humble Chirammal <hchiramm@redhat.com>
This commit is contained in:
Humble Chirammal 2021-10-01 09:45:45 +05:30 committed by mergify[bot]
parent 1171111a94
commit b778fe51a4
6 changed files with 226 additions and 20 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
e2e/configmap.go
View File

@ -96,15 +96,11 @@ func createCustomConfigMap(
for key := range clusterInfo {
clusterID = append(clusterID, key)
}
conmap := []util.ClusterInfo{
{
ClusterID: clusterID[0],
Monitors: mons,
},
{
ClusterID: clusterID[1],
Monitors: mons,
},
conmap := make([]util.ClusterInfo, len(clusterID))
for i, j := range clusterID {
conmap[i].ClusterID = j
conmap[i].Monitors = mons
}
// fill radosNamespace and subvolgroups

116
e2e/migration.go Normal file
View File

@ -0,0 +1,116 @@
package e2e
import (
"context"
"encoding/hex"
"fmt"
"strconv"
"strings"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/kubernetes/test/e2e/framework"
)
func validateRBDStaticMigrationPVDeletion(f *framework.Framework, appPath, scName string, isBlock bool) error {
opt := make(map[string]string)
var (
rbdImageName = "kubernetes-dynamic-pvc-e0b45b52-7e09-47d3-8f1b-806995fa4412"
pvName = "pv-name"
pvcName = "pvc-name"
namespace = f.UniqueName
sc = scName
provisionerAnnKey = "pv.kubernetes.io/provisioned-by"
provisionerAnnValue = "rbd.csi.ceph.com"
)
c := f.ClientSet
PVAnnMap := make(map[string]string)
PVAnnMap[provisionerAnnKey] = provisionerAnnValue
mons, err := getMons(rookNamespace, c)
if err != nil {
return fmt.Errorf("failed to get mons: %w", err)
}
mon := strings.Join(mons, ",")
size := staticPVSize
// create rbd image
cmd := fmt.Sprintf(
"rbd create %s --size=%s --image-feature=layering %s",
rbdImageName,
staticPVSize,
rbdOptions(defaultRBDPool))
_, stdErr, err := execCommandInToolBoxPod(f, cmd, rookNamespace)
if err != nil {
return err
}
if stdErr != "" {
return fmt.Errorf("failed to create rbd image %s", stdErr)
}
opt["migration"] = "true"
opt["monitors"] = mon
opt["imageFeatures"] = staticPVImageFeature
opt["pool"] = defaultRBDPool
opt["staticVolume"] = strconv.FormatBool(true)
opt["imageName"] = rbdImageName
// Make volumeID similar to the migration volumeID
volID := composeIntreeMigVolID(mon, rbdImageName)
pv := getStaticPV(
pvName,
volID,
size,
rbdNodePluginSecretName,
cephCSINamespace,
sc,
provisionerAnnValue,
isBlock,
opt,
PVAnnMap,
deletePolicy)
_, err = c.CoreV1().PersistentVolumes().Create(context.TODO(), pv, metav1.CreateOptions{})
if err != nil {
return fmt.Errorf("PV Create API error: %w", err)
}
pvc := getStaticPVC(pvcName, pvName, size, namespace, sc, isBlock)
_, err = c.CoreV1().PersistentVolumeClaims(pvc.Namespace).Create(context.TODO(), pvc, metav1.CreateOptions{})
if err != nil {
return fmt.Errorf("PVC Create API error: %w", err)
}
// bind pvc to app
app, err := loadApp(appPath)
if err != nil {
return err
}
app.Namespace = namespace
app.Spec.Volumes[0].PersistentVolumeClaim.ClaimName = pvcName
err = createApp(f.ClientSet, app, deployTimeout)
if err != nil {
return err
}
err = deletePVCAndApp("", f, pvc, app)
if err != nil {
return fmt.Errorf("failed to delete PVC and application with error %w", err)
}
return err
}
// composeIntreeMigVolID create a volID similar to intree migration volID
// the migration volID format looks like below
// mig-mons-<hash>-image-<UUID_<poolhash>
// nolint:lll // ex: "mig_mons-b7f67366bb43f32e07d8a261a7840da9_image-e0b45b52-7e09-47d3-8f1b-806995fa4412_706f6f6c5f7265706c6963615f706f6f6c
func composeIntreeMigVolID(mons, rbdImageName string) string {
poolField := hex.EncodeToString([]byte(defaultRBDPool))
monsField := monsPrefix + getMonsHash(mons)
imageUID := strings.Split(rbdImageName, intreeVolPrefix)[1:]
imageField := imagePrefix + imageUID[0]
vhSlice := []string{migIdentifier, monsField, imageField, poolField}
return strings.Join(vhSlice, "_")
}

41
e2e/rbd.go
View File

@ -366,6 +366,47 @@ var _ = Describe("RBD", func() {
}
})
}
By("validate RBD migration+static Block PVC Deletion", func() {
// create monitors hash by fetching monitors from the cluster.
mons, err := getMons(rookNamespace, c)
if err != nil {
e2elog.Failf("failed to get monitors %v", err)
}
mon := strings.Join(mons, ",")
inClusterID := getMonsHash(mon)
clusterInfo := map[string]map[string]string{}
clusterInfo[inClusterID] = map[string]string{}
// create custom configmap
err = createCustomConfigMap(f.ClientSet, rbdDirPath, clusterInfo)
if err != nil {
e2elog.Failf("failed to create configmap with error %v", err)
}
err = createRBDStorageClass(f.ClientSet, f, "migrationsc", nil, nil, deletePolicy)
if err != nil {
e2elog.Failf("failed to create storageclass with error %v", err)
}
// restart csi pods for the configmap to take effect.
err = recreateCSIRBDPods(f)
if err != nil {
e2elog.Failf("failed to recreate rbd csi pods with error %v", err)
}
err = validateRBDStaticMigrationPVDeletion(f, rawAppPath, "migrationsc", true)
if err != nil {
e2elog.Failf("failed to validate rbd migrated static block pv with error %v", err)
}
// validate created backend rbd images
validateRBDImageCount(f, 0, defaultRBDPool)
err = deleteConfigMap(rbdDirPath)
if err != nil {
e2elog.Failf("failed to delete configmap with error %v", err)
}
err = createConfigMap(rbdDirPath, f.ClientSet, f)
if err != nil {
e2elog.Failf("failed to create configmap with error %v", err)
}
})
By("create a PVC and validate owner", func() {
err := validateImageOwner(pvcPath, f)

19
e2e/rbd_helper.go
View File

@ -974,3 +974,22 @@ func waitToRemoveImagesFromTrash(f *framework.Framework, poolName string, t int)
return err
}
func recreateCSIRBDPods(f *framework.Framework) error {
err := deletePodWithLabel("app in (ceph-csi-rbd, csi-rbdplugin, csi-rbdplugin-provisioner)",
cephCSINamespace, false)
if err != nil {
return fmt.Errorf("failed to delete pods with labels with error %w", err)
}
// wait for csi pods to come up
err = waitForDaemonSets(rbdDaemonsetName, cephCSINamespace, f.ClientSet, deployTimeout)
if err != nil {
return fmt.Errorf("timeout waiting for daemonset pods with error %w", err)
}
err = waitForDeploymentComplete(rbdDeploymentName, cephCSINamespace, f.ClientSet, deployTimeout)
if err != nil {
return fmt.Errorf("timeout waiting for deployment to be in running state with error %w", err)
}
return nil
}

51
e2e/staticpvc.go
View File

@ -12,16 +12,26 @@ import (
"k8s.io/kubernetes/test/e2e/framework"
)
const (
staticPVSize = "4Gi"
staticPVImageFeature = "layering"
monsPrefix = "mons-"
imagePrefix = "image-"
migIdentifier = "mig"
intreeVolPrefix = "kubernetes-dynamic-pvc-"
)
// nolint:unparam // currently name receive pvName, this can change in the future
func getStaticPV(
name, volName, size, secretName, secretNS, sc, driverName string,
blockPV bool,
options map[string]string) *v1.PersistentVolume {
options, annotations map[string]string, policy v1.PersistentVolumeReclaimPolicy) *v1.PersistentVolume {
pv := &v1.PersistentVolume{
ObjectMeta: metav1.ObjectMeta{
Name: name,
},
Spec: v1.PersistentVolumeSpec{
PersistentVolumeReclaimPolicy: v1.PersistentVolumeReclaimRetain,
PersistentVolumeReclaimPolicy: policy,
Capacity: v1.ResourceList{
v1.ResourceStorage: resource.MustParse(size),
},
@ -49,10 +59,17 @@ func getStaticPV(
volumeMode := v1.PersistentVolumeFilesystem
pv.Spec.VolumeMode = &volumeMode
}
if len(annotations) > 0 {
pv.Annotations = make(map[string]string)
for k, v := range annotations {
pv.Annotations[k] = v
}
}
return pv
}
// nolint:unparam // currently name receive same name, this can change in the future
func getStaticPVC(name, pvName, size, ns, sc string, blockPVC bool) *v1.PersistentVolumeClaim {
pvc := &v1.PersistentVolumeClaim{
ObjectMeta: metav1.ObjectMeta{
@ -104,12 +121,12 @@ func validateRBDStaticPV(f *framework.Framework, appPath string, isBlock, checkI
}
// remove new line present in fsID
fsID = strings.Trim(fsID, "\n")
size := "4Gi"
size := staticPVSize
// create rbd image
cmd := fmt.Sprintf(
"rbd create %s --size=%d --image-feature=layering %s",
"rbd create %s --size=%s --image-feature=layering %s",
rbdImageName,
4096,
staticPVSize,
rbdOptions(defaultRBDPool))
_, e, err = execCommandInToolBoxPod(f, cmd, rookNamespace)
@ -121,7 +138,7 @@ func validateRBDStaticPV(f *framework.Framework, appPath string, isBlock, checkI
}
opt["clusterID"] = fsID
if !checkImgFeat {
opt["imageFeatures"] = "layering"
opt["imageFeatures"] = staticPVImageFeature
}
opt["pool"] = defaultRBDPool
opt["staticVolume"] = strconv.FormatBool(true)
@ -138,7 +155,8 @@ func validateRBDStaticPV(f *framework.Framework, appPath string, isBlock, checkI
sc,
"rbd.csi.ceph.com",
isBlock,
opt)
opt,
nil, retainPolicy)
_, err = c.CoreV1().PersistentVolumes().Create(context.TODO(), pv, metav1.CreateOptions{})
if err != nil {
@ -207,7 +225,7 @@ func validateRBDStaticMigrationPV(f *framework.Framework, appPath string, isBloc
return fmt.Errorf("failed to get mons: %w", err)
}
mon := strings.Join(mons, ",")
size := "4Gi"
size := staticPVSize
// create rbd image
cmd := fmt.Sprintf(
"rbd create %s --size=%d --image-feature=layering %s",
@ -225,7 +243,7 @@ func validateRBDStaticMigrationPV(f *framework.Framework, appPath string, isBloc
opt["migration"] = "true"
opt["monitors"] = mon
opt["imageFeatures"] = "layering"
opt["imageFeatures"] = staticPVImageFeature
opt["pool"] = defaultRBDPool
opt["staticVolume"] = strconv.FormatBool(true)
opt["imageName"] = rbdImageName
@ -238,7 +256,7 @@ func validateRBDStaticMigrationPV(f *framework.Framework, appPath string, isBloc
sc,
"rbd.csi.ceph.com",
isBlock,
opt)
opt, nil, retainPolicy)
_, err = c.CoreV1().PersistentVolumes().Create(context.TODO(), pv, metav1.CreateOptions{})
if err != nil {
@ -378,7 +396,18 @@ func validateCephFsStaticPV(f *framework.Framework, appPath, scPath string) erro
opt["fsName"] = fsName
opt["staticVolume"] = strconv.FormatBool(true)
opt["rootPath"] = rootPath
pv := getStaticPV(pvName, pvName, "4Gi", secretName, cephCSINamespace, sc, "cephfs.csi.ceph.com", false, opt)
pv := getStaticPV(
pvName,
pvName,
staticPVSize,
secretName,
cephCSINamespace,
sc,
"cephfs.csi.ceph.com",
false,
opt,
nil,
retainPolicy)
_, err = c.CoreV1().PersistentVolumes().Create(context.TODO(), pv, metav1.CreateOptions{})
if err != nil {
return fmt.Errorf("failed to create PV: %w", err)

5
e2e/utils.go
View File

@ -2,6 +2,7 @@ package e2e
import (
"context"
"crypto/md5" //nolint:gosec // hash generation
"encoding/base64"
"encoding/json"
"errors"
@ -102,6 +103,10 @@ func getMons(ns string, c kubernetes.Interface) ([]string, error) {
return services, nil
}
func getMonsHash(mons string) string {
return fmt.Sprintf("%x", md5.Sum([]byte(mons))) //nolint:gosec // hash generation
}
func getStorageClass(path string) (scv1.StorageClass, error) {
sc := scv1.StorageClass{}
err := unmarshal(path, &sc)