ceph-csi/e2e/staticpvc.go
Humble Chirammal cf6fdcb796 e2e: adjust migration e2e tests and introduce helper functions
This commit adjust existing migration e2e tests to a couple of tests
to cover the scenarios. The seperate filesystem and block tests have
been shrinked to single one and also introduced a couple of helper
functions to setup and teardown migraition specific secret,configmap
and sc. The static pv function has been renamed to a general name
while the tests were adjusted.

Signed-off-by: Humble Chirammal <hchiramm@redhat.com>
2021-12-20 07:34:43 +00:00

515 lines
13 KiB
Go

package e2e
import (
"context"
"fmt"
"strconv"
"strings"
v1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/api/resource"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/kubernetes/test/e2e/framework"
)
const (
staticPVSize = "4Gi"
staticPVNewSize = "8Gi"
staticPVImageFeature = "layering"
monsPrefix = "mons-"
imagePrefix = "image-"
migIdentifier = "mig"
intreeVolPrefix = "kubernetes-dynamic-pvc-"
)
func getStaticPV(
name, volName, size, secretName, secretNS, sc, driverName string,
blockPV bool,
options, annotations map[string]string, policy v1.PersistentVolumeReclaimPolicy) *v1.PersistentVolume {
pv := &v1.PersistentVolume{
ObjectMeta: metav1.ObjectMeta{
Name: name,
},
Spec: v1.PersistentVolumeSpec{
PersistentVolumeReclaimPolicy: policy,
Capacity: v1.ResourceList{
v1.ResourceStorage: resource.MustParse(size),
},
PersistentVolumeSource: v1.PersistentVolumeSource{
CSI: &v1.CSIPersistentVolumeSource{
Driver: driverName,
VolumeHandle: volName,
ReadOnly: false,
VolumeAttributes: options,
NodeStageSecretRef: &v1.SecretReference{
Name: secretName,
Namespace: secretNS,
},
},
},
StorageClassName: sc,
AccessModes: []v1.PersistentVolumeAccessMode{v1.ReadWriteOnce},
},
}
if blockPV {
volumeMode := v1.PersistentVolumeBlock
pv.Spec.VolumeMode = &volumeMode
} else {
volumeMode := v1.PersistentVolumeFilesystem
pv.Spec.VolumeMode = &volumeMode
}
if len(annotations) > 0 {
pv.Annotations = make(map[string]string)
for k, v := range annotations {
pv.Annotations[k] = v
}
}
return pv
}
func getStaticPVC(name, pvName, size, ns, sc string, blockPVC bool) *v1.PersistentVolumeClaim {
pvc := &v1.PersistentVolumeClaim{
ObjectMeta: metav1.ObjectMeta{
Name: name,
Namespace: ns,
},
Spec: v1.PersistentVolumeClaimSpec{
Resources: v1.ResourceRequirements{
Requests: v1.ResourceList{
v1.ResourceStorage: resource.MustParse(size),
},
},
AccessModes: []v1.PersistentVolumeAccessMode{v1.ReadWriteOnce},
VolumeName: pvName,
StorageClassName: &sc,
},
}
if blockPVC {
volumeMode := v1.PersistentVolumeBlock
pvc.Spec.VolumeMode = &volumeMode
} else {
volumeMode := v1.PersistentVolumeFilesystem
pvc.Spec.VolumeMode = &volumeMode
}
return pvc
}
func validateRBDStaticPV(f *framework.Framework, appPath string, isBlock, checkImgFeat bool) error {
opt := make(map[string]string)
var (
rbdImageName = "test-static-pv"
pvName = "pv-name"
pvcName = "pvc-name"
namespace = f.UniqueName
// minikube creates default class in cluster, we need to set dummy
// storageclass on PV and PVC to avoid storageclass name mismatch
sc = "storage-class"
)
c := f.ClientSet
fsID, e, err := execCommandInToolBoxPod(f, "ceph fsid", rookNamespace)
if err != nil {
return err
}
if e != "" {
return fmt.Errorf("failed to get fsid from ceph cluster %s", e)
}
// remove new line present in fsID
fsID = strings.Trim(fsID, "\n")
size := staticPVSize
// create rbd image
cmd := fmt.Sprintf(
"rbd create %s --size=%s --image-feature=layering %s",
rbdImageName,
staticPVSize,
rbdOptions(defaultRBDPool))
_, e, err = execCommandInToolBoxPod(f, cmd, rookNamespace)
if err != nil {
return err
}
if e != "" {
return fmt.Errorf("failed to create rbd image %s", e)
}
opt["clusterID"] = fsID
if !checkImgFeat {
opt["imageFeatures"] = staticPVImageFeature
}
opt["pool"] = defaultRBDPool
opt["staticVolume"] = strconv.FormatBool(true)
if radosNamespace != "" {
opt["radosNamespace"] = radosNamespace
}
pv := getStaticPV(
pvName,
rbdImageName,
size,
rbdNodePluginSecretName,
cephCSINamespace,
sc,
"rbd.csi.ceph.com",
isBlock,
opt,
nil, retainPolicy)
_, err = c.CoreV1().PersistentVolumes().Create(context.TODO(), pv, metav1.CreateOptions{})
if err != nil {
return fmt.Errorf("PV Create API error: %w", err)
}
pvc := getStaticPVC(pvcName, pvName, size, namespace, sc, isBlock)
_, err = c.CoreV1().PersistentVolumeClaims(pvc.Namespace).Create(context.TODO(), pvc, metav1.CreateOptions{})
if err != nil {
return fmt.Errorf("PVC Create API error: %w", err)
}
// bind pvc to app
app, err := loadApp(appPath)
if err != nil {
return err
}
app.Labels = make(map[string]string)
app.Labels[appKey] = appLabel
appOpt := metav1.ListOptions{
LabelSelector: fmt.Sprintf("%s=%s", appKey, appLabel),
}
app.Namespace = namespace
app.Spec.Volumes[0].PersistentVolumeClaim.ClaimName = pvcName
if checkImgFeat {
err = createAppErr(f.ClientSet, app, deployTimeout, "missing required parameter imageFeatures")
} else {
err = createApp(f.ClientSet, app, deployTimeout)
}
if err != nil {
return err
}
err = deletePod(app.Name, app.Namespace, f.ClientSet, deployTimeout)
if err != nil {
return err
}
// resize image only if the image is already mounted and formatted
if !checkImgFeat {
err = validateRBDStaticResize(f, app, &appOpt, pvc, rbdImageName)
if err != nil {
return err
}
}
err = c.CoreV1().PersistentVolumeClaims(pvc.Namespace).Delete(context.TODO(), pvc.Name, metav1.DeleteOptions{})
if err != nil {
return fmt.Errorf("failed to delete pvc: %w", err)
}
err = c.CoreV1().PersistentVolumes().Delete(context.TODO(), pv.Name, metav1.DeleteOptions{})
if err != nil {
return fmt.Errorf("failed to delete pv: %w", err)
}
cmd = fmt.Sprintf("rbd rm %s %s", rbdImageName, rbdOptions(defaultRBDPool))
_, _, err = execCommandInToolBoxPod(f, cmd, rookNamespace)
return err
}
func validateRBDStaticMigrationPVC(f *framework.Framework, appPath, scName string, isBlock bool) error {
opt := make(map[string]string)
var (
rbdImageName = "kubernetes-dynamic-pvc-e0b45b52-7e09-47d3-8f1b-806995fa4412"
pvName = "pv-name"
pvcName = "pvc-name"
namespace = f.UniqueName
sc = scName
provisionerAnnKey = "pv.kubernetes.io/provisioned-by"
provisionerAnnValue = "rbd.csi.ceph.com"
)
c := f.ClientSet
PVAnnMap := make(map[string]string)
PVAnnMap[provisionerAnnKey] = provisionerAnnValue
mons, err := getMons(rookNamespace, c)
if err != nil {
return fmt.Errorf("failed to get mons: %w", err)
}
mon := strings.Join(mons, ",")
size := staticPVSize
// create rbd image
cmd := fmt.Sprintf(
"rbd create %s --size=%s --image-feature=layering %s",
rbdImageName,
staticPVSize,
rbdOptions(defaultRBDPool))
_, stdErr, err := execCommandInToolBoxPod(f, cmd, rookNamespace)
if err != nil {
return err
}
if stdErr != "" {
return fmt.Errorf("failed to create rbd image %s", stdErr)
}
opt["migration"] = "true"
opt["clusterID"] = getMonsHash(mon)
opt["imageFeatures"] = staticPVImageFeature
opt["pool"] = defaultRBDPool
opt["staticVolume"] = strconv.FormatBool(true)
opt["imageName"] = rbdImageName
// Make volumeID similar to the migration volumeID
volID := composeIntreeMigVolID(mon, rbdImageName)
pv := getStaticPV(
pvName,
volID,
size,
rbdNodePluginSecretName,
cephCSINamespace,
sc,
provisionerAnnValue,
isBlock,
opt,
PVAnnMap,
deletePolicy)
_, err = c.CoreV1().PersistentVolumes().Create(context.TODO(), pv, metav1.CreateOptions{})
if err != nil {
return fmt.Errorf("PV Create API error: %w", err)
}
pvc := getStaticPVC(pvcName, pvName, size, namespace, sc, isBlock)
_, err = c.CoreV1().PersistentVolumeClaims(pvc.Namespace).Create(context.TODO(), pvc, metav1.CreateOptions{})
if err != nil {
return fmt.Errorf("PVC Create API error: %w", err)
}
// bind pvc to app
app, err := loadApp(appPath)
if err != nil {
return err
}
app.Namespace = namespace
app.Spec.Volumes[0].PersistentVolumeClaim.ClaimName = pvcName
err = createApp(f.ClientSet, app, deployTimeout)
if err != nil {
return err
}
err = deletePVCAndApp("", f, pvc, app)
if err != nil {
return fmt.Errorf("failed to delete PVC and application: %w", err)
}
return err
}
// nolint:gocyclo,cyclop // reduce complexity
func validateCephFsStaticPV(f *framework.Framework, appPath, scPath string) error {
opt := make(map[string]string)
var (
cephFsVolName = "testSubVol"
groupName = "testGroup"
fsName = "myfs"
pvName = "pv-name"
pvcName = "pvc-name"
namespace = f.UniqueName
// minikube creates default storage class in cluster, we need to set dummy
// storageclass on PV and PVC to avoid storageclass name mismatch
sc = "storage-class"
secretName = "cephfs-static-pv-sc" // #nosec
)
c := f.ClientSet
listOpt := metav1.ListOptions{
LabelSelector: "app=rook-ceph-tools",
}
fsID, e, err := execCommandInPod(f, "ceph fsid", rookNamespace, &listOpt)
if err != nil {
return err
}
if e != "" {
return fmt.Errorf("failed to get fsid from ceph cluster %s", e)
}
// remove new line present in fsID
fsID = strings.Trim(fsID, "\n")
// 4GiB in bytes
size := "4294967296"
// create subvolumegroup, command will work even if group is already present.
cmd := fmt.Sprintf("ceph fs subvolumegroup create %s %s", fsName, groupName)
_, e, err = execCommandInPod(f, cmd, rookNamespace, &listOpt)
if err != nil {
return err
}
if e != "" {
return fmt.Errorf("failed to create subvolumegroup: %s", e)
}
// create subvolume
cmd = fmt.Sprintf("ceph fs subvolume create %s %s %s --size %s", fsName, cephFsVolName, groupName, size)
_, e, err = execCommandInPod(f, cmd, rookNamespace, &listOpt)
if err != nil {
return err
}
if e != "" {
return fmt.Errorf("failed to create subvolume: %s", e)
}
// get rootpath
cmd = fmt.Sprintf("ceph fs subvolume getpath %s %s %s", fsName, cephFsVolName, groupName)
rootPath, e, err := execCommandInPod(f, cmd, rookNamespace, &listOpt)
if err != nil {
return err
}
if e != "" {
return fmt.Errorf("failed to get rootpath %s", e)
}
// remove new line present in rootPath
rootPath = strings.Trim(rootPath, "\n")
// create secret
secret, err := getSecret(scPath)
if err != nil {
return err
}
adminKey, e, err := execCommandInPod(f, "ceph auth get-key client.admin", rookNamespace, &listOpt)
if err != nil {
return err
}
if e != "" {
return fmt.Errorf("failed to get adminKey %s", e)
}
secret.StringData["userID"] = adminUser
secret.StringData["userKey"] = adminKey
secret.Name = secretName
secret.Namespace = cephCSINamespace
_, err = c.CoreV1().Secrets(cephCSINamespace).Create(context.TODO(), &secret, metav1.CreateOptions{})
if err != nil {
return fmt.Errorf("failed to create secret: %w", err)
}
opt["clusterID"] = fsID
opt["fsName"] = fsName
opt["staticVolume"] = strconv.FormatBool(true)
opt["rootPath"] = rootPath
pv := getStaticPV(
pvName,
pvName,
staticPVSize,
secretName,
cephCSINamespace,
sc,
"cephfs.csi.ceph.com",
false,
opt,
nil,
retainPolicy)
_, err = c.CoreV1().PersistentVolumes().Create(context.TODO(), pv, metav1.CreateOptions{})
if err != nil {
return fmt.Errorf("failed to create PV: %w", err)
}
pvc := getStaticPVC(pvcName, pvName, size, namespace, sc, false)
_, err = c.CoreV1().PersistentVolumeClaims(pvc.Namespace).Create(context.TODO(), pvc, metav1.CreateOptions{})
if err != nil {
return fmt.Errorf("failed to create PVC: %w", err)
}
// bind pvc to app
app, err := loadApp(appPath)
if err != nil {
return fmt.Errorf("failed to load app: %w", err)
}
app.Namespace = namespace
app.Spec.Volumes[0].PersistentVolumeClaim.ClaimName = pvcName
err = createApp(f.ClientSet, app, deployTimeout)
if err != nil {
return fmt.Errorf("failed to create pod: %w", err)
}
err = deletePod(app.Name, namespace, f.ClientSet, deployTimeout)
if err != nil {
return fmt.Errorf("failed to delete pod: %w", err)
}
err = c.CoreV1().PersistentVolumeClaims(pvc.Namespace).Delete(context.TODO(), pvc.Name, metav1.DeleteOptions{})
if err != nil {
return fmt.Errorf("failed to delete pvc: %w", err)
}
err = c.CoreV1().PersistentVolumes().Delete(context.TODO(), pv.Name, metav1.DeleteOptions{})
if err != nil {
return fmt.Errorf("failed to delete pv: %w", err)
}
err = c.CoreV1().Secrets(cephCSINamespace).Delete(context.TODO(), secret.Name, metav1.DeleteOptions{})
if err != nil {
return fmt.Errorf("failed to delete secret: %w", err)
}
// delete subvolume
cmd = fmt.Sprintf("ceph fs subvolume rm %s %s %s", fsName, cephFsVolName, groupName)
_, e, err = execCommandInPod(f, cmd, rookNamespace, &listOpt)
if err != nil {
return err
}
if e != "" {
return fmt.Errorf("failed to remove sub-volume %s", e)
}
// delete subvolume group
cmd = fmt.Sprintf("ceph fs subvolumegroup rm %s %s", fsName, groupName)
_, e, err = execCommandInPod(f, cmd, rookNamespace, &listOpt)
if err != nil {
return err
}
if e != "" {
return fmt.Errorf("failed to remove subvolume group %s", e)
}
return nil
}
func validateRBDStaticResize(
f *framework.Framework,
app *v1.Pod,
appOpt *metav1.ListOptions,
pvc *v1.PersistentVolumeClaim,
rbdImageName string) error {
// resize rbd image
size := staticPVNewSize
cmd := fmt.Sprintf(
"rbd resize %s --size=%s %s",
rbdImageName,
size,
rbdOptions(defaultRBDPool))
_, _, err := execCommandInToolBoxPod(f, cmd, rookNamespace)
if err != nil {
return err
}
err = createApp(f.ClientSet, app, deployTimeout)
if err != nil {
return err
}
// check size for the filesystem type PVC
if *pvc.Spec.VolumeMode == v1.PersistentVolumeFilesystem {
err = checkDirSize(app, f, appOpt, size)
if err != nil {
return err
}
}
return deletePod(app.Name, app.Namespace, f.ClientSet, deployTimeout)
}