mirror of
https://github.com/ceph/ceph-csi.git
synced 2024-12-18 02:50:30 +00:00
e2e: add tests using different accessModes and volumeModes for rbd-nbd
Add tests for RWX and ROX accessModes for Block and FileSystem Mode PVCs. Fixes: #2262 Signed-off-by: Prasanna Kumar Kalever <prasanna.kalever@redhat.com>
This commit is contained in:
Prasanna Kumar Kalever
mergify[bot]
parent
e5435c5bdc
commit
0fd4db92bb
334
e2e/rbd.go
334
e2e/rbd.go
@ -33,6 +33,7 @@ var (
|
||||
rbdDirPath = "../deploy/rbd/kubernetes/"
|
||||
examplePath = "../examples/"
|
||||
rbdExamplePath = examplePath + "/rbd/"
|
||||
e2eTemplatesPath = "../e2e/templates/"
|
||||
rbdDeploymentName = "csi-rbdplugin-provisioner"
|
||||
rbdDaemonsetName = "csi-rbdplugin"
|
||||
defaultRBDPool = "replicapool"
|
||||
@ -59,6 +60,8 @@ var (
|
||||
appBlockSmartClonePath = rbdExamplePath + "block-pod-clone.yaml"
|
||||
appEphemeralPath = rbdExamplePath + "pod-ephemeral.yaml"
|
||||
snapshotPath = rbdExamplePath + "snapshot.yaml"
|
||||
deployFSAppPath = e2eTemplatesPath + "rbd-fs-deployment.yaml"
|
||||
deployBlockAppPath = e2eTemplatesPath + "rbd-block-deployment.yaml"
|
||||
defaultCloneCount = 10
|
||||
|
||||
nbdMapOptions = "nbd:debug-rbd=20"
|
||||
@ -576,6 +579,337 @@ var _ = Describe("RBD", func() {
|
||||
}
|
||||
})
|
||||
|
||||
// NOTE: RWX is restricted for FileSystem VolumeMode at ceph-csi,
|
||||
// see pull#261 for more details.
|
||||
By("Create RWX+Block Mode PVC and bind to multiple pods via deployment using rbd-nbd mounter", func() {
|
||||
err := deleteResource(rbdExamplePath + "storageclass.yaml")
|
||||
if err != nil {
|
||||
e2elog.Failf("failed to delete storageclass: %v", err)
|
||||
}
|
||||
// Storage class with rbd-nbd mounter
|
||||
err = createRBDStorageClass(
|
||||
f.ClientSet,
|
||||
f,
|
||||
defaultSCName,
|
||||
nil,
|
||||
map[string]string{
|
||||
"mounter": "rbd-nbd",
|
||||
"mapOptions": nbdMapOptions,
|
||||
"cephLogStrategy": e2eDefaultCephLogStrategy,
|
||||
},
|
||||
deletePolicy)
|
||||
if err != nil {
|
||||
e2elog.Failf("failed to create storageclass: %v", err)
|
||||
}
|
||||
pvc, err := loadPVC(rawPvcPath)
|
||||
if err != nil {
|
||||
e2elog.Failf("failed to load PVC: %v", err)
|
||||
}
|
||||
pvc.Namespace = f.UniqueName
|
||||
pvc.Spec.AccessModes = []v1.PersistentVolumeAccessMode{v1.ReadWriteMany}
|
||||
|
||||
app, err := loadAppDeployment(deployBlockAppPath)
|
||||
if err != nil {
|
||||
e2elog.Failf("failed to load application deployment: %v", err)
|
||||
}
|
||||
app.Namespace = f.UniqueName
|
||||
|
||||
err = createPVCAndDeploymentApp(f, "", pvc, app, deployTimeout)
|
||||
if err != nil {
|
||||
e2elog.Failf("failed to create PVC and application: %v", err)
|
||||
}
|
||||
|
||||
err = waitForDeploymentComplete(f.ClientSet, app.Name, app.Namespace, deployTimeout)
|
||||
if err != nil {
|
||||
e2elog.Failf("timeout waiting for deployment to be in running state: %v", err)
|
||||
}
|
||||
|
||||
devPath := app.Spec.Template.Spec.Containers[0].VolumeDevices[0].DevicePath
|
||||
cmd := fmt.Sprintf("dd if=/dev/zero of=%s bs=1M count=10", devPath)
|
||||
|
||||
opt := metav1.ListOptions{
|
||||
LabelSelector: fmt.Sprintf("app=%s", app.Name),
|
||||
}
|
||||
podList, err := f.PodClientNS(app.Namespace).List(context.TODO(), opt)
|
||||
if err != nil {
|
||||
e2elog.Failf("get pod list failed: %v", err)
|
||||
}
|
||||
if len(podList.Items) != int(*app.Spec.Replicas) {
|
||||
e2elog.Failf("podlist contains %d items, expected %d items", len(podList.Items), *app.Spec.Replicas)
|
||||
}
|
||||
for _, pod := range podList.Items {
|
||||
_, _, err = execCommandInPodWithName(f, cmd, pod.Name, pod.Spec.Containers[0].Name, app.Namespace)
|
||||
if err != nil {
|
||||
e2elog.Failf("command %q failed: %v", cmd, err)
|
||||
}
|
||||
}
|
||||
|
||||
err = deletePVCAndDeploymentApp(f, "", pvc, app)
|
||||
if err != nil {
|
||||
e2elog.Failf("failed to delete PVC and application: %v", err)
|
||||
}
|
||||
// validate created backend rbd images
|
||||
validateRBDImageCount(f, 0, defaultRBDPool)
|
||||
// validate images in trash
|
||||
err = waitToRemoveImagesFromTrash(f, defaultRBDPool, deployTimeout)
|
||||
if err != nil {
|
||||
e2elog.Failf("failed to validate rbd images in pool %s trash: %v", rbdOptions(defaultRBDPool), err)
|
||||
}
|
||||
err = deleteResource(rbdExamplePath + "storageclass.yaml")
|
||||
if err != nil {
|
||||
e2elog.Failf("failed to delete storageclass: %v", err)
|
||||
}
|
||||
err = createRBDStorageClass(f.ClientSet, f, defaultSCName, nil, nil, deletePolicy)
|
||||
if err != nil {
|
||||
e2elog.Failf("failed to create storageclass: %v", err)
|
||||
}
|
||||
})
|
||||
|
||||
By("Create ROX+FS Mode PVC and bind to multiple pods via deployment using rbd-nbd mounter", func() {
|
||||
err := deleteResource(rbdExamplePath + "storageclass.yaml")
|
||||
if err != nil {
|
||||
e2elog.Failf("failed to delete storageclass: %v", err)
|
||||
}
|
||||
// Storage class with rbd-nbd mounter
|
||||
err = createRBDStorageClass(
|
||||
f.ClientSet,
|
||||
f,
|
||||
defaultSCName,
|
||||
nil,
|
||||
map[string]string{
|
||||
"mounter": "rbd-nbd",
|
||||
"mapOptions": nbdMapOptions,
|
||||
"cephLogStrategy": e2eDefaultCephLogStrategy,
|
||||
},
|
||||
deletePolicy)
|
||||
if err != nil {
|
||||
e2elog.Failf("failed to create storageclass: %v", err)
|
||||
}
|
||||
|
||||
// create PVC and bind it to an app
|
||||
pvc, err := loadPVC(pvcPath)
|
||||
if err != nil {
|
||||
e2elog.Failf("failed to load PVC: %v", err)
|
||||
}
|
||||
pvc.Namespace = f.UniqueName
|
||||
app, err := loadApp(appPath)
|
||||
if err != nil {
|
||||
e2elog.Failf("failed to load application: %v", err)
|
||||
}
|
||||
app.Namespace = f.UniqueName
|
||||
err = createPVCAndApp("", f, pvc, app, deployTimeout)
|
||||
if err != nil {
|
||||
e2elog.Failf("failed to create PVC and application: %v", err)
|
||||
}
|
||||
// validate created backend rbd images
|
||||
validateRBDImageCount(f, 1, defaultRBDPool)
|
||||
err = deletePod(app.Name, app.Namespace, f.ClientSet, deployTimeout)
|
||||
if err != nil {
|
||||
e2elog.Failf("failed to delete application: %v", err)
|
||||
}
|
||||
|
||||
// create clone PVC as ROX
|
||||
pvcClone, err := loadPVC(pvcSmartClonePath)
|
||||
if err != nil {
|
||||
e2elog.Failf("failed to load PVC: %v", err)
|
||||
}
|
||||
pvcClone.Spec.DataSource.Name = pvc.Name
|
||||
pvcClone.Namespace = f.UniqueName
|
||||
pvcClone.Spec.AccessModes = []v1.PersistentVolumeAccessMode{v1.ReadOnlyMany}
|
||||
appClone, err := loadAppDeployment(deployFSAppPath)
|
||||
if err != nil {
|
||||
e2elog.Failf("failed to load application deployment: %v", err)
|
||||
}
|
||||
appClone.Namespace = f.UniqueName
|
||||
appClone.Spec.Template.Spec.Volumes[0].PersistentVolumeClaim.ClaimName = pvcClone.Name
|
||||
appClone.Spec.Template.Spec.Volumes[0].PersistentVolumeClaim.ReadOnly = true
|
||||
err = createPVCAndDeploymentApp(f, "", pvcClone, appClone, deployTimeout)
|
||||
if err != nil {
|
||||
e2elog.Failf("failed to create PVC and application: %v", err)
|
||||
}
|
||||
|
||||
err = waitForDeploymentComplete(f.ClientSet, appClone.Name, appClone.Namespace, deployTimeout)
|
||||
if err != nil {
|
||||
e2elog.Failf("timeout waiting for deployment to be in running state: %v", err)
|
||||
}
|
||||
|
||||
// validate created backend rbd images
|
||||
validateRBDImageCount(f, 3, defaultRBDPool)
|
||||
|
||||
filePath := appClone.Spec.Template.Spec.Containers[0].VolumeMounts[0].MountPath + "/test"
|
||||
cmd := fmt.Sprintf("echo 'Hello World' > %s", filePath)
|
||||
|
||||
opt := metav1.ListOptions{
|
||||
LabelSelector: fmt.Sprintf("app=%s", appClone.Name),
|
||||
}
|
||||
podList, err := f.PodClientNS(appClone.Namespace).List(context.TODO(), opt)
|
||||
if err != nil {
|
||||
e2elog.Failf("get pod list failed: %v", err)
|
||||
}
|
||||
if len(podList.Items) != int(*appClone.Spec.Replicas) {
|
||||
e2elog.Failf("podlist contains %d items, expected %d items", len(podList.Items), *appClone.Spec.Replicas)
|
||||
}
|
||||
for _, pod := range podList.Items {
|
||||
var stdErr string
|
||||
_, stdErr, err = execCommandInPodWithName(f, cmd, pod.Name, pod.Spec.Containers[0].Name, appClone.Namespace)
|
||||
if err != nil {
|
||||
e2elog.Logf("command %q failed: %v", cmd, err)
|
||||
}
|
||||
readOnlyErr := fmt.Sprintf("cannot create %s: Read-only file system", filePath)
|
||||
if !strings.Contains(stdErr, readOnlyErr) {
|
||||
e2elog.Failf(stdErr)
|
||||
}
|
||||
}
|
||||
|
||||
err = deletePVCAndDeploymentApp(f, "", pvcClone, appClone)
|
||||
if err != nil {
|
||||
e2elog.Failf("failed to delete PVC and application: %v", err)
|
||||
}
|
||||
// delete parent pvc
|
||||
err = deletePVCAndValidatePV(f.ClientSet, pvc, deployTimeout)
|
||||
if err != nil {
|
||||
e2elog.Failf("failed to delete PVC: %v", err)
|
||||
}
|
||||
// validate created backend rbd images
|
||||
validateRBDImageCount(f, 0, defaultRBDPool)
|
||||
// validate images in trash
|
||||
err = waitToRemoveImagesFromTrash(f, defaultRBDPool, deployTimeout)
|
||||
if err != nil {
|
||||
e2elog.Failf("failed to validate rbd images in pool %s trash: %v", rbdOptions(defaultRBDPool), err)
|
||||
}
|
||||
err = deleteResource(rbdExamplePath + "storageclass.yaml")
|
||||
if err != nil {
|
||||
e2elog.Failf("failed to delete storageclass: %v", err)
|
||||
}
|
||||
err = createRBDStorageClass(f.ClientSet, f, defaultSCName, nil, nil, deletePolicy)
|
||||
if err != nil {
|
||||
e2elog.Failf("failed to create storageclass: %v", err)
|
||||
}
|
||||
})
|
||||
|
||||
By("Create ROX+Block Mode PVC and bind to multiple pods via deployment using rbd-nbd mounter", func() {
|
||||
err := deleteResource(rbdExamplePath + "storageclass.yaml")
|
||||
if err != nil {
|
||||
e2elog.Failf("failed to delete storageclass: %v", err)
|
||||
}
|
||||
// Storage class with rbd-nbd mounter
|
||||
err = createRBDStorageClass(
|
||||
f.ClientSet,
|
||||
f,
|
||||
defaultSCName,
|
||||
nil,
|
||||
map[string]string{
|
||||
"mounter": "rbd-nbd",
|
||||
"mapOptions": nbdMapOptions,
|
||||
"cephLogStrategy": e2eDefaultCephLogStrategy,
|
||||
},
|
||||
deletePolicy)
|
||||
if err != nil {
|
||||
e2elog.Failf("failed to create storageclass: %v", err)
|
||||
}
|
||||
|
||||
// create PVC and bind it to an app
|
||||
pvc, err := loadPVC(rawPvcPath)
|
||||
if err != nil {
|
||||
e2elog.Failf("failed to load PVC: %v", err)
|
||||
}
|
||||
pvc.Namespace = f.UniqueName
|
||||
app, err := loadApp(rawAppPath)
|
||||
if err != nil {
|
||||
e2elog.Failf("failed to load application: %v", err)
|
||||
}
|
||||
app.Namespace = f.UniqueName
|
||||
err = createPVCAndApp("", f, pvc, app, deployTimeout)
|
||||
if err != nil {
|
||||
e2elog.Failf("failed to create PVC and application: %v", err)
|
||||
}
|
||||
// validate created backend rbd images
|
||||
validateRBDImageCount(f, 1, defaultRBDPool)
|
||||
err = deletePod(app.Name, app.Namespace, f.ClientSet, deployTimeout)
|
||||
if err != nil {
|
||||
e2elog.Failf("failed to delete application: %v", err)
|
||||
}
|
||||
|
||||
// create clone PVC as ROX
|
||||
pvcClone, err := loadPVC(pvcBlockSmartClonePath)
|
||||
if err != nil {
|
||||
e2elog.Failf("failed to load PVC: %v", err)
|
||||
}
|
||||
pvcClone.Spec.DataSource.Name = pvc.Name
|
||||
pvcClone.Namespace = f.UniqueName
|
||||
pvcClone.Spec.AccessModes = []v1.PersistentVolumeAccessMode{v1.ReadOnlyMany}
|
||||
volumeMode := v1.PersistentVolumeBlock
|
||||
pvcClone.Spec.VolumeMode = &volumeMode
|
||||
appClone, err := loadAppDeployment(deployBlockAppPath)
|
||||
if err != nil {
|
||||
e2elog.Failf("failed to load application deployment: %v", err)
|
||||
}
|
||||
appClone.Namespace = f.UniqueName
|
||||
appClone.Spec.Template.Spec.Volumes[0].PersistentVolumeClaim.ClaimName = pvcClone.Name
|
||||
appClone.Spec.Template.Spec.Volumes[0].PersistentVolumeClaim.ReadOnly = true
|
||||
err = createPVCAndDeploymentApp(f, "", pvcClone, appClone, deployTimeout)
|
||||
if err != nil {
|
||||
e2elog.Failf("failed to create PVC and application: %v", err)
|
||||
}
|
||||
|
||||
err = waitForDeploymentComplete(f.ClientSet, appClone.Name, appClone.Namespace, deployTimeout)
|
||||
if err != nil {
|
||||
e2elog.Failf("timeout waiting for deployment to be in running state: %v", err)
|
||||
}
|
||||
|
||||
// validate created backend rbd images
|
||||
validateRBDImageCount(f, 3, defaultRBDPool)
|
||||
|
||||
devPath := appClone.Spec.Template.Spec.Containers[0].VolumeDevices[0].DevicePath
|
||||
cmd := fmt.Sprintf("dd if=/dev/zero of=%s bs=1M count=10", devPath)
|
||||
|
||||
opt := metav1.ListOptions{
|
||||
LabelSelector: fmt.Sprintf("app=%s", appClone.Name),
|
||||
}
|
||||
podList, err := f.PodClientNS(appClone.Namespace).List(context.TODO(), opt)
|
||||
if err != nil {
|
||||
e2elog.Failf("get pod list failed: %v", err)
|
||||
}
|
||||
if len(podList.Items) != int(*appClone.Spec.Replicas) {
|
||||
e2elog.Failf("podlist contains %d items, expected %d items", len(podList.Items), *appClone.Spec.Replicas)
|
||||
}
|
||||
for _, pod := range podList.Items {
|
||||
var stdErr string
|
||||
_, stdErr, err = execCommandInPodWithName(f, cmd, pod.Name, pod.Spec.Containers[0].Name, appClone.Namespace)
|
||||
if err != nil {
|
||||
e2elog.Logf("command %q failed: %v", cmd, err)
|
||||
}
|
||||
readOnlyErr := fmt.Sprintf("'%s': Operation not permitted", devPath)
|
||||
if !strings.Contains(stdErr, readOnlyErr) {
|
||||
e2elog.Failf(stdErr)
|
||||
}
|
||||
}
|
||||
err = deletePVCAndDeploymentApp(f, "", pvcClone, appClone)
|
||||
if err != nil {
|
||||
e2elog.Failf("failed to delete PVC and application: %v", err)
|
||||
}
|
||||
// delete parent pvc
|
||||
err = deletePVCAndValidatePV(f.ClientSet, pvc, deployTimeout)
|
||||
if err != nil {
|
||||
e2elog.Failf("failed to delete PVC: %v", err)
|
||||
}
|
||||
// validate created backend rbd images
|
||||
validateRBDImageCount(f, 0, defaultRBDPool)
|
||||
// validate images in trash
|
||||
err = waitToRemoveImagesFromTrash(f, defaultRBDPool, deployTimeout)
|
||||
if err != nil {
|
||||
e2elog.Failf("failed to validate rbd images in pool %s trash: %v", rbdOptions(defaultRBDPool), err)
|
||||
}
|
||||
err = deleteResource(rbdExamplePath + "storageclass.yaml")
|
||||
if err != nil {
|
||||
e2elog.Failf("failed to delete storageclass: %v", err)
|
||||
}
|
||||
err = createRBDStorageClass(f.ClientSet, f, defaultSCName, nil, nil, deletePolicy)
|
||||
if err != nil {
|
||||
e2elog.Failf("failed to create storageclass: %v", err)
|
||||
}
|
||||
})
|
||||
|
||||
By("perform IO on rbd-nbd volume after nodeplugin restart", func() {
|
||||
err := deleteResource(rbdExamplePath + "storageclass.yaml")
|
||||
if err != nil {
|
||||
|
||||
Loading…
Reference in New Issue
Block a user