mirror of
https://github.com/ceph/ceph-csi.git
synced 2024-11-10 00:10:20 +00:00
e2e: add tests using different accessModes and volumeModes for rbd-nbd
Add tests for RWX and ROX accessModes for Block and FileSystem Mode PVCs. Fixes: #2262 Signed-off-by: Prasanna Kumar Kalever <prasanna.kalever@redhat.com>
This commit is contained in:
Prasanna Kumar Kalever
mergify[bot]
parent
e5435c5bdc
commit
0fd4db92bb
334
e2e/rbd.go
334
e2e/rbd.go
@ -33,6 +33,7 @@ var (
|
|||||||
rbdDirPath = "../deploy/rbd/kubernetes/"
|
rbdDirPath = "../deploy/rbd/kubernetes/"
|
||||||
examplePath = "../examples/"
|
examplePath = "../examples/"
|
||||||
rbdExamplePath = examplePath + "/rbd/"
|
rbdExamplePath = examplePath + "/rbd/"
|
||||||
|
e2eTemplatesPath = "../e2e/templates/"
|
||||||
rbdDeploymentName = "csi-rbdplugin-provisioner"
|
rbdDeploymentName = "csi-rbdplugin-provisioner"
|
||||||
rbdDaemonsetName = "csi-rbdplugin"
|
rbdDaemonsetName = "csi-rbdplugin"
|
||||||
defaultRBDPool = "replicapool"
|
defaultRBDPool = "replicapool"
|
||||||
@ -59,6 +60,8 @@ var (
|
|||||||
appBlockSmartClonePath = rbdExamplePath + "block-pod-clone.yaml"
|
appBlockSmartClonePath = rbdExamplePath + "block-pod-clone.yaml"
|
||||||
appEphemeralPath = rbdExamplePath + "pod-ephemeral.yaml"
|
appEphemeralPath = rbdExamplePath + "pod-ephemeral.yaml"
|
||||||
snapshotPath = rbdExamplePath + "snapshot.yaml"
|
snapshotPath = rbdExamplePath + "snapshot.yaml"
|
||||||
|
deployFSAppPath = e2eTemplatesPath + "rbd-fs-deployment.yaml"
|
||||||
|
deployBlockAppPath = e2eTemplatesPath + "rbd-block-deployment.yaml"
|
||||||
defaultCloneCount = 10
|
defaultCloneCount = 10
|
||||||
|
|
||||||
nbdMapOptions = "nbd:debug-rbd=20"
|
nbdMapOptions = "nbd:debug-rbd=20"
|
||||||
@ -576,6 +579,337 @@ var _ = Describe("RBD", func() {
|
|||||||
}
|
}
|
||||||
})
|
})
|
||||||
|
|
||||||
|
// NOTE: RWX is restricted for FileSystem VolumeMode at ceph-csi,
|
||||||
|
// see pull#261 for more details.
|
||||||
|
By("Create RWX+Block Mode PVC and bind to multiple pods via deployment using rbd-nbd mounter", func() {
|
||||||
|
err := deleteResource(rbdExamplePath + "storageclass.yaml")
|
||||||
|
if err != nil {
|
||||||
|
e2elog.Failf("failed to delete storageclass: %v", err)
|
||||||
|
}
|
||||||
|
// Storage class with rbd-nbd mounter
|
||||||
|
err = createRBDStorageClass(
|
||||||
|
f.ClientSet,
|
||||||
|
f,
|
||||||
|
defaultSCName,
|
||||||
|
nil,
|
||||||
|
map[string]string{
|
||||||
|
"mounter": "rbd-nbd",
|
||||||
|
"mapOptions": nbdMapOptions,
|
||||||
|
"cephLogStrategy": e2eDefaultCephLogStrategy,
|
||||||
|
},
|
||||||
|
deletePolicy)
|
||||||
|
if err != nil {
|
||||||
|
e2elog.Failf("failed to create storageclass: %v", err)
|
||||||
|
}
|
||||||
|
pvc, err := loadPVC(rawPvcPath)
|
||||||
|
if err != nil {
|
||||||
|
e2elog.Failf("failed to load PVC: %v", err)
|
||||||
|
}
|
||||||
|
pvc.Namespace = f.UniqueName
|
||||||
|
pvc.Spec.AccessModes = []v1.PersistentVolumeAccessMode{v1.ReadWriteMany}
|
||||||
|
|
||||||
|
app, err := loadAppDeployment(deployBlockAppPath)
|
||||||
|
if err != nil {
|
||||||
|
e2elog.Failf("failed to load application deployment: %v", err)
|
||||||
|
}
|
||||||
|
app.Namespace = f.UniqueName
|
||||||
|
|
||||||
|
err = createPVCAndDeploymentApp(f, "", pvc, app, deployTimeout)
|
||||||
|
if err != nil {
|
||||||
|
e2elog.Failf("failed to create PVC and application: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
err = waitForDeploymentComplete(f.ClientSet, app.Name, app.Namespace, deployTimeout)
|
||||||
|
if err != nil {
|
||||||
|
e2elog.Failf("timeout waiting for deployment to be in running state: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
devPath := app.Spec.Template.Spec.Containers[0].VolumeDevices[0].DevicePath
|
||||||
|
cmd := fmt.Sprintf("dd if=/dev/zero of=%s bs=1M count=10", devPath)
|
||||||
|
|
||||||
|
opt := metav1.ListOptions{
|
||||||
|
LabelSelector: fmt.Sprintf("app=%s", app.Name),
|
||||||
|
}
|
||||||
|
podList, err := f.PodClientNS(app.Namespace).List(context.TODO(), opt)
|
||||||
|
if err != nil {
|
||||||
|
e2elog.Failf("get pod list failed: %v", err)
|
||||||
|
}
|
||||||
|
if len(podList.Items) != int(*app.Spec.Replicas) {
|
||||||
|
e2elog.Failf("podlist contains %d items, expected %d items", len(podList.Items), *app.Spec.Replicas)
|
||||||
|
}
|
||||||
|
for _, pod := range podList.Items {
|
||||||
|
_, _, err = execCommandInPodWithName(f, cmd, pod.Name, pod.Spec.Containers[0].Name, app.Namespace)
|
||||||
|
if err != nil {
|
||||||
|
e2elog.Failf("command %q failed: %v", cmd, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
err = deletePVCAndDeploymentApp(f, "", pvc, app)
|
||||||
|
if err != nil {
|
||||||
|
e2elog.Failf("failed to delete PVC and application: %v", err)
|
||||||
|
}
|
||||||
|
// validate created backend rbd images
|
||||||
|
validateRBDImageCount(f, 0, defaultRBDPool)
|
||||||
|
// validate images in trash
|
||||||
|
err = waitToRemoveImagesFromTrash(f, defaultRBDPool, deployTimeout)
|
||||||
|
if err != nil {
|
||||||
|
e2elog.Failf("failed to validate rbd images in pool %s trash: %v", rbdOptions(defaultRBDPool), err)
|
||||||
|
}
|
||||||
|
err = deleteResource(rbdExamplePath + "storageclass.yaml")
|
||||||
|
if err != nil {
|
||||||
|
e2elog.Failf("failed to delete storageclass: %v", err)
|
||||||
|
}
|
||||||
|
err = createRBDStorageClass(f.ClientSet, f, defaultSCName, nil, nil, deletePolicy)
|
||||||
|
if err != nil {
|
||||||
|
e2elog.Failf("failed to create storageclass: %v", err)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
By("Create ROX+FS Mode PVC and bind to multiple pods via deployment using rbd-nbd mounter", func() {
|
||||||
|
err := deleteResource(rbdExamplePath + "storageclass.yaml")
|
||||||
|
if err != nil {
|
||||||
|
e2elog.Failf("failed to delete storageclass: %v", err)
|
||||||
|
}
|
||||||
|
// Storage class with rbd-nbd mounter
|
||||||
|
err = createRBDStorageClass(
|
||||||
|
f.ClientSet,
|
||||||
|
f,
|
||||||
|
defaultSCName,
|
||||||
|
nil,
|
||||||
|
map[string]string{
|
||||||
|
"mounter": "rbd-nbd",
|
||||||
|
"mapOptions": nbdMapOptions,
|
||||||
|
"cephLogStrategy": e2eDefaultCephLogStrategy,
|
||||||
|
},
|
||||||
|
deletePolicy)
|
||||||
|
if err != nil {
|
||||||
|
e2elog.Failf("failed to create storageclass: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// create PVC and bind it to an app
|
||||||
|
pvc, err := loadPVC(pvcPath)
|
||||||
|
if err != nil {
|
||||||
|
e2elog.Failf("failed to load PVC: %v", err)
|
||||||
|
}
|
||||||
|
pvc.Namespace = f.UniqueName
|
||||||
|
app, err := loadApp(appPath)
|
||||||
|
if err != nil {
|
||||||
|
e2elog.Failf("failed to load application: %v", err)
|
||||||
|
}
|
||||||
|
app.Namespace = f.UniqueName
|
||||||
|
err = createPVCAndApp("", f, pvc, app, deployTimeout)
|
||||||
|
if err != nil {
|
||||||
|
e2elog.Failf("failed to create PVC and application: %v", err)
|
||||||
|
}
|
||||||
|
// validate created backend rbd images
|
||||||
|
validateRBDImageCount(f, 1, defaultRBDPool)
|
||||||
|
err = deletePod(app.Name, app.Namespace, f.ClientSet, deployTimeout)
|
||||||
|
if err != nil {
|
||||||
|
e2elog.Failf("failed to delete application: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// create clone PVC as ROX
|
||||||
|
pvcClone, err := loadPVC(pvcSmartClonePath)
|
||||||
|
if err != nil {
|
||||||
|
e2elog.Failf("failed to load PVC: %v", err)
|
||||||
|
}
|
||||||
|
pvcClone.Spec.DataSource.Name = pvc.Name
|
||||||
|
pvcClone.Namespace = f.UniqueName
|
||||||
|
pvcClone.Spec.AccessModes = []v1.PersistentVolumeAccessMode{v1.ReadOnlyMany}
|
||||||
|
appClone, err := loadAppDeployment(deployFSAppPath)
|
||||||
|
if err != nil {
|
||||||
|
e2elog.Failf("failed to load application deployment: %v", err)
|
||||||
|
}
|
||||||
|
appClone.Namespace = f.UniqueName
|
||||||
|
appClone.Spec.Template.Spec.Volumes[0].PersistentVolumeClaim.ClaimName = pvcClone.Name
|
||||||
|
appClone.Spec.Template.Spec.Volumes[0].PersistentVolumeClaim.ReadOnly = true
|
||||||
|
err = createPVCAndDeploymentApp(f, "", pvcClone, appClone, deployTimeout)
|
||||||
|
if err != nil {
|
||||||
|
e2elog.Failf("failed to create PVC and application: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
err = waitForDeploymentComplete(f.ClientSet, appClone.Name, appClone.Namespace, deployTimeout)
|
||||||
|
if err != nil {
|
||||||
|
e2elog.Failf("timeout waiting for deployment to be in running state: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// validate created backend rbd images
|
||||||
|
validateRBDImageCount(f, 3, defaultRBDPool)
|
||||||
|
|
||||||
|
filePath := appClone.Spec.Template.Spec.Containers[0].VolumeMounts[0].MountPath + "/test"
|
||||||
|
cmd := fmt.Sprintf("echo 'Hello World' > %s", filePath)
|
||||||
|
|
||||||
|
opt := metav1.ListOptions{
|
||||||
|
LabelSelector: fmt.Sprintf("app=%s", appClone.Name),
|
||||||
|
}
|
||||||
|
podList, err := f.PodClientNS(appClone.Namespace).List(context.TODO(), opt)
|
||||||
|
if err != nil {
|
||||||
|
e2elog.Failf("get pod list failed: %v", err)
|
||||||
|
}
|
||||||
|
if len(podList.Items) != int(*appClone.Spec.Replicas) {
|
||||||
|
e2elog.Failf("podlist contains %d items, expected %d items", len(podList.Items), *appClone.Spec.Replicas)
|
||||||
|
}
|
||||||
|
for _, pod := range podList.Items {
|
||||||
|
var stdErr string
|
||||||
|
_, stdErr, err = execCommandInPodWithName(f, cmd, pod.Name, pod.Spec.Containers[0].Name, appClone.Namespace)
|
||||||
|
if err != nil {
|
||||||
|
e2elog.Logf("command %q failed: %v", cmd, err)
|
||||||
|
}
|
||||||
|
readOnlyErr := fmt.Sprintf("cannot create %s: Read-only file system", filePath)
|
||||||
|
if !strings.Contains(stdErr, readOnlyErr) {
|
||||||
|
e2elog.Failf(stdErr)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
err = deletePVCAndDeploymentApp(f, "", pvcClone, appClone)
|
||||||
|
if err != nil {
|
||||||
|
e2elog.Failf("failed to delete PVC and application: %v", err)
|
||||||
|
}
|
||||||
|
// delete parent pvc
|
||||||
|
err = deletePVCAndValidatePV(f.ClientSet, pvc, deployTimeout)
|
||||||
|
if err != nil {
|
||||||
|
e2elog.Failf("failed to delete PVC: %v", err)
|
||||||
|
}
|
||||||
|
// validate created backend rbd images
|
||||||
|
validateRBDImageCount(f, 0, defaultRBDPool)
|
||||||
|
// validate images in trash
|
||||||
|
err = waitToRemoveImagesFromTrash(f, defaultRBDPool, deployTimeout)
|
||||||
|
if err != nil {
|
||||||
|
e2elog.Failf("failed to validate rbd images in pool %s trash: %v", rbdOptions(defaultRBDPool), err)
|
||||||
|
}
|
||||||
|
err = deleteResource(rbdExamplePath + "storageclass.yaml")
|
||||||
|
if err != nil {
|
||||||
|
e2elog.Failf("failed to delete storageclass: %v", err)
|
||||||
|
}
|
||||||
|
err = createRBDStorageClass(f.ClientSet, f, defaultSCName, nil, nil, deletePolicy)
|
||||||
|
if err != nil {
|
||||||
|
e2elog.Failf("failed to create storageclass: %v", err)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
By("Create ROX+Block Mode PVC and bind to multiple pods via deployment using rbd-nbd mounter", func() {
|
||||||
|
err := deleteResource(rbdExamplePath + "storageclass.yaml")
|
||||||
|
if err != nil {
|
||||||
|
e2elog.Failf("failed to delete storageclass: %v", err)
|
||||||
|
}
|
||||||
|
// Storage class with rbd-nbd mounter
|
||||||
|
err = createRBDStorageClass(
|
||||||
|
f.ClientSet,
|
||||||
|
f,
|
||||||
|
defaultSCName,
|
||||||
|
nil,
|
||||||
|
map[string]string{
|
||||||
|
"mounter": "rbd-nbd",
|
||||||
|
"mapOptions": nbdMapOptions,
|
||||||
|
"cephLogStrategy": e2eDefaultCephLogStrategy,
|
||||||
|
},
|
||||||
|
deletePolicy)
|
||||||
|
if err != nil {
|
||||||
|
e2elog.Failf("failed to create storageclass: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// create PVC and bind it to an app
|
||||||
|
pvc, err := loadPVC(rawPvcPath)
|
||||||
|
if err != nil {
|
||||||
|
e2elog.Failf("failed to load PVC: %v", err)
|
||||||
|
}
|
||||||
|
pvc.Namespace = f.UniqueName
|
||||||
|
app, err := loadApp(rawAppPath)
|
||||||
|
if err != nil {
|
||||||
|
e2elog.Failf("failed to load application: %v", err)
|
||||||
|
}
|
||||||
|
app.Namespace = f.UniqueName
|
||||||
|
err = createPVCAndApp("", f, pvc, app, deployTimeout)
|
||||||
|
if err != nil {
|
||||||
|
e2elog.Failf("failed to create PVC and application: %v", err)
|
||||||
|
}
|
||||||
|
// validate created backend rbd images
|
||||||
|
validateRBDImageCount(f, 1, defaultRBDPool)
|
||||||
|
err = deletePod(app.Name, app.Namespace, f.ClientSet, deployTimeout)
|
||||||
|
if err != nil {
|
||||||
|
e2elog.Failf("failed to delete application: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// create clone PVC as ROX
|
||||||
|
pvcClone, err := loadPVC(pvcBlockSmartClonePath)
|
||||||
|
if err != nil {
|
||||||
|
e2elog.Failf("failed to load PVC: %v", err)
|
||||||
|
}
|
||||||
|
pvcClone.Spec.DataSource.Name = pvc.Name
|
||||||
|
pvcClone.Namespace = f.UniqueName
|
||||||
|
pvcClone.Spec.AccessModes = []v1.PersistentVolumeAccessMode{v1.ReadOnlyMany}
|
||||||
|
volumeMode := v1.PersistentVolumeBlock
|
||||||
|
pvcClone.Spec.VolumeMode = &volumeMode
|
||||||
|
appClone, err := loadAppDeployment(deployBlockAppPath)
|
||||||
|
if err != nil {
|
||||||
|
e2elog.Failf("failed to load application deployment: %v", err)
|
||||||
|
}
|
||||||
|
appClone.Namespace = f.UniqueName
|
||||||
|
appClone.Spec.Template.Spec.Volumes[0].PersistentVolumeClaim.ClaimName = pvcClone.Name
|
||||||
|
appClone.Spec.Template.Spec.Volumes[0].PersistentVolumeClaim.ReadOnly = true
|
||||||
|
err = createPVCAndDeploymentApp(f, "", pvcClone, appClone, deployTimeout)
|
||||||
|
if err != nil {
|
||||||
|
e2elog.Failf("failed to create PVC and application: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
err = waitForDeploymentComplete(f.ClientSet, appClone.Name, appClone.Namespace, deployTimeout)
|
||||||
|
if err != nil {
|
||||||
|
e2elog.Failf("timeout waiting for deployment to be in running state: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// validate created backend rbd images
|
||||||
|
validateRBDImageCount(f, 3, defaultRBDPool)
|
||||||
|
|
||||||
|
devPath := appClone.Spec.Template.Spec.Containers[0].VolumeDevices[0].DevicePath
|
||||||
|
cmd := fmt.Sprintf("dd if=/dev/zero of=%s bs=1M count=10", devPath)
|
||||||
|
|
||||||
|
opt := metav1.ListOptions{
|
||||||
|
LabelSelector: fmt.Sprintf("app=%s", appClone.Name),
|
||||||
|
}
|
||||||
|
podList, err := f.PodClientNS(appClone.Namespace).List(context.TODO(), opt)
|
||||||
|
if err != nil {
|
||||||
|
e2elog.Failf("get pod list failed: %v", err)
|
||||||
|
}
|
||||||
|
if len(podList.Items) != int(*appClone.Spec.Replicas) {
|
||||||
|
e2elog.Failf("podlist contains %d items, expected %d items", len(podList.Items), *appClone.Spec.Replicas)
|
||||||
|
}
|
||||||
|
for _, pod := range podList.Items {
|
||||||
|
var stdErr string
|
||||||
|
_, stdErr, err = execCommandInPodWithName(f, cmd, pod.Name, pod.Spec.Containers[0].Name, appClone.Namespace)
|
||||||
|
if err != nil {
|
||||||
|
e2elog.Logf("command %q failed: %v", cmd, err)
|
||||||
|
}
|
||||||
|
readOnlyErr := fmt.Sprintf("'%s': Operation not permitted", devPath)
|
||||||
|
if !strings.Contains(stdErr, readOnlyErr) {
|
||||||
|
e2elog.Failf(stdErr)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
err = deletePVCAndDeploymentApp(f, "", pvcClone, appClone)
|
||||||
|
if err != nil {
|
||||||
|
e2elog.Failf("failed to delete PVC and application: %v", err)
|
||||||
|
}
|
||||||
|
// delete parent pvc
|
||||||
|
err = deletePVCAndValidatePV(f.ClientSet, pvc, deployTimeout)
|
||||||
|
if err != nil {
|
||||||
|
e2elog.Failf("failed to delete PVC: %v", err)
|
||||||
|
}
|
||||||
|
// validate created backend rbd images
|
||||||
|
validateRBDImageCount(f, 0, defaultRBDPool)
|
||||||
|
// validate images in trash
|
||||||
|
err = waitToRemoveImagesFromTrash(f, defaultRBDPool, deployTimeout)
|
||||||
|
if err != nil {
|
||||||
|
e2elog.Failf("failed to validate rbd images in pool %s trash: %v", rbdOptions(defaultRBDPool), err)
|
||||||
|
}
|
||||||
|
err = deleteResource(rbdExamplePath + "storageclass.yaml")
|
||||||
|
if err != nil {
|
||||||
|
e2elog.Failf("failed to delete storageclass: %v", err)
|
||||||
|
}
|
||||||
|
err = createRBDStorageClass(f.ClientSet, f, defaultSCName, nil, nil, deletePolicy)
|
||||||
|
if err != nil {
|
||||||
|
e2elog.Failf("failed to create storageclass: %v", err)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
By("perform IO on rbd-nbd volume after nodeplugin restart", func() {
|
By("perform IO on rbd-nbd volume after nodeplugin restart", func() {
|
||||||
err := deleteResource(rbdExamplePath + "storageclass.yaml")
|
err := deleteResource(rbdExamplePath + "storageclass.yaml")
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
|||||||
Loading…
Reference in New Issue
Block a user