mirror of
https://github.com/ceph/ceph-csi.git
synced 2024-11-09 16:00:22 +00:00
Add validation of backend rbd image and snapshots
Signed-off-by: Madhu Rajanna <madhupr007@gmail.com>
This commit is contained in:
parent
a083cb713f
commit
2e1fc352b3
@ -1,6 +1,7 @@
|
||||
package e2e
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
. "github.com/onsi/ginkgo" // nolint
|
||||
@ -43,7 +44,9 @@ var _ = Describe("cephfs", func() {
|
||||
cephfsFiles := getFilesinDirectory(cephfsDirPath)
|
||||
for _, file := range cephfsFiles {
|
||||
res, err := framework.RunKubectl("delete", "-f", cephfsDirPath+file.Name())
|
||||
framework.Logf("failed to delete resource in %s with err %v", res, err)
|
||||
if err != nil {
|
||||
framework.Logf("failed to delete resource in %s with err %v", res, err)
|
||||
}
|
||||
}
|
||||
deleteResource(cephfsExamplePath + "secret.yaml")
|
||||
deleteResource(cephfsExamplePath + "storageclass.yaml")
|
||||
@ -52,6 +55,8 @@ var _ = Describe("cephfs", func() {
|
||||
|
||||
Context("Test cephfs CSI", func() {
|
||||
It("Test cephfs CSI", func() {
|
||||
pvcPath := cephfsExamplePath + "pvc.yaml"
|
||||
appPath := cephfsExamplePath + "pod.yaml"
|
||||
By("checking provisioner statefulset is running")
|
||||
timeout := time.Duration(deployTimeout) * time.Minute
|
||||
err := framework.WaitForStatefulSetReplicasReady(cephfsDeploymentName, namespace, f.ClientSet, 1*time.Second, timeout)
|
||||
@ -67,16 +72,49 @@ var _ = Describe("cephfs", func() {
|
||||
|
||||
By("create and delete a PVC", func() {
|
||||
By("create a PVC and Bind it to an app", func() {
|
||||
pvcPath := cephfsExamplePath + "pvc.yaml"
|
||||
appPath := cephfsExamplePath + "pod.yaml"
|
||||
validatePVCAndAppBinding(pvcPath, appPath, f)
|
||||
|
||||
})
|
||||
|
||||
By("create a PVC and Bind it to an app with normal user", func() {
|
||||
pvcPath := cephfsExamplePath + "pvc.yaml"
|
||||
validateNormalUserPVCAccess(pvcPath, f)
|
||||
})
|
||||
|
||||
By("create/delete multiple PVCs and Apps", func() {
|
||||
totalCount := 2
|
||||
pvc, err := loadPVC(pvcPath)
|
||||
if err != nil {
|
||||
Fail(err.Error())
|
||||
}
|
||||
pvc.Namespace = f.UniqueName
|
||||
|
||||
app, err := loadApp(appPath)
|
||||
if err != nil {
|
||||
Fail(err.Error())
|
||||
}
|
||||
app.Namespace = f.UniqueName
|
||||
// create pvc and app
|
||||
for i := 0; i < totalCount; i++ {
|
||||
name := fmt.Sprintf("%s%d", f.UniqueName, i)
|
||||
err := createPVCAndApp(name, f, pvc, app)
|
||||
if err != nil {
|
||||
Fail(err.Error())
|
||||
}
|
||||
|
||||
}
|
||||
// TODO add cephfs backend validation
|
||||
|
||||
// delete pvc and app
|
||||
for i := 0; i < totalCount; i++ {
|
||||
name := fmt.Sprintf("%s%d", f.UniqueName, i)
|
||||
err := deletePVCAndApp(name, f, pvc, app)
|
||||
if err != nil {
|
||||
Fail(err.Error())
|
||||
}
|
||||
|
||||
}
|
||||
})
|
||||
|
||||
})
|
||||
|
||||
})
|
||||
|
@ -49,19 +49,26 @@ func createRBDPool() {
|
||||
}
|
||||
func deleteFileSystem() {
|
||||
commonPath := fmt.Sprintf("%s/%s", rookURL, "filesystem-test.yaml")
|
||||
framework.RunKubectlOrDie("delete", "-f", commonPath)
|
||||
_, err := framework.RunKubectl("delete", "-f", commonPath)
|
||||
if err != nil {
|
||||
framework.Logf("failed to delete file-system %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func deleteRBDPool() {
|
||||
commonPath := fmt.Sprintf("%s/%s", rookURL, "pool-test.yaml")
|
||||
framework.RunKubectlOrDie("delete", "-f", commonPath)
|
||||
_, err := framework.RunKubectl("delete", "-f", commonPath)
|
||||
if err != nil {
|
||||
framework.Logf("failed to delete pool %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func deployOperator(c kubernetes.Interface) {
|
||||
opPath := fmt.Sprintf("%s/%s", rookURL, "operator.yaml")
|
||||
|
||||
framework.RunKubectlOrDie("create", "-f", opPath)
|
||||
err := waitForDaemonSets("rook-ceph-agent", rookNS, c, deployTimeout)
|
||||
_, err := framework.RunKubectl("create", "-f", opPath)
|
||||
Expect(err).Should(BeNil())
|
||||
err = waitForDaemonSets("rook-ceph-agent", rookNS, c, deployTimeout)
|
||||
Expect(err).Should(BeNil())
|
||||
err = waitForDaemonSets("rook-discover", rookNS, c, deployTimeout)
|
||||
Expect(err).Should(BeNil())
|
||||
@ -109,5 +116,8 @@ func tearDownRook() {
|
||||
// TODO need to add selector for cleanup validation
|
||||
framework.Cleanup(opPath, rookNS)
|
||||
commonPath := fmt.Sprintf("%s/%s", rookURL, "common.yaml")
|
||||
framework.RunKubectlOrDie("delete", "-f", commonPath)
|
||||
_, err := framework.RunKubectl("delete", "-f", commonPath)
|
||||
if err != nil {
|
||||
framework.Logf("failed to delete rook common %v", err)
|
||||
}
|
||||
}
|
||||
|
86
e2e/rbd.go
86
e2e/rbd.go
@ -1,6 +1,7 @@
|
||||
package e2e
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
. "github.com/onsi/ginkgo" // nolint
|
||||
@ -46,7 +47,9 @@ var _ = Describe("RBD", func() {
|
||||
rbdFiles := getFilesinDirectory(rbdDirPath)
|
||||
for _, file := range rbdFiles {
|
||||
res, err := framework.RunKubectl("delete", "-f", rbdDirPath+file.Name())
|
||||
framework.Logf("failed to delete resource in %s with err %v", res, err)
|
||||
if err != nil {
|
||||
framework.Logf("failed to delete resource in %s with err %v", res, err)
|
||||
}
|
||||
}
|
||||
deleteRBDPool()
|
||||
deleteResource(rbdExamplePath + "secret.yaml")
|
||||
@ -56,6 +59,14 @@ var _ = Describe("RBD", func() {
|
||||
|
||||
Context("Test RBD CSI", func() {
|
||||
It("Test RBD CSI", func() {
|
||||
pvcPath := rbdExamplePath + "raw-block-pvc.yaml"
|
||||
appPath := rbdExamplePath + "raw-block-pod.yaml"
|
||||
rawPvcPath := rbdExamplePath + "raw-block-pvc.yaml"
|
||||
rawAppPath := rbdExamplePath + "raw-block-pod.yaml"
|
||||
pvcClonePath := rbdExamplePath + "pvc-restore.yaml"
|
||||
appClonePath := rbdExamplePath + "pod-restore.yaml"
|
||||
snapshotPath := rbdExamplePath + "snapshot.yaml"
|
||||
|
||||
By("checking provisioner statefulset is running")
|
||||
timeout := time.Duration(deployTimeout) * time.Minute
|
||||
err := framework.WaitForStatefulSetReplicasReady(rbdDeploymentName, namespace, f.ClientSet, 1*time.Second, timeout)
|
||||
@ -70,22 +81,15 @@ var _ = Describe("RBD", func() {
|
||||
}
|
||||
|
||||
By("create a PVC and Bind it to an app", func() {
|
||||
pvcPath := rbdExamplePath + "pvc.yaml"
|
||||
appPath := rbdExamplePath + "pod.yaml"
|
||||
validatePVCAndAppBinding(pvcPath, appPath, f)
|
||||
})
|
||||
|
||||
By("create a PVC and Bind it to an app with normal user", func() {
|
||||
pvcPath := rbdExamplePath + "pvc.yaml"
|
||||
validateNormalUserPVCAccess(pvcPath, f)
|
||||
})
|
||||
|
||||
By("create a PVC clone and Bind it to an app", func() {
|
||||
createRBDSnapshotClass(f)
|
||||
pvcPath := rbdExamplePath + "pvc.yaml"
|
||||
pvcClonePath := rbdExamplePath + "pvc-restore.yaml"
|
||||
appClonePath := rbdExamplePath + "pod-restore.yaml"
|
||||
snapshotPath := rbdExamplePath + "snapshot.yaml"
|
||||
pvc, err := loadPVC(pvcPath)
|
||||
if err != nil {
|
||||
Fail(err.Error())
|
||||
@ -97,6 +101,12 @@ var _ = Describe("RBD", func() {
|
||||
if err != nil {
|
||||
Fail(err.Error())
|
||||
}
|
||||
// validate created backend rbd images
|
||||
images := listRBDImages(f)
|
||||
if len(images) != 1 {
|
||||
framework.Logf("backend image count %d expected image count %d", len(images), 1)
|
||||
Fail("validate backend image failed")
|
||||
}
|
||||
snap := getSnapshot(snapshotPath)
|
||||
snap.Namespace = f.UniqueName
|
||||
snap.Spec.Source.Name = pvc.Name
|
||||
@ -105,6 +115,15 @@ var _ = Describe("RBD", func() {
|
||||
if err != nil {
|
||||
Fail(err.Error())
|
||||
}
|
||||
pool := "replicapool"
|
||||
snapList, err := listSnapshots(f, pool, images[0])
|
||||
if err != nil {
|
||||
Fail(err.Error())
|
||||
}
|
||||
if len(snapList) != 1 {
|
||||
framework.Logf("backend snapshot not matching kube snap count,snap count = % kube snap count %d", len(snapList), 1)
|
||||
Fail("validate backend snapshot failed")
|
||||
}
|
||||
|
||||
validatePVCAndAppBinding(pvcClonePath, appClonePath, f)
|
||||
|
||||
@ -119,9 +138,54 @@ var _ = Describe("RBD", func() {
|
||||
})
|
||||
|
||||
By("create a block type PVC and Bind it to an app", func() {
|
||||
pvcPath := rbdExamplePath + "raw-block-pvc.yaml"
|
||||
appPath := rbdExamplePath + "raw-block-pod.yaml"
|
||||
validatePVCAndAppBinding(pvcPath, appPath, f)
|
||||
validatePVCAndAppBinding(rawPvcPath, rawAppPath, f)
|
||||
})
|
||||
|
||||
By("create/delete multiple PVCs and Apps", func() {
|
||||
totalCount := 2
|
||||
pvc, err := loadPVC(pvcPath)
|
||||
if err != nil {
|
||||
Fail(err.Error())
|
||||
}
|
||||
pvc.Namespace = f.UniqueName
|
||||
|
||||
app, err := loadApp(appPath)
|
||||
if err != nil {
|
||||
Fail(err.Error())
|
||||
}
|
||||
app.Namespace = f.UniqueName
|
||||
// create pvc and app
|
||||
for i := 0; i < totalCount; i++ {
|
||||
name := fmt.Sprintf("%s%d", f.UniqueName, i)
|
||||
err := createPVCAndApp(name, f, pvc, app)
|
||||
if err != nil {
|
||||
Fail(err.Error())
|
||||
}
|
||||
|
||||
}
|
||||
// validate created backend rbd images
|
||||
images := listRBDImages(f)
|
||||
if len(images) != totalCount {
|
||||
framework.Logf("backend image creation not matching pvc count, image count = % pvc count %d", len(images), totalCount)
|
||||
Fail("validate multiple pvc failed")
|
||||
}
|
||||
|
||||
// delete pvc and app
|
||||
for i := 0; i < totalCount; i++ {
|
||||
name := fmt.Sprintf("%s%d", f.UniqueName, i)
|
||||
err := deletePVCAndApp(name, f, pvc, app)
|
||||
if err != nil {
|
||||
Fail(err.Error())
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
// validate created backend rbd images
|
||||
images = listRBDImages(f)
|
||||
if len(images) > 0 {
|
||||
framework.Logf("left out rbd backend images count %d", len(images))
|
||||
Fail("validate multiple pvc failed")
|
||||
}
|
||||
})
|
||||
})
|
||||
})
|
||||
|
93
e2e/utils.go
93
e2e/utils.go
@ -37,6 +37,13 @@ func getFilesinDirectory(path string) []os.FileInfo {
|
||||
|
||||
var poll = 2 * time.Second
|
||||
|
||||
type snapInfo struct {
|
||||
ID int64 `json:"id"`
|
||||
Name string `json:"name"`
|
||||
Size int64 `json:"size"`
|
||||
Timestamp string `json:"timestamp"`
|
||||
}
|
||||
|
||||
func waitForDaemonSets(name, ns string, c clientset.Interface, t int) error {
|
||||
timeout := time.Duration(t) * time.Minute
|
||||
start := time.Now()
|
||||
@ -507,6 +514,41 @@ func checkCephPods(ns string, c kubernetes.Interface, count, t int, opt *metav1.
|
||||
|
||||
}
|
||||
|
||||
// createPVCAndApp creates pvc and pod
|
||||
// if name is not empty same will be set as pvc and app name
|
||||
func createPVCAndApp(name string, f *framework.Framework, pvc *v1.PersistentVolumeClaim, app *v1.Pod) error {
|
||||
|
||||
if name != "" {
|
||||
pvc.Name = name
|
||||
app.Name = name
|
||||
app.Spec.Volumes[0].PersistentVolumeClaim.ClaimName = name
|
||||
}
|
||||
err := createPVCAndvalidatePV(f.ClientSet, pvc, deployTimeout)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
err = createApp(f.ClientSet, app, deployTimeout)
|
||||
return err
|
||||
}
|
||||
|
||||
// deletePVCAndApp delete pvc and pod
|
||||
// if name is not empty same will be set as pvc and app name
|
||||
func deletePVCAndApp(name string, f *framework.Framework, pvc *v1.PersistentVolumeClaim, app *v1.Pod) error {
|
||||
|
||||
if name != "" {
|
||||
pvc.Name = name
|
||||
app.Name = name
|
||||
app.Spec.Volumes[0].PersistentVolumeClaim.ClaimName = name
|
||||
}
|
||||
|
||||
err := deletePod(app.Name, app.Namespace, f.ClientSet, deployTimeout)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
err = deletePVCAndValidatePV(f.ClientSet, pvc, deployTimeout)
|
||||
return err
|
||||
}
|
||||
|
||||
func validatePVCAndAppBinding(pvcPath, appPath string, f *framework.Framework) {
|
||||
pvc, err := loadPVC(pvcPath)
|
||||
if pvc == nil {
|
||||
@ -514,27 +556,19 @@ func validatePVCAndAppBinding(pvcPath, appPath string, f *framework.Framework) {
|
||||
}
|
||||
pvc.Namespace = f.UniqueName
|
||||
framework.Logf("The PVC template %+v", pvc)
|
||||
err = createPVCAndvalidatePV(f.ClientSet, pvc, deployTimeout)
|
||||
if err != nil {
|
||||
Fail(err.Error())
|
||||
}
|
||||
|
||||
app, err := loadApp(appPath)
|
||||
if err != nil {
|
||||
Fail(err.Error())
|
||||
}
|
||||
app.Namespace = f.UniqueName
|
||||
err = createApp(f.ClientSet, app, deployTimeout)
|
||||
|
||||
err = createPVCAndApp("", f, pvc, app)
|
||||
if err != nil {
|
||||
Fail(err.Error())
|
||||
}
|
||||
|
||||
err = deletePod(app.Name, app.Namespace, f.ClientSet, deployTimeout)
|
||||
if err != nil {
|
||||
Fail(err.Error())
|
||||
}
|
||||
|
||||
err = deletePVCAndValidatePV(f.ClientSet, pvc, deployTimeout)
|
||||
err = deletePVCAndApp("", f, pvc, app)
|
||||
if err != nil {
|
||||
Fail(err.Error())
|
||||
}
|
||||
@ -626,6 +660,7 @@ func createSnapshot(snap *v1alpha1.VolumeSnapshot, t int) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
framework.Logf("snapshot with name %v created in %v namespace", snap.Name, snap.Namespace)
|
||||
|
||||
timeout := time.Duration(t) * time.Minute
|
||||
name := snap.Name
|
||||
@ -665,12 +700,11 @@ func deleteSnapshot(snap *v1alpha1.VolumeSnapshot, t int) error {
|
||||
timeout := time.Duration(t) * time.Minute
|
||||
name := snap.Name
|
||||
start := time.Now()
|
||||
framework.Logf("Waiting up to %v to be in Ready state", snap)
|
||||
framework.Logf("Waiting up to %v to be deleted", snap)
|
||||
|
||||
return wait.PollImmediate(poll, timeout, func() (bool, error) {
|
||||
framework.Logf("waiting for snapshot %s (%d seconds elapsed)", name, int(time.Since(start).Seconds()))
|
||||
framework.Logf("deleting snapshot %s (%d seconds elapsed)", name, int(time.Since(start).Seconds()))
|
||||
_, err := sclient.VolumeSnapshots(snap.Namespace).Get(name, metav1.GetOptions{})
|
||||
|
||||
if err == nil {
|
||||
return false, nil
|
||||
}
|
||||
@ -679,6 +713,35 @@ func deleteSnapshot(snap *v1alpha1.VolumeSnapshot, t int) error {
|
||||
return false, fmt.Errorf("get on deleted snapshot %v failed with error other than \"not found\": %v", name, err)
|
||||
}
|
||||
|
||||
return false, nil
|
||||
return true, nil
|
||||
})
|
||||
}
|
||||
|
||||
func listRBDImages(f *framework.Framework) []string {
|
||||
|
||||
opt := metav1.ListOptions{
|
||||
LabelSelector: "app=rook-ceph-tools",
|
||||
}
|
||||
stdout := execCommandInPod(f, "rbd ls --pool=replicapool --format=json", rookNS, &opt)
|
||||
|
||||
var imgInfos []string
|
||||
|
||||
err := json.Unmarshal([]byte(stdout), &imgInfos)
|
||||
if err != nil {
|
||||
Fail(err.Error())
|
||||
}
|
||||
return imgInfos
|
||||
}
|
||||
|
||||
func listSnapshots(f *framework.Framework, pool, imageName string) ([]snapInfo, error) {
|
||||
opt := metav1.ListOptions{
|
||||
LabelSelector: "app=rook-ceph-tools",
|
||||
}
|
||||
command := fmt.Sprintf("rbd snap ls %s/%s --format=json", pool, imageName)
|
||||
stdout := execCommandInPod(f, command, rookNS, &opt)
|
||||
|
||||
var snapInfos []snapInfo
|
||||
|
||||
err := json.Unmarshal([]byte(stdout), &snapInfos)
|
||||
return snapInfos, err
|
||||
}
|
||||
|
Loading…
Reference in New Issue
Block a user