mirrors/ceph-csi
1
0
Fork 0
mirror of https://github.com/ceph/ceph-csi.git synced 2024-11-09 16:00:22 +00:00
Code Issues Packages Projects Releases Wiki Activity

e2e: restart rbd-nbd process after nodeplugin reboot

Browse Source 
Bringup the rbd-nbd map/attach process on the rbd node plugin and expect the
IO to continue uninterrupted.

Signed-off-by: Prasanna Kumar Kalever <prasanna.kalever@redhat.com>
This commit is contained in:
Prasanna Kumar Kalever 2021-02-25 12:53:22 +05:30 committed by mergify[bot]
parent 7334c3b783
commit 819267112e
1 changed files with 134 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

134
e2e/rbd.go
View File

@ -5,6 +5,7 @@ import (
"fmt"
"io/ioutil"
"os"
"regexp"
"strings"
. "github.com/onsi/ginkgo" // nolint
@ -460,6 +461,139 @@ var _ = Describe("RBD", func() {
}
})
By("restart rbd-nbd process on nodeplugin and continue IO after nodeplugin restart", func() {
err := deleteResource(rbdExamplePath + "storageclass.yaml")
if err != nil {
e2elog.Failf("failed to delete storageclass with error %v", err)
}
// Tweak Storageclass to add netlink,reattach rbd-nbd mounter options
scOpts := map[string]string{
"mounter": "rbd-nbd",
"mapOptions": "try-netlink,reattach-timeout=180",
}
err = createRBDStorageClass(f.ClientSet, f, nil, scOpts, deletePolicy)
if err != nil {
e2elog.Failf("failed to create storageclass with error %v", err)
}
pvc, err := loadPVC(pvcPath)
if err != nil {
e2elog.Failf("failed to load PVC with error %v", err)
}
pvc.Namespace = f.UniqueName
app, err := loadApp(appPath)
if err != nil {
e2elog.Failf("failed to load application with error %v", err)
}
app.Namespace = f.UniqueName
label := map[string]string{
"app": app.Name,
}
app.Labels = label
app.Spec.Volumes[0].PersistentVolumeClaim.ClaimName = pvc.Name
err = createPVCAndApp("", f, pvc, app, deployTimeout)
if err != nil {
e2elog.Failf("failed to create PVC and application with error %v", err)
}
// validate created backend rbd images
validateRBDImageCount(f, 1)
selector, err := getDaemonSetLabelSelector(f, cephCSINamespace, rbdDaemonsetName)
if err != nil {
e2elog.Failf("failed to get the labels with error %v", err)
}
opt := metav1.ListOptions{
LabelSelector: selector,
}
uname, stdErr, err := execCommandInContainer(f, "uname -a", cephCSINamespace, "csi-rbdplugin", &opt)
if err != nil || stdErr != "" {
e2elog.Failf("failed to run uname cmd : %v, stdErr: %v ", err, stdErr)
}
e2elog.Logf("uname -a: %v", uname)
rpmv, stdErr, err := execCommandInContainer(f, "rpm -qa | grep rbd-nbd", cephCSINamespace, "csi-rbdplugin", &opt)
if err != nil || stdErr != "" {
e2elog.Failf("failed to run rpm -qa cmd : %v, stdErr: %v ", err, stdErr)
}
e2elog.Logf("rbd-nbd package version: %v", rpmv)
// Get details of rbd-nbd process
// # ps -eo 'cmd' | grep [r]bd-nbd
// /usr/bin/rbd-nbd --id cephcsi-rbd-node -m svc-name:6789 --keyfile=/tmp/csi/keys/keyfile attach --device /dev/nbd0 pool-name/image-name --try-netlink --reattach-timeout=180
mapCmd, stdErr, err := execCommandInContainer(f, "ps -eo 'cmd' | grep [r]bd-nbd", cephCSINamespace, "csi-rbdplugin", &opt)
if err != nil || stdErr != "" {
e2elog.Failf("failed to run ps cmd : %v, stdErr: %v ", err, stdErr)
}
e2elog.Logf("map command running before restart, mapCmd: %v", mapCmd)
rbdNodeKey, stdErr, err := execCommandInToolBoxPod(f, "ceph auth get-key client.cephcsi-rbd-node", rookNamespace)
if err != nil || stdErr != "" {
e2elog.Failf("error getting cephcsi-rbd-node key, err: %v, stdErr: %v ", err, stdErr)
}
// restart the rbd node plugin
err = deletePodWithLabel(selector, cephCSINamespace, false)
if err != nil {
e2elog.Failf("fail to delete pod with error %v", err)
}
// wait for nodeplugin pods to come up
err = waitForDaemonSets(rbdDaemonsetName, cephCSINamespace, f.ClientSet, deployTimeout)
if err != nil {
e2elog.Failf("timeout waiting for daemonset pods with error %v", err)
}
// Prepare the rbd-nbd with command args
attachCmd := strings.ReplaceAll(mapCmd, "map", "attach --device /dev/nbd0")
m1 := regexp.MustCompile(`/keyfile-[0-9]* `)
attachCmd = m1.ReplaceAllString(attachCmd, "/keyfile-test ")
e2elog.Logf("attach command to run after restart, attachCmd: %v", attachCmd)
// create the keyfile
_, stdErr, err = execCommandInContainer(f, fmt.Sprintf("echo %s > /tmp/csi/keys/keyfile-test", rbdNodeKey), cephCSINamespace, "csi-rbdplugin", &opt)
if err != nil || stdErr != "" {
e2elog.Failf("failed to write key to a file, err: %v, stdErr: %v ", err, stdErr)
}
_, stdErr, err = execCommandInContainer(f, attachCmd, cephCSINamespace, "csi-rbdplugin", &opt)
if err != nil || stdErr != "" {
e2elog.Failf("failed to run attach cmd err: %v, stdErr: %v ", err, stdErr)
}
runningAttachCmd, stdErr, err := execCommandInContainer(f, "ps -eo 'cmd' | grep [r]bd-nbd", cephCSINamespace, "csi-rbdplugin", &opt)
if err != nil || stdErr != "" {
e2elog.Failf("failed to run ps cmd : %v, stdErr: %v ", err, stdErr)
}
e2elog.Logf("attach command running after restart, runningAttachCmd: %v", runningAttachCmd)
appOpt := metav1.ListOptions{
LabelSelector: fmt.Sprintf("app=%s", app.Name),
}
// Write something to mountpoint and expect it to happen
filePath := app.Spec.Containers[0].VolumeMounts[0].MountPath + "/test"
_, stdErr, err = execCommandInPod(f, fmt.Sprintf("echo 'Hello World' > %s", filePath), app.Namespace, &appOpt)
if err != nil || stdErr != "" {
e2elog.Failf("failed to write IO, err: %v, stdErr: %v ", err, stdErr)
}
err = deletePVCAndApp("", f, pvc, app)
if err != nil {
e2elog.Failf("failed to delete PVC and application with error %v", err)
}
// validate created backend rbd images
validateRBDImageCount(f, 0)
err = deleteResource(rbdExamplePath + "storageclass.yaml")
if err != nil {
e2elog.Failf("failed to delete storageclass with error %v", err)
}
err = createRBDStorageClass(f.ClientSet, f, nil, nil, deletePolicy)
if err != nil {
e2elog.Failf("failed to create storageclass with error %v", err)
}
})
By("create a PVC and bind it to an app with encrypted RBD volume", func() {
err := deleteResource(rbdExamplePath + "storageclass.yaml")
if err != nil {