e2e: restart rbd-nbd process after nodeplugin reboot

Bringup the rbd-nbd map/attach process on the rbd node plugin and expect the
IO to continue uninterrupted.

Signed-off-by: Prasanna Kumar Kalever <prasanna.kalever@redhat.com>

e2e/rbd.go

@@ -5,6 +5,7 @@ import (
 	"fmt"
 	"io/ioutil"
 	"os"
+	"regexp"
 	"strings"
 
 	. "github.com/onsi/ginkgo" // nolint
@@ -460,6 +461,139 @@ var _ = Describe("RBD", func() {
 				}
 			})
 
+			By("restart rbd-nbd process on nodeplugin and continue IO after nodeplugin restart", func() {
+				err := deleteResource(rbdExamplePath + "storageclass.yaml")
+				if err != nil {
+					e2elog.Failf("failed to delete storageclass with error %v", err)
+				}
+				// Tweak Storageclass to add netlink,reattach rbd-nbd mounter options
+				scOpts := map[string]string{
+					"mounter":    "rbd-nbd",
+					"mapOptions": "try-netlink,reattach-timeout=180",
+				}
+				err = createRBDStorageClass(f.ClientSet, f, nil, scOpts, deletePolicy)
+				if err != nil {
+					e2elog.Failf("failed to create storageclass with error %v", err)
+				}
+
+				pvc, err := loadPVC(pvcPath)
+				if err != nil {
+					e2elog.Failf("failed to load PVC with error %v", err)
+				}
+				pvc.Namespace = f.UniqueName
+
+				app, err := loadApp(appPath)
+				if err != nil {
+					e2elog.Failf("failed to load application with error %v", err)
+				}
+				app.Namespace = f.UniqueName
+				label := map[string]string{
+					"app": app.Name,
+				}
+				app.Labels = label
+				app.Spec.Volumes[0].PersistentVolumeClaim.ClaimName = pvc.Name
+				err = createPVCAndApp("", f, pvc, app, deployTimeout)
+				if err != nil {
+					e2elog.Failf("failed to create PVC and application with error %v", err)
+				}
+				// validate created backend rbd images
+				validateRBDImageCount(f, 1)
+
+				selector, err := getDaemonSetLabelSelector(f, cephCSINamespace, rbdDaemonsetName)
+				if err != nil {
+					e2elog.Failf("failed to get the labels with error %v", err)
+				}
+
+				opt := metav1.ListOptions{
+					LabelSelector: selector,
+				}
+
+				uname, stdErr, err := execCommandInContainer(f, "uname -a", cephCSINamespace, "csi-rbdplugin", &opt)
+				if err != nil || stdErr != "" {
+					e2elog.Failf("failed to run uname cmd : %v, stdErr: %v ", err, stdErr)
+				}
+				e2elog.Logf("uname -a: %v", uname)
+				rpmv, stdErr, err := execCommandInContainer(f, "rpm -qa | grep rbd-nbd", cephCSINamespace, "csi-rbdplugin", &opt)
+				if err != nil || stdErr != "" {
+					e2elog.Failf("failed to run rpm -qa cmd : %v, stdErr: %v ", err, stdErr)
+				}
+				e2elog.Logf("rbd-nbd package version: %v", rpmv)
+
+				// Get details of rbd-nbd process
+				// # ps -eo 'cmd' | grep [r]bd-nbd
+				// /usr/bin/rbd-nbd --id cephcsi-rbd-node -m svc-name:6789 --keyfile=/tmp/csi/keys/keyfile attach --device /dev/nbd0 pool-name/image-name --try-netlink --reattach-timeout=180
+				mapCmd, stdErr, err := execCommandInContainer(f, "ps -eo 'cmd' | grep [r]bd-nbd", cephCSINamespace, "csi-rbdplugin", &opt)
+				if err != nil || stdErr != "" {
+					e2elog.Failf("failed to run ps cmd : %v, stdErr: %v ", err, stdErr)
+				}
+				e2elog.Logf("map command running before restart, mapCmd: %v", mapCmd)
+
+				rbdNodeKey, stdErr, err := execCommandInToolBoxPod(f, "ceph auth get-key client.cephcsi-rbd-node", rookNamespace)
+				if err != nil || stdErr != "" {
+					e2elog.Failf("error getting cephcsi-rbd-node key, err: %v, stdErr: %v ", err, stdErr)
+				}
+
+				// restart the rbd node plugin
+				err = deletePodWithLabel(selector, cephCSINamespace, false)
+				if err != nil {
+					e2elog.Failf("fail to delete pod with error %v", err)
+				}
+
+				// wait for nodeplugin pods to come up
+				err = waitForDaemonSets(rbdDaemonsetName, cephCSINamespace, f.ClientSet, deployTimeout)
+				if err != nil {
+					e2elog.Failf("timeout waiting for daemonset pods with error %v", err)
+				}
+
+				// Prepare the rbd-nbd with command args
+				attachCmd := strings.ReplaceAll(mapCmd, "map", "attach --device /dev/nbd0")
+				m1 := regexp.MustCompile(`/keyfile-[0-9]* `)
+				attachCmd = m1.ReplaceAllString(attachCmd, "/keyfile-test ")
+				e2elog.Logf("attach command to run after restart, attachCmd: %v", attachCmd)
+
+				// create the keyfile
+				_, stdErr, err = execCommandInContainer(f, fmt.Sprintf("echo %s > /tmp/csi/keys/keyfile-test", rbdNodeKey), cephCSINamespace, "csi-rbdplugin", &opt)
+				if err != nil || stdErr != "" {
+					e2elog.Failf("failed to write key to a file,  err: %v, stdErr: %v ", err, stdErr)
+				}
+
+				_, stdErr, err = execCommandInContainer(f, attachCmd, cephCSINamespace, "csi-rbdplugin", &opt)
+				if err != nil || stdErr != "" {
+					e2elog.Failf("failed to run attach cmd err: %v, stdErr: %v ", err, stdErr)
+				}
+
+				runningAttachCmd, stdErr, err := execCommandInContainer(f, "ps -eo 'cmd' | grep [r]bd-nbd", cephCSINamespace, "csi-rbdplugin", &opt)
+				if err != nil || stdErr != "" {
+					e2elog.Failf("failed to run ps cmd : %v, stdErr: %v ", err, stdErr)
+				}
+				e2elog.Logf("attach command running after restart, runningAttachCmd: %v", runningAttachCmd)
+
+				appOpt := metav1.ListOptions{
+					LabelSelector: fmt.Sprintf("app=%s", app.Name),
+				}
+				// Write something to mountpoint and expect it to happen
+				filePath := app.Spec.Containers[0].VolumeMounts[0].MountPath + "/test"
+				_, stdErr, err = execCommandInPod(f, fmt.Sprintf("echo 'Hello World' > %s", filePath), app.Namespace, &appOpt)
+				if err != nil || stdErr != "" {
+					e2elog.Failf("failed to write IO, err: %v, stdErr: %v ", err, stdErr)
+				}
+
+				err = deletePVCAndApp("", f, pvc, app)
+				if err != nil {
+					e2elog.Failf("failed to delete PVC and application with error %v", err)
+				}
+				// validate created backend rbd images
+				validateRBDImageCount(f, 0)
+				err = deleteResource(rbdExamplePath + "storageclass.yaml")
+				if err != nil {
+					e2elog.Failf("failed to delete storageclass with error %v", err)
+				}
+				err = createRBDStorageClass(f.ClientSet, f, nil, nil, deletePolicy)
+				if err != nil {
+					e2elog.Failf("failed to create storageclass with error %v", err)
+				}
+			})
+
 			By("create a PVC and bind it to an app with encrypted RBD volume", func() {
 				err := deleteResource(rbdExamplePath + "storageclass.yaml")
 				if err != nil {