diff --git a/e2e/rbd.go b/e2e/rbd.go index ccc4cb959..42fe0e097 100644 --- a/e2e/rbd.go +++ b/e2e/rbd.go @@ -162,7 +162,10 @@ func validateRBDImageCount(f *framework.Framework, count int, pool string) { e2elog.Failf("failed to list rbd images with error %v", err) } if len(imageList) != count { - e2elog.Failf("backend images not matching kubernetes resource count,image count %d kubernetes resource count %d", len(imageList), count) + e2elog.Failf( + "backend images not matching kubernetes resource count,image count %d kubernetes resource count %d", + len(imageList), + count) } } @@ -241,11 +244,15 @@ var _ = Describe("RBD", func() { if err != nil { e2elog.Failf("failed to delete configmap with error %v", err) } - err = c.CoreV1().Secrets(cephCSINamespace).Delete(context.TODO(), rbdProvisionerSecretName, metav1.DeleteOptions{}) + err = c.CoreV1(). + Secrets(cephCSINamespace). + Delete(context.TODO(), rbdProvisionerSecretName, metav1.DeleteOptions{}) if err != nil { e2elog.Failf("failed to delete provisioner secret with error %v", err) } - err = c.CoreV1().Secrets(cephCSINamespace).Delete(context.TODO(), rbdNodePluginSecretName, metav1.DeleteOptions{}) + err = c.CoreV1(). + Secrets(cephCSINamespace). + Delete(context.TODO(), rbdNodePluginSecretName, metav1.DeleteOptions{}) if err != nil { e2elog.Failf("failed to delete node secret with error %v", err) } @@ -331,7 +338,13 @@ var _ = Describe("RBD", func() { if err != nil { e2elog.Failf("failed to delete storageclass with error %v", err) } - err = createRBDStorageClass(f.ClientSet, f, defaultSCName, nil, map[string]string{"csi.storage.k8s.io/fstype": "ext4"}, deletePolicy) + err = createRBDStorageClass( + f.ClientSet, + f, + defaultSCName, + nil, + map[string]string{"csi.storage.k8s.io/fstype": "ext4"}, + deletePolicy) if err != nil { e2elog.Failf("failed to create storageclass with error %v", err) } @@ -356,7 +369,13 @@ var _ = Describe("RBD", func() { if err != nil { e2elog.Failf("failed to delete storageclass with error %v", err) } - err = createRBDStorageClass(f.ClientSet, f, defaultSCName, nil, map[string]string{"mounter": "rbd-nbd"}, deletePolicy) + err = createRBDStorageClass( + f.ClientSet, + f, + defaultSCName, + nil, + map[string]string{"mounter": "rbd-nbd"}, + deletePolicy) if err != nil { e2elog.Failf("failed to create storageclass with error %v", err) } @@ -382,7 +401,13 @@ var _ = Describe("RBD", func() { e2elog.Failf("failed to delete storageclass with error %v", err) } // Storage class with rbd-nbd mounter - err = createRBDStorageClass(f.ClientSet, f, defaultSCName, nil, map[string]string{"mounter": "rbd-nbd"}, deletePolicy) + err = createRBDStorageClass( + f.ClientSet, + f, + defaultSCName, + nil, + map[string]string{"mounter": "rbd-nbd"}, + deletePolicy) if err != nil { e2elog.Failf("failed to create storageclass with error %v", err) } @@ -438,7 +463,11 @@ var _ = Describe("RBD", func() { // For now to prove this isn't working, write something to // mountpoint and expect a failure as the processes are terminated. filePath := app.Spec.Containers[0].VolumeMounts[0].MountPath + "/test" - _, stdErr := execCommandInPodAndAllowFail(f, fmt.Sprintf("echo 'Hello World' > %s", filePath), app.Namespace, &opt) + _, stdErr := execCommandInPodAndAllowFail( + f, + fmt.Sprintf("echo 'Hello World' > %s", filePath), + app.Namespace, + &opt) IOErr := fmt.Sprintf("cannot create %s: Input/output error", filePath) if !strings.Contains(stdErr, IOErr) { e2elog.Failf(stdErr) @@ -514,7 +543,12 @@ var _ = Describe("RBD", func() { e2elog.Failf("failed to run uname cmd : %v, stdErr: %v ", err, stdErr) } e2elog.Logf("uname -a: %v", uname) - rpmv, stdErr, err := execCommandInContainer(f, "rpm -qa | grep rbd-nbd", cephCSINamespace, "csi-rbdplugin", &opt) + rpmv, stdErr, err := execCommandInContainer( + f, + "rpm -qa | grep rbd-nbd", + cephCSINamespace, + "csi-rbdplugin", + &opt) if err != nil || stdErr != "" { e2elog.Failf("failed to run rpm -qa cmd : %v, stdErr: %v ", err, stdErr) } @@ -522,14 +556,23 @@ var _ = Describe("RBD", func() { // Get details of rbd-nbd process // # ps -eo 'cmd' | grep [r]bd-nbd - // /usr/bin/rbd-nbd --id cephcsi-rbd-node -m svc-name:6789 --keyfile=/tmp/csi/keys/keyfile attach --device /dev/nbd0 pool-name/image-name --try-netlink --reattach-timeout=180 - mapCmd, stdErr, err := execCommandInContainer(f, "ps -eo 'cmd' | grep [r]bd-nbd", cephCSINamespace, "csi-rbdplugin", &opt) + // /usr/bin/rbd-nbd --id cephcsi-rbd-node -m svc-name:6789 --keyfile=/tmp/csi/keys/keyfile attach \ + // --device /dev/nbd0 pool-name/image-name --try-netlink --reattach-timeout=180 + mapCmd, stdErr, err := execCommandInContainer( + f, + "ps -eo 'cmd' | grep [r]bd-nbd", + cephCSINamespace, + "csi-rbdplugin", + &opt) if err != nil || stdErr != "" { e2elog.Failf("failed to run ps cmd : %v, stdErr: %v ", err, stdErr) } e2elog.Logf("map command running before restart, mapCmd: %v", mapCmd) - rbdNodeKey, stdErr, err := execCommandInToolBoxPod(f, "ceph auth get-key client.cephcsi-rbd-node", rookNamespace) + rbdNodeKey, stdErr, err := execCommandInToolBoxPod( + f, + "ceph auth get-key client.cephcsi-rbd-node", + rookNamespace) if err != nil || stdErr != "" { e2elog.Failf("error getting cephcsi-rbd-node key, err: %v, stdErr: %v ", err, stdErr) } @@ -553,7 +596,12 @@ var _ = Describe("RBD", func() { e2elog.Logf("attach command to run after restart, attachCmd: %v", attachCmd) // create the keyfile - _, stdErr, err = execCommandInContainer(f, fmt.Sprintf("echo %s > /tmp/csi/keys/keyfile-test", rbdNodeKey), cephCSINamespace, "csi-rbdplugin", &opt) + _, stdErr, err = execCommandInContainer( + f, + fmt.Sprintf("echo %s > /tmp/csi/keys/keyfile-test", rbdNodeKey), + cephCSINamespace, + "csi-rbdplugin", + &opt) if err != nil || stdErr != "" { e2elog.Failf("failed to write key to a file, err: %v, stdErr: %v ", err, stdErr) } @@ -563,7 +611,12 @@ var _ = Describe("RBD", func() { e2elog.Failf("failed to run attach cmd err: %v, stdErr: %v ", err, stdErr) } - runningAttachCmd, stdErr, err := execCommandInContainer(f, "ps -eo 'cmd' | grep [r]bd-nbd", cephCSINamespace, "csi-rbdplugin", &opt) + runningAttachCmd, stdErr, err := execCommandInContainer( + f, + "ps -eo 'cmd' | grep [r]bd-nbd", + cephCSINamespace, + "csi-rbdplugin", + &opt) if err != nil || stdErr != "" { e2elog.Failf("failed to run ps cmd : %v, stdErr: %v ", err, stdErr) } @@ -574,7 +627,11 @@ var _ = Describe("RBD", func() { } // Write something to mountpoint and expect it to happen filePath := app.Spec.Containers[0].VolumeMounts[0].MountPath + "/test" - _, stdErr, err = execCommandInPod(f, fmt.Sprintf("echo 'Hello World' > %s", filePath), app.Namespace, &appOpt) + _, stdErr, err = execCommandInPod( + f, + fmt.Sprintf("echo 'Hello World' > %s", filePath), + app.Namespace, + &appOpt) if err != nil || stdErr != "" { e2elog.Failf("failed to write IO, err: %v, stdErr: %v ", err, stdErr) } @@ -600,7 +657,13 @@ var _ = Describe("RBD", func() { if err != nil { e2elog.Failf("failed to delete storageclass with error %v", err) } - err = createRBDStorageClass(f.ClientSet, f, defaultSCName, nil, map[string]string{"encrypted": "true"}, deletePolicy) + err = createRBDStorageClass( + f.ClientSet, + f, + defaultSCName, + nil, + map[string]string{"encrypted": "true"}, + deletePolicy) if err != nil { e2elog.Failf("failed to create storageclass with error %v", err) } @@ -728,40 +791,64 @@ var _ = Describe("RBD", func() { } }) - By("create a PVC and Bind it to an app with journaling/exclusive-lock image-features and rbd-nbd mounter", func() { - err := deleteResource(rbdExamplePath + "storageclass.yaml") - if err != nil { - e2elog.Failf("failed to delete storageclass with error %v", err) - } - err = createRBDStorageClass(f.ClientSet, f, defaultSCName, nil, map[string]string{"imageFeatures": "layering,journaling,exclusive-lock", "mounter": "rbd-nbd"}, deletePolicy) - if err != nil { - e2elog.Failf("failed to create storageclass with error %v", err) - } - err = validatePVCAndAppBinding(pvcPath, appPath, f) - if err != nil { - e2elog.Failf("failed to validate pvc and application binding with error %v", err) - } - err = deleteResource(rbdExamplePath + "storageclass.yaml") - if err != nil { - e2elog.Failf("failed to delete storageclass with error %v", err) - } - err = createRBDStorageClass(f.ClientSet, f, defaultSCName, nil, nil, deletePolicy) - if err != nil { - e2elog.Failf("failed to create storageclass with error %v", err) - } - }) + By( + "create a PVC and Bind it to an app with journaling/exclusive-lock image-features and rbd-nbd mounter", + func() { + err := deleteResource(rbdExamplePath + "storageclass.yaml") + if err != nil { + e2elog.Failf("failed to delete storageclass with error %v", err) + } + err = createRBDStorageClass( + f.ClientSet, + f, + defaultSCName, + nil, + map[string]string{"imageFeatures": "layering,journaling,exclusive-lock", "mounter": "rbd-nbd"}, + deletePolicy) + if err != nil { + e2elog.Failf("failed to create storageclass with error %v", err) + } + err = validatePVCAndAppBinding(pvcPath, appPath, f) + if err != nil { + e2elog.Failf("failed to validate pvc and application binding with error %v", err) + } + err = deleteResource(rbdExamplePath + "storageclass.yaml") + if err != nil { + e2elog.Failf("failed to delete storageclass with error %v", err) + } + err = createRBDStorageClass(f.ClientSet, f, defaultSCName, nil, nil, deletePolicy) + if err != nil { + e2elog.Failf("failed to create storageclass with error %v", err) + } + }, + ) By("create a PVC clone and bind it to an app", func() { // snapshot beta is only supported from v1.17+ if k8sVersionGreaterEquals(f.ClientSet, 1, 17) { - validatePVCSnapshot(defaultCloneCount, pvcPath, appPath, snapshotPath, pvcClonePath, appClonePath, "", f) + validatePVCSnapshot( + defaultCloneCount, + pvcPath, + appPath, + snapshotPath, + pvcClonePath, + appClonePath, + "", + f) } }) By("create a PVC-PVC clone and bind it to an app", func() { // pvc clone is only supported from v1.16+ if k8sVersionGreaterEquals(f.ClientSet, 1, 16) { - validatePVCClone(defaultCloneCount, pvcPath, appPath, pvcSmartClonePath, appSmartClonePath, noPVCValidation, f) + validatePVCClone( + defaultCloneCount, + pvcPath, + appPath, + pvcSmartClonePath, + appSmartClonePath, + noPVCValidation, + f) } }) @@ -866,7 +953,14 @@ var _ = Describe("RBD", func() { } // pvc clone is only supported from v1.16+ if v.Major > "1" || (v.Major == "1" && v.Minor >= "16") { - validatePVCClone(defaultCloneCount, rawPvcPath, rawAppPath, pvcBlockSmartClonePath, appBlockSmartClonePath, noPVCValidation, f) + validatePVCClone( + defaultCloneCount, + rawPvcPath, + rawAppPath, + pvcBlockSmartClonePath, + appBlockSmartClonePath, + noPVCValidation, + f) } }) By("create/delete multiple PVCs and Apps", func() { @@ -928,7 +1022,13 @@ var _ = Describe("RBD", func() { if err != nil { e2elog.Failf("failed to delete storageclass with error %v", err) } - err = createRBDStorageClass(f.ClientSet, f, defaultSCName, nil, map[string]string{"csi.storage.k8s.io/fstype": "xfs"}, deletePolicy) + err = createRBDStorageClass( + f.ClientSet, + f, + defaultSCName, + nil, + map[string]string{"csi.storage.k8s.io/fstype": "xfs"}, + deletePolicy) if err != nil { e2elog.Failf("failed to create storageclass with error %v", err) } @@ -998,7 +1098,13 @@ var _ = Describe("RBD", func() { if err != nil { e2elog.Failf("failed to delete storageclass with error %v", err) } - err = createRBDStorageClass(f.ClientSet, f, defaultSCName, nil, map[string]string{"volumeNamePrefix": volumeNamePrefix}, deletePolicy) + err = createRBDStorageClass( + f.ClientSet, + f, + defaultSCName, + nil, + map[string]string{"volumeNamePrefix": volumeNamePrefix}, + deletePolicy) if err != nil { e2elog.Failf("failed to create storageclass with error %v", err) } @@ -1205,7 +1311,13 @@ var _ = Describe("RBD", func() { if err != nil { e2elog.Failf("failed to delete storageclass with error %v", err) } - err = createRBDStorageClass(f.ClientSet, f, defaultSCName, map[string]string{rbdMountOptions: "debug,invalidOption"}, nil, deletePolicy) + err = createRBDStorageClass( + f.ClientSet, + f, + defaultSCName, + map[string]string{rbdMountOptions: "debug,invalidOption"}, + nil, + deletePolicy) if err != nil { e2elog.Failf("failed to create storageclass with error %v", err) } @@ -1401,7 +1513,11 @@ var _ = Describe("RBD", func() { } filePath := appClone.Spec.Containers[0].VolumeMounts[0].MountPath + "/test" - _, stdErr := execCommandInPodAndAllowFail(f, fmt.Sprintf("echo 'Hello World' > %s", filePath), appClone.Namespace, &opt) + _, stdErr := execCommandInPodAndAllowFail( + f, + fmt.Sprintf("echo 'Hello World' > %s", filePath), + appClone.Namespace, + &opt) readOnlyErr := fmt.Sprintf("cannot create %s: Read-only file system", filePath) if !strings.Contains(stdErr, readOnlyErr) { e2elog.Failf(stdErr) @@ -1577,7 +1693,11 @@ var _ = Describe("RBD", func() { updateConfigMap("e2e-ns") // create rbd provisioner secret - key, err := createCephUser(f, keyringRBDNamespaceProvisionerUsername, rbdProvisionerCaps(defaultRBDPool, radosNamespace)) + key, err := createCephUser( + f, + keyringRBDNamespaceProvisionerUsername, + rbdProvisionerCaps(defaultRBDPool, radosNamespace), + ) if err != nil { e2elog.Failf("failed to create user %s with error %v", keyringRBDNamespaceProvisionerUsername, err) } @@ -1586,7 +1706,10 @@ var _ = Describe("RBD", func() { e2elog.Failf("failed to create provisioner secret with error %v", err) } // create rbd plugin secret - key, err = createCephUser(f, keyringRBDNamespaceNodePluginUsername, rbdNodePluginCaps(defaultRBDPool, radosNamespace)) + key, err = createCephUser( + f, + keyringRBDNamespaceNodePluginUsername, + rbdNodePluginCaps(defaultRBDPool, radosNamespace)) if err != nil { e2elog.Failf("failed to create user %s with error %v", keyringRBDNamespaceNodePluginUsername, err) } @@ -1704,7 +1827,9 @@ var _ = Describe("RBD", func() { if err != nil { e2elog.Failf("failed to delete user %s with error %v", keyringRBDNamespaceProvisionerUsername, err) } - err = c.CoreV1().Secrets(cephCSINamespace).Delete(context.TODO(), rbdNamespaceProvisionerSecretName, metav1.DeleteOptions{}) + err = c.CoreV1(). + Secrets(cephCSINamespace). + Delete(context.TODO(), rbdNamespaceProvisionerSecretName, metav1.DeleteOptions{}) if err != nil { e2elog.Failf("failed to delete provisioner secret with error %v", err) } @@ -1713,7 +1838,9 @@ var _ = Describe("RBD", func() { if err != nil { e2elog.Failf("failed to delete user %s with error %v", keyringRBDNamespaceNodePluginUsername, err) } - err = c.CoreV1().Secrets(cephCSINamespace).Delete(context.TODO(), rbdNamespaceNodePluginSecretName, metav1.DeleteOptions{}) + err = c.CoreV1(). + Secrets(cephCSINamespace). + Delete(context.TODO(), rbdNamespaceNodePluginSecretName, metav1.DeleteOptions{}) if err != nil { e2elog.Failf("failed to delete node secret with error %v", err) } @@ -1761,7 +1888,11 @@ var _ = Describe("RBD", func() { } filePath := app.Spec.Containers[0].VolumeMounts[0].MountPath + "/test" - _, stdErr := execCommandInPodAndAllowFail(f, fmt.Sprintf("echo 'Hello World' > %s", filePath), app.Namespace, &opt) + _, stdErr := execCommandInPodAndAllowFail( + f, + fmt.Sprintf("echo 'Hello World' > %s", filePath), + app.Namespace, + &opt) readOnlyErr := fmt.Sprintf("cannot create %s: Read-only file system", filePath) if !strings.Contains(stdErr, readOnlyErr) { e2elog.Failf(stdErr)