e2e: make use of ceph users created in e2e

This commit adds support to create and delete
new ceph users to test various functionalities.

Signed-off-by: Madhu Rajanna <madhupr007@gmail.com>

e2e/ceph_user.go

@@ -0,0 +1,93 @@
+package e2e
+
+import (
+	"fmt"
+	"strings"
+
+	"k8s.io/kubernetes/test/e2e/framework"
+)
+
+// #nosec because of the word `Secret`
+const (
+	// ceph user names
+	keyringRBDProvisionerUsername          = "cephcsi-rbd-provisioner"
+	keyringRBDNodePluginUsername           = "cephcsi-rbd-node"
+	keyringRBDNamespaceProvisionerUsername = "cephcsi-rbd-ns-provisioner"
+	keyringRBDNamespaceNodePluginUsername  = "cephcsi-rbd-ns-node"
+	keyringCephFSProvisionerUsername       = "cephcsi-cephfs-provisioner"
+	keyringCephFSNodePluginUsername        = "cephcsi-cephfs-node"
+	// secret names
+	rbdNodePluginSecretName           = "cephcsi-rbd-node"
+	rbdProvisionerSecretName          = "cephcsi-rbd-provisioner"
+	rbdNamespaceNodePluginSecretName  = "cephcsi-rbd-ns-node"
+	rbdNamespaceProvisionerSecretName = "cephcsi-rbd-ns-provisioner"
+	cephFSNodePluginSecretName        = "cephcsi-cephfs-node"
+	cephFSProvisionerSecretName       = "cephcsi-cephfs-provisioner"
+)
+
+// refer https://github.com/ceph/ceph-csi/blob/devel/docs/capabilities.md#rbd
+// for RBD caps.
+func rbdNodePluginCaps(pool, rbdNamespace string) []string {
+	caps := []string{
+		"mon", "'profile rbd'",
+		"mgr", "'allow rw'",
+	}
+	if rbdNamespace == "" {
+		caps = append(caps, "osd", "'profile rbd'")
+	} else {
+		caps = append(caps, fmt.Sprintf("osd 'profile rbd pool=%s namespace=%s'", pool, rbdNamespace))
+	}
+	return caps
+}
+
+func rbdProvisionerCaps(pool, rbdNamespace string) []string {
+	caps := []string{
+		"mon", "'profile rbd'",
+		"mgr", "'allow rw'",
+	}
+	if rbdNamespace == "" {
+		caps = append(caps, "osd", "'profile rbd'")
+	} else {
+		caps = append(caps, fmt.Sprintf("osd 'profile rbd pool=%s namespace=%s'", pool, rbdNamespace))
+	}
+	return caps
+}
+
+// refer https://github.com/ceph/ceph-csi/blob/devel/docs/capabilities.md#rbd
+// for cephFS caps.
+func cephFSNodePluginCaps() []string {
+	caps := []string{
+		"mon", "'allow r'",
+		"mgr", "'allow rw'",
+		"osd", "'allow rw tag cephfs *=*'",
+		"mds", "'allow rw'",
+	}
+	return caps
+}
+
+func cephFSProvisionerCaps() []string {
+	caps := []string{
+		"mon", "'allow r'",
+		"mgr", "'allow rw'",
+		"osd", "'allow rw tag cephfs metadata=*'",
+	}
+	return caps
+}
+
+func createCephUser(f *framework.Framework, user string, caps []string) (string, error) {
+	cmd := fmt.Sprintf("ceph auth get-or-create-key client.%s %s", user, strings.Join(caps, " "))
+	stdOut, stdErr, err := execCommandInToolBoxPod(f, cmd, rookNamespace)
+	if err != nil {
+		return "", err
+	}
+	if stdErr != "" {
+		return "", fmt.Errorf("failed to create user %s with error %v", cmd, stdErr)
+	}
+	return strings.TrimSpace(stdOut), nil
+}
+
+func deleteCephUser(f *framework.Framework, user string) error {
+	cmd := fmt.Sprintf("ceph auth del client.%s", user)
+	_, _, err := execCommandInToolBoxPod(f, cmd, rookNamespace)
+	return err
+}

e2e/cephfs.go

@@ -1,6 +1,7 @@
 package e2e
 
 import (
+	"context"
 	"fmt"
 	"strings"
 	"sync"
@@ -170,9 +171,23 @@ var _ = Describe("cephfs", func() {
 		if err != nil {
 			e2elog.Failf("failed to create configmap with error %v", err)
 		}
-		err = createCephfsSecret(f.ClientSet, f)
+		// create cephFS provisioner secret
+		key, err := createCephUser(f, keyringCephFSProvisionerUsername, cephFSProvisionerCaps())
 		if err != nil {
-			e2elog.Failf("failed to create secret with error %v", err)
+			e2elog.Failf("failed to create user %s with error %v", keyringCephFSProvisionerUsername, err)
+		}
+		err = createCephfsSecret(f, cephFSProvisionerSecretName, keyringCephFSProvisionerUsername, key)
+		if err != nil {
+			e2elog.Failf("failed to create provisioner secret with error %v", err)
+		}
+		// create cephFS plugin secret
+		key, err = createCephUser(f, keyringCephFSNodePluginUsername, cephFSNodePluginCaps())
+		if err != nil {
+			e2elog.Failf("failed to create user %s with error %v", keyringCephFSNodePluginUsername, err)
+		}
+		err = createCephfsSecret(f, cephFSNodePluginSecretName, keyringCephFSNodePluginUsername, key)
+		if err != nil {
+			e2elog.Failf("failed to create node secret with error %v", err)
 		}
 	})
 
@@ -195,9 +210,13 @@ var _ = Describe("cephfs", func() {
 		if err != nil {
 			e2elog.Failf("failed to delete configmap with error %v", err)
 		}
-		err = deleteResource(cephfsExamplePath + "secret.yaml")
+		err = c.CoreV1().Secrets(cephCSINamespace).Delete(context.TODO(), cephFSProvisionerSecretName, metav1.DeleteOptions{})
 		if err != nil {
-			e2elog.Failf("failed to delete secret with error %v", err)
+			e2elog.Failf("failed to delete provisioner secret with error %v", err)
+		}
+		err = c.CoreV1().Secrets(cephCSINamespace).Delete(context.TODO(), cephFSNodePluginSecretName, metav1.DeleteOptions{})
+		if err != nil {
+			e2elog.Failf("failed to delete node secret with error %v", err)
 		}
 		err = deleteResource(cephfsExamplePath + "storageclass.yaml")
 		if err != nil {
@@ -1015,6 +1034,16 @@ var _ = Describe("cephfs", func() {
 					e2elog.Failf("failed to delete PVC with error %v", err)
 				}
 			})
+			// delete cephFS provisioner secret
+			err := deleteCephUser(f, keyringCephFSProvisionerUsername)
+			if err != nil {
+				e2elog.Failf("failed to delete user %s with error %v", keyringCephFSProvisionerUsername, err)
+			}
+			// delete cephFS plugin secret
+			err = deleteCephUser(f, keyringCephFSNodePluginUsername)
+			if err != nil {
+				e2elog.Failf("failed to delete user %s with error %v", keyringCephFSNodePluginUsername, err)
+			}
 
 		})
 	})

e2e/cephfs_helper.go

@@ -41,14 +41,14 @@ func createCephfsStorageClass(c kubernetes.Interface, f *framework.Framework, en
 		return err
 	}
 	sc.Parameters["fsName"] = "myfs"
-	sc.Parameters["csi.storage.k8s.io/provisioner-secret-namespace"] = rookNamespace
+	sc.Parameters["csi.storage.k8s.io/provisioner-secret-namespace"] = cephCSINamespace
-	sc.Parameters["csi.storage.k8s.io/provisioner-secret-name"] = cephfsProvisionerSecretName
+	sc.Parameters["csi.storage.k8s.io/provisioner-secret-name"] = cephFSProvisionerSecretName
 
-	sc.Parameters["csi.storage.k8s.io/controller-expand-secret-namespace"] = rookNamespace
+	sc.Parameters["csi.storage.k8s.io/controller-expand-secret-namespace"] = cephCSINamespace
-	sc.Parameters["csi.storage.k8s.io/controller-expand-secret-name"] = cephfsProvisionerSecretName
+	sc.Parameters["csi.storage.k8s.io/controller-expand-secret-name"] = cephFSProvisionerSecretName
 
-	sc.Parameters["csi.storage.k8s.io/node-stage-secret-namespace"] = rookNamespace
+	sc.Parameters["csi.storage.k8s.io/node-stage-secret-namespace"] = cephCSINamespace
-	sc.Parameters["csi.storage.k8s.io/node-stage-secret-name"] = cephfsNodePluginSecretName
+	sc.Parameters["csi.storage.k8s.io/node-stage-secret-name"] = cephFSNodePluginSecretName
 
 	if enablePool {
 		sc.Parameters["pool"] = "myfs-data0"
@@ -80,25 +80,21 @@ func createCephfsStorageClass(c kubernetes.Interface, f *framework.Framework, en
 	return err
 }
 
-func createCephfsSecret(c kubernetes.Interface, f *framework.Framework) error {
+func createCephfsSecret(f *framework.Framework, secretName, userName, userKey string) error {
 	scPath := fmt.Sprintf("%s/%s", cephfsExamplePath, "secret.yaml")
 	sc, err := getSecret(scPath)
 	if err != nil {
 		return err
 	}
-	adminKey, stdErr, err := execCommandInToolBoxPod(f, "ceph auth get-key client.admin", rookNamespace)
+	if secretName != "" {
-	if err != nil {
+		sc.Name = secretName
-		return err
 	}
-	if stdErr != "" {
+	sc.StringData["adminID"] = userName
-		return fmt.Errorf("error getting admin key %v", stdErr)
+	sc.StringData["adminKey"] = userKey
-	}
-	sc.StringData["adminID"] = adminUser
-	sc.StringData["adminKey"] = adminKey
 	delete(sc.StringData, "userID")
 	delete(sc.StringData, "userKey")
 	sc.Namespace = cephCSINamespace
-	_, err = c.CoreV1().Secrets(cephCSINamespace).Create(context.TODO(), &sc, metav1.CreateOptions{})
+	_, err = f.ClientSet.CoreV1().Secrets(cephCSINamespace).Create(context.TODO(), &sc, metav1.CreateOptions{})
 	return err
 }
 

e2e/rbd.go

@@ -171,9 +171,23 @@ var _ = Describe("RBD", func() {
 		if err != nil {
 			e2elog.Failf("failed to create storageclass with error %v", err)
 		}
-		err = createRBDSecret(f.ClientSet, f)
+		// create rbd provisioner secret
+		key, err := createCephUser(f, keyringRBDProvisionerUsername, rbdProvisionerCaps("", ""))
 		if err != nil {
-			e2elog.Failf("failed to create secret with error %v", err)
+			e2elog.Failf("failed to create user %s with error %v", keyringRBDProvisionerUsername, err)
+		}
+		err = createRBDSecret(f, rbdProvisionerSecretName, keyringRBDProvisionerUsername, key)
+		if err != nil {
+			e2elog.Failf("failed to create provisioner secret with error %v", err)
+		}
+		// create rbd plugin secret
+		key, err = createCephUser(f, keyringRBDNodePluginUsername, rbdNodePluginCaps("", ""))
+		if err != nil {
+			e2elog.Failf("failed to create user %s with error %v", keyringRBDNodePluginUsername, err)
+		}
+		err = createRBDSecret(f, rbdNodePluginSecretName, keyringRBDNodePluginUsername, key)
+		if err != nil {
+			e2elog.Failf("failed to create node secret with error %v", err)
 		}
 		deployVault(f.ClientSet, deployTimeout)
 	})
@@ -198,9 +212,13 @@ var _ = Describe("RBD", func() {
 		if err != nil {
 			e2elog.Failf("failed to delete configmap with error %v", err)
 		}
-		err = deleteResource(rbdExamplePath + "secret.yaml")
+		err = c.CoreV1().Secrets(cephCSINamespace).Delete(context.TODO(), rbdProvisionerSecretName, metav1.DeleteOptions{})
 		if err != nil {
-			e2elog.Failf("failed to delete secret with error %v", err)
+			e2elog.Failf("failed to delete provisioner secret with error %v", err)
+		}
+		err = c.CoreV1().Secrets(cephCSINamespace).Delete(context.TODO(), rbdNodePluginSecretName, metav1.DeleteOptions{})
+		if err != nil {
+			e2elog.Failf("failed to delete node secret with error %v", err)
 		}
 		err = deleteResource(rbdExamplePath + "storageclass.yaml")
 		if err != nil {
@@ -1171,8 +1189,44 @@ var _ = Describe("RBD", func() {
 				}
 
 				updateConfigMap("e2e-ns")
+				// create rbd provisioner secret
+				key, err := createCephUser(f, keyringRBDNamespaceProvisionerUsername, rbdProvisionerCaps(defaultRBDPool, radosNamespace))
+				if err != nil {
+					e2elog.Failf("failed to create user %s with error %v", keyringRBDNamespaceProvisionerUsername, err)
+				}
+				err = createRBDSecret(f, rbdNamespaceProvisionerSecretName, keyringRBDNamespaceProvisionerUsername, key)
+				if err != nil {
+					e2elog.Failf("failed to create provisioner secret with error %v", err)
+				}
+				// create rbd plugin secret
+				key, err = createCephUser(f, keyringRBDNamespaceNodePluginUsername, rbdNodePluginCaps(defaultRBDPool, radosNamespace))
+				if err != nil {
+					e2elog.Failf("failed to create user %s with error %v", keyringRBDNamespaceNodePluginUsername, err)
+				}
+				err = createRBDSecret(f, rbdNamespaceNodePluginSecretName, keyringRBDNamespaceNodePluginUsername, key)
+				if err != nil {
+					e2elog.Failf("failed to create node secret with error %v", err)
+				}
 
-				err := validateImageOwner(pvcPath, f)
+				err = deleteResource(rbdExamplePath + "storageclass.yaml")
+				if err != nil {
+					e2elog.Failf("failed to delete storageclass with error %v", err)
+				}
+				param := make(map[string]string)
+				// override existing secrets
+				param["csi.storage.k8s.io/provisioner-secret-namespace"] = cephCSINamespace
+				param["csi.storage.k8s.io/provisioner-secret-name"] = rbdProvisionerSecretName
+				param["csi.storage.k8s.io/controller-expand-secret-namespace"] = cephCSINamespace
+				param["csi.storage.k8s.io/controller-expand-secret-name"] = rbdProvisionerSecretName
+				param["csi.storage.k8s.io/node-stage-secret-namespace"] = cephCSINamespace
+				param["csi.storage.k8s.io/node-stage-secret-name"] = rbdNodePluginSecretName
+
+				err = createRBDStorageClass(f.ClientSet, f, nil, param, deletePolicy)
+				if err != nil {
+					e2elog.Failf("failed to create storageclass with error %v", err)
+				}
+
+				err = validateImageOwner(pvcPath, f)
 				if err != nil {
 					e2elog.Failf("failed to validate owner of pvc with error %v", err)
 				}
@@ -1197,7 +1251,7 @@ var _ = Describe("RBD", func() {
 				// Resize Filesystem PVC and check application directory size
 				// Resize 0.3.0 is only supported from v1.15+
 				if k8sVersionGreaterEquals(f.ClientSet, 1, 15) {
-					err := resizePVCAndValidateSize(pvcPath, appPath, f)
+					err = resizePVCAndValidateSize(pvcPath, appPath, f)
 					if err != nil {
 						e2elog.Failf("failed to resize filesystem PVC %v", err)
 					}
@@ -1206,7 +1260,8 @@ var _ = Describe("RBD", func() {
 				// Create a PVC clone and bind it to an app within the namespace
 				// snapshot beta is only supported from v1.17+
 				if k8sVersionGreaterEquals(f.ClientSet, 1, 17) {
-					pvc, err := loadPVC(pvcPath)
+					var pvc = &v1.PersistentVolumeClaim{}
+					pvc, err = loadPVC(pvcPath)
 					if err != nil {
 						e2elog.Failf("failed to load PVC with error %v", err)
 					}
@@ -1246,6 +1301,32 @@ var _ = Describe("RBD", func() {
 					validateRBDImageCount(f, 0)
 				}
 
+				// delete RBD provisioner secret
+				err = deleteCephUser(f, keyringRBDNamespaceProvisionerUsername)
+				if err != nil {
+					e2elog.Failf("failed to delete user %s with error %v", keyringRBDNamespaceProvisionerUsername, err)
+				}
+				err = c.CoreV1().Secrets(cephCSINamespace).Delete(context.TODO(), rbdNamespaceProvisionerSecretName, metav1.DeleteOptions{})
+				if err != nil {
+					e2elog.Failf("failed to delete provisioner secret with error %v", err)
+				}
+				// delete RBD plugin secret
+				err = deleteCephUser(f, keyringRBDNamespaceNodePluginUsername)
+				if err != nil {
+					e2elog.Failf("failed to delete user %s with error %v", keyringRBDNamespaceNodePluginUsername, err)
+				}
+				err = c.CoreV1().Secrets(cephCSINamespace).Delete(context.TODO(), rbdNamespaceNodePluginSecretName, metav1.DeleteOptions{})
+				if err != nil {
+					e2elog.Failf("failed to delete node secret with error %v", err)
+				}
+				err = deleteResource(rbdExamplePath + "storageclass.yaml")
+				if err != nil {
+					e2elog.Failf("failed to delete storageclass with error %v", err)
+				}
+				err = createRBDStorageClass(f.ClientSet, f, nil, nil, deletePolicy)
+				if err != nil {
+					e2elog.Failf("failed to create storageclass with error %v", err)
+				}
 				updateConfigMap("")
 			})
 
@@ -1424,6 +1505,16 @@ var _ = Describe("RBD", func() {
 					e2elog.Failf("failed to delete PVC when pool not found with error %v", err)
 				}
 			})
+			// delete RBD provisioner secret
+			err := deleteCephUser(f, keyringRBDProvisionerUsername)
+			if err != nil {
+				e2elog.Failf("failed to delete user %s with error %v", keyringRBDProvisionerUsername, err)
+			}
+			// delete RBD plugin secret
+			err = deleteCephUser(f, keyringRBDNodePluginUsername)
+			if err != nil {
+				e2elog.Failf("failed to delete user %s with error %v", keyringRBDNodePluginUsername, err)
+			}
 		})
 	})
 })

e2e/rbd_helper.go

@@ -118,27 +118,19 @@ func createRadosNamespace(f *framework.Framework) error {
 	return nil
 }
 
-func createRBDSecret(c kubernetes.Interface, f *framework.Framework) error {
+func createRBDSecret(f *framework.Framework, secretName, userName, userKey string) error {
 	scPath := fmt.Sprintf("%s/%s", rbdExamplePath, "secret.yaml")
 	sc, err := getSecret(scPath)
 	if err != nil {
 		return err
 	}
-	adminKey, stdErr, err := execCommandInToolBoxPod(f, "ceph auth get-key client.admin", rookNamespace)
+	if secretName != "" {
-	if err != nil {
+		sc.Name = secretName
-		return err
 	}
-	if stdErr != "" {
+	sc.StringData["userID"] = userName
-		return fmt.Errorf("error getting admin key %v", stdErr)
+	sc.StringData["userKey"] = userKey
-	}
-	sc.StringData["userID"] = adminUser
-	sc.StringData["userKey"] = adminKey
 	sc.Namespace = cephCSINamespace
-	_, err = c.CoreV1().Secrets(cephCSINamespace).Create(context.TODO(), &sc, metav1.CreateOptions{})
+	_, err = f.ClientSet.CoreV1().Secrets(cephCSINamespace).Create(context.TODO(), &sc, metav1.CreateOptions{})
-	if err != nil {
-		return err
-	}
-
 	return err
 }
 

e2e/snapshot.go

@@ -117,6 +117,7 @@ func createRBDSnapshotClass(f *framework.Framework) error {
 	sc := getSnapshotClass(scPath)
 
 	sc.Parameters["csi.storage.k8s.io/snapshotter-secret-namespace"] = cephCSINamespace
+	sc.Parameters["csi.storage.k8s.io/snapshotter-secret-name"] = rbdProvisionerSecretName
 
 	fsID, stdErr, err := execCommandInToolBoxPod(f, "ceph fsid", rookNamespace)
 	if err != nil {
@@ -139,6 +140,7 @@ func createCephFSSnapshotClass(f *framework.Framework) error {
 	scPath := fmt.Sprintf("%s/%s", cephfsExamplePath, "snapshotclass.yaml")
 	sc := getSnapshotClass(scPath)
 	sc.Parameters["csi.storage.k8s.io/snapshotter-secret-namespace"] = cephCSINamespace
+	sc.Parameters["csi.storage.k8s.io/snapshotter-secret-name"] = cephFSProvisionerSecretName
 	fsID, stdErr, err := execCommandInToolBoxPod(f, "ceph fsid", rookNamespace)
 	if err != nil {
 		return err

e2e/staticpvc.go

@@ -119,7 +119,7 @@ func validateRBDStaticPV(f *framework.Framework, appPath string, isBlock bool) e
 		opt["radosNamespace"] = radosNamespace
 	}
 
-	pv := getStaticPV(pvName, rbdImageName, size, "csi-rbd-secret", cephCSINamespace, sc, "rbd.csi.ceph.com", isBlock, opt)
+	pv := getStaticPV(pvName, rbdImageName, size, rbdNodePluginSecretName, cephCSINamespace, sc, "rbd.csi.ceph.com", isBlock, opt)
 
 	_, err = c.CoreV1().PersistentVolumes().Create(context.TODO(), pv, metav1.CreateOptions{})
 	if err != nil {

e2e/upgrade-cephfs.go

@@ -64,10 +64,26 @@ var _ = Describe("CephFS Upgrade Testing", func() {
 		if err != nil {
 			e2elog.Failf("failed to create configmap with error %v", err)
 		}
-		err = createCephfsSecret(f.ClientSet, f)
+		var key string
+		// create cephFS provisioner secret
+		key, err = createCephUser(f, keyringCephFSProvisionerUsername, cephFSProvisionerCaps())
 		if err != nil {
-			e2elog.Failf("failed to create secret with error %v", err)
+			e2elog.Failf("failed to create user %s with error %v", keyringCephFSProvisionerUsername, err)
 		}
+		err = createCephfsSecret(f, cephFSProvisionerSecretName, keyringCephFSProvisionerUsername, key)
+		if err != nil {
+			e2elog.Failf("failed to create provisioner secret with error %v", err)
+		}
+		// create cephFS plugin secret
+		key, err = createCephUser(f, keyringCephFSNodePluginUsername, cephFSNodePluginCaps())
+		if err != nil {
+			e2elog.Failf("failed to create user %s with error %v", keyringCephFSNodePluginUsername, err)
+		}
+		err = createCephfsSecret(f, cephFSNodePluginSecretName, keyringCephFSNodePluginUsername, key)
+		if err != nil {
+			e2elog.Failf("failed to create node secret with error %v", err)
+		}
+
 		err = createCephFSSnapshotClass(f)
 		if err != nil {
 			e2elog.Failf("failed to create snapshotclass with error %v", err)
@@ -96,9 +112,13 @@ var _ = Describe("CephFS Upgrade Testing", func() {
 		if err != nil {
 			e2elog.Failf("failed to delete configmap with error %v", err)
 		}
-		err = deleteResource(cephfsExamplePath + "secret.yaml")
+		err = c.CoreV1().Secrets(cephCSINamespace).Delete(context.TODO(), cephFSProvisionerSecretName, metav1.DeleteOptions{})
 		if err != nil {
-			e2elog.Failf("failed to delete secret with error %v", err)
+			e2elog.Failf("failed to delete provisioner secret with error %v", err)
+		}
+		err = c.CoreV1().Secrets(cephCSINamespace).Delete(context.TODO(), cephFSNodePluginSecretName, metav1.DeleteOptions{})
+		if err != nil {
+			e2elog.Failf("failed to delete node secret with error %v", err)
 		}
 		err = deleteResource(cephfsExamplePath + "storageclass.yaml")
 		if err != nil {
@@ -373,6 +393,17 @@ var _ = Describe("CephFS Upgrade Testing", func() {
 			if err != nil {
 				e2elog.Failf("failed to delete pvc and application with error %v", err)
 			}
-		})
+			// delete cephFS provisioner secret
+			err = deleteCephUser(f, keyringCephFSProvisionerUsername)
+			if err != nil {
+				e2elog.Failf("failed to delete user %s with error %v", keyringCephFSProvisionerUsername, err)
+			}
+			// delete cephFS plugin secret
+			err = deleteCephUser(f, keyringCephFSNodePluginUsername)
+			if err != nil {
+				e2elog.Failf("failed to delete user %s with error %v", keyringCephFSNodePluginUsername, err)
+			}
+		})
+
 	})
 })

e2e/upgrade-rbd.go

@@ -67,9 +67,23 @@ var _ = Describe("RBD Upgrade Testing", func() {
 		if err != nil {
 			e2elog.Failf("failed to create storageclass with error %v", err)
 		}
-		err = createRBDSecret(f.ClientSet, f)
+		// create rbd provisioner secret
+		key, err := createCephUser(f, keyringRBDProvisionerUsername, rbdProvisionerCaps("", ""))
 		if err != nil {
-			e2elog.Failf("failed to create secret with error %v", err)
+			e2elog.Failf("failed to create user %s with error %v", keyringRBDProvisionerUsername, err)
+		}
+		err = createRBDSecret(f, rbdProvisionerSecretName, keyringRBDProvisionerUsername, key)
+		if err != nil {
+			e2elog.Failf("failed to create provisioner secret with error %v", err)
+		}
+		// create rbd plugin secret
+		key, err = createCephUser(f, keyringRBDNodePluginUsername, rbdNodePluginCaps("", ""))
+		if err != nil {
+			e2elog.Failf("failed to create user %s with error %v", keyringRBDNodePluginUsername, err)
+		}
+		err = createRBDSecret(f, rbdNodePluginSecretName, keyringRBDNodePluginUsername, key)
+		if err != nil {
+			e2elog.Failf("failed to create node secret with error %v", err)
 		}
 		err = createRBDSnapshotClass(f)
 		if err != nil {
@@ -105,9 +119,13 @@ var _ = Describe("RBD Upgrade Testing", func() {
 		if err != nil {
 			e2elog.Failf("failed to delete configmap with error %v", err)
 		}
-		err = deleteResource(rbdExamplePath + "secret.yaml")
+		err = c.CoreV1().Secrets(cephCSINamespace).Delete(context.TODO(), rbdProvisionerSecretName, metav1.DeleteOptions{})
 		if err != nil {
-			e2elog.Failf("failed to delete secret with error %v", err)
+			e2elog.Failf("failed to delete provisioner secret with error %v", err)
+		}
+		err = c.CoreV1().Secrets(cephCSINamespace).Delete(context.TODO(), rbdNodePluginSecretName, metav1.DeleteOptions{})
+		if err != nil {
+			e2elog.Failf("failed to delete node secret with error %v", err)
 		}
 		err = deleteResource(rbdExamplePath + "storageclass.yaml")
 		if err != nil {
@@ -381,6 +399,17 @@ var _ = Describe("RBD Upgrade Testing", func() {
 					e2elog.Failf("failed to delete pvc and application with error %v", err)
 				}
 			})
-		})
+			// delete RBD provisioner secret
+			err := deleteCephUser(f, keyringRBDProvisionerUsername)
+			if err != nil {
+				e2elog.Failf("failed to delete user %s with error %v", keyringRBDProvisionerUsername, err)
+			}
+			// delete RBD plugin secret
+			err = deleteCephUser(f, keyringRBDNodePluginUsername)
+			if err != nil {
+				e2elog.Failf("failed to delete user %s with error %v", keyringRBDNodePluginUsername, err)
+			}
+		})
+
 	})
 })

e2e/utils.go

@@ -28,16 +28,6 @@ const (
 	defaultNs     = "default"
 	vaultSecretNs = "/secret/ceph-csi/"
 
-	// rook created cephfs user
-	cephfsNodePluginSecretName  = "rook-csi-cephfs-node"
-	cephfsProvisionerSecretName = "rook-csi-cephfs-provisioner"
-
-	// Secret created inside the cephCSINamespace, can be modified. The
-	// Rook secrets get reconciled and changes are undone (needed for
-	// encryption).
-	rbdNodePluginSecretName  = "csi-rbd-secret"
-	rbdProvisionerSecretName = "csi-rbd-secret"
-
 	rookTolBoxPodLabel = "app=rook-ceph-tools"
 	rbdmountOptions    = "mountOptions"