diff --git a/e2e/ceph_user.go b/e2e/ceph_user.go new file mode 100644 index 000000000..cecfef1bc --- /dev/null +++ b/e2e/ceph_user.go @@ -0,0 +1,93 @@ +package e2e + +import ( + "fmt" + "strings" + + "k8s.io/kubernetes/test/e2e/framework" +) + +// #nosec because of the word `Secret` +const ( + // ceph user names + keyringRBDProvisionerUsername = "cephcsi-rbd-provisioner" + keyringRBDNodePluginUsername = "cephcsi-rbd-node" + keyringRBDNamespaceProvisionerUsername = "cephcsi-rbd-ns-provisioner" + keyringRBDNamespaceNodePluginUsername = "cephcsi-rbd-ns-node" + keyringCephFSProvisionerUsername = "cephcsi-cephfs-provisioner" + keyringCephFSNodePluginUsername = "cephcsi-cephfs-node" + // secret names + rbdNodePluginSecretName = "cephcsi-rbd-node" + rbdProvisionerSecretName = "cephcsi-rbd-provisioner" + rbdNamespaceNodePluginSecretName = "cephcsi-rbd-ns-node" + rbdNamespaceProvisionerSecretName = "cephcsi-rbd-ns-provisioner" + cephFSNodePluginSecretName = "cephcsi-cephfs-node" + cephFSProvisionerSecretName = "cephcsi-cephfs-provisioner" +) + +// refer https://github.com/ceph/ceph-csi/blob/devel/docs/capabilities.md#rbd +// for RBD caps. +func rbdNodePluginCaps(pool, rbdNamespace string) []string { + caps := []string{ + "mon", "'profile rbd'", + "mgr", "'allow rw'", + } + if rbdNamespace == "" { + caps = append(caps, "osd", "'profile rbd'") + } else { + caps = append(caps, fmt.Sprintf("osd 'profile rbd pool=%s namespace=%s'", pool, rbdNamespace)) + } + return caps +} + +func rbdProvisionerCaps(pool, rbdNamespace string) []string { + caps := []string{ + "mon", "'profile rbd'", + "mgr", "'allow rw'", + } + if rbdNamespace == "" { + caps = append(caps, "osd", "'profile rbd'") + } else { + caps = append(caps, fmt.Sprintf("osd 'profile rbd pool=%s namespace=%s'", pool, rbdNamespace)) + } + return caps +} + +// refer https://github.com/ceph/ceph-csi/blob/devel/docs/capabilities.md#rbd +// for cephFS caps. +func cephFSNodePluginCaps() []string { + caps := []string{ + "mon", "'allow r'", + "mgr", "'allow rw'", + "osd", "'allow rw tag cephfs *=*'", + "mds", "'allow rw'", + } + return caps +} + +func cephFSProvisionerCaps() []string { + caps := []string{ + "mon", "'allow r'", + "mgr", "'allow rw'", + "osd", "'allow rw tag cephfs metadata=*'", + } + return caps +} + +func createCephUser(f *framework.Framework, user string, caps []string) (string, error) { + cmd := fmt.Sprintf("ceph auth get-or-create-key client.%s %s", user, strings.Join(caps, " ")) + stdOut, stdErr, err := execCommandInToolBoxPod(f, cmd, rookNamespace) + if err != nil { + return "", err + } + if stdErr != "" { + return "", fmt.Errorf("failed to create user %s with error %v", cmd, stdErr) + } + return strings.TrimSpace(stdOut), nil +} + +func deleteCephUser(f *framework.Framework, user string) error { + cmd := fmt.Sprintf("ceph auth del client.%s", user) + _, _, err := execCommandInToolBoxPod(f, cmd, rookNamespace) + return err +} diff --git a/e2e/cephfs.go b/e2e/cephfs.go index ce78ea652..973caa29d 100644 --- a/e2e/cephfs.go +++ b/e2e/cephfs.go @@ -1,6 +1,7 @@ package e2e import ( + "context" "fmt" "strings" "sync" @@ -170,9 +171,23 @@ var _ = Describe("cephfs", func() { if err != nil { e2elog.Failf("failed to create configmap with error %v", err) } - err = createCephfsSecret(f.ClientSet, f) + // create cephFS provisioner secret + key, err := createCephUser(f, keyringCephFSProvisionerUsername, cephFSProvisionerCaps()) if err != nil { - e2elog.Failf("failed to create secret with error %v", err) + e2elog.Failf("failed to create user %s with error %v", keyringCephFSProvisionerUsername, err) + } + err = createCephfsSecret(f, cephFSProvisionerSecretName, keyringCephFSProvisionerUsername, key) + if err != nil { + e2elog.Failf("failed to create provisioner secret with error %v", err) + } + // create cephFS plugin secret + key, err = createCephUser(f, keyringCephFSNodePluginUsername, cephFSNodePluginCaps()) + if err != nil { + e2elog.Failf("failed to create user %s with error %v", keyringCephFSNodePluginUsername, err) + } + err = createCephfsSecret(f, cephFSNodePluginSecretName, keyringCephFSNodePluginUsername, key) + if err != nil { + e2elog.Failf("failed to create node secret with error %v", err) } }) @@ -195,9 +210,13 @@ var _ = Describe("cephfs", func() { if err != nil { e2elog.Failf("failed to delete configmap with error %v", err) } - err = deleteResource(cephfsExamplePath + "secret.yaml") + err = c.CoreV1().Secrets(cephCSINamespace).Delete(context.TODO(), cephFSProvisionerSecretName, metav1.DeleteOptions{}) if err != nil { - e2elog.Failf("failed to delete secret with error %v", err) + e2elog.Failf("failed to delete provisioner secret with error %v", err) + } + err = c.CoreV1().Secrets(cephCSINamespace).Delete(context.TODO(), cephFSNodePluginSecretName, metav1.DeleteOptions{}) + if err != nil { + e2elog.Failf("failed to delete node secret with error %v", err) } err = deleteResource(cephfsExamplePath + "storageclass.yaml") if err != nil { @@ -1015,6 +1034,16 @@ var _ = Describe("cephfs", func() { e2elog.Failf("failed to delete PVC with error %v", err) } }) + // delete cephFS provisioner secret + err := deleteCephUser(f, keyringCephFSProvisionerUsername) + if err != nil { + e2elog.Failf("failed to delete user %s with error %v", keyringCephFSProvisionerUsername, err) + } + // delete cephFS plugin secret + err = deleteCephUser(f, keyringCephFSNodePluginUsername) + if err != nil { + e2elog.Failf("failed to delete user %s with error %v", keyringCephFSNodePluginUsername, err) + } }) }) diff --git a/e2e/cephfs_helper.go b/e2e/cephfs_helper.go index 98ffe77dd..d24da5dd3 100644 --- a/e2e/cephfs_helper.go +++ b/e2e/cephfs_helper.go @@ -41,14 +41,14 @@ func createCephfsStorageClass(c kubernetes.Interface, f *framework.Framework, en return err } sc.Parameters["fsName"] = "myfs" - sc.Parameters["csi.storage.k8s.io/provisioner-secret-namespace"] = rookNamespace - sc.Parameters["csi.storage.k8s.io/provisioner-secret-name"] = cephfsProvisionerSecretName + sc.Parameters["csi.storage.k8s.io/provisioner-secret-namespace"] = cephCSINamespace + sc.Parameters["csi.storage.k8s.io/provisioner-secret-name"] = cephFSProvisionerSecretName - sc.Parameters["csi.storage.k8s.io/controller-expand-secret-namespace"] = rookNamespace - sc.Parameters["csi.storage.k8s.io/controller-expand-secret-name"] = cephfsProvisionerSecretName + sc.Parameters["csi.storage.k8s.io/controller-expand-secret-namespace"] = cephCSINamespace + sc.Parameters["csi.storage.k8s.io/controller-expand-secret-name"] = cephFSProvisionerSecretName - sc.Parameters["csi.storage.k8s.io/node-stage-secret-namespace"] = rookNamespace - sc.Parameters["csi.storage.k8s.io/node-stage-secret-name"] = cephfsNodePluginSecretName + sc.Parameters["csi.storage.k8s.io/node-stage-secret-namespace"] = cephCSINamespace + sc.Parameters["csi.storage.k8s.io/node-stage-secret-name"] = cephFSNodePluginSecretName if enablePool { sc.Parameters["pool"] = "myfs-data0" @@ -80,25 +80,21 @@ func createCephfsStorageClass(c kubernetes.Interface, f *framework.Framework, en return err } -func createCephfsSecret(c kubernetes.Interface, f *framework.Framework) error { +func createCephfsSecret(f *framework.Framework, secretName, userName, userKey string) error { scPath := fmt.Sprintf("%s/%s", cephfsExamplePath, "secret.yaml") sc, err := getSecret(scPath) if err != nil { return err } - adminKey, stdErr, err := execCommandInToolBoxPod(f, "ceph auth get-key client.admin", rookNamespace) - if err != nil { - return err + if secretName != "" { + sc.Name = secretName } - if stdErr != "" { - return fmt.Errorf("error getting admin key %v", stdErr) - } - sc.StringData["adminID"] = adminUser - sc.StringData["adminKey"] = adminKey + sc.StringData["adminID"] = userName + sc.StringData["adminKey"] = userKey delete(sc.StringData, "userID") delete(sc.StringData, "userKey") sc.Namespace = cephCSINamespace - _, err = c.CoreV1().Secrets(cephCSINamespace).Create(context.TODO(), &sc, metav1.CreateOptions{}) + _, err = f.ClientSet.CoreV1().Secrets(cephCSINamespace).Create(context.TODO(), &sc, metav1.CreateOptions{}) return err } diff --git a/e2e/rbd.go b/e2e/rbd.go index 14f39bf73..00d931dbc 100644 --- a/e2e/rbd.go +++ b/e2e/rbd.go @@ -171,9 +171,23 @@ var _ = Describe("RBD", func() { if err != nil { e2elog.Failf("failed to create storageclass with error %v", err) } - err = createRBDSecret(f.ClientSet, f) + // create rbd provisioner secret + key, err := createCephUser(f, keyringRBDProvisionerUsername, rbdProvisionerCaps("", "")) if err != nil { - e2elog.Failf("failed to create secret with error %v", err) + e2elog.Failf("failed to create user %s with error %v", keyringRBDProvisionerUsername, err) + } + err = createRBDSecret(f, rbdProvisionerSecretName, keyringRBDProvisionerUsername, key) + if err != nil { + e2elog.Failf("failed to create provisioner secret with error %v", err) + } + // create rbd plugin secret + key, err = createCephUser(f, keyringRBDNodePluginUsername, rbdNodePluginCaps("", "")) + if err != nil { + e2elog.Failf("failed to create user %s with error %v", keyringRBDNodePluginUsername, err) + } + err = createRBDSecret(f, rbdNodePluginSecretName, keyringRBDNodePluginUsername, key) + if err != nil { + e2elog.Failf("failed to create node secret with error %v", err) } deployVault(f.ClientSet, deployTimeout) }) @@ -198,9 +212,13 @@ var _ = Describe("RBD", func() { if err != nil { e2elog.Failf("failed to delete configmap with error %v", err) } - err = deleteResource(rbdExamplePath + "secret.yaml") + err = c.CoreV1().Secrets(cephCSINamespace).Delete(context.TODO(), rbdProvisionerSecretName, metav1.DeleteOptions{}) if err != nil { - e2elog.Failf("failed to delete secret with error %v", err) + e2elog.Failf("failed to delete provisioner secret with error %v", err) + } + err = c.CoreV1().Secrets(cephCSINamespace).Delete(context.TODO(), rbdNodePluginSecretName, metav1.DeleteOptions{}) + if err != nil { + e2elog.Failf("failed to delete node secret with error %v", err) } err = deleteResource(rbdExamplePath + "storageclass.yaml") if err != nil { @@ -1171,8 +1189,44 @@ var _ = Describe("RBD", func() { } updateConfigMap("e2e-ns") + // create rbd provisioner secret + key, err := createCephUser(f, keyringRBDNamespaceProvisionerUsername, rbdProvisionerCaps(defaultRBDPool, radosNamespace)) + if err != nil { + e2elog.Failf("failed to create user %s with error %v", keyringRBDNamespaceProvisionerUsername, err) + } + err = createRBDSecret(f, rbdNamespaceProvisionerSecretName, keyringRBDNamespaceProvisionerUsername, key) + if err != nil { + e2elog.Failf("failed to create provisioner secret with error %v", err) + } + // create rbd plugin secret + key, err = createCephUser(f, keyringRBDNamespaceNodePluginUsername, rbdNodePluginCaps(defaultRBDPool, radosNamespace)) + if err != nil { + e2elog.Failf("failed to create user %s with error %v", keyringRBDNamespaceNodePluginUsername, err) + } + err = createRBDSecret(f, rbdNamespaceNodePluginSecretName, keyringRBDNamespaceNodePluginUsername, key) + if err != nil { + e2elog.Failf("failed to create node secret with error %v", err) + } - err := validateImageOwner(pvcPath, f) + err = deleteResource(rbdExamplePath + "storageclass.yaml") + if err != nil { + e2elog.Failf("failed to delete storageclass with error %v", err) + } + param := make(map[string]string) + // override existing secrets + param["csi.storage.k8s.io/provisioner-secret-namespace"] = cephCSINamespace + param["csi.storage.k8s.io/provisioner-secret-name"] = rbdProvisionerSecretName + param["csi.storage.k8s.io/controller-expand-secret-namespace"] = cephCSINamespace + param["csi.storage.k8s.io/controller-expand-secret-name"] = rbdProvisionerSecretName + param["csi.storage.k8s.io/node-stage-secret-namespace"] = cephCSINamespace + param["csi.storage.k8s.io/node-stage-secret-name"] = rbdNodePluginSecretName + + err = createRBDStorageClass(f.ClientSet, f, nil, param, deletePolicy) + if err != nil { + e2elog.Failf("failed to create storageclass with error %v", err) + } + + err = validateImageOwner(pvcPath, f) if err != nil { e2elog.Failf("failed to validate owner of pvc with error %v", err) } @@ -1197,7 +1251,7 @@ var _ = Describe("RBD", func() { // Resize Filesystem PVC and check application directory size // Resize 0.3.0 is only supported from v1.15+ if k8sVersionGreaterEquals(f.ClientSet, 1, 15) { - err := resizePVCAndValidateSize(pvcPath, appPath, f) + err = resizePVCAndValidateSize(pvcPath, appPath, f) if err != nil { e2elog.Failf("failed to resize filesystem PVC %v", err) } @@ -1206,7 +1260,8 @@ var _ = Describe("RBD", func() { // Create a PVC clone and bind it to an app within the namespace // snapshot beta is only supported from v1.17+ if k8sVersionGreaterEquals(f.ClientSet, 1, 17) { - pvc, err := loadPVC(pvcPath) + var pvc = &v1.PersistentVolumeClaim{} + pvc, err = loadPVC(pvcPath) if err != nil { e2elog.Failf("failed to load PVC with error %v", err) } @@ -1246,6 +1301,32 @@ var _ = Describe("RBD", func() { validateRBDImageCount(f, 0) } + // delete RBD provisioner secret + err = deleteCephUser(f, keyringRBDNamespaceProvisionerUsername) + if err != nil { + e2elog.Failf("failed to delete user %s with error %v", keyringRBDNamespaceProvisionerUsername, err) + } + err = c.CoreV1().Secrets(cephCSINamespace).Delete(context.TODO(), rbdNamespaceProvisionerSecretName, metav1.DeleteOptions{}) + if err != nil { + e2elog.Failf("failed to delete provisioner secret with error %v", err) + } + // delete RBD plugin secret + err = deleteCephUser(f, keyringRBDNamespaceNodePluginUsername) + if err != nil { + e2elog.Failf("failed to delete user %s with error %v", keyringRBDNamespaceNodePluginUsername, err) + } + err = c.CoreV1().Secrets(cephCSINamespace).Delete(context.TODO(), rbdNamespaceNodePluginSecretName, metav1.DeleteOptions{}) + if err != nil { + e2elog.Failf("failed to delete node secret with error %v", err) + } + err = deleteResource(rbdExamplePath + "storageclass.yaml") + if err != nil { + e2elog.Failf("failed to delete storageclass with error %v", err) + } + err = createRBDStorageClass(f.ClientSet, f, nil, nil, deletePolicy) + if err != nil { + e2elog.Failf("failed to create storageclass with error %v", err) + } updateConfigMap("") }) @@ -1424,6 +1505,16 @@ var _ = Describe("RBD", func() { e2elog.Failf("failed to delete PVC when pool not found with error %v", err) } }) + // delete RBD provisioner secret + err := deleteCephUser(f, keyringRBDProvisionerUsername) + if err != nil { + e2elog.Failf("failed to delete user %s with error %v", keyringRBDProvisionerUsername, err) + } + // delete RBD plugin secret + err = deleteCephUser(f, keyringRBDNodePluginUsername) + if err != nil { + e2elog.Failf("failed to delete user %s with error %v", keyringRBDNodePluginUsername, err) + } }) }) }) diff --git a/e2e/rbd_helper.go b/e2e/rbd_helper.go index a52c235d7..6fb4d5de6 100644 --- a/e2e/rbd_helper.go +++ b/e2e/rbd_helper.go @@ -118,27 +118,19 @@ func createRadosNamespace(f *framework.Framework) error { return nil } -func createRBDSecret(c kubernetes.Interface, f *framework.Framework) error { +func createRBDSecret(f *framework.Framework, secretName, userName, userKey string) error { scPath := fmt.Sprintf("%s/%s", rbdExamplePath, "secret.yaml") sc, err := getSecret(scPath) if err != nil { return err } - adminKey, stdErr, err := execCommandInToolBoxPod(f, "ceph auth get-key client.admin", rookNamespace) - if err != nil { - return err + if secretName != "" { + sc.Name = secretName } - if stdErr != "" { - return fmt.Errorf("error getting admin key %v", stdErr) - } - sc.StringData["userID"] = adminUser - sc.StringData["userKey"] = adminKey + sc.StringData["userID"] = userName + sc.StringData["userKey"] = userKey sc.Namespace = cephCSINamespace - _, err = c.CoreV1().Secrets(cephCSINamespace).Create(context.TODO(), &sc, metav1.CreateOptions{}) - if err != nil { - return err - } - + _, err = f.ClientSet.CoreV1().Secrets(cephCSINamespace).Create(context.TODO(), &sc, metav1.CreateOptions{}) return err } diff --git a/e2e/snapshot.go b/e2e/snapshot.go index 6b231ec17..709b0a800 100644 --- a/e2e/snapshot.go +++ b/e2e/snapshot.go @@ -117,6 +117,7 @@ func createRBDSnapshotClass(f *framework.Framework) error { sc := getSnapshotClass(scPath) sc.Parameters["csi.storage.k8s.io/snapshotter-secret-namespace"] = cephCSINamespace + sc.Parameters["csi.storage.k8s.io/snapshotter-secret-name"] = rbdProvisionerSecretName fsID, stdErr, err := execCommandInToolBoxPod(f, "ceph fsid", rookNamespace) if err != nil { @@ -139,6 +140,7 @@ func createCephFSSnapshotClass(f *framework.Framework) error { scPath := fmt.Sprintf("%s/%s", cephfsExamplePath, "snapshotclass.yaml") sc := getSnapshotClass(scPath) sc.Parameters["csi.storage.k8s.io/snapshotter-secret-namespace"] = cephCSINamespace + sc.Parameters["csi.storage.k8s.io/snapshotter-secret-name"] = cephFSProvisionerSecretName fsID, stdErr, err := execCommandInToolBoxPod(f, "ceph fsid", rookNamespace) if err != nil { return err diff --git a/e2e/staticpvc.go b/e2e/staticpvc.go index aade234cb..75c0d2603 100644 --- a/e2e/staticpvc.go +++ b/e2e/staticpvc.go @@ -119,7 +119,7 @@ func validateRBDStaticPV(f *framework.Framework, appPath string, isBlock bool) e opt["radosNamespace"] = radosNamespace } - pv := getStaticPV(pvName, rbdImageName, size, "csi-rbd-secret", cephCSINamespace, sc, "rbd.csi.ceph.com", isBlock, opt) + pv := getStaticPV(pvName, rbdImageName, size, rbdNodePluginSecretName, cephCSINamespace, sc, "rbd.csi.ceph.com", isBlock, opt) _, err = c.CoreV1().PersistentVolumes().Create(context.TODO(), pv, metav1.CreateOptions{}) if err != nil { diff --git a/e2e/upgrade-cephfs.go b/e2e/upgrade-cephfs.go index 556917920..202b84317 100644 --- a/e2e/upgrade-cephfs.go +++ b/e2e/upgrade-cephfs.go @@ -64,10 +64,26 @@ var _ = Describe("CephFS Upgrade Testing", func() { if err != nil { e2elog.Failf("failed to create configmap with error %v", err) } - err = createCephfsSecret(f.ClientSet, f) + var key string + // create cephFS provisioner secret + key, err = createCephUser(f, keyringCephFSProvisionerUsername, cephFSProvisionerCaps()) if err != nil { - e2elog.Failf("failed to create secret with error %v", err) + e2elog.Failf("failed to create user %s with error %v", keyringCephFSProvisionerUsername, err) } + err = createCephfsSecret(f, cephFSProvisionerSecretName, keyringCephFSProvisionerUsername, key) + if err != nil { + e2elog.Failf("failed to create provisioner secret with error %v", err) + } + // create cephFS plugin secret + key, err = createCephUser(f, keyringCephFSNodePluginUsername, cephFSNodePluginCaps()) + if err != nil { + e2elog.Failf("failed to create user %s with error %v", keyringCephFSNodePluginUsername, err) + } + err = createCephfsSecret(f, cephFSNodePluginSecretName, keyringCephFSNodePluginUsername, key) + if err != nil { + e2elog.Failf("failed to create node secret with error %v", err) + } + err = createCephFSSnapshotClass(f) if err != nil { e2elog.Failf("failed to create snapshotclass with error %v", err) @@ -96,9 +112,13 @@ var _ = Describe("CephFS Upgrade Testing", func() { if err != nil { e2elog.Failf("failed to delete configmap with error %v", err) } - err = deleteResource(cephfsExamplePath + "secret.yaml") + err = c.CoreV1().Secrets(cephCSINamespace).Delete(context.TODO(), cephFSProvisionerSecretName, metav1.DeleteOptions{}) if err != nil { - e2elog.Failf("failed to delete secret with error %v", err) + e2elog.Failf("failed to delete provisioner secret with error %v", err) + } + err = c.CoreV1().Secrets(cephCSINamespace).Delete(context.TODO(), cephFSNodePluginSecretName, metav1.DeleteOptions{}) + if err != nil { + e2elog.Failf("failed to delete node secret with error %v", err) } err = deleteResource(cephfsExamplePath + "storageclass.yaml") if err != nil { @@ -373,6 +393,17 @@ var _ = Describe("CephFS Upgrade Testing", func() { if err != nil { e2elog.Failf("failed to delete pvc and application with error %v", err) } + // delete cephFS provisioner secret + err = deleteCephUser(f, keyringCephFSProvisionerUsername) + if err != nil { + e2elog.Failf("failed to delete user %s with error %v", keyringCephFSProvisionerUsername, err) + } + // delete cephFS plugin secret + err = deleteCephUser(f, keyringCephFSNodePluginUsername) + if err != nil { + e2elog.Failf("failed to delete user %s with error %v", keyringCephFSNodePluginUsername, err) + } }) + }) }) diff --git a/e2e/upgrade-rbd.go b/e2e/upgrade-rbd.go index 241906c1e..681ea184d 100644 --- a/e2e/upgrade-rbd.go +++ b/e2e/upgrade-rbd.go @@ -67,9 +67,23 @@ var _ = Describe("RBD Upgrade Testing", func() { if err != nil { e2elog.Failf("failed to create storageclass with error %v", err) } - err = createRBDSecret(f.ClientSet, f) + // create rbd provisioner secret + key, err := createCephUser(f, keyringRBDProvisionerUsername, rbdProvisionerCaps("", "")) if err != nil { - e2elog.Failf("failed to create secret with error %v", err) + e2elog.Failf("failed to create user %s with error %v", keyringRBDProvisionerUsername, err) + } + err = createRBDSecret(f, rbdProvisionerSecretName, keyringRBDProvisionerUsername, key) + if err != nil { + e2elog.Failf("failed to create provisioner secret with error %v", err) + } + // create rbd plugin secret + key, err = createCephUser(f, keyringRBDNodePluginUsername, rbdNodePluginCaps("", "")) + if err != nil { + e2elog.Failf("failed to create user %s with error %v", keyringRBDNodePluginUsername, err) + } + err = createRBDSecret(f, rbdNodePluginSecretName, keyringRBDNodePluginUsername, key) + if err != nil { + e2elog.Failf("failed to create node secret with error %v", err) } err = createRBDSnapshotClass(f) if err != nil { @@ -105,9 +119,13 @@ var _ = Describe("RBD Upgrade Testing", func() { if err != nil { e2elog.Failf("failed to delete configmap with error %v", err) } - err = deleteResource(rbdExamplePath + "secret.yaml") + err = c.CoreV1().Secrets(cephCSINamespace).Delete(context.TODO(), rbdProvisionerSecretName, metav1.DeleteOptions{}) if err != nil { - e2elog.Failf("failed to delete secret with error %v", err) + e2elog.Failf("failed to delete provisioner secret with error %v", err) + } + err = c.CoreV1().Secrets(cephCSINamespace).Delete(context.TODO(), rbdNodePluginSecretName, metav1.DeleteOptions{}) + if err != nil { + e2elog.Failf("failed to delete node secret with error %v", err) } err = deleteResource(rbdExamplePath + "storageclass.yaml") if err != nil { @@ -381,6 +399,17 @@ var _ = Describe("RBD Upgrade Testing", func() { e2elog.Failf("failed to delete pvc and application with error %v", err) } }) + // delete RBD provisioner secret + err := deleteCephUser(f, keyringRBDProvisionerUsername) + if err != nil { + e2elog.Failf("failed to delete user %s with error %v", keyringRBDProvisionerUsername, err) + } + // delete RBD plugin secret + err = deleteCephUser(f, keyringRBDNodePluginUsername) + if err != nil { + e2elog.Failf("failed to delete user %s with error %v", keyringRBDNodePluginUsername, err) + } }) + }) }) diff --git a/e2e/utils.go b/e2e/utils.go index 1250f3e3c..9be308acc 100644 --- a/e2e/utils.go +++ b/e2e/utils.go @@ -28,16 +28,6 @@ const ( defaultNs = "default" vaultSecretNs = "/secret/ceph-csi/" - // rook created cephfs user - cephfsNodePluginSecretName = "rook-csi-cephfs-node" - cephfsProvisionerSecretName = "rook-csi-cephfs-provisioner" - - // Secret created inside the cephCSINamespace, can be modified. The - // Rook secrets get reconciled and changes are undone (needed for - // encryption). - rbdNodePluginSecretName = "csi-rbd-secret" - rbdProvisionerSecretName = "csi-rbd-secret" - rookTolBoxPodLabel = "app=rook-ceph-tools" rbdmountOptions = "mountOptions"