e2e: add e2e for user secret based metadata encryption

This commit adds e2e for user secret based metadata encryption,
adds user-secret.yaml and makes required changes in kms-connection-details,
kms-config yamls.

Signed-off-by: Rakshith R <rar@redhat.com>

e2e/rbd.go

@@ -821,6 +821,108 @@ var _ = Describe("RBD", func() {
 				}
 			})
 
+			By("test RBD volume encryption with user secrets based SecretsMetadataKMS", func() {
+				err := deleteResource(rbdExamplePath + "storageclass.yaml")
+				if err != nil {
+					e2elog.Failf("failed to delete storageclass: %v", err)
+				}
+				scOpts := map[string]string{
+					"encrypted":       "true",
+					"encryptionKMSID": "user-ns-secrets-metadata-test",
+				}
+				err = createRBDStorageClass(f.ClientSet, f, defaultSCName, nil, scOpts, deletePolicy)
+				if err != nil {
+					e2elog.Failf("failed to create storageclass: %v", err)
+				}
+
+				// user provided namespace where secret will be created
+				namespace := cephCSINamespace
+
+				// create user Secret
+				secret, err := getSecret(vaultExamplePath + "user-secret.yaml")
+				if err != nil {
+					e2elog.Failf("failed to load user Secret: %v", err)
+				}
+				_, err = c.CoreV1().Secrets(namespace).Create(context.TODO(), &secret, metav1.CreateOptions{})
+				if err != nil {
+					e2elog.Failf("failed to create user Secret: %v", err)
+				}
+
+				err = validateEncryptedPVCAndAppBinding(pvcPath, appPath, "", f)
+				if err != nil {
+					e2elog.Failf("failed to validate encrypted pvc: %v", err)
+				}
+				// validate created backend rbd images
+				validateRBDImageCount(f, 0, defaultRBDPool)
+
+				// delete user secret
+				err = c.CoreV1().Secrets(namespace).Delete(context.TODO(), secret.Name, metav1.DeleteOptions{})
+				if err != nil {
+					e2elog.Failf("failed to delete user Secret: %v", err)
+				}
+
+				err = deleteResource(rbdExamplePath + "storageclass.yaml")
+				if err != nil {
+					e2elog.Failf("failed to delete storageclass: %v", err)
+				}
+				err = createRBDStorageClass(f.ClientSet, f, defaultSCName, nil, nil, deletePolicy)
+				if err != nil {
+					e2elog.Failf("failed to create storageclass: %v", err)
+				}
+			})
+
+			By(
+				"test RBD volume encryption with user secrets based SecretsMetadataKMS with tenant namespace",
+				func() {
+					err := deleteResource(rbdExamplePath + "storageclass.yaml")
+					if err != nil {
+						e2elog.Failf("failed to delete storageclass: %v", err)
+					}
+					scOpts := map[string]string{
+						"encrypted":       "true",
+						"encryptionKMSID": "user-secrets-metadata-test",
+					}
+					err = createRBDStorageClass(f.ClientSet, f, defaultSCName, nil, scOpts, deletePolicy)
+					if err != nil {
+						e2elog.Failf("failed to create storageclass: %v", err)
+					}
+
+					// PVC creation namespace where secret will be created
+					namespace := f.UniqueName
+
+					// create user Secret
+					secret, err := getSecret(vaultExamplePath + "user-secret.yaml")
+					if err != nil {
+						e2elog.Failf("failed to load user Secret: %v", err)
+					}
+					_, err = c.CoreV1().Secrets(namespace).Create(context.TODO(), &secret, metav1.CreateOptions{})
+					if err != nil {
+						e2elog.Failf("failed to create user Secret: %v", err)
+					}
+
+					err = validateEncryptedPVCAndAppBinding(pvcPath, appPath, "", f)
+					if err != nil {
+						e2elog.Failf("failed to validate encrypted pvc: %v", err)
+					}
+					// validate created backend rbd images
+					validateRBDImageCount(f, 0, defaultRBDPool)
+
+					// delete user secret
+					err = c.CoreV1().Secrets(namespace).Delete(context.TODO(), secret.Name, metav1.DeleteOptions{})
+					if err != nil {
+						e2elog.Failf("failed to delete user Secret: %v", err)
+					}
+
+					err = deleteResource(rbdExamplePath + "storageclass.yaml")
+					if err != nil {
+						e2elog.Failf("failed to delete storageclass: %v", err)
+					}
+					err = createRBDStorageClass(f.ClientSet, f, defaultSCName, nil, nil, deletePolicy)
+					if err != nil {
+						e2elog.Failf("failed to create storageclass: %v", err)
+					}
+				})
+
 			By(
 				"create a PVC and Bind it to an app with journaling/exclusive-lock image-features and rbd-nbd mounter",
 				func() {

examples/kms/vault/csi-kms-connection-details.yaml

@@ -35,6 +35,17 @@ data:
     {
       "encryptionKMSType": "metadata"
     }
+  user-ns-secrets-metadata-test: |-
+    {
+      "encryptionKMSType": "metadata",
+      "secretName": "storage-encryption-secret",
+      "secretNamespace": "default"
+    }
+  user-secrets-metadata-test: |-
+    {
+      "encryptionKMSType": "metadata",
+      "secretName": "storage-encryption-secret"
+    }
   aws-metadata-test: |-
     {
       "KMS_PROVIDER": "aws-metadata",

examples/kms/vault/kms-config.yaml

@@ -33,6 +33,15 @@ data:
       },
       "secrets-metadata-test": {
           "encryptionKMSType": "metadata"
+      },
+      "user-ns-secrets-metadata-test": {
+        "encryptionKMSType": "metadata",
+        "secretName": "storage-encryption-secret",
+        "secretNamespace": "default"
+      },
+      "user-secrets-metadata-test": {
+        "encryptionKMSType": "metadata",
+        "secretName": "storage-encryption-secret"
       }
     }
 metadata:

examples/kms/vault/user-secret.yaml

@@ -0,0 +1,11 @@
+---
+# This is the user secret containing encryptionPasspharse that can be
+# created in a Kubernetes Namespace for encrypting PVCs with the
+# "user-ns-secrets-metadata-test" or "user-secrets-metadata-test"
+# encryptionKMSID.
+apiVersion: v1
+kind: Secret
+metadata:
+  name: storage-encryption-secret
+stringData:
+  encryptionPassphrase: test-encryption