rbd: delete encryption key from KMS

when a Snapshot is encrypted during a CreateSnapshot
operation, the encryption key gets created in the KMS
when we delete the Snapshot the key from the KMS
should also gets deleted.

When we create a volume from snapshot we are copying
required information but we missed to copy the
encryption information, This commit adds the missing
information to delete the encryption key.

Signed-off-by: Madhu Rajanna <madhupr007@gmail.com>

e2e/rbd.go

@@ -497,7 +497,7 @@ var _ = Describe("RBD", func() {
 			By("create a PVC clone and bind it to an app", func() {
 				// snapshot beta is only supported from v1.17+
 				if k8sVersionGreaterEquals(f.ClientSet, 1, 17) {
-					validatePVCSnapshot(defaultCloneCount, pvcPath, appPath, snapshotPath, pvcClonePath, appClonePath, false, f)
+					validatePVCSnapshot(defaultCloneCount, pvcPath, appPath, snapshotPath, pvcClonePath, appClonePath, "", false, f)
 				}
 			})
 
@@ -508,7 +508,7 @@ var _ = Describe("RBD", func() {
 				}
 			})
 
-			By("create an encrypted PVC snapshot and restore it for an app", func() {
+			By("create an encrypted PVC snapshot and restore it for an app with VaultKMS", func() {
 				if !k8sVersionGreaterEquals(f.ClientSet, 1, 16) {
 					Skip("pvc clone is only supported from v1.16+")
 				}
@@ -519,14 +519,14 @@ var _ = Describe("RBD", func() {
 				}
 				scOpts := map[string]string{
 					"encrypted":       "true",
-					"encryptionKMSID": "secrets-metadata-test",
+					"encryptionKMSID": "vault-test",
 				}
 				err = createRBDStorageClass(f.ClientSet, f, nil, scOpts, deletePolicy)
 				if err != nil {
 					e2elog.Failf("failed to create storageclass with error %v", err)
 				}
 
-				validatePVCSnapshot(1, pvcPath, appPath, snapshotPath, pvcClonePath, appClonePath, true, f)
+				validatePVCSnapshot(1, pvcPath, appPath, snapshotPath, pvcClonePath, appClonePath, "vault", true, f)
 
 				err = deleteResource(rbdExamplePath + "storageclass.yaml")
 				if err != nil {

e2e/rbd_helper.go

@@ -245,7 +245,7 @@ func validateEncryptedPVCAndAppBinding(pvcPath, appPath, kms string, f *framewor
 		return err
 	}
 
-	if kmsIsVault(kms) || kms == "vaulttokens" {
+	if kmsIsVault(kms) || kms == vaultTokens {
 		// check new passphrase created
 		_, stdErr := readVaultSecret(imageData.csiVolumeHandle, kmsIsVault(kms), f)
 		if stdErr != "" {
@@ -258,7 +258,7 @@ func validateEncryptedPVCAndAppBinding(pvcPath, appPath, kms string, f *framewor
 		return err
 	}
 
-	if kmsIsVault(kms) || kms == "vaulttokens" {
+	if kmsIsVault(kms) || kms == vaultTokens {
 		// check new passphrase created
 		stdOut, _ := readVaultSecret(imageData.csiVolumeHandle, kmsIsVault(kms), f)
 		if stdOut != "" {

e2e/snapshot.go

@@ -168,3 +168,20 @@ func createCephFSSnapshotClass(f *framework.Framework) error {
 	_, err = sclient.SnapshotV1beta1().VolumeSnapshotClasses().Create(context.TODO(), &sc, metav1.CreateOptions{})
 	return err
 }
+
+func getVolumeSnapshotContent(namespace, snapshotName string) (*snapapi.VolumeSnapshotContent, error) {
+	sclient, err := newSnapshotClient()
+	if err != nil {
+		return nil, err
+	}
+	snapshot, err := sclient.SnapshotV1beta1().VolumeSnapshots(namespace).Get(context.TODO(), snapshotName, metav1.GetOptions{})
+	if err != nil {
+		return nil, err
+	}
+
+	volumeSnapshotContent, err := sclient.SnapshotV1beta1().VolumeSnapshotContents().Get(context.TODO(), *snapshot.Status.BoundVolumeSnapshotContentName, metav1.GetOptions{})
+	if err != nil {
+		return nil, err
+	}
+	return volumeSnapshotContent, nil
+}

e2e/utils.go

@@ -43,6 +43,9 @@ const (
 	// Default key and label for Listoptions
 	appKey   = "app"
 	appLabel = "write-data-in-pod"
+
+	// vaultTokens KMS type
+	vaultTokens = "vaulttokens"
 )
 
 var (
@@ -651,8 +654,8 @@ func validatePVCClone(totalCount int, sourcePvcPath, sourceAppPath, clonePvcPath
 	validateRBDImageCount(f, 0)
 }
 
-// nolint:gocyclo,gocognit // reduce complexity
+// nolint:gocyclo,gocognit,nestif // reduce complexity
-func validatePVCSnapshot(totalCount int, pvcPath, appPath, snapshotPath, pvcClonePath, appClonePath string, validateEncryption bool, f *framework.Framework) {
+func validatePVCSnapshot(totalCount int, pvcPath, appPath, snapshotPath, pvcClonePath, appClonePath, kms string, validateEncryption bool, f *framework.Framework) {
 	var wg sync.WaitGroup
 	wgErrs := make([]error, totalCount)
 	chErrs := make([]error, totalCount)
@@ -704,6 +707,20 @@ func validatePVCSnapshot(totalCount int, pvcPath, appPath, snapshotPath, pvcClon
 		go func(w *sync.WaitGroup, n int, s v1beta1.VolumeSnapshot) {
 			s.Name = fmt.Sprintf("%s%d", f.UniqueName, n)
 			wgErrs[n] = createSnapshot(&s, deployTimeout)
+			if wgErrs[n] == nil && validateEncryption {
+				if kmsIsVault(kms) || kms == vaultTokens {
+					content, sErr := getVolumeSnapshotContent(s.Namespace, s.Name)
+					if sErr != nil {
+						wgErrs[n] = fmt.Errorf("failed to get snapshotcontent for %s in namespace %s with error: %w", s.Name, s.Namespace, sErr)
+					} else {
+						// check new passphrase created
+						_, stdErr := readVaultSecret(*content.Status.SnapshotHandle, kmsIsVault(kms), f)
+						if stdErr != "" {
+							wgErrs[n] = fmt.Errorf("failed to read passphrase from vault: %s", stdErr)
+						}
+					}
+				}
+			}
 			w.Done()
 		}(&wg, i, snap)
 	}
@@ -861,7 +878,28 @@ func validatePVCSnapshot(totalCount int, pvcPath, appPath, snapshotPath, pvcClon
 	for i := 0; i < totalCount; i++ {
 		go func(w *sync.WaitGroup, n int, s v1beta1.VolumeSnapshot) {
 			s.Name = fmt.Sprintf("%s%d", f.UniqueName, n)
-			wgErrs[n] = deleteSnapshot(&s, deployTimeout)
+			content := &v1beta1.VolumeSnapshotContent{}
+			var err error
+			if validateEncryption {
+				if kmsIsVault(kms) || kms == vaultTokens {
+					content, err = getVolumeSnapshotContent(s.Namespace, s.Name)
+					if err != nil {
+						wgErrs[n] = fmt.Errorf("failed to get snapshotcontent for %s in namespace %s with error: %w", s.Name, s.Namespace, err)
+					}
+				}
+			}
+			if wgErrs[n] == nil {
+				wgErrs[n] = deleteSnapshot(&s, deployTimeout)
+				if wgErrs[n] == nil && validateEncryption {
+					if kmsIsVault(kms) || kms == vaultTokens {
+						// check passphrase deleted
+						stdOut, _ := readVaultSecret(*content.Status.SnapshotHandle, kmsIsVault(kms), f)
+						if stdOut != "" {
+							wgErrs[n] = fmt.Errorf("passphrase found in vault while should be deleted: %s", stdOut)
+						}
+					}
+				}
+			}
 			w.Done()
 		}(&wg, i, snap)
 	}

internal/rbd/snapshot.go

@@ -89,6 +89,10 @@ func generateVolFromSnap(rbdSnap *rbdSnapshot) *rbdVolume {
 	vol.RadosNamespace = rbdSnap.RadosNamespace
 	vol.RbdImageName = rbdSnap.RbdSnapName
 	vol.ImageID = rbdSnap.ImageID
+	// copyEncryptionConfig cannot be used here because the volume and the
+	// snapshot will have the same volumeID which cases the panic in
+	// copyEncryptionConfig function.
+	vol.encryption = rbdSnap.encryption
 	return vol
 }