From 1c59f0683e002a00393cee45589bab8a930186eb Mon Sep 17 00:00:00 2001 From: Madhu Rajanna Date: Mon, 26 Apr 2021 18:45:18 +0530 Subject: [PATCH] rbd: delete encryption key from KMS when a Snapshot is encrypted during a CreateSnapshot operation, the encryption key gets created in the KMS when we delete the Snapshot the key from the KMS should also gets deleted. When we create a volume from snapshot we are copying required information but we missed to copy the encryption information, This commit adds the missing information to delete the encryption key. Signed-off-by: Madhu Rajanna (cherry picked from commit c3bae17fcee0413dfb0a15a3524f0e7e0fc8d5d9) --- e2e/rbd.go | 8 ++++---- e2e/rbd_helper.go | 4 ++-- e2e/snapshot.go | 17 ++++++++++++++++ e2e/utils.go | 44 +++++++++++++++++++++++++++++++++++++--- internal/rbd/snapshot.go | 4 ++++ 5 files changed, 68 insertions(+), 9 deletions(-) diff --git a/e2e/rbd.go b/e2e/rbd.go index 3528bb685..7ff958302 100644 --- a/e2e/rbd.go +++ b/e2e/rbd.go @@ -497,7 +497,7 @@ var _ = Describe("RBD", func() { By("create a PVC clone and bind it to an app", func() { // snapshot beta is only supported from v1.17+ if k8sVersionGreaterEquals(f.ClientSet, 1, 17) { - validatePVCSnapshot(defaultCloneCount, pvcPath, appPath, snapshotPath, pvcClonePath, appClonePath, false, f) + validatePVCSnapshot(defaultCloneCount, pvcPath, appPath, snapshotPath, pvcClonePath, appClonePath, "", false, f) } }) @@ -508,7 +508,7 @@ var _ = Describe("RBD", func() { } }) - By("create an encrypted PVC snapshot and restore it for an app", func() { + By("create an encrypted PVC snapshot and restore it for an app with VaultKMS", func() { if !k8sVersionGreaterEquals(f.ClientSet, 1, 16) { Skip("pvc clone is only supported from v1.16+") } @@ -519,14 +519,14 @@ var _ = Describe("RBD", func() { } scOpts := map[string]string{ "encrypted": "true", - "encryptionKMSID": "secrets-metadata-test", + "encryptionKMSID": "vault-test", } err = createRBDStorageClass(f.ClientSet, f, nil, scOpts, deletePolicy) if err != nil { e2elog.Failf("failed to create storageclass with error %v", err) } - validatePVCSnapshot(1, pvcPath, appPath, snapshotPath, pvcClonePath, appClonePath, true, f) + validatePVCSnapshot(1, pvcPath, appPath, snapshotPath, pvcClonePath, appClonePath, "vault", true, f) err = deleteResource(rbdExamplePath + "storageclass.yaml") if err != nil { diff --git a/e2e/rbd_helper.go b/e2e/rbd_helper.go index 05093fee0..b7a9766b6 100644 --- a/e2e/rbd_helper.go +++ b/e2e/rbd_helper.go @@ -245,7 +245,7 @@ func validateEncryptedPVCAndAppBinding(pvcPath, appPath, kms string, f *framewor return err } - if kmsIsVault(kms) || kms == "vaulttokens" { + if kmsIsVault(kms) || kms == vaultTokens { // check new passphrase created _, stdErr := readVaultSecret(imageData.csiVolumeHandle, kmsIsVault(kms), f) if stdErr != "" { @@ -258,7 +258,7 @@ func validateEncryptedPVCAndAppBinding(pvcPath, appPath, kms string, f *framewor return err } - if kmsIsVault(kms) || kms == "vaulttokens" { + if kmsIsVault(kms) || kms == vaultTokens { // check new passphrase created stdOut, _ := readVaultSecret(imageData.csiVolumeHandle, kmsIsVault(kms), f) if stdOut != "" { diff --git a/e2e/snapshot.go b/e2e/snapshot.go index 91e0b7835..d8d3bf5b8 100644 --- a/e2e/snapshot.go +++ b/e2e/snapshot.go @@ -168,3 +168,20 @@ func createCephFSSnapshotClass(f *framework.Framework) error { _, err = sclient.SnapshotV1beta1().VolumeSnapshotClasses().Create(context.TODO(), &sc, metav1.CreateOptions{}) return err } + +func getVolumeSnapshotContent(namespace, snapshotName string) (*snapapi.VolumeSnapshotContent, error) { + sclient, err := newSnapshotClient() + if err != nil { + return nil, err + } + snapshot, err := sclient.SnapshotV1beta1().VolumeSnapshots(namespace).Get(context.TODO(), snapshotName, metav1.GetOptions{}) + if err != nil { + return nil, err + } + + volumeSnapshotContent, err := sclient.SnapshotV1beta1().VolumeSnapshotContents().Get(context.TODO(), *snapshot.Status.BoundVolumeSnapshotContentName, metav1.GetOptions{}) + if err != nil { + return nil, err + } + return volumeSnapshotContent, nil +} diff --git a/e2e/utils.go b/e2e/utils.go index f6145fdfd..6d7640d2d 100644 --- a/e2e/utils.go +++ b/e2e/utils.go @@ -43,6 +43,9 @@ const ( // Default key and label for Listoptions appKey = "app" appLabel = "write-data-in-pod" + + // vaultTokens KMS type + vaultTokens = "vaulttokens" ) var ( @@ -651,8 +654,8 @@ func validatePVCClone(totalCount int, sourcePvcPath, sourceAppPath, clonePvcPath validateRBDImageCount(f, 0) } -// nolint:gocyclo,gocognit // reduce complexity -func validatePVCSnapshot(totalCount int, pvcPath, appPath, snapshotPath, pvcClonePath, appClonePath string, validateEncryption bool, f *framework.Framework) { +// nolint:gocyclo,gocognit,nestif // reduce complexity +func validatePVCSnapshot(totalCount int, pvcPath, appPath, snapshotPath, pvcClonePath, appClonePath, kms string, validateEncryption bool, f *framework.Framework) { var wg sync.WaitGroup wgErrs := make([]error, totalCount) chErrs := make([]error, totalCount) @@ -704,6 +707,20 @@ func validatePVCSnapshot(totalCount int, pvcPath, appPath, snapshotPath, pvcClon go func(w *sync.WaitGroup, n int, s v1beta1.VolumeSnapshot) { s.Name = fmt.Sprintf("%s%d", f.UniqueName, n) wgErrs[n] = createSnapshot(&s, deployTimeout) + if wgErrs[n] == nil && validateEncryption { + if kmsIsVault(kms) || kms == vaultTokens { + content, sErr := getVolumeSnapshotContent(s.Namespace, s.Name) + if sErr != nil { + wgErrs[n] = fmt.Errorf("failed to get snapshotcontent for %s in namespace %s with error: %w", s.Name, s.Namespace, sErr) + } else { + // check new passphrase created + _, stdErr := readVaultSecret(*content.Status.SnapshotHandle, kmsIsVault(kms), f) + if stdErr != "" { + wgErrs[n] = fmt.Errorf("failed to read passphrase from vault: %s", stdErr) + } + } + } + } w.Done() }(&wg, i, snap) } @@ -861,7 +878,28 @@ func validatePVCSnapshot(totalCount int, pvcPath, appPath, snapshotPath, pvcClon for i := 0; i < totalCount; i++ { go func(w *sync.WaitGroup, n int, s v1beta1.VolumeSnapshot) { s.Name = fmt.Sprintf("%s%d", f.UniqueName, n) - wgErrs[n] = deleteSnapshot(&s, deployTimeout) + content := &v1beta1.VolumeSnapshotContent{} + var err error + if validateEncryption { + if kmsIsVault(kms) || kms == vaultTokens { + content, err = getVolumeSnapshotContent(s.Namespace, s.Name) + if err != nil { + wgErrs[n] = fmt.Errorf("failed to get snapshotcontent for %s in namespace %s with error: %w", s.Name, s.Namespace, err) + } + } + } + if wgErrs[n] == nil { + wgErrs[n] = deleteSnapshot(&s, deployTimeout) + if wgErrs[n] == nil && validateEncryption { + if kmsIsVault(kms) || kms == vaultTokens { + // check passphrase deleted + stdOut, _ := readVaultSecret(*content.Status.SnapshotHandle, kmsIsVault(kms), f) + if stdOut != "" { + wgErrs[n] = fmt.Errorf("passphrase found in vault while should be deleted: %s", stdOut) + } + } + } + } w.Done() }(&wg, i, snap) } diff --git a/internal/rbd/snapshot.go b/internal/rbd/snapshot.go index 8d9cf0efa..b10be0dce 100644 --- a/internal/rbd/snapshot.go +++ b/internal/rbd/snapshot.go @@ -89,6 +89,10 @@ func generateVolFromSnap(rbdSnap *rbdSnapshot) *rbdVolume { vol.RadosNamespace = rbdSnap.RadosNamespace vol.RbdImageName = rbdSnap.RbdSnapName vol.ImageID = rbdSnap.ImageID + // copyEncryptionConfig cannot be used here because the volume and the + // snapshot will have the same volumeID which cases the panic in + // copyEncryptionConfig function. + vol.encryption = rbdSnap.encryption return vol }