e2e: validate encryption keys in KMS

this commit is to validate if the encrypted
keys are created and deleted properly while
pvc-pvc clone images

Updates: #2022

Signed-off-by: Yati Padia <ypadia@redhat.com>

e2e/rbd.go

@@ -833,7 +833,7 @@ var _ = Describe("RBD", func() {
 						snapshotPath,
 						pvcClonePath,
 						appClonePath,
-						"",
+						noKms,
 						f)
 				}
 			})
@@ -847,6 +847,7 @@ var _ = Describe("RBD", func() {
 						appPath,
 						pvcSmartClonePath,
 						appSmartClonePath,
+						noKms,
 						noPVCValidation,
 						f)
 				}
@@ -868,7 +869,7 @@ var _ = Describe("RBD", func() {
 					e2elog.Failf("failed to create storageclass with error %v", err)
 				}
 
-				validatePVCClone(1, pvcPath, appPath, pvcSmartClonePath, appSmartClonePath, isThickPVC, f)
+				validatePVCClone(1, pvcPath, appPath, pvcSmartClonePath, appSmartClonePath, noKms, isThickPVC, f)
 
 				err = deleteResource(rbdExamplePath + "storageclass.yaml")
 				if err != nil {
@@ -928,7 +929,37 @@ var _ = Describe("RBD", func() {
 					e2elog.Failf("failed to create storageclass with error %v", err)
 				}
 
-				validatePVCClone(1, pvcPath, appPath, pvcSmartClonePath, appSmartClonePath, isEncryptedPVC, f)
+				validatePVCClone(1, pvcPath, appPath, pvcSmartClonePath, appSmartClonePath, "secrets-metadata", isEncryptedPVC, f)
+
+				err = deleteResource(rbdExamplePath + "storageclass.yaml")
+				if err != nil {
+					e2elog.Failf("failed to delete storageclass with error %v", err)
+				}
+				err = createRBDStorageClass(f.ClientSet, f, defaultSCName, nil, nil, deletePolicy)
+				if err != nil {
+					e2elog.Failf("failed to create storageclass with error %v", err)
+				}
+			})
+
+			By("create an encrypted PVC-PVC clone and bind it to an app with VaultKMS", func() {
+				if !k8sVersionGreaterEquals(f.ClientSet, 1, 16) {
+					Skip("pvc clone is only supported from v1.16+")
+				}
+
+				err := deleteResource(rbdExamplePath + "storageclass.yaml")
+				if err != nil {
+					e2elog.Failf("failed to delete storageclass with error %v", err)
+				}
+				scOpts := map[string]string{
+					"encrypted":       "true",
+					"encryptionKMSID": "vault-test",
+				}
+				err = createRBDStorageClass(f.ClientSet, f, defaultSCName, nil, scOpts, deletePolicy)
+				if err != nil {
+					e2elog.Failf("failed to create storageclass with error %v", err)
+				}
+
+				validatePVCClone(1, pvcPath, appPath, pvcSmartClonePath, appSmartClonePath, "vault", isEncryptedPVC, f)
 
 				err = deleteResource(rbdExamplePath + "storageclass.yaml")
 				if err != nil {
@@ -959,6 +990,7 @@ var _ = Describe("RBD", func() {
 						rawAppPath,
 						pvcBlockSmartClonePath,
 						appBlockSmartClonePath,
+						noKms,
 						noPVCValidation,
 						f)
 				}

e2e/utils.go

@@ -47,6 +47,8 @@ const (
 	// vaultTokens KMS type
 	vaultTokens = "vaulttokens"
 	noError     = ""
+
+	noKms = ""
 )
 
 var (
@@ -539,10 +541,10 @@ func writeDataAndCalChecksum(app *v1.Pod, opt *metav1.ListOptions, f *framework.
 	return checkSum, nil
 }
 
-// nolint:gocyclo,gocognit // reduce complexity
+// nolint:gocyclo,gocognit,nestif // reduce complexity
 func validatePVCClone(
 	totalCount int,
-	sourcePvcPath, sourceAppPath, clonePvcPath, clonePvcAppPath string,
+	sourcePvcPath, sourceAppPath, clonePvcPath, clonePvcAppPath, kms string,
 	validatePVC validateFunc,
 	f *framework.Framework) {
 	var wg sync.WaitGroup
@@ -609,6 +611,28 @@ func validatePVCClone(
 				LabelSelector: fmt.Sprintf("%s=%s", appKey, label[appKey]),
 			}
 			wgErrs[n] = createPVCAndApp(name, f, &p, &a, deployTimeout)
+			if wgErrs[n] == nil && kms != noKms {
+				if kmsIsVault(kms) || kms == vaultTokens {
+					imageData, sErr := getImageInfoFromPVC(p.Namespace, name, f)
+					if sErr != nil {
+						wgErrs[n] = fmt.Errorf(
+							"failed to get image info for %s namespace=%s volumehandle=%s error=%w",
+							name,
+							p.Namespace,
+							imageData.csiVolumeHandle,
+							sErr)
+					} else {
+						// check new passphrase created
+						stdOut, stdErr := readVaultSecret(imageData.csiVolumeHandle, kmsIsVault(kms), f)
+						if stdOut != "" {
+							e2elog.Logf("successfully read the passphrase from vault: %s", stdOut)
+						}
+						if stdErr != "" {
+							wgErrs[n] = fmt.Errorf("failed to read passphrase from vault: %s", stdErr)
+						}
+					}
+				}
+			}
 			if *pvc.Spec.VolumeMode == v1.PersistentVolumeFilesystem && wgErrs[n] == nil {
 				filePath := a.Spec.Containers[0].VolumeMounts[0].MountPath + "/test"
 				var checkSumClone string
@@ -622,7 +646,7 @@ func validatePVCClone(
 					e2elog.Logf("checksum didn't match. checksum=%s and checksumclone=%s", checkSum, checkSumClone)
 				}
 			}
-			if wgErrs[n] == nil && validatePVC != nil {
+			if wgErrs[n] == nil && validatePVC != nil && kms != noKms {
 				wgErrs[n] = validatePVC(f, &p, &a)
 			}
 			wg.Done()
@@ -671,7 +695,33 @@ func validatePVCClone(
 		go func(n int, p v1.PersistentVolumeClaim, a v1.Pod) {
 			name := fmt.Sprintf("%s%d", f.UniqueName, n)
 			p.Spec.DataSource.Name = name
+			var imageData imageInfoFromPVC
+			var sErr error
+			if kms != noKms {
+				if kmsIsVault(kms) || kms == vaultTokens {
+					imageData, sErr = getImageInfoFromPVC(p.Namespace, name, f)
+					if sErr != nil {
+						wgErrs[n] = fmt.Errorf(
+							"failed to get image info for %s namespace=%s volumehandle=%s error=%w",
+							name,
+							p.Namespace,
+							imageData.csiVolumeHandle,
+							sErr)
+					}
+				}
+			}
+			if wgErrs[n] == nil {
 				wgErrs[n] = deletePVCAndApp(name, f, &p, &a)
+				if wgErrs[n] == nil && kms != noKms {
+					if kmsIsVault(kms) || kms == vaultTokens {
+						// check passphrase deleted
+						stdOut, _ := readVaultSecret(imageData.csiVolumeHandle, kmsIsVault(kms), f)
+						if stdOut != "" {
+							wgErrs[n] = fmt.Errorf("passphrase found in vault while should be deleted: %s", stdOut)
+						}
+					}
+				}
+			}
 			wg.Done()
 		}(i, *pvcClone, *appClone)
 	}
@@ -747,7 +797,7 @@ func validatePVCSnapshot(
 		go func(n int, s snapapi.VolumeSnapshot) {
 			s.Name = fmt.Sprintf("%s%d", f.UniqueName, n)
 			wgErrs[n] = createSnapshot(&s, deployTimeout)
-			if wgErrs[n] == nil && kms != "" {
+			if wgErrs[n] == nil && kms != noKms {
 				if kmsIsVault(kms) || kms == vaultTokens {
 					content, sErr := getVolumeSnapshotContent(s.Namespace, s.Name)
 					if sErr != nil {
@@ -927,7 +977,7 @@ func validatePVCSnapshot(
 			s.Name = fmt.Sprintf("%s%d", f.UniqueName, n)
 			content := &snapapi.VolumeSnapshotContent{}
 			var err error
-			if kms != "" {
+			if kms != noKms {
 				if kmsIsVault(kms) || kms == vaultTokens {
 					content, err = getVolumeSnapshotContent(s.Namespace, s.Name)
 					if err != nil {
@@ -941,7 +991,7 @@ func validatePVCSnapshot(
 			}
 			if wgErrs[n] == nil {
 				wgErrs[n] = deleteSnapshot(&s, deployTimeout)
-				if wgErrs[n] == nil && kms != "" {
+				if wgErrs[n] == nil && kms != noKms {
 					if kmsIsVault(kms) || kms == vaultTokens {
 						// check passphrase deleted
 						stdOut, _ := readVaultSecret(*content.Status.SnapshotHandle, kmsIsVault(kms), f)