e2e: refactor KMS configuration and usage

This adds a new `kmsConfig` interface that can be used to validate
different KMS services and setting. It makes checking for the available
support easier, and fetching the passphrase simpler.

The basicKMS mirrors the current validation of the KMS implementations
that use secrets and metadata. vaultKMS can be used to validate the
passphrase stored in a Vault service.

Signed-off-by: Niels de Vos <ndevos@redhat.com>

e2e/kms.go

@@ -0,0 +1,102 @@
+package e2e
+
+import (
+	"fmt"
+	"strings"
+
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/kubernetes/test/e2e/framework"
+)
+
+const (
+	// defaultVaultBackendPath is the default VAULT_BACKEND_PATH for secrets
+	defaultVaultBackendPath = "secret/"
+)
+
+// kmsConfig is an interface that should be used when passing a configuration
+// for a KMS to validation functions. This allows the validation functions to
+// work independently from the actual KMS.
+type kmsConfig interface {
+	canGetPassphrase() bool
+	getPassphrase(f *framework.Framework, key string) (string, string)
+}
+
+// simpleKMS is to be used for KMS configurations that do not offer options to
+// validate the passphrase stored in the KMS.
+type simpleKMS struct {
+	provider string
+}
+
+// vaultConfig describes the configuration of the Hashicorp Vault service that
+// is used to store the encryption passphrase in.
+type vaultConfig struct {
+	*simpleKMS
+	backendPath string
+}
+
+// The following variables describe different KMS services as they are defined
+// in the kms-config.yaml file. These variables can be passed on to validation
+// functions when a StorageClass has a KMS enabled.
+var (
+	noKMS kmsConfig = nil
+
+	secretsMetadataKMS = &simpleKMS{
+		provider: "secrets-metadata",
+	}
+
+	vaultKMS = &vaultConfig{
+		simpleKMS: &simpleKMS{
+			provider: "vault",
+		},
+		backendPath: defaultVaultBackendPath + "ceph-csi/",
+	}
+	vaultTokensKMS = &vaultConfig{
+		simpleKMS: &simpleKMS{
+			provider: "vaulttokens",
+		},
+		backendPath: defaultVaultBackendPath,
+	}
+)
+
+func (sk *simpleKMS) String() string {
+	return sk.provider
+}
+
+// canGetPassphrase returns false for the basic KMS configuration as there is
+// currently no way to fetch the passphrase.
+func (sk *simpleKMS) canGetPassphrase() bool {
+	return false
+}
+
+func (sk *simpleKMS) getPassphrase(f *framework.Framework, key string) (string, string) {
+	return "", ""
+}
+
+func (vc *vaultConfig) String() string {
+	return fmt.Sprintf("%s (backend path %q)", vc.simpleKMS, vc.backendPath)
+}
+
+// canGetPassphrase returns true for the Hashicorp Vault KMS configurations as
+// the Vault CLI can be used to retrieve the passphrase.
+func (vc *vaultConfig) canGetPassphrase() bool {
+	return true
+}
+
+// getPassphrase method will execute few commands to try read the secret for
+// specified key from inside the vault container:
+//  * authenticate with vault and ignore any stdout (we do not need output)
+//  * issue get request for particular key
+// resulting in stdOut (first entry in tuple) - output that contains the key
+// or stdErr (second entry in tuple) - error getting the key.
+func (vc *vaultConfig) getPassphrase(f *framework.Framework, key string) (string, string) {
+	vaultAddr := fmt.Sprintf("http://vault.%s.svc.cluster.local:8200", cephCSINamespace)
+	loginCmd := fmt.Sprintf("vault login -address=%s sample_root_token_id > /dev/null", vaultAddr)
+	readSecret := fmt.Sprintf("vault kv get -address=%s -field=data %s%s",
+		vaultAddr, vc.backendPath, key)
+	cmd := fmt.Sprintf("%s && %s", loginCmd, readSecret)
+	opt := metav1.ListOptions{
+		LabelSelector: "app=vault",
+	}
+	stdOut, stdErr := execCommandInPodAndAllowFail(f, cmd, cephCSINamespace, &opt)
+	return strings.TrimSpace(stdOut), strings.TrimSpace(stdErr)
+}

e2e/rbd.go

@@ -697,7 +697,7 @@ var _ = Describe("RBD", func() {
 				if err != nil {
 					e2elog.Failf("failed to create storageclass with error %v", err)
 				}
-				err = validateEncryptedPVCAndAppBinding(pvcPath, appPath, "", f)
+				err = validateEncryptedPVCAndAppBinding(pvcPath, appPath, noKMS, f)
 				if err != nil {
 					e2elog.Failf("failed to validate encrypted pvc with error %v", err)
 				}
@@ -726,7 +726,7 @@ var _ = Describe("RBD", func() {
 				if err != nil {
 					e2elog.Failf("failed to create storageclass with error %v", err)
 				}
-				err = validateEncryptedPVCAndAppBinding(pvcPath, appPath, "vault", f)
+				err = validateEncryptedPVCAndAppBinding(pvcPath, appPath, vaultKMS, f)
 				if err != nil {
 					e2elog.Failf("failed to validate encrypted pvc with error %v", err)
 				}
@@ -769,7 +769,7 @@ var _ = Describe("RBD", func() {
 					e2elog.Failf("failed to create Secret with tenant token: %v", err)
 				}
 
-				err = validateEncryptedPVCAndAppBinding(pvcPath, appPath, "vaulttokens", f)
+				err = validateEncryptedPVCAndAppBinding(pvcPath, appPath, vaultTokensKMS, f)
 				if err != nil {
 					e2elog.Failf("failed to validate encrypted pvc with error %v", err)
 				}
@@ -805,7 +805,7 @@ var _ = Describe("RBD", func() {
 				if err != nil {
 					e2elog.Failf("failed to create storageclass with error %v", err)
 				}
-				err = validateEncryptedPVCAndAppBinding(pvcPath, appPath, "", f)
+				err = validateEncryptedPVCAndAppBinding(pvcPath, appPath, noKMS, f)
 				if err != nil {
 					e2elog.Failf("failed to validate encrypted pvc with error %v", err)
 				}
@@ -848,7 +848,7 @@ var _ = Describe("RBD", func() {
 					e2elog.Failf("failed to create user Secret: %v", err)
 				}
 
-				err = validateEncryptedPVCAndAppBinding(pvcPath, appPath, "", f)
+				err = validateEncryptedPVCAndAppBinding(pvcPath, appPath, noKMS, f)
 				if err != nil {
 					e2elog.Failf("failed to validate encrypted pvc: %v", err)
 				}
@@ -900,7 +900,7 @@ var _ = Describe("RBD", func() {
 						e2elog.Failf("failed to create user Secret: %v", err)
 					}
 
-					err = validateEncryptedPVCAndAppBinding(pvcPath, appPath, "", f)
+					err = validateEncryptedPVCAndAppBinding(pvcPath, appPath, noKMS, f)
 					if err != nil {
 						e2elog.Failf("failed to validate encrypted pvc: %v", err)
 					}
@@ -965,7 +965,7 @@ var _ = Describe("RBD", func() {
 						snapshotPath,
 						pvcClonePath,
 						appClonePath,
-						noKms,
+						noKMS,
 						f)
 				}
 			})
@@ -979,7 +979,7 @@ var _ = Describe("RBD", func() {
 						appPath,
 						pvcSmartClonePath,
 						appSmartClonePath,
-						noKms,
+						noKMS,
 						noPVCValidation,
 						f)
 				}
@@ -1001,7 +1001,7 @@ var _ = Describe("RBD", func() {
 					e2elog.Failf("failed to create storageclass with error %v", err)
 				}
 
-				validatePVCClone(1, pvcPath, appPath, pvcSmartClonePath, appSmartClonePath, noKms, isThickPVC, f)
+				validatePVCClone(1, pvcPath, appPath, pvcSmartClonePath, appSmartClonePath, noKMS, isThickPVC, f)
 
 				err = deleteResource(rbdExamplePath + "storageclass.yaml")
 				if err != nil {
@@ -1031,7 +1031,7 @@ var _ = Describe("RBD", func() {
 					e2elog.Failf("failed to create storageclass with error %v", err)
 				}
 
-				validatePVCSnapshot(1, pvcPath, appPath, snapshotPath, pvcClonePath, appClonePath, "vault", f)
+				validatePVCSnapshot(1, pvcPath, appPath, snapshotPath, pvcClonePath, appClonePath, vaultKMS, f)
 
 				err = deleteResource(rbdExamplePath + "storageclass.yaml")
 				if err != nil {
@@ -1061,7 +1061,7 @@ var _ = Describe("RBD", func() {
 					e2elog.Failf("failed to create storageclass with error %v", err)
 				}
 
-				validatePVCClone(1, pvcPath, appPath, pvcSmartClonePath, appSmartClonePath, "secrets-metadata", isEncryptedPVC, f)
+				validatePVCClone(1, pvcPath, appPath, pvcSmartClonePath, appSmartClonePath, secretsMetadataKMS, isEncryptedPVC, f)
 
 				err = deleteResource(rbdExamplePath + "storageclass.yaml")
 				if err != nil {
@@ -1091,7 +1091,7 @@ var _ = Describe("RBD", func() {
 					e2elog.Failf("failed to create storageclass with error %v", err)
 				}
 
-				validatePVCClone(1, pvcPath, appPath, pvcSmartClonePath, appSmartClonePath, "vault", isEncryptedPVC, f)
+				validatePVCClone(1, pvcPath, appPath, pvcSmartClonePath, appSmartClonePath, vaultKMS, isEncryptedPVC, f)
 
 				err = deleteResource(rbdExamplePath + "storageclass.yaml")
 				if err != nil {
@@ -1122,7 +1122,7 @@ var _ = Describe("RBD", func() {
 						rawAppPath,
 						pvcBlockSmartClonePath,
 						appBlockSmartClonePath,
-						noKms,
+						noKMS,
 						noPVCValidation,
 						f)
 				}

e2e/rbd_helper.go

@@ -253,10 +253,6 @@ func validateImageOwner(pvcPath string, f *framework.Framework) error {
 	return deletePVCAndValidatePV(f.ClientSet, pvc, deployTimeout)
 }
 
-func kmsIsVault(kms string) bool {
-	return kms == "vault"
-}
-
 func logErrors(f *framework.Framework, msg string, wgErrs []error) int {
 	failures := 0
 	for i, err := range wgErrs {
@@ -395,7 +391,7 @@ func validateCloneInDifferentPool(f *framework.Framework, snapshotPool, cloneSc,
 	return nil
 }
 
-func validateEncryptedPVCAndAppBinding(pvcPath, appPath, kms string, f *framework.Framework) error {
+func validateEncryptedPVCAndAppBinding(pvcPath, appPath string, kms kmsConfig, f *framework.Framework) error {
 	pvc, app, err := createPVCAndAppBinding(pvcPath, appPath, f, deployTimeout)
 	if err != nil {
 		return err
@@ -411,9 +407,9 @@ func validateEncryptedPVCAndAppBinding(pvcPath, appPath, kms string, f *framewor
 		return err
 	}
 
-	if kmsIsVault(kms) || kms == vaultTokens {
+	if kms != noKMS && kms.canGetPassphrase() {
 		// check new passphrase created
-		_, stdErr := readVaultSecret(imageData.csiVolumeHandle, kmsIsVault(kms), f)
+		_, stdErr := kms.getPassphrase(f, imageData.csiVolumeHandle)
 		if stdErr != "" {
 			return fmt.Errorf("failed to read passphrase from vault: %s", stdErr)
 		}
@@ -424,9 +420,9 @@ func validateEncryptedPVCAndAppBinding(pvcPath, appPath, kms string, f *framewor
 		return err
 	}
 
-	if kmsIsVault(kms) || kms == vaultTokens {
+	if kms != noKMS && kms.canGetPassphrase() {
 		// check new passphrase created
-		stdOut, _ := readVaultSecret(imageData.csiVolumeHandle, kmsIsVault(kms), f)
+		stdOut, _ := kms.getPassphrase(f, imageData.csiVolumeHandle)
 		if stdOut != "" {
 			return fmt.Errorf("passphrase found in vault while should be deleted: %s", stdOut)
 		}

e2e/utils.go

@@ -28,11 +28,6 @@ import (
 const (
 	defaultNs     = "default"
 	defaultSCName = ""
-	// vaultBackendPath is the default VAULT_BACKEND_PATH for secrets
-	vaultBackendPath = "secret/"
-	// vaultPassphrasePath is an advanced configuration option, only
-	// available for the VaultKMS (not VaultTokensKMS) provider.
-	vaultPassphrasePath = "ceph-csi/"
 
 	rookToolBoxPodLabel = "app=rook-ceph-tools"
 	rbdMountOptions     = "mountOptions"
@@ -44,11 +39,7 @@ const (
 	appKey   = "app"
 	appLabel = "write-data-in-pod"
 
-	// vaultTokens KMS type
+	noError = ""
-	vaultTokens = "vaulttokens"
-	noError     = ""
-
-	noKms = ""
 )
 
 var (
@@ -65,13 +56,11 @@ var (
 	rookNamespace    string
 	radosNamespace   string
 	ns               string
-	vaultAddr        string
 	poll             = 2 * time.Second
 )
 
 func initResources() {
 	ns = fmt.Sprintf("--namespace=%v", cephCSINamespace)
-	vaultAddr = fmt.Sprintf("http://vault.%s.svc.cluster.local:8200", cephCSINamespace)
 }
 
 func getMons(ns string, c kubernetes.Interface) ([]string, error) {
@@ -226,29 +215,6 @@ func getMountType(appName, appNamespace, mountPath string, f *framework.Framewor
 	return strings.TrimSpace(stdOut), nil
 }
 
-// readVaultSecret method will execute few commands to try read the secret for
-// specified key from inside the vault container:
-//  * authenticate with vault and ignore any stdout (we do not need output)
-//  * issue get request for particular key
-// resulting in stdOut (first entry in tuple) - output that contains the key
-// or stdErr (second entry in tuple) - error getting the key.
-func readVaultSecret(key string, usePassphrasePath bool, f *framework.Framework) (string, string) {
-	extraPath := vaultPassphrasePath
-	if !usePassphrasePath {
-		extraPath = ""
-	}
-
-	loginCmd := fmt.Sprintf("vault login -address=%s sample_root_token_id > /dev/null", vaultAddr)
-	readSecret := fmt.Sprintf("vault kv get -address=%s -field=data %s%s%s",
-		vaultAddr, vaultBackendPath, extraPath, key)
-	cmd := fmt.Sprintf("%s && %s", loginCmd, readSecret)
-	opt := metav1.ListOptions{
-		LabelSelector: "app=vault",
-	}
-	stdOut, stdErr := execCommandInPodAndAllowFail(f, cmd, cephCSINamespace, &opt)
-	return strings.TrimSpace(stdOut), strings.TrimSpace(stdErr)
-}
-
 func validateNormalUserPVCAccess(pvcPath string, f *framework.Framework) error {
 	pvc, err := loadPVC(pvcPath)
 	if err != nil {
@@ -544,7 +510,8 @@ func writeDataAndCalChecksum(app *v1.Pod, opt *metav1.ListOptions, f *framework.
 // nolint:gocyclo,gocognit,nestif // reduce complexity
 func validatePVCClone(
 	totalCount int,
-	sourcePvcPath, sourceAppPath, clonePvcPath, clonePvcAppPath, kms string,
+	sourcePvcPath, sourceAppPath, clonePvcPath, clonePvcAppPath string,
+	kms kmsConfig,
 	validatePVC validateFunc,
 	f *framework.Framework) {
 	var wg sync.WaitGroup
@@ -611,8 +578,8 @@ func validatePVCClone(
 				LabelSelector: fmt.Sprintf("%s=%s", appKey, label[appKey]),
 			}
 			wgErrs[n] = createPVCAndApp(name, f, &p, &a, deployTimeout)
-			if wgErrs[n] == nil && kms != noKms {
+			if wgErrs[n] == nil && kms != noKMS {
-				if kmsIsVault(kms) || kms == vaultTokens {
+				if kms.canGetPassphrase() {
 					imageData, sErr := getImageInfoFromPVC(p.Namespace, name, f)
 					if sErr != nil {
 						wgErrs[n] = fmt.Errorf(
@@ -623,7 +590,7 @@ func validatePVCClone(
 							sErr)
 					} else {
 						// check new passphrase created
-						stdOut, stdErr := readVaultSecret(imageData.csiVolumeHandle, kmsIsVault(kms), f)
+						stdOut, stdErr := kms.getPassphrase(f, imageData.csiVolumeHandle)
 						if stdOut != "" {
 							e2elog.Logf("successfully read the passphrase from vault: %s", stdOut)
 						}
@@ -646,7 +613,7 @@ func validatePVCClone(
 					e2elog.Logf("checksum didn't match. checksum=%s and checksumclone=%s", checkSum, checkSumClone)
 				}
 			}
-			if wgErrs[n] == nil && validatePVC != nil && kms != noKms {
+			if wgErrs[n] == nil && validatePVC != nil && kms != noKMS {
 				wgErrs[n] = validatePVC(f, &p, &a)
 			}
 			wg.Done()
@@ -697,8 +664,8 @@ func validatePVCClone(
 			p.Spec.DataSource.Name = name
 			var imageData imageInfoFromPVC
 			var sErr error
-			if kms != noKms {
+			if kms != noKMS {
-				if kmsIsVault(kms) || kms == vaultTokens {
+				if kms.canGetPassphrase() {
 					imageData, sErr = getImageInfoFromPVC(p.Namespace, name, f)
 					if sErr != nil {
 						wgErrs[n] = fmt.Errorf(
@@ -712,10 +679,10 @@ func validatePVCClone(
 			}
 			if wgErrs[n] == nil {
 				wgErrs[n] = deletePVCAndApp(name, f, &p, &a)
-				if wgErrs[n] == nil && kms != noKms {
+				if wgErrs[n] == nil && kms != noKMS {
-					if kmsIsVault(kms) || kms == vaultTokens {
+					if kms.canGetPassphrase() {
 						// check passphrase deleted
-						stdOut, _ := readVaultSecret(imageData.csiVolumeHandle, kmsIsVault(kms), f)
+						stdOut, _ := kms.getPassphrase(f, imageData.csiVolumeHandle)
 						if stdOut != "" {
 							wgErrs[n] = fmt.Errorf("passphrase found in vault while should be deleted: %s", stdOut)
 						}
@@ -744,7 +711,8 @@ func validatePVCClone(
 // nolint:gocyclo,gocognit,nestif // reduce complexity
 func validatePVCSnapshot(
 	totalCount int,
-	pvcPath, appPath, snapshotPath, pvcClonePath, appClonePath, kms string,
+	pvcPath, appPath, snapshotPath, pvcClonePath, appClonePath string,
+	kms kmsConfig,
 	f *framework.Framework) {
 	var wg sync.WaitGroup
 	wgErrs := make([]error, totalCount)
@@ -797,8 +765,8 @@ func validatePVCSnapshot(
 		go func(n int, s snapapi.VolumeSnapshot) {
 			s.Name = fmt.Sprintf("%s%d", f.UniqueName, n)
 			wgErrs[n] = createSnapshot(&s, deployTimeout)
-			if wgErrs[n] == nil && kms != noKms {
+			if wgErrs[n] == nil && kms != noKMS {
-				if kmsIsVault(kms) || kms == vaultTokens {
+				if kms.canGetPassphrase() {
 					content, sErr := getVolumeSnapshotContent(s.Namespace, s.Name)
 					if sErr != nil {
 						wgErrs[n] = fmt.Errorf(
@@ -808,7 +776,7 @@ func validatePVCSnapshot(
 							sErr)
 					} else {
 						// check new passphrase created
-						_, stdErr := readVaultSecret(*content.Status.SnapshotHandle, kmsIsVault(kms), f)
+						_, stdErr := kms.getPassphrase(f, *content.Status.SnapshotHandle)
 						if stdErr != "" {
 							wgErrs[n] = fmt.Errorf("failed to read passphrase from vault: %s", stdErr)
 						}
@@ -874,7 +842,7 @@ func validatePVCSnapshot(
 						checkSumClone)
 				}
 			}
-			if wgErrs[n] == nil && kms != "" {
+			if wgErrs[n] == nil && kms != noKMS {
 				wgErrs[n] = isEncryptedPVC(f, &p, &a)
 			}
 			wg.Done()
@@ -977,8 +945,8 @@ func validatePVCSnapshot(
 			s.Name = fmt.Sprintf("%s%d", f.UniqueName, n)
 			content := &snapapi.VolumeSnapshotContent{}
 			var err error
-			if kms != noKms {
+			if kms != noKMS {
-				if kmsIsVault(kms) || kms == vaultTokens {
+				if kms.canGetPassphrase() {
 					content, err = getVolumeSnapshotContent(s.Namespace, s.Name)
 					if err != nil {
 						wgErrs[n] = fmt.Errorf(
@@ -991,10 +959,10 @@ func validatePVCSnapshot(
 			}
 			if wgErrs[n] == nil {
 				wgErrs[n] = deleteSnapshot(&s, deployTimeout)
-				if wgErrs[n] == nil && kms != noKms {
+				if wgErrs[n] == nil && kms != noKMS {
-					if kmsIsVault(kms) || kms == vaultTokens {
+					if kms.canGetPassphrase() {
 						// check passphrase deleted
-						stdOut, _ := readVaultSecret(*content.Status.SnapshotHandle, kmsIsVault(kms), f)
+						stdOut, _ := kms.getPassphrase(f, *content.Status.SnapshotHandle)
 						if stdOut != "" {
 							wgErrs[n] = fmt.Errorf("passphrase found in vault while should be deleted: %s", stdOut)
 						}