e2e: add test for Vault with ServiceAccount per Tenant

Signed-off-by: Niels de Vos <ndevos@redhat.com>

e2e/deploy-vault.go

@@ -2,6 +2,7 @@ package e2e
 
 import (
 	"context"
+	"fmt"
 	"strings"
 
 	. "github.com/onsi/gomega" // nolint
@@ -12,11 +13,13 @@ import (
 )
 
 var (
-	vaultExamplePath = "../examples/kms/vault/"
+	vaultExamplePath     = "../examples/kms/vault/"
-	vaultServicePath = "vault.yaml"
+	vaultServicePath     = "vault.yaml"
-	vaultPSPPath     = "vault-psp.yaml"
+	vaultPSPPath         = "vault-psp.yaml"
-	vaultRBACPath    = "csi-vaulttokenreview-rbac.yaml"
+	vaultRBACPath        = "csi-vaulttokenreview-rbac.yaml"
-	vaultConfigPath  = "kms-config.yaml"
+	vaultConfigPath      = "kms-config.yaml"
+	vaultTenantPath      = "tenant-sa.yaml"
+	vaultTenantAdminPath = "tenant-sa-admin.yaml"
 )
 
 func deployVault(c kubernetes.Interface, deployTimeout int) {
@@ -91,3 +94,58 @@ func createORDeleteVault(action string) {
 		e2elog.Failf("failed to %s vault psp %v", action, err)
 	}
 }
+
+// createTenantServiceAccount uses the tenant-sa.yaml example file to create
+// the ServiceAccount for the tenant and configured Hashicorp Vault with a
+// kv-store that the ServiceAccount has access to.
+func createTenantServiceAccount(c kubernetes.Interface, ns string) error {
+	err := createORDeleteTenantServiceAccount("create", ns)
+	if err != nil {
+		return fmt.Errorf("failed to create ServiceAccount: %w", err)
+	}
+
+	// wait for the Job to finish
+	const jobName = "vault-tenant-sa"
+	err = waitForJobCompletion(c, cephCSINamespace, jobName, deployTimeout)
+	if err != nil {
+		return fmt.Errorf("job %s/%s did not succeed: %w", cephCSINamespace, jobName, err)
+	}
+
+	return nil
+}
+
+// deleteTenantServiceAccount removed the ServiceAccount and other objects that
+// were created with createTenantServiceAccount.
+func deleteTenantServiceAccount(ns string) {
+	err := createORDeleteTenantServiceAccount("delete", ns)
+	Expect(err).Should(BeNil())
+}
+
+// createORDeleteTenantServiceAccount is a helper that reads the tenant-sa.yaml
+// example file and replaces the default namespaces with the current deployment
+// configuration.
+func createORDeleteTenantServiceAccount(action, ns string) error {
+	_, err := framework.RunKubectl(ns, action, "-f", vaultExamplePath+vaultTenantPath)
+	if err != nil {
+		return fmt.Errorf("failed to %s tenant ServiceAccount: %w", action, err)
+	}
+
+	// the ServiceAccount needs permissions in Vault, the admin job sets that up
+	data, err := replaceNamespaceInTemplate(vaultExamplePath + vaultTenantAdminPath)
+	if err != nil {
+		return fmt.Errorf("failed to read content from %q: %w", vaultExamplePath+vaultTenantAdminPath, err)
+	}
+
+	// replace the value for TENANT_NAMESPACE
+	data = strings.ReplaceAll(data, "value: tenant", "value: "+ns)
+
+	// replace "default" in the URL to the Vault service
+	data = strings.ReplaceAll(data, "vault.default", "vault."+cephCSINamespace)
+
+	_, err = framework.RunKubectlInput(cephCSINamespace, data, action, "-f", "-")
+	if err != nil {
+		return fmt.Errorf("failed to %s ServiceAccount: %w", action, err)
+	}
+
+	return nil
+}

e2e/kms.go

@@ -56,6 +56,12 @@ var (
 		},
 		backendPath: defaultVaultBackendPath,
 	}
+	vaultTenantSAKMS = &vaultConfig{
+		simpleKMS: &simpleKMS{
+			provider: "vaulttenantsa",
+		},
+		backendPath: "tenant/",
+	}
 )
 
 func (sk *simpleKMS) String() string {

e2e/rbd.go

@@ -792,6 +792,42 @@ var _ = Describe("RBD", func() {
 				}
 			})
 
+			By("create a PVC and bind it to an app with encrypted RBD volume with VaultTenantSA KMS", func() {
+				err := deleteResource(rbdExamplePath + "storageclass.yaml")
+				if err != nil {
+					e2elog.Failf("failed to delete storageclass: %v", err)
+				}
+				scOpts := map[string]string{
+					"encrypted":       "true",
+					"encryptionKMSID": "vault-tenant-sa-test",
+				}
+				err = createRBDStorageClass(f.ClientSet, f, defaultSCName, nil, scOpts, deletePolicy)
+				if err != nil {
+					e2elog.Failf("failed to create storageclass: %v", err)
+				}
+
+				err = createTenantServiceAccount(f.ClientSet, f.UniqueName)
+				if err != nil {
+					e2elog.Failf("failed to create ServiceAccount: %v", err)
+				}
+				defer deleteTenantServiceAccount(f.UniqueName)
+
+				err = validateEncryptedPVCAndAppBinding(pvcPath, appPath, vaultTenantSAKMS, f)
+				if err != nil {
+					e2elog.Failf("failed to validate encrypted pvc: %v", err)
+				}
+				// validate created backend rbd images
+				validateRBDImageCount(f, 0, defaultRBDPool)
+				err = deleteResource(rbdExamplePath + "storageclass.yaml")
+				if err != nil {
+					e2elog.Failf("failed to delete storageclass: %v", err)
+				}
+				err = createRBDStorageClass(f.ClientSet, f, defaultSCName, nil, nil, deletePolicy)
+				if err != nil {
+					e2elog.Failf("failed to create storageclass: %v", err)
+				}
+			})
+
 			By("create a PVC and bind it to an app with encrypted RBD volume with SecretsMetadataKMS", func() {
 				err := deleteResource(rbdExamplePath + "storageclass.yaml")
 				if err != nil {

e2e/utils.go

@@ -13,11 +13,13 @@ import (
 	"time"
 
 	snapapi "github.com/kubernetes-csi/external-snapshotter/client/v4/apis/volumesnapshot/v1"
+	batch "k8s.io/api/batch/v1"
 	v1 "k8s.io/api/core/v1"
 	scv1 "k8s.io/api/storage/v1"
 	"k8s.io/apimachinery/pkg/api/resource"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/fields"
+	"k8s.io/apimachinery/pkg/util/wait"
 	utilyaml "k8s.io/apimachinery/pkg/util/yaml"
 	"k8s.io/client-go/kubernetes"
 	"k8s.io/kubernetes/test/e2e/framework"
@@ -1154,3 +1156,32 @@ func k8sVersionGreaterEquals(c kubernetes.Interface, major, minor int) bool {
 
 	return (v.Major > maj) || (v.Major == maj && v.Minor >= min)
 }
+
+// waitForJobCompletion polls the status of the given job and waits until the
+// jobs has succeeded or until the timeout is hit.
+func waitForJobCompletion(c kubernetes.Interface, ns, job string, timeout int) error {
+	t := time.Duration(timeout) * time.Minute
+	start := time.Now()
+
+	e2elog.Logf("waiting for Job %s/%s to be in state %q", ns, job, batch.JobComplete)
+
+	return wait.PollImmediate(poll, t, func() (bool, error) {
+		j, err := c.BatchV1().Jobs(ns).Get(context.TODO(), job, metav1.GetOptions{})
+		if err != nil {
+			if isRetryableAPIError(err) {
+				return false, nil
+			}
+			return false, fmt.Errorf("failed to get Job: %w", err)
+		}
+
+		if j.Status.CompletionTime != nil {
+			// Job has successfully completed
+			return true, nil
+		}
+
+		e2elog.Logf(
+			"Job %s/%s has not completed yet (%d seconds elapsed)",
+			ns, job, int(time.Since(start).Seconds()))
+		return false, nil
+	})
+}