From e3c7dea7d667a7057dcf0c011bcc3aed45843c75 Mon Sep 17 00:00:00 2001 From: Niels de Vos Date: Thu, 8 Jul 2021 10:33:17 +0200 Subject: [PATCH] e2e: add test for Vault with ServiceAccount per Tenant Signed-off-by: Niels de Vos --- e2e/deploy-vault.go | 68 +++++++++++++++++++++++++++++++++++++++++---- e2e/kms.go | 6 ++++ e2e/rbd.go | 36 ++++++++++++++++++++++++ e2e/utils.go | 31 +++++++++++++++++++++ 4 files changed, 136 insertions(+), 5 deletions(-) diff --git a/e2e/deploy-vault.go b/e2e/deploy-vault.go index dccb450e7..fbf9c5514 100644 --- a/e2e/deploy-vault.go +++ b/e2e/deploy-vault.go @@ -2,6 +2,7 @@ package e2e import ( "context" + "fmt" "strings" . "github.com/onsi/gomega" // nolint @@ -12,11 +13,13 @@ import ( ) var ( - vaultExamplePath = "../examples/kms/vault/" - vaultServicePath = "vault.yaml" - vaultPSPPath = "vault-psp.yaml" - vaultRBACPath = "csi-vaulttokenreview-rbac.yaml" - vaultConfigPath = "kms-config.yaml" + vaultExamplePath = "../examples/kms/vault/" + vaultServicePath = "vault.yaml" + vaultPSPPath = "vault-psp.yaml" + vaultRBACPath = "csi-vaulttokenreview-rbac.yaml" + vaultConfigPath = "kms-config.yaml" + vaultTenantPath = "tenant-sa.yaml" + vaultTenantAdminPath = "tenant-sa-admin.yaml" ) func deployVault(c kubernetes.Interface, deployTimeout int) { @@ -91,3 +94,58 @@ func createORDeleteVault(action string) { e2elog.Failf("failed to %s vault psp %v", action, err) } } + +// createTenantServiceAccount uses the tenant-sa.yaml example file to create +// the ServiceAccount for the tenant and configured Hashicorp Vault with a +// kv-store that the ServiceAccount has access to. +func createTenantServiceAccount(c kubernetes.Interface, ns string) error { + err := createORDeleteTenantServiceAccount("create", ns) + if err != nil { + return fmt.Errorf("failed to create ServiceAccount: %w", err) + } + + // wait for the Job to finish + const jobName = "vault-tenant-sa" + err = waitForJobCompletion(c, cephCSINamespace, jobName, deployTimeout) + if err != nil { + return fmt.Errorf("job %s/%s did not succeed: %w", cephCSINamespace, jobName, err) + } + + return nil +} + +// deleteTenantServiceAccount removed the ServiceAccount and other objects that +// were created with createTenantServiceAccount. +func deleteTenantServiceAccount(ns string) { + err := createORDeleteTenantServiceAccount("delete", ns) + Expect(err).Should(BeNil()) +} + +// createORDeleteTenantServiceAccount is a helper that reads the tenant-sa.yaml +// example file and replaces the default namespaces with the current deployment +// configuration. +func createORDeleteTenantServiceAccount(action, ns string) error { + _, err := framework.RunKubectl(ns, action, "-f", vaultExamplePath+vaultTenantPath) + if err != nil { + return fmt.Errorf("failed to %s tenant ServiceAccount: %w", action, err) + } + + // the ServiceAccount needs permissions in Vault, the admin job sets that up + data, err := replaceNamespaceInTemplate(vaultExamplePath + vaultTenantAdminPath) + if err != nil { + return fmt.Errorf("failed to read content from %q: %w", vaultExamplePath+vaultTenantAdminPath, err) + } + + // replace the value for TENANT_NAMESPACE + data = strings.ReplaceAll(data, "value: tenant", "value: "+ns) + + // replace "default" in the URL to the Vault service + data = strings.ReplaceAll(data, "vault.default", "vault."+cephCSINamespace) + + _, err = framework.RunKubectlInput(cephCSINamespace, data, action, "-f", "-") + if err != nil { + return fmt.Errorf("failed to %s ServiceAccount: %w", action, err) + } + + return nil +} diff --git a/e2e/kms.go b/e2e/kms.go index aae528a24..2d4d1f78b 100644 --- a/e2e/kms.go +++ b/e2e/kms.go @@ -56,6 +56,12 @@ var ( }, backendPath: defaultVaultBackendPath, } + vaultTenantSAKMS = &vaultConfig{ + simpleKMS: &simpleKMS{ + provider: "vaulttenantsa", + }, + backendPath: "tenant/", + } ) func (sk *simpleKMS) String() string { diff --git a/e2e/rbd.go b/e2e/rbd.go index fd52b8ec5..a1303ee4c 100644 --- a/e2e/rbd.go +++ b/e2e/rbd.go @@ -792,6 +792,42 @@ var _ = Describe("RBD", func() { } }) + By("create a PVC and bind it to an app with encrypted RBD volume with VaultTenantSA KMS", func() { + err := deleteResource(rbdExamplePath + "storageclass.yaml") + if err != nil { + e2elog.Failf("failed to delete storageclass: %v", err) + } + scOpts := map[string]string{ + "encrypted": "true", + "encryptionKMSID": "vault-tenant-sa-test", + } + err = createRBDStorageClass(f.ClientSet, f, defaultSCName, nil, scOpts, deletePolicy) + if err != nil { + e2elog.Failf("failed to create storageclass: %v", err) + } + + err = createTenantServiceAccount(f.ClientSet, f.UniqueName) + if err != nil { + e2elog.Failf("failed to create ServiceAccount: %v", err) + } + defer deleteTenantServiceAccount(f.UniqueName) + + err = validateEncryptedPVCAndAppBinding(pvcPath, appPath, vaultTenantSAKMS, f) + if err != nil { + e2elog.Failf("failed to validate encrypted pvc: %v", err) + } + // validate created backend rbd images + validateRBDImageCount(f, 0, defaultRBDPool) + err = deleteResource(rbdExamplePath + "storageclass.yaml") + if err != nil { + e2elog.Failf("failed to delete storageclass: %v", err) + } + err = createRBDStorageClass(f.ClientSet, f, defaultSCName, nil, nil, deletePolicy) + if err != nil { + e2elog.Failf("failed to create storageclass: %v", err) + } + }) + By("create a PVC and bind it to an app with encrypted RBD volume with SecretsMetadataKMS", func() { err := deleteResource(rbdExamplePath + "storageclass.yaml") if err != nil { diff --git a/e2e/utils.go b/e2e/utils.go index 376a862bf..c7f4b077e 100644 --- a/e2e/utils.go +++ b/e2e/utils.go @@ -13,11 +13,13 @@ import ( "time" snapapi "github.com/kubernetes-csi/external-snapshotter/client/v4/apis/volumesnapshot/v1" + batch "k8s.io/api/batch/v1" v1 "k8s.io/api/core/v1" scv1 "k8s.io/api/storage/v1" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/fields" + "k8s.io/apimachinery/pkg/util/wait" utilyaml "k8s.io/apimachinery/pkg/util/yaml" "k8s.io/client-go/kubernetes" "k8s.io/kubernetes/test/e2e/framework" @@ -1154,3 +1156,32 @@ func k8sVersionGreaterEquals(c kubernetes.Interface, major, minor int) bool { return (v.Major > maj) || (v.Major == maj && v.Minor >= min) } + +// waitForJobCompletion polls the status of the given job and waits until the +// jobs has succeeded or until the timeout is hit. +func waitForJobCompletion(c kubernetes.Interface, ns, job string, timeout int) error { + t := time.Duration(timeout) * time.Minute + start := time.Now() + + e2elog.Logf("waiting for Job %s/%s to be in state %q", ns, job, batch.JobComplete) + + return wait.PollImmediate(poll, t, func() (bool, error) { + j, err := c.BatchV1().Jobs(ns).Get(context.TODO(), job, metav1.GetOptions{}) + if err != nil { + if isRetryableAPIError(err) { + return false, nil + } + return false, fmt.Errorf("failed to get Job: %w", err) + } + + if j.Status.CompletionTime != nil { + // Job has successfully completed + return true, nil + } + + e2elog.Logf( + "Job %s/%s has not completed yet (%d seconds elapsed)", + ns, job, int(time.Since(start).Seconds())) + return false, nil + }) +}