package e2e import ( "context" "fmt" "strings" . "github.com/onsi/gomega" // nolint metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes" "k8s.io/kubernetes/test/e2e/framework" e2elog "k8s.io/kubernetes/test/e2e/framework/log" ) var ( vaultExamplePath = "../examples/kms/vault/" vaultServicePath = "vault.yaml" vaultPSPPath = "vault-psp.yaml" vaultRBACPath = "csi-vaulttokenreview-rbac.yaml" vaultConfigPath = "kms-config.yaml" vaultTenantPath = "tenant-sa.yaml" vaultTenantAdminPath = "tenant-sa-admin.yaml" ) func deployVault(c kubernetes.Interface, deployTimeout int) { // hack to make helm E2E pass as helm charts creates this configmap as part // of cephcsi deployment _, err := framework.RunKubectl( cephCSINamespace, "delete", "cm", "ceph-csi-encryption-kms-config", "--namespace", cephCSINamespace, "--ignore-not-found=true") Expect(err).Should(BeNil()) createORDeleteVault(kubectlCreate) opt := metav1.ListOptions{ LabelSelector: "app=vault", } pods, err := c.CoreV1().Pods(cephCSINamespace).List(context.TODO(), opt) Expect(err).Should(BeNil()) Expect(len(pods.Items)).Should(Equal(1)) name := pods.Items[0].Name err = waitForPodInRunningState(name, cephCSINamespace, c, deployTimeout, noError) Expect(err).Should(BeNil()) } func deleteVault() { createORDeleteVault(kubectlDelete) } func createORDeleteVault(action kubectlAction) { data, err := replaceNamespaceInTemplate(vaultExamplePath + vaultServicePath) if err != nil { e2elog.Failf("failed to read content from %s %v", vaultExamplePath+vaultServicePath, err) } data = strings.ReplaceAll(data, "vault.default", "vault."+cephCSINamespace) data = strings.ReplaceAll(data, "value: default", "value: "+cephCSINamespace) err = retryKubectlInput(cephCSINamespace, action, data, deployTimeout) if err != nil { e2elog.Failf("failed to %s vault statefulset %v", action, err) } data, err = replaceNamespaceInTemplate(vaultExamplePath + vaultRBACPath) if err != nil { e2elog.Failf("failed to read content from %s %v", vaultExamplePath+vaultRBACPath, err) } err = retryKubectlInput(cephCSINamespace, action, data, deployTimeout) if err != nil { e2elog.Failf("failed to %s vault statefulset %v", action, err) } data, err = replaceNamespaceInTemplate(vaultExamplePath + vaultConfigPath) if err != nil { e2elog.Failf("failed to read content from %s %v", vaultExamplePath+vaultConfigPath, err) } data = strings.ReplaceAll(data, "default", cephCSINamespace) err = retryKubectlInput(cephCSINamespace, action, data, deployTimeout) if err != nil { e2elog.Failf("failed to %s vault configmap %v", action, err) } data, err = replaceNamespaceInTemplate(vaultExamplePath + vaultPSPPath) if err != nil { e2elog.Failf("failed to read content from %s %v", vaultExamplePath+vaultPSPPath, err) } err = retryKubectlInput(cephCSINamespace, action, data, deployTimeout) if err != nil { e2elog.Failf("failed to %s vault psp %v", action, err) } } // createTenantServiceAccount uses the tenant-sa.yaml example file to create // the ServiceAccount for the tenant and configured Hashicorp Vault with a // kv-store that the ServiceAccount has access to. func createTenantServiceAccount(c kubernetes.Interface, ns string) error { err := createORDeleteTenantServiceAccount(kubectlCreate, ns) if err != nil { return fmt.Errorf("failed to create ServiceAccount: %w", err) } // wait for the Job to finish const jobName = "vault-tenant-sa" err = waitForJobCompletion(c, cephCSINamespace, jobName, deployTimeout) if err != nil { return fmt.Errorf("job %s/%s did not succeed: %w", cephCSINamespace, jobName, err) } return nil } // deleteTenantServiceAccount removed the ServiceAccount and other objects that // were created with createTenantServiceAccount. func deleteTenantServiceAccount(ns string) { err := createORDeleteTenantServiceAccount(kubectlDelete, ns) Expect(err).Should(BeNil()) } // createORDeleteTenantServiceAccount is a helper that reads the tenant-sa.yaml // example file and replaces the default namespaces with the current deployment // configuration. func createORDeleteTenantServiceAccount(action kubectlAction, ns string) error { err := retryKubectlFile(ns, action, vaultExamplePath+vaultTenantPath, deployTimeout) if err != nil { return fmt.Errorf("failed to %s tenant ServiceAccount: %w", action, err) } // the ServiceAccount needs permissions in Vault, the admin job sets that up data, err := replaceNamespaceInTemplate(vaultExamplePath + vaultTenantAdminPath) if err != nil { return fmt.Errorf("failed to read content from %q: %w", vaultExamplePath+vaultTenantAdminPath, err) } // replace the value for TENANT_NAMESPACE data = strings.ReplaceAll(data, "value: tenant", "value: "+ns) // replace "default" in the URL to the Vault service data = strings.ReplaceAll(data, "vault.default", "vault."+cephCSINamespace) err = retryKubectlInput(cephCSINamespace, action, data, deployTimeout) if err != nil { return fmt.Errorf("failed to %s ServiceAccount: %w", action, err) } return nil }