ceph-csi/e2e/deploy-vault.go

150 lines
4.9 KiB
Go
Raw Normal View History

package e2e
import (
"context"
"fmt"
"strings"
. "github.com/onsi/gomega" // nolint
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/client-go/kubernetes"
e2elog "k8s.io/kubernetes/test/e2e/framework/log"
)
var (
vaultExamplePath = "../examples/kms/vault/"
vaultServicePath = "vault.yaml"
vaultPSPPath = "vault-psp.yaml"
vaultRBACPath = "csi-vaulttokenreview-rbac.yaml"
vaultConfigPath = "kms-config.yaml"
vaultTenantPath = "tenant-sa.yaml"
vaultTenantAdminPath = "tenant-sa-admin.yaml"
)
func deployVault(c kubernetes.Interface, deployTimeout int) {
// hack to make helm E2E pass as helm charts creates this configmap as part
// of cephcsi deployment
err := retryKubectlArgs(
cephCSINamespace,
kubectlDelete,
deployTimeout,
"cm",
"ceph-csi-encryption-kms-config",
"--ignore-not-found=true")
Expect(err).Should(BeNil())
createORDeleteVault(kubectlCreate)
opt := metav1.ListOptions{
LabelSelector: "app=vault",
}
pods, err := c.CoreV1().Pods(cephCSINamespace).List(context.TODO(), opt)
Expect(err).Should(BeNil())
Expect(len(pods.Items)).Should(Equal(1))
name := pods.Items[0].Name
err = waitForPodInRunningState(name, cephCSINamespace, c, deployTimeout, noError)
Expect(err).Should(BeNil())
}
func deleteVault() {
createORDeleteVault(kubectlDelete)
}
func createORDeleteVault(action kubectlAction) {
data, err := replaceNamespaceInTemplate(vaultExamplePath + vaultServicePath)
if err != nil {
e2elog.Failf("failed to read content from %s %v", vaultExamplePath+vaultServicePath, err)
}
data = strings.ReplaceAll(data, "vault.default", "vault."+cephCSINamespace)
data = strings.ReplaceAll(data, "value: default", "value: "+cephCSINamespace)
err = retryKubectlInput(cephCSINamespace, action, data, deployTimeout)
if err != nil {
e2elog.Failf("failed to %s vault statefulset %v", action, err)
}
data, err = replaceNamespaceInTemplate(vaultExamplePath + vaultRBACPath)
if err != nil {
e2elog.Failf("failed to read content from %s %v", vaultExamplePath+vaultRBACPath, err)
}
err = retryKubectlInput(cephCSINamespace, action, data, deployTimeout)
if err != nil {
e2elog.Failf("failed to %s vault statefulset %v", action, err)
}
data, err = replaceNamespaceInTemplate(vaultExamplePath + vaultConfigPath)
if err != nil {
e2elog.Failf("failed to read content from %s %v", vaultExamplePath+vaultConfigPath, err)
}
data = strings.ReplaceAll(data, "default", cephCSINamespace)
err = retryKubectlInput(cephCSINamespace, action, data, deployTimeout)
if err != nil {
e2elog.Failf("failed to %s vault configmap %v", action, err)
}
data, err = replaceNamespaceInTemplate(vaultExamplePath + vaultPSPPath)
if err != nil {
e2elog.Failf("failed to read content from %s %v", vaultExamplePath+vaultPSPPath, err)
}
err = retryKubectlInput(cephCSINamespace, action, data, deployTimeout)
if err != nil {
e2elog.Failf("failed to %s vault psp %v", action, err)
}
}
// createTenantServiceAccount uses the tenant-sa.yaml example file to create
// the ServiceAccount for the tenant and configured Hashicorp Vault with a
// kv-store that the ServiceAccount has access to.
func createTenantServiceAccount(c kubernetes.Interface, ns string) error {
err := createORDeleteTenantServiceAccount(kubectlCreate, ns)
if err != nil {
return fmt.Errorf("failed to create ServiceAccount: %w", err)
}
// wait for the Job to finish
const jobName = "vault-tenant-sa"
err = waitForJobCompletion(c, cephCSINamespace, jobName, deployTimeout)
if err != nil {
return fmt.Errorf("job %s/%s did not succeed: %w", cephCSINamespace, jobName, err)
}
return nil
}
// deleteTenantServiceAccount removed the ServiceAccount and other objects that
// were created with createTenantServiceAccount.
func deleteTenantServiceAccount(ns string) {
err := createORDeleteTenantServiceAccount(kubectlDelete, ns)
Expect(err).Should(BeNil())
}
// createORDeleteTenantServiceAccount is a helper that reads the tenant-sa.yaml
// example file and replaces the default namespaces with the current deployment
// configuration.
func createORDeleteTenantServiceAccount(action kubectlAction, ns string) error {
err := retryKubectlFile(ns, action, vaultExamplePath+vaultTenantPath, deployTimeout)
if err != nil {
return fmt.Errorf("failed to %s tenant ServiceAccount: %w", action, err)
}
// the ServiceAccount needs permissions in Vault, the admin job sets that up
data, err := replaceNamespaceInTemplate(vaultExamplePath + vaultTenantAdminPath)
if err != nil {
return fmt.Errorf("failed to read content from %q: %w", vaultExamplePath+vaultTenantAdminPath, err)
}
// replace the value for TENANT_NAMESPACE
data = strings.ReplaceAll(data, "value: tenant", "value: "+ns)
// replace "default" in the URL to the Vault service
data = strings.ReplaceAll(data, "vault.default", "vault."+cephCSINamespace)
err = retryKubectlInput(cephCSINamespace, action, data, deployTimeout)
if err != nil {
return fmt.Errorf("failed to %s ServiceAccount: %w", action, err)
}
return nil
}