ceph-csi/e2e/deploy-vault.go

package e2e

import (
	"context"
	"fmt"
	"strings"

	. "github.com/onsi/gomega" // nolint
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/client-go/kubernetes"
	e2elog "k8s.io/kubernetes/test/e2e/framework/log"
)

var (
	vaultExamplePath     = "../examples/kms/vault/"
	vaultServicePath     = "vault.yaml"
	vaultPSPPath         = "vault-psp.yaml"
	vaultRBACPath        = "csi-vaulttokenreview-rbac.yaml"
	vaultConfigPath      = "kms-config.yaml"
	vaultTenantPath      = "tenant-sa.yaml"
	vaultTenantAdminPath = "tenant-sa-admin.yaml"
	vaultUserSecret      = "user-secret.yaml"
)

func deployVault(c kubernetes.Interface, deployTimeout int) {
	// hack to make helm E2E pass as helm charts creates this configmap as part
	// of cephcsi deployment
	err := retryKubectlArgs(
		cephCSINamespace,
		kubectlDelete,
		deployTimeout,
		"cm",
		"ceph-csi-encryption-kms-config",
		"--ignore-not-found=true")
	Expect(err).Should(BeNil())

	createORDeleteVault(kubectlCreate)
	opt := metav1.ListOptions{
		LabelSelector: "app=vault",
	}

	pods, err := c.CoreV1().Pods(cephCSINamespace).List(context.TODO(), opt)
	Expect(err).Should(BeNil())
	Expect(len(pods.Items)).Should(Equal(1))
	name := pods.Items[0].Name
	err = waitForPodInRunningState(name, cephCSINamespace, c, deployTimeout, noError)
	Expect(err).Should(BeNil())
}

func deleteVault() {
	createORDeleteVault(kubectlDelete)
}

func createORDeleteVault(action kubectlAction) {
	data, err := replaceNamespaceInTemplate(vaultExamplePath + vaultServicePath)
	if err != nil {
		e2elog.Failf("failed to read content from %s %v", vaultExamplePath+vaultServicePath, err)
	}

	data = strings.ReplaceAll(data, "vault.default", "vault."+cephCSINamespace)

	data = strings.ReplaceAll(data, "value: default", "value: "+cephCSINamespace)
	err = retryKubectlInput(cephCSINamespace, action, data, deployTimeout)
	if err != nil {
		e2elog.Failf("failed to %s vault statefulset %v", action, err)
	}

	data, err = replaceNamespaceInTemplate(vaultExamplePath + vaultRBACPath)
	if err != nil {
		e2elog.Failf("failed to read content from %s %v", vaultExamplePath+vaultRBACPath, err)
	}
	err = retryKubectlInput(cephCSINamespace, action, data, deployTimeout)
	if err != nil {
		e2elog.Failf("failed to %s vault statefulset %v", action, err)
	}

	data, err = replaceNamespaceInTemplate(vaultExamplePath + vaultConfigPath)
	if err != nil {
		e2elog.Failf("failed to read content from %s %v", vaultExamplePath+vaultConfigPath, err)
	}
	data = strings.ReplaceAll(data, "default", cephCSINamespace)
	err = retryKubectlInput(cephCSINamespace, action, data, deployTimeout)
	if err != nil {
		e2elog.Failf("failed to %s vault configmap %v", action, err)
	}

	data, err = replaceNamespaceInTemplate(vaultExamplePath + vaultPSPPath)
	if err != nil {
		e2elog.Failf("failed to read content from %s %v", vaultExamplePath+vaultPSPPath, err)
	}
	err = retryKubectlInput(cephCSINamespace, action, data, deployTimeout)
	if err != nil {
		e2elog.Failf("failed to %s vault psp %v", action, err)
	}
}

// createTenantServiceAccount uses the tenant-sa.yaml example file to create
// the ServiceAccount for the tenant and configured Hashicorp Vault with a
// kv-store that the ServiceAccount has access to.
func createTenantServiceAccount(c kubernetes.Interface, ns string) error {
	err := createORDeleteTenantServiceAccount(kubectlCreate, ns)
	if err != nil {
		return fmt.Errorf("failed to create ServiceAccount: %w", err)
	}

	// wait for the Job to finish
	const jobName = "vault-tenant-sa"
	err = waitForJobCompletion(c, cephCSINamespace, jobName, deployTimeout)
	if err != nil {
		return fmt.Errorf("job %s/%s did not succeed: %w", cephCSINamespace, jobName, err)
	}

	return nil
}

// deleteTenantServiceAccount removed the ServiceAccount and other objects that
// were created with createTenantServiceAccount.
func deleteTenantServiceAccount(ns string) {
	err := createORDeleteTenantServiceAccount(kubectlDelete, ns)
	Expect(err).Should(BeNil())
}

// createORDeleteTenantServiceAccount is a helper that reads the tenant-sa.yaml
// example file and replaces the default namespaces with the current deployment
// configuration.
func createORDeleteTenantServiceAccount(action kubectlAction, ns string) error {
	err := retryKubectlFile(ns, action, vaultExamplePath+vaultTenantPath, deployTimeout)
	if err != nil {
		return fmt.Errorf("failed to %s tenant ServiceAccount: %w", action, err)
	}

	// the ServiceAccount needs permissions in Vault, the admin job sets that up
	data, err := replaceNamespaceInTemplate(vaultExamplePath + vaultTenantAdminPath)
	if err != nil {
		return fmt.Errorf("failed to read content from %q: %w", vaultExamplePath+vaultTenantAdminPath, err)
	}

	// replace the value for TENANT_NAMESPACE
	data = strings.ReplaceAll(data, "value: tenant", "value: "+ns)

	// replace "default" in the URL to the Vault service
	data = strings.ReplaceAll(data, "vault.default", "vault."+cephCSINamespace)

	err = retryKubectlInput(cephCSINamespace, action, data, deployTimeout)
	if err != nil {
		return fmt.Errorf("failed to %s ServiceAccount: %w", action, err)
	}

	return nil
}