/* Copyright 2021 The Ceph-CSI Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package e2e import ( "context" "fmt" "strings" . "github.com/onsi/gomega" // nolint metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes" e2elog "k8s.io/kubernetes/test/e2e/framework/log" ) var ( vaultExamplePath = "../examples/kms/vault/" vaultServicePath = "vault.yaml" vaultPSPPath = "vault-psp.yaml" vaultRBACPath = "csi-vaulttokenreview-rbac.yaml" vaultConfigPath = "kms-config.yaml" vaultTenantPath = "tenant-sa.yaml" vaultTenantAdminPath = "tenant-sa-admin.yaml" vaultUserSecret = "user-secret.yaml" ) func deployVault(c kubernetes.Interface, deployTimeout int) { // hack to make helm E2E pass as helm charts creates this configmap as part // of cephcsi deployment err := retryKubectlArgs( cephCSINamespace, kubectlDelete, deployTimeout, "cm", "ceph-csi-encryption-kms-config", "--ignore-not-found=true") Expect(err).Should(BeNil()) createORDeleteVault(kubectlCreate) opt := metav1.ListOptions{ LabelSelector: "app=vault", } pods, err := c.CoreV1().Pods(cephCSINamespace).List(context.TODO(), opt) Expect(err).Should(BeNil()) Expect(len(pods.Items)).Should(Equal(1)) name := pods.Items[0].Name err = waitForPodInRunningState(name, cephCSINamespace, c, deployTimeout, noError) Expect(err).Should(BeNil()) } func deleteVault() { createORDeleteVault(kubectlDelete) } func createORDeleteVault(action kubectlAction) { data, err := replaceNamespaceInTemplate(vaultExamplePath + vaultServicePath) if err != nil { e2elog.Failf("failed to read content from %s %v", vaultExamplePath+vaultServicePath, err) } data = strings.ReplaceAll(data, "vault.default", "vault."+cephCSINamespace) data = strings.ReplaceAll(data, "value: default", "value: "+cephCSINamespace) err = retryKubectlInput(cephCSINamespace, action, data, deployTimeout) if err != nil { e2elog.Failf("failed to %s vault statefulset %v", action, err) } data, err = replaceNamespaceInTemplate(vaultExamplePath + vaultRBACPath) if err != nil { e2elog.Failf("failed to read content from %s %v", vaultExamplePath+vaultRBACPath, err) } err = retryKubectlInput(cephCSINamespace, action, data, deployTimeout) if err != nil { e2elog.Failf("failed to %s vault statefulset %v", action, err) } data, err = replaceNamespaceInTemplate(vaultExamplePath + vaultConfigPath) if err != nil { e2elog.Failf("failed to read content from %s %v", vaultExamplePath+vaultConfigPath, err) } data = strings.ReplaceAll(data, "default", cephCSINamespace) err = retryKubectlInput(cephCSINamespace, action, data, deployTimeout) if err != nil { e2elog.Failf("failed to %s vault configmap %v", action, err) } data, err = replaceNamespaceInTemplate(vaultExamplePath + vaultPSPPath) if err != nil { e2elog.Failf("failed to read content from %s %v", vaultExamplePath+vaultPSPPath, err) } err = retryKubectlInput(cephCSINamespace, action, data, deployTimeout) if err != nil { e2elog.Failf("failed to %s vault psp %v", action, err) } } // createTenantServiceAccount uses the tenant-sa.yaml example file to create // the ServiceAccount for the tenant and configured Hashicorp Vault with a // kv-store that the ServiceAccount has access to. func createTenantServiceAccount(c kubernetes.Interface, ns string) error { err := createORDeleteTenantServiceAccount(kubectlCreate, ns) if err != nil { return fmt.Errorf("failed to create ServiceAccount: %w", err) } // wait for the Job to finish const jobName = "vault-tenant-sa" err = waitForJobCompletion(c, cephCSINamespace, jobName, deployTimeout) if err != nil { return fmt.Errorf("job %s/%s did not succeed: %w", cephCSINamespace, jobName, err) } return nil } // deleteTenantServiceAccount removed the ServiceAccount and other objects that // were created with createTenantServiceAccount. func deleteTenantServiceAccount(ns string) { err := createORDeleteTenantServiceAccount(kubectlDelete, ns) Expect(err).Should(BeNil()) } // createORDeleteTenantServiceAccount is a helper that reads the tenant-sa.yaml // example file and replaces the default namespaces with the current deployment // configuration. func createORDeleteTenantServiceAccount(action kubectlAction, ns string) error { err := retryKubectlFile(ns, action, vaultExamplePath+vaultTenantPath, deployTimeout) if err != nil { return fmt.Errorf("failed to %s tenant ServiceAccount: %w", action, err) } // the ServiceAccount needs permissions in Vault, the admin job sets that up data, err := replaceNamespaceInTemplate(vaultExamplePath + vaultTenantAdminPath) if err != nil { return fmt.Errorf("failed to read content from %q: %w", vaultExamplePath+vaultTenantAdminPath, err) } // replace the value for TENANT_NAMESPACE data = strings.ReplaceAll(data, "value: tenant", "value: "+ns) // replace "default" in the URL to the Vault service data = strings.ReplaceAll(data, "vault.default", "vault."+cephCSINamespace) err = retryKubectlInput(cephCSINamespace, action, data, deployTimeout) if err != nil { return fmt.Errorf("failed to %s ServiceAccount: %w", action, err) } return nil }