mirror of
https://github.com/ceph/ceph-csi.git
synced 2025-06-14 10:53:34 +00:00
Adds per volume encryption with Vault integration
- adds proposal document for PVC encryption from PR448 - adds per-volume encription by generating encryption passphrase for each volume and storing it in a KMS - adds HashiCorp Vault integration as a KMS for encryption passphrases - avoids encrypting volume second time if it was already encrypted but no file system created - avoids unnecessary checks if volume is a mapped device when encryption was not requested - prevents resizing encrypted volumes (it is not currently supported) - prevents creating snapshots from encrypted volumes to prevent attack on encryption key (security guard until re-encryption of volumes implemented) Signed-off-by: Vasyl Purchel vasyl.purchel@workday.com Signed-off-by: Andrea Baglioni andrea.baglioni@workday.com Fixes #420 Fixes #744
This commit is contained in:
Vasyl Purchel
committed by
mergify[bot]
mergify[bot]
parent
1adef00c86
commit
419ad0dd8e
54
e2e/deploy-vault.go
Normal file
54
e2e/deploy-vault.go
Normal file
@ -0,0 +1,54 @@
|
||||
package e2e
|
||||
|
||||
import (
|
||||
. "github.com/onsi/gomega" // nolint
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
"k8s.io/client-go/kubernetes"
|
||||
"k8s.io/kubernetes/test/e2e/framework"
|
||||
e2elog "k8s.io/kubernetes/test/e2e/framework/log"
|
||||
)
|
||||
|
||||
var (
|
||||
vaultExamplePath = "../examples/kms/vault/"
|
||||
vaultServicePath = "vault.yaml"
|
||||
vaultPSPPath = "vault-psp.yaml"
|
||||
vaultRBACPath = "csi-vaulttokenreview-rbac.yaml"
|
||||
vaultConfigPath = "kms-config.yaml"
|
||||
)
|
||||
|
||||
func deployVault(c kubernetes.Interface, deployTimeout int) {
|
||||
framework.RunKubectlOrDie("create", "-f", vaultExamplePath+vaultServicePath)
|
||||
framework.RunKubectlOrDie("create", "-f", vaultExamplePath+vaultPSPPath)
|
||||
framework.RunKubectlOrDie("create", "-f", vaultExamplePath+vaultRBACPath)
|
||||
framework.RunKubectlOrDie("create", "-f", vaultExamplePath+vaultConfigPath)
|
||||
|
||||
opt := metav1.ListOptions{
|
||||
LabelSelector: "app=vault",
|
||||
}
|
||||
|
||||
pods, err := c.CoreV1().Pods("default").List(opt)
|
||||
Expect(err).Should(BeNil())
|
||||
Expect(len(pods.Items)).Should(Equal(1))
|
||||
name := pods.Items[0].Name
|
||||
err = waitForPodInRunningState(name, "default", c, deployTimeout)
|
||||
Expect(err).Should(BeNil())
|
||||
}
|
||||
|
||||
func deleteVault() {
|
||||
_, err := framework.RunKubectl("delete", "-f", vaultExamplePath+vaultServicePath)
|
||||
if err != nil {
|
||||
e2elog.Logf("failed to delete vault statefull set %v", err)
|
||||
}
|
||||
_, err = framework.RunKubectl("delete", "-f", vaultExamplePath+vaultRBACPath)
|
||||
if err != nil {
|
||||
e2elog.Logf("failed to delete vault statefull set %v", err)
|
||||
}
|
||||
_, err = framework.RunKubectl("delete", "-f", vaultExamplePath+vaultConfigPath)
|
||||
if err != nil {
|
||||
e2elog.Logf("failed to delete vault config map %v", err)
|
||||
}
|
||||
_, err = framework.RunKubectl("delete", "-f", vaultExamplePath+vaultPSPPath)
|
||||
if err != nil {
|
||||
e2elog.Logf("failed to delete vault psp %v", err)
|
||||
}
|
||||
}
|
||||
18
e2e/rbd.go
18
e2e/rbd.go
@ -76,7 +76,7 @@ var _ = Describe("RBD", func() {
|
||||
deployRBDPlugin()
|
||||
createRBDStorageClass(f.ClientSet, f, make(map[string]string))
|
||||
createRBDSecret(f.ClientSet, f)
|
||||
|
||||
deployVault(f.ClientSet, deployTimeout)
|
||||
})
|
||||
|
||||
AfterEach(func() {
|
||||
@ -91,6 +91,7 @@ var _ = Describe("RBD", func() {
|
||||
deleteResource(rbdExamplePath + "secret.yaml")
|
||||
deleteResource(rbdExamplePath + "storageclass.yaml")
|
||||
// deleteResource(rbdExamplePath + "snapshotclass.yaml")
|
||||
deleteVault()
|
||||
})
|
||||
|
||||
Context("Test RBD CSI", func() {
|
||||
@ -135,7 +136,20 @@ var _ = Describe("RBD", func() {
|
||||
By("create a PVC and Bind it to an app with encrypted RBD volume", func() {
|
||||
deleteResource(rbdExamplePath + "storageclass.yaml")
|
||||
createRBDStorageClass(f.ClientSet, f, map[string]string{"encrypted": "true"})
|
||||
validateEncryptedPVCAndAppBinding(pvcPath, appPath, f)
|
||||
validateEncryptedPVCAndAppBinding(pvcPath, appPath, "", f)
|
||||
deleteResource(rbdExamplePath + "storageclass.yaml")
|
||||
createRBDStorageClass(f.ClientSet, f, make(map[string]string))
|
||||
})
|
||||
|
||||
By("create a PVC and Bind it to an app with encrypted RBD volume with Vault KMS", func() {
|
||||
deleteResource(rbdExamplePath + "storageclass.yaml")
|
||||
scOpts := map[string]string{
|
||||
"encrypted": "true",
|
||||
"encryptionKMS": "vault",
|
||||
"encryptionKMSID": "vault-test",
|
||||
}
|
||||
createRBDStorageClass(f.ClientSet, f, scOpts)
|
||||
validateEncryptedPVCAndAppBinding(pvcPath, appPath, "vault", f)
|
||||
deleteResource(rbdExamplePath + "storageclass.yaml")
|
||||
createRBDStorageClass(f.ClientSet, f, make(map[string]string))
|
||||
})
|
||||
|
||||
86
e2e/utils.go
86
e2e/utils.go
@ -31,7 +31,9 @@ import (
|
||||
)
|
||||
|
||||
const (
|
||||
rookNS = "rook-ceph"
|
||||
rookNS = "rook-ceph"
|
||||
vaultAddr = "http://vault.default.svc.cluster.local:8200"
|
||||
vaultSecretNs = "/secret/ceph-csi/" // nolint: gosec, #nosec
|
||||
)
|
||||
|
||||
var poll = 2 * time.Second
|
||||
@ -108,14 +110,14 @@ func waitForDeploymentComplete(name, ns string, c clientset.Interface, t int) er
|
||||
return nil
|
||||
}
|
||||
|
||||
func execCommandInPod(f *framework.Framework, c, ns string, opt *metav1.ListOptions) (string, string) {
|
||||
func getCommandInPodOpts(f *framework.Framework, c, ns string, opt *metav1.ListOptions) framework.ExecOptions {
|
||||
cmd := []string{"/bin/sh", "-c", c}
|
||||
podList, err := f.PodClientNS(ns).List(*opt)
|
||||
framework.ExpectNoError(err)
|
||||
Expect(podList.Items).NotTo(BeNil())
|
||||
Expect(err).Should(BeNil())
|
||||
|
||||
podPot := framework.ExecOptions{
|
||||
return framework.ExecOptions{
|
||||
Command: cmd,
|
||||
PodName: podList.Items[0].Name,
|
||||
Namespace: ns,
|
||||
@ -125,6 +127,10 @@ func execCommandInPod(f *framework.Framework, c, ns string, opt *metav1.ListOpti
|
||||
CaptureStderr: true,
|
||||
PreserveWhitespace: true,
|
||||
}
|
||||
}
|
||||
|
||||
func execCommandInPod(f *framework.Framework, c, ns string, opt *metav1.ListOptions) (string, string) {
|
||||
podPot := getCommandInPodOpts(f, c, ns, opt)
|
||||
stdOut, stdErr, err := f.ExecWithOptions(podPot)
|
||||
if stdErr != "" {
|
||||
e2elog.Logf("stdErr occurred: %v", stdErr)
|
||||
@ -133,6 +139,15 @@ func execCommandInPod(f *framework.Framework, c, ns string, opt *metav1.ListOpti
|
||||
return stdOut, stdErr
|
||||
}
|
||||
|
||||
func execCommandInPodAndAllowFail(f *framework.Framework, c, ns string, opt *metav1.ListOptions) (string, string) {
|
||||
podPot := getCommandInPodOpts(f, c, ns, opt)
|
||||
stdOut, stdErr, err := f.ExecWithOptions(podPot)
|
||||
if err != nil {
|
||||
e2elog.Logf("command %s failed: %v", c, err)
|
||||
}
|
||||
return stdOut, stdErr
|
||||
}
|
||||
|
||||
func getMons(ns string, c kubernetes.Interface) []string {
|
||||
opt := metav1.ListOptions{
|
||||
LabelSelector: "app=rook-ceph-mon",
|
||||
@ -557,39 +572,22 @@ func validatePVCAndAppBinding(pvcPath, appPath string, f *framework.Framework) {
|
||||
}
|
||||
}
|
||||
|
||||
func getImageIDFromPVC(pvcNamespace, pvcName string, f *framework.Framework) (string, error) {
|
||||
func getRBDImageIds(pvcNamespace, pvcName string, f *framework.Framework) (string, string, error) {
|
||||
c := f.ClientSet.CoreV1()
|
||||
pvc, err := c.PersistentVolumeClaims(pvcNamespace).Get(pvcName, metav1.GetOptions{})
|
||||
if err != nil {
|
||||
return "", err
|
||||
return "", "", err
|
||||
}
|
||||
|
||||
pv, err := c.PersistentVolumes().Get(pvc.Spec.VolumeName, metav1.GetOptions{})
|
||||
if err != nil {
|
||||
return "", err
|
||||
return "", "", err
|
||||
}
|
||||
|
||||
imageIDRegex := regexp.MustCompile(`(\w+\-?){5}$`)
|
||||
imageID := imageIDRegex.FindString(pv.Spec.CSI.VolumeHandle)
|
||||
|
||||
return imageID, nil
|
||||
}
|
||||
func getRBDImageSpec(pvcNamespace, pvcName string, f *framework.Framework) (string, error) {
|
||||
imageID, err := getImageIDFromPVC(pvcNamespace, pvcName, f)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
return fmt.Sprintf("replicapool/csi-vol-%s", imageID), nil
|
||||
}
|
||||
|
||||
func getCephFSVolumeName(pvcNamespace, pvcName string, f *framework.Framework) (string, error) {
|
||||
imageID, err := getImageIDFromPVC(pvcNamespace, pvcName, f)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
return fmt.Sprintf("csi-vol-%s", imageID), nil
|
||||
return fmt.Sprintf("csi-vol-%s", imageID), pv.Spec.CSI.VolumeHandle, nil
|
||||
}
|
||||
|
||||
func getImageMeta(rbdImageSpec, metaKey string, f *framework.Framework) (string, error) {
|
||||
@ -616,13 +614,31 @@ func getMountType(appName, appNamespace, mountPath string, f *framework.Framewor
|
||||
return strings.TrimSpace(stdOut), nil
|
||||
}
|
||||
|
||||
func validateEncryptedPVCAndAppBinding(pvcPath, appPath string, f *framework.Framework) {
|
||||
// readVaultSecret method will execute few commands to try read the secret for
|
||||
// specified key from inside the vault container:
|
||||
// * authenticate with vault and ignore any stdout (we do not need output)
|
||||
// * issue get request for particular key
|
||||
// resulting in stdOut (first entry in tuple) - output that contains the key
|
||||
// or stdErr (second entry in tuple) - error getting the key
|
||||
func readVaultSecret(key string, f *framework.Framework) (string, string) {
|
||||
loginCmd := fmt.Sprintf("vault login -address=%s sample_root_token_id > /dev/null", vaultAddr)
|
||||
readSecret := fmt.Sprintf("vault kv get -address=%s %s%s", vaultAddr, vaultSecretNs, key)
|
||||
cmd := fmt.Sprintf("%s && %s", loginCmd, readSecret)
|
||||
opt := metav1.ListOptions{
|
||||
LabelSelector: "app=vault",
|
||||
}
|
||||
stdOut, stdErr := execCommandInPodAndAllowFail(f, cmd, "default", &opt)
|
||||
return strings.TrimSpace(stdOut), strings.TrimSpace(stdErr)
|
||||
}
|
||||
|
||||
func validateEncryptedPVCAndAppBinding(pvcPath, appPath, kms string, f *framework.Framework) {
|
||||
pvc, app := createPVCAndAppBinding(pvcPath, appPath, f)
|
||||
|
||||
rbdImageSpec, err := getRBDImageSpec(pvc.Namespace, pvc.Name, f)
|
||||
rbdImageID, rbdImageHandle, err := getRBDImageIds(pvc.Namespace, pvc.Name, f)
|
||||
if err != nil {
|
||||
Fail(err.Error())
|
||||
}
|
||||
rbdImageSpec := fmt.Sprintf("replicapool/%s", rbdImageID)
|
||||
encryptedState, err := getImageMeta(rbdImageSpec, ".rbd.csi.ceph.com/encrypted", f)
|
||||
if err != nil {
|
||||
Fail(err.Error())
|
||||
@ -636,10 +652,26 @@ func validateEncryptedPVCAndAppBinding(pvcPath, appPath string, f *framework.Fra
|
||||
}
|
||||
Expect(mountType).To(Equal("crypt"))
|
||||
|
||||
if kms == "vault" {
|
||||
// check new passphrase created
|
||||
_, stdErr := readVaultSecret(rbdImageHandle, f)
|
||||
if stdErr != "" {
|
||||
Fail(fmt.Sprintf("failed to read passphrase from vault: %s", stdErr))
|
||||
}
|
||||
}
|
||||
|
||||
err = deletePVCAndApp("", f, pvc, app)
|
||||
if err != nil {
|
||||
Fail(err.Error())
|
||||
}
|
||||
|
||||
if kms == "vault" {
|
||||
// check new passphrase created
|
||||
stdOut, _ := readVaultSecret(rbdImageHandle, f)
|
||||
if stdOut != "" {
|
||||
Fail(fmt.Sprintf("passphrase found in vault while should be deleted: %s", stdOut))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func deletePodWithLabel(label string) error {
|
||||
@ -794,7 +826,7 @@ func validateNormalUserPVCAccess(pvcPath string, f *framework.Framework) {
|
||||
// }
|
||||
|
||||
func deleteBackingCephFSVolume(f *framework.Framework, pvc *v1.PersistentVolumeClaim) error {
|
||||
volname, err := getCephFSVolumeName(pvc.Namespace, pvc.Name, f)
|
||||
volname, _, err := getRBDImageIds(pvc.Namespace, pvc.Name, f)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user