mirrors/ceph-csi
1
0
Fork 0
mirror of https://github.com/ceph/ceph-csi.git synced 2024-12-23 21:40:20 +00:00
Code Issues Packages Projects Releases Wiki Activity

e2e: refactor KMS configuration and usage

Browse Source 
This adds a new `kmsConfig` interface that can be used to validate
different KMS services and setting. It makes checking for the available
support easier, and fetching the passphrase simpler.

The basicKMS mirrors the current validation of the KMS implementations
that use secrets and metadata. vaultKMS can be used to validate the
passphrase stored in a Vault service.

Signed-off-by: Niels de Vos <ndevos@redhat.com>
This commit is contained in:
Niels de Vos 2021-07-09 13:49:23 +02:00 committed by mergify[bot]
parent 42f8d8d446
commit 855f9080d1
4 changed files with 143 additions and 77 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

102
e2e/kms.go Normal file
View File

@ -0,0 +1,102 @@
package e2e
import (
"fmt"
"strings"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/kubernetes/test/e2e/framework"
)
const (
// defaultVaultBackendPath is the default VAULT_BACKEND_PATH for secrets
defaultVaultBackendPath = "secret/"
)
// kmsConfig is an interface that should be used when passing a configuration
// for a KMS to validation functions. This allows the validation functions to
// work independently from the actual KMS.
type kmsConfig interface {
canGetPassphrase() bool
getPassphrase(f *framework.Framework, key string) (string, string)
}
// simpleKMS is to be used for KMS configurations that do not offer options to
// validate the passphrase stored in the KMS.
type simpleKMS struct {
provider string
}
// vaultConfig describes the configuration of the Hashicorp Vault service that
// is used to store the encryption passphrase in.
type vaultConfig struct {
*simpleKMS
backendPath string
}
// The following variables describe different KMS services as they are defined
// in the kms-config.yaml file. These variables can be passed on to validation
// functions when a StorageClass has a KMS enabled.
var (
noKMS kmsConfig = nil
secretsMetadataKMS = &simpleKMS{
provider: "secrets-metadata",
}
vaultKMS = &vaultConfig{
simpleKMS: &simpleKMS{
provider: "vault",
},
backendPath: defaultVaultBackendPath + "ceph-csi/",
}
vaultTokensKMS = &vaultConfig{
simpleKMS: &simpleKMS{
provider: "vaulttokens",
},
backendPath: defaultVaultBackendPath,
}
)
func (sk *simpleKMS) String() string {
return sk.provider
}
// canGetPassphrase returns false for the basic KMS configuration as there is
// currently no way to fetch the passphrase.
func (sk *simpleKMS) canGetPassphrase() bool {
return false
}
func (sk *simpleKMS) getPassphrase(f *framework.Framework, key string) (string, string) {
return "", ""
}
func (vc *vaultConfig) String() string {
return fmt.Sprintf("%s (backend path %q)", vc.simpleKMS, vc.backendPath)
}
// canGetPassphrase returns true for the Hashicorp Vault KMS configurations as
// the Vault CLI can be used to retrieve the passphrase.
func (vc *vaultConfig) canGetPassphrase() bool {
return true
}
// getPassphrase method will execute few commands to try read the secret for
// specified key from inside the vault container:
// * authenticate with vault and ignore any stdout (we do not need output)
// * issue get request for particular key
// resulting in stdOut (first entry in tuple) - output that contains the key
// or stdErr (second entry in tuple) - error getting the key.
func (vc *vaultConfig) getPassphrase(f *framework.Framework, key string) (string, string) {
vaultAddr := fmt.Sprintf("http://vault.%s.svc.cluster.local:8200", cephCSINamespace)
loginCmd := fmt.Sprintf("vault login -address=%s sample_root_token_id > /dev/null", vaultAddr)
readSecret := fmt.Sprintf("vault kv get -address=%s -field=data %s%s",
vaultAddr, vc.backendPath, key)
cmd := fmt.Sprintf("%s && %s", loginCmd, readSecret)
opt := metav1.ListOptions{
LabelSelector: "app=vault",
}
stdOut, stdErr := execCommandInPodAndAllowFail(f, cmd, cephCSINamespace, &opt)
return strings.TrimSpace(stdOut), strings.TrimSpace(stdErr)
}

26
e2e/rbd.go
View File

@ -697,7 +697,7 @@ var _ = Describe("RBD", func() {
if err != nil {
e2elog.Failf("failed to create storageclass with error %v", err)
}
err = validateEncryptedPVCAndAppBinding(pvcPath, appPath, "", f)
err = validateEncryptedPVCAndAppBinding(pvcPath, appPath, noKMS, f)
if err != nil {
e2elog.Failf("failed to validate encrypted pvc with error %v", err)
}
@ -726,7 +726,7 @@ var _ = Describe("RBD", func() {
if err != nil {
e2elog.Failf("failed to create storageclass with error %v", err)
}
err = validateEncryptedPVCAndAppBinding(pvcPath, appPath, "vault", f)
err = validateEncryptedPVCAndAppBinding(pvcPath, appPath, vaultKMS, f)
if err != nil {
e2elog.Failf("failed to validate encrypted pvc with error %v", err)
}
@ -769,7 +769,7 @@ var _ = Describe("RBD", func() {
e2elog.Failf("failed to create Secret with tenant token: %v", err)
}
err = validateEncryptedPVCAndAppBinding(pvcPath, appPath, "vaulttokens", f)
err = validateEncryptedPVCAndAppBinding(pvcPath, appPath, vaultTokensKMS, f)
if err != nil {
e2elog.Failf("failed to validate encrypted pvc with error %v", err)
}
@ -805,7 +805,7 @@ var _ = Describe("RBD", func() {
if err != nil {
e2elog.Failf("failed to create storageclass with error %v", err)
}
err = validateEncryptedPVCAndAppBinding(pvcPath, appPath, "", f)
err = validateEncryptedPVCAndAppBinding(pvcPath, appPath, noKMS, f)
if err != nil {
e2elog.Failf("failed to validate encrypted pvc with error %v", err)
}
@ -848,7 +848,7 @@ var _ = Describe("RBD", func() {
e2elog.Failf("failed to create user Secret: %v", err)
}
err = validateEncryptedPVCAndAppBinding(pvcPath, appPath, "", f)
err = validateEncryptedPVCAndAppBinding(pvcPath, appPath, noKMS, f)
if err != nil {
e2elog.Failf("failed to validate encrypted pvc: %v", err)
}
@ -900,7 +900,7 @@ var _ = Describe("RBD", func() {
e2elog.Failf("failed to create user Secret: %v", err)
}
err = validateEncryptedPVCAndAppBinding(pvcPath, appPath, "", f)
err = validateEncryptedPVCAndAppBinding(pvcPath, appPath, noKMS, f)
if err != nil {
e2elog.Failf("failed to validate encrypted pvc: %v", err)
}
@ -965,7 +965,7 @@ var _ = Describe("RBD", func() {
snapshotPath,
pvcClonePath,
appClonePath,
noKms,
noKMS,
f)
}
})
@ -979,7 +979,7 @@ var _ = Describe("RBD", func() {
appPath,
pvcSmartClonePath,
appSmartClonePath,
noKms,
noKMS,
noPVCValidation,
f)
}
@ -1001,7 +1001,7 @@ var _ = Describe("RBD", func() {
e2elog.Failf("failed to create storageclass with error %v", err)
}
validatePVCClone(1, pvcPath, appPath, pvcSmartClonePath, appSmartClonePath, noKms, isThickPVC, f)
validatePVCClone(1, pvcPath, appPath, pvcSmartClonePath, appSmartClonePath, noKMS, isThickPVC, f)
err = deleteResource(rbdExamplePath + "storageclass.yaml")
if err != nil {
@ -1031,7 +1031,7 @@ var _ = Describe("RBD", func() {
e2elog.Failf("failed to create storageclass with error %v", err)
}
validatePVCSnapshot(1, pvcPath, appPath, snapshotPath, pvcClonePath, appClonePath, "vault", f)
validatePVCSnapshot(1, pvcPath, appPath, snapshotPath, pvcClonePath, appClonePath, vaultKMS, f)
err = deleteResource(rbdExamplePath + "storageclass.yaml")
if err != nil {
@ -1061,7 +1061,7 @@ var _ = Describe("RBD", func() {
e2elog.Failf("failed to create storageclass with error %v", err)
}
validatePVCClone(1, pvcPath, appPath, pvcSmartClonePath, appSmartClonePath, "secrets-metadata", isEncryptedPVC, f)
validatePVCClone(1, pvcPath, appPath, pvcSmartClonePath, appSmartClonePath, secretsMetadataKMS, isEncryptedPVC, f)
err = deleteResource(rbdExamplePath + "storageclass.yaml")
if err != nil {
@ -1091,7 +1091,7 @@ var _ = Describe("RBD", func() {
e2elog.Failf("failed to create storageclass with error %v", err)
}
validatePVCClone(1, pvcPath, appPath, pvcSmartClonePath, appSmartClonePath, "vault", isEncryptedPVC, f)
validatePVCClone(1, pvcPath, appPath, pvcSmartClonePath, appSmartClonePath, vaultKMS, isEncryptedPVC, f)
err = deleteResource(rbdExamplePath + "storageclass.yaml")
if err != nil {
@ -1122,7 +1122,7 @@ var _ = Describe("RBD", func() {
rawAppPath,
pvcBlockSmartClonePath,
appBlockSmartClonePath,
noKms,
noKMS,
noPVCValidation,
f)
}

14
e2e/rbd_helper.go
View File

@ -253,10 +253,6 @@ func validateImageOwner(pvcPath string, f *framework.Framework) error {
return deletePVCAndValidatePV(f.ClientSet, pvc, deployTimeout)
}
func kmsIsVault(kms string) bool {
return kms == "vault"
}
func logErrors(f *framework.Framework, msg string, wgErrs []error) int {
failures := 0
for i, err := range wgErrs {
@ -395,7 +391,7 @@ func validateCloneInDifferentPool(f *framework.Framework, snapshotPool, cloneSc,
return nil
}
func validateEncryptedPVCAndAppBinding(pvcPath, appPath, kms string, f *framework.Framework) error {
func validateEncryptedPVCAndAppBinding(pvcPath, appPath string, kms kmsConfig, f *framework.Framework) error {
pvc, app, err := createPVCAndAppBinding(pvcPath, appPath, f, deployTimeout)
if err != nil {
return err
@ -411,9 +407,9 @@ func validateEncryptedPVCAndAppBinding(pvcPath, appPath, kms string, f *framewor
return err
}
if kmsIsVault(kms) || kms == vaultTokens {
if kms != noKMS && kms.canGetPassphrase() {
// check new passphrase created
_, stdErr := readVaultSecret(imageData.csiVolumeHandle, kmsIsVault(kms), f)
_, stdErr := kms.getPassphrase(f, imageData.csiVolumeHandle)
if stdErr != "" {
return fmt.Errorf("failed to read passphrase from vault: %s", stdErr)
}
@ -424,9 +420,9 @@ func validateEncryptedPVCAndAppBinding(pvcPath, appPath, kms string, f *framewor
return err
}
if kmsIsVault(kms) || kms == vaultTokens {
if kms != noKMS && kms.canGetPassphrase() {
// check new passphrase created
stdOut, _ := readVaultSecret(imageData.csiVolumeHandle, kmsIsVault(kms), f)
stdOut, _ := kms.getPassphrase(f, imageData.csiVolumeHandle)
if stdOut != "" {
return fmt.Errorf("passphrase found in vault while should be deleted: %s", stdOut)
}

78
e2e/utils.go
View File

@ -28,11 +28,6 @@ import (
const (
defaultNs = "default"
defaultSCName = ""
// vaultBackendPath is the default VAULT_BACKEND_PATH for secrets
vaultBackendPath = "secret/"
// vaultPassphrasePath is an advanced configuration option, only
// available for the VaultKMS (not VaultTokensKMS) provider.
vaultPassphrasePath = "ceph-csi/"
rookToolBoxPodLabel = "app=rook-ceph-tools"
rbdMountOptions = "mountOptions"
@ -44,11 +39,7 @@ const (
appKey = "app"
appLabel = "write-data-in-pod"
// vaultTokens KMS type
vaultTokens = "vaulttokens"
noError = ""
noKms = ""
noError = ""
)
var (
@ -65,13 +56,11 @@ var (
rookNamespace string
radosNamespace string
ns string
vaultAddr string
poll = 2 * time.Second
)
func initResources() {
ns = fmt.Sprintf("--namespace=%v", cephCSINamespace)
vaultAddr = fmt.Sprintf("http://vault.%s.svc.cluster.local:8200", cephCSINamespace)
}
func getMons(ns string, c kubernetes.Interface) ([]string, error) {
@ -226,29 +215,6 @@ func getMountType(appName, appNamespace, mountPath string, f *framework.Framewor
return strings.TrimSpace(stdOut), nil
}
// readVaultSecret method will execute few commands to try read the secret for
// specified key from inside the vault container:
// * authenticate with vault and ignore any stdout (we do not need output)
// * issue get request for particular key
// resulting in stdOut (first entry in tuple) - output that contains the key
// or stdErr (second entry in tuple) - error getting the key.
func readVaultSecret(key string, usePassphrasePath bool, f *framework.Framework) (string, string) {
extraPath := vaultPassphrasePath
if !usePassphrasePath {
extraPath = ""
}
loginCmd := fmt.Sprintf("vault login -address=%s sample_root_token_id > /dev/null", vaultAddr)
readSecret := fmt.Sprintf("vault kv get -address=%s -field=data %s%s%s",
vaultAddr, vaultBackendPath, extraPath, key)
cmd := fmt.Sprintf("%s && %s", loginCmd, readSecret)
opt := metav1.ListOptions{
LabelSelector: "app=vault",
}
stdOut, stdErr := execCommandInPodAndAllowFail(f, cmd, cephCSINamespace, &opt)
return strings.TrimSpace(stdOut), strings.TrimSpace(stdErr)
}
func validateNormalUserPVCAccess(pvcPath string, f *framework.Framework) error {
pvc, err := loadPVC(pvcPath)
if err != nil {
@ -544,7 +510,8 @@ func writeDataAndCalChecksum(app *v1.Pod, opt *metav1.ListOptions, f *framework.
// nolint:gocyclo,gocognit,nestif // reduce complexity
func validatePVCClone(
totalCount int,
sourcePvcPath, sourceAppPath, clonePvcPath, clonePvcAppPath, kms string,
sourcePvcPath, sourceAppPath, clonePvcPath, clonePvcAppPath string,
kms kmsConfig,
validatePVC validateFunc,
f *framework.Framework) {
var wg sync.WaitGroup
@ -611,8 +578,8 @@ func validatePVCClone(
LabelSelector: fmt.Sprintf("%s=%s", appKey, label[appKey]),
}
wgErrs[n] = createPVCAndApp(name, f, &p, &a, deployTimeout)
if wgErrs[n] == nil && kms != noKms {
if kmsIsVault(kms) || kms == vaultTokens {
if wgErrs[n] == nil && kms != noKMS {
if kms.canGetPassphrase() {
imageData, sErr := getImageInfoFromPVC(p.Namespace, name, f)
if sErr != nil {
wgErrs[n] = fmt.Errorf(
@ -623,7 +590,7 @@ func validatePVCClone(
sErr)
} else {
// check new passphrase created
stdOut, stdErr := readVaultSecret(imageData.csiVolumeHandle, kmsIsVault(kms), f)
stdOut, stdErr := kms.getPassphrase(f, imageData.csiVolumeHandle)
if stdOut != "" {
e2elog.Logf("successfully read the passphrase from vault: %s", stdOut)
}
@ -646,7 +613,7 @@ func validatePVCClone(
e2elog.Logf("checksum didn't match. checksum=%s and checksumclone=%s", checkSum, checkSumClone)
}
}
if wgErrs[n] == nil && validatePVC != nil && kms != noKms {
if wgErrs[n] == nil && validatePVC != nil && kms != noKMS {
wgErrs[n] = validatePVC(f, &p, &a)
}
wg.Done()
@ -697,8 +664,8 @@ func validatePVCClone(
p.Spec.DataSource.Name = name
var imageData imageInfoFromPVC
var sErr error
if kms != noKms {
if kmsIsVault(kms) || kms == vaultTokens {
if kms != noKMS {
if kms.canGetPassphrase() {
imageData, sErr = getImageInfoFromPVC(p.Namespace, name, f)
if sErr != nil {
wgErrs[n] = fmt.Errorf(
@ -712,10 +679,10 @@ func validatePVCClone(
}
if wgErrs[n] == nil {
wgErrs[n] = deletePVCAndApp(name, f, &p, &a)
if wgErrs[n] == nil && kms != noKms {
if kmsIsVault(kms) || kms == vaultTokens {
if wgErrs[n] == nil && kms != noKMS {
if kms.canGetPassphrase() {
// check passphrase deleted
stdOut, _ := readVaultSecret(imageData.csiVolumeHandle, kmsIsVault(kms), f)
stdOut, _ := kms.getPassphrase(f, imageData.csiVolumeHandle)
if stdOut != "" {
wgErrs[n] = fmt.Errorf("passphrase found in vault while should be deleted: %s", stdOut)
}
@ -744,7 +711,8 @@ func validatePVCClone(
// nolint:gocyclo,gocognit,nestif // reduce complexity
func validatePVCSnapshot(
totalCount int,
pvcPath, appPath, snapshotPath, pvcClonePath, appClonePath, kms string,
pvcPath, appPath, snapshotPath, pvcClonePath, appClonePath string,
kms kmsConfig,
f *framework.Framework) {
var wg sync.WaitGroup
wgErrs := make([]error, totalCount)
@ -797,8 +765,8 @@ func validatePVCSnapshot(
go func(n int, s snapapi.VolumeSnapshot) {
s.Name = fmt.Sprintf("%s%d", f.UniqueName, n)
wgErrs[n] = createSnapshot(&s, deployTimeout)
if wgErrs[n] == nil && kms != noKms {
if kmsIsVault(kms) || kms == vaultTokens {
if wgErrs[n] == nil && kms != noKMS {
if kms.canGetPassphrase() {
content, sErr := getVolumeSnapshotContent(s.Namespace, s.Name)
if sErr != nil {
wgErrs[n] = fmt.Errorf(
@ -808,7 +776,7 @@ func validatePVCSnapshot(
sErr)
} else {
// check new passphrase created
_, stdErr := readVaultSecret(*content.Status.SnapshotHandle, kmsIsVault(kms), f)
_, stdErr := kms.getPassphrase(f, *content.Status.SnapshotHandle)
if stdErr != "" {
wgErrs[n] = fmt.Errorf("failed to read passphrase from vault: %s", stdErr)
}
@ -874,7 +842,7 @@ func validatePVCSnapshot(
checkSumClone)
}
}
if wgErrs[n] == nil && kms != "" {
if wgErrs[n] == nil && kms != noKMS {
wgErrs[n] = isEncryptedPVC(f, &p, &a)
}
wg.Done()
@ -977,8 +945,8 @@ func validatePVCSnapshot(
s.Name = fmt.Sprintf("%s%d", f.UniqueName, n)
content := &snapapi.VolumeSnapshotContent{}
var err error
if kms != noKms {
if kmsIsVault(kms) || kms == vaultTokens {
if kms != noKMS {
if kms.canGetPassphrase() {
content, err = getVolumeSnapshotContent(s.Namespace, s.Name)
if err != nil {
wgErrs[n] = fmt.Errorf(
@ -991,10 +959,10 @@ func validatePVCSnapshot(
}
if wgErrs[n] == nil {
wgErrs[n] = deleteSnapshot(&s, deployTimeout)
if wgErrs[n] == nil && kms != noKms {
if kmsIsVault(kms) || kms == vaultTokens {
if wgErrs[n] == nil && kms != noKMS {
if kms.canGetPassphrase() {
// check passphrase deleted
stdOut, _ := readVaultSecret(*content.Status.SnapshotHandle, kmsIsVault(kms), f)
stdOut, _ := kms.getPassphrase(f, *content.Status.SnapshotHandle)
if stdOut != "" {
wgErrs[n] = fmt.Errorf("passphrase found in vault while should be deleted: %s", stdOut)
}