diff --git a/docs/design/proposals/encryption-with-vault-tokens.md b/docs/design/proposals/encryption-with-vault-tokens.md index e245b5a3d..7ae74a1a3 100644 --- a/docs/design/proposals/encryption-with-vault-tokens.md +++ b/docs/design/proposals/encryption-with-vault-tokens.md @@ -180,8 +180,73 @@ data: vaultBackendPath: "secret/ceph-csi-encryption/" vaultTLSServerName: "vault.infosec.example.org" vaultCAFromSecret: "vault-infosec-ca" + vaultClientCertFromSecret: "vault-client-cert" + vaultClientCertKeyFromSecret: "vault-client-cert-key" vaultCAVerify: "true" ``` Only parameters with the `vault`-prefix may be changed in the Kubernetes ConfigMap of the Tenant. + +### Certificates stored in the Tenants Kubernetes Namespace + +The `vaultCAFromSecret` , `vaultClientCertFromSecret` and +`vaultClientCertKeyFromSecret` secrets should be created in the namespace where +Ceph-CSI is deployed. The sample of secrets for the CA and client Certificate. + +#### CA Certificate to verify Vault server TLS certificate + +```yaml +--- +apiVersion: v1 +kind: secret +metadata: + name: vault-infosec-ca +stringData: + ca.cert: | + MIIC2DCCAcCgAwIBAgIBATANBgkqh... +``` + +#### Client Certificate for Vault connection + +```yaml +--- +apiVersion: v1 +kind: secret +metadata: + name: vault-client-cert +stringData: + tls.cert: | + BATANBgkqcCgAwIBAgIBATANBAwI... +``` + +#### Client Certificate key for Vault connection + +```yaml +--- +apiVersion: v1 +kind: secret +metadata: + name: vault-client-cert-key +stringData: + tls.key: | + KNSC2DVVXcCgkqcCgAwIBAgIwewrvx... +``` + +Its also possible that a user can create a single secret for the certificates +and update the configuration to fetch certificates from a secret. + +```yaml +--- +apiVersion: v1 +kind: secret +metadata: + name: vault-certificates +stringData: + ca.cert: | + MIIC2DCCAcCgAwIBAgIBATANBgkqh... + tls.cert: | + BATANBgkqcCgAwIBAgIBATANBAwI... + tls.key: | + KNSC2DVVXcCgkqcCgAwIBAgIwewrvx... +``` diff --git a/internal/util/crypto.go b/internal/util/crypto.go index 7b0c61e70..13fe216a7 100644 --- a/internal/util/crypto.go +++ b/internal/util/crypto.go @@ -141,7 +141,7 @@ func GetKMS(tenant, kmsID string, secrets map[string]string) (EncryptionKMS, err case kmsTypeVault: return InitVaultKMS(kmsID, kmsConfig, secrets) case kmsTypeVaultTokens: - return InitVaultTokensKMS(tenant, kmsID, kmsConfig, secrets) + return InitVaultTokensKMS(tenant, kmsID, kmsConfig) } return nil, fmt.Errorf("unknown encryption KMS type %s", kmsType) } diff --git a/internal/util/vault.go b/internal/util/vault.go index ec08df756..b44fb4063 100644 --- a/internal/util/vault.go +++ b/internal/util/vault.go @@ -113,7 +113,7 @@ func setConfigString(option *string, config map[string]interface{}, key string) // vc.connectVault(). // // nolint:gocyclo // iterating through many config options, not complex at all. -func (vc *vaultConnection) initConnection(kmsID string, config map[string]interface{}, secrets map[string]string) error { +func (vc *vaultConnection) initConnection(kmsID string, config map[string]interface{}) error { vaultConfig := make(map[string]interface{}) keyContext := make(map[string]string) @@ -183,18 +183,6 @@ func (vc *vaultConnection) initConnection(kmsID string, config map[string]interf if errors.Is(err, errConfigOptionInvalid) { return err } - // ignore errConfigOptionMissing, no default was set - if vaultCAFromSecret != "" { - caPEM, ok := secrets[vaultCAFromSecret] - if !ok { - return fmt.Errorf("missing vault CA in secret %s", vaultCAFromSecret) - } - - vaultConfig[api.EnvVaultCACert], err = createTempFile("vault-ca-cert", []byte(caPEM)) - if err != nil { - return fmt.Errorf("failed to create temporary file for Vault CA: %w", err) - } - } // update the existing config only if no config is available yet if vc.keyContext != nil { @@ -215,6 +203,38 @@ func (vc *vaultConnection) initConnection(kmsID string, config map[string]interf return nil } +// initCertificates sets VAULT_* environment variables in the vc.vaultConfig map, +// these settings will be used when connecting to the Vault service with +// vc.connectVault(). +// +func (vc *vaultConnection) initCertificates(config map[string]interface{}, secrets map[string]string) error { + vaultConfig := make(map[string]interface{}) + + vaultCAFromSecret := "" // optional + err := setConfigString(&vaultCAFromSecret, config, "vaultCAFromSecret") + if errors.Is(err, errConfigOptionInvalid) { + return err + } + // ignore errConfigOptionMissing, no default was set + if vaultCAFromSecret != "" { + caPEM, ok := secrets[vaultCAFromSecret] + if !ok { + return fmt.Errorf("missing vault CA in secret %s", vaultCAFromSecret) + } + + vaultConfig[api.EnvVaultCACert], err = createTempFile("vault-ca-cert", []byte(caPEM)) + if err != nil { + return fmt.Errorf("failed to create temporary file for Vault CA: %w", err) + } + // update the existing config + for key, value := range vaultConfig { + vc.vaultConfig[key] = value + } + } + + return nil +} + // connectVault creates a new connection to Vault. This should be called after // filling vc.vaultConfig. func (vc *vaultConnection) connectVault() error { @@ -242,11 +262,16 @@ func (vc *vaultConnection) Destroy() { // InitVaultKMS returns an interface to HashiCorp Vault KMS. func InitVaultKMS(kmsID string, config map[string]interface{}, secrets map[string]string) (EncryptionKMS, error) { kms := &VaultKMS{} - err := kms.initConnection(kmsID, config, secrets) + err := kms.initConnection(kmsID, config) if err != nil { return nil, fmt.Errorf("failed to initialize Vault connection: %w", err) } + err = kms.initCertificates(config, secrets) + if err != nil { + return nil, fmt.Errorf("failed to initialize Vault certificates: %w", err) + } + vaultAuthPath := vaultDefaultAuthPath err = setConfigString(&vaultAuthPath, config, "vaultAuthPath") if err != nil { diff --git a/internal/util/vault_tokens.go b/internal/util/vault_tokens.go index f445989f6..3d41691f1 100644 --- a/internal/util/vault_tokens.go +++ b/internal/util/vault_tokens.go @@ -20,9 +20,11 @@ import ( "context" "errors" "fmt" + "os" "github.com/hashicorp/vault/api" loss "github.com/libopenstorage/secrets" + apierrs "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) @@ -62,6 +64,8 @@ Example JSON structure in the KMS config is, "vaultBackendPath": "secret/", "vaultTLSServerName": "vault.default.svc.cluster.local", "vaultCAFromSecret": "vault-ca", + "vaultClientCertFromSecret": "vault-client-cert", + "vaultClientCertKeyFromSecret": "vault-client-cert-key", "vaultCAVerify": "false", "tenantConfigName": "ceph-csi-kms-config", "tenantTokenName": "ceph-csi-kms-token", @@ -89,9 +93,9 @@ type VaultTokensKMS struct { } // InitVaultTokensKMS returns an interface to HashiCorp Vault KMS. -func InitVaultTokensKMS(tenant, kmsID string, config map[string]interface{}, secrets map[string]string) (EncryptionKMS, error) { +func InitVaultTokensKMS(tenant, kmsID string, config map[string]interface{}) (EncryptionKMS, error) { kms := &VaultTokensKMS{} - err := kms.initConnection(kmsID, config, secrets) + err := kms.initConnection(kmsID, config) if err != nil { return nil, fmt.Errorf("failed to initialize Vault connection: %w", err) } @@ -100,7 +104,7 @@ func InitVaultTokensKMS(tenant, kmsID string, config map[string]interface{}, sec kms.ConfigName = vaultTokensDefaultConfigName kms.TokenName = vaultTokensDefaultTokenName - err = kms.parseConfig(config, secrets) + err = kms.parseConfig(config) if err != nil { return nil, err } @@ -116,7 +120,7 @@ func InitVaultTokensKMS(tenant, kmsID string, config map[string]interface{}, sec tenantConfig, ok := tenants[tenant] if ok { // override connection details from the tenant - err = kms.parseConfig(tenantConfig, secrets) + err = kms.parseConfig(tenantConfig) if err != nil { return nil, err } @@ -132,6 +136,10 @@ func InitVaultTokensKMS(tenant, kmsID string, config map[string]interface{}, sec return nil, fmt.Errorf("failed fetching token from %s/%s: %w", tenant, kms.TokenName, err) } + err = kms.initCertificates(config) + if err != nil { + return nil, fmt.Errorf("failed to initialize Vault certificates: %w", err) + } // connect to the Vault service err = kms.connectVault() if err != nil { @@ -144,8 +152,8 @@ func InitVaultTokensKMS(tenant, kmsID string, config map[string]interface{}, sec // parseConfig updates the kms.vaultConfig with the options from config and // secrets. This method can be called multiple times, i.e. to override // configuration options from tenants. -func (kms *VaultTokensKMS) parseConfig(config map[string]interface{}, secrets map[string]string) error { - err := kms.initConnection(kms.EncryptionKMSID, config, secrets) +func (kms *VaultTokensKMS) parseConfig(config map[string]interface{}) error { + err := kms.initConnection(kms.EncryptionKMSID, config) if err != nil { return err } @@ -163,6 +171,97 @@ func (kms *VaultTokensKMS) parseConfig(config map[string]interface{}, secrets ma return nil } +// initCertificates updates the kms.vaultConfig with the options from config +// it calls the kubernetes secrets and get the required data. + +// nolint:gocyclo // iterating through many config options, not complex at all. +func (kms *VaultTokensKMS) initCertificates(config map[string]interface{}) error { + vaultConfig := make(map[string]interface{}) + + csiNamespace := os.Getenv("POD_NAMESPACE") + vaultCAFromSecret := "" // optional + err := setConfigString(&vaultCAFromSecret, config, "vaultCAFromSecret") + if errors.Is(err, errConfigOptionInvalid) { + return err + } + // ignore errConfigOptionMissing, no default was set + if vaultCAFromSecret != "" { + cert, cErr := getCertificate(kms.Tenant, vaultCAFromSecret, "ca.cert") + if cErr != nil && !apierrs.IsNotFound(err) { + return fmt.Errorf("failed to get CA certificate from secret %s: %w", vaultCAFromSecret, cErr) + } + // if the certificate is not present in tenant namespace get it from + // cephcsi pod namespace + if apierrs.IsNotFound(cErr) { + cert, cErr = getCertificate(csiNamespace, vaultCAFromSecret, "ca.cert") + if cErr != nil { + return fmt.Errorf("failed to get CA certificate from secret %s: %w", vaultCAFromSecret, cErr) + } + } + vaultConfig[api.EnvVaultCACert], err = createTempFile("vault-ca-cert", []byte(cert)) + if err != nil { + return fmt.Errorf("failed to create temporary file for Vault CA: %w", err) + } + } + + vaultClientCertFromSecret := "" // optional + err = setConfigString(&vaultClientCertFromSecret, config, "vaultClientCertFromSecret") + if errors.Is(err, errConfigOptionInvalid) { + return err + } + // ignore errConfigOptionMissing, no default was set + if vaultClientCertFromSecret != "" { + cert, cErr := getCertificate(kms.Tenant, vaultClientCertFromSecret, "tls.cert") + if cErr != nil && !apierrs.IsNotFound(cErr) { + return fmt.Errorf("failed to get client certificate from secret %s: %w", vaultClientCertFromSecret, cErr) + } + // if the certificate is not present in tenant namespace get it from + // cephcsi pod namespace + if apierrs.IsNotFound(cErr) { + cert, cErr = getCertificate(csiNamespace, vaultClientCertFromSecret, "tls.cert") + if cErr != nil { + return fmt.Errorf("failed to get client certificate from secret %s: %w", vaultCAFromSecret, cErr) + } + } + vaultConfig[api.EnvVaultClientCert], err = createTempFile("vault-ca-cert", []byte(cert)) + if err != nil { + return fmt.Errorf("failed to create temporary file for Vault client certificate: %w", err) + } + } + + vaultClientCertKeyFromSecret := "" // optional + err = setConfigString(&vaultClientCertKeyFromSecret, config, "vaultClientCertKeyFromSecret") + if errors.Is(err, errConfigOptionInvalid) { + return err + } + + // ignore errConfigOptionMissing, no default was set + if vaultClientCertKeyFromSecret != "" { + certKey, err := getCertificate(kms.Tenant, vaultClientCertKeyFromSecret, "tls.key") + if err != nil && !apierrs.IsNotFound(err) { + return fmt.Errorf("failed to get client certificate key from secret %s: %w", vaultClientCertKeyFromSecret, err) + } + // if the certificate is not present in tenant namespace get it from + // cephcsi pod namespace + if apierrs.IsNotFound(err) { + certKey, err = getCertificate(csiNamespace, vaultClientCertFromSecret, "tls.key") + if err != nil { + return fmt.Errorf("failed to get client certificate key from secret %s: %w", vaultCAFromSecret, err) + } + } + vaultConfig[api.EnvVaultClientKey], err = createTempFile("vault-client-cert-key", []byte(certKey)) + if err != nil { + return fmt.Errorf("failed to create temporary file for Vault client cert key: %w", err) + } + } + + for key, value := range vaultConfig { + kms.vaultConfig[key] = value + } + + return nil +} + // GetPassphrase returns passphrase from Vault. The passphrase is stored in a // data.data.passphrase structure. func (kms *VaultTokensKMS) GetPassphrase(key string) (string, error) { @@ -225,3 +324,18 @@ func getToken(tenant, tokenName string) (string, error) { return string(token), nil } + +func getCertificate(tenant, secretName, key string) (string, error) { + c := NewK8sClient() + secret, err := c.CoreV1().Secrets(tenant).Get(context.TODO(), secretName, metav1.GetOptions{}) + if err != nil { + return "", err + } + + cert, ok := secret.Data[key] + if !ok { + return "", errors.New("failed to parse certificates") + } + + return string(cert), nil +} diff --git a/internal/util/vault_tokens_test.go b/internal/util/vault_tokens_test.go index 9864f192d..8e7c8d080 100644 --- a/internal/util/vault_tokens_test.go +++ b/internal/util/vault_tokens_test.go @@ -26,10 +26,9 @@ func TestParseConfig(t *testing.T) { kms := VaultTokensKMS{} config := make(map[string]interface{}) - secrets := make(map[string]string) // empty config map - err := kms.parseConfig(config, secrets) + err := kms.parseConfig(config) if !errors.Is(err, errConfigOptionMissing) { t.Errorf("unexpected error (%T): %s", err, err) } @@ -40,7 +39,7 @@ func TestParseConfig(t *testing.T) { config["tenantTokenName"] = vaultTokensDefaultTokenName // parsing with all required options - err = kms.parseConfig(config, secrets) + err = kms.parseConfig(config) switch { case err != nil: t.Errorf("unexpected error: %s", err) @@ -53,7 +52,7 @@ func TestParseConfig(t *testing.T) { // tenant "bob" uses a different kms.ConfigName bob := make(map[string]interface{}) bob["tenantConfigName"] = "the-config-from-bob" - err = kms.parseConfig(bob, secrets) + err = kms.parseConfig(bob) switch { case err != nil: t.Errorf("unexpected error: %s", err) @@ -75,10 +74,9 @@ func TestInitVaultTokensKMS(t *testing.T) { } config := make(map[string]interface{}) - secrets := make(map[string]string) // empty config map - _, err := InitVaultTokensKMS("bob", "vault-tokens-config", config, secrets) + _, err := InitVaultTokensKMS("bob", "vault-tokens-config", config) if !errors.Is(err, errConfigOptionMissing) { t.Errorf("unexpected error (%T): %s", err, err) } @@ -87,7 +85,7 @@ func TestInitVaultTokensKMS(t *testing.T) { config["vaultAddress"] = "https://vault.default.cluster.svc" // parsing with all required options - _, err = InitVaultTokensKMS("bob", "vault-tokens-config", config, secrets) + _, err = InitVaultTokensKMS("bob", "vault-tokens-config", config) if err != nil && !strings.Contains(err.Error(), "VAULT_TOKEN") { t.Errorf("unexpected error: %s", err) } @@ -97,7 +95,7 @@ func TestInitVaultTokensKMS(t *testing.T) { config["tenants"] = tenants // empty tenants list - _, err = InitVaultTokensKMS("bob", "vault-tokens-config", config, secrets) + _, err = InitVaultTokensKMS("bob", "vault-tokens-config", config) if err != nil && !strings.Contains(err.Error(), "VAULT_TOKEN") { t.Errorf("unexpected error: %s", err) } @@ -107,7 +105,7 @@ func TestInitVaultTokensKMS(t *testing.T) { config["tenants"].(map[string]interface{})["bob"] = bob bob["vaultAddress"] = "https://vault.bob.example.org" - _, err = InitVaultTokensKMS("bob", "vault-tokens-config", config, secrets) + _, err = InitVaultTokensKMS("bob", "vault-tokens-config", config) if err != nil && !strings.Contains(err.Error(), "VAULT_TOKEN") { t.Errorf("unexpected error: %s", err) }