util: add support for vault certificates

Added a option to pass the client certificate
and the client certificate key for the vault token
based encryption.

Signed-off-by: Madhu Rajanna <madhupr007@gmail.com>

docs/design/proposals/encryption-with-vault-tokens.md

@@ -180,8 +180,73 @@ data:
   vaultBackendPath: "secret/ceph-csi-encryption/"
   vaultTLSServerName: "vault.infosec.example.org"
   vaultCAFromSecret: "vault-infosec-ca"
+  vaultClientCertFromSecret: "vault-client-cert"
+  vaultClientCertKeyFromSecret: "vault-client-cert-key"
   vaultCAVerify: "true"
 ```
 
 Only parameters with the `vault`-prefix may be changed in the Kubernetes
 ConfigMap of the Tenant.
+
+### Certificates stored in the Tenants Kubernetes Namespace
+
+The `vaultCAFromSecret` , `vaultClientCertFromSecret` and
+`vaultClientCertKeyFromSecret` secrets should be created in the namespace where
+Ceph-CSI is deployed. The sample of secrets for the CA and client Certificate.
+
+#### CA Certificate to verify Vault server TLS certificate
+
+```yaml
+---
+apiVersion: v1
+kind: secret
+metadata:
+  name: vault-infosec-ca
+stringData:
+  ca.cert: |
+    MIIC2DCCAcCgAwIBAgIBATANBgkqh...
+```
+
+#### Client Certificate for Vault connection
+
+```yaml
+---
+apiVersion: v1
+kind: secret
+metadata:
+  name: vault-client-cert
+stringData:
+  tls.cert: |
+    BATANBgkqcCgAwIBAgIBATANBAwI...
+```
+
+#### Client Certificate key for Vault connection
+
+```yaml
+---
+apiVersion: v1
+kind: secret
+metadata:
+  name: vault-client-cert-key
+stringData:
+  tls.key: |
+    KNSC2DVVXcCgkqcCgAwIBAgIwewrvx...
+```
+
+Its also possible that a user can create a single secret for the certificates
+and update the configuration to fetch certificates from a secret.
+
+```yaml
+---
+apiVersion: v1
+kind: secret
+metadata:
+  name: vault-certificates
+stringData:
+  ca.cert: |
+    MIIC2DCCAcCgAwIBAgIBATANBgkqh...
+  tls.cert: |
+    BATANBgkqcCgAwIBAgIBATANBAwI...
+  tls.key: |
+    KNSC2DVVXcCgkqcCgAwIBAgIwewrvx...
+```

internal/util/crypto.go

@@ -141,7 +141,7 @@ func GetKMS(tenant, kmsID string, secrets map[string]string) (EncryptionKMS, err
 	case kmsTypeVault:
 		return InitVaultKMS(kmsID, kmsConfig, secrets)
 	case kmsTypeVaultTokens:
-		return InitVaultTokensKMS(tenant, kmsID, kmsConfig, secrets)
+		return InitVaultTokensKMS(tenant, kmsID, kmsConfig)
 	}
 	return nil, fmt.Errorf("unknown encryption KMS type %s", kmsType)
 }

internal/util/vault.go

@@ -113,7 +113,7 @@ func setConfigString(option *string, config map[string]interface{}, key string)
 // vc.connectVault().
 //
 // nolint:gocyclo // iterating through many config options, not complex at all.
-func (vc *vaultConnection) initConnection(kmsID string, config map[string]interface{}, secrets map[string]string) error {
+func (vc *vaultConnection) initConnection(kmsID string, config map[string]interface{}) error {
 	vaultConfig := make(map[string]interface{})
 	keyContext := make(map[string]string)
 
@@ -183,18 +183,6 @@ func (vc *vaultConnection) initConnection(kmsID string, config map[string]interf
 	if errors.Is(err, errConfigOptionInvalid) {
 		return err
 	}
-	// ignore errConfigOptionMissing, no default was set
-	if vaultCAFromSecret != "" {
-		caPEM, ok := secrets[vaultCAFromSecret]
-		if !ok {
-			return fmt.Errorf("missing vault CA in secret %s", vaultCAFromSecret)
-		}
-
-		vaultConfig[api.EnvVaultCACert], err = createTempFile("vault-ca-cert", []byte(caPEM))
-		if err != nil {
-			return fmt.Errorf("failed to create temporary file for Vault CA: %w", err)
-		}
-	}
 
 	// update the existing config only if no config is available yet
 	if vc.keyContext != nil {
@@ -215,6 +203,38 @@ func (vc *vaultConnection) initConnection(kmsID string, config map[string]interf
 	return nil
 }
 
+// initCertificates sets VAULT_* environment variables in the vc.vaultConfig map,
+// these settings will be used when connecting to the Vault service with
+// vc.connectVault().
+//
+func (vc *vaultConnection) initCertificates(config map[string]interface{}, secrets map[string]string) error {
+	vaultConfig := make(map[string]interface{})
+
+	vaultCAFromSecret := "" // optional
+	err := setConfigString(&vaultCAFromSecret, config, "vaultCAFromSecret")
+	if errors.Is(err, errConfigOptionInvalid) {
+		return err
+	}
+	// ignore errConfigOptionMissing, no default was set
+	if vaultCAFromSecret != "" {
+		caPEM, ok := secrets[vaultCAFromSecret]
+		if !ok {
+			return fmt.Errorf("missing vault CA in secret %s", vaultCAFromSecret)
+		}
+
+		vaultConfig[api.EnvVaultCACert], err = createTempFile("vault-ca-cert", []byte(caPEM))
+		if err != nil {
+			return fmt.Errorf("failed to create temporary file for Vault CA: %w", err)
+		}
+		// update the existing config
+		for key, value := range vaultConfig {
+			vc.vaultConfig[key] = value
+		}
+	}
+
+	return nil
+}
+
 // connectVault creates a new connection to Vault. This should be called after
 // filling vc.vaultConfig.
 func (vc *vaultConnection) connectVault() error {
@@ -242,11 +262,16 @@ func (vc *vaultConnection) Destroy() {
 // InitVaultKMS returns an interface to HashiCorp Vault KMS.
 func InitVaultKMS(kmsID string, config map[string]interface{}, secrets map[string]string) (EncryptionKMS, error) {
 	kms := &VaultKMS{}
-	err := kms.initConnection(kmsID, config, secrets)
+	err := kms.initConnection(kmsID, config)
 	if err != nil {
 		return nil, fmt.Errorf("failed to initialize Vault connection: %w", err)
 	}
 
+	err = kms.initCertificates(config, secrets)
+	if err != nil {
+		return nil, fmt.Errorf("failed to initialize Vault certificates: %w", err)
+	}
+
 	vaultAuthPath := vaultDefaultAuthPath
 	err = setConfigString(&vaultAuthPath, config, "vaultAuthPath")
 	if err != nil {

internal/util/vault_tokens.go

@@ -20,9 +20,11 @@ import (
 	"context"
 	"errors"
 	"fmt"
+	"os"
 
 	"github.com/hashicorp/vault/api"
 	loss "github.com/libopenstorage/secrets"
+	apierrs "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 )
 
@@ -62,6 +64,8 @@ Example JSON structure in the KMS config is,
         "vaultBackendPath": "secret/",
         "vaultTLSServerName": "vault.default.svc.cluster.local",
         "vaultCAFromSecret": "vault-ca",
+        "vaultClientCertFromSecret": "vault-client-cert",
+        "vaultClientCertKeyFromSecret": "vault-client-cert-key",
         "vaultCAVerify": "false",
         "tenantConfigName": "ceph-csi-kms-config",
         "tenantTokenName": "ceph-csi-kms-token",
@@ -89,9 +93,9 @@ type VaultTokensKMS struct {
 }
 
 // InitVaultTokensKMS returns an interface to HashiCorp Vault KMS.
-func InitVaultTokensKMS(tenant, kmsID string, config map[string]interface{}, secrets map[string]string) (EncryptionKMS, error) {
+func InitVaultTokensKMS(tenant, kmsID string, config map[string]interface{}) (EncryptionKMS, error) {
 	kms := &VaultTokensKMS{}
-	err := kms.initConnection(kmsID, config, secrets)
+	err := kms.initConnection(kmsID, config)
 	if err != nil {
 		return nil, fmt.Errorf("failed to initialize Vault connection: %w", err)
 	}
@@ -100,7 +104,7 @@ func InitVaultTokensKMS(tenant, kmsID string, config map[string]interface{}, sec
 	kms.ConfigName = vaultTokensDefaultConfigName
 	kms.TokenName = vaultTokensDefaultTokenName
 
-	err = kms.parseConfig(config, secrets)
+	err = kms.parseConfig(config)
 	if err != nil {
 		return nil, err
 	}
@@ -116,7 +120,7 @@ func InitVaultTokensKMS(tenant, kmsID string, config map[string]interface{}, sec
 				tenantConfig, ok := tenants[tenant]
 				if ok {
 					// override connection details from the tenant
-					err = kms.parseConfig(tenantConfig, secrets)
+					err = kms.parseConfig(tenantConfig)
 					if err != nil {
 						return nil, err
 					}
@@ -132,6 +136,10 @@ func InitVaultTokensKMS(tenant, kmsID string, config map[string]interface{}, sec
 		return nil, fmt.Errorf("failed fetching token from %s/%s: %w", tenant, kms.TokenName, err)
 	}
 
+	err = kms.initCertificates(config)
+	if err != nil {
+		return nil, fmt.Errorf("failed to initialize Vault certificates: %w", err)
+	}
 	// connect to the Vault service
 	err = kms.connectVault()
 	if err != nil {
@@ -144,8 +152,8 @@ func InitVaultTokensKMS(tenant, kmsID string, config map[string]interface{}, sec
 // parseConfig updates the kms.vaultConfig with the options from config and
 // secrets. This method can be called multiple times, i.e. to override
 // configuration options from tenants.
-func (kms *VaultTokensKMS) parseConfig(config map[string]interface{}, secrets map[string]string) error {
+func (kms *VaultTokensKMS) parseConfig(config map[string]interface{}) error {
-	err := kms.initConnection(kms.EncryptionKMSID, config, secrets)
+	err := kms.initConnection(kms.EncryptionKMSID, config)
 	if err != nil {
 		return err
 	}
@@ -163,6 +171,97 @@ func (kms *VaultTokensKMS) parseConfig(config map[string]interface{}, secrets ma
 	return nil
 }
 
+// initCertificates updates the kms.vaultConfig with the options from config
+// it calls the kubernetes secrets and get the required data.
+
+// nolint:gocyclo // iterating through many config options, not complex at all.
+func (kms *VaultTokensKMS) initCertificates(config map[string]interface{}) error {
+	vaultConfig := make(map[string]interface{})
+
+	csiNamespace := os.Getenv("POD_NAMESPACE")
+	vaultCAFromSecret := "" // optional
+	err := setConfigString(&vaultCAFromSecret, config, "vaultCAFromSecret")
+	if errors.Is(err, errConfigOptionInvalid) {
+		return err
+	}
+	// ignore errConfigOptionMissing, no default was set
+	if vaultCAFromSecret != "" {
+		cert, cErr := getCertificate(kms.Tenant, vaultCAFromSecret, "ca.cert")
+		if cErr != nil && !apierrs.IsNotFound(err) {
+			return fmt.Errorf("failed to get CA certificate from secret %s: %w", vaultCAFromSecret, cErr)
+		}
+		// if the certificate is not present in tenant namespace get it from
+		// cephcsi pod namespace
+		if apierrs.IsNotFound(cErr) {
+			cert, cErr = getCertificate(csiNamespace, vaultCAFromSecret, "ca.cert")
+			if cErr != nil {
+				return fmt.Errorf("failed to get CA certificate from secret %s: %w", vaultCAFromSecret, cErr)
+			}
+		}
+		vaultConfig[api.EnvVaultCACert], err = createTempFile("vault-ca-cert", []byte(cert))
+		if err != nil {
+			return fmt.Errorf("failed to create temporary file for Vault CA: %w", err)
+		}
+	}
+
+	vaultClientCertFromSecret := "" // optional
+	err = setConfigString(&vaultClientCertFromSecret, config, "vaultClientCertFromSecret")
+	if errors.Is(err, errConfigOptionInvalid) {
+		return err
+	}
+	// ignore errConfigOptionMissing, no default was set
+	if vaultClientCertFromSecret != "" {
+		cert, cErr := getCertificate(kms.Tenant, vaultClientCertFromSecret, "tls.cert")
+		if cErr != nil && !apierrs.IsNotFound(cErr) {
+			return fmt.Errorf("failed to get client certificate from secret %s: %w", vaultClientCertFromSecret, cErr)
+		}
+		// if the certificate is not present in tenant namespace get it from
+		// cephcsi pod namespace
+		if apierrs.IsNotFound(cErr) {
+			cert, cErr = getCertificate(csiNamespace, vaultClientCertFromSecret, "tls.cert")
+			if cErr != nil {
+				return fmt.Errorf("failed to get client certificate from secret %s: %w", vaultCAFromSecret, cErr)
+			}
+		}
+		vaultConfig[api.EnvVaultClientCert], err = createTempFile("vault-ca-cert", []byte(cert))
+		if err != nil {
+			return fmt.Errorf("failed to create temporary file for Vault client certificate: %w", err)
+		}
+	}
+
+	vaultClientCertKeyFromSecret := "" // optional
+	err = setConfigString(&vaultClientCertKeyFromSecret, config, "vaultClientCertKeyFromSecret")
+	if errors.Is(err, errConfigOptionInvalid) {
+		return err
+	}
+
+	// ignore errConfigOptionMissing, no default was set
+	if vaultClientCertKeyFromSecret != "" {
+		certKey, err := getCertificate(kms.Tenant, vaultClientCertKeyFromSecret, "tls.key")
+		if err != nil && !apierrs.IsNotFound(err) {
+			return fmt.Errorf("failed to get client certificate key from secret %s: %w", vaultClientCertKeyFromSecret, err)
+		}
+		// if the certificate is not present in tenant namespace get it from
+		// cephcsi pod namespace
+		if apierrs.IsNotFound(err) {
+			certKey, err = getCertificate(csiNamespace, vaultClientCertFromSecret, "tls.key")
+			if err != nil {
+				return fmt.Errorf("failed to get client certificate key from secret %s: %w", vaultCAFromSecret, err)
+			}
+		}
+		vaultConfig[api.EnvVaultClientKey], err = createTempFile("vault-client-cert-key", []byte(certKey))
+		if err != nil {
+			return fmt.Errorf("failed to create temporary file for Vault client cert key: %w", err)
+		}
+	}
+
+	for key, value := range vaultConfig {
+		kms.vaultConfig[key] = value
+	}
+
+	return nil
+}
+
 // GetPassphrase returns passphrase from Vault. The passphrase is stored in a
 // data.data.passphrase structure.
 func (kms *VaultTokensKMS) GetPassphrase(key string) (string, error) {
@@ -225,3 +324,18 @@ func getToken(tenant, tokenName string) (string, error) {
 
 	return string(token), nil
 }
+
+func getCertificate(tenant, secretName, key string) (string, error) {
+	c := NewK8sClient()
+	secret, err := c.CoreV1().Secrets(tenant).Get(context.TODO(), secretName, metav1.GetOptions{})
+	if err != nil {
+		return "", err
+	}
+
+	cert, ok := secret.Data[key]
+	if !ok {
+		return "", errors.New("failed to parse certificates")
+	}
+
+	return string(cert), nil
+}

internal/util/vault_tokens_test.go

@@ -26,10 +26,9 @@ func TestParseConfig(t *testing.T) {
 	kms := VaultTokensKMS{}
 
 	config := make(map[string]interface{})
-	secrets := make(map[string]string)
 
 	// empty config map
-	err := kms.parseConfig(config, secrets)
+	err := kms.parseConfig(config)
 	if !errors.Is(err, errConfigOptionMissing) {
 		t.Errorf("unexpected error (%T): %s", err, err)
 	}
@@ -40,7 +39,7 @@ func TestParseConfig(t *testing.T) {
 	config["tenantTokenName"] = vaultTokensDefaultTokenName
 
 	// parsing with all required options
-	err = kms.parseConfig(config, secrets)
+	err = kms.parseConfig(config)
 	switch {
 	case err != nil:
 		t.Errorf("unexpected error: %s", err)
@@ -53,7 +52,7 @@ func TestParseConfig(t *testing.T) {
 	// tenant "bob" uses a different kms.ConfigName
 	bob := make(map[string]interface{})
 	bob["tenantConfigName"] = "the-config-from-bob"
-	err = kms.parseConfig(bob, secrets)
+	err = kms.parseConfig(bob)
 	switch {
 	case err != nil:
 		t.Errorf("unexpected error: %s", err)
@@ -75,10 +74,9 @@ func TestInitVaultTokensKMS(t *testing.T) {
 	}
 
 	config := make(map[string]interface{})
-	secrets := make(map[string]string)
 
 	// empty config map
-	_, err := InitVaultTokensKMS("bob", "vault-tokens-config", config, secrets)
+	_, err := InitVaultTokensKMS("bob", "vault-tokens-config", config)
 	if !errors.Is(err, errConfigOptionMissing) {
 		t.Errorf("unexpected error (%T): %s", err, err)
 	}
@@ -87,7 +85,7 @@ func TestInitVaultTokensKMS(t *testing.T) {
 	config["vaultAddress"] = "https://vault.default.cluster.svc"
 
 	// parsing with all required options
-	_, err = InitVaultTokensKMS("bob", "vault-tokens-config", config, secrets)
+	_, err = InitVaultTokensKMS("bob", "vault-tokens-config", config)
 	if err != nil && !strings.Contains(err.Error(), "VAULT_TOKEN") {
 		t.Errorf("unexpected error: %s", err)
 	}
@@ -97,7 +95,7 @@ func TestInitVaultTokensKMS(t *testing.T) {
 	config["tenants"] = tenants
 
 	// empty tenants list
-	_, err = InitVaultTokensKMS("bob", "vault-tokens-config", config, secrets)
+	_, err = InitVaultTokensKMS("bob", "vault-tokens-config", config)
 	if err != nil && !strings.Contains(err.Error(), "VAULT_TOKEN") {
 		t.Errorf("unexpected error: %s", err)
 	}
@@ -107,7 +105,7 @@ func TestInitVaultTokensKMS(t *testing.T) {
 	config["tenants"].(map[string]interface{})["bob"] = bob
 	bob["vaultAddress"] = "https://vault.bob.example.org"
 
-	_, err = InitVaultTokensKMS("bob", "vault-tokens-config", config, secrets)
+	_, err = InitVaultTokensKMS("bob", "vault-tokens-config", config)
 	if err != nil && !strings.Contains(err.Error(), "VAULT_TOKEN") {
 		t.Errorf("unexpected error: %s", err)
 	}