Adds per volume encryption with Vault integration

- adds proposal document for PVC encryption from PR448
- adds per-volume encription by generating encryption passphrase
  for each volume and storing it in a KMS
- adds HashiCorp Vault integration as a KMS for encryption passphrases
- avoids encrypting volume second time if it was already encrypted but
  no file system created
- avoids unnecessary checks if volume is a mapped device when encryption
  was not requested
- prevents resizing encrypted volumes (it is not currently supported)
- prevents creating snapshots from encrypted volumes to prevent attack
  on encryption key (security guard until re-encryption of volumes
  implemented)

Signed-off-by: Vasyl Purchel vasyl.purchel@workday.com
Signed-off-by: Andrea Baglioni andrea.baglioni@workday.com

Fixes #420
Fixes #744
This commit is contained in:
Vasyl Purchel
2020-01-29 11:44:45 +00:00
committed by mergify[bot]
parent 1adef00c86
commit 419ad0dd8e
26 changed files with 1210 additions and 102 deletions

View File

@ -18,12 +18,15 @@ package util
import (
"context"
"encoding/base64"
"fmt"
"path"
"strings"
"github.com/pkg/errors"
"crypto/rand"
"k8s.io/klog"
)
@ -36,8 +39,123 @@ const (
// Encryption passphrase location in K8s secrets
encryptionPassphraseKey = "encryptionPassphrase"
// kmsConfigPath is the location of the vault config file
kmsConfigPath = "/etc/ceph-csi-encryption-kms-config/config.json"
// Passphrase size - 20 bytes is 160 bits to satisfy:
// https://tools.ietf.org/html/rfc6749#section-10.10
encryptionPassphraseSize = 20
)
// EncryptionKMS provides external Key Management System for encryption
// passphrases storage
type EncryptionKMS interface {
GetPassphrase(key string) (string, error)
SavePassphrase(key, value string) error
DeletePassphrase(key string) error
KmsConfig() string
}
// MissingPassphrase is an error instructing to generate new passphrase
type MissingPassphrase struct {
error
}
// SecretsKMS is default KMS implementation that means no KMS is in use
type SecretsKMS struct {
passphrase string
}
func initSecretsKMS(secrets map[string]string) (EncryptionKMS, error) {
passphraseValue, ok := secrets[encryptionPassphraseKey]
if !ok {
return nil, errors.New("missing encryption passphrase in secrets")
}
return SecretsKMS{passphrase: passphraseValue}, nil
}
// KmsConfig returns KMS configuration: "<kms-type>|<kms-id>"
func (kms SecretsKMS) KmsConfig() string {
return "secrets|kubernetes"
}
// GetPassphrase returns passphrase from Kubernetes secrets
func (kms SecretsKMS) GetPassphrase(key string) (string, error) {
return kms.passphrase, nil
}
// SavePassphrase is not implemented
func (kms SecretsKMS) SavePassphrase(key, value string) error {
return fmt.Errorf("save new passphrase is not implemented for Kubernetes secrets")
}
// DeletePassphrase is doing nothing as no new passphrases are saved with
// SecretsKMS
func (kms SecretsKMS) DeletePassphrase(key string) error {
return nil
}
// GetKMS returns an instance of Key Management System
func GetKMS(opts, secrets map[string]string) (EncryptionKMS, error) {
kmsType, ok := opts["encryptionKMS"]
if !ok || kmsType == "" || kmsType == "secrets" {
return initSecretsKMS(secrets)
}
if kmsType == "vault" {
return InitVaultKMS(opts, secrets)
}
return nil, fmt.Errorf("unknown encryption KMS type %s", kmsType)
}
// GetKMSConfig returns required keys for KMS to instantiate from it's config
// - map with kms type and ID keys
// - error if format is invalid
func GetKMSConfig(config string) (map[string]string, error) {
kmsConfigParts := strings.Split(config, "|")
if len(kmsConfigParts) != 2 {
return make(map[string]string), fmt.Errorf("failed to parse encryption KMS "+
"configuration from config string, expected <type>|<id>, got: %s", config)
}
return map[string]string{
"encryptionKMS": kmsConfigParts[0],
"encryptionKMSID": kmsConfigParts[1],
}, nil
}
// GetCryptoPassphrase Retrieves passphrase to encrypt volume
func GetCryptoPassphrase(ctx context.Context, volumeID string, kms EncryptionKMS) (string, error) {
passphrase, err := kms.GetPassphrase(volumeID)
if err == nil {
return passphrase, nil
}
if _, ok := err.(MissingPassphrase); ok {
klog.V(4).Infof(Log(ctx, "Encryption passphrase is missing for %s. Generating a new one"),
volumeID)
passphrase, err = generateNewEncryptionPassphrase()
if err != nil {
return "", fmt.Errorf("failed to generate passphrase for %s: %s", volumeID, err)
}
err = kms.SavePassphrase(volumeID, passphrase)
if err != nil {
return "", fmt.Errorf("failed to save the passphrase for %s: %s", volumeID, err)
}
return passphrase, nil
}
klog.Errorf(Log(ctx, "failed to get encryption passphrase for %s: %s"), volumeID, err)
return "", err
}
// generateNewEncryptionPassphrase generates a random passphrase for encryption
func generateNewEncryptionPassphrase() (string, error) {
bytesPassphrase := make([]byte, encryptionPassphraseSize)
_, err := rand.Read(bytesPassphrase)
if err != nil {
return "", err
}
return base64.URLEncoding.EncodeToString(bytesPassphrase), nil
}
// VolumeMapper returns file name and it's path to where encrypted device should be open
func VolumeMapper(volumeID string) (mapperFile, mapperFilePath string) {
mapperFile = mapperFilePrefix + volumeID
@ -45,15 +163,6 @@ func VolumeMapper(volumeID string) (mapperFile, mapperFilePath string) {
return mapperFile, mapperFilePath
}
// GetCryptoPassphrase Retrieves passphrase to encrypt volume
func GetCryptoPassphrase(secrets map[string]string) (string, error) {
val, ok := secrets[encryptionPassphraseKey]
if !ok {
return "", errors.New("missing encryption passphrase in secrets")
}
return val, nil
}
// EncryptVolume encrypts provided device with LUKS
func EncryptVolume(ctx context.Context, devicePath, passphrase string) error {
klog.V(4).Infof(Log(ctx, "Encrypting device %s with LUKS"), devicePath)

341
pkg/util/vault.go Normal file
View File

@ -0,0 +1,341 @@
/*
Copyright 2019 The Ceph-CSI Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package util
import (
"crypto/tls"
"crypto/x509"
"encoding/json"
"fmt"
"io"
"io/ioutil"
"net/http"
"strings"
)
const (
// path to service account token that will be used to authenticate with Vault
// #nosec
serviceAccountTokenPath = "/var/run/secrets/kubernetes.io/serviceaccount/token"
// vault configuration defaults
vaultDefaultAuthPath = "/v1/auth/kubernetes/login"
vaultDefaultRole = "csi-kubernetes"
vaultDefaultNamespace = ""
vaultDefaultPassphraseRoot = "/v1/secret"
vaultDefaultPassphrasePath = ""
// vault request headers
vaultTokenHeader = "X-Vault-Token" // nolint: gosec, #nosec
vaultNamespaceHeader = "X-Vault-Namespace"
)
/*
kmsKMS represents a Hashicorp Vault KMS configuration
Example JSON structure in the KMS config is,
[
{
"encryptionKMSID": "local_vault_unique_identifier",
"vaultAddress": "https://127.0.0.1:8500",
"vaultAuthPath": "/v1/auth/kubernetes/login",
"vaultRole": "csi-kubernetes",
"vaultNamespace": "",
"vaultPassphraseRoot": "/v1/secret",
"vaultPassphrasePath": "",
"vaultCAVerify": true,
"vaultCAFromSecret": "vault-ca"
},
...
]
*/
type VaultKMS struct {
EncryptionKMSID string `json:"encryptionKMSID"`
VaultAddress string `json:"vaultAddress"`
VaultAuthPath string `json:"vaultAuthPath"`
VaultRole string `json:"vaultRole"`
VaultNamespace string `json:"vaultNamespace"`
VaultPassphraseRoot string `json:"vaultPassphraseRoot"`
VaultPassphrasePath string `json:"vaultPassphrasePath"`
VaultCAVerify bool `json:"vaultCAVerify"`
VaultCAFromSecret string `json:"vaultCAFromSecret"`
vaultCA *x509.CertPool
}
// InitVaultKMS returns an interface to HashiCorp Vault KMS
func InitVaultKMS(opts, secrets map[string]string) (EncryptionKMS, error) {
var config []VaultKMS
vaultID, ok := opts["encryptionKMSID"]
if !ok {
return nil, fmt.Errorf("missing encryptionKMSID for vault as encryption KMS")
}
// #nosec
content, err := ioutil.ReadFile(kmsConfigPath)
if err != nil {
return nil, fmt.Errorf("error fetching vault configuration for vault ID (%s): (%s)",
vaultID, err)
}
err = json.Unmarshal(content, &config)
if err != nil {
return nil, fmt.Errorf("unmarshal failed: %v. raw buffer response: %s",
err, string(content))
}
for i := range config {
vault := &config[i]
if vault.EncryptionKMSID != vaultID {
continue
}
if vault.VaultAddress == "" {
return nil, fmt.Errorf("missing vaultAddress for vault as encryption KMS")
}
if vault.VaultAuthPath == "" {
vault.VaultAuthPath = vaultDefaultAuthPath
}
if vault.VaultRole == "" {
vault.VaultRole = vaultDefaultRole
}
if vault.VaultNamespace == "" {
vault.VaultNamespace = vaultDefaultNamespace
}
if vault.VaultPassphraseRoot == "" {
vault.VaultPassphraseRoot = vaultDefaultPassphraseRoot
}
if vault.VaultPassphrasePath == "" {
vault.VaultPassphrasePath = vaultDefaultPassphrasePath
}
if vault.VaultCAFromSecret != "" {
caPEM, ok := secrets[vault.VaultCAFromSecret]
if !ok {
return nil, fmt.Errorf("missing vault CA in secret %s", vault.VaultCAFromSecret)
}
roots := x509.NewCertPool()
ok = roots.AppendCertsFromPEM([]byte(caPEM))
if !ok {
return nil, fmt.Errorf("failed loading CA bundle for vault from secret %s",
vault.VaultCAFromSecret)
}
vault.vaultCA = roots
}
return vault, nil
}
return nil, fmt.Errorf("missing configuration for vault ID (%s)", vaultID)
}
// KmsConfig returns KMS configuration: "<kms-type>|<kms-id>"
func (kms *VaultKMS) KmsConfig() string {
return fmt.Sprintf("vault|%s", kms.EncryptionKMSID)
}
// GetPassphrase returns passphrase from Vault
func (kms *VaultKMS) GetPassphrase(key string) (string, error) {
var passphrase string
resp, err := kms.request("GET", kms.getKeyDataURI(key), nil)
if err != nil {
return "", fmt.Errorf("failed to retrieve passphrase for %s from vault: %s",
key, err)
}
defer resp.Body.Close()
if resp.StatusCode == 404 {
return "", MissingPassphrase{fmt.Errorf("passphrase for %s not found", key)}
}
err = kms.processError(resp, fmt.Sprintf("get passphrase for %s", key))
if err != nil {
return "", err
}
// parse resp as JSON and retrieve vault token
var result map[string]interface{}
err = json.NewDecoder(resp.Body).Decode(&result)
if err != nil {
return "", fmt.Errorf("failed parsing passphrase for %s from response: %s",
key, err)
}
data, ok := result["data"].(map[string]interface{})
if !ok {
return "", fmt.Errorf("failed parsing data for get passphrase request for %s", key)
}
data, ok = data["data"].(map[string]interface{})
if !ok {
return "", fmt.Errorf("failed parsing data.data for get passphrase request for %s", key)
}
passphrase, ok = data["passphrase"].(string)
if !ok {
return "", fmt.Errorf("failed parsing passphrase for get passphrase request for %s", key)
}
return passphrase, nil
}
// SavePassphrase saves new passphrase in Vault
func (kms *VaultKMS) SavePassphrase(key, value string) error {
data, err := json.Marshal(map[string]map[string]string{
"data": {
"passphrase": value,
},
})
if err != nil {
return fmt.Errorf("passphrase request data is broken: %s", err)
}
resp, err := kms.request("POST", kms.getKeyDataURI(key), data)
if err != nil {
return fmt.Errorf("failed to POST passphrase for %s to vault: %s", key, err)
}
defer resp.Body.Close()
err = kms.processError(resp, "save passphrase")
if err != nil {
return err
}
return nil
}
// DeletePassphrase deletes passphrase from Vault
func (kms *VaultKMS) DeletePassphrase(key string) error {
vaultToken, err := kms.getAccessToken()
if err != nil {
return fmt.Errorf("could not retrieve vault token to delete the passphrase at %s: %s",
key, err)
}
resp, err := kms.send("DELETE", kms.getKeyMetadataURI(key), &vaultToken, nil)
if err != nil {
return fmt.Errorf("delete passphrase at %s request to vault failed: %s", key, err)
}
defer resp.Body.Close()
if resp.StatusCode != 404 {
err = kms.processError(resp, "delete passphrase")
if err != nil {
return err
}
}
return nil
}
func (kms *VaultKMS) getKeyDataURI(key string) string {
return kms.VaultPassphraseRoot + "/data/" + kms.VaultPassphrasePath + key
}
func (kms *VaultKMS) getKeyMetadataURI(key string) string {
return kms.VaultPassphraseRoot + "/metadata/" + kms.VaultPassphrasePath + key
}
/*
getVaultAccessToken retrieves vault token using kubernetes authentication:
1. read jwt service account token from well known location
2. request token from vault using service account jwt token
Vault will verify service account jwt token with Kubernetes and return token
if the requester is allowed
*/
func (kms *VaultKMS) getAccessToken() (string, error) {
saToken, err := ioutil.ReadFile(serviceAccountTokenPath)
if err != nil {
return "", fmt.Errorf("service account token could not be read: %s", err)
}
data, err := json.Marshal(map[string]string{
"role": kms.VaultRole,
"jwt": string(saToken),
})
if err != nil {
return "", fmt.Errorf("vault token request data is broken: %s", err)
}
resp, err := kms.send("POST", kms.VaultAuthPath, nil, data)
if err != nil {
return "", fmt.Errorf("failed to retrieve vault token: %s", err)
}
defer resp.Body.Close()
err = kms.processError(resp, "retrieve vault token")
if err != nil {
return "", err
}
// parse resp as JSON and retrieve vault token
var result map[string]interface{}
err = json.NewDecoder(resp.Body).Decode(&result)
if err != nil {
return "", fmt.Errorf("failed parsing vaultToken from response: %s", err)
}
auth, ok := result["auth"].(map[string]interface{})
if !ok {
return "", fmt.Errorf("failed parsing vault token auth data")
}
vaultToken, ok := auth["client_token"].(string)
if !ok {
return "", fmt.Errorf("failed parsing vault client_token")
}
return vaultToken, nil
}
func (kms *VaultKMS) processError(resp *http.Response, action string) error {
if resp.StatusCode >= 200 || resp.StatusCode < 300 {
return nil
}
body, err := ioutil.ReadAll(resp.Body)
if err != nil {
return fmt.Errorf("failed to %s (%v), error body parsing failed: %s",
action, resp.StatusCode, err)
}
return fmt.Errorf("failed to %s (%v): %s", action, resp.StatusCode, body)
}
func (kms *VaultKMS) request(method, path string, data []byte) (*http.Response, error) {
vaultToken, err := kms.getAccessToken()
if err != nil {
return nil, err
}
return kms.send(method, path, &vaultToken, data)
}
func (kms *VaultKMS) send(method, path string, token *string, data []byte) (*http.Response, error) {
tlsConfig := &tls.Config{}
if !kms.VaultCAVerify {
tlsConfig.InsecureSkipVerify = true
}
if kms.vaultCA != nil {
tlsConfig.RootCAs = kms.vaultCA
}
netTransport := &http.Transport{TLSClientConfig: tlsConfig}
client := &http.Client{Transport: netTransport}
var dataToSend io.Reader
if data != nil {
dataToSend = strings.NewReader(string(data))
}
req, err := http.NewRequest(method, kms.VaultAddress+path, dataToSend)
if err != nil {
return nil, fmt.Errorf("could not create a Vault request: %s", err)
}
if kms.VaultNamespace != "" {
req.Header.Set(vaultNamespaceHeader, kms.VaultNamespace)
}
if token != nil {
req.Header.Set(vaultTokenHeader, *token)
}
return client.Do(req)
}

View File

@ -118,6 +118,9 @@ type CSIJournal struct {
// namespace in which the RADOS objects are stored, default is no namespace
namespace string
// encryptKMS in which encryption passphrase was saved, default is no encryption
encryptKMSKey string
}
// CSIVolumeJournal returns an instance of volume keys
@ -130,6 +133,7 @@ func NewCSIVolumeJournal() *CSIJournal {
namingPrefix: "csi-vol-",
cephSnapSourceKey: "",
namespace: "",
encryptKMSKey: "csi.volume.encryptKMS",
}
}
@ -143,6 +147,7 @@ func NewCSISnapshotJournal() *CSIJournal {
namingPrefix: "csi-snap-",
cephSnapSourceKey: "csi.source",
namespace: "",
encryptKMSKey: "csi.volume.encryptKMS",
}
}
@ -176,7 +181,7 @@ Return values:
there was no reservation found
- error: non-nil in case of any errors
*/
func (cj *CSIJournal) CheckReservation(ctx context.Context, monitors string, cr *Credentials, pool, reqName, parentName string) (string, error) {
func (cj *CSIJournal) CheckReservation(ctx context.Context, monitors string, cr *Credentials, pool, reqName, parentName, encryptionKmsConfig string) (string, error) {
var snapSource bool
if parentName != "" {
@ -199,7 +204,7 @@ func (cj *CSIJournal) CheckReservation(ctx context.Context, monitors string, cr
return "", err
}
savedReqName, savedReqParentName, err := cj.GetObjectUUIDData(ctx, monitors, cr, pool,
savedReqName, savedReqParentName, savedKms, err := cj.GetObjectUUIDData(ctx, monitors, cr, pool,
objUUID, snapSource)
if err != nil {
// error should specifically be not found, for image to be absent, any other error
@ -219,6 +224,14 @@ func (cj *CSIJournal) CheckReservation(ctx context.Context, monitors string, cr
reqName, objUUID, savedReqName)
}
if encryptionKmsConfig != "" {
if savedKms != encryptionKmsConfig {
return "", fmt.Errorf("internal state inconsistent, omap encryption KMS"+
" mismatch, request KMS (%s) volume UUID (%s) volume omap KMS (%s)",
encryptionKmsConfig, objUUID, savedKms)
}
}
if snapSource {
// check if source UUID key points back to the parent volume passed in
if savedReqParentName != parentName {
@ -310,7 +323,7 @@ Return values:
- string: Contains the UUID that was reserved for the passed in reqName
- error: non-nil in case of any errors
*/
func (cj *CSIJournal) ReserveName(ctx context.Context, monitors string, cr *Credentials, pool, reqName, parentName string) (string, error) {
func (cj *CSIJournal) ReserveName(ctx context.Context, monitors string, cr *Credentials, pool, reqName, parentName, encryptionKmsConfig string) (string, error) {
var snapSource bool
if parentName != "" {
@ -355,6 +368,14 @@ func (cj *CSIJournal) ReserveName(ctx context.Context, monitors string, cr *Cred
return "", err
}
if encryptionKmsConfig != "" {
err = SetOMapKeyValue(ctx, monitors, cr, pool, cj.namespace, cj.cephUUIDDirectoryPrefix+volUUID,
cj.encryptKMSKey, encryptionKmsConfig)
if err != nil {
return "", err
}
}
if snapSource {
// Update UUID directory to store source volume UUID in case of snapshots
err = SetOMapKeyValue(ctx, monitors, cr, pool, cj.namespace, cj.cephUUIDDirectoryPrefix+volUUID,
@ -372,30 +393,42 @@ GetObjectUUIDData fetches all keys from a UUID directory
Return values:
- string: Contains the request name for the passed in UUID
- string: Contains the parent image name for the passed in UUID, if it is a snapshot
- string: Contains encryption KMS, if it is an encrypted image
- error: non-nil in case of any errors
*/
func (cj *CSIJournal) GetObjectUUIDData(ctx context.Context, monitors string, cr *Credentials, pool, objectUUID string, snapSource bool) (string, string, error) {
func (cj *CSIJournal) GetObjectUUIDData(ctx context.Context, monitors string, cr *Credentials, pool, objectUUID string, snapSource bool) (string, string, string, error) {
var sourceName string
if snapSource && cj.cephSnapSourceKey == "" {
err := errors.New("invalid request, cephSnapSourceKey is nil")
return "", "", err
return "", "", "", err
}
// TODO: fetch all omap vals in one call, than make multiple listomapvals
requestName, err := GetOMapValue(ctx, monitors, cr, pool, cj.namespace,
cj.cephUUIDDirectoryPrefix+objectUUID, cj.csiNameKey)
if err != nil {
return "", "", err
return "", "", "", err
}
encryptionKmsConfig := ""
encryptionKmsConfig, err = GetOMapValue(ctx, monitors, cr, pool, cj.namespace,
cj.cephUUIDDirectoryPrefix+objectUUID, cj.encryptKMSKey)
if err != nil {
if _, ok := err.(ErrKeyNotFound); !ok {
klog.Errorf(Log(ctx, "=> GetObjectUUIDData encryptedKMS failed: %s (%s)"), cj.cephUUIDDirectoryPrefix+objectUUID, err)
return "", "", "", err
}
// ErrKeyNotFound means no encryption KMS was used
}
if snapSource {
sourceName, err = GetOMapValue(ctx, monitors, cr, pool, cj.namespace,
cj.cephUUIDDirectoryPrefix+objectUUID, cj.cephSnapSourceKey)
if err != nil {
return "", "", err
return "", "", "", err
}
}
return requestName, sourceName, nil
return requestName, sourceName, encryptionKmsConfig, nil
}