mirrors/ceph-csi
1
0
Fork 0
mirror of https://github.com/ceph/ceph-csi.git synced 2024-11-09 16:00:22 +00:00
Code Issues Packages Projects Releases Wiki Activity

Adds PVC encryption with LUKS

Browse Source 
Adds encryption in StorageClass as a parameter. Encryption passphrase is
stored in kubernetes secrets per StorageClass. Implements rbd volume
encryption relying on dm-crypt and cryptsetup using LUKS extension

The change is related to proposal made earlier. This is a first part of
the full feature that adds encryption with passphrase stored in secrets.

Signed-off-by: Vasyl Purchel vasyl.purchel@workday.com
Signed-off-by: Andrea Baglioni andrea.baglioni@workday.com
Signed-off-by: Ioannis Papaioannou ioannis.papaioannou@workday.com
Signed-off-by: Paul Mc Auley paul.mcauley@workday.com
Signed-off-by: Sergio de Carvalho sergio.carvalho@workday.com
This commit is contained in:
Vasyl Purchel 2019-12-13 11:41:32 +00:00 committed by mergify[bot]
parent 7c8e66e427
commit 166eaf700f
13 changed files with 619 additions and 39 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
Gopkg.lock generated
View File

@ -1380,6 +1380,7 @@
"k8s.io/apimachinery/pkg/util/validation",
"k8s.io/apimachinery/pkg/util/wait",
"k8s.io/apimachinery/pkg/util/yaml",
"k8s.io/apimachinery/pkg/fields",
"k8s.io/client-go/kubernetes",
"k8s.io/client-go/rest",
"k8s.io/client-go/tools/clientcmd",

57
docs/deploy-rbd.md
View File

@ -54,6 +54,7 @@ make image-cephcsi
| `csi.storage.k8s.io/provisioner-secret-name`, `csi.storage.k8s.io/node-stage-secret-name` | yes (for Kubernetes) | name of the Kubernetes Secret object containing Ceph client credentials. Both parameters should have the same value |
| `csi.storage.k8s.io/provisioner-secret-namespace`, `csi.storage.k8s.io/node-stage-secret-namespace` | yes (for Kubernetes) | namespaces of the above Secret objects |
| `mounter` | no | if set to `rbd-nbd`, use `rbd-nbd` on nodes that have `rbd-nbd` and `nbd` kernel modules to map rbd images |
| `encrypted` | no | disabled by default, use `"true"` to enable LUKS encryption on pvc and `"false"` to disable it. **Do not change for existing storageclasses** |
**NOTE:** An accompanying CSI configuration file, needs to be provided to the
running pods. Refer to [Creating CSI configuration](../examples/README.md#creating-csi-configuration)
@ -155,3 +156,59 @@ The Helm chart is located in `charts/ceph-csi-rbd`.
**Deploy Helm Chart:**
[See the Helm chart readme for installation instructions.](../charts/ceph-csi-rbd/README.md)
## Encryption for RBD volumes
> Enabling encryption on volumes created without encryption is **not supported**
>
> Enabling encryption for storage class that has PVs created without encryption
> is **not supported**
Volumes provisioned with Ceph RBD do not have encryption by default. It is
possible to encrypt them with ceph-csi by using LUKS encryption.
To enable encryption set `encrypted` option in storage class to `"true"` and
set encryption passphrase in kubernetes secrets under `encryptionPassphrase` key.
To use different passphrase you need to have different storage classes and point
to a different K8s secrets (different `csi.storage.k8s.io/node-stage-secret-name`
and `csi.storage.k8s.io/node-stage-secret-namespace`).
### Life-cycle for encrypted volumes
**Create volume**:
* create volume request received
* volume requested to be created in Ceph
* encrypted state "requiresEncryption" is saved in image-meta in Ceph
**Attach volume**:
* attach volume request received
* volume is attached to provisioner container
* on first time attachment
(no file system on the attached device, checked with blkid)
* device is encrypted with LUKS using a passphrase from K8s secrets
* image-meta updated to "encrypted" in Ceph
* device is open and device path is changed to use a mapper device
* mapper device is used instead of original one with usual workflow
**Detach volume**:
* mapper device closed and device path changed to original volume path
* volume is detached as usual
### Encryption configuration
To encrypt rbd volumes with LUKS you need to set encryption passphrase in
secrets under `encryptionPassphrase` key and switch `encrypted` option in
StorageClass to `"true"`. This is not supported for storage classes that already
have PVs provisioned.
### Encryption prerequisites
In order for encryption to work you need to make sure that `dm-crypt` kernel
module is enabled on the nodes running ceph-csi attachers.
If custom image is built for the rbd-plugin instance, make sure that it contains
`cryptsetup` tool installed to be able to use encryption.

8
e2e/rbd.go
View File

@ -125,6 +125,14 @@ var _ = Describe("RBD", func() {
createRBDStorageClass(f.ClientSet, f, make(map[string]string))
})
By("create a PVC and Bind it to an app with encrypted RBD volume", func() {
deleteResource(rbdExamplePath + "storageclass.yaml")
createRBDStorageClass(f.ClientSet, f, map[string]string{"encrypted": "true"})
validateEncryptedPVCAndAppBinding(pvcPath, appPath, f)
deleteResource(rbdExamplePath + "storageclass.yaml")
createRBDStorageClass(f.ClientSet, f, make(map[string]string))
})
// skipping snapshot testing
// By("create a PVC clone and Bind it to an app", func() {

77
e2e/utils.go
View File

@ -5,6 +5,7 @@ import (
"encoding/json"
"fmt"
"io/ioutil"
"regexp"
"strings"
"time"
@ -20,6 +21,7 @@ import (
apierrs "k8s.io/apimachinery/pkg/api/errors"
"k8s.io/apimachinery/pkg/api/resource"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/fields"
"k8s.io/apimachinery/pkg/util/wait"
utilyaml "k8s.io/apimachinery/pkg/util/yaml"
"k8s.io/client-go/kubernetes"
@ -553,7 +555,7 @@ func deletePVCAndApp(name string, f *framework.Framework, pvc *v1.PersistentVolu
return err
}
func validatePVCAndAppBinding(pvcPath, appPath string, f *framework.Framework) {
func createPVCAndAppBinding(pvcPath, appPath string, f *framework.Framework) (*v1.PersistentVolumeClaim, *v1.Pod) {
pvc, err := loadPVC(pvcPath)
if pvc == nil {
Fail(err.Error())
@ -572,11 +574,84 @@ func validatePVCAndAppBinding(pvcPath, appPath string, f *framework.Framework) {
Fail(err.Error())
}
return pvc, app
}
func validatePVCAndAppBinding(pvcPath, appPath string, f *framework.Framework) {
pvc, app := createPVCAndAppBinding(pvcPath, appPath, f)
err := deletePVCAndApp("", f, pvc, app)
if err != nil {
Fail(err.Error())
}
}
func getRBDImageSpec(pvcNamespace, pvcName string, f *framework.Framework) (string, error) {
c := f.ClientSet.CoreV1()
pvc, err := c.PersistentVolumeClaims(pvcNamespace).Get(pvcName, metav1.GetOptions{})
if err != nil {
return "", err
}
pv, err := c.PersistentVolumes().Get(pvc.Spec.VolumeName, metav1.GetOptions{})
if err != nil {
return "", err
}
imageIDRegex := regexp.MustCompile(`(\w+\-?){5}$`)
imageID := imageIDRegex.FindString(pv.Spec.CSI.VolumeHandle)
return fmt.Sprintf("replicapool/csi-vol-%s", imageID), nil
}
func getImageMeta(rbdImageSpec, metaKey string, f *framework.Framework) (string, error) {
cmd := fmt.Sprintf("rbd image-meta get %s %s", rbdImageSpec, metaKey)
opt := metav1.ListOptions{
LabelSelector: "app=rook-ceph-tools",
}
stdOut, stdErr := execCommandInPod(f, cmd, rookNS, &opt)
if stdErr != "" {
return strings.TrimSpace(stdOut), fmt.Errorf(stdErr)
}
return strings.TrimSpace(stdOut), nil
}
func getMountType(appName, appNamespace, mountPath string, f *framework.Framework) (string, error) {
opt := metav1.ListOptions{
FieldSelector: fields.OneTermEqualSelector("metadata.name", appName).String(),
}
cmd := fmt.Sprintf("lsblk -o TYPE,MOUNTPOINT | grep '%s' | awk '{print $1}'", mountPath)
stdOut, stdErr := execCommandInPod(f, cmd, appNamespace, &opt)
if stdErr != "" {
return strings.TrimSpace(stdOut), fmt.Errorf(stdErr)
}
return strings.TrimSpace(stdOut), nil
}
func validateEncryptedPVCAndAppBinding(pvcPath, appPath string, f *framework.Framework) {
pvc, app := createPVCAndAppBinding(pvcPath, appPath, f)
rbdImageSpec, err := getRBDImageSpec(pvc.Namespace, pvc.Name, f)
if err != nil {
Fail(err.Error())
}
encryptedState, err := getImageMeta(rbdImageSpec, ".rbd.csi.ceph.com/encrypted", f)
if err != nil {
Fail(err.Error())
}
Expect(encryptedState).To(Equal("encrypted"))
volumeMountPath := app.Spec.Containers[0].VolumeMounts[0].MountPath
mountType, err := getMountType(app.Name, app.Namespace, volumeMountPath, f)
if err != nil {
Fail(err.Error())
}
Expect(mountType).To(Equal("crypt"))
err = deletePVCAndApp("", f, pvc, app)
if err != nil {
Fail(err.Error())
}
}
func deletePodWithLabel(label string) error {
_, err := framework.RunKubectl("delete", "po", "-l", label)
if err != nil {

3
examples/rbd/secret.yaml
View File

@ -10,3 +10,6 @@ stringData:
# specified in the storage class
userID: <plaintext ID>
userKey: <Ceph auth key corresponding to ID above>
# Encryption passphrase
encryptionPassphrase: test_passphrase

5
examples/rbd/storageclass.yaml
View File

@ -38,6 +38,11 @@ parameters:
csi.storage.k8s.io/fstype: ext4
# uncomment the following to use rbd-nbd as mounter on supported nodes
# mounter: rbd-nbd
# Instruct the plugin it has to encrypt the volume
# By default it is disabled. Valid values are “true” or “false”.
# A string is expected here, i.e. “true”, not true.
# encrypted: "true"
reclaimPolicy: Delete
allowVolumeExpansion: true
mountOptions:

23
pkg/rbd/controllerserver.go
View File

@ -149,6 +149,14 @@ func (cs *ControllerServer) CreateVolume(ctx context.Context, req *csi.CreateVol
return nil, status.Error(codes.Internal, err.Error())
}
if found {
if rbdVol.Encrypted {
err = ensureEncryptionMetadataSet(ctx, cr, rbdVol)
if err != nil {
klog.Errorf(util.Log(ctx, err.Error()))
return nil, err
}
}
return &csi.CreateVolumeResponse{
Volume: &csi.Volume{
VolumeId: rbdVol.VolID,
@ -176,6 +184,20 @@ func (cs *ControllerServer) CreateVolume(ctx context.Context, req *csi.CreateVol
return nil, err
}
if rbdVol.Encrypted {
err = ensureEncryptionMetadataSet(ctx, cr, rbdVol)
if err != nil {
klog.Errorf(util.Log(ctx, "failed to save encryption status, deleting image %s"),
rbdVol.RbdImageName)
if deleteErr := deleteImage(ctx, rbdVol, cr); err != nil {
klog.Errorf(util.Log(ctx, "failed to delete rbd image: %s/%s with error: %v"),
rbdVol.Pool, rbdVol.RbdImageName, deleteErr)
return nil, deleteErr
}
return nil, err
}
}
return &csi.CreateVolumeResponse{
Volume: &csi.Volume{
VolumeId: rbdVol.VolID,
@ -211,6 +233,7 @@ func (cs *ControllerServer) createBackingImage(ctx context.Context, rbdVol *rbdV
return nil
}
func (cs *ControllerServer) checkSnapshot(ctx context.Context, req *csi.CreateVolumeRequest, rbdVol *rbdVolume) error {
snapshot := req.VolumeContentSource.GetSnapshot()
if snapshot == nil {

155
pkg/rbd/nodeserver.go
View File

@ -87,25 +87,14 @@ func (ns *NodeServer) NodeStageVolume(ctx context.Context, req *csi.NodeStageVol
}
defer ns.VolumeLocks.Release(volID)
isLegacyVolume := false
volName, err := getVolumeName(volID)
if err != nil {
// error ErrInvalidVolID may mean this is an 1.0.0 version volume, check for name
// pattern match in addition to error to ensure this is a likely v1.0.0 volume
if _, ok := err.(ErrInvalidVolID); !ok || !isLegacyVolumeID(volID) {
return nil, status.Error(codes.InvalidArgument, err.Error())
}
volName, err = getLegacyVolumeName(req.GetStagingTargetPath())
if err != nil {
return nil, status.Error(codes.InvalidArgument, err.Error())
}
isLegacyVolume = true
}
stagingParentPath := req.GetStagingTargetPath()
stagingTargetPath := stagingParentPath + "/" + volID
isLegacyVolume, volName, err := getVolumeNameByID(volID, stagingParentPath)
if err != nil {
return nil, status.Error(codes.InvalidArgument, err.Error())
}
var isNotMnt bool
// check if stagingPath is already mounted
isNotMnt, err = mount.IsNotMountPoint(ns.mounter, stagingTargetPath)
@ -123,6 +112,7 @@ func (ns *NodeServer) NodeStageVolume(ctx context.Context, req *csi.NodeStageVol
return nil, status.Error(codes.Internal, err.Error())
}
volOptions.RbdImageName = volName
volOptions.VolID = req.GetVolumeId()
isMounted := false
isStagePathCreated := false
@ -145,7 +135,15 @@ func (ns *NodeServer) NodeStageVolume(ctx context.Context, req *csi.NodeStageVol
if err != nil {
return nil, status.Error(codes.Internal, err.Error())
}
klog.V(4).Infof(util.Log(ctx, "rbd image: %s/%s was successfully mapped at %s\n"), req.GetVolumeId(), volOptions.Pool, devicePath)
klog.V(4).Infof(util.Log(ctx, "rbd image: %s/%s was successfully mapped at %s\n"),
req.GetVolumeId(), volOptions.Pool, devicePath)
if volOptions.Encrypted {
devicePath, err = ns.processEncryptedDevice(ctx, volOptions, devicePath, cr, req.GetSecrets())
if err != nil {
return nil, status.Error(codes.Internal, err.Error())
}
}
err = ns.createStageMountPoint(ctx, stagingTargetPath, isBlock)
if err != nil {
@ -194,7 +192,7 @@ func (ns *NodeServer) undoStagingTransaction(ctx context.Context, stagingParentP
// Unmapping rbd device
if devicePath != "" {
err = detachRBDDevice(ctx, devicePath)
err = detachRBDDevice(ctx, devicePath, volID)
if err != nil {
klog.Errorf(util.Log(ctx, "failed to unmap rbd device: %s for volume %s with error: %v"), devicePath, volID, err)
// continue on failure to delete the stash file, as kubernetes will fail to delete the staging path otherwise
@ -273,18 +271,6 @@ func (ns *NodeServer) NodePublishVolume(ctx context.Context, req *csi.NodePublis
return &csi.NodePublishVolumeResponse{}, nil
}
func getVolumeName(volID string) (string, error) {
var vi util.CSIIdentifier
err := vi.DecomposeCSIID(volID)
if err != nil {
err = fmt.Errorf("error decoding volume ID (%s) (%s)", err, volID)
return "", ErrInvalidVolID{err}
}
return volJournal.NamingPrefix() + vi.ObjectUUID, nil
}
func getLegacyVolumeName(mountPath string) (string, error) {
var volName string
@ -521,7 +507,7 @@ func (ns *NodeServer) NodeUnstageVolume(ctx context.Context, req *csi.NodeUnstag
// Unmapping rbd device
imageSpec := imgInfo.Pool + "/" + imgInfo.ImageName
if err = detachRBDImageOrDeviceSpec(ctx, imageSpec, true, imgInfo.NbdAccess); err != nil {
if err = detachRBDImageOrDeviceSpec(ctx, imageSpec, true, imgInfo.NbdAccess, req.GetVolumeId()); err != nil {
klog.Errorf(util.Log(ctx, "error unmapping volume (%s) from staging path (%s): (%v)"), req.GetVolumeId(), stagingTargetPath, err)
return nil, status.Error(codes.Internal, err.Error())
}
@ -613,3 +599,110 @@ func (ns *NodeServer) NodeGetCapabilities(ctx context.Context, req *csi.NodeGetC
},
}, nil
}
func (ns *NodeServer) processEncryptedDevice(ctx context.Context, volOptions *rbdVolume, devicePath string, cr *util.Credentials, secrets map[string]string) (string, error) {
imageSpec := volOptions.Pool + "/" + volOptions.RbdImageName
encrypted, err := util.CheckRbdImageEncrypted(ctx, cr, volOptions.Monitors, imageSpec)
if err != nil {
klog.Errorf(util.Log(ctx, "failed to get encryption status for rbd image %s: %v"),
imageSpec, err)
return "", err
}
if encrypted == rbdImageRequiresEncryption {
diskMounter := &mount.SafeFormatAndMount{Interface: ns.mounter, Exec: mount.NewOsExec()}
// TODO: update this when adding support for static (pre-provisioned) PVs
var existingFormat string
existingFormat, err = diskMounter.GetDiskFormat(devicePath)
if err != nil {
return "", fmt.Errorf("failed to get disk format for path %s, error: %v", devicePath, err)
}
if existingFormat != "" {
return "", fmt.Errorf("can not encrypt rbdImage %s that already has file system: %s",
imageSpec, existingFormat)
}
err = encryptDevice(ctx, volOptions, secrets, cr, devicePath)
if err != nil {
return "", fmt.Errorf("failed to encrypt rbd image %s: %v", imageSpec, err)
}
} else if encrypted != rbdImageEncrypted {
return "", fmt.Errorf("rbd image %s found mounted with unexpected encryption status %s",
imageSpec, encrypted)
}
devicePath, err = openEncryptedDevice(ctx, volOptions, devicePath, secrets)
if err != nil {
return "", err
}
return devicePath, nil
}
func encryptDevice(ctx context.Context, rbdVol *rbdVolume, secret map[string]string, cr *util.Credentials, devicePath string) error {
passphrase, err := util.GetCryptoPassphrase(secret)
if err != nil {
klog.Errorf(util.Log(ctx, "failed to get crypto passphrase for %s/%s: %v"),
rbdVol.Pool, rbdVol.RbdImageName, err)
return err
}
if err = util.EncryptVolume(ctx, devicePath, passphrase); err != nil {
err = fmt.Errorf("failed to encrypt volume %s/%s: %v", rbdVol.Pool, rbdVol.RbdImageName, err)
klog.Errorf(util.Log(ctx, err.Error()))
return err
}
imageSpec := rbdVol.Pool + "/" + rbdVol.RbdImageName
err = util.SaveRbdImageEncryptionStatus(ctx, cr, rbdVol.Monitors, imageSpec, rbdImageEncrypted)
return err
}
func openEncryptedDevice(ctx context.Context, volOptions *rbdVolume, devicePath string, secrets map[string]string) (string, error) {
passphrase, err := util.GetCryptoPassphrase(secrets)
if err != nil {
klog.Errorf(util.Log(ctx, "failed to get passphrase for encrypted device %s/%s: %v"),
volOptions.Pool, volOptions.RbdImageName, err)
return "", status.Error(codes.Internal, err.Error())
}
mapperFile, mapperFilePath := util.VolumeMapper(volOptions.VolID)
isOpen, err := util.IsDeviceOpen(ctx, mapperFilePath)
if err != nil {
klog.Errorf(util.Log(ctx, "failed to check device %s encryption status: %s"), devicePath, err)
return devicePath, err
}
if isOpen {
klog.V(4).Infof(util.Log(ctx, "encrypted device is already open at %s"), mapperFilePath)
} else {
err = util.OpenEncryptedVolume(ctx, devicePath, mapperFile, passphrase)
if err != nil {
klog.Errorf(util.Log(ctx, "failed to open device %s/%s: %v"),
volOptions.Pool, volOptions.RbdImageName, err)
return devicePath, err
}
}
return mapperFilePath, nil
}
func getVolumeNameByID(volID, stagingTargetPath string) (bool, string, error) {
volName, err := getVolumeName(volID)
if err != nil {
// error ErrInvalidVolID may mean this is an 1.0.0 version volume, check for name
// pattern match in addition to error to ensure this is a likely v1.0.0 volume
if _, ok := err.(ErrInvalidVolID); !ok || !isLegacyVolumeID(volID) {
return false, "", status.Error(codes.InvalidArgument, err.Error())
}
volName, err = getLegacyVolumeName(stagingTargetPath)
if err != nil {
return false, "", status.Error(codes.InvalidArgument, err.Error())
}
return true, volName, nil
}
return false, volName, nil
}

27
pkg/rbd/rbd_attach.go
View File

@ -225,7 +225,7 @@ func createPath(ctx context.Context, volOpt *rbdVolume, cr *util.Credentials) (s
klog.Warningf(util.Log(ctx, "rbd: map error %v, rbd output: %s"), err, string(output))
// unmap rbd image if connection timeout
if strings.Contains(err.Error(), rbdMapConnectionTimeout) {
detErr := detachRBDImageOrDeviceSpec(ctx, imagePath, true, isNbd)
detErr := detachRBDImageOrDeviceSpec(ctx, imagePath, true, isNbd, volOpt.VolID)
if detErr != nil {
klog.Warningf(util.Log(ctx, "rbd: %s unmap error %v"), imagePath, detErr)
}
@ -260,21 +260,38 @@ func waitForrbdImage(ctx context.Context, backoff wait.Backoff, volOptions *rbdV
return err
}
func detachRBDDevice(ctx context.Context, devicePath string) error {
func detachRBDDevice(ctx context.Context, devicePath, volumeID string) error {
nbdType := false
if strings.HasPrefix(devicePath, "/dev/nbd") {
nbdType = true
}
return detachRBDImageOrDeviceSpec(ctx, devicePath, false, nbdType)
return detachRBDImageOrDeviceSpec(ctx, devicePath, false, nbdType, volumeID)
}
// detachRBDImageOrDeviceSpec detaches an rbd imageSpec or devicePath, with additional checking
// when imageSpec is used to decide if image is already unmapped
func detachRBDImageOrDeviceSpec(ctx context.Context, imageOrDeviceSpec string, isImageSpec, ndbType bool) error {
var err error
func detachRBDImageOrDeviceSpec(ctx context.Context, imageOrDeviceSpec string, isImageSpec, ndbType bool, volumeID string) error {
var output []byte
mapperFile, mapperPath := util.VolumeMapper(volumeID)
mappedDevice, mapper, err := util.DeviceEncryptionStatus(ctx, mapperPath)
if err != nil {
klog.Errorf(util.Log(ctx, "error determining LUKS device on %s, %s: %s"),
mapperPath, imageOrDeviceSpec, err)
return err
}
if len(mapper) > 0 {
// mapper found, so it is open Luks device
err = util.CloseEncryptedVolume(ctx, mapperFile)
if err != nil {
klog.Warningf(util.Log(ctx, "error closing LUKS device on %s, %s: %s"),
mapperPath, imageOrDeviceSpec, err)
return err
}
imageOrDeviceSpec = mappedDevice
}
accessType := accessTypeKRbd
if ndbType {
accessType = accessTypeNbd

47
pkg/rbd/rbd_util.go
View File

@ -24,6 +24,7 @@ import (
"os"
"os/exec"
"path/filepath"
"strconv"
"strings"
"time"
@ -53,6 +54,10 @@ const (
rbdTaskRemoveCmdInvalidString1 = "no valid command found"
rbdTaskRemoveCmdInvalidString2 = "Error EINVAL: invalid command"
rbdTaskRemoveCmdAccessDeniedMessage = "Error EACCES:"
// Encryption statuses for RbdImage
rbdImageEncrypted = "encrypted"
rbdImageRequiresEncryption = "requiresEncryption"
)
// rbdVolume represents a CSI volume and its RBD image specifics
@ -71,15 +76,16 @@ type rbdVolume struct {
DataPool string
ImageFormat string `json:"imageFormat"`
ImageFeatures string `json:"imageFeatures"`
VolSize int64 `json:"volSize"`
AdminID string `json:"adminId"`
UserID string `json:"userId"`
Mounter string `json:"mounter"`
DisableInUseChecks bool `json:"disableInUseChecks"`
ClusterID string `json:"clusterId"`
RequestName string
VolName string `json:"volName"`
MonValueFromSecret string `json:"monValueFromSecret"`
VolSize int64 `json:"volSize"`
DisableInUseChecks bool `json:"disableInUseChecks"`
Encrypted bool
}
// rbdSnapshot represents a CSI snapshot and its RBD snapshot specifics
@ -486,6 +492,16 @@ func genVolFromVolumeOptions(ctx context.Context, volOptions, credentials map[st
rbdVol.Mounter = rbdDefaultMounter
}
rbdVol.Encrypted = false
encrypted, ok := volOptions["encrypted"]
if ok {
rbdVol.Encrypted, err = strconv.ParseBool(encrypted)
if err != nil {
return nil, fmt.Errorf(
"invalid value set in 'encrypted': %s (should be \"true\" or \"false\")", encrypted)
}
}
return rbdVol, nil
}
@ -831,3 +847,30 @@ func resizeRBDImage(rbdVol *rbdVolume, newSize int64, cr *util.Credentials) erro
return nil
}
func getVolumeName(volID string) (string, error) {
var vi util.CSIIdentifier
err := vi.DecomposeCSIID(volID)
if err != nil {
err = fmt.Errorf("error decoding volume ID (%s) (%s)", err, volID)
return "", ErrInvalidVolID{err}
}
return volJournal.NamingPrefix() + vi.ObjectUUID, nil
}
func ensureEncryptionMetadataSet(ctx context.Context, cr *util.Credentials, rbdVol *rbdVolume) error {
rbdImageName, err := getVolumeName(rbdVol.VolID)
if err != nil {
return err
}
imageSpec := rbdVol.Pool + "/" + rbdImageName
err = util.SaveRbdImageEncryptionStatus(ctx, cr, rbdVol.Monitors, imageSpec, rbdImageRequiresEncryption)
if err != nil {
return fmt.Errorf("failed to save encryption status for %s: %v", imageSpec, err)
}
return nil
}

45
pkg/util/cephcmds.go
View File

@ -280,3 +280,48 @@ func RemoveObject(ctx context.Context, monitors string, cr *Credentials, poolNam
return nil
}
// SetImageMeta sets image metadata
func SetImageMeta(ctx context.Context, cr *Credentials, monitors, imageSpec, key, value string) error {
args := []string{
"-m", monitors,
"--id", cr.ID,
"--keyfile=" + cr.KeyFile,
"-c", CephConfigPath,
"image-meta", "set", imageSpec,
key, value,
}
_, _, err := ExecCommand("rbd", args[:]...)
if err != nil {
klog.Errorf(Log(ctx, "failed setting image metadata (%s) for (%s): (%v)"), key, imageSpec, err)
return err
}
return nil
}
// GetImageMeta gets image metadata
func GetImageMeta(ctx context.Context, cr *Credentials, monitors, imageSpec, key string) (string, error) {
args := []string{
"-m", monitors,
"--id", cr.ID,
"--keyfile=" + cr.KeyFile,
"-c", CephConfigPath,
"image-meta", "get", imageSpec,
key,
}
stdout, stderr, err := ExecCommand("rbd", args[:]...)
if err != nil {
stdoutanderr := strings.Join([]string{string(stdout), string(stderr)}, " ")
if strings.Contains(stdoutanderr, "failed to get metadata "+key+" of image : (2) No such file or directory") {
return "", ErrKeyNotFound{imageSpec + " " + key, err}
}
klog.Errorf(Log(ctx, "failed getting image metadata (%s) for (%s): (%v)"), key, imageSpec, err)
return "", err
}
return string(stdout), nil
}

143
pkg/util/crypto.go Normal file
View File

@ -0,0 +1,143 @@
/*
Copyright 2019 The Ceph-CSI Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package util
import (
"context"
"fmt"
"path"
"strings"
"github.com/pkg/errors"
"k8s.io/klog"
)
const (
mapperFilePrefix = "luks-rbd-"
mapperFilePathPrefix = "/dev/mapper"
// image metadata key for encryption
encryptionMetaKey = ".rbd.csi.ceph.com/encrypted"
// Encryption passphrase location in K8s secrets
encryptionPassphraseKey = "encryptionPassphrase"
)
// VolumeMapper returns file name and it's path to where encrypted device should be open
func VolumeMapper(volumeID string) (mapperFile, mapperFilePath string) {
mapperFile = mapperFilePrefix + volumeID
mapperFilePath = path.Join(mapperFilePathPrefix, mapperFile)
return mapperFile, mapperFilePath
}
// GetCryptoPassphrase Retrieves passphrase to encrypt volume
func GetCryptoPassphrase(secrets map[string]string) (string, error) {
val, ok := secrets[encryptionPassphraseKey]
if !ok {
return "", errors.New("missing encryption passphrase in secrets")
}
return val, nil
}
// EncryptVolume encrypts provided device with LUKS
func EncryptVolume(ctx context.Context, devicePath, passphrase string) error {
klog.V(4).Infof(Log(ctx, "Encrypting device %s with LUKS"), devicePath)
if _, _, err := LuksFormat(devicePath, passphrase); err != nil {
return errors.Wrapf(err, "failed to encrypt device %s with LUKS", devicePath)
}
return nil
}
// OpenEncryptedVolume opens volume so that it can be used by the client
func OpenEncryptedVolume(ctx context.Context, devicePath, mapperFile, passphrase string) error {
klog.V(4).Infof(Log(ctx, "Opening device %s with LUKS on %s"), devicePath, mapperFile)
_, _, err := LuksOpen(devicePath, mapperFile, passphrase)
return err
}
// CloseEncryptedVolume closes encrypted volume so it can be detached
func CloseEncryptedVolume(ctx context.Context, mapperFile string) error {
klog.V(4).Infof(Log(ctx, "Closing LUKS device %s"), mapperFile)
_, _, err := LuksClose(mapperFile)
return err
}
// IsDeviceOpen determines if encrypted device is already open
func IsDeviceOpen(ctx context.Context, device string) (bool, error) {
_, mappedFile, err := DeviceEncryptionStatus(ctx, device)
return (mappedFile != ""), err
}
// DeviceEncryptionStatus looks to identify if the passed device is a LUKS mapping
// and if so what the device is and the mapper name as used by LUKS.
// If not, just returns the original device and an empty string.
func DeviceEncryptionStatus(ctx context.Context, devicePath string) (mappedDevice, mapper string, err error) {
if !strings.HasPrefix(devicePath, mapperFilePathPrefix) {
return devicePath, "", nil
}
mapPath := strings.TrimPrefix(devicePath, mapperFilePathPrefix+"/")
stdout, _, err := LuksStatus(mapPath)
if err != nil {
klog.V(4).Infof(Log(ctx, "device %s is not an active LUKS device: %v"), devicePath, err)
return devicePath, "", nil
}
lines := strings.Split(string(stdout), "\n")
if len(lines) < 1 {
return "", "", fmt.Errorf("device encryption status returned no stdout for %s", devicePath)
}
if !strings.HasSuffix(lines[0], " is active.") {
// Implies this is not a LUKS device
return devicePath, "", nil
}
for i := 1; i < len(lines); i++ {
kv := strings.SplitN(strings.TrimSpace(lines[i]), ":", 2)
if len(kv) < 1 {
return "", "", fmt.Errorf("device encryption status output for %s is badly formatted: %s",
devicePath, lines[i])
}
if strings.Compare(kv[0], "device") == 0 {
return strings.TrimSpace(kv[1]), mapPath, nil
}
}
// Identified as LUKS, but failed to identify a mapped device
return "", "", fmt.Errorf("mapped device not found in path %s", devicePath)
}
// CheckRbdImageEncrypted verifies if rbd image was encrypted when created
func CheckRbdImageEncrypted(ctx context.Context, cr *Credentials, monitors, imageSpec string) (string, error) {
value, err := GetImageMeta(ctx, cr, monitors, imageSpec, encryptionMetaKey)
if err != nil {
klog.Errorf(Log(ctx, "checking image %s encrypted state metadata failed: %s"), imageSpec, err)
return "", err
}
encrypted := strings.TrimSpace(value)
klog.V(4).Infof(Log(ctx, "image %s encrypted state metadata reports %q"), imageSpec, encrypted)
return encrypted, nil
}
// SaveRbdImageEncryptionStatus sets image metadata for encryption status
func SaveRbdImageEncryptionStatus(ctx context.Context, cr *Credentials, monitors, imageSpec, status string) error {
err := SetImageMeta(ctx, cr, monitors, imageSpec, encryptionMetaKey, status)
if err != nil {
err = fmt.Errorf("failed to save image metadata encryption status for %s: %v", imageSpec, err.Error())
klog.Errorf(Log(ctx, err.Error()))
return err
}
return nil
}

67
pkg/util/cryptsetup.go Normal file
View File

@ -0,0 +1,67 @@
/*
Copyright 2019 The Ceph-CSI Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package util
import (
"bytes"
"fmt"
"os/exec"
"strings"
)
// LuksFormat sets up volume as an encrypted LUKS partition
func LuksFormat(devicePath, passphrase string) (stdout, stderr []byte, err error) {
return execCryptsetupCommand(&passphrase, "-q", "luksFormat", "--hash", "sha256", devicePath, "-d", "/dev/stdin")
}
// LuksOpen opens LUKS encrypted partition and sets up a mapping
func LuksOpen(devicePath, mapperFile, passphrase string) (stdout, stderr []byte, err error) {
return execCryptsetupCommand(&passphrase, "luksOpen", devicePath, mapperFile, "-d", "/dev/stdin")
}
// LuksClose removes existing mapping
func LuksClose(mapperFile string) (stdout, stderr []byte, err error) {
return execCryptsetupCommand(nil, "luksClose", mapperFile)
}
// LuksStatus returns encryption status of a provided device
func LuksStatus(mapperFile string) (stdout, stderr []byte, err error) {
return execCryptsetupCommand(nil, "status", mapperFile)
}
func execCryptsetupCommand(stdin *string, args ...string) (stdout, stderr []byte, err error) {
var (
program = "cryptsetup"
cmd = exec.Command(program, args...) // nolint: gosec, #nosec
sanitizedArgs = StripSecretInArgs(args)
stdoutBuf bytes.Buffer
stderrBuf bytes.Buffer
)
cmd.Stdout = &stdoutBuf
cmd.Stderr = &stderrBuf
if stdin != nil {
cmd.Stdin = strings.NewReader(*stdin)
}
if err := cmd.Run(); err != nil {
return stdoutBuf.Bytes(), stderrBuf.Bytes(), fmt.Errorf("an error (%v)"+
" occurred while running %s args: %v", err, program, sanitizedArgs)
}
return stdoutBuf.Bytes(), nil, nil
}