Adds PVC encryption with LUKS

Adds encryption in StorageClass as a parameter. Encryption passphrase is stored in kubernetes secrets per StorageClass. Implements rbd volume encryption relying on dm-crypt and cryptsetup using LUKS extension The change is related to proposal made earlier. This is a first part of the full feature that adds encryption with passphrase stored in secrets. Signed-off-by: Vasyl Purchel vasyl.purchel@workday.com Signed-off-by: Andrea Baglioni andrea.baglioni@workday.com Signed-off-by: Ioannis Papaioannou ioannis.papaioannou@workday.com Signed-off-by: Paul Mc Auley paul.mcauley@workday.com Signed-off-by: Sergio de Carvalho sergio.carvalho@workday.com
2025-06-13 10:33:35 +00:00 · 2019-12-13 11:41:32 +00:00
parent 7c8e66e427
commit 166eaf700f
13 changed files with 619 additions and 39 deletions
--- a/pkg/rbd/controllerserver.go
+++ b/pkg/rbd/controllerserver.go
@ -149,6 +149,14 @@ func (cs *ControllerServer) CreateVolume(ctx context.Context, req *csi.CreateVol
 		return nil, status.Error(codes.Internal, err.Error())
 	}
 	if found {
+		if rbdVol.Encrypted {
+			err = ensureEncryptionMetadataSet(ctx, cr, rbdVol)
+			if err != nil {
+				klog.Errorf(util.Log(ctx, err.Error()))
+				return nil, err
+			}
+		}
+
 		return &csi.CreateVolumeResponse{
 			Volume: &csi.Volume{
 				VolumeId:      rbdVol.VolID,
@ -176,6 +184,20 @@ func (cs *ControllerServer) CreateVolume(ctx context.Context, req *csi.CreateVol
 		return nil, err
 	}

+	if rbdVol.Encrypted {
+		err = ensureEncryptionMetadataSet(ctx, cr, rbdVol)
+		if err != nil {
+			klog.Errorf(util.Log(ctx, "failed to save encryption status, deleting image %s"),
+				rbdVol.RbdImageName)
+			if deleteErr := deleteImage(ctx, rbdVol, cr); err != nil {
+				klog.Errorf(util.Log(ctx, "failed to delete rbd image: %s/%s with error: %v"),
+					rbdVol.Pool, rbdVol.RbdImageName, deleteErr)
+				return nil, deleteErr
+			}
+			return nil, err
+		}
+	}
+
 	return &csi.CreateVolumeResponse{
 		Volume: &csi.Volume{
 			VolumeId:      rbdVol.VolID,
@ -211,6 +233,7 @@ func (cs *ControllerServer) createBackingImage(ctx context.Context, rbdVol *rbdV

 	return nil
 }
+
 func (cs *ControllerServer) checkSnapshot(ctx context.Context, req *csi.CreateVolumeRequest, rbdVol *rbdVolume) error {
 	snapshot := req.VolumeContentSource.GetSnapshot()
 	if snapshot == nil {
--- a/pkg/rbd/nodeserver.go
+++ b/pkg/rbd/nodeserver.go
@ -87,25 +87,14 @@ func (ns *NodeServer) NodeStageVolume(ctx context.Context, req *csi.NodeStageVol
 	}
 	defer ns.VolumeLocks.Release(volID)

-	isLegacyVolume := false
-	volName, err := getVolumeName(volID)
-	if err != nil {
-		// error ErrInvalidVolID may mean this is an 1.0.0 version volume, check for name
-		// pattern match in addition to error to ensure this is a likely v1.0.0 volume
-		if _, ok := err.(ErrInvalidVolID); !ok || !isLegacyVolumeID(volID) {
-			return nil, status.Error(codes.InvalidArgument, err.Error())
-		}
-
-		volName, err = getLegacyVolumeName(req.GetStagingTargetPath())
-		if err != nil {
-			return nil, status.Error(codes.InvalidArgument, err.Error())
-		}
-		isLegacyVolume = true
-	}
-
 	stagingParentPath := req.GetStagingTargetPath()
 	stagingTargetPath := stagingParentPath + "/" + volID

+	isLegacyVolume, volName, err := getVolumeNameByID(volID, stagingParentPath)
+	if err != nil {
+		return nil, status.Error(codes.InvalidArgument, err.Error())
+	}
+
 	var isNotMnt bool
 	// check if stagingPath is already mounted
 	isNotMnt, err = mount.IsNotMountPoint(ns.mounter, stagingTargetPath)
@ -123,6 +112,7 @@ func (ns *NodeServer) NodeStageVolume(ctx context.Context, req *csi.NodeStageVol
 		return nil, status.Error(codes.Internal, err.Error())
 	}
 	volOptions.RbdImageName = volName
+	volOptions.VolID = req.GetVolumeId()

 	isMounted := false
 	isStagePathCreated := false
@ -145,7 +135,15 @@ func (ns *NodeServer) NodeStageVolume(ctx context.Context, req *csi.NodeStageVol
 	if err != nil {
 		return nil, status.Error(codes.Internal, err.Error())
 	}
-	klog.V(4).Infof(util.Log(ctx, "rbd image: %s/%s was successfully mapped at %s\n"), req.GetVolumeId(), volOptions.Pool, devicePath)
+	klog.V(4).Infof(util.Log(ctx, "rbd image: %s/%s was successfully mapped at %s\n"),
+		req.GetVolumeId(), volOptions.Pool, devicePath)
+
+	if volOptions.Encrypted {
+		devicePath, err = ns.processEncryptedDevice(ctx, volOptions, devicePath, cr, req.GetSecrets())
+		if err != nil {
+			return nil, status.Error(codes.Internal, err.Error())
+		}
+	}

 	err = ns.createStageMountPoint(ctx, stagingTargetPath, isBlock)
 	if err != nil {
@ -194,7 +192,7 @@ func (ns *NodeServer) undoStagingTransaction(ctx context.Context, stagingParentP

 	// Unmapping rbd device
 	if devicePath != "" {
-		err = detachRBDDevice(ctx, devicePath)
+		err = detachRBDDevice(ctx, devicePath, volID)
 		if err != nil {
 			klog.Errorf(util.Log(ctx, "failed to unmap rbd device: %s for volume %s with error: %v"), devicePath, volID, err)
 			// continue on failure to delete the stash file, as kubernetes will fail to delete the staging path otherwise
@ -273,18 +271,6 @@ func (ns *NodeServer) NodePublishVolume(ctx context.Context, req *csi.NodePublis
 	return &csi.NodePublishVolumeResponse{}, nil
 }

-func getVolumeName(volID string) (string, error) {
-	var vi util.CSIIdentifier
-
-	err := vi.DecomposeCSIID(volID)
-	if err != nil {
-		err = fmt.Errorf("error decoding volume ID (%s) (%s)", err, volID)
-		return "", ErrInvalidVolID{err}
-	}
-
-	return volJournal.NamingPrefix() + vi.ObjectUUID, nil
-}
-
 func getLegacyVolumeName(mountPath string) (string, error) {
 	var volName string

@ -521,7 +507,7 @@ func (ns *NodeServer) NodeUnstageVolume(ctx context.Context, req *csi.NodeUnstag

 	// Unmapping rbd device
 	imageSpec := imgInfo.Pool + "/" + imgInfo.ImageName
-	if err = detachRBDImageOrDeviceSpec(ctx, imageSpec, true, imgInfo.NbdAccess); err != nil {
+	if err = detachRBDImageOrDeviceSpec(ctx, imageSpec, true, imgInfo.NbdAccess, req.GetVolumeId()); err != nil {
 		klog.Errorf(util.Log(ctx, "error unmapping volume (%s) from staging path (%s): (%v)"), req.GetVolumeId(), stagingTargetPath, err)
 		return nil, status.Error(codes.Internal, err.Error())
 	}
@ -613,3 +599,110 @@ func (ns *NodeServer) NodeGetCapabilities(ctx context.Context, req *csi.NodeGetC
 		},
 	}, nil
 }
+
+func (ns *NodeServer) processEncryptedDevice(ctx context.Context, volOptions *rbdVolume, devicePath string, cr *util.Credentials, secrets map[string]string) (string, error) {
+	imageSpec := volOptions.Pool + "/" + volOptions.RbdImageName
+	encrypted, err := util.CheckRbdImageEncrypted(ctx, cr, volOptions.Monitors, imageSpec)
+	if err != nil {
+		klog.Errorf(util.Log(ctx, "failed to get encryption status for rbd image %s: %v"),
+			imageSpec, err)
+		return "", err
+	}
+
+	if encrypted == rbdImageRequiresEncryption {
+		diskMounter := &mount.SafeFormatAndMount{Interface: ns.mounter, Exec: mount.NewOsExec()}
+		// TODO: update this when adding support for static (pre-provisioned) PVs
+		var existingFormat string
+		existingFormat, err = diskMounter.GetDiskFormat(devicePath)
+		if err != nil {
+			return "", fmt.Errorf("failed to get disk format for path %s, error: %v", devicePath, err)
+		}
+		if existingFormat != "" {
+			return "", fmt.Errorf("can not encrypt rbdImage %s that already has file system: %s",
+				imageSpec, existingFormat)
+		}
+		err = encryptDevice(ctx, volOptions, secrets, cr, devicePath)
+		if err != nil {
+			return "", fmt.Errorf("failed to encrypt rbd image %s: %v", imageSpec, err)
+		}
+	} else if encrypted != rbdImageEncrypted {
+		return "", fmt.Errorf("rbd image %s found mounted with unexpected encryption status %s",
+			imageSpec, encrypted)
+	}
+
+	devicePath, err = openEncryptedDevice(ctx, volOptions, devicePath, secrets)
+	if err != nil {
+		return "", err
+	}
+
+	return devicePath, nil
+}
+
+func encryptDevice(ctx context.Context, rbdVol *rbdVolume, secret map[string]string, cr *util.Credentials, devicePath string) error {
+	passphrase, err := util.GetCryptoPassphrase(secret)
+	if err != nil {
+		klog.Errorf(util.Log(ctx, "failed to get crypto passphrase for %s/%s: %v"),
+			rbdVol.Pool, rbdVol.RbdImageName, err)
+		return err
+	}
+
+	if err = util.EncryptVolume(ctx, devicePath, passphrase); err != nil {
+		err = fmt.Errorf("failed to encrypt volume %s/%s: %v", rbdVol.Pool, rbdVol.RbdImageName, err)
+		klog.Errorf(util.Log(ctx, err.Error()))
+		return err
+	}
+
+	imageSpec := rbdVol.Pool + "/" + rbdVol.RbdImageName
+	err = util.SaveRbdImageEncryptionStatus(ctx, cr, rbdVol.Monitors, imageSpec, rbdImageEncrypted)
+
+	return err
+}
+
+func openEncryptedDevice(ctx context.Context, volOptions *rbdVolume, devicePath string, secrets map[string]string) (string, error) {
+	passphrase, err := util.GetCryptoPassphrase(secrets)
+	if err != nil {
+		klog.Errorf(util.Log(ctx, "failed to get passphrase for encrypted device %s/%s: %v"),
+			volOptions.Pool, volOptions.RbdImageName, err)
+		return "", status.Error(codes.Internal, err.Error())
+	}
+
+	mapperFile, mapperFilePath := util.VolumeMapper(volOptions.VolID)
+
+	isOpen, err := util.IsDeviceOpen(ctx, mapperFilePath)
+	if err != nil {
+		klog.Errorf(util.Log(ctx, "failed to check device %s encryption status: %s"), devicePath, err)
+		return devicePath, err
+	}
+	if isOpen {
+		klog.V(4).Infof(util.Log(ctx, "encrypted device is already open at %s"), mapperFilePath)
+	} else {
+		err = util.OpenEncryptedVolume(ctx, devicePath, mapperFile, passphrase)
+		if err != nil {
+			klog.Errorf(util.Log(ctx, "failed to open device %s/%s: %v"),
+				volOptions.Pool, volOptions.RbdImageName, err)
+			return devicePath, err
+		}
+	}
+
+	return mapperFilePath, nil
+}
+
+func getVolumeNameByID(volID, stagingTargetPath string) (bool, string, error) {
+	volName, err := getVolumeName(volID)
+	if err != nil {
+		// error ErrInvalidVolID may mean this is an 1.0.0 version volume, check for name
+		// pattern match in addition to error to ensure this is a likely v1.0.0 volume
+		if _, ok := err.(ErrInvalidVolID); !ok || !isLegacyVolumeID(volID) {
+			return false, "", status.Error(codes.InvalidArgument, err.Error())
+		}
+
+		volName, err = getLegacyVolumeName(stagingTargetPath)
+		if err != nil {
+			return false, "", status.Error(codes.InvalidArgument, err.Error())
+		}
+
+		return true, volName, nil
+	}
+
+	return false, volName, nil
+}
--- a/pkg/rbd/rbd_attach.go
+++ b/pkg/rbd/rbd_attach.go
@ -225,7 +225,7 @@ func createPath(ctx context.Context, volOpt *rbdVolume, cr *util.Credentials) (s
 		klog.Warningf(util.Log(ctx, "rbd: map error %v, rbd output: %s"), err, string(output))
 		// unmap rbd image if connection timeout
 		if strings.Contains(err.Error(), rbdMapConnectionTimeout) {
-			detErr := detachRBDImageOrDeviceSpec(ctx, imagePath, true, isNbd)
+			detErr := detachRBDImageOrDeviceSpec(ctx, imagePath, true, isNbd, volOpt.VolID)
 			if detErr != nil {
 				klog.Warningf(util.Log(ctx, "rbd: %s unmap error %v"), imagePath, detErr)
 			}
@ -260,21 +260,38 @@ func waitForrbdImage(ctx context.Context, backoff wait.Backoff, volOptions *rbdV
 	return err
 }

-func detachRBDDevice(ctx context.Context, devicePath string) error {
+func detachRBDDevice(ctx context.Context, devicePath, volumeID string) error {
 	nbdType := false
 	if strings.HasPrefix(devicePath, "/dev/nbd") {
 		nbdType = true
 	}

-	return detachRBDImageOrDeviceSpec(ctx, devicePath, false, nbdType)
+	return detachRBDImageOrDeviceSpec(ctx, devicePath, false, nbdType, volumeID)
 }

 // detachRBDImageOrDeviceSpec detaches an rbd imageSpec or devicePath, with additional checking
 // when imageSpec is used to decide if image is already unmapped
-func detachRBDImageOrDeviceSpec(ctx context.Context, imageOrDeviceSpec string, isImageSpec, ndbType bool) error {
-	var err error
+func detachRBDImageOrDeviceSpec(ctx context.Context, imageOrDeviceSpec string, isImageSpec, ndbType bool, volumeID string) error {
 	var output []byte

+	mapperFile, mapperPath := util.VolumeMapper(volumeID)
+	mappedDevice, mapper, err := util.DeviceEncryptionStatus(ctx, mapperPath)
+	if err != nil {
+		klog.Errorf(util.Log(ctx, "error determining LUKS device on %s, %s: %s"),
+			mapperPath, imageOrDeviceSpec, err)
+		return err
+	}
+	if len(mapper) > 0 {
+		// mapper found, so it is open Luks device
+		err = util.CloseEncryptedVolume(ctx, mapperFile)
+		if err != nil {
+			klog.Warningf(util.Log(ctx, "error closing LUKS device on %s, %s: %s"),
+				mapperPath, imageOrDeviceSpec, err)
+			return err
+		}
+		imageOrDeviceSpec = mappedDevice
+	}
+
 	accessType := accessTypeKRbd
 	if ndbType {
 		accessType = accessTypeNbd
--- a/pkg/rbd/rbd_util.go
+++ b/pkg/rbd/rbd_util.go
@ -24,6 +24,7 @@ import (
 	"os"
 	"os/exec"
 	"path/filepath"
+	"strconv"
 	"strings"
 	"time"

@ -53,6 +54,10 @@ const (
 	rbdTaskRemoveCmdInvalidString1      = "no valid command found"
 	rbdTaskRemoveCmdInvalidString2      = "Error EINVAL: invalid command"
 	rbdTaskRemoveCmdAccessDeniedMessage = "Error EACCES:"
+
+	// Encryption statuses for RbdImage
+	rbdImageEncrypted          = "encrypted"
+	rbdImageRequiresEncryption = "requiresEncryption"
 )

 // rbdVolume represents a CSI volume and its RBD image specifics
@ -71,15 +76,16 @@ type rbdVolume struct {
 	DataPool           string
 	ImageFormat        string `json:"imageFormat"`
 	ImageFeatures      string `json:"imageFeatures"`
-	VolSize            int64  `json:"volSize"`
 	AdminID            string `json:"adminId"`
 	UserID             string `json:"userId"`
 	Mounter            string `json:"mounter"`
-	DisableInUseChecks bool   `json:"disableInUseChecks"`
 	ClusterID          string `json:"clusterId"`
 	RequestName        string
 	VolName            string `json:"volName"`
 	MonValueFromSecret string `json:"monValueFromSecret"`
+	VolSize            int64  `json:"volSize"`
+	DisableInUseChecks bool   `json:"disableInUseChecks"`
+	Encrypted          bool
 }

 // rbdSnapshot represents a CSI snapshot and its RBD snapshot specifics
@ -486,6 +492,16 @@ func genVolFromVolumeOptions(ctx context.Context, volOptions, credentials map[st
 		rbdVol.Mounter = rbdDefaultMounter
 	}

+	rbdVol.Encrypted = false
+	encrypted, ok := volOptions["encrypted"]
+	if ok {
+		rbdVol.Encrypted, err = strconv.ParseBool(encrypted)
+		if err != nil {
+			return nil, fmt.Errorf(
+				"invalid value set in 'encrypted': %s (should be \"true\" or \"false\")", encrypted)
+		}
+	}
+
 	return rbdVol, nil
 }

@ -831,3 +847,30 @@ func resizeRBDImage(rbdVol *rbdVolume, newSize int64, cr *util.Credentials) erro

 	return nil
 }
+
+func getVolumeName(volID string) (string, error) {
+	var vi util.CSIIdentifier
+
+	err := vi.DecomposeCSIID(volID)
+	if err != nil {
+		err = fmt.Errorf("error decoding volume ID (%s) (%s)", err, volID)
+		return "", ErrInvalidVolID{err}
+	}
+
+	return volJournal.NamingPrefix() + vi.ObjectUUID, nil
+}
+
+func ensureEncryptionMetadataSet(ctx context.Context, cr *util.Credentials, rbdVol *rbdVolume) error {
+	rbdImageName, err := getVolumeName(rbdVol.VolID)
+	if err != nil {
+		return err
+	}
+	imageSpec := rbdVol.Pool + "/" + rbdImageName
+
+	err = util.SaveRbdImageEncryptionStatus(ctx, cr, rbdVol.Monitors, imageSpec, rbdImageRequiresEncryption)
+	if err != nil {
+		return fmt.Errorf("failed to save encryption status for %s: %v", imageSpec, err)
+	}
+
+	return nil
+}