rbd: fscrypt file encryption support

Integrate basic fscrypt functionality into RBD initialization. To
activate file encryption instead of block introduce the new
'encryptionType' storage class key.

Signed-off-by: Marcel Lauhoff <marcel.lauhoff@suse.com>

internal/rbd/encryption.go

@@ -63,6 +63,11 @@ const (
 	oldMetadataDEK = ".rbd.csi.ceph.com/dek"
 
 	encryptionPassphraseSize = 20
+
+	// rbdDefaultEncryptionType is the default to use when the
+	// user did not specify an "encryptionType", but set
+	// "encryption": true.
+	rbdDefaultEncryptionType = util.EncryptionTypeBlock
 )
 
 // checkRbdImageEncrypted verifies if rbd image was encrypted when created.
@@ -98,11 +103,20 @@ func (ri *rbdImage) isBlockEncrypted() bool {
 	return ri.blockEncryption != nil
 }
 
-// isBlockDeviceEncrypted returns `true` if the filesystem on the rbdImage is (or needs to be) encrypted.
+// isFileEncrypted returns `true` if the filesystem on the rbdImage is (or needs to be) encrypted.
 func (ri *rbdImage) isFileEncrypted() bool {
 	return ri.fileEncryption != nil
 }
 
+func IsFileEncrypted(ctx context.Context, volOptions map[string]string) (bool, error) {
+	_, encType, err := ParseEncryptionOpts(ctx, volOptions)
+	if err != nil {
+		return false, err
+	}
+
+	return encType == util.EncryptionTypeFile, nil
+}
+
 // setupBlockEncryption configures the metadata of the RBD image for encryption:
 // - the Data-Encryption-Key (DEK) will be generated stored for use by the KMS;
 // - the RBD image will be marked to support encryption in its metadata.
@@ -137,7 +151,7 @@ func (ri *rbdImage) setupBlockEncryption(ctx context.Context) error {
 // (Usecase: Restoring snapshot into a storageclass with different encryption config).
 func (ri *rbdImage) copyEncryptionConfig(cp *rbdImage, copyOnlyPassphrase bool) error {
 	// nothing to do if parent image is not encrypted.
-	if !ri.isBlockEncrypted() {
+	if !ri.isBlockEncrypted() && !ri.isFileEncrypted() {
 		return nil
 	}
 
@@ -146,25 +160,54 @@ func (ri *rbdImage) copyEncryptionConfig(cp *rbdImage, copyOnlyPassphrase bool)
 			"set!? Call stack: %s", ri, cp, ri.VolID, util.CallStack())
 	}
 
-	// get the unencrypted passphrase
-	passphrase, err := ri.blockEncryption.GetCryptoPassphrase(ri.VolID)
-	if err != nil {
-		return fmt.Errorf("failed to fetch passphrase for %q: %w",
-			ri, err)
-	}
+	if ri.isBlockEncrypted() {
+		// get the unencrypted passphrase
+		passphrase, err := ri.blockEncryption.GetCryptoPassphrase(ri.VolID)
+		if err != nil {
+			return fmt.Errorf("failed to fetch passphrase for %q: %w",
+				ri, err)
+		}
 
-	if !copyOnlyPassphrase {
-		cp.blockEncryption, err = util.NewVolumeEncryption(ri.blockEncryption.GetID(), ri.blockEncryption.KMS)
-		if errors.Is(err, util.ErrDEKStoreNeeded) {
-			cp.blockEncryption.SetDEKStore(cp)
+		if !copyOnlyPassphrase {
+			cp.blockEncryption, err = util.NewVolumeEncryption(ri.blockEncryption.GetID(), ri.blockEncryption.KMS)
+			if errors.Is(err, util.ErrDEKStoreNeeded) {
+				cp.blockEncryption.SetDEKStore(cp)
+			}
+		}
+
+		// re-encrypt the plain passphrase for the cloned volume
+		err = cp.blockEncryption.StoreCryptoPassphrase(cp.VolID, passphrase)
+		if err != nil {
+			return fmt.Errorf("failed to store passphrase for %q: %w",
+				cp, err)
 		}
 	}
 
-	// re-encrypt the plain passphrase for the cloned volume
-	err = cp.blockEncryption.StoreCryptoPassphrase(cp.VolID, passphrase)
-	if err != nil {
-		return fmt.Errorf("failed to store passphrase for %q: %w",
-			cp, err)
+	if ri.isFileEncrypted() && !copyOnlyPassphrase {
+		var err error
+		cp.fileEncryption, err = util.NewVolumeEncryption(ri.fileEncryption.GetID(), ri.fileEncryption.KMS)
+		if errors.Is(err, util.ErrDEKStoreNeeded) {
+			_, err := ri.fileEncryption.KMS.GetSecret("")
+			if errors.Is(err, kmsapi.ErrGetSecretUnsupported) {
+				return err
+			}
+		}
+	}
+
+	if ri.isFileEncrypted() && ri.fileEncryption.KMS.RequiresDEKStore() == kmsapi.DEKStoreIntegrated {
+		// get the unencrypted passphrase
+		passphrase, err := ri.fileEncryption.GetCryptoPassphrase(ri.VolID)
+		if err != nil {
+			return fmt.Errorf("failed to fetch passphrase for %q: %w",
+				ri, err)
+		}
+
+		// re-encrypt the plain passphrase for the cloned volume
+		err = cp.fileEncryption.StoreCryptoPassphrase(cp.VolID, passphrase)
+		if err != nil {
+			return fmt.Errorf("failed to store passphrase for %q: %w",
+				cp, err)
+		}
 	}
 
 	// copy encryption status for the original volume
@@ -173,6 +216,7 @@ func (ri *rbdImage) copyEncryptionConfig(cp *rbdImage, copyOnlyPassphrase bool)
 		return fmt.Errorf("failed to get encryption status for %q: %w",
 			ri, err)
 	}
+
 	err = cp.ensureEncryptionMetadataSet(status)
 	if err != nil {
 		return fmt.Errorf("failed to store encryption status for %q: "+
@@ -185,12 +229,12 @@ func (ri *rbdImage) copyEncryptionConfig(cp *rbdImage, copyOnlyPassphrase bool)
 // repairEncryptionConfig checks the encryption state of the current rbdImage,
 // and makes sure that the destination rbdImage has the same configuration.
 func (ri *rbdImage) repairEncryptionConfig(dest *rbdImage) error {
-	if !ri.isBlockEncrypted() {
+	if !ri.isBlockEncrypted() && !ri.isFileEncrypted() {
 		return nil
 	}
 
 	// if ri is encrypted, copy its configuration in case it is missing
-	if !dest.isBlockEncrypted() {
+	if !dest.isBlockEncrypted() && !dest.isFileEncrypted() {
 		// dest needs to be connected to the cluster, otherwise it will
 		// not be possible to write any metadata
 		if dest.conn == nil {
@@ -262,14 +306,22 @@ func (rv *rbdVolume) openEncryptedDevice(ctx context.Context, devicePath string)
 }
 
 func (ri *rbdImage) initKMS(ctx context.Context, volOptions, credentials map[string]string) error {
-	kmsID, err := ri.ParseEncryptionOpts(ctx, volOptions)
+	kmsID, encType, err := ParseEncryptionOpts(ctx, volOptions)
 	if err != nil {
 		return err
 	} else if kmsID == "" {
 		return nil
 	}
 
-	err = ri.configureBlockDeviceEncryption(kmsID, credentials)
+	switch encType {
+	case util.EncryptionTypeBlock:
+		err = ri.configureBlockEncryption(kmsID, credentials)
+	case util.EncryptionTypeFile:
+		err = ri.configureFileEncryption(kmsID, credentials)
+	case util.EncryptionTypeInvalid:
+		return fmt.Errorf("invalid encryption type")
+	}
+
 	if err != nil {
 		return fmt.Errorf("invalid encryption kms configuration: %w", err)
 	}
@@ -278,10 +330,10 @@ func (ri *rbdImage) initKMS(ctx context.Context, volOptions, credentials map[str
 }
 
 // ParseEncryptionOpts returns kmsID and sets Owner attribute.
-func (ri *rbdImage) ParseEncryptionOpts(
+func ParseEncryptionOpts(
 	ctx context.Context,
 	volOptions map[string]string,
-) (string, error) {
+) (string, util.EncryptionType, error) {
 	var (
 		err              error
 		ok               bool
@@ -289,14 +341,16 @@ func (ri *rbdImage) ParseEncryptionOpts(
 	)
 	encrypted, ok = volOptions["encrypted"]
 	if !ok {
-		return "", nil
+		return "", util.EncryptionTypeInvalid, err
 	}
 	kmsID, err = util.FetchEncryptionKMSID(encrypted, volOptions["encryptionKMSID"])
 	if err != nil {
-		return "", err
+		return "", util.EncryptionTypeInvalid, err
 	}
 
-	return kmsID, nil
+	encType := util.FetchEncryptionType(volOptions, rbdDefaultEncryptionType)
+
+	return kmsID, encType, nil
 }
 
 // configureBlockDeviceEncryption sets up the VolumeEncryption for this rbdImage. Once
@@ -318,6 +372,31 @@ func (ri *rbdImage) configureBlockEncryption(kmsID string, credentials map[strin
 	return nil
 }
 
+// configureBlockDeviceEncryption sets up the VolumeEncryption for this rbdImage. Once
+// configured, use isEncrypted() to see if the volume supports encryption.
+func (ri *rbdImage) configureFileEncryption(kmsID string, credentials map[string]string) error {
+	kms, err := kmsapi.GetKMS(ri.Owner, kmsID, credentials)
+	if err != nil {
+		return err
+	}
+
+	ri.fileEncryption, err = util.NewVolumeEncryption(kmsID, kms)
+
+	if errors.Is(err, util.ErrDEKStoreNeeded) {
+		// fscrypt uses secrets directly from the KMS.
+		// Therefore we do not support an additional DEK
+		// store. Since not all "metadata" KMS support
+		// GetSecret, test for support here. Postpone any
+		// other error handling
+		_, err := ri.fileEncryption.KMS.GetSecret("")
+		if errors.Is(err, kmsapi.ErrGetSecretUnsupported) {
+			return err
+		}
+	}
+
+	return nil
+}
+
 // StoreDEK saves the DEK in the metadata, overwrites any existing contents.
 func (ri *rbdImage) StoreDEK(volumeID, dek string) error {
 	if ri.VolID == "" {

internal/rbd/nodeserver.go

@@ -27,6 +27,7 @@ import (
 	csicommon "github.com/ceph/ceph-csi/internal/csi-common"
 	"github.com/ceph/ceph-csi/internal/journal"
 	"github.com/ceph/ceph-csi/internal/util"
+	"github.com/ceph/ceph-csi/internal/util/fscrypt"
 	"github.com/ceph/ceph-csi/internal/util/log"
 
 	librbd "github.com/ceph/go-ceph/rbd"
@@ -433,6 +434,12 @@ func (ns *NodeServer) stageTransaction(
 		transaction.isBlockEncrypted = true
 	}
 
+	if volOptions.isFileEncrypted() {
+		if err = fscrypt.InitializeNode(ctx); err != nil {
+			return transaction, err
+		}
+	}
+
 	stagingTargetPath := getStagingTargetPath(req)
 
 	isBlock := req.GetVolumeCapability().GetBlock() != nil
@@ -444,12 +451,20 @@ func (ns *NodeServer) stageTransaction(
 	transaction.isStagePathCreated = true
 
 	// nodeStage Path
-	err = ns.mountVolumeToStagePath(ctx, req, staticVol, stagingTargetPath, devicePath)
+	err = ns.mountVolumeToStagePath(ctx, req, staticVol, stagingTargetPath, devicePath, volOptions.isFileEncrypted())
 	if err != nil {
 		return transaction, err
 	}
 	transaction.isMounted = true
 
+	if volOptions.isFileEncrypted() {
+		log.DebugLog(ctx, "rbd fscrypt: trying to unlock filesystem on %s image %q", stagingTargetPath, volOptions.VolID)
+		err = fscrypt.Unlock(ctx, volOptions.fileEncryption, stagingTargetPath, volOptions.VolID)
+		if err != nil {
+			return transaction, err
+		}
+	}
+
 	// As we are supporting the restore of a volume to a bigger size and
 	// creating bigger size clone from a volume, we need to check filesystem
 	// resize is required, if required resize filesystem.
@@ -691,6 +706,17 @@ func (ns *NodeServer) NodePublishVolume(
 		return &csi.NodePublishVolumeResponse{}, nil
 	}
 
+	fileEncrypted, err := IsFileEncrypted(ctx, req.GetVolumeContext())
+	if err != nil {
+		return nil, status.Error(codes.Internal, err.Error())
+	}
+	if fileEncrypted {
+		stagingPath = fscrypt.AppendEncyptedSubdirectory(stagingPath)
+		if err = fscrypt.IsDirectoryUnlocked(stagingPath, req.GetVolumeCapability().GetMount().GetFsType()); err != nil {
+			return nil, status.Error(codes.Internal, err.Error())
+		}
+	}
+
 	// Publish Path
 	err = ns.mountVolume(ctx, stagingPath, req)
 	if err != nil {
@@ -707,6 +733,7 @@ func (ns *NodeServer) mountVolumeToStagePath(
 	req *csi.NodeStageVolumeRequest,
 	staticVol bool,
 	stagingPath, devicePath string,
+	fileEncryption bool,
 ) error {
 	readOnly := false
 	fsType := req.GetVolumeCapability().GetMount().GetFsType()
@@ -751,7 +778,11 @@ func (ns *NodeServer) mountVolumeToStagePath(
 		args := []string{}
 		switch fsType {
 		case "ext4":
-			args = []string{"-m0", "-Enodiscard,lazy_itable_init=1,lazy_journal_init=1", devicePath}
+			args = []string{"-m0", "-Enodiscard,lazy_itable_init=1,lazy_journal_init=1"}
+			if fileEncryption {
+				args = append(args, "-Oencrypt")
+			}
+			args = append(args, devicePath)
 		case "xfs":
 			args = []string{"-K", devicePath}
 			// always disable reflink

internal/rbd/rbd_journal.go

@@ -568,7 +568,7 @@ func RegenerateJournal(
 
 	rbdVol.Owner = owner
 
-	kmsID, err = rbdVol.ParseEncryptionOpts(ctx, volumeAttributes)
+	kmsID, _, err = ParseEncryptionOpts(ctx, volumeAttributes)
 	if err != nil {
 		return "", err
 	}

internal/rbd/rbd_util.go

@@ -137,7 +137,7 @@ type rbdImage struct {
 	// fileEncryption provides access to optional VolumeEncryption functions (e.g fscrypt)
 	fileEncryption *util.VolumeEncryption
 
-	CreatedAt  *timestamp.Timestamp
+	CreatedAt *timestamp.Timestamp
 
 	// conn is a connection to the Ceph cluster obtained from a ConnPool
 	conn *util.ClusterConnection
@@ -393,6 +393,9 @@ func (ri *rbdImage) Destroy() {
 	if ri.isBlockEncrypted() {
 		ri.blockEncryption.Destroy()
 	}
+	if ri.isFileEncrypted() {
+		ri.fileEncryption.Destroy()
+	}
 }
 
 // String returns the image-spec (pool/{namespace/}image) format of the image.
@@ -631,9 +634,16 @@ func (ri *rbdImage) deleteImage(ctx context.Context) error {
 	}
 
 	if ri.isBlockEncrypted() {
-		log.DebugLog(ctx, "rbd: going to remove DEK for %q", ri)
+		log.DebugLog(ctx, "rbd: going to remove DEK for %q (block encryption)", ri)
 		if err = ri.blockEncryption.RemoveDEK(ri.VolID); err != nil {
-			log.WarningLog(ctx, "failed to clean the passphrase for volume %s: %s", ri.VolID, err)
+			log.WarningLog(ctx, "failed to clean the passphrase for volume %s (block encryption): %s", ri.VolID, err)
+		}
+	}
+
+	if ri.isFileEncrypted() {
+		log.DebugLog(ctx, "rbd: going to remove DEK for %q (file encryption)", ri)
+		if err = ri.fileEncryption.RemoveDEK(ri.VolID); err != nil {
+			log.WarningLog(ctx, "failed to clean the passphrase for volume %s (file encryption): %s", ri.VolID, err)
 		}
 	}
 
@@ -1967,11 +1977,13 @@ func (ri *rbdImage) getOrigSnapName(snapID uint64) (string, error) {
 }
 
 func (ri *rbdImage) isCompatibleEncryption(dst *rbdImage) error {
+	riEncrypted := ri.isBlockEncrypted() || ri.isFileEncrypted()
+	dstEncrypted := dst.isBlockEncrypted() || dst.isFileEncrypted()
 	switch {
-	case ri.isBlockEncrypted() && !dst.isBlockEncrypted():
+	case riEncrypted && !dstEncrypted:
 		return fmt.Errorf("cannot create unencrypted volume from encrypted volume %q", ri)
 
-	case !ri.isBlockEncrypted() && dst.isBlockEncrypted():
+	case !riEncrypted && dstEncrypted:
 		return fmt.Errorf("cannot create encrypted volume from unencrypted volume %q", ri)
 	}