rbd: add backwards compatible encryption in NodeStageVolume

When a volume was provisioned by an old Ceph-CSI provisioner, the
metadata of the RBD image will contain `requiresEncryption` to indicate
a passphrase needs to be created. New Ceph-CSI provisioners create the
passphrase in the CreateVolume request, and set `encryptionPrepared`
instead.

When a new node-plugin detects that `requiresEncryption` is set in the
RBD image metadata, it will fallback to the old behaviour.

In case `encryptionPrepared` is read from the RBD image metadata, the
passphrase is used to cryptsetup/format the image.

Signed-off-by: Niels de Vos <ndevos@redhat.com>

docs/deploy-rbd.md

@@ -189,7 +189,8 @@ possible to encrypt them with ceph-csi by using LUKS encryption.
 
 * create volume request received
 * volume requested to be created in Ceph
-* encrypted state "requiresEncryption" is saved in image-meta in Ceph
+* new passphrase is generated and stored in selected KMS if KMS is in use
+* encrypted state "encryptionPrepared" is saved in image-meta in Ceph
 
 **Attach volume**:
 
@@ -197,8 +198,8 @@ possible to encrypt them with ceph-csi by using LUKS encryption.
 * volume is attached to provisioner container
 * on first time attachment
   (no file system on the attached device, checked with blkid)
-  * new passphrase is generated and stored in selected KMS if KMS is in use
+  * passphrase is retrieved from selected KMS if KMS is in use
-  * device is encrypted with LUKS using a passphrase from K8s secrets.
+  * device is encrypted with LUKS using a passphrase from K8s Secret or KMS
   * image-meta updated to "encrypted" in Ceph
 * passphrase is retrieved from selected KMS if KMS is in use
 * device is open and device path is changed to use a mapper device

internal/rbd/encryption.go

@@ -18,18 +18,37 @@ package rbd
 
 import (
 	"context"
+	"errors"
 	"fmt"
 	"strings"
 
 	"github.com/ceph/ceph-csi/internal/util"
+
+	librbd "github.com/ceph/go-ceph/rbd"
 )
 
+// rbdEncryptionState describes the status of the process where the image is
+// with respect to being encrypted.
 type rbdEncryptionState string
 
 const (
-	// Encryption statuses for RbdImage
+	// rbdImageEncryptionUnknown means the image is not encrypted, or the
+	// metadata of the image can not be fetched.
 	rbdImageEncryptionUnknown = rbdEncryptionState("")
+	// rbdImageEncrypted is set in the image metadata after the image has
+	// been formatted with cryptsetup. Future usage of the image should
+	// unlock the image before mounting.
 	rbdImageEncrypted = rbdEncryptionState("encrypted")
+	// rbdImageEncryptionPrepared gets set in the image metadata once the
+	// passphrase for the image has been generated and stored in the KMS.
+	// When using the image for the first time, it needs to be encrypted
+	// with cryptsetup before updating the state to `rbdImageEncrypted`.
+	rbdImageEncryptionPrepared = rbdEncryptionState("encryptionPrepared")
+
+	// rbdImageRequiresEncryption has been deprecated, it is used only for
+	// volumes that have been created with an old provisioner, were never
+	// attached/mounted and now get staged by a new node-plugin
+	// TODO: remove this backwards compatibility support
 	rbdImageRequiresEncryption = rbdEncryptionState("requiresEncryption")
 
 	// image metadata key for encryption
@@ -39,13 +58,16 @@ const (
 // checkRbdImageEncrypted verifies if rbd image was encrypted when created.
 func (rv *rbdVolume) checkRbdImageEncrypted(ctx context.Context) (rbdEncryptionState, error) {
 	value, err := rv.GetMetadata(encryptionMetaKey)
-	if err != nil {
+	if errors.Is(err, librbd.ErrNotFound) {
-		util.ErrorLog(ctx, "checking image %s encrypted state metadata failed: %s", rv, err)
+		util.DebugLog(ctx, "image %s encrypted state not set", rv.String())
+		return rbdImageEncryptionUnknown, nil
+	} else if err != nil {
+		util.ErrorLog(ctx, "checking image %s encrypted state metadata failed: %s", rv.String(), err)
 		return rbdImageEncryptionUnknown, err
 	}
 
 	encrypted := rbdEncryptionState(strings.TrimSpace(value))
-	util.DebugLog(ctx, "image %s encrypted state metadata reports %q", rv, encrypted)
+	util.DebugLog(ctx, "image %s encrypted state metadata reports %q", rv.String(), encrypted)
 	return encrypted, nil
 }
 
@@ -69,7 +91,7 @@ func (rv *rbdVolume) setupEncryption(ctx context.Context) error {
 		return err
 	}
 
-	err = rv.ensureEncryptionMetadataSet(rbdImageRequiresEncryption)
+	err = rv.ensureEncryptionMetadataSet(rbdImageEncryptionPrepared)
 	if err != nil {
 		util.ErrorLog(ctx, "failed to save encryption status, deleting "+
 			"image %s: %s", rv.String(), err)

internal/rbd/nodeserver.go

@@ -790,7 +790,24 @@ func (ns *NodeServer) processEncryptedDevice(ctx context.Context, volOptions *rb
 		return "", err
 	}
 
-	if encrypted == rbdImageRequiresEncryption {
+	switch {
+	case encrypted == rbdImageRequiresEncryption:
+		// If we get here, it means the image was created with a
+		// ceph-csi version that creates a passphrase for the encrypted
+		// device in NodeStage. New versions moved that to
+		// CreateVolume.
+		// Use the same setupEncryption() as CreateVolume does, and
+		// continue with the common process to crypt-format the device.
+		err = volOptions.setupEncryption(ctx)
+		if err != nil {
+			util.ErrorLog(ctx, "failed to setup encryption for rbd"+
+				"image %s: %v", imageSpec, err)
+			return "", err
+		}
+
+		// make sure we continue with the encrypting of the device
+		fallthrough
+	case encrypted == rbdImageEncryptionPrepared:
 		diskMounter := &mount.SafeFormatAndMount{Interface: ns.mounter, Exec: utilexec.New()}
 		// TODO: update this when adding support for static (pre-provisioned) PVs
 		var existingFormat string
@@ -816,7 +833,7 @@ func (ns *NodeServer) processEncryptedDevice(ctx context.Context, volOptions *rb
 			return "", fmt.Errorf("can not encrypt rbdImage %s that already has file system: %s",
 				imageSpec, existingFormat)
 		}
-	} else if encrypted != rbdImageEncrypted {
+	case encrypted != rbdImageEncrypted:
 		return "", fmt.Errorf("rbd image %s found mounted with unexpected encryption status %s",
 			imageSpec, encrypted)
 	}