From a5af5a13829a3b7f058b67aab6442ecb5c32756b Mon Sep 17 00:00:00 2001 From: Praveen M Date: Thu, 28 Mar 2024 17:10:11 +0530 Subject: [PATCH] rbd: add additional space for encrypted volumes issue: when a block-mode pvc is created with encryption enabled there is some space reserved for the encryption metadata. Which doesn't allows users to write extact amount of data that they have requested for. solution: create pvc with extra space needed for the encryption metadata. The extra space is added during the CreateVolume and ExpandVolume operations. And while returning the response remove the extra space so the client/user gets the requested size reported. Signed-off-by: Praveen M --- PendingReleaseNotes.md | 1 + internal/rbd/controllerserver.go | 22 ++++++++++++++ internal/rbd/encryption.go | 8 +++++ internal/rbd/rbd_util.go | 52 ++++++++++++++++++++++++++++++-- internal/util/cryptsetup.go | 13 +++++++- 5 files changed, 92 insertions(+), 4 deletions(-) diff --git a/PendingReleaseNotes.md b/PendingReleaseNotes.md index a77eb58d4..e2ae845f8 100644 --- a/PendingReleaseNotes.md +++ b/PendingReleaseNotes.md @@ -12,5 +12,6 @@ - deploy: radosNamespaceCephFS can be configured for ceph-csi-cephfs chart in [PR](https://github.com/ceph/ceph-csi/pull/4652) - build: update ceph release to squid in [PR](https://github.com/ceph/ceph-csi/pull/4735) - build: CentOS Stream 9 is used as OS in the container-images [PR](https://github.com/ceph/ceph-csi/pull/4735) +- rbd: add additional space for encrypted volumes for Luks2 header in [PR] (https://github.com/ceph/ceph-csi/pull/4582) ## NOTE diff --git a/internal/rbd/controllerserver.go b/internal/rbd/controllerserver.go index f0ac0a479..6a7c1c251 100644 --- a/internal/rbd/controllerserver.go +++ b/internal/rbd/controllerserver.go @@ -1235,6 +1235,17 @@ func (cs *ControllerServer) CreateSnapshot( return nil, status.Error(codes.Internal, err.Error()) } + err = vol.Connect(cr) + if err != nil { + return nil, status.Error(codes.Internal, err.Error()) + } + defer vol.Destroy(ctx) + + err = vol.getImageInfo() + if err != nil { + return nil, status.Error(codes.Internal, err.Error()) + } + csiSnap, err := vol.toSnapshot().ToCSI(ctx) if err != nil { return nil, status.Error(codes.Internal, err.Error()) @@ -1294,6 +1305,17 @@ func cloneFromSnapshot( } } + err = rbdSnap.Connect(cr) + if err != nil { + return nil, status.Error(codes.Internal, err.Error()) + } + defer rbdSnap.Destroy(ctx) + + err = rbdSnap.getImageInfo() + if err != nil { + return nil, status.Error(codes.Internal, err.Error()) + } + csiSnap, err := rbdSnap.ToCSI(ctx) if err != nil { return nil, status.Error(codes.Internal, err.Error()) diff --git a/internal/rbd/encryption.go b/internal/rbd/encryption.go index 4c27bb6e2..bbe6d7e12 100644 --- a/internal/rbd/encryption.go +++ b/internal/rbd/encryption.go @@ -59,6 +59,9 @@ const ( metadataDEK = "rbd.csi.ceph.com/dek" oldMetadataDEK = ".rbd.csi.ceph.com/dek" + // luks2 header size metadata key. + luks2HeaderSizeKey = "rbd.csi.ceph.com/luks2HeaderSize" + encryptionPassphraseSize = 20 // rbdDefaultEncryptionType is the default to use when the @@ -130,6 +133,11 @@ func (ri *rbdImage) setupBlockEncryption(ctx context.Context) error { return err } + err = ri.SetMetadata(luks2HeaderSizeKey, strconv.FormatUint(util.Luks2HeaderSize, 10)) + if err != nil { + return fmt.Errorf("failed to save %s metadata on image: %w", luks2HeaderSizeKey, err) + } + err = ri.ensureEncryptionMetadataSet(rbdImageEncryptionPrepared) if err != nil { log.ErrorLog(ctx, "failed to save encryption status, deleting "+ diff --git a/internal/rbd/rbd_util.go b/internal/rbd/rbd_util.go index a69b0e632..eed63cec2 100644 --- a/internal/rbd/rbd_util.go +++ b/internal/rbd/rbd_util.go @@ -449,8 +449,16 @@ func createImage(ctx context.Context, pOpts *rbdVolume, cr *util.Credentials) er return fmt.Errorf("failed to get IOContext: %w", err) } - err = librbd.CreateImage(pOpts.ioctx, pOpts.RbdImageName, - uint64(util.RoundOffVolSize(pOpts.VolSize)*helpers.MiB), options) + size := uint64(util.RoundOffVolSize(pOpts.VolSize) * helpers.MiB) + if pOpts.isBlockEncrypted() { + // When a block-mode PVC is created with encryption enabled, + // some space is reserved for the LUKS2 header. + // Add the LUKS2 header size to the image size so that the user has at least + // the requested size. + size += util.Luks2HeaderSize + } + + err = librbd.CreateImage(pOpts.ioctx, pOpts.RbdImageName, size, options) if err != nil { return fmt.Errorf("failed to create rbd image: %w", err) } @@ -1604,6 +1612,26 @@ func (ri *rbdImage) GetCreationTime(ctx context.Context) (*time.Time, error) { return ri.CreatedAt, nil } +// getLuks2HeaderSizeSet returns the value of the LUKS2 header size +// set in the image metadata (size returned in MiB). +func (ri *rbdImage) getLuks2HeaderSizeSet() (uint64, error) { + value, err := ri.GetMetadata(luks2HeaderSizeKey) + if err != nil { + if !errors.Is(err, librbd.ErrNotFound) { + return 0, err + } + + return 0, nil + } + + headerSize, parseErr := strconv.ParseUint(value, 10, 64) + if parseErr != nil { + return 0, parseErr + } + + return headerSize, nil +} + // getImageInfo queries rbd about the given image and returns its metadata, and returns // ErrImageNotFound if provided image is not found. func (ri *rbdImage) getImageInfo() error { @@ -1620,6 +1648,14 @@ func (ri *rbdImage) getImageInfo() error { // TODO: can rv.VolSize not be a uint64? Or initialize it to -1? ri.VolSize = int64(imageInfo.Size) + // If the luks2HeaderSizeKey metadata is set + // reduce the extra size of the LUKS header from the image size. + headerSize, err := ri.getLuks2HeaderSizeSet() + if err != nil { + return err + } + ri.VolSize -= int64(headerSize) + features, err := image.GetFeatures() if err != nil { return err @@ -1869,7 +1905,17 @@ func (ri *rbdImage) resize(newSize int64) error { } defer image.Close() - err = image.Resize(uint64(util.RoundOffVolSize(newSize) * helpers.MiB)) + size := uint64(util.RoundOffVolSize(newSize) * helpers.MiB) + + // If the luks2HeaderSizeKey metadata is set + // add the extra size of the LUKS header to the image size. + headerSize, err := ri.getLuks2HeaderSizeSet() + if err != nil { + return err + } + size += headerSize + + err = image.Resize(size) if err != nil { return err } diff --git a/internal/util/cryptsetup.go b/internal/util/cryptsetup.go index 06e2028f3..57dae438c 100644 --- a/internal/util/cryptsetup.go +++ b/internal/util/cryptsetup.go @@ -26,10 +26,17 @@ import ( "github.com/ceph/ceph-csi/internal/util/file" "github.com/ceph/ceph-csi/internal/util/log" + + "k8s.io/cloud-provider/volume/helpers" ) // Limit memory used by Argon2i PBKDF to 32 MiB. -const cryptsetupPBKDFMemoryLimit = 32 << 10 // 32768 KiB +const ( + cryptsetupPBKDFMemoryLimit = 32 << 10 // 32768 KiB + luks2MetadataSize = 32 << 7 // 4096 KiB + luks2KeySlotsSize = 32 << 8 // 8192 KiB + Luks2HeaderSize = uint64((((2 * luks2MetadataSize) + luks2KeySlotsSize) * helpers.KiB)) +) // LuksFormat sets up volume as an encrypted LUKS partition. func LuksFormat(devicePath, passphrase string) (string, string, error) { @@ -41,6 +48,10 @@ func LuksFormat(devicePath, passphrase string) (string, string, error) { "luks2", "--hash", "sha256", + "--luks2-metadata-size", + strconv.Itoa(luks2MetadataSize)+"k", + "--luks2-keyslots-size", + strconv.Itoa(luks2KeySlotsSize)+"k", "--pbkdf-memory", strconv.Itoa(cryptsetupPBKDFMemoryLimit), devicePath,