From f32e2d5dda3a88bc933ab7c44db232707751c227 Mon Sep 17 00:00:00 2001 From: Praveen M Date: Thu, 28 Mar 2024 17:10:11 +0530 Subject: [PATCH] rbd: add additional space for encrypted volumes issue: when a block-mode pvc is created with encryption enabled there is some space reserved for the encryption metadata. Which doesn't allows users to write extact amount of data that they have requested for. solution: create pvc with extra space needed for the encryption metadata. The extra space is added during the CreateVolume and ExpandVolume operations. And while returning the response remove the extra space so the client/user gets the requested size reported. Signed-off-by: Praveen M --- internal/rbd/controllerserver.go | 22 +++++++++++ internal/rbd/encryption.go | 8 ++++ internal/rbd/rbd_util.go | 53 ++++++++++++++++++++++++-- internal/util/cryptsetup/cryptsetup.go | 13 ++++++- 4 files changed, 91 insertions(+), 5 deletions(-) diff --git a/internal/rbd/controllerserver.go b/internal/rbd/controllerserver.go index 8283970f5..1802693f4 100644 --- a/internal/rbd/controllerserver.go +++ b/internal/rbd/controllerserver.go @@ -1226,6 +1226,17 @@ func (cs *ControllerServer) CreateSnapshot( return nil, status.Error(codes.Internal, err.Error()) } + err = vol.Connect(cr) + if err != nil { + return nil, status.Error(codes.Internal, err.Error()) + } + defer vol.Destroy(ctx) + + err = vol.getImageInfo() + if err != nil { + return nil, status.Error(codes.Internal, err.Error()) + } + csiSnap, err := vol.toSnapshot().ToCSI(ctx) if err != nil { return nil, status.Error(codes.Internal, err.Error()) @@ -1285,6 +1296,17 @@ func cloneFromSnapshot( } } + err = rbdSnap.Connect(cr) + if err != nil { + return nil, status.Error(codes.Internal, err.Error()) + } + defer rbdSnap.Destroy(ctx) + + err = rbdSnap.getImageInfo() + if err != nil { + return nil, status.Error(codes.Internal, err.Error()) + } + csiSnap, err := rbdSnap.ToCSI(ctx) if err != nil { return nil, status.Error(codes.Internal, err.Error()) diff --git a/internal/rbd/encryption.go b/internal/rbd/encryption.go index eadef94fb..f8e78d73f 100644 --- a/internal/rbd/encryption.go +++ b/internal/rbd/encryption.go @@ -60,6 +60,9 @@ const ( metadataDEK = "rbd.csi.ceph.com/dek" oldMetadataDEK = ".rbd.csi.ceph.com/dek" + // luks2 header size metadata key. + luks2HeaderSizeKey = "rbd.csi.ceph.com/luks2HeaderSize" + encryptionPassphraseSize = 20 // rbdDefaultEncryptionType is the default to use when the @@ -131,6 +134,11 @@ func (ri *rbdImage) setupBlockEncryption(ctx context.Context) error { return err } + err = ri.SetMetadata(luks2HeaderSizeKey, strconv.FormatUint(cryptsetup.Luks2HeaderSize, 10)) + if err != nil { + return fmt.Errorf("failed to save %s metadata on image: %w", luks2HeaderSizeKey, err) + } + err = ri.ensureEncryptionMetadataSet(rbdImageEncryptionPrepared) if err != nil { log.ErrorLog(ctx, "failed to save encryption status, deleting "+ diff --git a/internal/rbd/rbd_util.go b/internal/rbd/rbd_util.go index 274981bfe..c7d31c3ed 100644 --- a/internal/rbd/rbd_util.go +++ b/internal/rbd/rbd_util.go @@ -30,6 +30,7 @@ import ( "github.com/ceph/ceph-csi/internal/rbd/types" "github.com/ceph/ceph-csi/internal/util" + "github.com/ceph/ceph-csi/internal/util/cryptsetup" "github.com/ceph/ceph-csi/internal/util/log" "github.com/ceph/go-ceph/rados" @@ -453,8 +454,16 @@ func createImage(ctx context.Context, pOpts *rbdVolume, cr *util.Credentials) er return fmt.Errorf("failed to get IOContext: %w", err) } - err = librbd.CreateImage(pOpts.ioctx, pOpts.RbdImageName, - uint64(util.RoundOffVolSize(pOpts.VolSize)*helpers.MiB), options) + size := uint64(util.RoundOffVolSize(pOpts.VolSize) * helpers.MiB) + if pOpts.isBlockEncrypted() { + // When a block-mode PVC is created with encryption enabled, + // some space is reserved for the LUKS2 header. + // Add the LUKS2 header size to the image size so that the user has at least + // the requested size. + size += cryptsetup.Luks2HeaderSize + } + + err = librbd.CreateImage(pOpts.ioctx, pOpts.RbdImageName, size, options) if err != nil { return fmt.Errorf("failed to create rbd image: %w", err) } @@ -1643,6 +1652,26 @@ func (ri *rbdImage) GetCreationTime(ctx context.Context) (*time.Time, error) { return ri.CreatedAt, nil } +// getLuks2HeaderSizeSet returns the value of the LUKS2 header size +// set in the image metadata (size returned in MiB). +func (ri *rbdImage) getLuks2HeaderSizeSet() (uint64, error) { + value, err := ri.GetMetadata(luks2HeaderSizeKey) + if err != nil { + if !errors.Is(err, librbd.ErrNotFound) { + return 0, err + } + + return 0, nil + } + + headerSize, parseErr := strconv.ParseUint(value, 10, 64) + if parseErr != nil { + return 0, parseErr + } + + return headerSize, nil +} + // getImageInfo queries rbd about the given image and returns its metadata, and returns // ErrImageNotFound if provided image is not found. func (ri *rbdImage) getImageInfo() error { @@ -1659,6 +1688,14 @@ func (ri *rbdImage) getImageInfo() error { // TODO: can rv.VolSize not be a uint64? Or initialize it to -1? ri.VolSize = int64(imageInfo.Size) + // If the luks2HeaderSizeKey metadata is set + // reduce the extra size of the LUKS header from the image size. + headerSize, err := ri.getLuks2HeaderSizeSet() + if err != nil { + return err + } + ri.VolSize -= int64(headerSize) + features, err := image.GetFeatures() if err != nil { return err @@ -1908,7 +1945,17 @@ func (ri *rbdImage) resize(newSize int64) error { } defer image.Close() - err = image.Resize(uint64(util.RoundOffVolSize(newSize) * helpers.MiB)) + size := uint64(util.RoundOffVolSize(newSize) * helpers.MiB) + + // If the luks2HeaderSizeKey metadata is set + // add the extra size of the LUKS header to the image size. + headerSize, err := ri.getLuks2HeaderSizeSet() + if err != nil { + return err + } + size += headerSize + + err = image.Resize(size) if err != nil { return err } diff --git a/internal/util/cryptsetup/cryptsetup.go b/internal/util/cryptsetup/cryptsetup.go index 7423fb417..9f58f1b55 100644 --- a/internal/util/cryptsetup/cryptsetup.go +++ b/internal/util/cryptsetup/cryptsetup.go @@ -30,6 +30,8 @@ import ( "github.com/ceph/ceph-csi/internal/util/file" "github.com/ceph/ceph-csi/internal/util/log" "github.com/ceph/ceph-csi/internal/util/stripsecrets" + + "k8s.io/cloud-provider/volume/helpers" ) const ( @@ -37,7 +39,10 @@ const ( ExecutionTimeout = 2*time.Minute + 30*time.Second // Limit memory used by Argon2i PBKDF to 32 MiB. - pkdbfMemoryLimit = 32 << 10 // 32768 KiB + cryptsetupPBKDFMemoryLimit = 32 << 10 // 32768 KiB + luks2MetadataSize = 32 << 7 // 4096 KiB + luks2KeySlotsSize = 32 << 8 // 8192 KiB + Luks2HeaderSize = uint64((((2 * luks2MetadataSize) + luks2KeySlotsSize) * helpers.KiB)) ) // LuksWrapper is a struct that provides a context-aware wrapper around cryptsetup commands. @@ -74,8 +79,12 @@ func (l *luksWrapper) Format(devicePath, passphrase string) (string, string, err "luks2", "--hash", "sha256", + "--luks2-metadata-size", + strconv.Itoa(luks2MetadataSize)+"k", + "--luks2-keyslots-size", + strconv.Itoa(luks2KeySlotsSize)+"k", "--pbkdf-memory", - strconv.Itoa(pkdbfMemoryLimit), + strconv.Itoa(cryptsetupPBKDFMemoryLimit), devicePath, "-d", "/dev/stdin")