From 526ff95f10a6640eeded03a26495ca848a029507 Mon Sep 17 00:00:00 2001 From: Prasanna Kumar Kalever Date: Thu, 8 Jul 2021 20:36:42 +0530 Subject: [PATCH] rbd: add support to expand encrypted volume Previously in ControllerExpandVolume() we had a check for encrypted volumes and we use to fail for all expand requests on an encrypted volume. Also for Block VolumeMode PVCs NodeExpandVolume used to be ignored/skipped. With these changes, we add support for the expansion of encrypted volumes. Also for raw Block VolumeMode PVCs with Encryption we call NodeExpandVolume. That said, With LUKS1, cryptsetup utility doesn't prompt for a passphrase on resizing the crypto mapper device. This is because LUKS1 devices don't use kernel keyring for volume keys. Whereas, LUKS2 devices use kernel keyring for volume key by default, i.e. cryptsetup utility asks for a passphrase if it detects volume key was previously passed to dm-crypt via kernel keyring service, we are overriding the default by --disable-keyring option during cryptsetup open command. So that at the time of crypto mapper device resize we will not be prompted for any passphrase. Fixes: #1469 Signed-off-by: Prasanna Kumar Kalever --- internal/rbd/controllerserver.go | 32 +++++++++---------- internal/rbd/nodeserver.go | 53 +++++++++++++++++++------------- internal/util/cryptsetup.go | 4 ++- 3 files changed, 50 insertions(+), 39 deletions(-) diff --git a/internal/rbd/controllerserver.go b/internal/rbd/controllerserver.go index 48d630352..cc3896ec4 100644 --- a/internal/rbd/controllerserver.go +++ b/internal/rbd/controllerserver.go @@ -1355,12 +1355,6 @@ func (cs *ControllerServer) ControllerExpandVolume( return nil, status.Error(codes.InvalidArgument, "capacityRange cannot be empty") } - nodeExpansion := false - // Get the nodeexpansion flag set based on the volume mode - if req.GetVolumeCapability().GetBlock() == nil { - nodeExpansion = true - } - // lock out parallel requests against the same volume ID if acquired := cs.VolumeLocks.TryAcquire(volID); !acquired { util.ErrorLog(ctx, util.VolumeOperationAlreadyExistsFmt, volID) @@ -1369,14 +1363,6 @@ func (cs *ControllerServer) ControllerExpandVolume( } defer cs.VolumeLocks.Release(volID) - // lock out volumeID for clone and delete operation - if err := cs.OperationLocks.GetExpandLock(volID); err != nil { - util.ErrorLog(ctx, err.Error()) - - return nil, status.Error(codes.Aborted, err.Error()) - } - defer cs.OperationLocks.ReleaseExpandLock(volID) - cr, err := util.NewUserCredentials(req.GetSecrets()) if err != nil { return nil, status.Error(codes.Internal, err.Error()) @@ -1399,11 +1385,23 @@ func (cs *ControllerServer) ControllerExpandVolume( return nil, err } - if rbdVol.isEncrypted() { - return nil, status.Errorf(codes.InvalidArgument, "encrypted volumes do not support resize (%s)", - rbdVol) + // NodeExpansion is needed for PersistentVolumes with, + // 1. Filesystem VolumeMode with & without Encryption and + // 2. Block VolumeMode with Encryption + // Hence set nodeExpansion flag based on VolumeMode and Encryption status + nodeExpansion := true + if req.GetVolumeCapability().GetBlock() != nil && !rbdVol.isEncrypted() { + nodeExpansion = false } + // lock out volumeID for clone and delete operation + if err = cs.OperationLocks.GetExpandLock(volID); err != nil { + util.ErrorLog(ctx, err.Error()) + + return nil, status.Error(codes.Aborted, err.Error()) + } + defer cs.OperationLocks.ReleaseExpandLock(volID) + // always round up the request size in bytes to the nearest MiB/GiB volSize := util.RoundOffBytes(req.GetCapacityRange().GetRequiredBytes()) diff --git a/internal/rbd/nodeserver.go b/internal/rbd/nodeserver.go index aee6a6ab2..6f6642028 100644 --- a/internal/rbd/nodeserver.go +++ b/internal/rbd/nodeserver.go @@ -875,38 +875,49 @@ func (ns *NodeServer) NodeExpandVolume( } defer ns.VolumeLocks.Release(volumeID) - devicePath, err := getDevicePath(ctx, volumePath) - if err != nil { - return nil, status.Error(codes.Internal, err.Error()) - } - - // TODO check size and return success or error - volumePath += "/" + volumeID - resizer := mount.NewResizeFs(utilexec.New()) - ok, err := resizer.Resize(devicePath, volumePath) - if !ok { - return nil, status.Errorf(codes.Internal, "rbd: resize failed on path %s, error: %v", req.GetVolumePath(), err) - } - - return &csi.NodeExpandVolumeResponse{}, nil -} - -func getDevicePath(ctx context.Context, volumePath string) (string, error) { imgInfo, err := lookupRBDImageMetadataStash(volumePath) if err != nil { util.ErrorLog(ctx, "failed to find image metadata: %v", err) } - device, found := findDeviceMappingImage( + devicePath, found := findDeviceMappingImage( ctx, imgInfo.Pool, imgInfo.RadosNamespace, imgInfo.ImageName, imgInfo.NbdAccess) - if found { - return device, nil + if !found { + return nil, status.Errorf(codes.Internal, + "failed to get device for stagingtarget path %v", volumePath) } - return "", fmt.Errorf("failed to get device for stagingtarget path %v", volumePath) + mapperFile, mapperPath := util.VolumeMapper(volumeID) + if imgInfo.Encrypted { + // The volume is encrypted, resize an active mapping + err = util.ResizeEncryptedVolume(ctx, mapperFile) + if err != nil { + util.ErrorLog(ctx, "failed to resize device %s, mapper %s: %w", + devicePath, mapperFile, err) + + return nil, status.Errorf(codes.Internal, + "failed to resize device %s, mapper %s: %v", devicePath, mapperFile, err) + } + // Use mapper device path for fs resize + devicePath = mapperPath + } + + if req.GetVolumeCapability().GetBlock() == nil { + // TODO check size and return success or error + volumePath += "/" + volumeID + resizer := mount.NewResizeFs(utilexec.New()) + var ok bool + ok, err = resizer.Resize(devicePath, volumePath) + if !ok { + return nil, status.Errorf(codes.Internal, + "rbd: resize failed on path %s, error: %v", req.GetVolumePath(), err) + } + } + + return &csi.NodeExpandVolumeResponse{}, nil } // NodeGetCapabilities returns the supported capabilities of the node server. diff --git a/internal/util/cryptsetup.go b/internal/util/cryptsetup.go index dbdb45102..ce9e4f018 100644 --- a/internal/util/cryptsetup.go +++ b/internal/util/cryptsetup.go @@ -40,7 +40,9 @@ func LuksFormat(devicePath, passphrase string) (stdout, stderr []byte, err error // LuksOpen opens LUKS encrypted partition and sets up a mapping. func LuksOpen(devicePath, mapperFile, passphrase string) (stdout, stderr []byte, err error) { - return execCryptsetupCommand(&passphrase, "luksOpen", devicePath, mapperFile, "-d", "/dev/stdin") + // cryptsetup option --disable-keyring (introduced with cryptsetup v2.0.0) + // will be ignored with luks1 + return execCryptsetupCommand(&passphrase, "luksOpen", devicePath, mapperFile, "--disable-keyring", "-d", "/dev/stdin") } // LuksResize resizes LUKS encrypted partition.