mirrors/ceph-csi
1
0
Fork 0
mirror of https://github.com/ceph/ceph-csi.git synced 2024-11-17 20:00:23 +00:00
Code Issues Packages Projects Releases Wiki Activity

rbd: utilize cookie support from rbd for nbd

Browse Source 
Problem:
On remap/attach of device (i.e. nodeplugin restart), there is no way
for rbd-nbd to defend if the backend storage is matching with the initial
backend storage.

Say, if an initial map request for backend "pool1/image1" got mapped to
/dev/nbd0 and the userspace process is terminated (on nodeplugin restart).
A next remap/attach (nodeplugin start) request within reattach-timeout is
allowed to use /dev/nbd0 for a different backend "pool1/image2"

For example, an operation like below could be dangerous:

$ sudo rbd-nbd map --try-netlink rbd-pool/ext4-image
/dev/nbd0
$ sudo blkid /dev/nbd0
/dev/nbd0: UUID="bfc444b4-64b1-418f-8b36-6e0d170cfc04" TYPE="ext4"
$ sudo pkill -15 rbd-nbd   <-- nodeplugin terminate
$ sudo rbd-nbd attach --try-netlink --device /dev/nbd0 rbd-pool/xfs-image
/dev/nbd0
$ sudo blkid /dev/nbd0
/dev/nbd0: UUID="d29bf343-6570-4069-a9ea-2fa156ced908" TYPE="xfs"

Solution:
rbd-nbd/kernel now provides a way to keep some metadata in sysfs to identify
between the device and the backend, so that when a remap/attach request is
made, rbd-nbd can compare and avoid such dangerous operations.

With the provided solution, as part of the initial map request, backend
cookie (ceph-csi VOLID) can be stored in the sysfs per device config, so
that on a remap/attach request rbd-nbd will check and validate if the
backend per device cookie matches with the initial map backend with the help
of cookie.

At Ceph-csi we use VOLID as device cookie, which will be unique, we pass
the VOLID as cookie at map and use the same at the time of attach, that
way rbd-nbd can identify backends and their matching devices.

Requires:
https://github.com/ceph/ceph/pull/41323
https://lkml.org/lkml/2021/4/29/274

Signed-off-by: Prasanna Kumar Kalever <prasanna.kalever@redhat.com>
This commit is contained in:
Prasanna Kumar Kalever 2021-09-21 18:43:10 +05:30 committed by mergify[bot]
parent 793b22cf27
commit 3686b6da8b
1 changed files with 12 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

17
internal/rbd/rbd_attach.go
View File

@ -250,7 +250,7 @@ func attachRBDImage(ctx context.Context, volOptions *rbdVolume, device string, c
return devicePath, err
}
func appendNbdDeviceTypeAndOptions(cmdArgs []string, isThick bool, userOptions string) []string {
func appendNbdDeviceTypeAndOptions(cmdArgs []string, isThick bool, userOptions, cookie string) []string {
cmdArgs = append(cmdArgs, "--device-type", accessTypeNbd)
isUnmap := CheckSliceContains(cmdArgs, "unmap")
@ -265,6 +265,10 @@ func appendNbdDeviceTypeAndOptions(cmdArgs []string, isThick bool, userOptions s
cmdArgs = append(cmdArgs, "--options", fmt.Sprintf("%s=%d", setNbdIOTimeout, defaultNbdIOTimeout))
}
if hasNBDCookieSupport {
cmdArgs = append(cmdArgs, "--options", fmt.Sprintf("cookie=%s", cookie))
}
if isThick {
// When an image is thick-provisioned, any discard/unmap/trim
// requests should not free extents.
@ -309,7 +313,7 @@ func appendKRbdDeviceTypeAndOptions(cmdArgs []string, isThick bool, userOptions
// appendRbdNbdCliOptions append mandatory options and convert list of useroptions
// provided for rbd integrated cli to rbd-nbd cli format specific.
func appendRbdNbdCliOptions(cmdArgs []string, userOptions string) []string {
func appendRbdNbdCliOptions(cmdArgs []string, userOptions, cookie string) []string {
if !strings.Contains(userOptions, useNbdNetlink) {
cmdArgs = append(cmdArgs, fmt.Sprintf("--%s", useNbdNetlink))
}
@ -319,6 +323,9 @@ func appendRbdNbdCliOptions(cmdArgs []string, userOptions string) []string {
if !strings.Contains(userOptions, setNbdIOTimeout) {
cmdArgs = append(cmdArgs, fmt.Sprintf("--%s=%d", setNbdIOTimeout, defaultNbdIOTimeout))
}
if hasNBDCookieSupport {
cmdArgs = append(cmdArgs, fmt.Sprintf("--cookie=%s", cookie))
}
if userOptions != "" {
options := strings.Split(userOptions, ",")
for _, opt := range options {
@ -362,11 +369,11 @@ func createPath(ctx context.Context, volOpt *rbdVolume, device string, cr *util.
// TODO: use rbd cli for attach/detach in the future
cli = rbdNbdMounter
mapArgs = append(mapArgs, "attach", imagePath, "--device", device)
mapArgs = appendRbdNbdCliOptions(mapArgs, volOpt.MapOptions)
mapArgs = appendRbdNbdCliOptions(mapArgs, volOpt.MapOptions, volOpt.VolID)
} else {
mapArgs = append(mapArgs, "map", imagePath)
if isNbd {
mapArgs = appendNbdDeviceTypeAndOptions(mapArgs, isThick, volOpt.MapOptions)
mapArgs = appendNbdDeviceTypeAndOptions(mapArgs, isThick, volOpt.MapOptions, volOpt.VolID)
} else {
mapArgs = appendKRbdDeviceTypeAndOptions(mapArgs, isThick, volOpt.MapOptions)
}
@ -476,7 +483,7 @@ func detachRBDImageOrDeviceSpec(
unmapArgs := []string{"unmap", dArgs.imageOrDeviceSpec}
if dArgs.isNbd {
unmapArgs = appendNbdDeviceTypeAndOptions(unmapArgs, false, dArgs.unmapOptions)
unmapArgs = appendNbdDeviceTypeAndOptions(unmapArgs, false, dArgs.unmapOptions, dArgs.volumeID)
} else {
unmapArgs = appendKRbdDeviceTypeAndOptions(unmapArgs, false, dArgs.unmapOptions)
}