Add multiNodeWritable option for RBD Volumes

This change adds the ability to define a `multiNodeWritable` option in
the Storage Class.

This change does a number of things:
1. Allow multi-node-multi-writer access modes if the SC options is
enabled
2. Bypass the watcher checks for MultiNodeMultiWriter Volumes
3. Maintains existing watcher checks for SingleNodeWriter access modes
regardless of the StorageClass option.

fix lint-errors

pkg/rbd/controllerserver.go

@@ -21,6 +21,7 @@ import (
 	"os/exec"
 	"sort"
 	"strconv"
+	"strings"
 	"syscall"
 
 	"github.com/ceph/ceph-csi/pkg/csi-common"
@@ -92,7 +93,16 @@ func (cs *ControllerServer) validateVolumeReq(req *csi.CreateVolumeRequest) erro
 func parseVolCreateRequest(req *csi.CreateVolumeRequest) (*rbdVolume, error) {
 	// TODO (sbezverk) Last check for not exceeding total storage capacity
 
-	rbdVol, err := getRBDVolumeOptions(req.GetParameters())
+	// MultiNodeWriters are accepted but they're only for special cases, and we skip the watcher checks for them which isn't the greatest
+	// let's make sure we ONLY skip that if the user is requesting a MULTI Node accessible mode
+	disableMultiWriter := true
+	for _, am := range req.VolumeCapabilities {
+		if am.GetAccessMode().GetMode() != csi.VolumeCapability_AccessMode_SINGLE_NODE_WRITER {
+			disableMultiWriter = false
+		}
+	}
+
+	rbdVol, err := getRBDVolumeOptions(req.GetParameters(), disableMultiWriter)
 	if err != nil {
 		return nil, err
 	}
@@ -330,11 +340,20 @@ func (cs *ControllerServer) ListVolumes(ctx context.Context, req *csi.ListVolume
 // ValidateVolumeCapabilities checks whether the volume capabilities requested
 // are supported.
 func (cs *ControllerServer) ValidateVolumeCapabilities(ctx context.Context, req *csi.ValidateVolumeCapabilitiesRequest) (*csi.ValidateVolumeCapabilitiesResponse, error) {
-	for _, cap := range req.VolumeCapabilities {
-		if cap.GetAccessMode().GetMode() != csi.VolumeCapability_AccessMode_SINGLE_NODE_WRITER {
-			return &csi.ValidateVolumeCapabilitiesResponse{Message: ""}, nil
+	params := req.GetParameters()
+	multiWriter := params["multiNodeWritable"]
+	if strings.ToLower(multiWriter) == "enabled" {
+		klog.V(3).Info("detected multiNodeWritable parameter in Storage Class, allowing multi-node access modes")
+
+	} else {
+		for _, cap := range req.VolumeCapabilities {
+			if cap.GetAccessMode().GetMode() != csi.VolumeCapability_AccessMode_SINGLE_NODE_WRITER {
+				return &csi.ValidateVolumeCapabilitiesResponse{Message: ""}, nil
+			}
 		}
+
 	}
+
 	return &csi.ValidateVolumeCapabilitiesResponse{
 		Confirmed: &csi.ValidateVolumeCapabilitiesResponse_Confirmed{
 			VolumeCapabilities: req.VolumeCapabilities,

pkg/rbd/nodeserver.go

@@ -70,10 +70,18 @@ func (ns *NodeServer) NodePublishVolume(ctx context.Context, req *csi.NodePublis
 	if !notMnt {
 		return &csi.NodePublishVolumeResponse{}, nil
 	}
-	volOptions, err := getRBDVolumeOptions(req.GetVolumeContext())
+
+	ignoreMultiWriterEnabled := true
+	if req.VolumeCapability.AccessMode.Mode != csi.VolumeCapability_AccessMode_SINGLE_NODE_WRITER {
+		ignoreMultiWriterEnabled = false
+	}
+
+	volOptions, err := getRBDVolumeOptions(req.GetVolumeContext(), ignoreMultiWriterEnabled)
 	if err != nil {
 		return nil, err
 	}
+	// Check access mode settings in the request, even if SC is RW-Many, if the request is a normal Single Writer volume, we ignore this setting and proceed as normal
+
 	volOptions.VolName = volName
 	// Mapping RBD image
 	devicePath, err := attachRBDImage(volOptions, volOptions.UserID, req.GetSecrets())

pkg/rbd/rbd.go

@@ -102,7 +102,12 @@ func (r *Driver) Run(driverName, nodeID, endpoint string, containerized bool, ca
 		csi.ControllerServiceCapability_RPC_LIST_SNAPSHOTS,
 		csi.ControllerServiceCapability_RPC_CLONE_VOLUME,
 	})
-	r.cd.AddVolumeCapabilityAccessModes([]csi.VolumeCapability_AccessMode_Mode{csi.VolumeCapability_AccessMode_SINGLE_NODE_WRITER})
+
+	// TODO: JDG Should also look at remaining modes like MULT_NODE_READER (SINGLE_READER)
+	r.cd.AddVolumeCapabilityAccessModes(
+		[]csi.VolumeCapability_AccessMode_Mode{
+			csi.VolumeCapability_AccessMode_SINGLE_NODE_WRITER,
+			csi.VolumeCapability_AccessMode_MULTI_NODE_MULTI_WRITER})
 
 	// Create GRPC servers
 	r.ids = NewIdentityServer(r.cd)

pkg/rbd/rbd_attach.go

@@ -313,8 +313,16 @@ func waitForrbdImage(backoff wait.Backoff, volOptions *rbdVolume, userID string,
 		if err != nil {
 			return false, fmt.Errorf("fail to check rbd image status with: (%v), rbd output: (%s)", err, rbdOutput)
 		}
+		// In the case of multiattach we want to short circuit the retries when used (so r`if used; return used`)
+		// otherwise we're setting this to false which translates to !ok, which means backoff and try again
+		// NOTE: we ONLY do this if an multi-node access mode is requested for this volume
+		if (strings.ToLower(volOptions.MultiNodeWritable) == "enabled") && (used) {
+			klog.V(2).Info("detected MultiNodeWritable enabled, ignoring watcher in-use result")
+			return used, nil
+		}
 		return !used, nil
 	})
+
 	// return error if rbd image has not become available for the specified timeout
 	if err == wait.ErrWaitTimeout {
 		return fmt.Errorf("rbd image %s is still being used", imagePath)

pkg/rbd/rbd_util.go

@@ -51,6 +51,7 @@ type rbdVolume struct {
 	AdminID            string `json:"adminId"`
 	UserID             string `json:"userId"`
 	Mounter            string `json:"mounter"`
+	MultiNodeWritable  string `json:"multiNodeWritable"`
 }
 
 type rbdSnapshot struct {
@@ -226,7 +227,7 @@ func execCommand(command string, args []string) ([]byte, error) {
 	return cmd.CombinedOutput()
 }
 
-func getRBDVolumeOptions(volOptions map[string]string) (*rbdVolume, error) {
+func getRBDVolumeOptions(volOptions map[string]string, ignoreMultiNodeWritable bool) (*rbdVolume, error) {
 	var ok bool
 	rbdVol := &rbdVolume{}
 	rbdVol.Pool, ok = volOptions["pool"]
@@ -260,6 +261,12 @@ func getRBDVolumeOptions(volOptions map[string]string) (*rbdVolume, error) {
 
 	}
 	getCredsFromVol(rbdVol, volOptions)
+
+	klog.V(3).Infof("ignoreMultiNodeWritable flag in parse getRBDVolumeOptions is: %v", ignoreMultiNodeWritable)
+	// If the volume we're working with is NOT requesting multi-node attach then don't treat it special, ignore the setting in the SC and just keep our watcher checks
+	if !ignoreMultiNodeWritable {
+		rbdVol.MultiNodeWritable = volOptions["multiNodeWritable"]
+	}
 	return rbdVol, nil
 }