Add support for erasure pool in rbd

Allow specifying different metadata and data pools in a
CSI RBD StorageClass

Fixes: #199
Fixes: https://github.com/rook/rook/issues/2650
Fixes: https://github.com/rook/rook/issues/3763

Signed-off-by: Madhu Rajanna <madhupr007@gmail.com>

docs/deploy-rbd.md

@@ -50,6 +50,7 @@ make image-cephcsi
 | --------------------------------------------------------------------------------------------------- | -------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
 | `clusterID`                                                                                         | yes                  | String representing a Ceph cluster, must be unique across all Ceph clusters in use for provisioning, cannot be greater than 36 bytes in length, and should remain immutable for the lifetime of the Ceph cluster in use |
 | `pool`                                                                                              | yes                  | Ceph pool into which the RBD image shall be created                                                                                                                                                                     |
+| `dataPool`                                                                                          | no                   | Ceph pool used for the data of the RBD images.                                                                                                                                                                          |
 | `imageFormat`                                                                                       | no                   | RBD image format. Defaults to `2`. See [man pages](http://docs.ceph.com/docs/mimic/man/8/rbd/#cmdoption-rbd-image-format)                                                                                               |
 | `imageFeatures`                                                                                     | no                   | RBD image features. Available for `imageFormat=2`. CSI RBD currently supports only `layering` feature. See [man pages](http://docs.ceph.com/docs/mimic/man/8/rbd/#cmdoption-rbd-image-feature)                          |
 | `csi.storage.k8s.io/provisioner-secret-name`, `csi.storage.k8s.io/node-stage-secret-name`           | yes (for Kubernetes) | name of the Kubernetes Secret object containing Ceph client credentials. Both parameters should have the same value                                                                                                     |

examples/rbd/storageclass.yaml

@@ -13,8 +13,12 @@ parameters:
    # csi-config-map-sample.yaml, to accompany the string chosen to
    # represent the Ceph cluster in clusterID below
    clusterID: <cluster-id>
-
+   # If you want to use erasure coded pool with RBD, you need to create
-   # Ceph pool into which the RBD image shall be created
+   # two pools. one erasure coded and one replicated.
+   # You need to specify the replicated pool here in the `pool` parameter, it is
+   # used for the metadata of the images.
+   # The erasure coded pool must be set as the `dataPool` parameter below.
+   # dataPool: ec-data-pool
    pool: rbd
 
    # RBD image format. Defaults to "2".

pkg/rbd/controllerserver.go

@@ -60,6 +60,10 @@ func (cs *ControllerServer) validateVolumeReq(ctx context.Context, req *csi.Crea
 	if value, ok := options["pool"]; !ok || value == "" {
 		return status.Error(codes.InvalidArgument, "missing or empty pool name to provision volume from")
 	}
+
+	if value, ok := options["dataPool"]; ok && value == "" {
+		return status.Error(codes.InvalidArgument, "empty datapool name to provision volume from")
+	}
 	return nil
 }
 

pkg/rbd/rbd_util.go

@@ -67,6 +67,7 @@ type rbdVolume struct {
 	VolID              string `json:"volID"`
 	Monitors           string `json:"monitors"`
 	Pool               string `json:"pool"`
+	DataPool           string
 	ImageFormat        string `json:"imageFormat"`
 	ImageFeatures      string `json:"imageFeatures"`
 	VolSize            int64  `json:"volSize"`
@@ -122,15 +123,27 @@ func createImage(ctx context.Context, pOpts *rbdVolume, volSz int64, cr *util.Cr
 	volSzMiB := fmt.Sprintf("%dM", volSz)
 
 	if pOpts.ImageFormat == rbdImageFormat2 {
-		klog.V(4).Infof(util.Log(ctx, "rbd: create %s size %s format %s (features: %s) using mon %s, pool %s"),
+		logMsg := "rbd: create %s size %s format %s (features: %s) using mon %s, pool %s "
+		if pOpts.DataPool != "" {
+			logMsg += fmt.Sprintf("data pool %s", pOpts.DataPool)
+		}
+		klog.V(4).Infof(util.Log(ctx, logMsg),
 			image, volSzMiB, pOpts.ImageFormat, pOpts.ImageFeatures, pOpts.Monitors, pOpts.Pool)
 	} else {
-		klog.V(4).Infof(util.Log(ctx, "rbd: create %s size %s format %s using mon %s, pool %s"), image, volSzMiB, pOpts.ImageFormat, pOpts.Monitors, pOpts.Pool)
+		logMsg := "rbd: create %s size %s format %s using mon %s, pool %s "
+		if pOpts.DataPool != "" {
+			logMsg += fmt.Sprintf("data pool %s", pOpts.DataPool)
 		}
+		klog.V(4).Infof(util.Log(ctx, logMsg), image, volSzMiB, pOpts.ImageFormat, pOpts.Monitors, pOpts.Pool)
+	}
+
 	args := []string{"create", image, "--size", volSzMiB, "--pool", pOpts.Pool, "--id", cr.ID, "-m", pOpts.Monitors, "--keyfile=" + cr.KeyFile, "--image-format", pOpts.ImageFormat}
 	if pOpts.ImageFormat == rbdImageFormat2 {
 		args = append(args, "--image-feature", pOpts.ImageFeatures)
 	}
+	if pOpts.DataPool != "" {
+		args = append(args, "--data-pool", pOpts.DataPool)
+	}
 	output, err := execCommand("rbd", args)
 
 	if err != nil {
@@ -452,6 +465,8 @@ func genVolFromVolumeOptions(ctx context.Context, volOptions, credentials map[st
 		return nil, errors.New("missing required parameter pool")
 	}
 
+	rbdVol.DataPool = volOptions["dataPool"]
+
 	if isLegacyVolume {
 		err = updateMons(rbdVol, volOptions, credentials)
 		if err != nil {