rbd: add maxsnapshotsonimage flag

Added maxsnapshotsonimage flag to flatten
the older rbd images on the chain to avoid
issue in krbd.The limit is in krbd since it
only allocate 1 4KiB page to handle all the
snapshot ids for an image.

The max limit is 510 as per
https://github.com/torvalds/linux/blob/
aaa2faab4ed8e5fe0111e04d6e168c028fe2987f/drivers/block/rbd.c#L98
in cephcsi we arekeeping the default to 450 to reserve 10%
to avoid issues.
Signed-off-by: Madhu Rajanna <madhupr007@gmail.com>

charts/ceph-csi-rbd/templates/provisioner-deployment.yaml

@@ -116,6 +116,7 @@ spec:
             - "--drivername=$(DRIVER_NAME)"
             - "--rbdhardmaxclonedepth={{ .Values.provisioner.hardMaxCloneDepth }}"
             - "--rbdsoftmaxclonedepth={{ .Values.provisioner.softMaxCloneDepth }}"
+            - "--maxsnapshotsonimage={{ .Values.provisioner.maxSnapshotsOnImage }}"
             {{- if .Values.provisioner.skipForceFlatten }}
             - "--skipforceflatten={{ .Values.provisioner.skipForceFlatten }}"
             {{- end }}

charts/ceph-csi-rbd/values.yaml

@@ -113,6 +113,8 @@ provisioner:
   # Soft limit for maximum number of nested volume clones that are taken before
   # a flatten occurs
   softMaxCloneDepth: 4
+  # Maximum number of snapshots allowed on rbd image without flattening
+  maxSnapshotsOnImage: 450
   # skip image flattening if kernel support mapping of rbd images
   # which has the deep-flatten feature
   # skipForceFlatten: false

cmd/cephcsi.go

@@ -79,7 +79,7 @@ func init() {
 
 	flag.UintVar(&conf.RbdHardMaxCloneDepth, "rbdhardmaxclonedepth", 8, "Hard limit for maximum number of nested volume clones that are taken before a flatten occurs")
 	flag.UintVar(&conf.RbdSoftMaxCloneDepth, "rbdsoftmaxclonedepth", 4, "Soft limit for maximum number of nested volume clones that are taken before a flatten occurs")
-
+	flag.UintVar(&conf.MaxSnapshotsOnImage, "maxsnapshotsonimage", 450, "Maximum number of snapshots allowed on rbd image without flattening")
 	flag.BoolVar(&conf.SkipForceFlatten, "skipforceflatten", false,
 		"skip image flattening if kernel support mapping of rbd images which has the deep-flatten feature")
 
@@ -182,6 +182,7 @@ func main() {
 	switch conf.Vtype {
 	case rbdType:
 		validateCloneDepthFlag(&conf)
+		validateMaxSnaphostFlag(&conf)
 		driver := rbd.NewDriver()
 		driver.Run(&conf, cp)
 
@@ -212,3 +213,13 @@ func validateCloneDepthFlag(conf *util.Config) {
 		klog.Fatalln("rbdsoftmaxclonedepth flag value should not be greater than rbdhardmaxclonedepth")
 	}
 }
+
+func validateMaxSnaphostFlag(conf *util.Config) {
+	// maximum number of snapshots on an image are 510 [1] and 16 images in
+	// a parent/child chain [2],keeping snapshot limit to 500 to avoid issues.
+	// [1] https://github.com/torvalds/linux/blob/master/drivers/block/rbd.c#L98
+	// [2] https://github.com/torvalds/linux/blob/master/drivers/block/rbd.c#L92
+	if conf.MaxSnapshotsOnImage == 0 || conf.MaxSnapshotsOnImage > 500 {
+		klog.Fatalln("maxsnapshotsonimage flag value should be between 1 and 500")
+	}
+}

docs/deploy-rbd.md

@@ -45,6 +45,7 @@ make image-cephcsi
 | `--rbdhardmaxclonedepth` | `8`                   | Hard limit for maximum number of nested volume clones that are taken before a flatten occurs                                                                                                                                                                                         |
 | `--rbdsoftmaxclonedepth` | `4`                   | Soft limit for maximum number of nested volume clones that are taken before a flatten occurs                                                                                                                                                                                         |
 | `--skipforceflatten`     | `false`               | skip image flattening on kernel < 5.2 which support mapping of rbd images which has the deep-flatten feature                                                                                                                                                                         |
+| `--maxsnapshotsonimage`  | `450`                 | Maximum number of snapshots allowed on rbd image without flattening                                                                                                                                                                                                                  |
 
 **Available volume parameters:**
 

internal/rbd/driver.go

@@ -58,8 +58,8 @@ var (
 
 	// rbdSoftMaxCloneDepth is the soft limit for maximum number of nested volume clones that are taken before a flatten occurs
 	rbdSoftMaxCloneDepth uint
-
+	maxSnapshotsOnImage  uint
-	skipForceFlatten bool
+	skipForceFlatten     bool
 )
 
 // NewDriver returns new rbd driver
@@ -114,6 +114,7 @@ func (r *Driver) Run(conf *util.Config, cachePersister util.CachePersister) {
 	rbdHardMaxCloneDepth = conf.RbdHardMaxCloneDepth
 	rbdSoftMaxCloneDepth = conf.RbdSoftMaxCloneDepth
 	skipForceFlatten = conf.SkipForceFlatten
+	maxSnapshotsOnImage = conf.MaxSnapshotsOnImage
 	// Create instances of the volume and snapshot journal
 	volJournal = journal.NewCSIVolumeJournal(CSIInstanceID)
 	snapJournal = journal.NewCSISnapshotJournal(CSIInstanceID)

internal/util/util.go

@@ -108,6 +108,11 @@ type Config struct {
 
 	// RbdSoftMaxCloneDepth is the soft limit for maximum number of nested volume clones that are taken before a flatten occurs
 	RbdSoftMaxCloneDepth uint
+
+	// MaxSnapshotsOnImage represents the maximum number of snapshots allowed
+	// on rbd image without flattening, once the limit is reached cephcsi will
+	// start flattening the older rbd images to allow more snapshots
+	MaxSnapshotsOnImage uint
 }
 
 // CreatePersistanceStorage creates storage path and initializes new cache