rbd: add minsnapshotsonimage flag

An rbd image can have a maximum number of
snapshots defined by maxsnapshotsonimage
On the limit is reached the cephcsi will
start flattening the older snapshots and
returns the ABORT error message, The Request
comes after this as to wait till all the
images are flattened (this will increase the
PVC creation time.  Instead of waiting till
the maximum snapshots on an RBD image, we can
have a soft limit, once the limit reached
cephcsi will start flattening the task to
break the chain. With this PVC  creation time
will only be affected when the hard limit
(minsnapshotsonimage) reached.

Signed-off-by: Madhu Rajanna <madhupr007@gmail.com>

charts/ceph-csi-rbd/templates/provisioner-deployment.yaml

@@ -133,6 +133,7 @@ spec:
             - "--rbdhardmaxclonedepth={{ .Values.provisioner.hardMaxCloneDepth }}"
             - "--rbdsoftmaxclonedepth={{ .Values.provisioner.softMaxCloneDepth }}"
             - "--maxsnapshotsonimage={{ .Values.provisioner.maxSnapshotsOnImage }}"
+            - "--minsnapshotsonimage={{ .Values.provisioner.minSnapshotsOnImage }}"
             {{- if .Values.provisioner.skipForceFlatten }}
             - "--skipforceflatten={{ .Values.provisioner.skipForceFlatten }}"
             {{- end }}

charts/ceph-csi-rbd/values.yaml

@@ -120,6 +120,8 @@ provisioner:
   softMaxCloneDepth: 4
   # Maximum number of snapshots allowed on rbd image without flattening
   maxSnapshotsOnImage: 450
+  # Minimum number of snapshots allowed on rbd image to trigger flattening
+  minSnapshotsOnImage: 250
   # skip image flattening if kernel support mapping of rbd images
   # which has the deep-flatten feature
   # skipForceFlatten: false

cmd/cephcsi.go

@@ -78,6 +78,7 @@ func init() {
 	flag.UintVar(&conf.RbdHardMaxCloneDepth, "rbdhardmaxclonedepth", 8, "Hard limit for maximum number of nested volume clones that are taken before a flatten occurs")
 	flag.UintVar(&conf.RbdSoftMaxCloneDepth, "rbdsoftmaxclonedepth", 4, "Soft limit for maximum number of nested volume clones that are taken before a flatten occurs")
 	flag.UintVar(&conf.MaxSnapshotsOnImage, "maxsnapshotsonimage", 450, "Maximum number of snapshots allowed on rbd image without flattening")
+	flag.UintVar(&conf.MinSnapshotsOnImage, "minsnapshotsonimage", 250, "Minimum number of snapshots required on rbd image to start flattening")
 	flag.BoolVar(&conf.SkipForceFlatten, "skipforceflatten", false,
 		"skip image flattening if kernel support mapping of rbd images which has the deep-flatten feature")
 
@@ -204,6 +205,10 @@ func validateMaxSnaphostFlag(conf *util.Config) {
 	if conf.MaxSnapshotsOnImage == 0 || conf.MaxSnapshotsOnImage > 500 {
 		logAndExit("maxsnapshotsonimage flag value should be between 1 and 500")
 	}
+
+	if conf.MinSnapshotsOnImage > conf.MaxSnapshotsOnImage {
+		logAndExit("minsnapshotsonimage flag value should be less than maxsnapshotsonimage")
+	}
 }
 
 func logAndExit(msg string) {

internal/rbd/controllerserver.go

@@ -342,10 +342,12 @@ func flattenParentImage(ctx context.Context, rbdVol *rbdVolume, cr *util.Credent
 	return nil
 }
 
-// check snapshots on the rbd image, as we have limit from krbd that
+// check snapshots on the rbd image, as we have limit from krbd that an image
-// an image cannot have more than 510 snapshot at a given point of time.
+// cannot have more than 510 snapshot at a given point of time. If the
-// If the snapshots are more than the `maxSnapshotsOnImage` Add a task to
+// snapshots are more than the `maxSnapshotsOnImage` Add a task to flatten all
-// flatten all the temporary cloned images.
+// the temporary cloned images and return ABORT error message. If the snapshots
+// are more than the `minSnapshotOnImage` Add a task to flatten all the
+// temporary cloned images.
 func flattenTemporaryClonedImages(ctx context.Context, rbdVol *rbdVolume, cr *util.Credentials) error {
 	snaps, err := rbdVol.listSnapshots()
 	if err != nil {
@@ -356,6 +358,7 @@ func flattenTemporaryClonedImages(ctx context.Context, rbdVol *rbdVolume, cr *ut
 	}
 
 	if len(snaps) > int(maxSnapshotsOnImage) {
+		util.DebugLog(ctx, "snapshots count %d on image: %s reached configured hard limit %d", len(snaps), rbdVol, maxSnapshotsOnImage)
 		err = flattenClonedRbdImages(
 			ctx,
 			snaps,
@@ -368,6 +371,24 @@ func flattenTemporaryClonedImages(ctx context.Context, rbdVol *rbdVolume, cr *ut
 		}
 		return status.Errorf(codes.ResourceExhausted, "rbd image %s has %d snapshots", rbdVol, len(snaps))
 	}
+
+	if len(snaps) > int(minSnapshotsOnImageToStartFlatten) {
+		util.DebugLog(ctx, "snapshots count %d on image: %s reached configured soft limit %d", len(snaps), rbdVol, minSnapshotsOnImageToStartFlatten)
+		// If we start flattening all the snapshots at one shot the volume
+		// creation time will be affected,so we will flatten only the extra
+		// snapshots.
+		snaps = snaps[minSnapshotsOnImageToStartFlatten-1:]
+		err = flattenClonedRbdImages(
+			ctx,
+			snaps,
+			rbdVol.Pool,
+			rbdVol.Monitors,
+			rbdVol.RbdImageName,
+			cr)
+		if err != nil {
+			return status.Error(codes.Internal, err.Error())
+		}
+	}
 	return nil
 }
 

internal/rbd/driver.go

@@ -55,6 +55,7 @@ var (
 	// rbdSoftMaxCloneDepth is the soft limit for maximum number of nested volume clones that are taken before a flatten occurs
 	rbdSoftMaxCloneDepth              uint
 	maxSnapshotsOnImage               uint
+	minSnapshotsOnImageToStartFlatten uint
 	skipForceFlatten                  bool
 )
 
@@ -111,6 +112,7 @@ func (r *Driver) Run(conf *util.Config) {
 	rbdSoftMaxCloneDepth = conf.RbdSoftMaxCloneDepth
 	skipForceFlatten = conf.SkipForceFlatten
 	maxSnapshotsOnImage = conf.MaxSnapshotsOnImage
+	minSnapshotsOnImageToStartFlatten = conf.MinSnapshotsOnImage
 	// Create instances of the volume and snapshot journal
 	volJournal = journal.NewCSIVolumeJournal(CSIInstanceID)
 	snapJournal = journal.NewCSISnapshotJournal(CSIInstanceID)

internal/util/util.go

@@ -107,6 +107,11 @@ type Config struct {
 	// on rbd image without flattening, once the limit is reached cephcsi will
 	// start flattening the older rbd images to allow more snapshots
 	MaxSnapshotsOnImage uint
+
+	// MinSnapshotsOnImage represents the soft limit for maximum number of
+	// snapshots allowed on rbd image without flattening, once the soft limit is
+	// reached cephcsi will start flattening the older rbd images.
+	MinSnapshotsOnImage uint
 }
 
 // ValidateDriverName validates the driver name.

scripts/install-helm.sh

@@ -125,7 +125,7 @@ install_cephcsi_helm_charts() {
     # deleting configmap as a workaround to avoid configmap already present
     # issue when installing ceph-csi-rbd
     kubectl delete cm ceph-csi-config --namespace ${NAMESPACE}
-    "${HELM}" install --namespace ${NAMESPACE} --set provisioner.fullnameOverride=csi-rbdplugin-provisioner --set nodeplugin.fullnameOverride=csi-rbdplugin --set configMapName=ceph-csi-config --set provisioner.podSecurityPolicy.enabled=true --set nodeplugin.podSecurityPolicy.enabled=true --set provisioner.replicaCount=1 ${RBD_CHART_NAME} "${SCRIPT_DIR}"/../charts/ceph-csi-rbd --set topology.enabled=true --set topology.domainLabels="{${NODE_LABEL_REGION},${NODE_LABEL_ZONE}}" --set provisioner.maxSnapshotsOnImage=3
+    "${HELM}" install --namespace ${NAMESPACE} --set provisioner.fullnameOverride=csi-rbdplugin-provisioner --set nodeplugin.fullnameOverride=csi-rbdplugin --set configMapName=ceph-csi-config --set provisioner.podSecurityPolicy.enabled=true --set nodeplugin.podSecurityPolicy.enabled=true --set provisioner.replicaCount=1 ${RBD_CHART_NAME} "${SCRIPT_DIR}"/../charts/ceph-csi-rbd --set topology.enabled=true --set topology.domainLabels="{${NODE_LABEL_REGION},${NODE_LABEL_ZONE}}" --set provisioner.maxSnapshotsOnImage=3 --set provisioner.minSnapshotsOnImage=2
 
     check_deployment_status app=ceph-csi-rbd ${NAMESPACE}
     check_daemonset_status app=ceph-csi-rbd ${NAMESPACE}