rbd: add minsnapshotsonimage flag

An rbd image can have a maximum number of
snapshots defined by maxsnapshotsonimage
On the limit is reached the cephcsi will
start flattening the older snapshots and
returns the ABORT error message, The Request
comes after this as to wait till all the
images are flattened (this will increase the
PVC creation time.  Instead of waiting till
the maximum snapshots on an RBD image, we can
have a soft limit, once the limit reached
cephcsi will start flattening the task to
break the chain. With this PVC  creation time
will only be affected when the hard limit
(minsnapshotsonimage) reached.

Signed-off-by: Madhu Rajanna <madhupr007@gmail.com>
This commit is contained in:
Madhu Rajanna 2020-11-17 09:04:29 +05:30 committed by mergify[bot]
parent 880b5bb427
commit 8d3a44d0c4
7 changed files with 44 additions and 8 deletions

View File

@ -133,6 +133,7 @@ spec:
- "--rbdhardmaxclonedepth={{ .Values.provisioner.hardMaxCloneDepth }}" - "--rbdhardmaxclonedepth={{ .Values.provisioner.hardMaxCloneDepth }}"
- "--rbdsoftmaxclonedepth={{ .Values.provisioner.softMaxCloneDepth }}" - "--rbdsoftmaxclonedepth={{ .Values.provisioner.softMaxCloneDepth }}"
- "--maxsnapshotsonimage={{ .Values.provisioner.maxSnapshotsOnImage }}" - "--maxsnapshotsonimage={{ .Values.provisioner.maxSnapshotsOnImage }}"
- "--minsnapshotsonimage={{ .Values.provisioner.minSnapshotsOnImage }}"
{{- if .Values.provisioner.skipForceFlatten }} {{- if .Values.provisioner.skipForceFlatten }}
- "--skipforceflatten={{ .Values.provisioner.skipForceFlatten }}" - "--skipforceflatten={{ .Values.provisioner.skipForceFlatten }}"
{{- end }} {{- end }}

View File

@ -120,6 +120,8 @@ provisioner:
softMaxCloneDepth: 4 softMaxCloneDepth: 4
# Maximum number of snapshots allowed on rbd image without flattening # Maximum number of snapshots allowed on rbd image without flattening
maxSnapshotsOnImage: 450 maxSnapshotsOnImage: 450
# Minimum number of snapshots allowed on rbd image to trigger flattening
minSnapshotsOnImage: 250
# skip image flattening if kernel support mapping of rbd images # skip image flattening if kernel support mapping of rbd images
# which has the deep-flatten feature # which has the deep-flatten feature
# skipForceFlatten: false # skipForceFlatten: false

View File

@ -78,6 +78,7 @@ func init() {
flag.UintVar(&conf.RbdHardMaxCloneDepth, "rbdhardmaxclonedepth", 8, "Hard limit for maximum number of nested volume clones that are taken before a flatten occurs") flag.UintVar(&conf.RbdHardMaxCloneDepth, "rbdhardmaxclonedepth", 8, "Hard limit for maximum number of nested volume clones that are taken before a flatten occurs")
flag.UintVar(&conf.RbdSoftMaxCloneDepth, "rbdsoftmaxclonedepth", 4, "Soft limit for maximum number of nested volume clones that are taken before a flatten occurs") flag.UintVar(&conf.RbdSoftMaxCloneDepth, "rbdsoftmaxclonedepth", 4, "Soft limit for maximum number of nested volume clones that are taken before a flatten occurs")
flag.UintVar(&conf.MaxSnapshotsOnImage, "maxsnapshotsonimage", 450, "Maximum number of snapshots allowed on rbd image without flattening") flag.UintVar(&conf.MaxSnapshotsOnImage, "maxsnapshotsonimage", 450, "Maximum number of snapshots allowed on rbd image without flattening")
flag.UintVar(&conf.MinSnapshotsOnImage, "minsnapshotsonimage", 250, "Minimum number of snapshots required on rbd image to start flattening")
flag.BoolVar(&conf.SkipForceFlatten, "skipforceflatten", false, flag.BoolVar(&conf.SkipForceFlatten, "skipforceflatten", false,
"skip image flattening if kernel support mapping of rbd images which has the deep-flatten feature") "skip image flattening if kernel support mapping of rbd images which has the deep-flatten feature")
@ -204,6 +205,10 @@ func validateMaxSnaphostFlag(conf *util.Config) {
if conf.MaxSnapshotsOnImage == 0 || conf.MaxSnapshotsOnImage > 500 { if conf.MaxSnapshotsOnImage == 0 || conf.MaxSnapshotsOnImage > 500 {
logAndExit("maxsnapshotsonimage flag value should be between 1 and 500") logAndExit("maxsnapshotsonimage flag value should be between 1 and 500")
} }
if conf.MinSnapshotsOnImage > conf.MaxSnapshotsOnImage {
logAndExit("minsnapshotsonimage flag value should be less than maxsnapshotsonimage")
}
} }
func logAndExit(msg string) { func logAndExit(msg string) {

View File

@ -342,10 +342,12 @@ func flattenParentImage(ctx context.Context, rbdVol *rbdVolume, cr *util.Credent
return nil return nil
} }
// check snapshots on the rbd image, as we have limit from krbd that // check snapshots on the rbd image, as we have limit from krbd that an image
// an image cannot have more than 510 snapshot at a given point of time. // cannot have more than 510 snapshot at a given point of time. If the
// If the snapshots are more than the `maxSnapshotsOnImage` Add a task to // snapshots are more than the `maxSnapshotsOnImage` Add a task to flatten all
// flatten all the temporary cloned images. // the temporary cloned images and return ABORT error message. If the snapshots
// are more than the `minSnapshotOnImage` Add a task to flatten all the
// temporary cloned images.
func flattenTemporaryClonedImages(ctx context.Context, rbdVol *rbdVolume, cr *util.Credentials) error { func flattenTemporaryClonedImages(ctx context.Context, rbdVol *rbdVolume, cr *util.Credentials) error {
snaps, err := rbdVol.listSnapshots() snaps, err := rbdVol.listSnapshots()
if err != nil { if err != nil {
@ -356,6 +358,7 @@ func flattenTemporaryClonedImages(ctx context.Context, rbdVol *rbdVolume, cr *ut
} }
if len(snaps) > int(maxSnapshotsOnImage) { if len(snaps) > int(maxSnapshotsOnImage) {
util.DebugLog(ctx, "snapshots count %d on image: %s reached configured hard limit %d", len(snaps), rbdVol, maxSnapshotsOnImage)
err = flattenClonedRbdImages( err = flattenClonedRbdImages(
ctx, ctx,
snaps, snaps,
@ -368,6 +371,24 @@ func flattenTemporaryClonedImages(ctx context.Context, rbdVol *rbdVolume, cr *ut
} }
return status.Errorf(codes.ResourceExhausted, "rbd image %s has %d snapshots", rbdVol, len(snaps)) return status.Errorf(codes.ResourceExhausted, "rbd image %s has %d snapshots", rbdVol, len(snaps))
} }
if len(snaps) > int(minSnapshotsOnImageToStartFlatten) {
util.DebugLog(ctx, "snapshots count %d on image: %s reached configured soft limit %d", len(snaps), rbdVol, minSnapshotsOnImageToStartFlatten)
// If we start flattening all the snapshots at one shot the volume
// creation time will be affected,so we will flatten only the extra
// snapshots.
snaps = snaps[minSnapshotsOnImageToStartFlatten-1:]
err = flattenClonedRbdImages(
ctx,
snaps,
rbdVol.Pool,
rbdVol.Monitors,
rbdVol.RbdImageName,
cr)
if err != nil {
return status.Error(codes.Internal, err.Error())
}
}
return nil return nil
} }

View File

@ -53,9 +53,10 @@ var (
rbdHardMaxCloneDepth uint rbdHardMaxCloneDepth uint
// rbdSoftMaxCloneDepth is the soft limit for maximum number of nested volume clones that are taken before a flatten occurs // rbdSoftMaxCloneDepth is the soft limit for maximum number of nested volume clones that are taken before a flatten occurs
rbdSoftMaxCloneDepth uint rbdSoftMaxCloneDepth uint
maxSnapshotsOnImage uint maxSnapshotsOnImage uint
skipForceFlatten bool minSnapshotsOnImageToStartFlatten uint
skipForceFlatten bool
) )
// NewDriver returns new rbd driver. // NewDriver returns new rbd driver.
@ -111,6 +112,7 @@ func (r *Driver) Run(conf *util.Config) {
rbdSoftMaxCloneDepth = conf.RbdSoftMaxCloneDepth rbdSoftMaxCloneDepth = conf.RbdSoftMaxCloneDepth
skipForceFlatten = conf.SkipForceFlatten skipForceFlatten = conf.SkipForceFlatten
maxSnapshotsOnImage = conf.MaxSnapshotsOnImage maxSnapshotsOnImage = conf.MaxSnapshotsOnImage
minSnapshotsOnImageToStartFlatten = conf.MinSnapshotsOnImage
// Create instances of the volume and snapshot journal // Create instances of the volume and snapshot journal
volJournal = journal.NewCSIVolumeJournal(CSIInstanceID) volJournal = journal.NewCSIVolumeJournal(CSIInstanceID)
snapJournal = journal.NewCSISnapshotJournal(CSIInstanceID) snapJournal = journal.NewCSISnapshotJournal(CSIInstanceID)

View File

@ -107,6 +107,11 @@ type Config struct {
// on rbd image without flattening, once the limit is reached cephcsi will // on rbd image without flattening, once the limit is reached cephcsi will
// start flattening the older rbd images to allow more snapshots // start flattening the older rbd images to allow more snapshots
MaxSnapshotsOnImage uint MaxSnapshotsOnImage uint
// MinSnapshotsOnImage represents the soft limit for maximum number of
// snapshots allowed on rbd image without flattening, once the soft limit is
// reached cephcsi will start flattening the older rbd images.
MinSnapshotsOnImage uint
} }
// ValidateDriverName validates the driver name. // ValidateDriverName validates the driver name.

View File

@ -125,7 +125,7 @@ install_cephcsi_helm_charts() {
# deleting configmap as a workaround to avoid configmap already present # deleting configmap as a workaround to avoid configmap already present
# issue when installing ceph-csi-rbd # issue when installing ceph-csi-rbd
kubectl delete cm ceph-csi-config --namespace ${NAMESPACE} kubectl delete cm ceph-csi-config --namespace ${NAMESPACE}
"${HELM}" install --namespace ${NAMESPACE} --set provisioner.fullnameOverride=csi-rbdplugin-provisioner --set nodeplugin.fullnameOverride=csi-rbdplugin --set configMapName=ceph-csi-config --set provisioner.podSecurityPolicy.enabled=true --set nodeplugin.podSecurityPolicy.enabled=true --set provisioner.replicaCount=1 ${RBD_CHART_NAME} "${SCRIPT_DIR}"/../charts/ceph-csi-rbd --set topology.enabled=true --set topology.domainLabels="{${NODE_LABEL_REGION},${NODE_LABEL_ZONE}}" --set provisioner.maxSnapshotsOnImage=3 "${HELM}" install --namespace ${NAMESPACE} --set provisioner.fullnameOverride=csi-rbdplugin-provisioner --set nodeplugin.fullnameOverride=csi-rbdplugin --set configMapName=ceph-csi-config --set provisioner.podSecurityPolicy.enabled=true --set nodeplugin.podSecurityPolicy.enabled=true --set provisioner.replicaCount=1 ${RBD_CHART_NAME} "${SCRIPT_DIR}"/../charts/ceph-csi-rbd --set topology.enabled=true --set topology.domainLabels="{${NODE_LABEL_REGION},${NODE_LABEL_ZONE}}" --set provisioner.maxSnapshotsOnImage=3 --set provisioner.minSnapshotsOnImage=2
check_deployment_status app=ceph-csi-rbd ${NAMESPACE} check_deployment_status app=ceph-csi-rbd ${NAMESPACE}
check_daemonset_status app=ceph-csi-rbd ${NAMESPACE} check_daemonset_status app=ceph-csi-rbd ${NAMESPACE}