From 8ef7143e6cbfc1e948009e8150096b11cd839fc7 Mon Sep 17 00:00:00 2001 From: Madhu Rajanna Date: Wed, 1 Jul 2020 10:57:11 +0530 Subject: [PATCH] rbd: add maxsnapshotsonimage flag Added maxsnapshotsonimage flag to flatten the older rbd images on the chain to avoid issue in krbd.The limit is in krbd since it only allocate 1 4KiB page to handle all the snapshot ids for an image. The max limit is 510 as per https://github.com/torvalds/linux/blob/ aaa2faab4ed8e5fe0111e04d6e168c028fe2987f/drivers/block/rbd.c#L98 in cephcsi we arekeeping the default to 450 to reserve 10% to avoid issues. Signed-off-by: Madhu Rajanna --- .../templates/provisioner-deployment.yaml | 1 + charts/ceph-csi-rbd/values.yaml | 2 ++ cmd/cephcsi.go | 13 ++++++++++++- docs/deploy-rbd.md | 1 + internal/rbd/driver.go | 5 +++-- internal/util/util.go | 5 +++++ 6 files changed, 24 insertions(+), 3 deletions(-) diff --git a/charts/ceph-csi-rbd/templates/provisioner-deployment.yaml b/charts/ceph-csi-rbd/templates/provisioner-deployment.yaml index 560203cc3..290ea8112 100644 --- a/charts/ceph-csi-rbd/templates/provisioner-deployment.yaml +++ b/charts/ceph-csi-rbd/templates/provisioner-deployment.yaml @@ -116,6 +116,7 @@ spec: - "--drivername=$(DRIVER_NAME)" - "--rbdhardmaxclonedepth={{ .Values.provisioner.hardMaxCloneDepth }}" - "--rbdsoftmaxclonedepth={{ .Values.provisioner.softMaxCloneDepth }}" + - "--maxsnapshotsonimage={{ .Values.provisioner.maxSnapshotsOnImage }}" {{- if .Values.provisioner.skipForceFlatten }} - "--skipforceflatten={{ .Values.provisioner.skipForceFlatten }}" {{- end }} diff --git a/charts/ceph-csi-rbd/values.yaml b/charts/ceph-csi-rbd/values.yaml index cf86f09f8..eb0acd7dd 100644 --- a/charts/ceph-csi-rbd/values.yaml +++ b/charts/ceph-csi-rbd/values.yaml @@ -113,6 +113,8 @@ provisioner: # Soft limit for maximum number of nested volume clones that are taken before # a flatten occurs softMaxCloneDepth: 4 + # Maximum number of snapshots allowed on rbd image without flattening + maxSnapshotsOnImage: 450 # skip image flattening if kernel support mapping of rbd images # which has the deep-flatten feature # skipForceFlatten: false diff --git a/cmd/cephcsi.go b/cmd/cephcsi.go index c0484d026..573fecc60 100644 --- a/cmd/cephcsi.go +++ b/cmd/cephcsi.go @@ -79,7 +79,7 @@ func init() { flag.UintVar(&conf.RbdHardMaxCloneDepth, "rbdhardmaxclonedepth", 8, "Hard limit for maximum number of nested volume clones that are taken before a flatten occurs") flag.UintVar(&conf.RbdSoftMaxCloneDepth, "rbdsoftmaxclonedepth", 4, "Soft limit for maximum number of nested volume clones that are taken before a flatten occurs") - + flag.UintVar(&conf.MaxSnapshotsOnImage, "maxsnapshotsonimage", 450, "Maximum number of snapshots allowed on rbd image without flattening") flag.BoolVar(&conf.SkipForceFlatten, "skipforceflatten", false, "skip image flattening if kernel support mapping of rbd images which has the deep-flatten feature") @@ -182,6 +182,7 @@ func main() { switch conf.Vtype { case rbdType: validateCloneDepthFlag(&conf) + validateMaxSnaphostFlag(&conf) driver := rbd.NewDriver() driver.Run(&conf, cp) @@ -212,3 +213,13 @@ func validateCloneDepthFlag(conf *util.Config) { klog.Fatalln("rbdsoftmaxclonedepth flag value should not be greater than rbdhardmaxclonedepth") } } + +func validateMaxSnaphostFlag(conf *util.Config) { + // maximum number of snapshots on an image are 510 [1] and 16 images in + // a parent/child chain [2],keeping snapshot limit to 500 to avoid issues. + // [1] https://github.com/torvalds/linux/blob/master/drivers/block/rbd.c#L98 + // [2] https://github.com/torvalds/linux/blob/master/drivers/block/rbd.c#L92 + if conf.MaxSnapshotsOnImage == 0 || conf.MaxSnapshotsOnImage > 500 { + klog.Fatalln("maxsnapshotsonimage flag value should be between 1 and 500") + } +} diff --git a/docs/deploy-rbd.md b/docs/deploy-rbd.md index b02839278..a56e1134c 100644 --- a/docs/deploy-rbd.md +++ b/docs/deploy-rbd.md @@ -45,6 +45,7 @@ make image-cephcsi | `--rbdhardmaxclonedepth` | `8` | Hard limit for maximum number of nested volume clones that are taken before a flatten occurs | | `--rbdsoftmaxclonedepth` | `4` | Soft limit for maximum number of nested volume clones that are taken before a flatten occurs | | `--skipforceflatten` | `false` | skip image flattening on kernel < 5.2 which support mapping of rbd images which has the deep-flatten feature | +| `--maxsnapshotsonimage` | `450` | Maximum number of snapshots allowed on rbd image without flattening | **Available volume parameters:** diff --git a/internal/rbd/driver.go b/internal/rbd/driver.go index 5cca15797..b59654e22 100644 --- a/internal/rbd/driver.go +++ b/internal/rbd/driver.go @@ -58,8 +58,8 @@ var ( // rbdSoftMaxCloneDepth is the soft limit for maximum number of nested volume clones that are taken before a flatten occurs rbdSoftMaxCloneDepth uint - - skipForceFlatten bool + maxSnapshotsOnImage uint + skipForceFlatten bool ) // NewDriver returns new rbd driver @@ -114,6 +114,7 @@ func (r *Driver) Run(conf *util.Config, cachePersister util.CachePersister) { rbdHardMaxCloneDepth = conf.RbdHardMaxCloneDepth rbdSoftMaxCloneDepth = conf.RbdSoftMaxCloneDepth skipForceFlatten = conf.SkipForceFlatten + maxSnapshotsOnImage = conf.MaxSnapshotsOnImage // Create instances of the volume and snapshot journal volJournal = journal.NewCSIVolumeJournal(CSIInstanceID) snapJournal = journal.NewCSISnapshotJournal(CSIInstanceID) diff --git a/internal/util/util.go b/internal/util/util.go index dc130324b..7db122d5a 100644 --- a/internal/util/util.go +++ b/internal/util/util.go @@ -108,6 +108,11 @@ type Config struct { // RbdSoftMaxCloneDepth is the soft limit for maximum number of nested volume clones that are taken before a flatten occurs RbdSoftMaxCloneDepth uint + + // MaxSnapshotsOnImage represents the maximum number of snapshots allowed + // on rbd image without flattening, once the limit is reached cephcsi will + // start flattening the older rbd images to allow more snapshots + MaxSnapshotsOnImage uint } // CreatePersistanceStorage creates storage path and initializes new cache