mirrors/ceph-csi
1
0
Fork 0
mirror of https://github.com/ceph/ceph-csi.git synced 2024-12-18 02:50:30 +00:00
Code Issues Packages Projects Releases Wiki Activity

rbd: add hardlimt and softlimit flag

Browse Source 
added Hardlimit and Softlimit flags for cephcsi
arguments. When the Softlimit is reached cephcsi
will start a background task to flatten the rbd
image and return success and if the hardlimit
is reached it will start a background task
to flatten the rbd image and return ready
to use as false to make sure that the image
will not be used until it is flatten.

Signed-off-by: Madhu Rajanna <madhupr007@gmail.com>
This commit is contained in:
Madhu Rajanna 2020-06-24 12:14:02 +05:30 committed by mergify[bot]
parent 3e7fa93256
commit 9b518726ab
7 changed files with 56 additions and 17 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
charts/ceph-csi-rbd/templates/provisioner-deployment.yaml
View File

@ -114,6 +114,8 @@ spec:
- "--endpoint=$(CSI_ENDPOINT)" - "--endpoint=$(CSI_ENDPOINT)"
- "--v=5" - "--v=5"
- "--drivername=$(DRIVER_NAME)" - "--drivername=$(DRIVER_NAME)"
- "--rbdhardmaxclonedepth={{ .Values.provisioner.hardMaxCloneDepth }}"
- "--rbdsoftmaxclonedepth={{ .Values.provisioner.softMaxCloneDepth }}"
env: env:
- name: POD_IP - name: POD_IP
valueFrom: valueFrom:

6
charts/ceph-csi-rbd/values.yaml
View File

@ -107,6 +107,12 @@ provisioner:
replicaCount: 3 replicaCount: 3
# Timeout for waiting for creation or deletion of a volume # Timeout for waiting for creation or deletion of a volume
timeout: 60s timeout: 60s
# Hard limit for maximum number of nested volume clones that are taken before
# a flatten occurs
hardMaxCloneDepth: 8
# Soft limit for maximum number of nested volume clones that are taken before
# a flatten occurs
softMaxCloneDepth: 4
httpMetrics: httpMetrics:
# Metrics only available for cephcsi/cephcsi => 1.2.0 # Metrics only available for cephcsi/cephcsi => 1.2.0

14
cmd/cephcsi.go
View File

@ -77,6 +77,8 @@ func init() {
flag.StringVar(&conf.HistogramOption, "histogramoption", "0.5,2,6", flag.StringVar(&conf.HistogramOption, "histogramoption", "0.5,2,6",
"[DEPRECATED] Histogram option for grpc metrics, should be comma separated value, ex:= 0.5,2,6 where start=0.5 factor=2, count=6") "[DEPRECATED] Histogram option for grpc metrics, should be comma separated value, ex:= 0.5,2,6 where start=0.5 factor=2, count=6")
flag.UintVar(&conf.RbdHardMaxCloneDepth, "rbdhardmaxclonedepth", 8, "Hard limit for maximum number of nested volume clones that are taken before a flatten occurs")
flag.UintVar(&conf.RbdSoftMaxCloneDepth, "rbdsoftmaxclonedepth", 4, "Soft limit for maximum number of nested volume clones that are taken before a flatten occurs")
flag.BoolVar(&conf.Version, "version", false, "Print cephcsi version information") flag.BoolVar(&conf.Version, "version", false, "Print cephcsi version information")
klog.InitFlags(nil) klog.InitFlags(nil)
@ -175,6 +177,7 @@ func main() {
klog.V(1).Infof("Starting driver type: %v with name: %v", conf.Vtype, dname) klog.V(1).Infof("Starting driver type: %v with name: %v", conf.Vtype, dname)
switch conf.Vtype { switch conf.Vtype {
case rbdType: case rbdType:
validateCloneDepthFlag(&conf)
driver := rbd.NewDriver() driver := rbd.NewDriver()
driver.Run(&conf, cp) driver.Run(&conf, cp)
@ -194,3 +197,14 @@ func main() {
os.Exit(0) os.Exit(0)
} }
func validateCloneDepthFlag(conf *util.Config) {
// keeping hardlimit to 14 as max to avoid max image depth
if conf.RbdHardMaxCloneDepth == 0 || conf.RbdHardMaxCloneDepth > 14 {
klog.Fatalln("rbdhardmaxclonedepth flag value should be between 1 and 14")
}
if conf.RbdSoftMaxCloneDepth > conf.RbdHardMaxCloneDepth {
klog.Fatalln("rbdsoftmaxclonedepth flag value should not be greater than rbdhardmaxclonedepth")
}
}

2
deploy/rbd/kubernetes/csi-rbdplugin-provisioner.yaml
View File

@ -112,6 +112,8 @@ spec:
- "--v=5" - "--v=5"
- "--drivername=rbd.csi.ceph.com" - "--drivername=rbd.csi.ceph.com"
- "--pidlimit=-1" - "--pidlimit=-1"
- "--rbdhardmaxclonedepth=8"
- "--rbdsoftmaxclonedepth=4"
env: env:
- name: POD_IP - name: POD_IP
valueFrom: valueFrom:

36
docs/deploy-rbd.md
View File

@ -26,22 +26,24 @@ make image-cephcsi
**Available command line arguments:** **Available command line arguments:**
| Option | Default value | Description | | Option | Default value | Description |
|-----------------------|-----------------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| | ------------------------ | --------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ |
| `--endpoint` | `unix://tmp/csi.sock` | CSI endpoint, must be a UNIX socket | | `--endpoint` | `unix://tmp/csi.sock` | CSI endpoint, must be a UNIX socket |
| `--drivername` | `rbd.csi.ceph.com` | Name of the driver (Kubernetes: `provisioner` field in StorageClass must correspond to this value) | | `--drivername` | `rbd.csi.ceph.com` | Name of the driver (Kubernetes: `provisioner` field in StorageClass must correspond to this value) |
| `--nodeid` | _empty_ | This node's ID | | `--nodeid` | _empty_ | This node's ID |
| `--type` | _empty_ | Driver type `[rbd | cephfs]` If the driver type is set to `rbd` it will act as a `rbd plugin` or if it's set to `cephfs` will act as a `cephfs plugin` | | `--type` | _empty_ | Driver type `[rbd | cephfs]` If the driver type is set to `rbd` it will act as a `rbd plugin` or if it's set to `cephfs` will act as a `cephfs plugin` |
| `--instanceid` | "default" | Unique ID distinguishing this instance of Ceph CSI among other instances, when sharing Ceph clusters across CSI instances for provisioning | | `--instanceid` | "default" | Unique ID distinguishing this instance of Ceph CSI among other instances, when sharing Ceph clusters across CSI instances for provisioning |
| `--metadatastorage` | _empty_ | Points to where legacy (1.0.0 or older plugin versions) metadata about provisioned volumes are kept, as file or in as k8s configmap (`node` or `k8s_configmap` respectively) | | `--metadatastorage` | _empty_ | Points to where legacy (1.0.0 or older plugin versions) metadata about provisioned volumes are kept, as file or in as k8s configmap (`node` or `k8s_configmap` respectively) |
| `--pidlimit` | _0_ | Configure the PID limit in cgroups. The container runtime can restrict the number of processes/tasks which can cause problems while provisioning (or deleting) a large number of volumes. A value of `-1` configures the limit to the maximum, `0` does not configure limits at all. | | `--pidlimit` | _0_ | Configure the PID limit in cgroups. The container runtime can restrict the number of processes/tasks which can cause problems while provisioning (or deleting) a large number of volumes. A value of `-1` configures the limit to the maximum, `0` does not configure limits at all. |
| `--metricsport` | `8080` | TCP port for liveness metrics requests | | `--metricsport` | `8080` | TCP port for liveness metrics requests |
| `--metricspath` | `"/metrics"` | Path of prometheus endpoint where metrics will be available | | `--metricspath` | `"/metrics"` | Path of prometheus endpoint where metrics will be available |
| `--enablegrpcmetrics` | `false` | [Deprecated] Enable grpc metrics collection and start prometheus server | | `--enablegrpcmetrics` | `false` | [Deprecated] Enable grpc metrics collection and start prometheus server |
| `--polltime` | `"60s"` | Time interval in between each poll | | `--polltime` | `"60s"` | Time interval in between each poll |
| `--timeout` | `"3s"` | Probe timeout in seconds | | `--timeout` | `"3s"` | Probe timeout in seconds |
| `--histogramoption` | `0.5,2,6` | [Deprecated] Histogram option for grpc metrics, should be comma separated value (ex:= "0.5,2,6" where start=0.5 factor=2, count=6) | | `--histogramoption` | `0.5,2,6` | [Deprecated] Histogram option for grpc metrics, should be comma separated value (ex:= "0.5,2,6" where start=0.5 factor=2, count=6) |
| `--domainlabels` | _empty_ | Kubernetes node labels to use as CSI domain labels for topology aware provisioning, should be a comma separated value (ex:= "failure-domain/region,failure-domain/zone") | | `--domainlabels` | _empty_ | Kubernetes node labels to use as CSI domain labels for topology aware provisioning, should be a comma separated value (ex:= "failure-domain/region,failure-domain/zone") |
| `--rbdhardmaxclonedepth` | `8` | Hard limit for maximum number of nested volume clones that are taken before a flatten occurs |
| `--rbdsoftmaxclonedepth` | `4` | Soft limit for maximum number of nested volume clones that are taken before a flatten occurs |
**Available volume parameters:** **Available volume parameters:**
@ -50,7 +52,7 @@ make image-cephcsi
| `clusterID` | yes | String representing a Ceph cluster, must be unique across all Ceph clusters in use for provisioning, cannot be greater than 36 bytes in length, and should remain immutable for the lifetime of the Ceph cluster in use | | `clusterID` | yes | String representing a Ceph cluster, must be unique across all Ceph clusters in use for provisioning, cannot be greater than 36 bytes in length, and should remain immutable for the lifetime of the Ceph cluster in use |
| `pool` | yes | Ceph pool into which the RBD image shall be created | | `pool` | yes | Ceph pool into which the RBD image shall be created |
| `dataPool` | no | Ceph pool used for the data of the RBD images. | | `dataPool` | no | Ceph pool used for the data of the RBD images. |
| `volumeNamePrefix` | no | Prefix to use for naming RBD images (defaults to `csi-vol-`). | | `volumeNamePrefix` | no | Prefix to use for naming RBD images (defaults to `csi-vol-`). |
| `snapshotNamePrefix` | no | Prefix to use for naming RBD snapshot images (defaults to `csi-snap-`). | | `snapshotNamePrefix` | no | Prefix to use for naming RBD snapshot images (defaults to `csi-snap-`). |
| `imageFeatures` | no | RBD image features. CSI RBD currently supports only `layering` feature. See [man pages](http://docs.ceph.com/docs/mimic/man/8/rbd/#cmdoption-rbd-image-feature) | | `imageFeatures` | no | RBD image features. CSI RBD currently supports only `layering` feature. See [man pages](http://docs.ceph.com/docs/mimic/man/8/rbd/#cmdoption-rbd-image-feature) |
| `csi.storage.k8s.io/provisioner-secret-name`, `csi.storage.k8s.io/node-stage-secret-name` | yes (for Kubernetes) | name of the Kubernetes Secret object containing Ceph client credentials. Both parameters should have the same value | | `csi.storage.k8s.io/provisioner-secret-name`, `csi.storage.k8s.io/node-stage-secret-name` | yes (for Kubernetes) | name of the Kubernetes Secret object containing Ceph client credentials. Both parameters should have the same value |

8
internal/rbd/driver.go
View File

@ -53,6 +53,11 @@ var (
// VolumeName to backing RBD images // VolumeName to backing RBD images
volJournal *journal.Config volJournal *journal.Config
snapJournal *journal.Config snapJournal *journal.Config
// rbdHardMaxCloneDepth is the hard limit for maximum number of nested volume clones that are taken before a flatten occurs
rbdHardMaxCloneDepth uint
// rbdSoftMaxCloneDepth is the soft limit for maximum number of nested volume clones that are taken before a flatten occurs
rbdSoftMaxCloneDepth uint
) )
// NewDriver returns new rbd driver // NewDriver returns new rbd driver
@ -103,6 +108,9 @@ func (r *Driver) Run(conf *util.Config, cachePersister util.CachePersister) {
CSIInstanceID = conf.InstanceID CSIInstanceID = conf.InstanceID
} }
// update clone soft and hard limit
rbdHardMaxCloneDepth = conf.RbdHardMaxCloneDepth
rbdSoftMaxCloneDepth = conf.RbdSoftMaxCloneDepth
// Create instances of the volume and snapshot journal // Create instances of the volume and snapshot journal
volJournal = journal.NewCSIVolumeJournal(CSIInstanceID) volJournal = journal.NewCSIVolumeJournal(CSIInstanceID)
snapJournal = journal.NewCSISnapshotJournal(CSIInstanceID) snapJournal = journal.NewCSISnapshotJournal(CSIInstanceID)

5
internal/util/util.go
View File

@ -98,6 +98,11 @@ type Config struct {
// cephfs related flags // cephfs related flags
ForceKernelCephFS bool // force to use the ceph kernel client even if the kernel is < 4.17 ForceKernelCephFS bool // force to use the ceph kernel client even if the kernel is < 4.17
// RbdHardMaxCloneDepth is the hard limit for maximum number of nested volume clones that are taken before a flatten occurs
RbdHardMaxCloneDepth uint
// RbdSoftMaxCloneDepth is the soft limit for maximum number of nested volume clones that are taken before a flatten occurs
RbdSoftMaxCloneDepth uint
} }
// CreatePersistanceStorage creates storage path and initializes new cache // CreatePersistanceStorage creates storage path and initializes new cache