From 10258710215b8c9630de3b871b14abbcacea8331 Mon Sep 17 00:00:00 2001 From: takeaki-matsumoto Date: Thu, 7 Jul 2022 00:46:12 +0900 Subject: [PATCH] cephfs: Support mount option on nodeplugin add mount options on nodeplugin side Signed-off-by: takeaki-matsumoto --- charts/ceph-csi-cephfs/README.md | 2 ++ .../templates/nodeplugin-daemonset.yaml | 2 ++ charts/ceph-csi-cephfs/values.yaml | 5 +++++ cmd/cephcsi.go | 10 ++++++++++ docs/deploy-cephfs.md | 2 ++ internal/cephfs/driver.go | 18 +++++++++++++----- internal/cephfs/nodeserver.go | 13 +++++++++++-- internal/util/util.go | 7 ++++++- 8 files changed, 51 insertions(+), 8 deletions(-) diff --git a/charts/ceph-csi-cephfs/README.md b/charts/ceph-csi-cephfs/README.md index 18e57fea5..51a0b4a15 100644 --- a/charts/ceph-csi-cephfs/README.md +++ b/charts/ceph-csi-cephfs/README.md @@ -99,6 +99,8 @@ charts and their default values. | `nodeplugin.nodeSelector` | Kubernetes `nodeSelector` to add to the Daemonset | `{}` | | `nodeplugin.tolerations` | List of Kubernetes `tolerations` to add to the Daemonset | `{}` | | `nodeplugin.forcecephkernelclient` | Set to true to enable Ceph Kernel clients on kernel < 4.17 which support quotas | `true` | +| `nodeplugin.kernelmountoptions` | Comma separated string of mount options accepted by cephfs kernel mounter quotas | `""` | +| `nodeplugin.fusemountoptions` | Comma separated string of mount options accepted by ceph-fuse mounter quotas | `""` | | `nodeplugin.podSecurityPolicy.enabled` | If true, create & use [Pod Security Policy resources](https://kubernetes.io/docs/concepts/policy/pod-security-policy/). | `false` | | `provisioner.name` | Specifies the name of provisioner | `provisioner` | | `provisioner.replicaCount` | Specifies the replicaCount | `3` | diff --git a/charts/ceph-csi-cephfs/templates/nodeplugin-daemonset.yaml b/charts/ceph-csi-cephfs/templates/nodeplugin-daemonset.yaml index cc8b3f734..1a499bb60 100644 --- a/charts/ceph-csi-cephfs/templates/nodeplugin-daemonset.yaml +++ b/charts/ceph-csi-cephfs/templates/nodeplugin-daemonset.yaml @@ -72,6 +72,8 @@ spec: {{- if .Values.nodeplugin.forcecephkernelclient }} - "--forcecephkernelclient={{ .Values.nodeplugin.forcecephkernelclient }}" {{- end }} + - "--kernelmountoptions={{ .Values.nodeplugin.kernelmountoptions }}" + - "--fusemountoptions={{ .Values.nodeplugin.fusemountoptions }}" - "--endpoint=$(CSI_ENDPOINT)" - "--v={{ .Values.logLevel }}" - "--drivername=$(DRIVER_NAME)" diff --git a/charts/ceph-csi-cephfs/values.yaml b/charts/ceph-csi-cephfs/values.yaml index 351e2d183..0baea5b8b 100644 --- a/charts/ceph-csi-cephfs/values.yaml +++ b/charts/ceph-csi-cephfs/values.yaml @@ -102,6 +102,11 @@ nodeplugin: # on kernel < 4.17 which support quotas # forcecephkernelclient: true + # common mount options to apply all mounting + # example: kernelmountoptions: "recover_session=clean" + kernelmountoptions: "" + fusemountoptions: "" + # If true, create & use Pod Security Policy resources # https://kubernetes.io/docs/concepts/policy/pod-security-policy/ podSecurityPolicy: diff --git a/cmd/cephcsi.go b/cmd/cephcsi.go index 71b3ce6ea..c0a657a0c 100644 --- a/cmd/cephcsi.go +++ b/cmd/cephcsi.go @@ -88,6 +88,16 @@ func init() { "forcecephkernelclient", false, "enable Ceph Kernel clients on kernel < 4.17 which support quotas") + flag.StringVar( + &conf.KernelMountOptions, + "kernelmountoptions", + "", + "Comma separated string of mount options accepted by cephfs kernel mounter") + flag.StringVar( + &conf.FuseMountOptions, + "fusemountoptions", + "", + "Comma separated string of mount options accepted by ceph-fuse mounter") // liveness/grpc metrics related flags flag.IntVar(&conf.MetricsPort, "metricsport", 8080, "TCP port for liveness/grpc metrics requests") diff --git a/docs/deploy-cephfs.md b/docs/deploy-cephfs.md index 181e8d1f3..b135fc7dc 100644 --- a/docs/deploy-cephfs.md +++ b/docs/deploy-cephfs.md @@ -59,6 +59,8 @@ that should be resolved in v14.2.3. | `--timeout` | `3s` | Probe timeout in seconds | | `--histogramoption` | `0.5,2,6` | [Deprecated] Histogram option for grpc metrics, should be comma separated value (ex:= "0.5,2,6" where start=0.5 factor=2, count=6) | | `--forcecephkernelclient` | `false` | Force enabling Ceph Kernel clients for mounting on kernels < 4.17 | +| `--kernelmountoptions` | _empty_ | Comma separated string of mount options accepted by cephfs kernel mounter | +| `--fusemountoptions` | _empty_ | Comma separated string of mount options accepted by ceph-fuse mounter | | `--domainlabels` | _empty_ | Kubernetes node labels to use as CSI domain labels for topology aware provisioning, should be a comma separated value (ex:= "failure-domain/region,failure-domain/zone") | **NOTE:** The parameter `-forcecephkernelclient` enables the Kernel diff --git a/internal/cephfs/driver.go b/internal/cephfs/driver.go index 4b7295b0b..b9e39cb68 100644 --- a/internal/cephfs/driver.go +++ b/internal/cephfs/driver.go @@ -64,10 +64,18 @@ func NewControllerServer(d *csicommon.CSIDriver) *ControllerServer { } // NewNodeServer initialize a node server for ceph CSI driver. -func NewNodeServer(d *csicommon.CSIDriver, t string, topology map[string]string) *NodeServer { +func NewNodeServer( + d *csicommon.CSIDriver, + t string, + topology map[string]string, + kernelMountOptions string, + fuseMountOptions string, +) *NodeServer { return &NodeServer{ - DefaultNodeServer: csicommon.NewDefaultNodeServer(d, t, topology), - VolumeLocks: util.NewVolumeLocks(), + DefaultNodeServer: csicommon.NewDefaultNodeServer(d, t, topology), + VolumeLocks: util.NewVolumeLocks(), + kernelMountOptions: kernelMountOptions, + fuseMountOptions: fuseMountOptions, } } @@ -122,7 +130,7 @@ func (fs *Driver) Run(conf *util.Config) { if err != nil { log.FatalLogMsg(err.Error()) } - fs.ns = NewNodeServer(fs.cd, conf.Vtype, topology) + fs.ns = NewNodeServer(fs.cd, conf.Vtype, topology, conf.KernelMountOptions, conf.FuseMountOptions) } if conf.IsControllerServer { @@ -133,7 +141,7 @@ func (fs *Driver) Run(conf *util.Config) { if err != nil { log.FatalLogMsg(err.Error()) } - fs.ns = NewNodeServer(fs.cd, conf.Vtype, topology) + fs.ns = NewNodeServer(fs.cd, conf.Vtype, topology, conf.KernelMountOptions, conf.FuseMountOptions) fs.cs = NewControllerServer(fs.cd) } diff --git a/internal/cephfs/nodeserver.go b/internal/cephfs/nodeserver.go index c6a2eb691..51fd3337c 100644 --- a/internal/cephfs/nodeserver.go +++ b/internal/cephfs/nodeserver.go @@ -43,7 +43,9 @@ type NodeServer struct { *csicommon.DefaultNodeServer // A map storing all volumes with ongoing operations so that additional operations // for that same volume (as defined by VolumeID) return an Aborted error - VolumeLocks *util.VolumeLocks + VolumeLocks *util.VolumeLocks + kernelMountOptions string + fuseMountOptions string } func getCredentialsForVolume( @@ -225,7 +227,7 @@ func (ns *NodeServer) NodeStageVolume( return &csi.NodeStageVolumeResponse{}, nil } -func (*NodeServer) mount( +func (ns *NodeServer) mount( ctx context.Context, mnt mounter.VolumeMounter, volOptions *store.VolumeOptions, @@ -244,6 +246,13 @@ func (*NodeServer) mount( log.DebugLog(ctx, "cephfs: mounting volume %s with %s", volID, mnt.Name()) + switch mnt.(type) { + case *mounter.FuseMounter: + volOptions.FuseMountOptions = util.MountOptionsAdd(volOptions.FuseMountOptions, ns.fuseMountOptions) + case *mounter.KernelMounter: + volOptions.KernelMountOptions = util.MountOptionsAdd(volOptions.KernelMountOptions, ns.kernelMountOptions) + } + const readOnly = "ro" if volCap.AccessMode.Mode == csi.VolumeCapability_AccessMode_MULTI_NODE_READER_ONLY || diff --git a/internal/util/util.go b/internal/util/util.go index 4c36e3679..1527adcfb 100644 --- a/internal/util/util.go +++ b/internal/util/util.go @@ -98,7 +98,12 @@ type Config struct { MetricsPath string // path of prometheus endpoint where metrics will be available HistogramOption string // Histogram option for grpc metrics, should be comma separated value, // ex:= "0.5,2,6" where start=0.5 factor=2, count=6 - MetricsIP string // TCP port for liveness/ metrics requests + MetricsIP string // TCP port for liveness/ metrics requests + + // mount option related flags + KernelMountOptions string // Comma separated string of mount options accepted by cephfs kernel mounter + FuseMountOptions string // Comma separated string of mount options accepted by ceph-fuse mounter + PidLimit int // PID limit to configure through cgroups") MetricsPort int // TCP port for liveness/grpc metrics requests PollTime time.Duration // time interval in seconds between each poll