From 0da4bd5151f385dfbbdd1f182f5ee1d946d00525 Mon Sep 17 00:00:00 2001 From: Madhu Rajanna Date: Wed, 14 Aug 2019 12:12:17 +0530 Subject: [PATCH] start controller or node server based on config if both controller and nodeserver flags are set/unset cephcsi will start both server, if only one flag is set, it will start relavent service. Signed-off-by: Madhu Rajanna --- cmd/cephcsi.go | 2 + .../v1.13/csi-cephfsplugin-provisioner.yaml | 1 + .../kubernetes/v1.13/csi-cephfsplugin.yaml | 1 + .../templates/nodeplugin-daemonset.yaml | 1 + .../templates/provisioner-statefulset.yaml | 1 + .../v1.14+/csi-cephfsplugin-provisioner.yaml | 1 + .../kubernetes/v1.14+/csi-cephfsplugin.yaml | 1 + .../templates/nodeplugin-daemonset.yaml | 3 +- .../templates/provisioner-deployment.yaml | 1 + .../v1.13/csi-rbdplugin-provisioner.yaml | 1 + .../rbd/kubernetes/v1.13/csi-rbdplugin.yaml | 1 + .../templates/nodeplugin-daemonset.yaml | 1 + .../templates/provisioner-statefulset.yaml | 1 + .../v1.14+/csi-rbdplugin-provisioner.yaml | 1 + .../rbd/kubernetes/v1.14+/csi-rbdplugin.yaml | 1 + .../templates/nodeplugin-daemonset.yaml | 1 + .../templates/provisioner-deployment.yaml | 1 + pkg/cephfs/driver.go | 28 +++++++---- pkg/rbd/driver.go | 47 ++++++++++++------- pkg/util/util.go | 18 +++---- 20 files changed, 78 insertions(+), 35 deletions(-) diff --git a/cmd/cephcsi.go b/cmd/cephcsi.go index 3505ceecd..d65114894 100644 --- a/cmd/cephcsi.go +++ b/cmd/cephcsi.go @@ -55,6 +55,8 @@ func init() { flag.StringVar(&conf.MetadataStorage, "metadatastorage", "", "metadata persistence method [node|k8s_configmap]") flag.StringVar(&conf.PluginPath, "pluginpath", "/var/lib/kubelet/plugins/", "the location of cephcsi plugin") flag.IntVar(&conf.PidLimit, "pidlimit", 0, "the PID limit to configure through cgroups") + flag.BoolVar(&conf.IsControllerServer, "controllerserver", false, "start cephcsi controller server") + flag.BoolVar(&conf.IsNodeServer, "nodeserver", false, "start cephcsi node server") // rbd related flags flag.BoolVar(&conf.Containerized, "containerized", true, "whether run as containerized") diff --git a/deploy/cephfs/kubernetes/v1.13/csi-cephfsplugin-provisioner.yaml b/deploy/cephfs/kubernetes/v1.13/csi-cephfsplugin-provisioner.yaml index 0502e3615..1a7bf7797 100644 --- a/deploy/cephfs/kubernetes/v1.13/csi-cephfsplugin-provisioner.yaml +++ b/deploy/cephfs/kubernetes/v1.13/csi-cephfsplugin-provisioner.yaml @@ -68,6 +68,7 @@ spec: args: - "--nodeid=$(NODE_ID)" - "--type=cephfs" + - "--controllerserver=true" - "--endpoint=$(CSI_ENDPOINT)" - "--v=5" - "--drivername=cephfs.csi.ceph.com" diff --git a/deploy/cephfs/kubernetes/v1.13/csi-cephfsplugin.yaml b/deploy/cephfs/kubernetes/v1.13/csi-cephfsplugin.yaml index e817a51a1..c6e716161 100644 --- a/deploy/cephfs/kubernetes/v1.13/csi-cephfsplugin.yaml +++ b/deploy/cephfs/kubernetes/v1.13/csi-cephfsplugin.yaml @@ -53,6 +53,7 @@ spec: args: - "--nodeid=$(NODE_ID)" - "--type=cephfs" + - "--nodeserver=true" - "--endpoint=$(CSI_ENDPOINT)" - "--v=5" - "--drivername=cephfs.csi.ceph.com" diff --git a/deploy/cephfs/kubernetes/v1.13/helm/ceph-csi-cephfs/templates/nodeplugin-daemonset.yaml b/deploy/cephfs/kubernetes/v1.13/helm/ceph-csi-cephfs/templates/nodeplugin-daemonset.yaml index a3c105114..4587b4c27 100644 --- a/deploy/cephfs/kubernetes/v1.13/helm/ceph-csi-cephfs/templates/nodeplugin-daemonset.yaml +++ b/deploy/cephfs/kubernetes/v1.13/helm/ceph-csi-cephfs/templates/nodeplugin-daemonset.yaml @@ -68,6 +68,7 @@ spec: args : - "--nodeid=$(NODE_ID)" - "--type=cephfs" + - "--nodeserver=true" - "--endpoint=$(CSI_ENDPOINT)" - "--v=5" - "--drivername=$(DRIVER_NAME)" diff --git a/deploy/cephfs/kubernetes/v1.13/helm/ceph-csi-cephfs/templates/provisioner-statefulset.yaml b/deploy/cephfs/kubernetes/v1.13/helm/ceph-csi-cephfs/templates/provisioner-statefulset.yaml index 7ae18fbfd..90a7c6691 100644 --- a/deploy/cephfs/kubernetes/v1.13/helm/ceph-csi-cephfs/templates/provisioner-statefulset.yaml +++ b/deploy/cephfs/kubernetes/v1.13/helm/ceph-csi-cephfs/templates/provisioner-statefulset.yaml @@ -71,6 +71,7 @@ spec: args : - "--nodeid=$(NODE_ID)" - "--type=cephfs" + - "--controllerserver=true" - "--endpoint=$(CSI_ENDPOINT)" - "--v=5" - "--drivername=$(DRIVER_NAME)" diff --git a/deploy/cephfs/kubernetes/v1.14+/csi-cephfsplugin-provisioner.yaml b/deploy/cephfs/kubernetes/v1.14+/csi-cephfsplugin-provisioner.yaml index a593b1fdc..3be2f87d0 100644 --- a/deploy/cephfs/kubernetes/v1.14+/csi-cephfsplugin-provisioner.yaml +++ b/deploy/cephfs/kubernetes/v1.14+/csi-cephfsplugin-provisioner.yaml @@ -71,6 +71,7 @@ spec: args: - "--nodeid=$(NODE_ID)" - "--type=cephfs" + - "--controllerserver=true" - "--endpoint=$(CSI_ENDPOINT)" - "--v=5" - "--drivername=cephfs.csi.ceph.com" diff --git a/deploy/cephfs/kubernetes/v1.14+/csi-cephfsplugin.yaml b/deploy/cephfs/kubernetes/v1.14+/csi-cephfsplugin.yaml index 45f59a371..d3d5adfb6 100644 --- a/deploy/cephfs/kubernetes/v1.14+/csi-cephfsplugin.yaml +++ b/deploy/cephfs/kubernetes/v1.14+/csi-cephfsplugin.yaml @@ -53,6 +53,7 @@ spec: args: - "--nodeid=$(NODE_ID)" - "--type=cephfs" + - "--nodeserver=true" - "--endpoint=$(CSI_ENDPOINT)" - "--v=5" - "--drivername=cephfs.csi.ceph.com" diff --git a/deploy/cephfs/kubernetes/v1.14+/helm/ceph-csi-cephfs/templates/nodeplugin-daemonset.yaml b/deploy/cephfs/kubernetes/v1.14+/helm/ceph-csi-cephfs/templates/nodeplugin-daemonset.yaml index 29e82e5b1..31099e685 100644 --- a/deploy/cephfs/kubernetes/v1.14+/helm/ceph-csi-cephfs/templates/nodeplugin-daemonset.yaml +++ b/deploy/cephfs/kubernetes/v1.14+/helm/ceph-csi-cephfs/templates/nodeplugin-daemonset.yaml @@ -68,6 +68,7 @@ spec: args : - "--nodeid=$(NODE_ID)" - "--type=cephfs" + - "--nodeserver=true" - "--endpoint=$(CSI_ENDPOINT)" - "--v=5" - "--drivername=$(DRIVER_NAME)" @@ -180,4 +181,4 @@ spec: {{- if .Values.nodeplugin.tolerations }} tolerations: {{ toYaml .Values.nodeplugin.tolerations | indent 8 -}} -{{- end -}} \ No newline at end of file +{{- end -}} diff --git a/deploy/cephfs/kubernetes/v1.14+/helm/ceph-csi-cephfs/templates/provisioner-deployment.yaml b/deploy/cephfs/kubernetes/v1.14+/helm/ceph-csi-cephfs/templates/provisioner-deployment.yaml index ef49d3e2d..4be3a7cee 100644 --- a/deploy/cephfs/kubernetes/v1.14+/helm/ceph-csi-cephfs/templates/provisioner-deployment.yaml +++ b/deploy/cephfs/kubernetes/v1.14+/helm/ceph-csi-cephfs/templates/provisioner-deployment.yaml @@ -73,6 +73,7 @@ spec: args : - "--nodeid=$(NODE_ID)" - "--type=cephfs" + - "--controllerserver=true" - "--endpoint=$(CSI_ENDPOINT)" - "--v=5" - "--drivername=$(DRIVER_NAME)" diff --git a/deploy/rbd/kubernetes/v1.13/csi-rbdplugin-provisioner.yaml b/deploy/rbd/kubernetes/v1.13/csi-rbdplugin-provisioner.yaml index 37c823460..22d5217b2 100644 --- a/deploy/rbd/kubernetes/v1.13/csi-rbdplugin-provisioner.yaml +++ b/deploy/rbd/kubernetes/v1.13/csi-rbdplugin-provisioner.yaml @@ -83,6 +83,7 @@ spec: args: - "--nodeid=$(NODE_ID)" - "--type=rbd" + - "--controllerserver=true" - "--endpoint=$(CSI_ENDPOINT)" - "--v=5" - "--drivername=rbd.csi.ceph.com" diff --git a/deploy/rbd/kubernetes/v1.13/csi-rbdplugin.yaml b/deploy/rbd/kubernetes/v1.13/csi-rbdplugin.yaml index fc2ba4be2..de62dfa3a 100644 --- a/deploy/rbd/kubernetes/v1.13/csi-rbdplugin.yaml +++ b/deploy/rbd/kubernetes/v1.13/csi-rbdplugin.yaml @@ -56,6 +56,7 @@ spec: args: - "--nodeid=$(NODE_ID)" - "--type=rbd" + - "--nodeserver=true" - "--endpoint=$(CSI_ENDPOINT)" - "--v=5" - "--drivername=rbd.csi.ceph.com" diff --git a/deploy/rbd/kubernetes/v1.13/helm/ceph-csi-rbd/templates/nodeplugin-daemonset.yaml b/deploy/rbd/kubernetes/v1.13/helm/ceph-csi-rbd/templates/nodeplugin-daemonset.yaml index 831777b50..07941b255 100644 --- a/deploy/rbd/kubernetes/v1.13/helm/ceph-csi-rbd/templates/nodeplugin-daemonset.yaml +++ b/deploy/rbd/kubernetes/v1.13/helm/ceph-csi-rbd/templates/nodeplugin-daemonset.yaml @@ -70,6 +70,7 @@ spec: args : - "--nodeid=$(NODE_ID)" - "--type=rbd" + - "--nodeserver=true" - "--endpoint=$(CSI_ENDPOINT)" - "--v=5" - "--drivername=$(DRIVER_NAME)" diff --git a/deploy/rbd/kubernetes/v1.13/helm/ceph-csi-rbd/templates/provisioner-statefulset.yaml b/deploy/rbd/kubernetes/v1.13/helm/ceph-csi-rbd/templates/provisioner-statefulset.yaml index 6d595b066..1c67d2f29 100644 --- a/deploy/rbd/kubernetes/v1.13/helm/ceph-csi-rbd/templates/provisioner-statefulset.yaml +++ b/deploy/rbd/kubernetes/v1.13/helm/ceph-csi-rbd/templates/provisioner-statefulset.yaml @@ -89,6 +89,7 @@ spec: args : - "--nodeid=$(NODE_ID)" - "--type=rbd" + - "--controllerserver=true" - "--endpoint=$(CSI_ENDPOINT)" - "--v=5" - "--drivername=$(DRIVER_NAME)" diff --git a/deploy/rbd/kubernetes/v1.14+/csi-rbdplugin-provisioner.yaml b/deploy/rbd/kubernetes/v1.14+/csi-rbdplugin-provisioner.yaml index 5642a934d..16f7ad174 100644 --- a/deploy/rbd/kubernetes/v1.14+/csi-rbdplugin-provisioner.yaml +++ b/deploy/rbd/kubernetes/v1.14+/csi-rbdplugin-provisioner.yaml @@ -87,6 +87,7 @@ spec: args: - "--nodeid=$(NODE_ID)" - "--type=rbd" + - "--controllerserver=true" - "--endpoint=$(CSI_ENDPOINT)" - "--v=5" - "--drivername=rbd.csi.ceph.com" diff --git a/deploy/rbd/kubernetes/v1.14+/csi-rbdplugin.yaml b/deploy/rbd/kubernetes/v1.14+/csi-rbdplugin.yaml index fc2ba4be2..de62dfa3a 100644 --- a/deploy/rbd/kubernetes/v1.14+/csi-rbdplugin.yaml +++ b/deploy/rbd/kubernetes/v1.14+/csi-rbdplugin.yaml @@ -56,6 +56,7 @@ spec: args: - "--nodeid=$(NODE_ID)" - "--type=rbd" + - "--nodeserver=true" - "--endpoint=$(CSI_ENDPOINT)" - "--v=5" - "--drivername=rbd.csi.ceph.com" diff --git a/deploy/rbd/kubernetes/v1.14+/helm/ceph-csi-rbd/templates/nodeplugin-daemonset.yaml b/deploy/rbd/kubernetes/v1.14+/helm/ceph-csi-rbd/templates/nodeplugin-daemonset.yaml index 95e3dfbbf..9b2790104 100644 --- a/deploy/rbd/kubernetes/v1.14+/helm/ceph-csi-rbd/templates/nodeplugin-daemonset.yaml +++ b/deploy/rbd/kubernetes/v1.14+/helm/ceph-csi-rbd/templates/nodeplugin-daemonset.yaml @@ -70,6 +70,7 @@ spec: args : - "--nodeid=$(NODE_ID)" - "--type=rbd" + - "--nodeserver=true" - "--endpoint=$(CSI_ENDPOINT)" - "--v=5" - "--drivername=$(DRIVER_NAME)" diff --git a/deploy/rbd/kubernetes/v1.14+/helm/ceph-csi-rbd/templates/provisioner-deployment.yaml b/deploy/rbd/kubernetes/v1.14+/helm/ceph-csi-rbd/templates/provisioner-deployment.yaml index d57d36802..270a551b6 100644 --- a/deploy/rbd/kubernetes/v1.14+/helm/ceph-csi-rbd/templates/provisioner-deployment.yaml +++ b/deploy/rbd/kubernetes/v1.14+/helm/ceph-csi-rbd/templates/provisioner-deployment.yaml @@ -92,6 +92,7 @@ spec: args : - "--nodeid=$(NODE_ID)" - "--type=rbd" + - "--controllerserver=true" - "--endpoint=$(CSI_ENDPOINT)" - "--v=5" - "--drivername=$(DRIVER_NAME)" diff --git a/pkg/cephfs/driver.go b/pkg/cephfs/driver.go index 2b620b5a7..ffa95b859 100644 --- a/pkg/cephfs/driver.go +++ b/pkg/cephfs/driver.go @@ -147,20 +147,30 @@ func (fs *Driver) Run(conf *util.Config, cachePersister util.CachePersister) { klog.Fatalln("failed to initialize CSI driver") } - fs.cd.AddControllerServiceCapabilities([]csi.ControllerServiceCapability_RPC_Type{ - csi.ControllerServiceCapability_RPC_CREATE_DELETE_VOLUME, - }) - - fs.cd.AddVolumeCapabilityAccessModes([]csi.VolumeCapability_AccessMode_Mode{ - csi.VolumeCapability_AccessMode_MULTI_NODE_MULTI_WRITER, - }) + if conf.IsControllerServer || !conf.IsNodeServer { + fs.cd.AddControllerServiceCapabilities([]csi.ControllerServiceCapability_RPC_Type{ + csi.ControllerServiceCapability_RPC_CREATE_DELETE_VOLUME, + }) + fs.cd.AddVolumeCapabilityAccessModes([]csi.VolumeCapability_AccessMode_Mode{ + csi.VolumeCapability_AccessMode_MULTI_NODE_MULTI_WRITER, + }) + } // Create gRPC servers fs.is = NewIdentityServer(fs.cd) - fs.ns = NewNodeServer(fs.cd, conf.Vtype) - fs.cs = NewControllerServer(fs.cd, cachePersister) + if conf.IsNodeServer { + fs.ns = NewNodeServer(fs.cd, conf.Vtype) + } + + if conf.IsControllerServer { + fs.cs = NewControllerServer(fs.cd, cachePersister) + } + if !conf.IsControllerServer && !conf.IsNodeServer { + fs.ns = NewNodeServer(fs.cd, conf.Vtype) + fs.cs = NewControllerServer(fs.cd, cachePersister) + } server := csicommon.NewNonBlockingGRPCServer() server.Start(conf.Endpoint, fs.is, fs.cs, fs.ns) diff --git a/pkg/rbd/driver.go b/pkg/rbd/driver.go index 361c201fc..10da8efc2 100644 --- a/pkg/rbd/driver.go +++ b/pkg/rbd/driver.go @@ -121,28 +121,41 @@ func (r *Driver) Run(conf *util.Config, cachePersister util.CachePersister) { if r.cd == nil { klog.Fatalln("Failed to initialize CSI Driver.") } - r.cd.AddControllerServiceCapabilities([]csi.ControllerServiceCapability_RPC_Type{ - csi.ControllerServiceCapability_RPC_CREATE_DELETE_VOLUME, - csi.ControllerServiceCapability_RPC_CREATE_DELETE_SNAPSHOT, - csi.ControllerServiceCapability_RPC_CLONE_VOLUME, - }) - - // We only support the multi-writer option when using block, but it's a supported capability for the plugin in general - // In addition, we want to add the remaining modes like MULTI_NODE_READER_ONLY, - // MULTI_NODE_SINGLE_WRITER etc, but need to do some verification of RO modes first - // will work those as follow up features - r.cd.AddVolumeCapabilityAccessModes( - []csi.VolumeCapability_AccessMode_Mode{csi.VolumeCapability_AccessMode_SINGLE_NODE_WRITER, - csi.VolumeCapability_AccessMode_MULTI_NODE_MULTI_WRITER}) + if conf.IsControllerServer || !conf.IsNodeServer { + r.cd.AddControllerServiceCapabilities([]csi.ControllerServiceCapability_RPC_Type{ + csi.ControllerServiceCapability_RPC_CREATE_DELETE_VOLUME, + csi.ControllerServiceCapability_RPC_CREATE_DELETE_SNAPSHOT, + csi.ControllerServiceCapability_RPC_CLONE_VOLUME, + }) + // We only support the multi-writer option when using block, but it's a supported capability for the plugin in general + // In addition, we want to add the remaining modes like MULTI_NODE_READER_ONLY, + // MULTI_NODE_SINGLE_WRITER etc, but need to do some verification of RO modes first + // will work those as follow up features + r.cd.AddVolumeCapabilityAccessModes( + []csi.VolumeCapability_AccessMode_Mode{csi.VolumeCapability_AccessMode_SINGLE_NODE_WRITER, + csi.VolumeCapability_AccessMode_MULTI_NODE_MULTI_WRITER}) + } // Create GRPC servers r.ids = NewIdentityServer(r.cd) - r.ns, err = NewNodeServer(r.cd, conf.Containerized, conf.Vtype) - if err != nil { - klog.Fatalf("failed to start node server, err %v\n", err) + + if conf.IsNodeServer { + r.ns, err = NewNodeServer(r.cd, conf.Containerized, conf.Vtype) + if err != nil { + klog.Fatalf("failed to start node server, err %v\n", err) + } } - r.cs = NewControllerServer(r.cd, cachePersister) + if conf.IsControllerServer { + r.cs = NewControllerServer(r.cd, cachePersister) + } + if !conf.IsControllerServer && !conf.IsNodeServer { + r.ns, err = NewNodeServer(r.cd, conf.Containerized, conf.Vtype) + if err != nil { + klog.Fatalf("failed to start node server, err %v\n", err) + } + r.cs = NewControllerServer(r.cd, cachePersister) + } s := csicommon.NewNonBlockingGRPCServer() s.Start(conf.Endpoint, r.ids, r.cs, r.ns) diff --git a/pkg/util/util.go b/pkg/util/util.go index 58794a50a..69549003f 100644 --- a/pkg/util/util.go +++ b/pkg/util/util.go @@ -54,14 +54,16 @@ var ( // Config holds the parameters list which can be configured type Config struct { // common flags - Vtype string // driver type [rbd|cephfs|liveness] - Endpoint string // CSI endpoint - DriverName string // name of the driver - NodeID string // node id - InstanceID string // unique ID distinguishing this instance of Ceph CSI - MetadataStorage string // metadata persistence method [node|k8s_configmap] - PluginPath string // location of cephcsi plugin - PidLimit int // PID limit to configure through cgroups") + Vtype string // driver type [rbd|cephfs|liveness] + Endpoint string // CSI endpoint + DriverName string // name of the driver + NodeID string // node id + InstanceID string // unique ID distinguishing this instance of Ceph CSI + MetadataStorage string // metadata persistence method [node|k8s_configmap] + PluginPath string // location of cephcsi plugin + PidLimit int // PID limit to configure through cgroups") + IsControllerServer bool // if set to true start provisoner server + IsNodeServer bool // if set to true start node server // rbd related flags Containerized bool // whether run as containerized