Use Deployment with leader election instead of StatefulSet

Deployment behaves better when a node gets disconnected from the rest of
the cluster - new provisioner leader is elected in ~15 seconds, while
it may take up to 5 minutes for StatefulSet to start a new replica.

Refer: 52d1fbcf9d

Fixes: #335

Signed-off-by: Madhu Rajanna <madhupr007@gmail.com>

deploy/rbd/helm/templates/provisioner-deployment.yaml

@@ -1,5 +1,5 @@
-kind: StatefulSet
-apiVersion: apps/v1beta1
+kind: Deployment
+apiVersion: apps/v1
 metadata:
   name: {{ include "ceph-csi-rbd.provisioner.fullname" . }}
   labels:
@@ -9,7 +9,6 @@ metadata:
     release: {{ .Release.Name }}
     heritage: {{ .Release.Service }}
 spec:
-  serviceName: {{ include "ceph-csi-rbd.provisioner.fullname" . }}
   replicas: {{ .Values.provisioner.replicas }}
   selector:
     matchLabels:
@@ -32,6 +31,7 @@ spec:
           args:
             - "--csi-address=$(ADDRESS)"
             - "--v=5"
+            - "--leader-election-type=leases"
           env:
             - name: ADDRESS
               value: "{{ .Values.socketDir }}/{{ .Values.socketFile }}"

deploy/rbd/helm/templates/provisioner-role.yaml

@@ -1,4 +1,4 @@
-{{- if .Values.rbac.create -}} 
+{{- if .Values.rbac.create -}}
 kind: Role
 apiVersion: rbac.authorization.k8s.io/v1
 metadata:
@@ -13,4 +13,7 @@ rules:
   - apiGroups: [""]
     resources: ["configmaps"]
     verbs: ["get", "list", "watch", "create", "delete"]
+  - apiGroups: ["coordination.k8s.io"]
+    resources: ["leases"]
+    verbs: ["get", "watch", "list", "delete", "update", "create"]
 {{- end -}}

deploy/rbd/helm/templates/provisioner-service.yaml

@@ -1,18 +0,0 @@
-kind: Service
-apiVersion: v1
-metadata:
-  name: {{ include "ceph-csi-rbd.provisioner.fullname" . }}
-  labels:
-    app: {{ include "ceph-csi-rbd.name" . }}
-    chart: {{ include "ceph-csi-rbd.chart" . }}
-    component: {{ .Values.provisioner.name }}
-    release: {{ .Release.Name }}
-    heritage: {{ .Release.Service }}
-spec:
-  selector:
-    app: {{ include "ceph-csi-rbd.name" . }}
-    component: {{ .Values.provisioner.name }}
-    release: {{ .Release.Name }}
-  ports:
-    - name: dummy
-      port: 12345

deploy/rbd/helm/values.yaml

@@ -67,7 +67,7 @@ nodeplugin:
 provisioner:
   name: provisioner
 
-  replicaCount: 1
+  replicaCount: 3
 
   image:
     repository: quay.io/k8scsi/csi-provisioner

deploy/rbd/kubernetes/csi-provisioner-rbac.yaml

@@ -87,6 +87,9 @@ rules:
   - apiGroups: [""]
     resources: ["configmaps"]
     verbs: ["get", "list", "watch", "create", "delete"]
+  - apiGroups: ["coordination.k8s.io"]
+    resources: ["leases"]
+    verbs: ["get", "watch", "list", "delete", "update", "create"]
 
 ---
 kind: RoleBinding

deploy/rbd/kubernetes/csi-rbdplugin-provisioner.yaml

@@ -1,25 +1,13 @@
 ---
-kind: Service
-apiVersion: v1
-metadata:
-  name: csi-rbdplugin-provisioner
-  labels:
-    app: csi-rbdplugin-provisioner
-spec:
-  selector:
-    app: csi-rbdplugin-provisioner
-  ports:
-    - name: dummy
-      port: 12345
-
----
-kind: StatefulSet
+kind: Deployment
 apiVersion: apps/v1beta1
 metadata:
   name: csi-rbdplugin-provisioner
 spec:
-  serviceName: "csi-rbdplugin-provisioner"
-  replicas: 1
+  replicas: 3
+  selector:
+    matchLabels:
+      app: csi-rbdplugin-provisioner
   template:
     metadata:
       labels:
@@ -32,6 +20,7 @@ spec:
           args:
             - "--csi-address=$(ADDRESS)"
             - "--v=5"
+            - --leader-election-type=leases"
           env:
             - name: ADDRESS
               value: unix:///csi/csi-provisioner.sock