Use Deployment with leader election instead of StatefulSet

Deployment behaves better when a node gets disconnected from the rest of
the cluster - new provisioner leader is elected in ~15 seconds, while
it may take up to 5 minutes for StatefulSet to start a new replica.

Refer: 52d1fbcf9d

Fixes: #335

Signed-off-by: Madhu Rajanna <madhupr007@gmail.com>

deploy/cephfs/helm/templates/provisioner-deployment.yaml

@@ -1,5 +1,5 @@
-kind: StatefulSet
-apiVersion: apps/v1beta1
+kind: Deployment
+apiVersion: apps/v1
 metadata:
   name: {{ include "ceph-csi-cephfs.provisioner.fullname" . }}
   labels:
@@ -9,7 +9,6 @@ metadata:
     release: {{ .Release.Name }}
     heritage: {{ .Release.Service }}
 spec:
-  serviceName: {{ include "ceph-csi-cephfs.provisioner.fullname" . }}
   replicas: {{ .Values.provisioner.replicas }}
   selector:
     matchLabels:
@@ -31,6 +30,7 @@ spec:
           image: "{{ .Values.provisioner.image.repository }}:{{ .Values.provisioner.image.tag }}"
           args:
             - "--csi-address=$(ADDRESS)"
+            - "--leader-election-type=leases"
             - "--v=5"
           env:
             - name: ADDRESS

deploy/cephfs/helm/templates/provisioner-role.yaml

@@ -16,4 +16,7 @@ rules:
   - apiGroups: [""]
     resources: ["configmaps"]
     verbs: ["get", "list", "watch", "create", "delete"]
+  - apiGroups: ["coordination.k8s.io"]
+    resources: ["leases"]
+    verbs: ["get", "watch", "list", "delete", "update", "create"]
 {{- end -}}

deploy/cephfs/helm/templates/provisioner-service.yaml

@@ -1,18 +0,0 @@
-kind: Service
-apiVersion: v1
-metadata:
-  name: {{ include "ceph-csi-cephfs.provisioner.fullname" . }}
-  labels:
-    app: {{ include "ceph-csi-cephfs.name" . }}
-    chart: {{ include "ceph-csi-cephfs.chart" . }}
-    component: {{ .Values.provisioner.name }}
-    release: {{ .Release.Name }}
-    heritage: {{ .Release.Service }}
-spec:
-  selector:
-    app: {{ include "ceph-csi-cephfs.name" . }}
-    component: {{ .Values.provisioner.name }}
-    release: {{ .Release.Name }}
-  ports:
-    - name: dummy
-      port: 12345

deploy/cephfs/helm/values.yaml

@@ -66,7 +66,7 @@ nodeplugin:
 provisioner:
   name: provisioner
 
-  replicaCount: 1
+  replicaCount: 3
 
   image:
     repository: quay.io/k8scsi/csi-provisioner

deploy/cephfs/kubernetes/csi-cephfsplugin-provisioner.yaml

@@ -1,25 +1,13 @@
 ---
-kind: Service
-apiVersion: v1
+kind: Deployment
+apiVersion: apps/v1
 metadata:
   name: csi-cephfsplugin-provisioner
-  labels:
-    app: csi-cephfsplugin-provisioner
 spec:
+  replicas: 3
   selector:
+    matchLabels:
       app: csi-cephfsplugin-provisioner
-  ports:
-    - name: dummy
-      port: 12345
-
----
-kind: StatefulSet
-apiVersion: apps/v1beta1
-metadata:
-  name: csi-cephfsplugin-provisioner
-spec:
-  serviceName: "csi-cephfsplugin-provisioner"
-  replicas: 1
   template:
     metadata:
       labels:
@@ -32,6 +20,7 @@ spec:
           args:
             - "--csi-address=$(ADDRESS)"
             - "--v=5"
+            - "--leader-election-type=leases"
           env:
             - name: ADDRESS
               value: unix:///csi/csi-provisioner.sock

deploy/cephfs/kubernetes/csi-provisioner-rbac.yaml

@@ -75,6 +75,9 @@ rules:
   - apiGroups: [""]
     resources: ["configmaps"]
     verbs: ["get", "list", "create", "delete"]
+  - apiGroups: ["coordination.k8s.io"]
+    resources: ["leases"]
+    verbs: ["get", "watch", "list", "delete", "update", "create"]
 
 ---
 kind: RoleBinding

deploy/rbd/helm/templates/provisioner-deployment.yaml

@@ -1,5 +1,5 @@
-kind: StatefulSet
-apiVersion: apps/v1beta1
+kind: Deployment
+apiVersion: apps/v1
 metadata:
   name: {{ include "ceph-csi-rbd.provisioner.fullname" . }}
   labels:
@@ -9,7 +9,6 @@ metadata:
     release: {{ .Release.Name }}
     heritage: {{ .Release.Service }}
 spec:
-  serviceName: {{ include "ceph-csi-rbd.provisioner.fullname" . }}
   replicas: {{ .Values.provisioner.replicas }}
   selector:
     matchLabels:
@@ -32,6 +31,7 @@ spec:
           args:
             - "--csi-address=$(ADDRESS)"
             - "--v=5"
+            - "--leader-election-type=leases"
           env:
             - name: ADDRESS
               value: "{{ .Values.socketDir }}/{{ .Values.socketFile }}"

deploy/rbd/helm/templates/provisioner-role.yaml

@@ -13,4 +13,7 @@ rules:
   - apiGroups: [""]
     resources: ["configmaps"]
     verbs: ["get", "list", "watch", "create", "delete"]
+  - apiGroups: ["coordination.k8s.io"]
+    resources: ["leases"]
+    verbs: ["get", "watch", "list", "delete", "update", "create"]
 {{- end -}}

deploy/rbd/helm/templates/provisioner-service.yaml

@@ -1,18 +0,0 @@
-kind: Service
-apiVersion: v1
-metadata:
-  name: {{ include "ceph-csi-rbd.provisioner.fullname" . }}
-  labels:
-    app: {{ include "ceph-csi-rbd.name" . }}
-    chart: {{ include "ceph-csi-rbd.chart" . }}
-    component: {{ .Values.provisioner.name }}
-    release: {{ .Release.Name }}
-    heritage: {{ .Release.Service }}
-spec:
-  selector:
-    app: {{ include "ceph-csi-rbd.name" . }}
-    component: {{ .Values.provisioner.name }}
-    release: {{ .Release.Name }}
-  ports:
-    - name: dummy
-      port: 12345

deploy/rbd/helm/values.yaml

@@ -67,7 +67,7 @@ nodeplugin:
 provisioner:
   name: provisioner
 
-  replicaCount: 1
+  replicaCount: 3
 
   image:
     repository: quay.io/k8scsi/csi-provisioner

deploy/rbd/kubernetes/csi-provisioner-rbac.yaml

@@ -87,6 +87,9 @@ rules:
   - apiGroups: [""]
     resources: ["configmaps"]
     verbs: ["get", "list", "watch", "create", "delete"]
+  - apiGroups: ["coordination.k8s.io"]
+    resources: ["leases"]
+    verbs: ["get", "watch", "list", "delete", "update", "create"]
 
 ---
 kind: RoleBinding

deploy/rbd/kubernetes/csi-rbdplugin-provisioner.yaml

@@ -1,25 +1,13 @@
 ---
-kind: Service
-apiVersion: v1
-metadata:
-  name: csi-rbdplugin-provisioner
-  labels:
-    app: csi-rbdplugin-provisioner
-spec:
-  selector:
-    app: csi-rbdplugin-provisioner
-  ports:
-    - name: dummy
-      port: 12345
-
----
-kind: StatefulSet
+kind: Deployment
 apiVersion: apps/v1beta1
 metadata:
   name: csi-rbdplugin-provisioner
 spec:
-  serviceName: "csi-rbdplugin-provisioner"
-  replicas: 1
+  replicas: 3
+  selector:
+    matchLabels:
+      app: csi-rbdplugin-provisioner
   template:
     metadata:
       labels:
@@ -32,6 +20,7 @@ spec:
           args:
             - "--csi-address=$(ADDRESS)"
             - "--v=5"
+            - --leader-election-type=leases"
           env:
             - name: ADDRESS
               value: unix:///csi/csi-provisioner.sock

e2e/cephfs.go

@@ -1,8 +1,6 @@
 package e2e
 
 import (
-	"time"
-
 	. "github.com/onsi/ginkgo" // nolint
 
 	"k8s.io/kubernetes/test/e2e/framework"
@@ -54,9 +52,8 @@ var _ = Describe("cephfs", func() {
 
 	Context("Test cephfs CSI", func() {
 		It("Test cephfs CSI", func() {
-			By("checking provisioner statefulset is running")
-			timeout := time.Duration(deployTimeout) * time.Minute
-			err := framework.WaitForStatefulSetReplicasReady("csi-cephfsplugin-provisioner", "default", f.ClientSet, 1*time.Second, timeout)
+			By("checking provisioner deployment is completed")
+			err := waitForDeploymentComplete("csi-cephfsplugin-provisioner", "default", f.ClientSet, deployTimeout)
 			if err != nil {
 				Fail(err.Error())
 			}

e2e/rbd.go

@@ -1,8 +1,6 @@
 package e2e
 
 import (
-	"time"
-
 	. "github.com/onsi/ginkgo" // nolint
 
 	"k8s.io/kubernetes/test/e2e/framework"
@@ -55,9 +53,8 @@ var _ = Describe("RBD", func() {
 
 	Context("Test RBD CSI", func() {
 		It("Test RBD CSI", func() {
-			By("checking provisioner statefulset is running")
-			timeout := time.Duration(deployTimeout) * time.Minute
-			err := framework.WaitForStatefulSetReplicasReady("csi-rbdplugin-provisioner", "default", f.ClientSet, 1*time.Second, timeout)
+			By("checking provisioner deployment is completed")
+			err := waitForDeploymentComplete("csi-rbdplugin-provisioner", "default", f.ClientSet, deployTimeout)
 			if err != nil {
 				Fail(err.Error())
 			}