From a151bec94b93f0f8ddbe50b572bcbe94e4c344ae Mon Sep 17 00:00:00 2001 From: Madhu Rajanna Date: Mon, 10 Jun 2019 09:38:06 +0530 Subject: [PATCH] Use Deployment with leader election instead of StatefulSet Deployment behaves better when a node gets disconnected from the rest of the cluster - new provisioner leader is elected in ~15 seconds, while it may take up to 5 minutes for StatefulSet to start a new replica. Refer: https://github.com/kubernetes-csi/external-provisioner/commit/52d1fbcf9d52d35053164bb0409aa97dd3e7777a Fixes: #335 Signed-off-by: Madhu Rajanna --- ...ulset.yaml => provisioner-deployment.yaml} | 6 ++--- .../helm/templates/provisioner-role.yaml | 5 +++- .../helm/templates/provisioner-service.yaml | 18 --------------- deploy/cephfs/helm/values.yaml | 2 +- .../csi-cephfsplugin-provisioner.yaml | 23 +++++-------------- .../kubernetes/csi-provisioner-rbac.yaml | 3 +++ ...ulset.yaml => provisioner-deployment.yaml} | 6 ++--- .../rbd/helm/templates/provisioner-role.yaml | 5 +++- .../helm/templates/provisioner-service.yaml | 18 --------------- deploy/rbd/helm/values.yaml | 2 +- .../rbd/kubernetes/csi-provisioner-rbac.yaml | 3 +++ .../kubernetes/csi-rbdplugin-provisioner.yaml | 23 +++++-------------- e2e/cephfs.go | 7 ++---- e2e/rbd.go | 7 ++---- 14 files changed, 38 insertions(+), 90 deletions(-) rename deploy/cephfs/helm/templates/{provisioner-statefulset.yaml => provisioner-deployment.yaml} (97%) delete mode 100644 deploy/cephfs/helm/templates/provisioner-service.yaml rename deploy/rbd/helm/templates/{provisioner-statefulset.yaml => provisioner-deployment.yaml} (97%) delete mode 100644 deploy/rbd/helm/templates/provisioner-service.yaml diff --git a/deploy/cephfs/helm/templates/provisioner-statefulset.yaml b/deploy/cephfs/helm/templates/provisioner-deployment.yaml similarity index 97% rename from deploy/cephfs/helm/templates/provisioner-statefulset.yaml rename to deploy/cephfs/helm/templates/provisioner-deployment.yaml index cc073df49..1a933232a 100644 --- a/deploy/cephfs/helm/templates/provisioner-statefulset.yaml +++ b/deploy/cephfs/helm/templates/provisioner-deployment.yaml @@ -1,5 +1,5 @@ -kind: StatefulSet -apiVersion: apps/v1beta1 +kind: Deployment +apiVersion: apps/v1 metadata: name: {{ include "ceph-csi-cephfs.provisioner.fullname" . }} labels: @@ -9,7 +9,6 @@ metadata: release: {{ .Release.Name }} heritage: {{ .Release.Service }} spec: - serviceName: {{ include "ceph-csi-cephfs.provisioner.fullname" . }} replicas: {{ .Values.provisioner.replicas }} selector: matchLabels: @@ -31,6 +30,7 @@ spec: image: "{{ .Values.provisioner.image.repository }}:{{ .Values.provisioner.image.tag }}" args: - "--csi-address=$(ADDRESS)" + - "--leader-election-type=leases" - "--v=5" env: - name: ADDRESS diff --git a/deploy/cephfs/helm/templates/provisioner-role.yaml b/deploy/cephfs/helm/templates/provisioner-role.yaml index c6f28c40e..dc572ed24 100644 --- a/deploy/cephfs/helm/templates/provisioner-role.yaml +++ b/deploy/cephfs/helm/templates/provisioner-role.yaml @@ -1,4 +1,4 @@ -{{- if .Values.rbac.create -}} +{{- if .Values.rbac.create -}} kind: Role apiVersion: rbac.authorization.k8s.io/v1 metadata: @@ -16,4 +16,7 @@ rules: - apiGroups: [""] resources: ["configmaps"] verbs: ["get", "list", "watch", "create", "delete"] + - apiGroups: ["coordination.k8s.io"] + resources: ["leases"] + verbs: ["get", "watch", "list", "delete", "update", "create"] {{- end -}} diff --git a/deploy/cephfs/helm/templates/provisioner-service.yaml b/deploy/cephfs/helm/templates/provisioner-service.yaml deleted file mode 100644 index 93d62ffb7..000000000 --- a/deploy/cephfs/helm/templates/provisioner-service.yaml +++ /dev/null @@ -1,18 +0,0 @@ -kind: Service -apiVersion: v1 -metadata: - name: {{ include "ceph-csi-cephfs.provisioner.fullname" . }} - labels: - app: {{ include "ceph-csi-cephfs.name" . }} - chart: {{ include "ceph-csi-cephfs.chart" . }} - component: {{ .Values.provisioner.name }} - release: {{ .Release.Name }} - heritage: {{ .Release.Service }} -spec: - selector: - app: {{ include "ceph-csi-cephfs.name" . }} - component: {{ .Values.provisioner.name }} - release: {{ .Release.Name }} - ports: - - name: dummy - port: 12345 diff --git a/deploy/cephfs/helm/values.yaml b/deploy/cephfs/helm/values.yaml index 7b59410c3..f67787577 100644 --- a/deploy/cephfs/helm/values.yaml +++ b/deploy/cephfs/helm/values.yaml @@ -66,7 +66,7 @@ nodeplugin: provisioner: name: provisioner - replicaCount: 1 + replicaCount: 3 image: repository: quay.io/k8scsi/csi-provisioner diff --git a/deploy/cephfs/kubernetes/csi-cephfsplugin-provisioner.yaml b/deploy/cephfs/kubernetes/csi-cephfsplugin-provisioner.yaml index d4732358f..eedae2bc0 100644 --- a/deploy/cephfs/kubernetes/csi-cephfsplugin-provisioner.yaml +++ b/deploy/cephfs/kubernetes/csi-cephfsplugin-provisioner.yaml @@ -1,25 +1,13 @@ --- -kind: Service -apiVersion: v1 +kind: Deployment +apiVersion: apps/v1 metadata: name: csi-cephfsplugin-provisioner - labels: - app: csi-cephfsplugin-provisioner spec: + replicas: 3 selector: - app: csi-cephfsplugin-provisioner - ports: - - name: dummy - port: 12345 - ---- -kind: StatefulSet -apiVersion: apps/v1beta1 -metadata: - name: csi-cephfsplugin-provisioner -spec: - serviceName: "csi-cephfsplugin-provisioner" - replicas: 1 + matchLabels: + app: csi-cephfsplugin-provisioner template: metadata: labels: @@ -32,6 +20,7 @@ spec: args: - "--csi-address=$(ADDRESS)" - "--v=5" + - "--leader-election-type=leases" env: - name: ADDRESS value: unix:///csi/csi-provisioner.sock diff --git a/deploy/cephfs/kubernetes/csi-provisioner-rbac.yaml b/deploy/cephfs/kubernetes/csi-provisioner-rbac.yaml index dc67c1b39..4cbd8cad2 100644 --- a/deploy/cephfs/kubernetes/csi-provisioner-rbac.yaml +++ b/deploy/cephfs/kubernetes/csi-provisioner-rbac.yaml @@ -75,6 +75,9 @@ rules: - apiGroups: [""] resources: ["configmaps"] verbs: ["get", "list", "create", "delete"] + - apiGroups: ["coordination.k8s.io"] + resources: ["leases"] + verbs: ["get", "watch", "list", "delete", "update", "create"] --- kind: RoleBinding diff --git a/deploy/rbd/helm/templates/provisioner-statefulset.yaml b/deploy/rbd/helm/templates/provisioner-deployment.yaml similarity index 97% rename from deploy/rbd/helm/templates/provisioner-statefulset.yaml rename to deploy/rbd/helm/templates/provisioner-deployment.yaml index 12751b90d..e9034f131 100644 --- a/deploy/rbd/helm/templates/provisioner-statefulset.yaml +++ b/deploy/rbd/helm/templates/provisioner-deployment.yaml @@ -1,5 +1,5 @@ -kind: StatefulSet -apiVersion: apps/v1beta1 +kind: Deployment +apiVersion: apps/v1 metadata: name: {{ include "ceph-csi-rbd.provisioner.fullname" . }} labels: @@ -9,7 +9,6 @@ metadata: release: {{ .Release.Name }} heritage: {{ .Release.Service }} spec: - serviceName: {{ include "ceph-csi-rbd.provisioner.fullname" . }} replicas: {{ .Values.provisioner.replicas }} selector: matchLabels: @@ -32,6 +31,7 @@ spec: args: - "--csi-address=$(ADDRESS)" - "--v=5" + - "--leader-election-type=leases" env: - name: ADDRESS value: "{{ .Values.socketDir }}/{{ .Values.socketFile }}" diff --git a/deploy/rbd/helm/templates/provisioner-role.yaml b/deploy/rbd/helm/templates/provisioner-role.yaml index 798174985..6294b4a27 100644 --- a/deploy/rbd/helm/templates/provisioner-role.yaml +++ b/deploy/rbd/helm/templates/provisioner-role.yaml @@ -1,4 +1,4 @@ -{{- if .Values.rbac.create -}} +{{- if .Values.rbac.create -}} kind: Role apiVersion: rbac.authorization.k8s.io/v1 metadata: @@ -13,4 +13,7 @@ rules: - apiGroups: [""] resources: ["configmaps"] verbs: ["get", "list", "watch", "create", "delete"] + - apiGroups: ["coordination.k8s.io"] + resources: ["leases"] + verbs: ["get", "watch", "list", "delete", "update", "create"] {{- end -}} diff --git a/deploy/rbd/helm/templates/provisioner-service.yaml b/deploy/rbd/helm/templates/provisioner-service.yaml deleted file mode 100644 index 9bbcfcf1f..000000000 --- a/deploy/rbd/helm/templates/provisioner-service.yaml +++ /dev/null @@ -1,18 +0,0 @@ -kind: Service -apiVersion: v1 -metadata: - name: {{ include "ceph-csi-rbd.provisioner.fullname" . }} - labels: - app: {{ include "ceph-csi-rbd.name" . }} - chart: {{ include "ceph-csi-rbd.chart" . }} - component: {{ .Values.provisioner.name }} - release: {{ .Release.Name }} - heritage: {{ .Release.Service }} -spec: - selector: - app: {{ include "ceph-csi-rbd.name" . }} - component: {{ .Values.provisioner.name }} - release: {{ .Release.Name }} - ports: - - name: dummy - port: 12345 diff --git a/deploy/rbd/helm/values.yaml b/deploy/rbd/helm/values.yaml index f7c5b33ff..9ba9f81ec 100644 --- a/deploy/rbd/helm/values.yaml +++ b/deploy/rbd/helm/values.yaml @@ -67,7 +67,7 @@ nodeplugin: provisioner: name: provisioner - replicaCount: 1 + replicaCount: 3 image: repository: quay.io/k8scsi/csi-provisioner diff --git a/deploy/rbd/kubernetes/csi-provisioner-rbac.yaml b/deploy/rbd/kubernetes/csi-provisioner-rbac.yaml index 5fad126c9..d665bd7cf 100644 --- a/deploy/rbd/kubernetes/csi-provisioner-rbac.yaml +++ b/deploy/rbd/kubernetes/csi-provisioner-rbac.yaml @@ -87,6 +87,9 @@ rules: - apiGroups: [""] resources: ["configmaps"] verbs: ["get", "list", "watch", "create", "delete"] + - apiGroups: ["coordination.k8s.io"] + resources: ["leases"] + verbs: ["get", "watch", "list", "delete", "update", "create"] --- kind: RoleBinding diff --git a/deploy/rbd/kubernetes/csi-rbdplugin-provisioner.yaml b/deploy/rbd/kubernetes/csi-rbdplugin-provisioner.yaml index 782b2a99f..9470f1b72 100644 --- a/deploy/rbd/kubernetes/csi-rbdplugin-provisioner.yaml +++ b/deploy/rbd/kubernetes/csi-rbdplugin-provisioner.yaml @@ -1,25 +1,13 @@ --- -kind: Service -apiVersion: v1 -metadata: - name: csi-rbdplugin-provisioner - labels: - app: csi-rbdplugin-provisioner -spec: - selector: - app: csi-rbdplugin-provisioner - ports: - - name: dummy - port: 12345 - ---- -kind: StatefulSet +kind: Deployment apiVersion: apps/v1beta1 metadata: name: csi-rbdplugin-provisioner spec: - serviceName: "csi-rbdplugin-provisioner" - replicas: 1 + replicas: 3 + selector: + matchLabels: + app: csi-rbdplugin-provisioner template: metadata: labels: @@ -32,6 +20,7 @@ spec: args: - "--csi-address=$(ADDRESS)" - "--v=5" + - --leader-election-type=leases" env: - name: ADDRESS value: unix:///csi/csi-provisioner.sock diff --git a/e2e/cephfs.go b/e2e/cephfs.go index 324529b58..1429e0d96 100644 --- a/e2e/cephfs.go +++ b/e2e/cephfs.go @@ -1,8 +1,6 @@ package e2e import ( - "time" - . "github.com/onsi/ginkgo" // nolint "k8s.io/kubernetes/test/e2e/framework" @@ -54,9 +52,8 @@ var _ = Describe("cephfs", func() { Context("Test cephfs CSI", func() { It("Test cephfs CSI", func() { - By("checking provisioner statefulset is running") - timeout := time.Duration(deployTimeout) * time.Minute - err := framework.WaitForStatefulSetReplicasReady("csi-cephfsplugin-provisioner", "default", f.ClientSet, 1*time.Second, timeout) + By("checking provisioner deployment is completed") + err := waitForDeploymentComplete("csi-cephfsplugin-provisioner", "default", f.ClientSet, deployTimeout) if err != nil { Fail(err.Error()) } diff --git a/e2e/rbd.go b/e2e/rbd.go index 5d01e2769..5a8fe5a01 100644 --- a/e2e/rbd.go +++ b/e2e/rbd.go @@ -1,8 +1,6 @@ package e2e import ( - "time" - . "github.com/onsi/ginkgo" // nolint "k8s.io/kubernetes/test/e2e/framework" @@ -55,9 +53,8 @@ var _ = Describe("RBD", func() { Context("Test RBD CSI", func() { It("Test RBD CSI", func() { - By("checking provisioner statefulset is running") - timeout := time.Duration(deployTimeout) * time.Minute - err := framework.WaitForStatefulSetReplicasReady("csi-rbdplugin-provisioner", "default", f.ClientSet, 1*time.Second, timeout) + By("checking provisioner deployment is completed") + err := waitForDeploymentComplete("csi-rbdplugin-provisioner", "default", f.ClientSet, deployTimeout) if err != nil { Fail(err.Error()) }