Use Deployment with leader election instead of StatefulSet

Deployment behaves better when a node gets disconnected from the rest of
the cluster - new provisioner leader is elected in ~15 seconds, while
it may take up to 5 minutes for StatefulSet to start a new replica.

Refer: 52d1fbcf9d

Fixes: #335

Signed-off-by: Madhu Rajanna <madhupr007@gmail.com>
This commit is contained in:
Madhu Rajanna 2019-06-10 09:38:06 +05:30
parent 0e61098522
commit a151bec94b
14 changed files with 38 additions and 90 deletions

View File

@ -1,5 +1,5 @@
kind: StatefulSet
apiVersion: apps/v1beta1
kind: Deployment
apiVersion: apps/v1
metadata:
name: {{ include "ceph-csi-cephfs.provisioner.fullname" . }}
labels:
@ -9,7 +9,6 @@ metadata:
release: {{ .Release.Name }}
heritage: {{ .Release.Service }}
spec:
serviceName: {{ include "ceph-csi-cephfs.provisioner.fullname" . }}
replicas: {{ .Values.provisioner.replicas }}
selector:
matchLabels:
@ -31,6 +30,7 @@ spec:
image: "{{ .Values.provisioner.image.repository }}:{{ .Values.provisioner.image.tag }}"
args:
- "--csi-address=$(ADDRESS)"
- "--leader-election-type=leases"
- "--v=5"
env:
- name: ADDRESS

View File

@ -1,4 +1,4 @@
{{- if .Values.rbac.create -}}
{{- if .Values.rbac.create -}}
kind: Role
apiVersion: rbac.authorization.k8s.io/v1
metadata:
@ -16,4 +16,7 @@ rules:
- apiGroups: [""]
resources: ["configmaps"]
verbs: ["get", "list", "watch", "create", "delete"]
- apiGroups: ["coordination.k8s.io"]
resources: ["leases"]
verbs: ["get", "watch", "list", "delete", "update", "create"]
{{- end -}}

View File

@ -1,18 +0,0 @@
kind: Service
apiVersion: v1
metadata:
name: {{ include "ceph-csi-cephfs.provisioner.fullname" . }}
labels:
app: {{ include "ceph-csi-cephfs.name" . }}
chart: {{ include "ceph-csi-cephfs.chart" . }}
component: {{ .Values.provisioner.name }}
release: {{ .Release.Name }}
heritage: {{ .Release.Service }}
spec:
selector:
app: {{ include "ceph-csi-cephfs.name" . }}
component: {{ .Values.provisioner.name }}
release: {{ .Release.Name }}
ports:
- name: dummy
port: 12345

View File

@ -66,7 +66,7 @@ nodeplugin:
provisioner:
name: provisioner
replicaCount: 1
replicaCount: 3
image:
repository: quay.io/k8scsi/csi-provisioner

View File

@ -1,25 +1,13 @@
---
kind: Service
apiVersion: v1
kind: Deployment
apiVersion: apps/v1
metadata:
name: csi-cephfsplugin-provisioner
labels:
app: csi-cephfsplugin-provisioner
spec:
replicas: 3
selector:
app: csi-cephfsplugin-provisioner
ports:
- name: dummy
port: 12345
---
kind: StatefulSet
apiVersion: apps/v1beta1
metadata:
name: csi-cephfsplugin-provisioner
spec:
serviceName: "csi-cephfsplugin-provisioner"
replicas: 1
matchLabels:
app: csi-cephfsplugin-provisioner
template:
metadata:
labels:
@ -32,6 +20,7 @@ spec:
args:
- "--csi-address=$(ADDRESS)"
- "--v=5"
- "--leader-election-type=leases"
env:
- name: ADDRESS
value: unix:///csi/csi-provisioner.sock

View File

@ -75,6 +75,9 @@ rules:
- apiGroups: [""]
resources: ["configmaps"]
verbs: ["get", "list", "create", "delete"]
- apiGroups: ["coordination.k8s.io"]
resources: ["leases"]
verbs: ["get", "watch", "list", "delete", "update", "create"]
---
kind: RoleBinding

View File

@ -1,5 +1,5 @@
kind: StatefulSet
apiVersion: apps/v1beta1
kind: Deployment
apiVersion: apps/v1
metadata:
name: {{ include "ceph-csi-rbd.provisioner.fullname" . }}
labels:
@ -9,7 +9,6 @@ metadata:
release: {{ .Release.Name }}
heritage: {{ .Release.Service }}
spec:
serviceName: {{ include "ceph-csi-rbd.provisioner.fullname" . }}
replicas: {{ .Values.provisioner.replicas }}
selector:
matchLabels:
@ -32,6 +31,7 @@ spec:
args:
- "--csi-address=$(ADDRESS)"
- "--v=5"
- "--leader-election-type=leases"
env:
- name: ADDRESS
value: "{{ .Values.socketDir }}/{{ .Values.socketFile }}"

View File

@ -1,4 +1,4 @@
{{- if .Values.rbac.create -}}
{{- if .Values.rbac.create -}}
kind: Role
apiVersion: rbac.authorization.k8s.io/v1
metadata:
@ -13,4 +13,7 @@ rules:
- apiGroups: [""]
resources: ["configmaps"]
verbs: ["get", "list", "watch", "create", "delete"]
- apiGroups: ["coordination.k8s.io"]
resources: ["leases"]
verbs: ["get", "watch", "list", "delete", "update", "create"]
{{- end -}}

View File

@ -1,18 +0,0 @@
kind: Service
apiVersion: v1
metadata:
name: {{ include "ceph-csi-rbd.provisioner.fullname" . }}
labels:
app: {{ include "ceph-csi-rbd.name" . }}
chart: {{ include "ceph-csi-rbd.chart" . }}
component: {{ .Values.provisioner.name }}
release: {{ .Release.Name }}
heritage: {{ .Release.Service }}
spec:
selector:
app: {{ include "ceph-csi-rbd.name" . }}
component: {{ .Values.provisioner.name }}
release: {{ .Release.Name }}
ports:
- name: dummy
port: 12345

View File

@ -67,7 +67,7 @@ nodeplugin:
provisioner:
name: provisioner
replicaCount: 1
replicaCount: 3
image:
repository: quay.io/k8scsi/csi-provisioner

View File

@ -87,6 +87,9 @@ rules:
- apiGroups: [""]
resources: ["configmaps"]
verbs: ["get", "list", "watch", "create", "delete"]
- apiGroups: ["coordination.k8s.io"]
resources: ["leases"]
verbs: ["get", "watch", "list", "delete", "update", "create"]
---
kind: RoleBinding

View File

@ -1,25 +1,13 @@
---
kind: Service
apiVersion: v1
metadata:
name: csi-rbdplugin-provisioner
labels:
app: csi-rbdplugin-provisioner
spec:
selector:
app: csi-rbdplugin-provisioner
ports:
- name: dummy
port: 12345
---
kind: StatefulSet
kind: Deployment
apiVersion: apps/v1beta1
metadata:
name: csi-rbdplugin-provisioner
spec:
serviceName: "csi-rbdplugin-provisioner"
replicas: 1
replicas: 3
selector:
matchLabels:
app: csi-rbdplugin-provisioner
template:
metadata:
labels:
@ -32,6 +20,7 @@ spec:
args:
- "--csi-address=$(ADDRESS)"
- "--v=5"
- --leader-election-type=leases"
env:
- name: ADDRESS
value: unix:///csi/csi-provisioner.sock

View File

@ -1,8 +1,6 @@
package e2e
import (
"time"
. "github.com/onsi/ginkgo" // nolint
"k8s.io/kubernetes/test/e2e/framework"
@ -54,9 +52,8 @@ var _ = Describe("cephfs", func() {
Context("Test cephfs CSI", func() {
It("Test cephfs CSI", func() {
By("checking provisioner statefulset is running")
timeout := time.Duration(deployTimeout) * time.Minute
err := framework.WaitForStatefulSetReplicasReady("csi-cephfsplugin-provisioner", "default", f.ClientSet, 1*time.Second, timeout)
By("checking provisioner deployment is completed")
err := waitForDeploymentComplete("csi-cephfsplugin-provisioner", "default", f.ClientSet, deployTimeout)
if err != nil {
Fail(err.Error())
}

View File

@ -1,8 +1,6 @@
package e2e
import (
"time"
. "github.com/onsi/ginkgo" // nolint
"k8s.io/kubernetes/test/e2e/framework"
@ -55,9 +53,8 @@ var _ = Describe("RBD", func() {
Context("Test RBD CSI", func() {
It("Test RBD CSI", func() {
By("checking provisioner statefulset is running")
timeout := time.Duration(deployTimeout) * time.Minute
err := framework.WaitForStatefulSetReplicasReady("csi-rbdplugin-provisioner", "default", f.ClientSet, 1*time.Second, timeout)
By("checking provisioner deployment is completed")
err := waitForDeploymentComplete("csi-rbdplugin-provisioner", "default", f.ClientSet, deployTimeout)
if err != nil {
Fail(err.Error())
}