2018-12-10 19:25:34 +00:00
|
|
|
kind: DaemonSet
|
2019-06-19 09:05:44 +00:00
|
|
|
apiVersion: apps/v1
|
2018-12-10 19:25:34 +00:00
|
|
|
metadata:
|
|
|
|
name: {{ include "ceph-csi-rbd.nodeplugin.fullname" . }}
|
2019-07-29 10:36:24 +00:00
|
|
|
namespace: {{ .Release.Namespace }}
|
2018-12-10 19:25:34 +00:00
|
|
|
labels:
|
|
|
|
app: {{ include "ceph-csi-rbd.name" . }}
|
|
|
|
chart: {{ include "ceph-csi-rbd.chart" . }}
|
|
|
|
component: {{ .Values.nodeplugin.name }}
|
|
|
|
release: {{ .Release.Name }}
|
|
|
|
heritage: {{ .Release.Service }}
|
|
|
|
spec:
|
|
|
|
selector:
|
|
|
|
matchLabels:
|
|
|
|
app: {{ include "ceph-csi-rbd.name" . }}
|
|
|
|
component: {{ .Values.nodeplugin.name }}
|
|
|
|
release: {{ .Release.Name }}
|
2019-07-03 10:02:36 +00:00
|
|
|
updateStrategy:
|
2020-01-02 08:48:30 +00:00
|
|
|
type: {{ .Values.nodeplugin.updateStrategy }}
|
2018-12-10 19:25:34 +00:00
|
|
|
template:
|
|
|
|
metadata:
|
|
|
|
labels:
|
|
|
|
app: {{ include "ceph-csi-rbd.name" . }}
|
|
|
|
chart: {{ include "ceph-csi-rbd.chart" . }}
|
|
|
|
component: {{ .Values.nodeplugin.name }}
|
|
|
|
release: {{ .Release.Name }}
|
|
|
|
heritage: {{ .Release.Service }}
|
|
|
|
spec:
|
|
|
|
serviceAccountName: {{ include "ceph-csi-rbd.serviceAccountName.nodeplugin" . }}
|
|
|
|
hostNetwork: true
|
2019-04-22 21:35:39 +00:00
|
|
|
hostPID: true
|
2020-05-05 08:27:38 +00:00
|
|
|
{{- if .Values.nodeplugin.priorityClassName }}
|
|
|
|
priorityClassName: {{ .Values.nodeplugin.priorityClassName }}
|
|
|
|
{{- end }}
|
2018-12-10 19:25:34 +00:00
|
|
|
# to use e.g. Rook orchestrated cluster, and mons' FQDN is
|
|
|
|
# resolved through k8s service, set dns policy to cluster first
|
|
|
|
dnsPolicy: ClusterFirstWithHostNet
|
|
|
|
containers:
|
|
|
|
- name: driver-registrar
|
2020-01-13 04:14:18 +00:00
|
|
|
# This is necessary only for systems with SELinux, where
|
|
|
|
# non-privileged sidecar containers cannot access unix domain socket
|
|
|
|
# created by privileged CSI driver container.
|
|
|
|
securityContext:
|
|
|
|
privileged: true
|
2018-12-10 19:25:34 +00:00
|
|
|
image: "{{ .Values.nodeplugin.registrar.image.repository }}:{{ .Values.nodeplugin.registrar.image.tag }}"
|
2019-09-26 09:55:12 +00:00
|
|
|
imagePullPolicy: {{ .Values.nodeplugin.registrar.image.pullPolicy }}
|
2018-12-10 19:25:34 +00:00
|
|
|
args:
|
2020-10-20 06:49:10 +00:00
|
|
|
- "--v={{ .Values.logLevel }}"
|
2019-09-26 09:55:12 +00:00
|
|
|
- "--csi-address=/csi/{{ .Values.pluginSocketFile }}"
|
2021-02-02 12:05:30 +00:00
|
|
|
- "--kubelet-registration-path={{ .Values.kubeletDir }}/plugins/{{ .Values.driverName }}/{{ .Values.pluginSocketFile }}"
|
2018-12-10 19:25:34 +00:00
|
|
|
env:
|
|
|
|
- name: KUBE_NODE_NAME
|
|
|
|
valueFrom:
|
|
|
|
fieldRef:
|
|
|
|
fieldPath: spec.nodeName
|
|
|
|
volumeMounts:
|
2019-07-03 10:02:36 +00:00
|
|
|
- name: socket-dir
|
2019-02-05 21:29:16 +00:00
|
|
|
mountPath: /csi
|
2018-12-10 19:25:34 +00:00
|
|
|
- name: registration-dir
|
|
|
|
mountPath: /registration
|
|
|
|
resources:
|
|
|
|
{{ toYaml .Values.nodeplugin.registrar.resources | indent 12 }}
|
|
|
|
- name: csi-rbdplugin
|
|
|
|
image: "{{ .Values.nodeplugin.plugin.image.repository }}:{{ .Values.nodeplugin.plugin.image.tag }}"
|
2019-09-26 09:55:12 +00:00
|
|
|
imagePullPolicy: {{ .Values.nodeplugin.plugin.image.pullPolicy }}
|
|
|
|
args:
|
2018-12-10 19:25:34 +00:00
|
|
|
- "--nodeid=$(NODE_ID)"
|
2021-05-31 11:11:01 +00:00
|
|
|
- "--pluginpath={{ .Values.kubeletDir }}/plugins"
|
rbd: add volume healer
Problem:
-------
For rbd nbd userspace mounter backends, after a restart of the nodeplugin
all the mounts will start seeing IO errors. This is because, for rbd-nbd
backends there will be a userspace mount daemon running per volume, post
restart of the nodeplugin pod, there is no way to restore the daemons
back to life.
Solution:
--------
The volume healer is a one-time activity that is triggered at the startup
time of the rbd nodeplugin. It navigates through the list of volume
attachments on the node and acts accordingly.
For now, it is limited to nbd type storage only, but it is flexible and
can be extended in the future for other backend types as needed.
From a few feets above:
This solves a severe problem for nbd backed csi volumes. The healer while
going through the list of volume attachments on the node, if finds the
volume is in attached state and is of type nbd, then it will attempt to
fix the rbd-nbd volumes by sending a NodeStageVolume request with the
required volume attributes like secrets, device name, image attributes,
and etc.. which will finally help start the required rbd-nbd daemons in
the nodeplugin csi-rbdplugin container. This will allow reattaching the
backend images with the right nbd device, thus allowing the applications
to perform IO without any interruptions even after a nodeplugin restart.
Signed-off-by: Prasanna Kumar Kalever <prasanna.kalever@redhat.com>
2021-05-31 11:13:54 +00:00
|
|
|
- "--stagingpath={{ .Values.kubeletDir }}/plugins/kubernetes.io/csi/pv/"
|
2019-05-24 11:03:33 +00:00
|
|
|
- "--type=rbd"
|
2019-08-14 06:42:17 +00:00
|
|
|
- "--nodeserver=true"
|
2019-09-26 09:55:12 +00:00
|
|
|
- "--pidlimit=-1"
|
2018-12-10 19:25:34 +00:00
|
|
|
- "--endpoint=$(CSI_ENDPOINT)"
|
2021-12-22 07:09:27 +00:00
|
|
|
- "--csi-addons-endpoint=$(CSI_ADDONS_ENDPOINT)"
|
2020-10-20 06:49:10 +00:00
|
|
|
- "--v={{ .Values.logLevel }}"
|
2019-03-13 05:09:58 +00:00
|
|
|
- "--drivername=$(DRIVER_NAME)"
|
2020-03-18 17:38:40 +00:00
|
|
|
{{- if .Values.topology.enabled }}
|
|
|
|
- "--domainlabels={{ .Values.topology.domainLabels | join "," }}"
|
2021-03-25 11:14:46 +00:00
|
|
|
{{- end }}
|
|
|
|
{{- if .Values.nodeplugin.profiling.enabled }}
|
|
|
|
- "--enableprofiling={{ .Values.nodeplugin.profiling.enabled }}"
|
2020-03-18 17:38:40 +00:00
|
|
|
{{- end }}
|
2018-12-10 19:25:34 +00:00
|
|
|
env:
|
2019-08-21 09:28:02 +00:00
|
|
|
- name: POD_IP
|
|
|
|
valueFrom:
|
|
|
|
fieldRef:
|
|
|
|
fieldPath: status.podIP
|
2019-03-13 05:09:58 +00:00
|
|
|
- name: DRIVER_NAME
|
|
|
|
value: {{ .Values.driverName }}
|
2018-12-10 19:25:34 +00:00
|
|
|
- name: NODE_ID
|
|
|
|
valueFrom:
|
|
|
|
fieldRef:
|
|
|
|
fieldPath: spec.nodeName
|
|
|
|
- name: CSI_ENDPOINT
|
2019-09-26 09:55:12 +00:00
|
|
|
value: "unix:///csi/{{ .Values.pluginSocketFile }}"
|
2021-12-22 07:09:27 +00:00
|
|
|
- name: CSI_ADDONS_ENDPOINT
|
|
|
|
value: "unix:///csi/csi-addons.sock"
|
2019-09-26 09:55:12 +00:00
|
|
|
securityContext:
|
|
|
|
privileged: true
|
|
|
|
capabilities:
|
|
|
|
add: ["SYS_ADMIN"]
|
|
|
|
allowPrivilegeEscalation: true
|
2018-12-10 19:25:34 +00:00
|
|
|
volumeMounts:
|
2019-07-03 10:02:36 +00:00
|
|
|
- name: socket-dir
|
2019-09-26 09:55:12 +00:00
|
|
|
mountPath: /csi
|
2018-12-10 19:25:34 +00:00
|
|
|
- mountPath: /dev
|
|
|
|
name: host-dev
|
2020-01-29 14:47:41 +00:00
|
|
|
- mountPath: /run/mount
|
|
|
|
name: host-mount
|
2018-12-10 19:25:34 +00:00
|
|
|
- mountPath: /sys
|
|
|
|
name: host-sys
|
2022-02-15 23:13:39 +00:00
|
|
|
{{- if .Values.selinuxMount }}
|
2021-08-02 14:57:11 +00:00
|
|
|
- mountPath: /etc/selinux
|
|
|
|
name: etc-selinux
|
|
|
|
readOnly: true
|
2022-02-15 23:13:39 +00:00
|
|
|
{{- end }}
|
2018-12-10 19:25:34 +00:00
|
|
|
- mountPath: /lib/modules
|
|
|
|
name: lib-modules
|
|
|
|
readOnly: true
|
2019-04-22 21:35:39 +00:00
|
|
|
- name: ceph-csi-config
|
|
|
|
mountPath: /etc/ceph-csi-config/
|
2021-09-01 14:56:15 +00:00
|
|
|
- name: ceph-config
|
|
|
|
mountPath: /etc/ceph/
|
2020-01-29 11:44:45 +00:00
|
|
|
- name: ceph-csi-encryption-kms-config
|
|
|
|
mountPath: /etc/ceph-csi-encryption-kms-config/
|
2019-09-26 09:55:12 +00:00
|
|
|
- name: plugin-dir
|
2020-11-27 14:23:35 +00:00
|
|
|
mountPath: {{ .Values.kubeletDir }}/plugins
|
2019-09-26 09:55:12 +00:00
|
|
|
mountPropagation: "Bidirectional"
|
|
|
|
- name: mountpoint-dir
|
2020-11-27 14:23:35 +00:00
|
|
|
mountPath: {{ .Values.kubeletDir }}/pods
|
2019-09-26 09:55:12 +00:00
|
|
|
mountPropagation: "Bidirectional"
|
2019-06-25 19:29:17 +00:00
|
|
|
- name: keys-tmp-dir
|
|
|
|
mountPath: /tmp/csi/keys
|
2021-08-12 13:07:54 +00:00
|
|
|
- name: ceph-logdir
|
|
|
|
mountPath: /var/log/ceph
|
2022-03-02 10:30:48 +00:00
|
|
|
- name: oidc-token
|
|
|
|
mountPath: /var/run/secrets/tokens
|
|
|
|
readOnly: true
|
2018-12-10 19:25:34 +00:00
|
|
|
resources:
|
2019-06-20 19:30:40 +00:00
|
|
|
{{ toYaml .Values.nodeplugin.plugin.resources | indent 12 }}
|
2019-09-26 09:55:12 +00:00
|
|
|
{{- if .Values.nodeplugin.httpMetrics.enabled }}
|
2019-06-20 19:30:40 +00:00
|
|
|
- name: liveness-prometheus
|
2020-01-13 04:14:18 +00:00
|
|
|
securityContext:
|
|
|
|
privileged: true
|
2019-06-20 19:30:40 +00:00
|
|
|
image: "{{ .Values.nodeplugin.plugin.image.repository }}:{{ .Values.nodeplugin.plugin.image.tag }}"
|
2019-09-26 09:55:12 +00:00
|
|
|
imagePullPolicy: {{ .Values.nodeplugin.plugin.image.pullPolicy }}
|
2019-06-20 19:30:40 +00:00
|
|
|
args:
|
|
|
|
- "--type=liveness"
|
|
|
|
- "--endpoint=$(CSI_ENDPOINT)"
|
2019-09-26 09:55:12 +00:00
|
|
|
- "--metricsport={{ .Values.nodeplugin.httpMetrics.containerPort }}"
|
2019-08-21 09:28:02 +00:00
|
|
|
- "--metricspath=/metrics"
|
2019-06-20 19:30:40 +00:00
|
|
|
- "--polltime=60s"
|
|
|
|
- "--timeout=3s"
|
|
|
|
env:
|
|
|
|
- name: CSI_ENDPOINT
|
2019-09-26 09:55:12 +00:00
|
|
|
value: "unix:///csi/{{ .Values.pluginSocketFile }}"
|
2019-06-20 19:30:40 +00:00
|
|
|
- name: POD_IP
|
|
|
|
valueFrom:
|
|
|
|
fieldRef:
|
|
|
|
fieldPath: status.podIP
|
2022-02-04 11:18:59 +00:00
|
|
|
ports:
|
|
|
|
- containerPort: {{ .Values.nodeplugin.httpMetrics.containerPort }}
|
|
|
|
name: metrics
|
|
|
|
protocol: TCP
|
2019-06-20 19:30:40 +00:00
|
|
|
volumeMounts:
|
2019-09-26 09:55:12 +00:00
|
|
|
- name: socket-dir
|
|
|
|
mountPath: /csi
|
2019-06-20 19:30:40 +00:00
|
|
|
resources:
|
2018-12-10 19:25:34 +00:00
|
|
|
{{ toYaml .Values.nodeplugin.plugin.resources | indent 12 }}
|
2019-09-26 09:55:12 +00:00
|
|
|
{{- end }}
|
2018-12-10 19:25:34 +00:00
|
|
|
volumes:
|
2019-07-03 10:02:36 +00:00
|
|
|
- name: socket-dir
|
2018-12-10 19:25:34 +00:00
|
|
|
hostPath:
|
2021-02-02 12:05:30 +00:00
|
|
|
path: "{{ .Values.kubeletDir }}/plugins/{{ .Values.driverName }}"
|
2018-12-10 19:25:34 +00:00
|
|
|
type: DirectoryOrCreate
|
|
|
|
- name: registration-dir
|
|
|
|
hostPath:
|
2020-11-27 14:23:35 +00:00
|
|
|
path: {{ .Values.kubeletDir }}/plugins_registry
|
2018-12-10 19:25:34 +00:00
|
|
|
type: Directory
|
2019-07-03 10:02:36 +00:00
|
|
|
- name: plugin-dir
|
2018-12-10 19:25:34 +00:00
|
|
|
hostPath:
|
2020-11-27 14:23:35 +00:00
|
|
|
path: {{ .Values.kubeletDir }}/plugins
|
2018-12-10 19:25:34 +00:00
|
|
|
type: Directory
|
2019-07-03 10:02:36 +00:00
|
|
|
- name: mountpoint-dir
|
|
|
|
hostPath:
|
2020-11-27 14:23:35 +00:00
|
|
|
path: {{ .Values.kubeletDir }}/pods
|
2019-07-03 10:02:36 +00:00
|
|
|
type: DirectoryOrCreate
|
2021-08-12 13:07:54 +00:00
|
|
|
- name: ceph-logdir
|
|
|
|
hostPath:
|
2021-09-01 12:24:42 +00:00
|
|
|
path: {{ .Values.cephLogDirHostPath }}
|
2021-08-12 13:07:54 +00:00
|
|
|
type: DirectoryOrCreate
|
2018-12-10 19:25:34 +00:00
|
|
|
- name: host-dev
|
|
|
|
hostPath:
|
|
|
|
path: /dev
|
2020-01-23 08:29:53 +00:00
|
|
|
- name: host-mount
|
|
|
|
hostPath:
|
|
|
|
path: /run/mount
|
2018-12-10 19:25:34 +00:00
|
|
|
- name: host-sys
|
|
|
|
hostPath:
|
|
|
|
path: /sys
|
2022-02-15 23:13:39 +00:00
|
|
|
{{- if .Values.selinuxMount }}
|
2021-08-02 14:57:11 +00:00
|
|
|
- name: etc-selinux
|
|
|
|
hostPath:
|
|
|
|
path: /etc/selinux
|
2022-02-15 23:13:39 +00:00
|
|
|
{{- end }}
|
2018-12-10 19:25:34 +00:00
|
|
|
- name: lib-modules
|
|
|
|
hostPath:
|
|
|
|
path: /lib/modules
|
2021-09-01 14:56:15 +00:00
|
|
|
- name: ceph-config
|
|
|
|
configMap:
|
2022-02-12 17:01:10 +00:00
|
|
|
name: {{ .Values.cephConfConfigMapName | quote }}
|
2019-04-22 21:35:39 +00:00
|
|
|
- name: ceph-csi-config
|
|
|
|
configMap:
|
|
|
|
name: {{ .Values.configMapName | quote }}
|
2020-04-10 15:55:21 +00:00
|
|
|
{{- if .Values.configMapKey }}
|
|
|
|
items:
|
|
|
|
- key: {{ .Values.configMapKey | quote }}
|
|
|
|
path: config.json
|
|
|
|
{{- end }}
|
2020-01-29 11:44:45 +00:00
|
|
|
- name: ceph-csi-encryption-kms-config
|
|
|
|
configMap:
|
|
|
|
name: {{ .Values.kmsConfigMapName | quote }}
|
2019-06-25 19:29:17 +00:00
|
|
|
- name: keys-tmp-dir
|
|
|
|
emptyDir: {
|
|
|
|
medium: "Memory"
|
|
|
|
}
|
2022-03-02 10:30:48 +00:00
|
|
|
- name: oidc-token
|
|
|
|
projected:
|
|
|
|
sources:
|
|
|
|
- serviceAccountToken:
|
|
|
|
path: oidc-token
|
|
|
|
expirationSeconds: 3600
|
|
|
|
audience: ceph-csi-kms
|
2019-07-29 10:36:24 +00:00
|
|
|
{{- if .Values.nodeplugin.affinity }}
|
2018-12-10 19:25:34 +00:00
|
|
|
affinity:
|
2019-12-03 08:18:59 +00:00
|
|
|
{{ toYaml .Values.nodeplugin.affinity | indent 8 -}}
|
2019-07-29 10:36:24 +00:00
|
|
|
{{- end -}}
|
|
|
|
{{- if .Values.nodeplugin.nodeSelector }}
|
2018-12-10 19:25:34 +00:00
|
|
|
nodeSelector:
|
2019-07-29 10:36:24 +00:00
|
|
|
{{ toYaml .Values.nodeplugin.nodeSelector | indent 8 -}}
|
|
|
|
{{- end -}}
|
|
|
|
{{- if .Values.nodeplugin.tolerations }}
|
2018-12-10 19:25:34 +00:00
|
|
|
tolerations:
|
2019-07-29 10:36:24 +00:00
|
|
|
{{ toYaml .Values.nodeplugin.tolerations | indent 8 -}}
|
|
|
|
{{- end -}}
|