helm: Support setting annotations for nodePlugin and provisioner

Adding annotation support to both the CephFS and RBD charts.  Support
setting the DaemonSet and Pod level annotations for the nodeplugin.
Support setting the Deployment and Pod level annotations for the
provisioner.

Signed-off-by: Mike Vollman <mike@reportallusa.com>

.pre-commit-config.yaml

@@ -20,6 +20,7 @@ repos:
       - id: check-json
       - id: check-yaml
         args: [--multi]
+        exclude: ^charts/ceph-csi-(cephfs|rbd)/templates/
 
       # Makes sure that files end in a new line.
       - id: end-of-file-fixer

PendingReleaseNotes.md

@@ -9,5 +9,6 @@
 - CSI metrics for sidecars are now exposed at `POD_IP`:`SIDECAR_ENDPOINT`/`metrics`
   path. Check sidecar container spec for `SIDECAR_ENDPOINT`
   value [PR](https://github.com/ceph/ceph-csi/pull/4887)
+- helm: Support setting nodepluigin and provisioner annotations
 
 ## NOTE

charts/ceph-csi-cephfs/README.md

@@ -131,6 +131,8 @@ charts and their default values.
 | `nodeplugin.plugin.image.tag`                  | Image tag                                                                                                                                            | `canary`                                           |
 | `nodeplugin.plugin.image.pullPolicy`           | Image pull policy                                                                                                                                    | `IfNotPresent`                                     |
 | `nodeplugin.podSecurityContext`                | Specifies pod-level security context.                                                                                                                | `{}`                                               |
+| `nodeplugin.annotations`                       | Specifies DaemonSet level annotations.                                                                                                               | `{}`                                               |
+| `nodeplugin.podAnnotations`                    | Specifies pod-level annotations.                                                                                                                     | `{}`                                               |
 | `nodeplugin.nodeSelector`                      | Kubernetes `nodeSelector` to add to the Daemonset                                                                                                    | `{}`                                               |
 | `nodeplugin.tolerations`                       | List of Kubernetes `tolerations` to add to the Daemonset                                                                                             | `{}`                                               |
 | `nodeplugin.forcecephkernelclient`             | Set to true to enable Ceph Kernel clients on kernel < 4.17 which support quotas                                                                      | `true`                                             |
@@ -167,6 +169,8 @@ charts and their default values.
 | `provisioner.tolerations`                      | Specifies the tolerations for provisioner deployment                                                                                                 | `{}`                                               |
 | `provisioner.affinity`                         | Specifies the affinity for provisioner deployment                                                                                                    | `{}`                                               |
 | `provisioner.podSecurityContext`               | Specifies pod-level security context.                                                                                                                | `{}`                                               |
+| `provisioner.annotations`                      | Specifies Deployment level annotations.                                                                                                              | `{}`                                               |
+| `provisioner.podAnnotations`                   | Specifies pod-level annotations.                                                                                                                     | `{}`                                               |
 | `provisionerSocketFile`                        | The filename of the provisioner socket                                                                                                               | `csi-provisioner.sock`                             |
 | `pluginSocketFile`                             | The filename of the plugin socket                                                                                                                    | `csi.sock`                                         |
 | `readAffinity.enabled` | Enable read affinity for CephFS subvolumes. Recommended to set to true if running kernel 5.8 or newer. | `false` |

charts/ceph-csi-cephfs/templates/nodeplugin-daemonset.yaml

@@ -10,6 +10,10 @@ metadata:
     release: {{ .Release.Name }}
     heritage: {{ .Release.Service }}
     {{- with .Values.commonLabels }}{{ toYaml . | trim | nindent 4 }}{{- end }}
+{{- if .Values.nodeplugin.annotations }}
+  annotations:
+    {{- toYaml .Values.nodeplugin.annotations | nindent 4 -}}
+{{- end }}
 spec:
   selector:
     matchLabels:
@@ -27,6 +31,10 @@ spec:
         release: {{ .Release.Name }}
         heritage: {{ .Release.Service }}
         {{- with .Values.commonLabels }}{{ toYaml . | trim | nindent 8 }}{{- end }}
+    {{- if .Values.nodeplugin.podAnnotations }}
+      annotations:
+        {{- toYaml .Values.nodeplugin.podAnnotations | nindent 8 -}}
+    {{- end }}
     spec:
       securityContext: {{ toYaml .Values.nodeplugin.podSecurityContext | nindent 8 }}
       serviceAccountName: {{ include "ceph-csi-cephfs.serviceAccountName.nodeplugin" . }}

charts/ceph-csi-cephfs/templates/provisioner-deployment.yaml

@@ -10,6 +10,10 @@ metadata:
     release: {{ .Release.Name }}
     heritage: {{ .Release.Service }}
     {{- with .Values.commonLabels }}{{ toYaml . | trim | nindent 4 }}{{- end }}
+{{- if .Values.provisioner.annotations }}
+  annotations:
+    {{- toYaml .Values.provisioner.annotations | nindent 4 -}}
+{{- end }}
 spec:
   replicas: {{ .Values.provisioner.replicaCount }}
   strategy:
@@ -32,6 +36,10 @@ spec:
         release: {{ .Release.Name }}
         heritage: {{ .Release.Service }}
         {{- with .Values.commonLabels }}{{ toYaml . | trim | nindent 8 }}{{- end }}
+    {{- if .Values.provisioner.podAnnotations }}
+      annotations:
+        {{- toYaml .Values.provisioner.podAnnotations | nindent 8 -}}
+    {{- end }}
     spec:
 {{- if gt (int .Values.provisioner.replicaCount) 1 }}
       affinity:

charts/ceph-csi-cephfs/values.yaml

@@ -130,8 +130,12 @@ nodeplugin:
 
   affinity: {}
 
+  annotations: {}
+
   podSecurityContext: {}
 
+  podAnnotations: {}
+
   # Set to true to enable Ceph Kernel clients
   # on kernel < 4.17 which support quotas
   # forcecephkernelclient: true
@@ -262,8 +266,12 @@ provisioner:
 
   affinity: {}
 
+  annotations: {}
+
   podSecurityContext: {}
 
+  podAnnotations: {}
+
 # readAffinity:
 # Enable read affinity for CephFS subvolumes. Recommended to
 # set to true if running kernel 5.8 or newer.

charts/ceph-csi-rbd/README.md

@@ -133,6 +133,8 @@ charts and their default values.
 | `nodeplugin.plugin.image.tag`                  | Image tag                                                                                                                                            | `canary`                                           |
 | `nodeplugin.plugin.image.pullPolicy`           | Image pull policy                                                                                                                                    | `IfNotPresent`                                     |
 | `nodeplugin.podSecurityContext`                | Specifies pod-level security context.                                                                                                                | `{}`                                               |
+| `nodeplugin.annotations`                       | Specifies DaemonSet level annotations.                                                                                                               | `{}`                                               |
+| `nodeplugin.podAnnotations`                    | Specifies pod-level annotations.                                                                                                                     | `{}`                                               |
 | `nodeplugin.nodeSelector`                      | Kubernetes `nodeSelector` to add to the Daemonset                                                                                                    | `{}`                                               |
 | `nodeplugin.tolerations`                       | List of Kubernetes `tolerations` to add to the Daemonset                                                                                             | `{}`                                               |
 | `provisioner.name`                             | Specifies the name of provisioner                                                                                                                    | `provisioner`                                      |
@@ -157,6 +159,8 @@ charts and their default values.
 | `provisioner.provisioner.args.httpEndpointPort`    | Specifies http server port for diagnostics, health checks and metrics                                                                                    | `""`                                               |
 | `provisioner.provisioner.extraArgs`            | Specifies extra arguments for the provisioner sidecar                                                                                                | `[]`                                               |
 | `provisioner.podSecurityContext`               | Specifies pod-level security context.                                                                                                                | `{}`                                               |
+| `provisioner.annotations`                      | Specifies Deployment level annotations.                                                                                                              | `{}`                                               |
+| `provisioner.podAnnotations`                   | Specifies pod-level annotations.                                                                                                                     | `{}`                                               |
 | `provisioner.attacher.name`                    | Specifies the name of csi-attacher sidecar                                                                                                           | `attacher`                                         |
 | `provisioner.attacher.enabled`                 | Specifies whether attacher sidecar is enabled                                                                                                        | `true`                                             |
 | `provisioner.attacher.image.repository`        | Specifies the csi-attacher image repository URL                                                                                                      | `registry.k8s.io/sig-storage/csi-attacher`              |

charts/ceph-csi-rbd/templates/nodeplugin-daemonset.yaml

@@ -10,6 +10,10 @@ metadata:
     release: {{ .Release.Name }}
     heritage: {{ .Release.Service }}
     {{- with .Values.commonLabels }}{{ toYaml . | trim | nindent 4 }}{{- end }}
+{{- if .Values.nodeplugin.annotations }}
+  annotations:
+    {{- toYaml .Values.nodeplugin.annotations | nindent 4 -}}
+{{- end }}
 spec:
   selector:
     matchLabels:
@@ -27,6 +31,10 @@ spec:
         release: {{ .Release.Name }}
         heritage: {{ .Release.Service }}
         {{- with .Values.commonLabels }}{{ toYaml . | trim | nindent 8 }}{{- end }}
+    {{- if .Values.nodeplugin.podAnnotations }}
+      annotations:
+        {{- toYaml .Values.nodeplugin.podAnnotations | nindent 8 -}}
+    {{- end }}
     spec:
       securityContext: {{ toYaml .Values.nodeplugin.podSecurityContext | nindent 8 }}
       serviceAccountName: {{ include "ceph-csi-rbd.serviceAccountName.nodeplugin" . }}

charts/ceph-csi-rbd/templates/provisioner-deployment.yaml

@@ -10,6 +10,10 @@ metadata:
     release: {{ .Release.Name }}
     heritage: {{ .Release.Service }}
     {{- with .Values.commonLabels }}{{ toYaml . | trim | nindent 4 }}{{- end }}
+{{- if .Values.provisioner.annotations }}
+  annotations:
+    {{- toYaml .Values.provisioner.annotations | nindent 4 -}}
+{{- end }}
 spec:
   replicas: {{ .Values.provisioner.replicaCount }}
   strategy:
@@ -32,6 +36,10 @@ spec:
         release: {{ .Release.Name }}
         heritage: {{ .Release.Service }}
         {{- with .Values.commonLabels }}{{ toYaml . | trim | nindent 8 }}{{- end }}
+    {{- if .Values.provisioner.podAnnotations }}
+      annotations:
+        {{- toYaml .Values.provisioner.podAnnotations | nindent 8 -}}
+    {{- end }}
     spec:
 {{- if gt (int .Values.provisioner.replicaCount) 1 }}
       affinity:

charts/ceph-csi-rbd/values.yaml

@@ -159,8 +159,12 @@ nodeplugin:
 
   affinity: {}
 
+  annotations: {}
+
   podSecurityContext: {}
 
+  podAnnotations: {}
+
 provisioner:
   name: provisioner
   replicaCount: 3
@@ -318,8 +322,12 @@ provisioner:
 
   affinity: {}
 
+  annotations: {}
+
   podSecurityContext: {}
 
+  podAnnotations: {}
+
 topology:
   # domainLabels define which node labels to use as domains
   # for CSI nodeplugins to advertise their domains