From dfd855066707badfd582bd5fe2499d58500f5f20 Mon Sep 17 00:00:00 2001 From: Nikhil-Ladha Date: Fri, 4 Oct 2024 12:21:23 +0530 Subject: [PATCH] cephfs: expose csi metrics of sidecars Expose csi metrics of sidecars deployed by cephfs driver Signed-off-by: Nikhil-Ladha --- charts/ceph-csi-cephfs/README.md | 15 ++++--- .../templates/provisioner-deployment.yaml | 39 +++++++++++++++++++ charts/ceph-csi-cephfs/values.yaml | 21 ++++++++-- .../csi-cephfsplugin-provisioner.yaml | 31 +++++++++++++++ 4 files changed, 96 insertions(+), 10 deletions(-) diff --git a/charts/ceph-csi-cephfs/README.md b/charts/ceph-csi-cephfs/README.md index 1be352f3f..edb3b0a1d 100644 --- a/charts/ceph-csi-cephfs/README.md +++ b/charts/ceph-csi-cephfs/README.md @@ -148,18 +148,21 @@ charts and their default values. | `provisioner.provisioner.image.repository` | Specifies the csi-provisioner image repository URL | `registry.k8s.io/sig-storage/csi-provisioner` | | `provisioner.provisioner.image.tag` | Specifies image tag | `v5.0.1` | | `provisioner.provisioner.image.pullPolicy` | Specifies pull policy | `IfNotPresent` | -| `provisioner.provisioner.image.extraArgs` | Specifies extra arguments for the provisioner sidecar | `[]` | +| `provisioner.provisioner.args.httpEndpointPort` | Specifies http server port for diagnostics, health checks and metrics | `""` | +| `provisioner.provisioner.extraArgs` | Specifies extra arguments for the provisioner sidecar | `[]` | +| `provisioner.resizer.name` | Specifies the name of csi-resizer sidecar | `resizer` | +| `provisioner.resizer.enabled` | Specifies whether resizer sidecar is enabled | `true` | | `provisioner.resizer.image.repository` | Specifies the csi-resizer image repository URL | `registry.k8s.io/sig-storage/csi-resizer` | | `provisioner.resizer.image.tag` | Specifies image tag | `v1.11.1` | | `provisioner.resizer.image.pullPolicy` | Specifies pull policy | `IfNotPresent` | -| `provisioner.resizer.image.extraArgs` | Specifies extra arguments for the resizer sidecar | `[]` | -| `provisioner.resizer.name` | Specifies the name of csi-resizer sidecar | `resizer` | -| `provisioner.resizer.enabled` | Specifies whether resizer sidecar is enabled | `true` | +| `provisioner.resizer.args.httpEndpointPort` | Specifies http server port for diagnostics, health checks and metrics | `""` | +| `provisioner.resizer.extraArgs` | Specifies extra arguments for the resizer sidecar | `[]` | | `provisioner.snapshotter.image.repository` | Specifies the csi-snapshotter image repository URL | `registry.k8s.io/sig-storage/csi-snapshotter` | | `provisioner.snapshotter.image.tag` | Specifies image tag | `v8.0.1` | | `provisioner.snapshotter.image.pullPolicy` | Specifies pull policy | `IfNotPresent` | -| `provisioner.snapshotter.image.extraArgs` | Specifies extra arguments for the snapshotter sidecar | `[]` | -| `provisioner.snapshotter.args.enableVolumeGroupSnapshots` | enables the creation of volume group snapshots | `false` | +| `provisioner.snapshotter.args.enableVolumeGroupSnapshots` | enables the creation of volume group snapshots | `false` | +| `provisioner.snapshotter.args.httpEndpointPort` | Specifies http server port for diagnostics, health checks and metrics | `""` | +| `provisioner.snapshotter.extraArgs` | Specifies extra arguments for the snapshotter sidecar | `[]` | | `provisioner.nodeSelector` | Specifies the node selector for provisioner deployment | `{}` | | `provisioner.tolerations` | Specifies the tolerations for provisioner deployment | `{}` | | `provisioner.affinity` | Specifies the affinity for provisioner deployment | `{}` | diff --git a/charts/ceph-csi-cephfs/templates/provisioner-deployment.yaml b/charts/ceph-csi-cephfs/templates/provisioner-deployment.yaml index 14b0f2c0c..17e2fb809 100644 --- a/charts/ceph-csi-cephfs/templates/provisioner-deployment.yaml +++ b/charts/ceph-csi-cephfs/templates/provisioner-deployment.yaml @@ -136,12 +136,25 @@ spec: - "--extra-create-metadata=true" - "--feature-gates=HonorPVReclaimPolicy=true" - "--prevent-volume-mode-conversion=true" +{{- if and .Values.provisioner.provisioner.args .Values.provisioner.provisioner.args.httpEndpointPort }} + - "--http-endpoint=$(POD_IP):{{ .Values.provisioner.provisioner.args.httpEndpointPort }}" +{{- end }} {{- range .Values.provisioner.provisioner.extraArgs }} - "--{{ . }}" {{- end }} env: - name: ADDRESS value: "unix:///csi/{{ .Values.provisionerSocketFile }}" +{{- if and .Values.provisioner.provisioner.args .Values.provisioner.provisioner.args.httpEndpointPort }} + - name: POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + ports: + - containerPort: {{ .Values.provisioner.provisioner.args.httpEndpointPort }} + name: http-endpoint + protocol: TCP +{{- end }} volumeMounts: - name: socket-dir mountPath: /csi @@ -157,12 +170,25 @@ spec: - "--leader-election=true" - "--extra-create-metadata=true" - "--enable-volume-group-snapshots={{.Values.provisioner.snapshotter.args.enableVolumeGroupSnapshots }}" +{{- if and .Values.provisioner.snapshotter.args .Values.provisioner.snapshotter.args.httpEndpointPort }} + - "--http-endpoint=$(POD_IP):{{ .Values.provisioner.snapshotter.args.httpEndpointPort }}" +{{- end }} {{- range .Values.provisioner.snapshotter.extraArgs }} - "--{{ . }}" {{- end }} env: - name: ADDRESS value: "unix:///csi/{{ .Values.provisionerSocketFile }}" +{{- if and .Values.provisioner.snapshotter.args .Values.provisioner.snapshotter.args.httpEndpointPort }} + - name: POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + ports: + - containerPort: {{ .Values.provisioner.snapshotter.args.httpEndpointPort }} + name: http-endpoint + protocol: TCP +{{- end }} volumeMounts: - name: socket-dir mountPath: /csi @@ -180,12 +206,25 @@ spec: - "--retry-interval-start=500ms" - "--handle-volume-inuse-error=false" - "--feature-gates=RecoverVolumeExpansionFailure=true" +{{- if and .Values.provisioner.resizer.args .Values.provisioner.resizer.args.httpEndpointPort }} + - "--http-endpoint=$(POD_IP):{{ .Values.provisioner.resizer.args.httpEndpointPort }}" +{{- end }} {{- range .Values.provisioner.resizer.extraArgs }} - "--{{ . }}" {{- end }} env: - name: ADDRESS value: "unix:///csi/{{ .Values.provisionerSocketFile }}" +{{- if and .Values.provisioner.resizer.args .Values.provisioner.resizer.args.httpEndpointPort }} + - name: POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + ports: + - containerPort: {{ .Values.provisioner.resizer.args.httpEndpointPort }} + name: http-endpoint + protocol: TCP +{{- end }} volumeMounts: - name: socket-dir mountPath: /csi diff --git a/charts/ceph-csi-cephfs/values.yaml b/charts/ceph-csi-cephfs/values.yaml index 7d73851e6..ec67204c6 100644 --- a/charts/ceph-csi-cephfs/values.yaml +++ b/charts/ceph-csi-cephfs/values.yaml @@ -210,6 +210,11 @@ provisioner: tag: v5.0.1 pullPolicy: IfNotPresent resources: {} + args: + # # httpEndpointPort specifies the http server port for + # # diagnostics, health checks and metrics. + # # Uncomment below line to enable the `http-endpoint` arg for sidecar + # httpEndpointPort: 8090 ## For further options, check ## https://github.com/kubernetes-csi/external-provisioner#command-line-options extraArgs: [] @@ -225,6 +230,11 @@ provisioner: tag: v1.11.1 pullPolicy: IfNotPresent resources: {} + args: + # # httpEndpointPort specifies the http server port for + # # diagnostics, health checks and metrics. + # # Uncomment below line to enable the `http-endpoint` arg for sidecar + # httpEndpointPort: 8091 ## For further options, check ## https://github.com/kubernetes-csi/external-resizer#recommended-optional-arguments extraArgs: [] @@ -235,13 +245,16 @@ provisioner: tag: v8.0.1 pullPolicy: IfNotPresent resources: {} - ## For further options, check - ## https://github.com/kubernetes-csi/external-snapshotter#csi-external-snapshotter-sidecar-command-line-options - extraArgs: [] - args: # enableVolumeGroupSnapshots enables support for volume group snapshots enableVolumeGroupSnapshots: false + # httpEndpointPort specifies the http server port for + # diagnostics, health checks and metrics. + # Uncomment below line to enable the `http-endpoint` arg for sidecar + # httpEndpointPort: 8092 + ## For further options, check + ## https://github.com/kubernetes-csi/external-snapshotter#csi-external-snapshotter-sidecar-command-line-options + extraArgs: [] nodeSelector: {} diff --git a/deploy/cephfs/kubernetes/csi-cephfsplugin-provisioner.yaml b/deploy/cephfs/kubernetes/csi-cephfsplugin-provisioner.yaml index 6685b16cf..9443b3ea4 100644 --- a/deploy/cephfs/kubernetes/csi-cephfsplugin-provisioner.yaml +++ b/deploy/cephfs/kubernetes/csi-cephfsplugin-provisioner.yaml @@ -102,10 +102,19 @@ spec: - "--feature-gates=HonorPVReclaimPolicy=true" - "--prevent-volume-mode-conversion=true" - "--extra-create-metadata=true" + - "--http-endpoint=$(POD_IP):8090" env: - name: ADDRESS value: unix:///csi/csi-provisioner.sock + - name: POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP imagePullPolicy: "IfNotPresent" + ports: + - containerPort: 8090 + name: http-endpoint + protocol: TCP volumeMounts: - name: socket-dir mountPath: /csi @@ -119,10 +128,19 @@ spec: - "--retry-interval-start=500ms" - "--handle-volume-inuse-error=false" - "--feature-gates=RecoverVolumeExpansionFailure=true" + - "--http-endpoint=$(POD_IP):8091" env: - name: ADDRESS value: unix:///csi/csi-provisioner.sock + - name: POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP imagePullPolicy: "IfNotPresent" + ports: + - containerPort: 8091 + name: http-endpoint + protocol: TCP volumeMounts: - name: socket-dir mountPath: /csi @@ -135,10 +153,19 @@ spec: - "--leader-election=true" - "--extra-create-metadata=true" - "--enable-volume-group-snapshots=true" + - "--http-endpoint=$(POD_IP):8092" env: - name: ADDRESS value: unix:///csi/csi-provisioner.sock + - name: POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP imagePullPolicy: "IfNotPresent" + ports: + - containerPort: 8092 + name: http-endpoint + protocol: TCP volumeMounts: - name: socket-dir mountPath: /csi @@ -158,6 +185,10 @@ spec: valueFrom: fieldRef: fieldPath: status.podIP + ports: + - containerPort: 8681 + name: http-metrics + protocol: TCP volumeMounts: - name: socket-dir mountPath: /csi