From 7e26beb51e9cf007a59335db4fadde86341f10f6 Mon Sep 17 00:00:00 2001 From: Praveen M Date: Fri, 17 Nov 2023 11:59:54 +0530 Subject: [PATCH] helm: add option to enable read affinity for CephFS This commit adds --enable-read-affinity flag to enable read affinity for CephFS. Signed-off-by: Praveen M --- charts/ceph-csi-cephfs/README.md | 2 ++ .../templates/nodeplugin-clusterrole.yaml | 19 +++++++++++++++++ .../nodeplugin-clusterrolebinding.yaml | 21 +++++++++++++++++++ .../templates/nodeplugin-daemonset.yaml | 4 ++++ charts/ceph-csi-cephfs/values.yaml | 11 ++++++++++ scripts/install-helm.sh | 2 +- 6 files changed, 58 insertions(+), 1 deletion(-) create mode 100644 charts/ceph-csi-cephfs/templates/nodeplugin-clusterrole.yaml create mode 100644 charts/ceph-csi-cephfs/templates/nodeplugin-clusterrolebinding.yaml diff --git a/charts/ceph-csi-cephfs/README.md b/charts/ceph-csi-cephfs/README.md index 2826d0536..31678e9ea 100644 --- a/charts/ceph-csi-cephfs/README.md +++ b/charts/ceph-csi-cephfs/README.md @@ -164,6 +164,8 @@ charts and their default values. | `provisioner.podSecurityPolicy.enabled` | Specifies whether podSecurityPolicy is enabled | `false` | | `provisionerSocketFile` | The filename of the provisioner socket | `csi-provisioner.sock` | | `pluginSocketFile` | The filename of the plugin socket | `csi.sock` | +| `readAffinity.enabled` | Enable read affinity for CephFS subvolumes. Recommended to set to true if running kernel 5.8 or newer. | `false` | +| `readAffinity.crushLocationLabels` | Define which node labels to use as CRUSH location. This should correspond to the values set in the CRUSH map. For more information, click [here](https://github.com/ceph/ceph-csi/blob/v3.9.0/docs/deploy-rbd.md#read-affinity-using-crush-locations-for-rbd-volumes)| `[]` | | `kubeletDir` | Kubelet working directory | `/var/lib/kubelet` | | `driverName` | Name of the csi-driver | `cephfs.csi.ceph.com` | | `configMapName` | Name of the configmap which contains cluster configuration | `ceph-csi-config` | diff --git a/charts/ceph-csi-cephfs/templates/nodeplugin-clusterrole.yaml b/charts/ceph-csi-cephfs/templates/nodeplugin-clusterrole.yaml new file mode 100644 index 000000000..e425f1840 --- /dev/null +++ b/charts/ceph-csi-cephfs/templates/nodeplugin-clusterrole.yaml @@ -0,0 +1,19 @@ +{{- if .Values.rbac.create -}} +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: {{ include "ceph-csi-cephfs.nodeplugin.fullname" . }} + labels: + app: {{ include "ceph-csi-cephfs.name" . }} + chart: {{ include "ceph-csi-cephfs.chart" . }} + component: {{ .Values.nodeplugin.name }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} + {{- with .Values.commonLabels }}{{ toYaml . | trim | nindent 4 }}{{- end }} +rules: +{{- if and .Values.readAffinity .Values.readAffinity.enabled }} + - apiGroups: [""] + resources: ["nodes"] + verbs: ["get"] +{{- end }} +{{- end -}} diff --git a/charts/ceph-csi-cephfs/templates/nodeplugin-clusterrolebinding.yaml b/charts/ceph-csi-cephfs/templates/nodeplugin-clusterrolebinding.yaml new file mode 100644 index 000000000..fd8d5787a --- /dev/null +++ b/charts/ceph-csi-cephfs/templates/nodeplugin-clusterrolebinding.yaml @@ -0,0 +1,21 @@ +{{- if .Values.rbac.create -}} +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: {{ include "ceph-csi-cephfs.nodeplugin.fullname" . }} + labels: + app: {{ include "ceph-csi-cephfs.name" . }} + chart: {{ include "ceph-csi-cephfs.chart" . }} + component: {{ .Values.nodeplugin.name }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} + {{- with .Values.commonLabels }}{{ toYaml . | trim | nindent 4 }}{{- end }} +subjects: + - kind: ServiceAccount + name: {{ include "ceph-csi-cephfs.serviceAccountName.nodeplugin" . }} + namespace: {{ .Release.Namespace }} +roleRef: + kind: ClusterRole + name: {{ include "ceph-csi-cephfs.nodeplugin.fullname" . }} + apiGroup: rbac.authorization.k8s.io +{{- end -}} diff --git a/charts/ceph-csi-cephfs/templates/nodeplugin-daemonset.yaml b/charts/ceph-csi-cephfs/templates/nodeplugin-daemonset.yaml index d3a5b9df8..264e50ea9 100644 --- a/charts/ceph-csi-cephfs/templates/nodeplugin-daemonset.yaml +++ b/charts/ceph-csi-cephfs/templates/nodeplugin-daemonset.yaml @@ -85,6 +85,10 @@ spec: - "--drivername=$(DRIVER_NAME)" {{- if .Values.nodeplugin.profiling.enabled }} - "--enableprofiling={{ .Values.nodeplugin.profiling.enabled }}" +{{- end }} + - "--enable-read-affinity={{ and .Values.readAffinity .Values.readAffinity.enabled }}" +{{- if and .Values.readAffinity .Values.readAffinity.enabled }} + - "--crush-location-labels={{ .Values.readAffinity.crushLocationLabels | join "," }}" {{- end }} env: - name: POD_IP diff --git a/charts/ceph-csi-cephfs/values.yaml b/charts/ceph-csi-cephfs/values.yaml index 552f58bb0..96336e4d4 100644 --- a/charts/ceph-csi-cephfs/values.yaml +++ b/charts/ceph-csi-cephfs/values.yaml @@ -222,6 +222,17 @@ provisioner: affinity: {} +# readAffinity: +# Enable read affinity for CephFS subvolumes. Recommended to +# set to true if running kernel 5.8 or newer. +# enabled: false +# Define which node labels to use as CRUSH location. +# This should correspond to the values set in the CRUSH map. +# NOTE: the value here serves as an example +# crushLocationLabels: +# - topology.kubernetes.io/region +# - topology.kubernetes.io/zone + # Mount the host /etc/selinux inside pods to support # selinux-enabled filesystems selinuxMount: true diff --git a/scripts/install-helm.sh b/scripts/install-helm.sh index fe4e5d35b..2fe6beebb 100755 --- a/scripts/install-helm.sh +++ b/scripts/install-helm.sh @@ -175,7 +175,7 @@ install_cephcsi_helm_charts() { fi # install ceph-csi-cephfs and ceph-csi-rbd charts # shellcheck disable=SC2086 - "${HELM}" install --namespace ${NAMESPACE} --set provisioner.fullnameOverride=csi-cephfsplugin-provisioner --set nodeplugin.fullnameOverride=csi-cephfsplugin --set configMapName=ceph-csi-config --set provisioner.replicaCount=1 --set-json='commonLabels={"app.kubernetes.io/name": "ceph-csi-cephfs", "app.kubernetes.io/managed-by": "helm"}' ${SET_SC_TEMPLATE_VALUES} ${CEPHFS_SECRET_TEMPLATE_VALUES} ${CEPHFS_CHART_NAME} "${SCRIPT_DIR}"/../charts/ceph-csi-cephfs + "${HELM}" install --namespace ${NAMESPACE} --set provisioner.fullnameOverride=csi-cephfsplugin-provisioner --set nodeplugin.fullnameOverride=csi-cephfsplugin --set configMapName=ceph-csi-config --set provisioner.replicaCount=1 --set-json='commonLabels={"app.kubernetes.io/name": "ceph-csi-cephfs", "app.kubernetes.io/managed-by": "helm"}' ${SET_SC_TEMPLATE_VALUES} ${CEPHFS_SECRET_TEMPLATE_VALUES} ${CEPHFS_CHART_NAME} "${SCRIPT_DIR}"/../charts/ceph-csi-cephfs --set readAffinity.enabled=true --set readAffinity.crushLocationLabels="{${CRUSH_LOCATION_REGION_LABEL},${CRUSH_LOCATION_ZONE_LABEL}}" check_deployment_status app=ceph-csi-cephfs "${NAMESPACE}" check_daemonset_status app=ceph-csi-cephfs "${NAMESPACE}"