From cc407d157ec33e69d8b33652d5232c9187569624 Mon Sep 17 00:00:00 2001 From: Antoine C Date: Mon, 1 Apr 2024 23:49:57 +0100 Subject: [PATCH] helm: support encryption config in ceph-csi-cephfs chart this chart currently lack the ability to properly configure encryption, as well as granting sufficent permission to allow controllers to access secret when needed. Signed-off-by: Antoine C --- charts/ceph-csi-cephfs/README.md | 20 +++++++++++++ .../templates/encryptionkms-configmap.yaml | 15 ++++++++++ .../templates/nodeplugin-clusterrole.yaml | 5 ++++ .../templates/nodeplugin-role.yaml | 21 ++++++++++++++ .../templates/nodeplugin-rolebinding.yaml | 24 ++++++++++++++++ .../templates/storageclass.yaml | 6 ++++ charts/ceph-csi-cephfs/values.yaml | 28 +++++++++++++++++++ scripts/install-helm.sh | 1 + 8 files changed, 120 insertions(+) create mode 100644 charts/ceph-csi-cephfs/templates/encryptionkms-configmap.yaml create mode 100644 charts/ceph-csi-cephfs/templates/nodeplugin-role.yaml create mode 100644 charts/ceph-csi-cephfs/templates/nodeplugin-rolebinding.yaml diff --git a/charts/ceph-csi-cephfs/README.md b/charts/ceph-csi-cephfs/README.md index 7ed366d71..aeafa0a77 100644 --- a/charts/ceph-csi-cephfs/README.md +++ b/charts/ceph-csi-cephfs/README.md @@ -66,6 +66,23 @@ version. We recommend not to use `--reuse-values` in case there are new defaults AND compare your currently used values with the new default values. +### Enabling encryption support + +To enable FSCrypt support, you will need to include the KMS configuration in +`encryptionKMSConfig`. + +Here is a `values.yaml` example using a Kubernetes secret (`kubernetes` KMS) + +```yaml +encryptionKMSConfig: + encryptionKMSType: "metadata" + secretName: "cephfs-encryption-passphrase" # This secret needs to contain the passphrase as the key `encryptionPassphrase` + secretNamespace: "my-namespace" +storageClass: + encrypted: true + encryptionKMSID: kubernetes +``` + #### Known Issues Upgrading - When upgrading to version >=3.7.0, you might encounter an error that the @@ -115,6 +132,7 @@ charts and their default values. | `serviceAccounts.provisioner.create` | Specifies whether a provisioner ServiceAccount should be created | `true` | | `serviceAccounts.provisioner.name` | The name of the provisioner ServiceAccount of provisioner to use. If not set and create is true, a name is generated using the fullname | "" | | `csiConfig` | Configuration for the CSI to connect to the cluster | [] | +| `encryptionKMSConfig` | Configuration for the encryption KMS | `{}` | | `commonLabels` | Labels to apply to all resources | `{}` | | `logLevel` | Set logging level for csi containers. Supported values from 0 to 5. 0 for general useful logs, 5 for trace level verbosity. | `5` | | `sidecarLogLevel` | Set logging level for csi sidecar containers. Supported values from 0 to 5. 0 for general useful logs, 5 for trace level verbosity. | `1` | @@ -184,6 +202,8 @@ charts and their default values. | `storageClass.name` | Specifies the cephFS StorageClass name | `csi-cephfs-sc` | | `storageClass.annotations` | Specifies the annotations for the cephFS storageClass | `[]` | | `storageClass.clusterID` | String representing a Ceph cluster to provision storage from | `` | +| `storageClass.encrypted` | Specifies whether volume should be encrypted. Set it to true if you want to enable encryption | `""` | +| `storageClass.encryptionKMSID` | Specifies the encryption kms id | `""` | | `storageClass.fsName` | CephFS filesystem name into which the volume shall be created | `myfs` | | `storageClass.pool` | Ceph pool into which volume data shall be stored | `""` | | `storageClass.fuseMountOptions` | Comma separated string of Ceph-fuse mount options | `""` | diff --git a/charts/ceph-csi-cephfs/templates/encryptionkms-configmap.yaml b/charts/ceph-csi-cephfs/templates/encryptionkms-configmap.yaml new file mode 100644 index 000000000..543297b62 --- /dev/null +++ b/charts/ceph-csi-cephfs/templates/encryptionkms-configmap.yaml @@ -0,0 +1,15 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ .Values.kmsConfigMapName | quote }} + namespace: {{ .Release.Namespace }} + labels: + app: {{ include "ceph-csi-cephfs.name" . }} + chart: {{ include "ceph-csi-cephfs.chart" . }} + component: {{ .Values.nodeplugin.name }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} + {{- with .Values.commonLabels }}{{ toYaml . | trim | nindent 4 }}{{- end }} +data: + config.json: |- +{{ toJson .Values.encryptionKMSConfig | indent 4 -}} diff --git a/charts/ceph-csi-cephfs/templates/nodeplugin-clusterrole.yaml b/charts/ceph-csi-cephfs/templates/nodeplugin-clusterrole.yaml index 632672ebd..d11e3eef0 100644 --- a/charts/ceph-csi-cephfs/templates/nodeplugin-clusterrole.yaml +++ b/charts/ceph-csi-cephfs/templates/nodeplugin-clusterrole.yaml @@ -3,6 +3,7 @@ kind: ClusterRole apiVersion: rbac.authorization.k8s.io/v1 metadata: name: {{ include "ceph-csi-cephfs.nodeplugin.fullname" . }} + namespace: {{ .Release.Namespace }} labels: app: {{ include "ceph-csi-cephfs.name" . }} chart: {{ include "ceph-csi-cephfs.chart" . }} @@ -14,10 +15,14 @@ rules: - apiGroups: [""] resources: ["nodes"] verbs: ["get"] + # allow to read Vault Token and connection options from the Tenants namespace - apiGroups: [""] resources: ["configmaps"] verbs: ["get"] +{{- if and .Values.encryptionKMSConfig .Values.encryptionKMSConfig.secretNamespace (not (eq .Values.encryptionKMSConfig.secretNamespace .Release.Namespace)) }} + # allow to read the encryption key used with the metadata KMS - apiGroups: [""] resources: ["secrets"] verbs: ["get"] {{- end -}} +{{- end -}} diff --git a/charts/ceph-csi-cephfs/templates/nodeplugin-role.yaml b/charts/ceph-csi-cephfs/templates/nodeplugin-role.yaml new file mode 100644 index 000000000..d498284aa --- /dev/null +++ b/charts/ceph-csi-cephfs/templates/nodeplugin-role.yaml @@ -0,0 +1,21 @@ +{{- if .Values.rbac.create -}} +{{- if and .Values.encryptionKMSConfig .Values.encryptionKMSConfig.secretNamespace .Values.encryptionKMSConfig.secretName (eq .Values.encryptionKMSConfig.secretNamespace .Release.Namespace) -}} +kind: Role +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: {{ include "ceph-csi-cephfs.nodeplugin.fullname" . }} + labels: + app: {{ include "ceph-csi-cephfs.name" . }} + chart: {{ include "ceph-csi-cephfs.chart" . }} + component: {{ .Values.nodeplugin.name }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} + {{- with .Values.commonLabels }}{{ toYaml . | trim | nindent 4 }}{{- end }} +rules: + # allow to read the encryption key used with the metadata KMS + - apiGroups: [""] + resources: ["secrets"] + verbs: ["get"] + resourceNames: [{{ .Values.encryptionKMSConfig.secretName | quote }}] +{{- end -}} +{{- end -}} diff --git a/charts/ceph-csi-cephfs/templates/nodeplugin-rolebinding.yaml b/charts/ceph-csi-cephfs/templates/nodeplugin-rolebinding.yaml new file mode 100644 index 000000000..c76e96f2f --- /dev/null +++ b/charts/ceph-csi-cephfs/templates/nodeplugin-rolebinding.yaml @@ -0,0 +1,24 @@ +{{- if .Values.rbac.create -}} +{{- if and (eq .Values.encryptionKMSConfig.encryptionKMSType "metadata") (eq .Values.encryptionKMSConfig.secretNamespace .Release.Namespace) -}} +kind: RoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: {{ include "ceph-csi-cephfs.nodeplugin.fullname" . }} + namespace: {{ .Release.Namespace }} + labels: + app: {{ include "ceph-csi-cephfs.name" . }} + chart: {{ include "ceph-csi-cephfs.chart" . }} + component: {{ .Values.nodeplugin.name }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} + {{- with .Values.commonLabels }}{{ toYaml . | trim | nindent 4 }}{{- end }} +subjects: + - kind: ServiceAccount + name: {{ include "ceph-csi-cephfs.serviceAccountName.nodeplugin" . }} + namespace: {{ .Release.Namespace }} +roleRef: + kind: Role + name: {{ include "ceph-csi-cephfs.nodeplugin.fullname" . }} + apiGroup: rbac.authorization.k8s.io +{{- end -}} +{{- end -}} diff --git a/charts/ceph-csi-cephfs/templates/storageclass.yaml b/charts/ceph-csi-cephfs/templates/storageclass.yaml index a21c99824..35e5c5e43 100644 --- a/charts/ceph-csi-cephfs/templates/storageclass.yaml +++ b/charts/ceph-csi-cephfs/templates/storageclass.yaml @@ -20,6 +20,12 @@ parameters: {{- if .Values.storageClass.pool }} pool: {{ .Values.storageClass.pool }} {{- end }} +{{- if .Values.storageClass.encrypted }} + encrypted: "{{ .Values.storageClass.encrypted }}" +{{- end }} +{{- if .Values.storageClass.encryptionKMSID }} + encryptionKMSID: {{ .Values.storageClass.encryptionKMSID }} +{{- end }} {{- if .Values.storageClass.fuseMountOptions }} fuseMountOptions: "{{ .Values.storageClass.fuseMountOptions }}" {{- end }} diff --git a/charts/ceph-csi-cephfs/values.yaml b/charts/ceph-csi-cephfs/values.yaml index 814b6a418..f05b605b6 100644 --- a/charts/ceph-csi-cephfs/values.yaml +++ b/charts/ceph-csi-cephfs/values.yaml @@ -31,6 +31,20 @@ serviceAccounts: # radosNamespace: "csi" csiConfig: [] +# Configuration for the encryption KMS +# yamllint disable-line rule:line-length +# Ref: https://github.com/ceph/ceph-csi/blob/devel/docs/deploy-cephfs.md#cephfs-volume-encryption +# Example: +# encryptionKMSConfig: +# encryptionKMSType: vault +# vaultAddress: https://vault.example.com +# vaultAuthPath: /v1/auth/kubernetes/login +# vaultRole: csi-kubernetes +# vaultPassphraseRoot: /v1/secret +# vaultPassphrasePath: ceph-csi/ +# vaultCAVerify: "true" +encryptionKMSConfig: {} + # Labels to apply to all resources commonLabels: {} @@ -329,6 +343,18 @@ storageClass: # If omitted, defaults to "csi-vol-". # volumeNamePrefix: "foo-bar-" volumeNamePrefix: "" + + # (optional) Instruct the plugin it has to encrypt the volume + # By default it is disabled. Valid values are "true" or "false". + # A string is expected here, i.e. "true", not true. + # encrypted: "true" + encrypted: "" + + # (optional) Use external key management system for encryption passphrases by + # specifying a unique ID matching KMS ConfigMap. The ID is only used for + # correlation to configmap entry. + encryptionKMSID: "" + # The secrets have to contain user and/or Ceph admin credentials. provisionerSecret: csi-cephfs-secret # If the Namespaces are not specified, the secrets are assumed to @@ -400,6 +426,8 @@ configMapName: ceph-csi-config externallyManagedConfigmap: false # Name of the configmap used for ceph.conf cephConfConfigMapName: ceph-config +# Name of the configmap used for encryption kms configuration +kmsConfigMapName: ceph-csi-encryption-kms-config # CephFS RadosNamespace used to store CSI specific objects and keys. # radosNamespaceCephFS: csi # Unique ID distinguishing this instance of Ceph CSI among other instances, diff --git a/scripts/install-helm.sh b/scripts/install-helm.sh index 429e633b1..eae2d51e6 100755 --- a/scripts/install-helm.sh +++ b/scripts/install-helm.sh @@ -188,6 +188,7 @@ install_cephcsi_helm_charts() { # issue when installing ceph-csi-rbd kubectl_retry delete cm ceph-csi-config --namespace "${NAMESPACE}" kubectl_retry delete cm ceph-config --namespace "${NAMESPACE}" + kubectl_retry delete cm ceph-csi-encryption-kms-config --namespace "${NAMESPACE}" # shellcheck disable=SC2086 "${HELM}" install --namespace ${NAMESPACE} --set provisioner.fullnameOverride=csi-rbdplugin-provisioner --set nodeplugin.fullnameOverride=csi-rbdplugin --set configMapName=ceph-csi-config --set provisioner.replicaCount=1 --set-json='commonLabels={"app.kubernetes.io/name": "ceph-csi-rbd", "app.kubernetes.io/managed-by": "helm"}' ${SET_SC_TEMPLATE_VALUES} ${RBD_SECRET_TEMPLATE_VALUES} ${RBD_CHART_NAME} "${SCRIPT_DIR}"/../charts/ceph-csi-rbd --set topology.domainLabels="{${NODE_LABEL_REGION},${NODE_LABEL_ZONE}}" --set provisioner.maxSnapshotsOnImage=3 --set provisioner.minSnapshotsOnImage=2 ${READ_AFFINITY_VALUES} --set provisioner.snapshotter.args.enableVolumeGroupSnapshots=true