diff --git a/deploy/cephfs/kubernetes/csi-cephfsplugin-provisioner.yaml b/deploy/cephfs/kubernetes/v1.13/csi-cephfsplugin-provisioner.yaml similarity index 100% rename from deploy/cephfs/kubernetes/csi-cephfsplugin-provisioner.yaml rename to deploy/cephfs/kubernetes/v1.13/csi-cephfsplugin-provisioner.yaml diff --git a/deploy/cephfs/kubernetes/csi-cephfsplugin.yaml b/deploy/cephfs/kubernetes/v1.13/csi-cephfsplugin.yaml similarity index 100% rename from deploy/cephfs/kubernetes/csi-cephfsplugin.yaml rename to deploy/cephfs/kubernetes/v1.13/csi-cephfsplugin.yaml diff --git a/deploy/cephfs/kubernetes/csi-config-map.yaml b/deploy/cephfs/kubernetes/v1.13/csi-config-map.yaml similarity index 100% rename from deploy/cephfs/kubernetes/csi-config-map.yaml rename to deploy/cephfs/kubernetes/v1.13/csi-config-map.yaml diff --git a/deploy/cephfs/kubernetes/csi-nodeplugin-rbac.yaml b/deploy/cephfs/kubernetes/v1.13/csi-nodeplugin-rbac.yaml similarity index 100% rename from deploy/cephfs/kubernetes/csi-nodeplugin-rbac.yaml rename to deploy/cephfs/kubernetes/v1.13/csi-nodeplugin-rbac.yaml diff --git a/deploy/cephfs/kubernetes/csi-provisioner-rbac.yaml b/deploy/cephfs/kubernetes/v1.13/csi-provisioner-rbac.yaml similarity index 100% rename from deploy/cephfs/kubernetes/csi-provisioner-rbac.yaml rename to deploy/cephfs/kubernetes/v1.13/csi-provisioner-rbac.yaml diff --git a/deploy/cephfs/helm/.helmignore b/deploy/cephfs/kubernetes/v1.13/helm/.helmignore similarity index 100% rename from deploy/cephfs/helm/.helmignore rename to deploy/cephfs/kubernetes/v1.13/helm/.helmignore diff --git a/deploy/cephfs/helm/Chart.yaml b/deploy/cephfs/kubernetes/v1.13/helm/Chart.yaml similarity index 100% rename from deploy/cephfs/helm/Chart.yaml rename to deploy/cephfs/kubernetes/v1.13/helm/Chart.yaml diff --git a/deploy/cephfs/helm/README.md b/deploy/cephfs/kubernetes/v1.13/helm/README.md similarity index 100% rename from deploy/cephfs/helm/README.md rename to deploy/cephfs/kubernetes/v1.13/helm/README.md diff --git a/deploy/cephfs/helm/templates/NOTES.txt b/deploy/cephfs/kubernetes/v1.13/helm/templates/NOTES.txt similarity index 100% rename from deploy/cephfs/helm/templates/NOTES.txt rename to deploy/cephfs/kubernetes/v1.13/helm/templates/NOTES.txt diff --git a/deploy/cephfs/helm/templates/_helpers.tpl b/deploy/cephfs/kubernetes/v1.13/helm/templates/_helpers.tpl similarity index 100% rename from deploy/cephfs/helm/templates/_helpers.tpl rename to deploy/cephfs/kubernetes/v1.13/helm/templates/_helpers.tpl diff --git a/deploy/cephfs/helm/templates/csidriver-crd.yaml b/deploy/cephfs/kubernetes/v1.13/helm/templates/csidriver-crd.yaml similarity index 100% rename from deploy/cephfs/helm/templates/csidriver-crd.yaml rename to deploy/cephfs/kubernetes/v1.13/helm/templates/csidriver-crd.yaml diff --git a/deploy/cephfs/helm/templates/csiplugin-configmap.yaml b/deploy/cephfs/kubernetes/v1.13/helm/templates/csiplugin-configmap.yaml similarity index 100% rename from deploy/cephfs/helm/templates/csiplugin-configmap.yaml rename to deploy/cephfs/kubernetes/v1.13/helm/templates/csiplugin-configmap.yaml diff --git a/deploy/cephfs/helm/templates/nodeplugin-clusterrole.yaml b/deploy/cephfs/kubernetes/v1.13/helm/templates/nodeplugin-clusterrole.yaml similarity index 100% rename from deploy/cephfs/helm/templates/nodeplugin-clusterrole.yaml rename to deploy/cephfs/kubernetes/v1.13/helm/templates/nodeplugin-clusterrole.yaml diff --git a/deploy/cephfs/helm/templates/nodeplugin-clusterrolebinding.yaml b/deploy/cephfs/kubernetes/v1.13/helm/templates/nodeplugin-clusterrolebinding.yaml similarity index 100% rename from deploy/cephfs/helm/templates/nodeplugin-clusterrolebinding.yaml rename to deploy/cephfs/kubernetes/v1.13/helm/templates/nodeplugin-clusterrolebinding.yaml diff --git a/deploy/cephfs/helm/templates/nodeplugin-daemonset.yaml b/deploy/cephfs/kubernetes/v1.13/helm/templates/nodeplugin-daemonset.yaml similarity index 100% rename from deploy/cephfs/helm/templates/nodeplugin-daemonset.yaml rename to deploy/cephfs/kubernetes/v1.13/helm/templates/nodeplugin-daemonset.yaml diff --git a/deploy/cephfs/helm/templates/nodeplugin-rules-clusterrole.yaml b/deploy/cephfs/kubernetes/v1.13/helm/templates/nodeplugin-rules-clusterrole.yaml similarity index 100% rename from deploy/cephfs/helm/templates/nodeplugin-rules-clusterrole.yaml rename to deploy/cephfs/kubernetes/v1.13/helm/templates/nodeplugin-rules-clusterrole.yaml diff --git a/deploy/cephfs/helm/templates/nodeplugin-serviceaccount.yaml b/deploy/cephfs/kubernetes/v1.13/helm/templates/nodeplugin-serviceaccount.yaml similarity index 100% rename from deploy/cephfs/helm/templates/nodeplugin-serviceaccount.yaml rename to deploy/cephfs/kubernetes/v1.13/helm/templates/nodeplugin-serviceaccount.yaml diff --git a/deploy/cephfs/helm/templates/provisioner-clusterrole.yaml b/deploy/cephfs/kubernetes/v1.13/helm/templates/provisioner-clusterrole.yaml similarity index 100% rename from deploy/cephfs/helm/templates/provisioner-clusterrole.yaml rename to deploy/cephfs/kubernetes/v1.13/helm/templates/provisioner-clusterrole.yaml diff --git a/deploy/cephfs/helm/templates/provisioner-clusterrolebinding.yaml b/deploy/cephfs/kubernetes/v1.13/helm/templates/provisioner-clusterrolebinding.yaml similarity index 100% rename from deploy/cephfs/helm/templates/provisioner-clusterrolebinding.yaml rename to deploy/cephfs/kubernetes/v1.13/helm/templates/provisioner-clusterrolebinding.yaml diff --git a/deploy/cephfs/helm/templates/provisioner-role.yaml b/deploy/cephfs/kubernetes/v1.13/helm/templates/provisioner-role.yaml similarity index 100% rename from deploy/cephfs/helm/templates/provisioner-role.yaml rename to deploy/cephfs/kubernetes/v1.13/helm/templates/provisioner-role.yaml diff --git a/deploy/cephfs/helm/templates/provisioner-rolebinding.yaml b/deploy/cephfs/kubernetes/v1.13/helm/templates/provisioner-rolebinding.yaml similarity index 100% rename from deploy/cephfs/helm/templates/provisioner-rolebinding.yaml rename to deploy/cephfs/kubernetes/v1.13/helm/templates/provisioner-rolebinding.yaml diff --git a/deploy/cephfs/helm/templates/provisioner-rules-clusterrole.yaml b/deploy/cephfs/kubernetes/v1.13/helm/templates/provisioner-rules-clusterrole.yaml similarity index 100% rename from deploy/cephfs/helm/templates/provisioner-rules-clusterrole.yaml rename to deploy/cephfs/kubernetes/v1.13/helm/templates/provisioner-rules-clusterrole.yaml diff --git a/deploy/cephfs/helm/templates/provisioner-service.yaml b/deploy/cephfs/kubernetes/v1.13/helm/templates/provisioner-service.yaml similarity index 100% rename from deploy/cephfs/helm/templates/provisioner-service.yaml rename to deploy/cephfs/kubernetes/v1.13/helm/templates/provisioner-service.yaml diff --git a/deploy/cephfs/helm/templates/provisioner-serviceaccount.yaml b/deploy/cephfs/kubernetes/v1.13/helm/templates/provisioner-serviceaccount.yaml similarity index 100% rename from deploy/cephfs/helm/templates/provisioner-serviceaccount.yaml rename to deploy/cephfs/kubernetes/v1.13/helm/templates/provisioner-serviceaccount.yaml diff --git a/deploy/cephfs/helm/templates/provisioner-statefulset.yaml b/deploy/cephfs/kubernetes/v1.13/helm/templates/provisioner-statefulset.yaml similarity index 100% rename from deploy/cephfs/helm/templates/provisioner-statefulset.yaml rename to deploy/cephfs/kubernetes/v1.13/helm/templates/provisioner-statefulset.yaml diff --git a/deploy/cephfs/helm/values.yaml b/deploy/cephfs/kubernetes/v1.13/helm/values.yaml similarity index 100% rename from deploy/cephfs/helm/values.yaml rename to deploy/cephfs/kubernetes/v1.13/helm/values.yaml diff --git a/deploy/cephfs/kubernetes/v1.14+/csi-cephfsplugin-provisioner.yaml b/deploy/cephfs/kubernetes/v1.14+/csi-cephfsplugin-provisioner.yaml new file mode 100644 index 000000000..23dfbf622 --- /dev/null +++ b/deploy/cephfs/kubernetes/v1.14+/csi-cephfsplugin-provisioner.yaml @@ -0,0 +1,108 @@ +--- +kind: Deployment +apiVersion: apps/v1 +metadata: + name: csi-cephfsplugin-provisioner +spec: + selector: + matchLabels: + app: csi-cephfsplugin-provisioner + replicas: 3 + template: + metadata: + labels: + app: csi-cephfsplugin-provisioner + spec: + serviceAccount: cephfs-csi-provisioner + containers: + - name: csi-provisioner + image: quay.io/k8scsi/csi-provisioner:v1.3.0 + args: + - "--csi-address=$(ADDRESS)" + - "--v=5" + - "--timeout=60s" + - "--enable-leader-election=true" + - "--leader-election-type=leases" + - "--retry-interval-start=500ms" + env: + - name: ADDRESS + value: unix:///csi/csi-provisioner.sock + imagePullPolicy: "IfNotPresent" + volumeMounts: + - name: socket-dir + mountPath: /csi + - name: csi-cephfsplugin-attacher + image: quay.io/k8scsi/csi-attacher:v1.2.0 + args: + - "--v=5" + - "--csi-address=$(ADDRESS)" + - "leader-election=true" + - "--leader-election-type=leases" + env: + - name: ADDRESS + value: /csi/csi-provisioner.sock + imagePullPolicy: "IfNotPresent" + volumeMounts: + - name: socket-dir + mountPath: /csi + - name: csi-cephfsplugin + securityContext: + privileged: true + capabilities: + add: ["SYS_ADMIN"] + # for stable functionality replace canary with latest release version + image: quay.io/cephcsi/cephcsi:canary + args: + - "--nodeid=$(NODE_ID)" + - "--type=cephfs" + - "--endpoint=$(CSI_ENDPOINT)" + - "--v=5" + - "--drivername=cephfs.csi.ceph.com" + - "--metadatastorage=k8s_configmap" + env: + - name: NODE_ID + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: CSI_ENDPOINT + value: unix:///csi/csi-provisioner.sock + imagePullPolicy: "IfNotPresent" + volumeMounts: + - name: socket-dir + mountPath: /csi + - name: host-sys + mountPath: /sys + - name: lib-modules + mountPath: /lib/modules + readOnly: true + - name: host-dev + mountPath: /dev + - name: ceph-csi-config + mountPath: /etc/ceph-csi-config/ + - name: keys-tmp-dir + mountPath: /tmp/csi/keys + volumes: + - name: socket-dir + hostPath: + path: /var/lib/kubelet/plugins/cephfs.csi.ceph.com + type: DirectoryOrCreate + - name: host-sys + hostPath: + path: /sys + - name: lib-modules + hostPath: + path: /lib/modules + - name: host-dev + hostPath: + path: /dev + - name: ceph-csi-config + configMap: + name: ceph-csi-config + - name: keys-tmp-dir + emptyDir: { + medium: "Memory" + } diff --git a/deploy/cephfs/kubernetes/v1.14+/csi-cephfsplugin.yaml b/deploy/cephfs/kubernetes/v1.14+/csi-cephfsplugin.yaml new file mode 100644 index 000000000..ac6dbd21c --- /dev/null +++ b/deploy/cephfs/kubernetes/v1.14+/csi-cephfsplugin.yaml @@ -0,0 +1,129 @@ +--- +kind: DaemonSet +apiVersion: apps/v1 +metadata: + name: csi-cephfsplugin +spec: + selector: + matchLabels: + app: csi-cephfsplugin + template: + metadata: + labels: + app: csi-cephfsplugin + spec: + serviceAccount: cephfs-csi-nodeplugin + hostNetwork: true + # to use e.g. Rook orchestrated cluster, and mons' FQDN is + # resolved through k8s service, set dns policy to cluster first + dnsPolicy: ClusterFirstWithHostNet + containers: + - name: driver-registrar + image: quay.io/k8scsi/csi-node-driver-registrar:v1.1.0 + args: + - "--v=5" + - "--csi-address=/csi/csi.sock" + - "--kubelet-registration-path=/var/lib/kubelet/plugins/cephfs.csi.ceph.com/csi.sock" + lifecycle: + preStop: + exec: + command: [ + "/bin/sh", "-c", + "rm -rf /registration/csi-cephfsplugin \ + /registration/csi-cephfsplugin-reg.sock" + ] + env: + - name: KUBE_NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + volumeMounts: + - name: socket-dir + mountPath: /csi + - name: registration-dir + mountPath: /registration + - name: csi-cephfsplugin + securityContext: + privileged: true + capabilities: + add: ["SYS_ADMIN"] + allowPrivilegeEscalation: true + # for stable functionality replace canary with latest release version + image: quay.io/cephcsi/cephcsi:canary + args: + - "--nodeid=$(NODE_ID)" + - "--type=cephfs" + - "--endpoint=$(CSI_ENDPOINT)" + - "--v=5" + - "--drivername=cephfs.csi.ceph.com" + - "--metadatastorage=k8s_configmap" + - "--mountcachedir=/mount-cache-dir" + env: + - name: NODE_ID + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: CSI_ENDPOINT + value: unix:///csi/csi.sock + imagePullPolicy: "IfNotPresent" + volumeMounts: + - name: mount-cache-dir + mountPath: /mount-cache-dir + - name: socket-dir + mountPath: /csi + - name: mountpoint-dir + mountPath: /var/lib/kubelet/pods + mountPropagation: Bidirectional + - name: plugin-dir + mountPath: /var/lib/kubelet/plugins + mountPropagation: "Bidirectional" + - name: host-sys + mountPath: /sys + - name: lib-modules + mountPath: /lib/modules + readOnly: true + - name: host-dev + mountPath: /dev + - name: ceph-csi-config + mountPath: /etc/ceph-csi-config/ + - name: keys-tmp-dir + mountPath: /tmp/csi/keys + volumes: + - name: mount-cache-dir + emptyDir: {} + - name: socket-dir + hostPath: + path: /var/lib/kubelet/plugins/cephfs.csi.ceph.com/ + type: DirectoryOrCreate + - name: registration-dir + hostPath: + path: /var/lib/kubelet/plugins_registry/ + type: Directory + - name: mountpoint-dir + hostPath: + path: /var/lib/kubelet/pods + type: DirectoryOrCreate + - name: plugin-dir + hostPath: + path: /var/lib/kubelet/plugins + type: Directory + - name: host-sys + hostPath: + path: /sys + - name: lib-modules + hostPath: + path: /lib/modules + - name: host-dev + hostPath: + path: /dev + - name: ceph-csi-config + configMap: + name: ceph-csi-config + - name: keys-tmp-dir + emptyDir: { + medium: "Memory" + } diff --git a/deploy/rbd/kubernetes/csi-config-map.yaml b/deploy/cephfs/kubernetes/v1.14+/csi-config-map.yaml similarity index 100% rename from deploy/rbd/kubernetes/csi-config-map.yaml rename to deploy/cephfs/kubernetes/v1.14+/csi-config-map.yaml diff --git a/deploy/cephfs/kubernetes/v1.14+/csi-nodeplugin-rbac.yaml b/deploy/cephfs/kubernetes/v1.14+/csi-nodeplugin-rbac.yaml new file mode 100644 index 000000000..4e4026bcc --- /dev/null +++ b/deploy/cephfs/kubernetes/v1.14+/csi-nodeplugin-rbac.yaml @@ -0,0 +1,53 @@ +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: cephfs-csi-nodeplugin + +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: cephfs-csi-nodeplugin +aggregationRule: + clusterRoleSelectors: + - matchLabels: + rbac.cephfs.csi.ceph.com/aggregate-to-cephfs-csi-nodeplugin: "true" +rules: [] +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: cephfs-csi-nodeplugin-rules + labels: + rbac.cephfs.csi.ceph.com/aggregate-to-cephfs-csi-nodeplugin: "true" +rules: + - apiGroups: [""] + resources: ["configmaps"] + verbs: ["get", "list"] + - apiGroups: [""] + resources: ["nodes"] + verbs: ["get", "list", "update"] + - apiGroups: [""] + resources: ["namespaces"] + verbs: ["get", "list"] + - apiGroups: [""] + resources: ["persistentvolumes"] + verbs: ["get", "list", "watch", "update"] + - apiGroups: ["storage.k8s.io"] + resources: ["volumeattachments"] + verbs: ["get", "list", "watch", "update"] + +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: cephfs-csi-nodeplugin +subjects: + - kind: ServiceAccount + name: cephfs-csi-nodeplugin + namespace: default +roleRef: + kind: ClusterRole + name: cephfs-csi-nodeplugin + apiGroup: rbac.authorization.k8s.io diff --git a/deploy/cephfs/kubernetes/v1.14+/csi-provisioner-rbac.yaml b/deploy/cephfs/kubernetes/v1.14+/csi-provisioner-rbac.yaml new file mode 100644 index 000000000..4cbd8cad2 --- /dev/null +++ b/deploy/cephfs/kubernetes/v1.14+/csi-provisioner-rbac.yaml @@ -0,0 +1,97 @@ +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: cephfs-csi-provisioner + +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: cephfs-external-provisioner-runner +aggregationRule: + clusterRoleSelectors: + - matchLabels: + rbac.cephfs.csi.ceph.com/aggregate-to-cephfs-external-provisioner-runner: "true" +rules: [] +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: cephfs-external-provisioner-runner-rules + labels: + rbac.cephfs.csi.ceph.com/aggregate-to-cephfs-external-provisioner-runner: "true" +rules: + - apiGroups: [""] + resources: ["nodes"] + verbs: ["get", "list", "watch"] + - apiGroups: [""] + resources: ["secrets"] + verbs: ["get", "list"] + - apiGroups: [""] + resources: ["events"] + verbs: ["list", "watch", "create", "update", "patch"] + - apiGroups: [""] + resources: ["persistentvolumes"] + verbs: ["get", "list", "watch", "create", "delete"] + - apiGroups: [""] + resources: ["persistentvolumeclaims"] + verbs: ["get", "list", "watch", "update"] + - apiGroups: ["storage.k8s.io"] + resources: ["storageclasses"] + verbs: ["get", "list", "watch"] + - apiGroups: ["csi.storage.k8s.io"] + resources: ["csinodeinfos"] + verbs: ["get", "list", "watch"] + - apiGroups: ["storage.k8s.io"] + resources: ["volumeattachments"] + verbs: ["get", "list", "watch", "update"] + +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: cephfs-csi-provisioner-role +subjects: + - kind: ServiceAccount + name: cephfs-csi-provisioner + namespace: default +roleRef: + kind: ClusterRole + name: cephfs-external-provisioner-runner + apiGroup: rbac.authorization.k8s.io + +--- +kind: Role +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + # replace with non-default namespace name + namespace: default + name: cephfs-external-provisioner-cfg +rules: + - apiGroups: [""] + resources: ["endpoints"] + verbs: ["get", "watch", "list", "delete", "update", "create"] + - apiGroups: [""] + resources: ["configmaps"] + verbs: ["get", "list", "create", "delete"] + - apiGroups: ["coordination.k8s.io"] + resources: ["leases"] + verbs: ["get", "watch", "list", "delete", "update", "create"] + +--- +kind: RoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: cephfs-csi-provisioner-role-cfg + # replace with non-default namespace name + namespace: default +subjects: + - kind: ServiceAccount + name: cephfs-csi-provisioner + # replace with non-default namespace name + namespace: default +roleRef: + kind: Role + name: cephfs-external-provisioner-cfg + apiGroup: rbac.authorization.k8s.io diff --git a/deploy/rbd/helm/.helmignore b/deploy/cephfs/kubernetes/v1.14+/helm/.helmignore similarity index 100% rename from deploy/rbd/helm/.helmignore rename to deploy/cephfs/kubernetes/v1.14+/helm/.helmignore diff --git a/deploy/cephfs/kubernetes/v1.14+/helm/Chart.yaml b/deploy/cephfs/kubernetes/v1.14+/helm/Chart.yaml new file mode 100644 index 000000000..5efb6e250 --- /dev/null +++ b/deploy/cephfs/kubernetes/v1.14+/helm/Chart.yaml @@ -0,0 +1,14 @@ +--- +apiVersion: v1 +appVersion: "1.0.0" +description: "Container Storage Interface (CSI) driver, +provisioner, and attacher for Ceph cephfs" +name: ceph-csi-cephfs +version: 0.8.0 +keywords: + - ceph + - cephfs + - ceph-csi +home: https://github.com/ceph/ceph-csi +sources: + - https://github.com/ceph/ceph-csi/tree/csi-v1.0/deploy/cephfs/helm diff --git a/deploy/cephfs/kubernetes/v1.14+/helm/README.md b/deploy/cephfs/kubernetes/v1.14+/helm/README.md new file mode 100644 index 000000000..fcd4a98a8 --- /dev/null +++ b/deploy/cephfs/kubernetes/v1.14+/helm/README.md @@ -0,0 +1,29 @@ +# ceph-csi-cephfs + +The ceph-csi-cephfs chart adds cephfs volume support to your cluster. + +## Install Chart + +To install the Chart into your Kubernetes cluster + +```bash +helm install --namespace "ceph-csi-cephfs" --name "ceph-csi-cephfs" ceph-csi/ceph-csi-cephfs +``` + +After installation succeeds, you can get a status of Chart + +```bash +helm status "ceph-csi-cephfs" +``` + +If you want to delete your Chart, use this command + +```bash +helm delete --purge "ceph-csi-cephfs" +``` + +If you want to delete the namespace, use this command + +```bash +kubectl delete namespace ceph-csi-rbd +``` diff --git a/deploy/cephfs/kubernetes/v1.14+/helm/templates/NOTES.txt b/deploy/cephfs/kubernetes/v1.14+/helm/templates/NOTES.txt new file mode 100644 index 000000000..3af9f2b57 --- /dev/null +++ b/deploy/cephfs/kubernetes/v1.14+/helm/templates/NOTES.txt @@ -0,0 +1,2 @@ +Examples on how to configure a storage class and start using the driver are here: +https://github.com/ceph/ceph-csi/tree/csi-v1.0/examples/cephfs diff --git a/deploy/cephfs/kubernetes/v1.14+/helm/templates/_helpers.tpl b/deploy/cephfs/kubernetes/v1.14+/helm/templates/_helpers.tpl new file mode 100644 index 000000000..635cca67b --- /dev/null +++ b/deploy/cephfs/kubernetes/v1.14+/helm/templates/_helpers.tpl @@ -0,0 +1,90 @@ +{{/* vim: set filetype=mustache: */}} +{{/* +Expand the name of the chart. +*/}} +{{- define "ceph-csi-cephfs.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "ceph-csi-cephfs.fullname" -}} +{{- if .Values.fullnameOverride -}} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- $name := default .Chart.Name .Values.nameOverride -}} +{{- if contains $name .Release.Name -}} +{{- .Release.Name | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}} +{{- end -}} +{{- end -}} +{{- end -}} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "ceph-csi-cephfs.nodeplugin.fullname" -}} +{{- if .Values.nodeplugin.fullnameOverride -}} +{{- .Values.nodeplugin.fullnameOverride | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- $name := default .Chart.Name .Values.nameOverride -}} +{{- if contains $name .Release.Name -}} +{{- printf "%s-%s" .Release.Name .Values.nodeplugin.name | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- printf "%s-%s-%s" .Release.Name $name .Values.nodeplugin.name | trunc 63 | trimSuffix "-" -}} +{{- end -}} +{{- end -}} +{{- end -}} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "ceph-csi-cephfs.provisioner.fullname" -}} +{{- if .Values.provisioner.fullnameOverride -}} +{{- .Values.provisioner.fullnameOverride | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- $name := default .Chart.Name .Values.nameOverride -}} +{{- if contains $name .Release.Name -}} +{{- printf "%s-%s" .Release.Name .Values.provisioner.name | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- printf "%s-%s-%s" .Release.Name $name .Values.provisioner.name | trunc 63 | trimSuffix "-" -}} +{{- end -}} +{{- end -}} +{{- end -}} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "ceph-csi-cephfs.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Create the name of the service account to use +*/}} +{{- define "ceph-csi-cephfs.serviceAccountName.nodeplugin" -}} +{{- if .Values.serviceAccounts.nodeplugin.create -}} + {{ default (include "ceph-csi-cephfs.nodeplugin.fullname" .) .Values.serviceAccounts.nodeplugin.name }} +{{- else -}} + {{ default "default" .Values.serviceAccounts.nodeplugin.name }} +{{- end -}} +{{- end -}} + +{{/* +Create the name of the service account to use +*/}} +{{- define "ceph-csi-cephfs.serviceAccountName.provisioner" -}} +{{- if .Values.serviceAccounts.provisioner.create -}} + {{ default (include "ceph-csi-cephfs.provisioner.fullname" .) .Values.serviceAccounts.provisioner.name }} +{{- else -}} + {{ default "default" .Values.serviceAccounts.provisioner.name }} +{{- end -}} +{{- end -}} diff --git a/deploy/rbd/helm/templates/csidriver-crd.yaml b/deploy/cephfs/kubernetes/v1.14+/helm/templates/csidriver-crd.yaml similarity index 100% rename from deploy/rbd/helm/templates/csidriver-crd.yaml rename to deploy/cephfs/kubernetes/v1.14+/helm/templates/csidriver-crd.yaml diff --git a/deploy/cephfs/kubernetes/v1.14+/helm/templates/csiplugin-configmap.yaml b/deploy/cephfs/kubernetes/v1.14+/helm/templates/csiplugin-configmap.yaml new file mode 100644 index 000000000..66b4a5545 --- /dev/null +++ b/deploy/cephfs/kubernetes/v1.14+/helm/templates/csiplugin-configmap.yaml @@ -0,0 +1,14 @@ +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ .Values.configMapName | quote }} + labels: + app: {{ include "ceph-csi-cephfs.name" . }} + chart: {{ include "ceph-csi-cephfs.chart" . }} + component: {{ .Values.provisioner.name }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} +data: + config.json: |- + [] diff --git a/deploy/cephfs/kubernetes/v1.14+/helm/templates/nodeplugin-clusterrole.yaml b/deploy/cephfs/kubernetes/v1.14+/helm/templates/nodeplugin-clusterrole.yaml new file mode 100644 index 000000000..4ed09bf9e --- /dev/null +++ b/deploy/cephfs/kubernetes/v1.14+/helm/templates/nodeplugin-clusterrole.yaml @@ -0,0 +1,17 @@ +{{- if .Values.rbac.create -}} +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: {{ include "ceph-csi-cephfs.nodeplugin.fullname" . }} + labels: + app: {{ include "ceph-csi-cephfs.name" . }} + chart: {{ include "ceph-csi-cephfs.chart" . }} + component: {{ .Values.nodeplugin.name }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} +aggregationRule: + clusterRoleSelectors: + - matchLabels: + rbac.cephfs.csi.ceph.com/aggregate-to-{{ include "ceph-csi-cephfs.nodeplugin.fullname" . }}: "true" +rules: [] +{{- end -}} diff --git a/deploy/cephfs/kubernetes/v1.14+/helm/templates/nodeplugin-clusterrolebinding.yaml b/deploy/cephfs/kubernetes/v1.14+/helm/templates/nodeplugin-clusterrolebinding.yaml new file mode 100644 index 000000000..24e21351c --- /dev/null +++ b/deploy/cephfs/kubernetes/v1.14+/helm/templates/nodeplugin-clusterrolebinding.yaml @@ -0,0 +1,20 @@ +{{- if .Values.rbac.create -}} +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: {{ include "ceph-csi-cephfs.nodeplugin.fullname" . }} + labels: + app: {{ include "ceph-csi-cephfs.name" . }} + chart: {{ include "ceph-csi-cephfs.chart" . }} + component: {{ .Values.nodeplugin.name }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} +subjects: + - kind: ServiceAccount + name: {{ include "ceph-csi-cephfs.serviceAccountName.nodeplugin" . }} + namespace: {{ .Release.Namespace }} +roleRef: + kind: ClusterRole + name: {{ include "ceph-csi-cephfs.nodeplugin.fullname" . }} + apiGroup: rbac.authorization.k8s.io +{{- end -}} diff --git a/deploy/cephfs/kubernetes/v1.14+/helm/templates/nodeplugin-daemonset.yaml b/deploy/cephfs/kubernetes/v1.14+/helm/templates/nodeplugin-daemonset.yaml new file mode 100644 index 000000000..2b341be87 --- /dev/null +++ b/deploy/cephfs/kubernetes/v1.14+/helm/templates/nodeplugin-daemonset.yaml @@ -0,0 +1,162 @@ +kind: DaemonSet +apiVersion: apps/v1 +metadata: + name: {{ include "ceph-csi-cephfs.nodeplugin.fullname" . }} + labels: + app: {{ include "ceph-csi-cephfs.name" . }} + chart: {{ include "ceph-csi-cephfs.chart" . }} + component: {{ .Values.nodeplugin.name }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} +spec: + selector: + matchLabels: + app: {{ include "ceph-csi-cephfs.name" . }} + component: {{ .Values.nodeplugin.name }} + release: {{ .Release.Name }} + template: + metadata: + labels: + app: {{ include "ceph-csi-cephfs.name" . }} + chart: {{ include "ceph-csi-cephfs.chart" . }} + component: {{ .Values.nodeplugin.name }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} + spec: + serviceAccountName: {{ include "ceph-csi-cephfs.serviceAccountName.nodeplugin" . }} + hostNetwork: true + hostPID: true + # to use e.g. Rook orchestrated cluster, and mons' FQDN is + # resolved through k8s service, set dns policy to cluster first + dnsPolicy: ClusterFirstWithHostNet + containers: + - name: driver-registrar + image: "{{ .Values.nodeplugin.registrar.image.repository }}:{{ .Values.nodeplugin.registrar.image.tag }}" + args: + - "--v=5" + - "--csi-address=/csi/{{ .Values.socketFile }}" + - "--kubelet-registration-path={{ .Values.socketDir }}/{{ .Values.socketFile }}" + lifecycle: + preStop: + exec: + command: [ + "/bin/sh", "-c", + 'rm -rf /registration/{{ .Values.driverName }} + /registration/{{ .Values.driverName }}-reg.sock' + ] + env: + - name: KUBE_NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + imagePullPolicy: {{ .Values.nodeplugin.registrar.image.pullPolicy }} + volumeMounts: + - name: plugin-dir + mountPath: /csi + - name: registration-dir + mountPath: /registration + resources: +{{ toYaml .Values.nodeplugin.registrar.resources | indent 12 }} + - name: csi-cephfsplugin + securityContext: + privileged: true + capabilities: + add: ["SYS_ADMIN"] + allowPrivilegeEscalation: true + image: "{{ .Values.nodeplugin.plugin.image.repository }}:{{ .Values.nodeplugin.plugin.image.tag }}" + args : + - "--nodeid=$(NODE_ID)" + - "--type=cephfs" + - "--endpoint=$(CSI_ENDPOINT)" + - "--v=5" + - "--drivername=$(DRIVER_NAME)" + - "--metadatastorage=k8s_configmap" + - "--mountcachedir=/mount-cache-dir" + env: + - name: HOST_ROOTFS + value: "/rootfs" + - name: DRIVER_NAME + value: {{ .Values.driverName }} + - name: NODE_ID + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: CSI_ENDPOINT + value: "unix:/{{ .Values.socketDir }}/{{ .Values.socketFile }}" + imagePullPolicy: {{ .Values.nodeplugin.plugin.image.pullPolicy }} + volumeMounts: + - name: mount-cache-dir + mountPath: /mount-cache-dir + - name: socket-dir + mountPath: {{ .Values.socketDir }} + - name: plugin-dir + mountPath: {{ .Values.pluginDir }} + mountPropagation: "Bidirectional" + - name: mointpoint-dir + mountPath: /var/lib/kubelet/pods + mountPropagation: "Bidirectional" + - mountPath: /dev + name: host-dev + - mountPath: /rootfs + name: host-rootfs + - mountPath: /sys + name: host-sys + - mountPath: /lib/modules + name: lib-modules + readOnly: true + - name: ceph-csi-config + mountPath: /etc/ceph-csi-config/ + - name: keys-tmp-dir + mountPath: /tmp/csi/keys + resources: +{{ toYaml .Values.nodeplugin.plugin.resources | indent 12 }} + volumes: + - name: mount-cache-dir + emptyDir: {} + - name: socket-dir + hostPath: + path: {{ .Values.socketDir }} + type: DirectoryOrCreate + - name: registration-dir + hostPath: + path: {{ .Values.registrationDir }} + type: Directory + - name: plugin-dir + hostPath: + path: {{ .Values.pluginDir }} + type: Directory + - name: mountpoint-dir + hostPath: + path: /var/lib/kubelet/pods + type: DirectoryOrCreate + - name: host-dev + hostPath: + path: /dev + - name: host-rootfs + hostPath: + path: / + - name: host-sys + hostPath: + path: /sys + - name: lib-modules + hostPath: + path: /lib/modules + - name: ceph-csi-config + configMap: + name: {{ .Values.configMapName | quote }} + - name: keys-tmp-dir + emptyDir: { + medium: "Memory" + } + {{- if .Values.nodeplugin.affinity -}} + affinity: +{{ toYaml .Values.nodeplugin.affinity . | indent 8 }} + {{- end -}} + {{- if .Values.nodeplugin.nodeSelector -}} + nodeSelector: +{{ toYaml .Values.nodeplugin.nodeSelector | indent 8 }} + {{- end -}} + {{- if .Values.nodeplugin.tolerations -}} + tolerations: +{{ toYaml .Values.nodeplugin.tolerations | indent 8 }} + {{- end -}} diff --git a/deploy/cephfs/kubernetes/v1.14+/helm/templates/nodeplugin-rules-clusterrole.yaml b/deploy/cephfs/kubernetes/v1.14+/helm/templates/nodeplugin-rules-clusterrole.yaml new file mode 100644 index 000000000..8b90d9ed8 --- /dev/null +++ b/deploy/cephfs/kubernetes/v1.14+/helm/templates/nodeplugin-rules-clusterrole.yaml @@ -0,0 +1,32 @@ +{{- if .Values.rbac.create -}} +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: {{ include "ceph-csi-cephfs.nodeplugin.fullname" . }}-rules + labels: + app: {{ include "ceph-csi-cephfs.name" . }} + chart: {{ include "ceph-csi-cephfs.chart" . }} + component: {{ .Values.nodeplugin.name }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} + rbac.cephfs.csi.ceph.com/aggregate-to-{{ include "ceph-csi-cephfs.nodeplugin.fullname" . }}: "true" +rules: + - apiGroups: [""] + resources: ["configmaps"] + verbs: ["get", "list"] + - apiGroups: [""] + resources: ["nodes"] + verbs: ["get", "list", "update"] + - apiGroups: [""] + resources: ["namespaces"] + verbs: ["get", "list"] + - apiGroups: [""] + resources: ["persistentvolumes"] + verbs: ["get", "list", "watch", "update"] + - apiGroups: ["storage.k8s.io"] + resources: ["volumeattachments"] + verbs: ["get", "list", "watch", "update"] + - apiGroups: [""] + resources: ["configmaps"] + verbs: ["get", "list"] +{{- end -}} diff --git a/deploy/cephfs/kubernetes/v1.14+/helm/templates/nodeplugin-serviceaccount.yaml b/deploy/cephfs/kubernetes/v1.14+/helm/templates/nodeplugin-serviceaccount.yaml new file mode 100644 index 000000000..88bd8f1bc --- /dev/null +++ b/deploy/cephfs/kubernetes/v1.14+/helm/templates/nodeplugin-serviceaccount.yaml @@ -0,0 +1,12 @@ +{{- if .Values.serviceAccounts.nodeplugin.create -}} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "ceph-csi-cephfs.serviceAccountName.nodeplugin" . }} + labels: + app: {{ include "ceph-csi-cephfs.name" . }} + chart: {{ include "ceph-csi-cephfs.chart" . }} + component: {{ .Values.nodeplugin.name }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} +{{- end -}} diff --git a/deploy/cephfs/kubernetes/v1.14+/helm/templates/provisioner-clusterrole.yaml b/deploy/cephfs/kubernetes/v1.14+/helm/templates/provisioner-clusterrole.yaml new file mode 100644 index 000000000..c656b139a --- /dev/null +++ b/deploy/cephfs/kubernetes/v1.14+/helm/templates/provisioner-clusterrole.yaml @@ -0,0 +1,17 @@ +{{- if .Values.rbac.create -}} +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: {{ include "ceph-csi-cephfs.provisioner.fullname" . }} + labels: + app: {{ include "ceph-csi-cephfs.name" . }} + chart: {{ include "ceph-csi-cephfs.chart" . }} + component: {{ .Values.provisioner.name }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} +aggregationRule: + clusterRoleSelectors: + - matchLabels: + rbac.cephfs.csi.ceph.com/aggregate-to-{{ include "ceph-csi-cephfs.provisioner.fullname" . }}: "true" +rules: [] +{{- end -}} diff --git a/deploy/cephfs/kubernetes/v1.14+/helm/templates/provisioner-clusterrolebinding.yaml b/deploy/cephfs/kubernetes/v1.14+/helm/templates/provisioner-clusterrolebinding.yaml new file mode 100644 index 000000000..82d5d1316 --- /dev/null +++ b/deploy/cephfs/kubernetes/v1.14+/helm/templates/provisioner-clusterrolebinding.yaml @@ -0,0 +1,20 @@ +{{- if .Values.rbac.create -}} +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: {{ include "ceph-csi-cephfs.provisioner.fullname" . }} + labels: + app: {{ include "ceph-csi-cephfs.name" . }} + chart: {{ include "ceph-csi-cephfs.chart" . }} + component: {{ .Values.provisioner.name }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} +subjects: + - kind: ServiceAccount + name: {{ include "ceph-csi-cephfs.serviceAccountName.provisioner" . }} + namespace: {{ .Release.Namespace }} +roleRef: + kind: ClusterRole + name: {{ include "ceph-csi-cephfs.provisioner.fullname" . }} + apiGroup: rbac.authorization.k8s.io +{{- end -}} diff --git a/deploy/cephfs/kubernetes/v1.14+/helm/templates/provisioner-deployment.yaml b/deploy/cephfs/kubernetes/v1.14+/helm/templates/provisioner-deployment.yaml new file mode 100644 index 000000000..e8da7b2e3 --- /dev/null +++ b/deploy/cephfs/kubernetes/v1.14+/helm/templates/provisioner-deployment.yaml @@ -0,0 +1,125 @@ +kind: Deployment +apiVersion: apps/v1 +metadata: + name: {{ include "ceph-csi-cephfs.provisioner.fullname" . }} + labels: + app: {{ include "ceph-csi-cephfs.name" . }} + chart: {{ include "ceph-csi-cephfs.chart" . }} + component: {{ .Values.provisioner.name }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} +spec: + replicas: {{ .Values.provisioner.replicas }} + selector: + matchLabels: + app: {{ include "ceph-csi-cephfs.name" . }} + component: {{ .Values.provisioner.name }} + release: {{ .Release.Name }} + template: + metadata: + labels: + app: {{ include "ceph-csi-cephfs.name" . }} + chart: {{ include "ceph-csi-cephfs.chart" . }} + component: {{ .Values.provisioner.name }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} + spec: + serviceAccountName: {{ include "ceph-csi-cephfs.serviceAccountName.provisioner" . }} + containers: + - name: csi-provisioner + image: "{{ .Values.provisioner.image.repository }}:{{ .Values.provisioner.image.tag }}" + args: + - "--csi-address=$(ADDRESS)" + - "--v=5" + - "--timeout=60s" + - "--enable-leader-election=true" + - "--leader-election-type=leases" + - "--retry-interval-start=500ms" + env: + - name: ADDRESS + value: "{{ .Values.socketDir }}/{{ .Values.socketFile }}" + imagePullPolicy: {{ .Values.provisioner.image.pullPolicy }} + volumeMounts: + - name: socket-dir + mountPath: {{ .Values.socketDir }} + resources: +{{ toYaml .Values.provisioner.resources | indent 12 }} + {{ if .Values.attacher.enabled }} + - name: csi-attacher + image: "{{ .Values.attacher.image.repository }}:{{ .Values.attacher.image.tag }}" + args: + - "--v=5" + - "--csi-address=$(ADDRESS)" + - "leader-election=true" + - "--leader-election-type=leases" + env: + - name: ADDRESS + value: "{{ .Values.socketDir }}/{{ .Values.socketFile }}" + imagePullPolicy: {{ .Values.attacher.image.pullPolicy }} + volumeMounts: + - name: socket-dir + mountPath: {{ .Values.socketDir }} + {{ end }} + - name: csi-cephfsplugin + securityContext: + privileged: true + capabilities: + add: ["SYS_ADMIN"] + allowPrivilegeEscalation: true + image: "{{ .Values.nodeplugin.plugin.image.repository }}:{{ .Values.nodeplugin.plugin.image.tag }}" + args : + - "--nodeid=$(NODE_ID)" + - "--type=cephfs" + - "--endpoint=$(CSI_ENDPOINT)" + - "--v=5" + - "--drivername=$(DRIVER_NAME)" + - "--metadatastorage=k8s_configmap" + env: + - name: HOST_ROOTFS + value: "/rootfs" + - name: DRIVER_NAME + value: {{ .Values.driverName }} + - name: NODE_ID + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: CSI_ENDPOINT + value: "unix:/{{ .Values.socketDir }}/{{ .Values.socketFile }}" + imagePullPolicy: {{ .Values.nodeplugin.plugin.image.pullPolicy }} + volumeMounts: + - name: socket-dir + mountPath: {{ .Values.socketDir }} + - name: host-rootfs + mountPath: "/rootfs" + - name: ceph-csi-config + mountPath: /etc/ceph-csi-config/ + - name: keys-tmp-dir + mountPath: /tmp/csi/keys + resources: +{{ toYaml .Values.nodeplugin.plugin.resources | indent 12 }} + volumes: + - name: socket-dir + emptyDir: {} +#FIXME this seems way too much. Why is it needed at all for this? + - name: host-rootfs + hostPath: + path: / + - name: ceph-csi-config + configMap: + name: {{ .Values.configMapName | quote }} + - name: keys-tmp-dir + emptyDir: { + medium: "Memory" + } + {{- if .Values.provisioner.affinity -}} + affinity: +{{ toYaml .Values.provisioner.affinity . | indent 8 }} + {{- end -}} + {{- if .Values.provisioner.nodeSelector -}} + nodeSelector: +{{ toYaml .Values.provisioner.nodeSelector | indent 8 }} + {{- end -}} + {{- if .Values.provisioner.tolerations -}} + tolerations: +{{ toYaml .Values.provisioner.tolerations | indent 8 }} + {{- end -}} diff --git a/deploy/cephfs/kubernetes/v1.14+/helm/templates/provisioner-role.yaml b/deploy/cephfs/kubernetes/v1.14+/helm/templates/provisioner-role.yaml new file mode 100644 index 000000000..dc572ed24 --- /dev/null +++ b/deploy/cephfs/kubernetes/v1.14+/helm/templates/provisioner-role.yaml @@ -0,0 +1,22 @@ +{{- if .Values.rbac.create -}} +kind: Role +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: {{ include "ceph-csi-cephfs.provisioner.fullname" . }} + labels: + app: {{ include "ceph-csi-cephfs.name" . }} + chart: {{ include "ceph-csi-cephfs.chart" . }} + component: {{ .Values.provisioner.name }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} +rules: + - apiGroups: [""] + resources: ["endpoints"] + verbs: ["get", "watch", "list", "delete", "update", "create"] + - apiGroups: [""] + resources: ["configmaps"] + verbs: ["get", "list", "watch", "create", "delete"] + - apiGroups: ["coordination.k8s.io"] + resources: ["leases"] + verbs: ["get", "watch", "list", "delete", "update", "create"] +{{- end -}} diff --git a/deploy/cephfs/kubernetes/v1.14+/helm/templates/provisioner-rolebinding.yaml b/deploy/cephfs/kubernetes/v1.14+/helm/templates/provisioner-rolebinding.yaml new file mode 100644 index 000000000..63dc9503b --- /dev/null +++ b/deploy/cephfs/kubernetes/v1.14+/helm/templates/provisioner-rolebinding.yaml @@ -0,0 +1,21 @@ +{{- if .Values.rbac.create -}} +kind: RoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: {{ include "ceph-csi-cephfs.provisioner.fullname" . }} + labels: + app: {{ include "ceph-csi-cephfs.name" . }} + chart: {{ include "ceph-csi-cephfs.chart" . }} + component: {{ .Values.provisioner.name }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} +subjects: + - kind: ServiceAccount + name: {{ include "ceph-csi-cephfs.serviceAccountName.provisioner" . }} + namespace: {{ .Release.Namespace }} +roleRef: + kind: Role + name: {{ include "ceph-csi-cephfs.provisioner.fullname" . }} + apiGroup: rbac.authorization.k8s.io + namespace: {{ .Release.Namespace }} +{{- end -}} diff --git a/deploy/cephfs/kubernetes/v1.14+/helm/templates/provisioner-rules-clusterrole.yaml b/deploy/cephfs/kubernetes/v1.14+/helm/templates/provisioner-rules-clusterrole.yaml new file mode 100644 index 000000000..2308b60cf --- /dev/null +++ b/deploy/cephfs/kubernetes/v1.14+/helm/templates/provisioner-rules-clusterrole.yaml @@ -0,0 +1,40 @@ +{{- if .Values.rbac.create -}} +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: {{ include "ceph-csi-cephfs.provisioner.fullname" . }}-rules + labels: + app: {{ include "ceph-csi-cephfs.name" . }} + chart: {{ include "ceph-csi-cephfs.chart" . }} + component: {{ .Values.provisioner.name }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} + rbac.cephfs.csi.ceph.com/aggregate-to-{{ include "ceph-csi-cephfs.provisioner.fullname" . }}: "true" +rules: + - apiGroups: [""] + resources: ["nodes"] + verbs: ["get", "list", "watch"] + - apiGroups: [""] + resources: ["secrets"] + verbs: ["get", "list"] + - apiGroups: [""] + resources: ["persistentvolumes"] + verbs: ["get", "list", "watch", "create", "delete"] + - apiGroups: [""] + resources: ["persistentvolumeclaims"] + verbs: ["get", "list", "watch", "update"] + - apiGroups: ["storage.k8s.io"] + resources: ["storageclasses"] + verbs: ["get", "list", "watch"] + - apiGroups: [""] + resources: ["events"] + verbs: ["list", "watch", "create", "update", "patch"] + - apiGroups: ["csi.storage.k8s.io"] + resources: ["csinodeinfos"] + verbs: ["get", "list", "watch"] + {{ if .Values.attacher.enabled }} + - apiGroups: ["storage.k8s.io"] + resources: ["volumeattachments"] + verbs: ["get", "list", "watch", "update"] + {{ end }} +{{- end -}} diff --git a/deploy/cephfs/kubernetes/v1.14+/helm/templates/provisioner-serviceaccount.yaml b/deploy/cephfs/kubernetes/v1.14+/helm/templates/provisioner-serviceaccount.yaml new file mode 100644 index 000000000..2c1d9f74f --- /dev/null +++ b/deploy/cephfs/kubernetes/v1.14+/helm/templates/provisioner-serviceaccount.yaml @@ -0,0 +1,12 @@ +{{- if .Values.serviceAccounts.provisioner.create -}} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "ceph-csi-cephfs.serviceAccountName.provisioner" . }} + labels: + app: {{ include "ceph-csi-cephfs.name" . }} + chart: {{ include "ceph-csi-cephfs.chart" . }} + component: {{ .Values.provisioner.name }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} +{{- end -}} diff --git a/deploy/cephfs/kubernetes/v1.14+/helm/values.yaml b/deploy/cephfs/kubernetes/v1.14+/helm/values.yaml new file mode 100644 index 000000000..af81e11b0 --- /dev/null +++ b/deploy/cephfs/kubernetes/v1.14+/helm/values.yaml @@ -0,0 +1,80 @@ +--- +rbac: + create: true + +serviceAccounts: + attacher: + create: true + name: + nodeplugin: + create: true + name: + provisioner: + create: true + name: + +socketDir: /var/lib/kubelet/plugins/cephfs.csi.ceph.com +socketFile: csi.sock +registrationDir: /var/lib/kubelet/plugins_registry +pluginDir: /var/lib/kubelet/plugins +driverName: cephfs.csi.ceph.com +configMapName: ceph-csi-config +attacher: + name: attacher + enabled: true + image: + repository: quay.io/k8scsi/csi-attacher + tag: v1.2.0 + pullPolicy: IfNotPresent + + resources: {} + + nodeSelector: {} + + tolerations: [] + + affinity: {} + +nodeplugin: + name: nodeplugin + + registrar: + image: + repository: quay.io/k8scsi/csi-node-driver-registrar + tag: v1.1.0 + pullPolicy: IfNotPresent + + resources: {} + + plugin: + image: + repository: quay.io/cephcsi/cephcsi + # for stable functionality replace canary with latest release version + tag: canary + pullPolicy: IfNotPresent + + resources: {} + + nodeSelector: {} + + tolerations: [] + + affinity: {} + +provisioner: + name: provisioner + + replicaCount: 3 + + image: + repository: quay.io/k8scsi/csi-provisioner + tag: v1.3.0 + pullPolicy: IfNotPresent + + resources: {} + + nodeSelector: {} + + tolerations: [] + + affinity: {} diff --git a/deploy/rbd/kubernetes/v1.13/csi-config-map.yaml b/deploy/rbd/kubernetes/v1.13/csi-config-map.yaml new file mode 100644 index 000000000..3efb0c1be --- /dev/null +++ b/deploy/rbd/kubernetes/v1.13/csi-config-map.yaml @@ -0,0 +1,8 @@ +--- +apiVersion: v1 +kind: ConfigMap +data: + config.json: |- + [] +metadata: + name: ceph-csi-config diff --git a/deploy/rbd/kubernetes/csi-nodeplugin-rbac.yaml b/deploy/rbd/kubernetes/v1.13/csi-nodeplugin-rbac.yaml similarity index 100% rename from deploy/rbd/kubernetes/csi-nodeplugin-rbac.yaml rename to deploy/rbd/kubernetes/v1.13/csi-nodeplugin-rbac.yaml diff --git a/deploy/rbd/kubernetes/csi-provisioner-rbac.yaml b/deploy/rbd/kubernetes/v1.13/csi-provisioner-rbac.yaml similarity index 100% rename from deploy/rbd/kubernetes/csi-provisioner-rbac.yaml rename to deploy/rbd/kubernetes/v1.13/csi-provisioner-rbac.yaml diff --git a/deploy/rbd/kubernetes/csi-rbdplugin-provisioner.yaml b/deploy/rbd/kubernetes/v1.13/csi-rbdplugin-provisioner.yaml similarity index 100% rename from deploy/rbd/kubernetes/csi-rbdplugin-provisioner.yaml rename to deploy/rbd/kubernetes/v1.13/csi-rbdplugin-provisioner.yaml diff --git a/deploy/rbd/kubernetes/csi-rbdplugin.yaml b/deploy/rbd/kubernetes/v1.13/csi-rbdplugin.yaml similarity index 100% rename from deploy/rbd/kubernetes/csi-rbdplugin.yaml rename to deploy/rbd/kubernetes/v1.13/csi-rbdplugin.yaml diff --git a/deploy/rbd/kubernetes/v1.13/helm/.helmignore b/deploy/rbd/kubernetes/v1.13/helm/.helmignore new file mode 100644 index 000000000..f0c131944 --- /dev/null +++ b/deploy/rbd/kubernetes/v1.13/helm/.helmignore @@ -0,0 +1,21 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*~ +# Various IDEs +.project +.idea/ +*.tmproj diff --git a/deploy/rbd/helm/Chart.yaml b/deploy/rbd/kubernetes/v1.13/helm/Chart.yaml similarity index 100% rename from deploy/rbd/helm/Chart.yaml rename to deploy/rbd/kubernetes/v1.13/helm/Chart.yaml diff --git a/deploy/rbd/helm/README.md b/deploy/rbd/kubernetes/v1.13/helm/README.md similarity index 100% rename from deploy/rbd/helm/README.md rename to deploy/rbd/kubernetes/v1.13/helm/README.md diff --git a/deploy/rbd/helm/templates/NOTES.txt b/deploy/rbd/kubernetes/v1.13/helm/templates/NOTES.txt similarity index 100% rename from deploy/rbd/helm/templates/NOTES.txt rename to deploy/rbd/kubernetes/v1.13/helm/templates/NOTES.txt diff --git a/deploy/rbd/helm/templates/_helpers.tpl b/deploy/rbd/kubernetes/v1.13/helm/templates/_helpers.tpl similarity index 100% rename from deploy/rbd/helm/templates/_helpers.tpl rename to deploy/rbd/kubernetes/v1.13/helm/templates/_helpers.tpl diff --git a/deploy/rbd/kubernetes/v1.13/helm/templates/csidriver-crd.yaml b/deploy/rbd/kubernetes/v1.13/helm/templates/csidriver-crd.yaml new file mode 100644 index 000000000..4c5021a72 --- /dev/null +++ b/deploy/rbd/kubernetes/v1.13/helm/templates/csidriver-crd.yaml @@ -0,0 +1,10 @@ +--- +{{ if not .Values.attacher.enabled }} +apiVersion: storage.k8s.io/v1beta1 +kind: CSIDriver +metadata: + name: {{ .Values.driverName }} +spec: + attachRequired: false + podInfoOnMount: false +{{ end }} diff --git a/deploy/rbd/helm/templates/csiplugin-configmap.yaml b/deploy/rbd/kubernetes/v1.13/helm/templates/csiplugin-configmap.yaml similarity index 100% rename from deploy/rbd/helm/templates/csiplugin-configmap.yaml rename to deploy/rbd/kubernetes/v1.13/helm/templates/csiplugin-configmap.yaml diff --git a/deploy/rbd/helm/templates/nodeplugin-clusterrole.yaml b/deploy/rbd/kubernetes/v1.13/helm/templates/nodeplugin-clusterrole.yaml similarity index 100% rename from deploy/rbd/helm/templates/nodeplugin-clusterrole.yaml rename to deploy/rbd/kubernetes/v1.13/helm/templates/nodeplugin-clusterrole.yaml diff --git a/deploy/rbd/helm/templates/nodeplugin-clusterrolebinding.yaml b/deploy/rbd/kubernetes/v1.13/helm/templates/nodeplugin-clusterrolebinding.yaml similarity index 100% rename from deploy/rbd/helm/templates/nodeplugin-clusterrolebinding.yaml rename to deploy/rbd/kubernetes/v1.13/helm/templates/nodeplugin-clusterrolebinding.yaml diff --git a/deploy/rbd/helm/templates/nodeplugin-daemonset.yaml b/deploy/rbd/kubernetes/v1.13/helm/templates/nodeplugin-daemonset.yaml similarity index 100% rename from deploy/rbd/helm/templates/nodeplugin-daemonset.yaml rename to deploy/rbd/kubernetes/v1.13/helm/templates/nodeplugin-daemonset.yaml diff --git a/deploy/rbd/helm/templates/nodeplugin-rules-clusterrole.yaml b/deploy/rbd/kubernetes/v1.13/helm/templates/nodeplugin-rules-clusterrole.yaml similarity index 100% rename from deploy/rbd/helm/templates/nodeplugin-rules-clusterrole.yaml rename to deploy/rbd/kubernetes/v1.13/helm/templates/nodeplugin-rules-clusterrole.yaml diff --git a/deploy/rbd/helm/templates/nodeplugin-serviceaccount.yaml b/deploy/rbd/kubernetes/v1.13/helm/templates/nodeplugin-serviceaccount.yaml similarity index 100% rename from deploy/rbd/helm/templates/nodeplugin-serviceaccount.yaml rename to deploy/rbd/kubernetes/v1.13/helm/templates/nodeplugin-serviceaccount.yaml diff --git a/deploy/rbd/helm/templates/provisioner-clusterrole.yaml b/deploy/rbd/kubernetes/v1.13/helm/templates/provisioner-clusterrole.yaml similarity index 100% rename from deploy/rbd/helm/templates/provisioner-clusterrole.yaml rename to deploy/rbd/kubernetes/v1.13/helm/templates/provisioner-clusterrole.yaml diff --git a/deploy/rbd/helm/templates/provisioner-clusterrolebinding.yaml b/deploy/rbd/kubernetes/v1.13/helm/templates/provisioner-clusterrolebinding.yaml similarity index 100% rename from deploy/rbd/helm/templates/provisioner-clusterrolebinding.yaml rename to deploy/rbd/kubernetes/v1.13/helm/templates/provisioner-clusterrolebinding.yaml diff --git a/deploy/rbd/helm/templates/provisioner-role.yaml b/deploy/rbd/kubernetes/v1.13/helm/templates/provisioner-role.yaml similarity index 100% rename from deploy/rbd/helm/templates/provisioner-role.yaml rename to deploy/rbd/kubernetes/v1.13/helm/templates/provisioner-role.yaml diff --git a/deploy/rbd/helm/templates/provisioner-rolebinding.yaml b/deploy/rbd/kubernetes/v1.13/helm/templates/provisioner-rolebinding.yaml similarity index 100% rename from deploy/rbd/helm/templates/provisioner-rolebinding.yaml rename to deploy/rbd/kubernetes/v1.13/helm/templates/provisioner-rolebinding.yaml diff --git a/deploy/rbd/helm/templates/provisioner-rules-clusterrole.yaml b/deploy/rbd/kubernetes/v1.13/helm/templates/provisioner-rules-clusterrole.yaml similarity index 100% rename from deploy/rbd/helm/templates/provisioner-rules-clusterrole.yaml rename to deploy/rbd/kubernetes/v1.13/helm/templates/provisioner-rules-clusterrole.yaml diff --git a/deploy/rbd/helm/templates/provisioner-service.yaml b/deploy/rbd/kubernetes/v1.13/helm/templates/provisioner-service.yaml similarity index 100% rename from deploy/rbd/helm/templates/provisioner-service.yaml rename to deploy/rbd/kubernetes/v1.13/helm/templates/provisioner-service.yaml diff --git a/deploy/rbd/helm/templates/provisioner-serviceaccount.yaml b/deploy/rbd/kubernetes/v1.13/helm/templates/provisioner-serviceaccount.yaml similarity index 100% rename from deploy/rbd/helm/templates/provisioner-serviceaccount.yaml rename to deploy/rbd/kubernetes/v1.13/helm/templates/provisioner-serviceaccount.yaml diff --git a/deploy/rbd/helm/templates/provisioner-statefulset.yaml b/deploy/rbd/kubernetes/v1.13/helm/templates/provisioner-statefulset.yaml similarity index 100% rename from deploy/rbd/helm/templates/provisioner-statefulset.yaml rename to deploy/rbd/kubernetes/v1.13/helm/templates/provisioner-statefulset.yaml diff --git a/deploy/rbd/helm/values.yaml b/deploy/rbd/kubernetes/v1.13/helm/values.yaml similarity index 100% rename from deploy/rbd/helm/values.yaml rename to deploy/rbd/kubernetes/v1.13/helm/values.yaml diff --git a/deploy/rbd/kubernetes/v1.14+/csi-config-map.yaml b/deploy/rbd/kubernetes/v1.14+/csi-config-map.yaml new file mode 100644 index 000000000..3efb0c1be --- /dev/null +++ b/deploy/rbd/kubernetes/v1.14+/csi-config-map.yaml @@ -0,0 +1,8 @@ +--- +apiVersion: v1 +kind: ConfigMap +data: + config.json: |- + [] +metadata: + name: ceph-csi-config diff --git a/deploy/rbd/kubernetes/v1.14+/csi-nodeplugin-rbac.yaml b/deploy/rbd/kubernetes/v1.14+/csi-nodeplugin-rbac.yaml new file mode 100644 index 000000000..4479cfff9 --- /dev/null +++ b/deploy/rbd/kubernetes/v1.14+/csi-nodeplugin-rbac.yaml @@ -0,0 +1,56 @@ +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: rbd-csi-nodeplugin + +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: rbd-csi-nodeplugin +aggregationRule: + clusterRoleSelectors: + - matchLabels: + rbac.rbd.csi.ceph.com/aggregate-to-rbd-csi-nodeplugin: "true" +rules: [] +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: rbd-csi-nodeplugin-rules + labels: + rbac.rbd.csi.ceph.com/aggregate-to-rbd-csi-nodeplugin: "true" +rules: + - apiGroups: [""] + resources: ["secrets"] + verbs: ["get", "list"] + - apiGroups: [""] + resources: ["nodes"] + verbs: ["get", "list", "update"] + - apiGroups: [""] + resources: ["namespaces"] + verbs: ["get", "list"] + - apiGroups: [""] + resources: ["persistentvolumes"] + verbs: ["get", "list", "watch", "update"] + - apiGroups: ["storage.k8s.io"] + resources: ["volumeattachments"] + verbs: ["get", "list", "watch", "update"] + - apiGroups: [""] + resources: ["configmaps"] + verbs: ["get", "list"] + +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: rbd-csi-nodeplugin +subjects: + - kind: ServiceAccount + name: rbd-csi-nodeplugin + namespace: default +roleRef: + kind: ClusterRole + name: rbd-csi-nodeplugin + apiGroup: rbac.authorization.k8s.io diff --git a/deploy/rbd/kubernetes/v1.14+/csi-provisioner-rbac.yaml b/deploy/rbd/kubernetes/v1.14+/csi-provisioner-rbac.yaml new file mode 100644 index 000000000..3054998b4 --- /dev/null +++ b/deploy/rbd/kubernetes/v1.14+/csi-provisioner-rbac.yaml @@ -0,0 +1,112 @@ +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: rbd-csi-provisioner + +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: rbd-external-provisioner-runner +aggregationRule: + clusterRoleSelectors: + - matchLabels: + rbac.rbd.csi.ceph.com/aggregate-to-rbd-external-provisioner-runner: "true" +rules: [] +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: rbd-external-provisioner-runner-rules + labels: + rbac.rbd.csi.ceph.com/aggregate-to-rbd-external-provisioner-runner: "true" +rules: + - apiGroups: [""] + resources: ["nodes"] + verbs: ["get", "list", "watch"] + - apiGroups: [""] + resources: ["secrets"] + verbs: ["get", "list"] + - apiGroups: [""] + resources: ["events"] + verbs: ["list", "watch", "create", "update", "patch"] + - apiGroups: [""] + resources: ["persistentvolumes"] + verbs: ["get", "list", "watch", "create", "update", "delete"] + - apiGroups: [""] + resources: ["persistentvolumeclaims"] + verbs: ["get", "list", "watch", "update"] + - apiGroups: ["storage.k8s.io"] + resources: ["storageclasses"] + verbs: ["get", "list", "watch"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshots"] + verbs: ["get", "list", "watch", "update"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshotcontents"] + verbs: ["create", "get", "list", "watch", "update", "delete"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshotclasses"] + verbs: ["get", "list", "watch"] + - apiGroups: ["apiextensions.k8s.io"] + resources: ["customresourcedefinitions"] + verbs: ["create", "list", "watch", "delete", "get", "update"] + - apiGroups: ["csi.storage.k8s.io"] + resources: ["csinodeinfos"] + verbs: ["get", "list", "watch"] + - apiGroups: ["storage.k8s.io"] + resources: ["volumeattachments"] + verbs: ["get", "list", "watch", "update"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshots/status"] + verbs: ["update"] + +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: rbd-csi-provisioner-role +subjects: + - kind: ServiceAccount + name: rbd-csi-provisioner + namespace: default +roleRef: + kind: ClusterRole + name: rbd-external-provisioner-runner + apiGroup: rbac.authorization.k8s.io + +--- +kind: Role +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + # replace with non-default namespace name + namespace: default + name: rbd-external-provisioner-cfg +rules: + - apiGroups: [""] + resources: ["endpoints"] + verbs: ["get", "watch", "list", "delete", "update", "create"] + - apiGroups: [""] + resources: ["configmaps"] + verbs: ["get", "list", "watch", "create", "delete"] + - apiGroups: ["coordination.k8s.io"] + resources: ["leases"] + verbs: ["get", "watch", "list", "delete", "update", "create"] + +--- +kind: RoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: rbd-csi-provisioner-role-cfg + # replace with non-default namespace name + namespace: default +subjects: + - kind: ServiceAccount + name: rbd-csi-provisioner + # replace with non-default namespace name + namespace: default +roleRef: + kind: Role + name: rbd-external-provisioner-cfg + apiGroup: rbac.authorization.k8s.io diff --git a/deploy/rbd/kubernetes/v1.14+/csi-rbdplugin-provisioner.yaml b/deploy/rbd/kubernetes/v1.14+/csi-rbdplugin-provisioner.yaml new file mode 100644 index 000000000..5644aa011 --- /dev/null +++ b/deploy/rbd/kubernetes/v1.14+/csi-rbdplugin-provisioner.yaml @@ -0,0 +1,127 @@ +--- +kind: Deployment +apiVersion: apps/v1 +metadata: + name: csi-rbdplugin-provisioner +spec: + replicas: 3 + selector: + matchLabels: + app: csi-rbdplugin-provisioner + template: + metadata: + labels: + app: csi-rbdplugin-provisioner + spec: + serviceAccount: rbd-csi-provisioner + containers: + - name: csi-provisioner + image: quay.io/k8scsi/csi-provisioner:v1.3.0 + args: + - "--csi-address=$(ADDRESS)" + - "--v=5" + - "--timeout=60s" + - "--retry-interval-start=500ms" + - "--enable-leader-election=true" + - "--leader-election-type=leases" + env: + - name: ADDRESS + value: unix:///csi/csi-provisioner.sock + imagePullPolicy: "IfNotPresent" + volumeMounts: + - name: socket-dir + mountPath: /csi + - name: csi-snapshotter + image: quay.io/k8scsi/csi-snapshotter:v1.2.0 + args: + - "--csi-address=$(ADDRESS)" + - "--v=5" + - "--timeout=60s" + - "leader-election=true" + env: + - name: ADDRESS + value: unix:///csi/csi-provisioner.sock + imagePullPolicy: Always + securityContext: + privileged: true + volumeMounts: + - name: socket-dir + mountPath: /csi + - name: csi-attacher + image: quay.io/k8scsi/csi-attacher:v1.2.0 + args: + - "--v=5" + - "--csi-address=$(ADDRESS)" + - "--leader-election=true" + - "--leader-election-type=leases" + env: + - name: ADDRESS + value: /csi/csi-provisioner.sock + imagePullPolicy: "IfNotPresent" + volumeMounts: + - name: socket-dir + mountPath: /csi + - name: csi-rbdplugin + securityContext: + privileged: true + capabilities: + add: ["SYS_ADMIN"] + # for stable functionality replace canary with latest release version + image: quay.io/cephcsi/cephcsi:canary + args: + - "--nodeid=$(NODE_ID)" + - "--type=rbd" + - "--endpoint=$(CSI_ENDPOINT)" + - "--v=5" + - "--drivername=rbd.csi.ceph.com" + - "--containerized=true" + env: + - name: HOST_ROOTFS + value: "/rootfs" + - name: NODE_ID + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: CSI_ENDPOINT + value: unix:///csi/csi-provisioner.sock + imagePullPolicy: "IfNotPresent" + volumeMounts: + - name: socket-dir + mountPath: /csi + - mountPath: /dev + name: host-dev + - mountPath: /rootfs + name: host-rootfs + - mountPath: /sys + name: host-sys + - mountPath: /lib/modules + name: lib-modules + readOnly: true + - name: ceph-csi-config + mountPath: /etc/ceph-csi-config/ + - name: keys-tmp-dir + mountPath: /tmp/csi/keys + volumes: + - name: host-dev + hostPath: + path: /dev + - name: host-rootfs + hostPath: + path: / + - name: host-sys + hostPath: + path: /sys + - name: lib-modules + hostPath: + path: /lib/modules + - name: socket-dir + hostPath: + path: /var/lib/kubelet/plugins/rbd.csi.ceph.com + type: DirectoryOrCreate + - name: ceph-csi-config + configMap: + name: ceph-csi-config + - name: keys-tmp-dir + emptyDir: { + medium: "Memory" + } diff --git a/deploy/rbd/kubernetes/v1.14+/csi-rbdplugin.yaml b/deploy/rbd/kubernetes/v1.14+/csi-rbdplugin.yaml new file mode 100644 index 000000000..e676aa279 --- /dev/null +++ b/deploy/rbd/kubernetes/v1.14+/csi-rbdplugin.yaml @@ -0,0 +1,130 @@ +--- +kind: DaemonSet +apiVersion: apps/v1 +metadata: + name: csi-rbdplugin +spec: + selector: + matchLabels: + app: csi-rbdplugin + updateStrategy: + type: OnDelete + template: + metadata: + labels: + app: csi-rbdplugin + spec: + serviceAccount: rbd-csi-nodeplugin + hostNetwork: true + hostPID: true + # to use e.g. Rook orchestrated cluster, and mons' FQDN is + # resolved through k8s service, set dns policy to cluster first + dnsPolicy: ClusterFirstWithHostNet + containers: + - name: driver-registrar + image: quay.io/k8scsi/csi-node-driver-registrar:v1.0.2 + args: + - "--v=5" + - "--csi-address=/csi/csi.sock" + - "--kubelet-registration-path=/var/lib/kubelet/plugins/rbd.csi.ceph.com/csi.sock" + lifecycle: + preStop: + exec: + command: [ + "/bin/sh", "-c", + "rm -rf /registration/rbd.csi.ceph.com \ + /registration/rbd.csi.ceph.com-reg.sock" + ] + env: + - name: KUBE_NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + volumeMounts: + - name: socket-dir + mountPath: /csi + - name: registration-dir + mountPath: /registration + - name: csi-rbdplugin + securityContext: + privileged: true + capabilities: + add: ["SYS_ADMIN"] + allowPrivilegeEscalation: true + # for stable functionality replace canary with latest release version + image: quay.io/cephcsi/cephcsi:canary + args: + - "--nodeid=$(NODE_ID)" + - "--type=rbd" + - "--endpoint=$(CSI_ENDPOINT)" + - "--v=5" + - "--drivername=rbd.csi.ceph.com" + - "--containerized=true" + env: + - name: HOST_ROOTFS + value: "/rootfs" + - name: NODE_ID + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: CSI_ENDPOINT + value: unix:///csi/csi.sock + imagePullPolicy: "IfNotPresent" + volumeMounts: + - name: socket-dir + mountPath: /csi + - mountPath: /dev + name: host-dev + - mountPath: /rootfs + name: host-rootfs + - mountPath: /sys + name: host-sys + - mountPath: /lib/modules + name: lib-modules + readOnly: true + - name: ceph-csi-config + mountPath: /etc/ceph-csi-config/ + - name: plugin-dir + mountPath: /var/lib/kubelet/plugins + mountPropagation: "Bidirectional" + - name: mountpoint-dir + mountPath: /var/lib/kubelet/pods + mountPropagation: "Bidirectional" + - name: keys-tmp-dir + mountPath: /tmp/csi/keys + volumes: + - name: socket-dir + hostPath: + path: /var/lib/kubelet/plugins/rbd.csi.ceph.com + type: DirectoryOrCreate + - name: plugin-dir + hostPath: + path: /var/lib/kubelet/plugins + type: Directory + - name: mountpoint-dir + hostPath: + path: /var/lib/kubelet/pods + type: DirectoryOrCreate + - name: registration-dir + hostPath: + path: /var/lib/kubelet/plugins_registry/ + type: Directory + - name: host-dev + hostPath: + path: /dev + - name: host-rootfs + hostPath: + path: / + - name: host-sys + hostPath: + path: /sys + - name: lib-modules + hostPath: + path: /lib/modules + - name: ceph-csi-config + configMap: + name: ceph-csi-config + - name: keys-tmp-dir + emptyDir: { + medium: "Memory" + } diff --git a/deploy/rbd/kubernetes/v1.14+/helm/.helmignore b/deploy/rbd/kubernetes/v1.14+/helm/.helmignore new file mode 100644 index 000000000..f0c131944 --- /dev/null +++ b/deploy/rbd/kubernetes/v1.14+/helm/.helmignore @@ -0,0 +1,21 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*~ +# Various IDEs +.project +.idea/ +*.tmproj diff --git a/deploy/rbd/kubernetes/v1.14+/helm/Chart.yaml b/deploy/rbd/kubernetes/v1.14+/helm/Chart.yaml new file mode 100644 index 000000000..a8a1595d0 --- /dev/null +++ b/deploy/rbd/kubernetes/v1.14+/helm/Chart.yaml @@ -0,0 +1,14 @@ +--- +apiVersion: v1 +appVersion: "1.0.0" +description: "Container Storage Interface (CSI) driver, +provisioner, snapshotter, and attacher for Ceph RBD" +name: ceph-csi-rbd +version: 0.8.0 +keywords: + - ceph + - rbd + - ceph-csi +home: https://github.com/ceph/ceph-csi +sources: + - https://github.com/ceph/ceph-csi/tree/csi-v1.0/deploy/rbd/helm diff --git a/deploy/rbd/kubernetes/v1.14+/helm/README.md b/deploy/rbd/kubernetes/v1.14+/helm/README.md new file mode 100644 index 000000000..250c10add --- /dev/null +++ b/deploy/rbd/kubernetes/v1.14+/helm/README.md @@ -0,0 +1,29 @@ +# ceph-csi-rbd + +The ceph-csi-rbd chart adds rbd volume support to your cluster. + +## Install Chart + +To install the Chart into your Kubernetes cluster + +```bash +helm install --namespace "ceph-csi-rbd" --name "ceph-csi-rbd" ceph-csi/ceph-csi-rbd +``` + +After installation succeeds, you can get a status of Chart + +```bash +helm status "ceph-csi-rbd" +``` + +If you want to delete your Chart, use this command + +```bash +helm delete --purge "ceph-csi-rbd" +``` + +If you want to delete the namespace, use this command + +```bash +kubectl delete namespace ceph-csi-rbd +``` diff --git a/deploy/rbd/kubernetes/v1.14+/helm/templates/NOTES.txt b/deploy/rbd/kubernetes/v1.14+/helm/templates/NOTES.txt new file mode 100644 index 000000000..fc1e29ca5 --- /dev/null +++ b/deploy/rbd/kubernetes/v1.14+/helm/templates/NOTES.txt @@ -0,0 +1,2 @@ +Examples on how to configure a storage class and start using the driver are here: +https://github.com/ceph/ceph-csi/tree/csi-v1.0/examples/rbd diff --git a/deploy/rbd/kubernetes/v1.14+/helm/templates/_helpers.tpl b/deploy/rbd/kubernetes/v1.14+/helm/templates/_helpers.tpl new file mode 100644 index 000000000..0a2613d63 --- /dev/null +++ b/deploy/rbd/kubernetes/v1.14+/helm/templates/_helpers.tpl @@ -0,0 +1,90 @@ +{{/* vim: set filetype=mustache: */}} +{{/* +Expand the name of the chart. +*/}} +{{- define "ceph-csi-rbd.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "ceph-csi-rbd.fullname" -}} +{{- if .Values.fullnameOverride -}} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- $name := default .Chart.Name .Values.nameOverride -}} +{{- if contains $name .Release.Name -}} +{{- .Release.Name | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}} +{{- end -}} +{{- end -}} +{{- end -}} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "ceph-csi-rbd.nodeplugin.fullname" -}} +{{- if .Values.nodeplugin.fullnameOverride -}} +{{- .Values.nodeplugin.fullnameOverride | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- $name := default .Chart.Name .Values.nameOverride -}} +{{- if contains $name .Release.Name -}} +{{- printf "%s-%s" .Release.Name .Values.nodeplugin.name | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- printf "%s-%s-%s" .Release.Name $name .Values.nodeplugin.name | trunc 63 | trimSuffix "-" -}} +{{- end -}} +{{- end -}} +{{- end -}} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "ceph-csi-rbd.provisioner.fullname" -}} +{{- if .Values.provisioner.fullnameOverride -}} +{{- .Values.provisioner.fullnameOverride | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- $name := default .Chart.Name .Values.nameOverride -}} +{{- if contains $name .Release.Name -}} +{{- printf "%s-%s" .Release.Name .Values.provisioner.name | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- printf "%s-%s-%s" .Release.Name $name .Values.provisioner.name | trunc 63 | trimSuffix "-" -}} +{{- end -}} +{{- end -}} +{{- end -}} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "ceph-csi-rbd.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Create the name of the service account to use +*/}} +{{- define "ceph-csi-rbd.serviceAccountName.nodeplugin" -}} +{{- if .Values.serviceAccounts.nodeplugin.create -}} + {{ default (include "ceph-csi-rbd.nodeplugin.fullname" .) .Values.serviceAccounts.nodeplugin.name }} +{{- else -}} + {{ default "default" .Values.serviceAccounts.nodeplugin.name }} +{{- end -}} +{{- end -}} + +{{/* +Create the name of the service account to use +*/}} +{{- define "ceph-csi-rbd.serviceAccountName.provisioner" -}} +{{- if .Values.serviceAccounts.provisioner.create -}} + {{ default (include "ceph-csi-rbd.provisioner.fullname" .) .Values.serviceAccounts.provisioner.name }} +{{- else -}} + {{ default "default" .Values.serviceAccounts.provisioner.name }} +{{- end -}} +{{- end -}} diff --git a/deploy/rbd/kubernetes/v1.14+/helm/templates/csidriver-crd.yaml b/deploy/rbd/kubernetes/v1.14+/helm/templates/csidriver-crd.yaml new file mode 100644 index 000000000..4c5021a72 --- /dev/null +++ b/deploy/rbd/kubernetes/v1.14+/helm/templates/csidriver-crd.yaml @@ -0,0 +1,10 @@ +--- +{{ if not .Values.attacher.enabled }} +apiVersion: storage.k8s.io/v1beta1 +kind: CSIDriver +metadata: + name: {{ .Values.driverName }} +spec: + attachRequired: false + podInfoOnMount: false +{{ end }} diff --git a/deploy/rbd/kubernetes/v1.14+/helm/templates/csiplugin-configmap.yaml b/deploy/rbd/kubernetes/v1.14+/helm/templates/csiplugin-configmap.yaml new file mode 100644 index 000000000..f7c7de5f5 --- /dev/null +++ b/deploy/rbd/kubernetes/v1.14+/helm/templates/csiplugin-configmap.yaml @@ -0,0 +1,14 @@ +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ .Values.configMapName | quote }} + labels: + app: {{ include "ceph-csi-rbd.name" . }} + chart: {{ include "ceph-csi-rbd.chart" . }} + component: {{ .Values.nodeplugin.name }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} +data: + config.json: |- + [] diff --git a/deploy/rbd/kubernetes/v1.14+/helm/templates/nodeplugin-clusterrole.yaml b/deploy/rbd/kubernetes/v1.14+/helm/templates/nodeplugin-clusterrole.yaml new file mode 100644 index 000000000..091cc2201 --- /dev/null +++ b/deploy/rbd/kubernetes/v1.14+/helm/templates/nodeplugin-clusterrole.yaml @@ -0,0 +1,17 @@ +{{- if .Values.rbac.create -}} +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: {{ include "ceph-csi-rbd.nodeplugin.fullname" . }} + labels: + app: {{ include "ceph-csi-rbd.name" . }} + chart: {{ include "ceph-csi-rbd.chart" . }} + component: {{ .Values.nodeplugin.name }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} +aggregationRule: + clusterRoleSelectors: + - matchLabels: + rbac.rbd.csi.ceph.com/aggregate-to-{{ include "ceph-csi-rbd.nodeplugin.fullname" . }}: "true" +rules: [] +{{- end -}} diff --git a/deploy/rbd/kubernetes/v1.14+/helm/templates/nodeplugin-clusterrolebinding.yaml b/deploy/rbd/kubernetes/v1.14+/helm/templates/nodeplugin-clusterrolebinding.yaml new file mode 100644 index 000000000..86abf2054 --- /dev/null +++ b/deploy/rbd/kubernetes/v1.14+/helm/templates/nodeplugin-clusterrolebinding.yaml @@ -0,0 +1,20 @@ +{{- if .Values.rbac.create -}} +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: {{ include "ceph-csi-rbd.nodeplugin.fullname" . }} + labels: + app: {{ include "ceph-csi-rbd.name" . }} + chart: {{ include "ceph-csi-rbd.chart" . }} + component: {{ .Values.nodeplugin.name }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} +subjects: + - kind: ServiceAccount + name: {{ include "ceph-csi-rbd.serviceAccountName.nodeplugin" . }} + namespace: {{ .Release.Namespace }} +roleRef: + kind: ClusterRole + name: {{ include "ceph-csi-rbd.nodeplugin.fullname" . }} + apiGroup: rbac.authorization.k8s.io +{{- end -}} diff --git a/deploy/rbd/kubernetes/v1.14+/helm/templates/nodeplugin-daemonset.yaml b/deploy/rbd/kubernetes/v1.14+/helm/templates/nodeplugin-daemonset.yaml new file mode 100644 index 000000000..331f57920 --- /dev/null +++ b/deploy/rbd/kubernetes/v1.14+/helm/templates/nodeplugin-daemonset.yaml @@ -0,0 +1,159 @@ +kind: DaemonSet +apiVersion: apps/v1 +metadata: + name: {{ include "ceph-csi-rbd.nodeplugin.fullname" . }} + labels: + app: {{ include "ceph-csi-rbd.name" . }} + chart: {{ include "ceph-csi-rbd.chart" . }} + component: {{ .Values.nodeplugin.name }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} +spec: + selector: + matchLabels: + app: {{ include "ceph-csi-rbd.name" . }} + component: {{ .Values.nodeplugin.name }} + release: {{ .Release.Name }} + updateStrategy: + type: OnDelete + template: + metadata: + labels: + app: {{ include "ceph-csi-rbd.name" . }} + chart: {{ include "ceph-csi-rbd.chart" . }} + component: {{ .Values.nodeplugin.name }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} + spec: + serviceAccountName: {{ include "ceph-csi-rbd.serviceAccountName.nodeplugin" . }} + hostNetwork: true + hostPID: true + # to use e.g. Rook orchestrated cluster, and mons' FQDN is + # resolved through k8s service, set dns policy to cluster first + dnsPolicy: ClusterFirstWithHostNet + containers: + - name: driver-registrar + image: "{{ .Values.nodeplugin.registrar.image.repository }}:{{ .Values.nodeplugin.registrar.image.tag }}" + args: + - "--v=5" + - "--csi-address=/csi/{{ .Values.socketFile }}" + - "--kubelet-registration-path={{ .Values.socketDir }}/{{ .Values.socketFile }}" + lifecycle: + preStop: + exec: + command: [ + "/bin/sh", "-c", + 'rm -rf /registration/{{ .Values.driverName }} + /registration/{{ .Values.driverName }}-reg.sock' + ] + env: + - name: KUBE_NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + imagePullPolicy: {{ .Values.nodeplugin.registrar.image.pullPolicy }} + volumeMounts: + - name: socket-dir + mountPath: /csi + - name: registration-dir + mountPath: /registration + resources: +{{ toYaml .Values.nodeplugin.registrar.resources | indent 12 }} + - name: csi-rbdplugin + securityContext: + privileged: true + capabilities: + add: ["SYS_ADMIN"] + allowPrivilegeEscalation: true + image: "{{ .Values.nodeplugin.plugin.image.repository }}:{{ .Values.nodeplugin.plugin.image.tag }}" + args : + - "--nodeid=$(NODE_ID)" + - "--type=rbd" + - "--endpoint=$(CSI_ENDPOINT)" + - "--v=5" + - "--drivername=$(DRIVER_NAME)" + - "--containerized=true" + env: + - name: HOST_ROOTFS + value: "/rootfs" + - name: DRIVER_NAME + value: {{ .Values.driverName }} + - name: NODE_ID + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: CSI_ENDPOINT + value: "unix:/{{ .Values.socketDir }}/{{ .Values.socketFile }}" + imagePullPolicy: {{ .Values.nodeplugin.plugin.image.pullPolicy }} + volumeMounts: + - name: socket-dir + mountPath: {{ .Values.socketDir }} + - name: plugin-dir + mountPath: {{ .Values.pluginDir }} + mountPropagation: "Bidirectional" + - name: mointpoint-dir + mountPath: /var/lib/kubelet/pods + mountPropagation: "Bidirectional" + - mountPath: /dev + name: host-dev + - mountPath: /rootfs + name: host-rootfs + - mountPath: /sys + name: host-sys + - mountPath: /lib/modules + name: lib-modules + readOnly: true + - name: ceph-csi-config + mountPath: /etc/ceph-csi-config/ + - name: keys-tmp-dir + mountPath: /tmp/csi/keys + resources: +{{ toYaml .Values.nodeplugin.plugin.resources | indent 12 }} + volumes: + - name: socket-dir + hostPath: + path: {{ .Values.socketDir }} + type: DirectoryOrCreate + - name: registration-dir + hostPath: + path: {{ .Values.registrationDir }} + type: Directory + - name: plugin-dir + hostPath: + path: {{ .Values.pluginDir }} + type: Directory + - name: mountpoint-dir + hostPath: + path: /var/lib/kubelet/pods + type: DirectoryOrCreate + - name: host-dev + hostPath: + path: /dev + - name: host-rootfs + hostPath: + path: / + - name: host-sys + hostPath: + path: /sys + - name: lib-modules + hostPath: + path: /lib/modules + - name: ceph-csi-config + configMap: + name: {{ .Values.configMapName | quote }} + - name: keys-tmp-dir + emptyDir: { + medium: "Memory" + } + {{- if .Values.nodeplugin.affinity -}} + affinity: +{{ toYaml .Values.nodeplugin.affinity . | indent 8 }} + {{- end -}} + {{- if .Values.nodeplugin.nodeSelector -}} + nodeSelector: +{{ toYaml .Values.nodeplugin.nodeSelector | indent 8 }} + {{- end -}} + {{- if .Values.nodeplugin.tolerations -}} + tolerations: +{{ toYaml .Values.nodeplugin.tolerations | indent 8 }} + {{- end -}} diff --git a/deploy/rbd/kubernetes/v1.14+/helm/templates/nodeplugin-rules-clusterrole.yaml b/deploy/rbd/kubernetes/v1.14+/helm/templates/nodeplugin-rules-clusterrole.yaml new file mode 100644 index 000000000..d80ebaea0 --- /dev/null +++ b/deploy/rbd/kubernetes/v1.14+/helm/templates/nodeplugin-rules-clusterrole.yaml @@ -0,0 +1,29 @@ +{{- if .Values.rbac.create -}} +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: {{ include "ceph-csi-rbd.nodeplugin.fullname" . }}-rules + labels: + app: {{ include "ceph-csi-rbd.name" . }} + chart: {{ include "ceph-csi-rbd.chart" . }} + component: {{ .Values.nodeplugin.name }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} + rbac.rbd.csi.ceph.com/aggregate-to-{{ include "ceph-csi-rbd.nodeplugin.fullname" . }}: "true" +rules: + - apiGroups: [""] + resources: ["nodes"] + verbs: ["get", "list", "update"] + - apiGroups: [""] + resources: ["namespaces"] + verbs: ["get", "list"] + - apiGroups: [""] + resources: ["persistentvolumes"] + verbs: ["get", "list", "watch", "update"] + - apiGroups: ["storage.k8s.io"] + resources: ["volumeattachments"] + verbs: ["get", "list", "watch", "update"] + - apiGroups: [""] + resources: ["configmaps"] + verbs: ["get", "list"] +{{- end -}} diff --git a/deploy/rbd/kubernetes/v1.14+/helm/templates/nodeplugin-serviceaccount.yaml b/deploy/rbd/kubernetes/v1.14+/helm/templates/nodeplugin-serviceaccount.yaml new file mode 100644 index 000000000..9dd41e4f6 --- /dev/null +++ b/deploy/rbd/kubernetes/v1.14+/helm/templates/nodeplugin-serviceaccount.yaml @@ -0,0 +1,12 @@ +{{- if .Values.serviceAccounts.nodeplugin.create -}} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "ceph-csi-rbd.serviceAccountName.nodeplugin" . }} + labels: + app: {{ include "ceph-csi-rbd.name" . }} + chart: {{ include "ceph-csi-rbd.chart" . }} + component: {{ .Values.nodeplugin.name }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} +{{- end -}} diff --git a/deploy/rbd/kubernetes/v1.14+/helm/templates/provisioner-clusterrole.yaml b/deploy/rbd/kubernetes/v1.14+/helm/templates/provisioner-clusterrole.yaml new file mode 100644 index 000000000..21745ed49 --- /dev/null +++ b/deploy/rbd/kubernetes/v1.14+/helm/templates/provisioner-clusterrole.yaml @@ -0,0 +1,17 @@ +{{- if .Values.rbac.create -}} +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: {{ include "ceph-csi-rbd.provisioner.fullname" . }} + labels: + app: {{ include "ceph-csi-rbd.name" . }} + chart: {{ include "ceph-csi-rbd.chart" . }} + component: {{ .Values.provisioner.name }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} +aggregationRule: + clusterRoleSelectors: + - matchLabels: + rbac.rbd.csi.ceph.com/aggregate-to-{{ include "ceph-csi-rbd.provisioner.fullname" . }}: "true" +rules: [] +{{- end -}} diff --git a/deploy/rbd/kubernetes/v1.14+/helm/templates/provisioner-clusterrolebinding.yaml b/deploy/rbd/kubernetes/v1.14+/helm/templates/provisioner-clusterrolebinding.yaml new file mode 100644 index 000000000..5a086103a --- /dev/null +++ b/deploy/rbd/kubernetes/v1.14+/helm/templates/provisioner-clusterrolebinding.yaml @@ -0,0 +1,20 @@ +{{- if .Values.rbac.create -}} +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: {{ include "ceph-csi-rbd.provisioner.fullname" . }} + labels: + app: {{ include "ceph-csi-rbd.name" . }} + chart: {{ include "ceph-csi-rbd.chart" . }} + component: {{ .Values.provisioner.name }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} +subjects: + - kind: ServiceAccount + name: {{ include "ceph-csi-rbd.serviceAccountName.provisioner" . }} + namespace: {{ .Release.Namespace }} +roleRef: + kind: ClusterRole + name: {{ include "ceph-csi-rbd.provisioner.fullname" . }} + apiGroup: rbac.authorization.k8s.io +{{- end -}} diff --git a/deploy/rbd/kubernetes/v1.14+/helm/templates/provisioner-deployment.yaml b/deploy/rbd/kubernetes/v1.14+/helm/templates/provisioner-deployment.yaml new file mode 100644 index 000000000..84e9a3806 --- /dev/null +++ b/deploy/rbd/kubernetes/v1.14+/helm/templates/provisioner-deployment.yaml @@ -0,0 +1,143 @@ +kind: Deployment +apiVersion: apps/v1 +metadata: + name: {{ include "ceph-csi-rbd.provisioner.fullname" . }} + labels: + app: {{ include "ceph-csi-rbd.name" . }} + chart: {{ include "ceph-csi-rbd.chart" . }} + component: {{ .Values.provisioner.name }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} +spec: + replicas: {{ .Values.provisioner.replicas }} + selector: + matchLabels: + app: {{ include "ceph-csi-rbd.name" . }} + component: {{ .Values.provisioner.name }} + release: {{ .Release.Name }} + template: + metadata: + labels: + app: {{ include "ceph-csi-rbd.name" . }} + chart: {{ include "ceph-csi-rbd.chart" . }} + component: {{ .Values.provisioner.name }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} + spec: + serviceAccountName: {{ include "ceph-csi-rbd.serviceAccountName.provisioner" . }} + containers: + - name: csi-provisioner + image: "{{ .Values.provisioner.image.repository }}:{{ .Values.provisioner.image.tag }}" + args: + - "--csi-address=$(ADDRESS)" + - "--v=5" + - "--timeout=60s" + - "--enable-leader-election=true" + - "--leader-election-type=leases" + - "--retry-interval-start=500ms" + env: + - name: ADDRESS + value: "{{ .Values.socketDir }}/{{ .Values.socketFile }}" + imagePullPolicy: {{ .Values.provisioner.image.pullPolicy }} + volumeMounts: + - name: socket-dir + mountPath: {{ .Values.socketDir }} + resources: +{{ toYaml .Values.provisioner.resources | indent 12 }} + - name: csi-snapshotter + image: {{ .Values.snapshotter.image.repository }}:{{ .Values.snapshotter.image.tag }} + imagePullPolicy: {{ .Values.nodeplugin.plugin.image.pullPolicy }} + args: + - "--csi-address=$(ADDRESS)" + - "--v=5" + - "--timeout=60s" + - "leader-election=true" + env: + - name: ADDRESS + value: "{{ .Values.socketDir }}/{{ .Values.socketFile }}" + securityContext: + privileged: true + volumeMounts: + - name: socket-dir + mountPath: {{ .Values.socketDir }} + resources: +{{ toYaml .Values.snapshotter.resources | indent 12 }} + {{ if .Values.attacher.enabled }} + - name: csi-attacher + image: "{{ .Values.attacher.image.repository }}:{{ .Values.attacher.image.tag }}" + args: + - "--v=5" + - "--csi-address=$(ADDRESS)" + - "leader-election=true" + - "--leader-election-type=leases" + env: + - name: ADDRESS + value: "{{ .Values.socketDir }}/{{ .Values.socketFile }}" + imagePullPolicy: {{ .Values.attacher.image.pullPolicy }} + volumeMounts: + - name: socket-dir + mountPath: {{ .Values.socketDir }} + {{ end }} + - name: csi-rbdplugin + securityContext: + privileged: true + capabilities: + add: ["SYS_ADMIN"] + allowPrivilegeEscalation: true + image: "{{ .Values.nodeplugin.plugin.image.repository }}:{{ .Values.nodeplugin.plugin.image.tag }}" + args : + - "--nodeid=$(NODE_ID)" + - "--type=rbd" + - "--endpoint=$(CSI_ENDPOINT)" + - "--v=5" + - "--drivername=$(DRIVER_NAME)" + - "--containerized=true" + env: + - name: HOST_ROOTFS + value: "/rootfs" + - name: DRIVER_NAME + value: {{ .Values.driverName }} + - name: NODE_ID + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: CSI_ENDPOINT + value: "unix:/{{ .Values.socketDir }}/{{ .Values.socketFile }}" + imagePullPolicy: {{ .Values.nodeplugin.plugin.image.pullPolicy }} + volumeMounts: + - name: socket-dir + mountPath: {{ .Values.socketDir }} + - name: host-rootfs + mountPath: "/rootfs" + - name: ceph-csi-config + mountPath: /etc/ceph-csi-config/ + - name: keys-tmp-dir + mountPath: /tmp/csi/keys + resources: +{{ toYaml .Values.nodeplugin.plugin.resources | indent 12 }} + volumes: + - name: socket-dir + emptyDir: {} +#FIXME this seems way too much. Why is it needed at all for this? + - name: host-rootfs + hostPath: + path: / + - name: ceph-csi-config + configMap: + name: {{ .Values.configMapName | quote }} + - name: keys-tmp-dir + emptyDir: { + medium: "Memory" + } + {{- if .Values.provisioner.affinity -}} + affinity: +{{ toYaml .Values.provisioner.affinity . | indent 8 }} + {{- end -}} + {{- if .Values.provisioner.nodeSelector -}} + nodeSelector: +{{ toYaml .Values.provisioner.nodeSelector | indent 8 }} + {{- end -}} + {{- if .Values.provisioner.tolerations -}} + tolerations: +{{ toYaml .Values.provisioner.tolerations | indent 8 }} + {{- end -}} diff --git a/deploy/rbd/kubernetes/v1.14+/helm/templates/provisioner-role.yaml b/deploy/rbd/kubernetes/v1.14+/helm/templates/provisioner-role.yaml new file mode 100644 index 000000000..6294b4a27 --- /dev/null +++ b/deploy/rbd/kubernetes/v1.14+/helm/templates/provisioner-role.yaml @@ -0,0 +1,19 @@ +{{- if .Values.rbac.create -}} +kind: Role +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: {{ include "ceph-csi-rbd.provisioner.fullname" . }} + labels: + app: {{ include "ceph-csi-rbd.name" . }} + chart: {{ include "ceph-csi-rbd.chart" . }} + component: {{ .Values.provisioner.name }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} +rules: + - apiGroups: [""] + resources: ["configmaps"] + verbs: ["get", "list", "watch", "create", "delete"] + - apiGroups: ["coordination.k8s.io"] + resources: ["leases"] + verbs: ["get", "watch", "list", "delete", "update", "create"] +{{- end -}} diff --git a/deploy/rbd/kubernetes/v1.14+/helm/templates/provisioner-rolebinding.yaml b/deploy/rbd/kubernetes/v1.14+/helm/templates/provisioner-rolebinding.yaml new file mode 100644 index 000000000..34f8356bb --- /dev/null +++ b/deploy/rbd/kubernetes/v1.14+/helm/templates/provisioner-rolebinding.yaml @@ -0,0 +1,21 @@ +{{- if .Values.rbac.create -}} +kind: RoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: {{ include "ceph-csi-rbd.provisioner.fullname" . }} + labels: + app: {{ include "ceph-csi-rbd.name" . }} + chart: {{ include "ceph-csi-rbd.chart" . }} + component: {{ .Values.provisioner.name }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} +subjects: + - kind: ServiceAccount + name: {{ include "ceph-csi-rbd.serviceAccountName.provisioner" . }} + namespace: {{ .Release.Namespace }} +roleRef: + kind: Role + name: {{ include "ceph-csi-rbd.provisioner.fullname" . }} + apiGroup: rbac.authorization.k8s.io + namespace: {{ .Release.Namespace }} +{{- end -}} diff --git a/deploy/rbd/kubernetes/v1.14+/helm/templates/provisioner-rules-clusterrole.yaml b/deploy/rbd/kubernetes/v1.14+/helm/templates/provisioner-rules-clusterrole.yaml new file mode 100644 index 000000000..9f2885c66 --- /dev/null +++ b/deploy/rbd/kubernetes/v1.14+/helm/templates/provisioner-rules-clusterrole.yaml @@ -0,0 +1,58 @@ +{{- if .Values.rbac.create -}} +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: {{ include "ceph-csi-rbd.provisioner.fullname" . }}-rules + labels: + app: {{ include "ceph-csi-rbd.name" . }} + chart: {{ include "ceph-csi-rbd.chart" . }} + component: {{ .Values.provisioner.name }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} + rbac.rbd.csi.ceph.com/aggregate-to-{{ include "ceph-csi-rbd.provisioner.fullname" . }}: "true" +rules: + - apiGroups: [""] + resources: ["nodes"] + verbs: ["get", "list", "watch"] + - apiGroups: [""] + resources: ["secrets"] + verbs: ["get", "list"] + - apiGroups: [""] + resources: ["persistentvolumes"] + verbs: ["get", "list", "watch", "create", "update", "delete"] + - apiGroups: [""] + resources: ["persistentvolumeclaims"] + verbs: ["get", "list", "watch", "update"] + - apiGroups: ["storage.k8s.io"] + resources: ["storageclasses"] + verbs: ["get", "list", "watch"] + - apiGroups: [""] + resources: ["events"] + verbs: ["list", "watch", "create", "update", "patch"] + - apiGroups: [""] + resources: ["endpoints"] + verbs: ["get", "create", "update"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshots"] + verbs: ["get", "list", "watch", "update"] + {{ if .Values.attacher.enabled }} + - apiGroups: ["storage.k8s.io"] + resources: ["volumeattachments"] + verbs: ["get", "list", "watch", "update"] + {{ end }} + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshotcontents"] + verbs: ["create", "get", "list", "watch", "update", "delete"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshotclasses"] + verbs: ["get", "list", "watch"] + - apiGroups: ["apiextensions.k8s.io"] + resources: ["customresourcedefinitions"] + verbs: ["create", "list", "watch", "delete", "get", "update"] + - apiGroups: ["csi.storage.k8s.io"] + resources: ["csinodeinfos"] + verbs: ["get", "list", "watch"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshots/status"] + verbs: ["update"] +{{- end -}} diff --git a/deploy/rbd/kubernetes/v1.14+/helm/templates/provisioner-serviceaccount.yaml b/deploy/rbd/kubernetes/v1.14+/helm/templates/provisioner-serviceaccount.yaml new file mode 100644 index 000000000..4c0f76297 --- /dev/null +++ b/deploy/rbd/kubernetes/v1.14+/helm/templates/provisioner-serviceaccount.yaml @@ -0,0 +1,12 @@ +{{- if .Values.serviceAccounts.provisioner.create -}} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "ceph-csi-rbd.serviceAccountName.provisioner" . }} + labels: + app: {{ include "ceph-csi-rbd.name" . }} + chart: {{ include "ceph-csi-rbd.chart" . }} + component: {{ .Values.provisioner.name }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} +{{- end -}} diff --git a/deploy/rbd/kubernetes/v1.14+/helm/values.yaml b/deploy/rbd/kubernetes/v1.14+/helm/values.yaml new file mode 100644 index 000000000..b5c81a4e3 --- /dev/null +++ b/deploy/rbd/kubernetes/v1.14+/helm/values.yaml @@ -0,0 +1,89 @@ +--- +rbac: + create: true + +serviceAccounts: + attacher: + create: true + name: + nodeplugin: + create: true + name: + provisioner: + create: true + name: + +socketDir: /var/lib/kubelet/plugins/rbd.csi.ceph.com +socketFile: csi.sock +registrationDir: /var/lib/kubelet/plugins_registry +pluginDir: /var/lib/kubelet/plugins +driverName: rbd.csi.ceph.com +configMapName: ceph-csi-config + +attacher: + name: attacher + enabled: true + image: + repository: quay.io/k8scsi/csi-attacher + tag: v1.2.0 + pullPolicy: IfNotPresent + + resources: {} + + nodeSelector: {} + + tolerations: [] + + affinity: {} + +nodeplugin: + name: nodeplugin + + registrar: + image: + repository: quay.io/k8scsi/csi-node-driver-registrar + tag: v1.1.0 + pullPolicy: IfNotPresent + + resources: {} + + plugin: + image: + repository: quay.io/cephcsi/cephcsi + # for stable functionality replace canary with latest release version + tag: canary + pullPolicy: IfNotPresent + + resources: {} + + nodeSelector: {} + + tolerations: [] + + affinity: {} + +provisioner: + name: provisioner + + replicaCount: 3 + + image: + repository: quay.io/k8scsi/csi-provisioner + tag: v1.3.0 + pullPolicy: IfNotPresent + + resources: {} + + nodeSelector: {} + + tolerations: [] + + affinity: {} + +snapshotter: + image: + repository: quay.io/k8scsi/csi-snapshotter + tag: v1.2.0 + pullPolicy: IfNotPresent + + resources: {} diff --git a/docs/deploy-cephfs.md b/docs/deploy-cephfs.md index bc5ca0d50..fcffbf954 100644 --- a/docs/deploy-cephfs.md +++ b/docs/deploy-cephfs.md @@ -42,18 +42,17 @@ that should be resolved in v14.2.3. **Available command line arguments:** -| Option | Default value | Description | -| ----------------- | --------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `--endpoint` | `unix://tmp/csi.sock` | CSI endpoint, must be a UNIX socket | -| `--drivername` | `cephfs.csi.ceph.com` | Name of the driver (Kubernetes: `provisioner` field in StorageClass must correspond to this value) | -| `--nodeid` | _empty_ | This node's ID | -| `--type` | _empty_ | Driver type `[rbd | cephfs]` If the driver type is set to `rbd` it will act as a `rbd plugin` or if it's set to `cephfs` will act as a `cephfs plugin` | -| `--volumemounter` | _empty_ | Default volume mounter. Available options are `kernel` and `fuse`. This is the mount method used if volume parameters don't specify otherwise. If left unspecified, the driver will first probe for `ceph-fuse` in system's path and will choose Ceph kernel client if probing failed. | -| `--mountcachedir` | _empty_ | Volume mount cache info save dir. If left unspecified, the dirver will not record mount info, or it will save mount info and when driver restart it will remount volume it cached. | -| `--instanceid` | "default" | Unique ID distinguishing this instance of Ceph CSI among other instances, when sharing Ceph clusters across CSI instances for provisioning | -| `--pluginpath` | "/var/lib/kubelet/plugins/" | The location of cephcsi plugin on host | - -| `--metadatastorage` | _empty_ | Points to where older (1.0.0 or older plugin versions) metadata about provisioned volumes are kept, as file or in as k8s configmap (`node` or `k8s_configmap` respectively) | +| Option | Default value | Description | +| ------------------- | --------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `--endpoint` | `unix://tmp/csi.sock` | CSI endpoint, must be a UNIX socket | +| `--drivername` | `cephfs.csi.ceph.com` | Name of the driver (Kubernetes: `provisioner` field in StorageClass must correspond to this value) | +| `--nodeid` | _empty_ | This node's ID | +| `--type` | _empty_ | Driver type `[rbd | cephfs]` If the driver type is set to `rbd` it will act as a `rbd plugin` or if it's set to `cephfs` will act as a `cephfs plugin` | +| `--volumemounter` | _empty_ | Default volume mounter. Available options are `kernel` and `fuse`. This is the mount method used if volume parameters don't specify otherwise. If left unspecified, the driver will first probe for `ceph-fuse` in system's path and will choose Ceph kernel client if probing failed. | +| `--mountcachedir` | _empty_ | Volume mount cache info save dir. If left unspecified, the dirver will not record mount info, or it will save mount info and when driver restart it will remount volume it cached. | +| `--instanceid` | "default" | Unique ID distinguishing this instance of Ceph CSI among other instances, when sharing Ceph clusters across CSI instances for provisioning | +| `--pluginpath` | "/var/lib/kubelet/plugins/" | The location of cephcsi plugin on host | +| `--metadatastorage` | _empty_ | Points to where older (1.0.0 or older plugin versions) metadata about provisioned volumes are kept, as file or in as k8s configmap (`node` or `k8s_configmap` respectively) | **Available environmental variables:** @@ -102,7 +101,11 @@ for a zero-sized volume means no quota attribute will be set. ## Deployment with Kubernetes -Requires Kubernetes 1.13 +Requires Kubernetes 1.13+ + +if your cluster version is 1.13.x please use [cephfs v1.13 +templates](../deploy/cephfs/kubernetes/v1.13) or else use [cephfs v1.14+ +templates](../deploy/cephfs/kubernetes/v1.14+) Your Kubernetes cluster must allow privileged pods (i.e. `--allow-privileged` flag must be set to true for both the API server and the kubelet). Moreover, as diff --git a/docs/deploy-rbd.md b/docs/deploy-rbd.md index 8fafe57a5..b3e61b20f 100644 --- a/docs/deploy-rbd.md +++ b/docs/deploy-rbd.md @@ -26,15 +26,15 @@ make image-cephcsi **Available command line arguments:** -| Option | Default value | Description | -| ------------------- | --------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `--endpoint` | `unix://tmp/csi.sock` | CSI endpoint, must be a UNIX socket | -| `--drivername` | `rbd.csi.ceph.com` | Name of the driver (Kubernetes: `provisioner` field in StorageClass must correspond to this value) | -| `--nodeid` | _empty_ | This node's ID | -| `--type` | _empty_ | Driver type `[rbd | cephfs]` If the driver type is set to `rbd` it will act as a `rbd plugin` or if it's set to `cephfs` will act as a `cephfs plugin` | -| `--containerized` | true | Whether running in containerized mode | -| `--instanceid` | "default" | Unique ID distinguishing this instance of Ceph CSI among other instances, when sharing Ceph clusters across CSI instances for provisioning | -| `--metadatastorage` | _empty_ | Points to where legacy (1.0.0 or older plugin versions) metadata about provisioned volumes are kept, as file or in as k8s configmap (`node` or `k8s_configmap` respectively) | +| Option | Default value | Description | +| ------------------- | --------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `--endpoint` | `unix://tmp/csi.sock` | CSI endpoint, must be a UNIX socket | +| `--drivername` | `rbd.csi.ceph.com` | Name of the driver (Kubernetes: `provisioner` field in StorageClass must correspond to this value) | +| `--nodeid` | _empty_ | This node's ID | +| `--type` | _empty_ | Driver type `[rbd | cephfs]` If the driver type is set to `rbd` it will act as a `rbd plugin` or if it's set to `cephfs` will act as a `cephfs plugin` | +| `--containerized` | true | Whether running in containerized mode | +| `--instanceid` | "default" | Unique ID distinguishing this instance of Ceph CSI among other instances, when sharing Ceph clusters across CSI instances for provisioning | +| `--metadatastorage` | _empty_ | Points to where legacy (1.0.0 or older plugin versions) metadata about provisioned volumes are kept, as file or in as k8s configmap (`node` or `k8s_configmap` respectively) | **Available environmental variables:** @@ -67,7 +67,11 @@ is required for provisioning new RBD images. ## Deployment with Kubernetes -Requires Kubernetes 1.11 +Requires Kubernetes 1.13+ + +if your cluster version is 1.13.x please use [rbd v1.13 +templates](../deploy/rbd/kubernetes/v1.13) or else use [rbd v1.14+ +templates](../deploy/rbd/kubernetes/v1.14+) Your Kubernetes cluster must allow privileged pods (i.e. `--allow-privileged` flag must be set to true for both the API server and the kubelet). Moreover, as diff --git a/e2e/cephfs.go b/e2e/cephfs.go index 79a5bc4c6..96b208cb5 100644 --- a/e2e/cephfs.go +++ b/e2e/cephfs.go @@ -6,6 +6,7 @@ import ( . "github.com/onsi/ginkgo" // nolint + clientset "k8s.io/client-go/kubernetes" "k8s.io/kubernetes/test/e2e/framework" e2elog "k8s.io/kubernetes/test/e2e/framework/log" ) @@ -17,10 +18,15 @@ var ( cephfsNodePluginRBAC = "csi-nodeplugin-rbac.yaml" cephfsDeploymentName = "csi-cephfsplugin-provisioner" cephfsDeamonSetName = "csi-cephfsplugin" - cephfsDirPath = "../deploy/cephfs/kubernetes/" + cephfsDirPath = "../deploy/cephfs/kubernetes" cephfsExamplePath = "../examples/cephfs/" ) +func updateCephfsDirPath(c clientset.Interface) { + version := getKubeVersionToDeploy(c) + cephfsDirPath = fmt.Sprintf("%s/%s/", cephfsDirPath, version) +} + func deployCephfsPlugin() { // deploy provisioner framework.RunKubectlOrDie("create", "-f", cephfsDirPath+cephfsProvisioner) @@ -34,8 +40,9 @@ var _ = Describe("cephfs", func() { f := framework.NewDefaultFramework("cephfs") // deploy cephfs CSI BeforeEach(func() { + updateCephfsDirPath(f.ClientSet) createFileSystem(f.ClientSet) - createConfigMap(f.ClientSet, f) + createConfigMap(cephfsDirPath, f.ClientSet, f) deployCephfsPlugin() createCephfsStorageClass(f.ClientSet, f) createCephfsSecret(f.ClientSet, f) @@ -58,9 +65,16 @@ var _ = Describe("cephfs", func() { It("Test cephfs CSI", func() { pvcPath := cephfsExamplePath + "pvc.yaml" appPath := cephfsExamplePath + "pod.yaml" - By("checking provisioner statefulset is running") + + By("checking provisioner statefulset/deployment is running") timeout := time.Duration(deployTimeout) * time.Minute - err := framework.WaitForStatefulSetReplicasReady(cephfsDeploymentName, namespace, f.ClientSet, 1*time.Second, timeout) + var err error + sts := deployProvAsSTS(f.ClientSet) + if sts { + err = framework.WaitForStatefulSetReplicasReady(cephfsDeploymentName, namespace, f.ClientSet, 1*time.Second, timeout) + } else { + err = waitForDeploymentComplete(cephfsDeploymentName, namespace, f.ClientSet, deployTimeout) + } if err != nil { Fail(err.Error()) } diff --git a/e2e/rbd.go b/e2e/rbd.go index 6ddd1f1d9..c769aaf11 100644 --- a/e2e/rbd.go +++ b/e2e/rbd.go @@ -6,6 +6,7 @@ import ( . "github.com/onsi/ginkgo" // nolint + clientset "k8s.io/client-go/kubernetes" "k8s.io/kubernetes/test/e2e/framework" e2elog "k8s.io/kubernetes/test/e2e/framework/log" ) @@ -15,14 +16,19 @@ var ( rbdProvisionerRBAC = "csi-provisioner-rbac.yaml" rbdNodePlugin = "csi-rbdplugin.yaml" rbdNodePluginRBAC = "csi-nodeplugin-rbac.yaml" - rbdConfigMap = "csi-config-map.yaml" - rbdDirPath = "../deploy/rbd/kubernetes/" + configMap = "csi-config-map.yaml" + rbdDirPath = "../deploy/rbd/kubernetes" rbdExamplePath = "../examples/rbd/" rbdDeploymentName = "csi-rbdplugin-provisioner" rbdDaemonsetName = "csi-rbdplugin" namespace = "default" ) +func updaterbdDirPath(c clientset.Interface) { + version := getKubeVersionToDeploy(c) + rbdDirPath = fmt.Sprintf("%s/%s/", rbdDirPath, version) +} + func deployRBDPlugin() { // deploy provisioner framework.RunKubectlOrDie("create", "-f", rbdDirPath+rbdProvisioner) @@ -36,8 +42,9 @@ var _ = Describe("RBD", func() { f := framework.NewDefaultFramework("rbd") // deploy RBD CSI BeforeEach(func() { + updaterbdDirPath(f.ClientSet) createRBDPool() - createConfigMap(f.ClientSet, f) + createConfigMap(rbdDirPath, f.ClientSet, f) deployRBDPlugin() createRBDStorageClass(f.ClientSet, f) createRBDSecret(f.ClientSet, f) @@ -68,9 +75,15 @@ var _ = Describe("RBD", func() { appClonePath := rbdExamplePath + "pod-restore.yaml" snapshotPath := rbdExamplePath + "snapshot.yaml" - By("checking provisioner statefulset is running") + By("checking provisioner statefulset/deployment is running") timeout := time.Duration(deployTimeout) * time.Minute - err := framework.WaitForStatefulSetReplicasReady(rbdDeploymentName, namespace, f.ClientSet, 1*time.Second, timeout) + var err error + sts := deployProvAsSTS(f.ClientSet) + if sts { + err = framework.WaitForStatefulSetReplicasReady(rbdDeploymentName, namespace, f.ClientSet, 1*time.Second, timeout) + } else { + err = waitForDeploymentComplete(rbdDeploymentName, namespace, f.ClientSet, deployTimeout) + } if err != nil { Fail(err.Error()) } diff --git a/e2e/utils.go b/e2e/utils.go index 115f40c28..4bc17ffcf 100644 --- a/e2e/utils.go +++ b/e2e/utils.go @@ -9,6 +9,8 @@ import ( "strings" "time" + "k8s.io/klog" + "github.com/kubernetes-csi/external-snapshotter/pkg/apis/volumesnapshot/v1alpha1" snapClient "github.com/kubernetes-csi/external-snapshotter/pkg/client/clientset/versioned/typed/volumesnapshot/v1alpha1" . "github.com/onsi/ginkgo" // nolint @@ -45,6 +47,31 @@ type snapInfo struct { Timestamp string `json:"timestamp"` } +func deployProvAsSTS(c clientset.Interface) bool { + // kubeMinor to use deployment instead of statefulset for provisioner + const kubeMinor = "14" + v, err := c.Discovery().ServerVersion() + if err != nil { + klog.Errorf("failed to get server version with error %v", err) + return false + } + if v.Minor < kubeMinor { + return true + } + return false +} + +func getKubeVersionToDeploy(c clientset.Interface) string { + sts := deployProvAsSTS(c) + version := "" + if sts { + version = "v1.13" + } else { + version = "v1.14+" + } + return version +} + func waitForDaemonSets(name, ns string, c clientset.Interface, t int) error { timeout := time.Duration(t) * time.Minute start := time.Now() @@ -97,7 +124,7 @@ func waitForDeploymentComplete(name, ns string, c clientset.Interface, t int) er return true, nil } - reason = fmt.Sprintf("deployment status: %#v", deployment.Status) + reason = fmt.Sprintf("deployment status: %#v", deployment.Status.String()) e2elog.Logf(reason) return false, nil @@ -234,8 +261,8 @@ func createRBDSnapshotClass(f *framework.Framework) { Expect(err).Should(BeNil()) } -func createConfigMap(c kubernetes.Interface, f *framework.Framework) { - path := rbdDirPath + rbdConfigMap +func createConfigMap(pluginPath string, c kubernetes.Interface, f *framework.Framework) { + path := pluginPath + configMap cm := v1.ConfigMap{} err := unmarshal(path, &cm) Expect(err).Should(BeNil()) diff --git a/examples/cephfs/plugin-deploy.sh b/examples/cephfs/plugin-deploy.sh index b7799de3d..785f9d64b 100755 --- a/examples/cephfs/plugin-deploy.sh +++ b/examples/cephfs/plugin-deploy.sh @@ -3,7 +3,7 @@ deployment_base="${1}" if [[ -z $deployment_base ]]; then - deployment_base="../../deploy/cephfs/kubernetes" + deployment_base="../../deploy/cephfs/kubernetes/v1.14+" fi cd "$deployment_base" || exit 1 diff --git a/examples/cephfs/plugin-teardown.sh b/examples/cephfs/plugin-teardown.sh index 65eda0d49..166f8cf2d 100755 --- a/examples/cephfs/plugin-teardown.sh +++ b/examples/cephfs/plugin-teardown.sh @@ -3,7 +3,7 @@ deployment_base="${1}" if [[ -z $deployment_base ]]; then - deployment_base="../../deploy/cephfs/kubernetes" + deployment_base="../../deploy/cephfs/kubernetes/v1.14+" fi cd "$deployment_base" || exit 1 diff --git a/examples/rbd/plugin-deploy.sh b/examples/rbd/plugin-deploy.sh index 06f853e48..79bb94678 100755 --- a/examples/rbd/plugin-deploy.sh +++ b/examples/rbd/plugin-deploy.sh @@ -3,7 +3,7 @@ deployment_base="${1}" if [[ -z $deployment_base ]]; then - deployment_base="../../deploy/rbd/kubernetes" + deployment_base="../../deploy/rbd/kubernetes/v1.14+" fi cd "$deployment_base" || exit 1 diff --git a/examples/rbd/plugin-teardown.sh b/examples/rbd/plugin-teardown.sh index e045aba16..ffd2717b8 100755 --- a/examples/rbd/plugin-teardown.sh +++ b/examples/rbd/plugin-teardown.sh @@ -3,7 +3,7 @@ deployment_base="${1}" if [[ -z $deployment_base ]]; then - deployment_base="../../deploy/rbd/kubernetes" + deployment_base="../../deploy/rbd/kubernetes/v1.14+" fi cd "$deployment_base" || exit 1 diff --git a/scripts/lint-text.sh b/scripts/lint-text.sh index c7bcf2f9f..90862f251 100755 --- a/scripts/lint-text.sh +++ b/scripts/lint-text.sh @@ -44,6 +44,6 @@ run_check '.*\.(ba)?sh' bash -n # Install via: pip install yamllint # disable yamlint chekck for helm chats -run_check '.*\.ya?ml' yamllint -s -d "{extends: default, rules: {line-length: {allow-non-breakable-inline-mappings: true}},ignore: deploy/*/helm/templates/*.yaml}" +run_check '.*\.ya?ml' yamllint -s -d "{extends: default, rules: {line-length: {allow-non-breakable-inline-mappings: true}},ignore: deploy/*/kubernetes/*/helm/templates/*.yaml}" echo "ALL OK."