deploy: add deployment artifacts for NFS support

These deployment files are heavily based on the CephFS deployment.

Deploying an environment with these files work for me in minikube. This
should make it possible to add e2e testing as well.

Signed-off-by: Niels de Vos <ndevos@redhat.com>
This commit is contained in:
Niels de Vos 2022-03-30 10:06:07 +02:00 committed by mergify[bot]
parent 591cd694ab
commit 804e2715d8
14 changed files with 746 additions and 0 deletions

View File

@ -0,0 +1,74 @@
/*
Copyright 2022 The Ceph-CSI Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package nfs
import (
"bytes"
_ "embed"
"fmt"
"text/template"
"github.com/ghodss/yaml"
v1 "k8s.io/api/core/v1"
)
//go:embed csi-config-map.yaml
var csiConfigMap string
type CSIConfigMapValues struct {
Name string
}
var CSIConfigMapDefaults = CSIConfigMapValues{
Name: "ceph-csi-config",
}
// NewCSIConfigMap takes a name from the CSIConfigMapValues struct and relaces
// the value in the template. A ConfigMap object is returned which can be
// created in the Kubernetes cluster.
func NewCSIConfigMap(values CSIConfigMapValues) (*v1.ConfigMap, error) {
data, err := NewCSIConfigMapYAML(values)
if err != nil {
return nil, err
}
cm := &v1.ConfigMap{}
err = yaml.Unmarshal([]byte(data), cm)
if err != nil {
return nil, fmt.Errorf("failed convert YAML to %T: %w", cm, err)
}
return cm, nil
}
// NewCSIConfigMapYAML takes a name from the CSIConfigMapValues struct and
// relaces the value in the template. A ConfigMap object in YAML is returned
// which can be created in the Kubernetes cluster.
func NewCSIConfigMapYAML(values CSIConfigMapValues) (string, error) {
var buf bytes.Buffer
tmpl, err := template.New("CSIConfigMap").Parse(csiConfigMap)
if err != nil {
return "", fmt.Errorf("failed to parse template: %w", err)
}
err = tmpl.Execute(&buf, values)
if err != nil {
return "", fmt.Errorf("failed to replace values in template: %w", err)
}
return buf.String(), nil
}

View File

@ -0,0 +1,8 @@
---
apiVersion: v1
kind: ConfigMap
metadata:
name: "{{ .Name }}"
data:
config.json: |-
[]

View File

@ -0,0 +1,38 @@
/*
Copyright 2022 The Ceph-CSI Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package nfs
import (
"testing"
"github.com/stretchr/testify/require"
)
func TestNewCSIConfigMap(t *testing.T) {
cm, err := NewCSIConfigMap(CSIConfigMapDefaults)
require.NoError(t, err)
require.NotNil(t, cm)
require.Equal(t, cm.Name, CSIConfigMapDefaults.Name)
}
func TestNewCSIConfigMapYAML(t *testing.T) {
yaml, err := NewCSIConfigMapYAML(CSIConfigMapDefaults)
require.NoError(t, err)
require.NotEqual(t, "", yaml)
}

View File

@ -16,6 +16,7 @@
all: \ all: \
scc.yaml \ scc.yaml \
nfs/kubernetes/csidriver.yaml \ nfs/kubernetes/csidriver.yaml \
nfs/kubernetes/csi-config-map.yaml \
rbd/kubernetes/csidriver.yaml \ rbd/kubernetes/csidriver.yaml \
rbd/kubernetes/csi-config-map.yaml rbd/kubernetes/csi-config-map.yaml
@ -25,6 +26,9 @@ scc.yaml: ../api/deploy/ocp/scc.yaml ../api/deploy/ocp/scc.go
nfs/kubernetes/csidriver.yaml: ../api/deploy/kubernetes/nfs/csidriver.yaml ../api/deploy/kubernetes/nfs/csidriver.go nfs/kubernetes/csidriver.yaml: ../api/deploy/kubernetes/nfs/csidriver.yaml ../api/deploy/kubernetes/nfs/csidriver.go
$(MAKE) -C ../tools generate-deploy $(MAKE) -C ../tools generate-deploy
nfs/kubernetes/csi-config-map.yaml: ../api/deploy/kubernetes/nfs/csi-config-map.*
$(MAKE) -C ../tools generate-deploy
rbd/kubernetes/csidriver.yaml: ../api/deploy/kubernetes/rbd/csidriver.yaml ../api/deploy/kubernetes/rbd/csidriver.go rbd/kubernetes/csidriver.yaml: ../api/deploy/kubernetes/rbd/csidriver.yaml ../api/deploy/kubernetes/rbd/csidriver.go
$(MAKE) -C ../tools generate-deploy $(MAKE) -C ../tools generate-deploy

View File

@ -0,0 +1,15 @@
#
# /!\ DO NOT MODIFY THIS FILE
#
# This file has been automatically generated by Ceph-CSI yamlgen.
# The source for the contents can be found in the api/deploy directory, make
# your modifications there.
#
---
apiVersion: v1
kind: ConfigMap
metadata:
name: "ceph-csi-config"
data:
config.json: |-
[]

View File

@ -0,0 +1,123 @@
---
kind: Service
apiVersion: v1
metadata:
name: csi-nfsplugin-provisioner
labels:
app: csi-metrics
spec:
selector:
app: csi-nfsplugin-provisioner
ports:
- name: http-metrics
port: 8080
protocol: TCP
targetPort: 8682
---
apiVersion: apps/v1
kind: Deployment
metadata:
name: csi-nfsplugin-provisioner
spec:
selector:
matchLabels:
app: csi-nfsplugin-provisioner
replicas: 3
template:
metadata:
labels:
app: csi-nfsplugin-provisioner
spec:
affinity:
podAntiAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
- labelSelector:
matchExpressions:
- key: app
operator: In
values:
- csi-nfsplugin-provisioner
topologyKey: "kubernetes.io/hostname"
containers:
- name: csi-provisioner
image: k8s.gcr.io/sig-storage/csi-provisioner:v3.1.0
args:
- "--csi-address=$(ADDRESS)"
- "--v=5"
- "--timeout=150s"
- "--leader-election=true"
- "--retry-interval-start=500ms"
env:
- name: ADDRESS
value: unix:///csi/csi-provisioner.sock
imagePullPolicy: "IfNotPresent"
volumeMounts:
- name: socket-dir
mountPath: /csi
- name: csi-nfsplugin
# for stable functionality replace canary with latest release version
image: quay.io/cephcsi/cephcsi:canary
args:
- "--nodeid=$(NODE_ID)"
- "--type=nfs"
- "--controllerserver=true"
- "--endpoint=$(CSI_ENDPOINT)"
- "--v=5"
- "--drivername=nfs.csi.ceph.com"
- "--pidlimit=-1"
- "--enableprofiling=false"
env:
- name: POD_IP
valueFrom:
fieldRef:
fieldPath: status.podIP
- name: NODE_ID
valueFrom:
fieldRef:
fieldPath: spec.nodeName
- name: CSI_ENDPOINT
value: unix:///csi/csi-provisioner.sock
imagePullPolicy: "IfNotPresent"
volumeMounts:
- name: socket-dir
mountPath: /csi
- name: host-sys
mountPath: /sys
- name: ceph-csi-config
mountPath: /etc/ceph-csi-config/
- name: keys-tmp-dir
mountPath: /tmp/csi/keys
- name: liveness-prometheus
image: quay.io/cephcsi/cephcsi:canary
args:
- "--type=liveness"
- "--endpoint=$(CSI_ENDPOINT)"
- "--metricsport=8682"
- "--metricspath=/metrics"
- "--polltime=60s"
- "--timeout=3s"
env:
- name: CSI_ENDPOINT
value: unix:///csi/csi-provisioner.sock
- name: POD_IP
valueFrom:
fieldRef:
fieldPath: status.podIP
volumeMounts:
- name: socket-dir
mountPath: /csi
imagePullPolicy: "IfNotPresent"
serviceAccountName: nfs-csi-provisioner
volumes:
- emptyDir:
medium: Memory
name: socket-dir
- name: host-sys
hostPath:
path: /sys
- name: ceph-csi-config
configMap:
name: ceph-csi-config
- emptyDir:
medium: Memory
name: keys-tmp-dir

View File

@ -0,0 +1,155 @@
---
apiVersion: apps/v1
kind: DaemonSet
metadata:
name: csi-nfs-node
spec:
selector:
matchLabels:
app: csi-nfs-node
template:
metadata:
labels:
app: csi-nfs-node
spec:
containers:
- args:
- --csi-address=/csi/csi.sock
- --probe-timeout=3s
- --health-port=29653
- --v=2
image: k8s.gcr.io/sig-storage/livenessprobe:v2.5.0
imagePullPolicy: IfNotPresent
name: liveness-probe
resources:
limits:
memory: 100Mi
requests:
cpu: 10m
memory: 20Mi
terminationMessagePath: /dev/termination-log
terminationMessagePolicy: File
volumeMounts:
- mountPath: /csi
name: socket-dir
- args:
- --v=5
- --csi-address=/csi/csi.sock
- --kubelet-registration-path=$(DRIVER_REG_SOCK_PATH)
env:
- name: DRIVER_REG_SOCK_PATH
value: /var/lib/kubelet/plugins/nfs.csi.ceph.com/csi.sock
- name: KUBE_NODE_NAME
valueFrom:
fieldRef:
apiVersion: v1
fieldPath: spec.nodeName
image: k8s.gcr.io/sig-storage/csi-node-driver-registrar:v2.4.0
imagePullPolicy: IfNotPresent
livenessProbe:
exec:
command:
- /csi-node-driver-registrar
- --kubelet-registration-path=$(DRIVER_REG_SOCK_PATH)
- --mode=kubelet-registration-probe
failureThreshold: 3
initialDelaySeconds: 30
periodSeconds: 10
successThreshold: 1
timeoutSeconds: 15
name: node-driver-registrar
resources:
limits:
memory: 100Mi
requests:
cpu: 10m
memory: 20Mi
securityContext:
privileged: true
terminationMessagePath: /dev/termination-log
terminationMessagePolicy: File
volumeMounts:
- mountPath: /csi
name: socket-dir
- mountPath: /registration
name: registration-dir
- args:
- -v=5
- --drivername=nfs.csi.ceph.com
- --nodeid=$(NODE_ID)
- --endpoint=$(CSI_ENDPOINT)
env:
- name: NODE_ID
valueFrom:
fieldRef:
apiVersion: v1
fieldPath: spec.nodeName
- name: CSI_ENDPOINT
value: unix:///csi/csi.sock
image: mcr.microsoft.com/k8s/csi/nfs-csi:v3.1.0
imagePullPolicy: IfNotPresent
livenessProbe:
failureThreshold: 5
httpGet:
path: /healthz
port: healthz
scheme: HTTP
initialDelaySeconds: 30
periodSeconds: 30
successThreshold: 1
timeoutSeconds: 10
name: nfs
ports:
- containerPort: 29653
hostPort: 29653
name: healthz
protocol: TCP
resources:
limits:
memory: 300Mi
requests:
cpu: 10m
memory: 20Mi
securityContext:
allowPrivilegeEscalation: true
capabilities:
add:
- SYS_ADMIN
privileged: true
terminationMessagePath: /dev/termination-log
terminationMessagePolicy: File
volumeMounts:
- mountPath: /csi
name: socket-dir
- mountPath: /var/lib/kubelet/pods
mountPropagation: Bidirectional
name: pods-mount-dir
dnsPolicy: ClusterFirstWithHostNet
hostNetwork: true
nodeSelector:
kubernetes.io/os: linux
restartPolicy: Always
schedulerName: default-scheduler
securityContext: {}
serviceAccountName: nfs-csi-nodeplugin
terminationGracePeriodSeconds: 30
tolerations:
- operator: Exists
volumes:
- hostPath:
path: /var/lib/kubelet/plugins/nfs.csi.ceph.com
type: DirectoryOrCreate
name: socket-dir
- hostPath:
path: /var/lib/kubelet/pods
type: Directory
name: pods-mount-dir
- hostPath:
path: /var/lib/kubelet/plugins_registry
type: Directory
name: registration-dir
updateStrategy:
rollingUpdate:
maxSurge: 0
maxUnavailable: 1
type: RollingUpdate

View File

@ -0,0 +1,75 @@
---
apiVersion: policy/v1beta1
kind: PodSecurityPolicy
metadata:
name: nfs-csi-nodeplugin-psp
spec:
allowPrivilegeEscalation: true
allowedCapabilities:
- 'SYS_ADMIN'
fsGroup:
rule: RunAsAny
privileged: true
hostNetwork: true
hostPID: true
runAsUser:
rule: RunAsAny
seLinux:
rule: RunAsAny
supplementalGroups:
rule: RunAsAny
volumes:
- 'configMap'
- 'emptyDir'
- 'projected'
- 'secret'
- 'hostPath'
allowedHostPaths:
- pathPrefix: '/dev'
readOnly: false
- pathPrefix: '/run/mount'
readOnly: false
- pathPrefix: '/sys'
readOnly: false
- pathPrefix: '/etc/selinux'
readOnly: true
- pathPrefix: '/lib/modules'
readOnly: true
- pathPrefix: '/var/lib/kubelet/pods'
readOnly: false
- pathPrefix: '/var/lib/kubelet/plugins/nfs.csi.ceph.com'
readOnly: false
- pathPrefix: '/var/lib/kubelet/plugins_registry'
readOnly: false
- pathPrefix: '/var/lib/kubelet/plugins'
readOnly: false
---
kind: Role
apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: nfs-csi-nodeplugin-psp
# replace with non-default namespace name
namespace: default
rules:
- apiGroups: ['policy']
resources: ['podsecuritypolicies']
verbs: ['use']
resourceNames: ['nfs-csi-nodeplugin-psp']
---
kind: RoleBinding
apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: nfs-csi-nodeplugin-psp
# replace with non-default namespace name
namespace: default
subjects:
- kind: ServiceAccount
name: nfs-csi-nodeplugin
# replace with non-default namespace name
namespace: default
roleRef:
kind: Role
name: nfs-csi-nodeplugin-psp
apiGroup: rbac.authorization.k8s.io

View File

@ -0,0 +1,27 @@
---
apiVersion: v1
kind: ServiceAccount
metadata:
name: nfs-csi-nodeplugin
---
kind: ClusterRole
apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: nfs-csi-nodeplugin
rules:
- apiGroups: [""]
resources: ["nodes"]
verbs: ["get"]
---
kind: ClusterRoleBinding
apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: nfs-csi-nodeplugin
subjects:
- kind: ServiceAccount
name: nfs-csi-nodeplugin
namespace: default
roleRef:
kind: ClusterRole
name: nfs-csi-nodeplugin
apiGroup: rbac.authorization.k8s.io

View File

@ -0,0 +1,55 @@
---
apiVersion: policy/v1beta1
kind: PodSecurityPolicy
metadata:
name: nfs-csi-provisioner-psp
spec:
fsGroup:
rule: RunAsAny
runAsUser:
rule: RunAsAny
seLinux:
rule: RunAsAny
supplementalGroups:
rule: RunAsAny
volumes:
- 'configMap'
- 'emptyDir'
- 'projected'
- 'secret'
- 'hostPath'
allowedHostPaths:
- pathPrefix: '/dev'
readOnly: false
- pathPrefix: '/sys'
readOnly: false
- pathPrefix: '/lib/modules'
readOnly: true
---
kind: Role
apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: nfs-csi-provisioner-psp
# replace with non-default namespace name
namespace: default
rules:
- apiGroups: ['policy']
resources: ['podsecuritypolicies']
verbs: ['use']
resourceNames: ['nfs-csi-provisioner-psp']
---
kind: RoleBinding
apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: nfs-csi-provisioner-psp
# replace with non-default namespace name
namespace: default
subjects:
- kind: ServiceAccount
name: nfs-csi-provisioner
# replace with non-default namespace name
namespace: default
roleRef:
kind: Role
name: nfs-csi-provisioner-psp
apiGroup: rbac.authorization.k8s.io

View File

@ -0,0 +1,85 @@
---
apiVersion: v1
kind: ServiceAccount
metadata:
name: nfs-csi-provisioner
---
kind: ClusterRole
apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: nfs-external-provisioner-runner
rules:
- apiGroups: [""]
resources: ["nodes"]
verbs: ["get", "list", "watch"]
- apiGroups: [""]
resources: ["secrets"]
verbs: ["get", "list"]
- apiGroups: [""]
resources: ["events"]
verbs: ["list", "watch", "create", "update", "patch"]
- apiGroups: [""]
resources: ["persistentvolumes"]
verbs: ["get", "list", "watch", "create", "delete", "patch"]
- apiGroups: [""]
resources: ["persistentvolumeclaims"]
verbs: ["get", "list", "watch", "update"]
- apiGroups: ["storage.k8s.io"]
resources: ["storageclasses"]
verbs: ["get", "list", "watch"]
- apiGroups: ["storage.k8s.io"]
resources: ["volumeattachments"]
verbs: ["get", "list", "watch", "update", "patch"]
- apiGroups: ["storage.k8s.io"]
resources: ["volumeattachments/status"]
verbs: ["patch"]
- apiGroups: [""]
resources: ["persistentvolumeclaims/status"]
verbs: ["update", "patch"]
- apiGroups: ["storage.k8s.io"]
resources: ["csinodes"]
verbs: ["get", "list", "watch"]
---
kind: ClusterRoleBinding
apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: nfs-csi-provisioner-role
subjects:
- kind: ServiceAccount
name: nfs-csi-provisioner
namespace: default
roleRef:
kind: ClusterRole
name: nfs-external-provisioner-runner
apiGroup: rbac.authorization.k8s.io
---
kind: Role
apiVersion: rbac.authorization.k8s.io/v1
metadata:
# replace with non-default namespace name
namespace: default
name: nfs-external-provisioner-cfg
rules:
# remove this once we stop supporting v1.0.0
- apiGroups: [""]
resources: ["configmaps"]
verbs: ["get", "list", "create", "delete"]
- apiGroups: ["coordination.k8s.io"]
resources: ["leases"]
verbs: ["get", "watch", "list", "delete", "update", "create"]
---
kind: RoleBinding
apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: nfs-csi-provisioner-role-cfg
# replace with non-default namespace name
namespace: default
subjects:
- kind: ServiceAccount
name: nfs-csi-provisioner
# replace with non-default namespace name
namespace: default
roleRef:
kind: Role
name: nfs-external-provisioner-cfg
apiGroup: rbac.authorization.k8s.io

View File

@ -53,6 +53,11 @@ var yamlArtifacts = []deploymentArtifact{
reflect.ValueOf(nfs.NewCSIDriverYAML), reflect.ValueOf(nfs.NewCSIDriverYAML),
reflect.ValueOf(nfs.CSIDriverDefaults), reflect.ValueOf(nfs.CSIDriverDefaults),
}, },
{
"../deploy/nfs/kubernetes/csi-config-map.yaml",
reflect.ValueOf(nfs.NewCSIConfigMapYAML),
reflect.ValueOf(nfs.CSIConfigMapDefaults),
},
{ {
"../deploy/rbd/kubernetes/csidriver.yaml", "../deploy/rbd/kubernetes/csidriver.yaml",
reflect.ValueOf(rbd.NewCSIDriverYAML), reflect.ValueOf(rbd.NewCSIDriverYAML),

View File

@ -0,0 +1,74 @@
/*
Copyright 2022 The Ceph-CSI Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package nfs
import (
"bytes"
_ "embed"
"fmt"
"text/template"
"github.com/ghodss/yaml"
v1 "k8s.io/api/core/v1"
)
//go:embed csi-config-map.yaml
var csiConfigMap string
type CSIConfigMapValues struct {
Name string
}
var CSIConfigMapDefaults = CSIConfigMapValues{
Name: "ceph-csi-config",
}
// NewCSIConfigMap takes a name from the CSIConfigMapValues struct and relaces
// the value in the template. A ConfigMap object is returned which can be
// created in the Kubernetes cluster.
func NewCSIConfigMap(values CSIConfigMapValues) (*v1.ConfigMap, error) {
data, err := NewCSIConfigMapYAML(values)
if err != nil {
return nil, err
}
cm := &v1.ConfigMap{}
err = yaml.Unmarshal([]byte(data), cm)
if err != nil {
return nil, fmt.Errorf("failed convert YAML to %T: %w", cm, err)
}
return cm, nil
}
// NewCSIConfigMapYAML takes a name from the CSIConfigMapValues struct and
// relaces the value in the template. A ConfigMap object in YAML is returned
// which can be created in the Kubernetes cluster.
func NewCSIConfigMapYAML(values CSIConfigMapValues) (string, error) {
var buf bytes.Buffer
tmpl, err := template.New("CSIConfigMap").Parse(csiConfigMap)
if err != nil {
return "", fmt.Errorf("failed to parse template: %w", err)
}
err = tmpl.Execute(&buf, values)
if err != nil {
return "", fmt.Errorf("failed to replace values in template: %w", err)
}
return buf.String(), nil
}

View File

@ -0,0 +1,8 @@
---
apiVersion: v1
kind: ConfigMap
metadata:
name: "{{ .Name }}"
data:
config.json: |-
[]