deploy: modify nfs daemonset to use cephcsi nfs nodeserver

This commit makes modification to nfs daemonset to use
nfs nodeserver. `nfs.NetNamespaceFilePath` example is
added.

Signed-off-by: Rakshith R <rar@redhat.com>

deploy/nfs/kubernetes/csi-nfsplugin.yaml

@@ -2,154 +2,134 @@
 apiVersion: apps/v1
 kind: DaemonSet
 metadata:
-  name: csi-nfs-node
+  name: csi-nfsplugin
 spec:
   selector:
     matchLabels:
-      app: csi-nfs-node
+      app: csi-nfsplugin
   template:
     metadata:
       labels:
-        app: csi-nfs-node
+        app: csi-nfsplugin
     spec:
+      serviceAccountName: nfs-csi-nodeplugin
+      priorityClassName: system-node-critical
+      hostNetwork: true
+      hostPID: true
+      # to use e.g. Rook orchestrated cluster, and mons' FQDN is
+      # resolved through k8s service, set dns policy to cluster first
+      dnsPolicy: ClusterFirstWithHostNet
       containers:
-        - args:
-            - --csi-address=/csi/csi.sock
-            - --probe-timeout=3s
-            - --health-port=29653
-            - --v=2
-          image: registry.k8s.io/sig-storage/livenessprobe:v2.7.0
-          imagePullPolicy: IfNotPresent
-          name: liveness-probe
-          resources:
-            limits:
-              memory: 100Mi
-            requests:
-              cpu: 10m
-              memory: 20Mi
-          terminationMessagePath: /dev/termination-log
-          terminationMessagePolicy: File
-          volumeMounts:
-            - mountPath: /csi
-              name: socket-dir
-        - args:
-            - --v=1
-            - --csi-address=/csi/csi.sock
-            - --kubelet-registration-path=$(DRIVER_REG_SOCK_PATH)
+        - name: driver-registrar
+          # This is necessary only for systems with SELinux, where
+          # non-privileged sidecar containers cannot access unix domain socket
+          # created by privileged CSI driver container.
+          securityContext:
+            privileged: true
+            allowPrivilegeEscalation: true
+          image: registry.k8s.io/sig-storage/csi-node-driver-registrar:v2.5.1
+          args:
+            - "--v=1"
+            - "--csi-address=/csi/csi.sock"
+            - "--kubelet-registration-path=/var/lib/kubelet/plugins/nfs.csi.ceph.com/csi.sock"
           env:
-            - name: DRIVER_REG_SOCK_PATH
-              value: /var/lib/kubelet/plugins/nfs.csi.ceph.com/csi.sock
             - name: KUBE_NODE_NAME
               valueFrom:
                 fieldRef:
-                  apiVersion: v1
                   fieldPath: spec.nodeName
-          image: registry.k8s.io/sig-storage/csi-node-driver-registrar:v2.5.1
-          imagePullPolicy: IfNotPresent
-          livenessProbe:
-            exec:
-              command:
-                - /csi-node-driver-registrar
-                - --kubelet-registration-path=$(DRIVER_REG_SOCK_PATH)
-                - --mode=kubelet-registration-probe
-            failureThreshold: 3
-            initialDelaySeconds: 30
-            periodSeconds: 10
-            successThreshold: 1
-            timeoutSeconds: 15
-          name: node-driver-registrar
-          resources:
-            limits:
-              memory: 100Mi
-            requests:
-              cpu: 10m
-              memory: 20Mi
+          volumeMounts:
+            - name: socket-dir
+              mountPath: /csi
+            - name: registration-dir
+              mountPath: /registration
+        - name: csi-nfsplugin
           securityContext:
             privileged: true
-          terminationMessagePath: /dev/termination-log
-          terminationMessagePolicy: File
-          volumeMounts:
-            - mountPath: /csi
-              name: socket-dir
-            - mountPath: /registration
-              name: registration-dir
-        - args:
-            - -v=1
-            - --drivername=nfs.csi.ceph.com
-            - --nodeid=$(NODE_ID)
-            - --endpoint=$(CSI_ENDPOINT)
+            capabilities:
+              add: ["SYS_ADMIN"]
+            allowPrivilegeEscalation: true
+          # for stable functionality replace canary with latest release version
+          image: quay.io/cephcsi/cephcsi:canary
+          args:
+            - "--nodeid=$(NODE_ID)"
+            - "--type=nfs"
+            - "--nodeserver=true"
+            - "--endpoint=$(CSI_ENDPOINT)"
+            - "--v=5"
+            - "--drivername=nfs.csi.ceph.com"
+            - "--enableprofiling=false"
           env:
+            - name: POD_IP
+              valueFrom:
+                fieldRef:
+                  fieldPath: status.podIP
             - name: NODE_ID
               valueFrom:
                 fieldRef:
-                  apiVersion: v1
                   fieldPath: spec.nodeName
             - name: CSI_ENDPOINT
               value: unix:///csi/csi.sock
-          image: registry.k8s.io/sig-storage/nfsplugin:v4.0.0
-          imagePullPolicy: IfNotPresent
-          livenessProbe:
-            failureThreshold: 5
-            httpGet:
-              path: /healthz
-              port: healthz
-              scheme: HTTP
-            initialDelaySeconds: 30
-            periodSeconds: 30
-            successThreshold: 1
-            timeoutSeconds: 10
-          name: nfs
-          ports:
-            - containerPort: 29653
-              hostPort: 29653
-              name: healthz
-              protocol: TCP
-          resources:
-            limits:
-              memory: 300Mi
-            requests:
-              cpu: 10m
-              memory: 20Mi
-          securityContext:
-            allowPrivilegeEscalation: true
-            capabilities:
-              add:
-                - SYS_ADMIN
-            privileged: true
-          terminationMessagePath: /dev/termination-log
-          terminationMessagePolicy: File
+          imagePullPolicy: "IfNotPresent"
           volumeMounts:
-            - mountPath: /csi
-              name: socket-dir
-            - mountPath: /var/lib/kubelet/pods
+            - name: socket-dir
+              mountPath: /csi
+            - name: mountpoint-dir
+              mountPath: /var/lib/kubelet/pods
               mountPropagation: Bidirectional
-              name: pods-mount-dir
-      dnsPolicy: ClusterFirstWithHostNet
-      hostNetwork: true
-      nodeSelector:
-        kubernetes.io/os: linux
-      restartPolicy: Always
-      schedulerName: default-scheduler
-      securityContext: {}
-      serviceAccountName: nfs-csi-nodeplugin
-      terminationGracePeriodSeconds: 30
-      tolerations:
-        - operator: Exists
+            - name: plugin-dir
+              mountPath: /var/lib/kubelet/plugins
+              mountPropagation: "Bidirectional"
+            - name: host-sys
+              mountPath: /sys
+            - name: etc-selinux
+              mountPath: /etc/selinux
+              readOnly: true
+            - name: lib-modules
+              mountPath: /lib/modules
+              readOnly: true
+            - name: host-dev
+              mountPath: /dev
+            - name: host-mount
+              mountPath: /run/mount
+            - name: ceph-config
+              mountPath: /etc/ceph/
+            - name: ceph-csi-config
+              mountPath: /etc/ceph-csi-config/
       volumes:
-        - hostPath:
-            path: /var/lib/kubelet/plugins/nfs.csi.ceph.com
+        - name: socket-dir
+          hostPath:
+            path: /var/lib/kubelet/plugins/nfs.csi.ceph.com/
             type: DirectoryOrCreate
-          name: socket-dir
-        - hostPath:
+        - name: registration-dir
+          hostPath:
+            path: /var/lib/kubelet/plugins_registry/
+            type: Directory
+        - name: mountpoint-dir
+          hostPath:
             path: /var/lib/kubelet/pods
+            type: DirectoryOrCreate
+        - name: plugin-dir
+          hostPath:
+            path: /var/lib/kubelet/plugins
             type: Directory
-          name: pods-mount-dir
-        - hostPath:
-            path: /var/lib/kubelet/plugins_registry
-            type: Directory
-          name: registration-dir
-  updateStrategy:
-    rollingUpdate:
-      maxSurge: 0
-      maxUnavailable: 1
-    type: RollingUpdate
+        - name: host-sys
+          hostPath:
+            path: /sys
+        - name: etc-selinux
+          hostPath:
+            path: /etc/selinux
+        - name: lib-modules
+          hostPath:
+            path: /lib/modules
+        - name: host-dev
+          hostPath:
+            path: /dev
+        - name: host-mount
+          hostPath:
+            path: /run/mount
+        - name: ceph-config
+          configMap:
+            name: ceph-config
+        - name: ceph-csi-config
+          configMap:
+            name: ceph-csi-config

e2e/nfs.go

@@ -43,7 +43,7 @@ var (
 	nfsNodePluginPSP   = "csi-nodeplugin-psp.yaml"
 	nfsRookCephNFS     = "rook-nfs.yaml"
 	nfsDeploymentName  = "csi-nfsplugin-provisioner"
-	nfsDeamonSetName   = "csi-nfs-node"
+	nfsDeamonSetName   = "csi-nfsplugin"
 	nfsDirPath         = "../deploy/nfs/kubernetes/"
 	nfsExamplePath     = examplePath + "nfs/"
 	nfsPoolName        = ".nfs"
@@ -235,7 +235,7 @@ func unmountNFSVolume(f *framework.Framework, appName, pvcName string) error {
 		cmd,
 		nfsDeamonSetName,
 		pod.Spec.NodeName,
-		"nfs", // name of the container
+		"csi-nfsplugin", // name of the container
 		cephCSINamespace)
 	if stdErr != "" {
 		e2elog.Logf("StdErr occurred: %s", stdErr)
@@ -299,7 +299,7 @@ var _ = Describe("nfs", func() {
 			// log provisioner
 			logsCSIPods("app=csi-nfsplugin-provisioner", c)
 			// log node plugin
-			logsCSIPods("app=csi-nfs-node", c)
+			logsCSIPods("app=csi-nfsplugin", c)
 
 			// log all details from the namespace where Ceph-CSI is deployed
 			framework.DumpAllNamespaceInfo(c, cephCSINamespace)

examples/README.md

@@ -52,15 +52,15 @@ option `clusterID`, can now be created on the cluster.
 
 ## Running CephCSI with pod networking
 
-The current problem with Pod Networking, is when a CephFS/RBD volume is mounted
-in a pod using Ceph CSI and then the CSI CephFS/RBD plugin is restarted or
+The current problem with Pod Networking, is when a CephFS/RBD/NFS volume is mounted
+in a pod using Ceph CSI and then the CSI CephFS/RBD/NFS plugin is restarted or
 terminated (e.g. by restarting or deleting its DaemonSet), all operations on
 the volume become blocked, even after restarting the CSI pods.
 
 The only workaround is to restart the node where the Ceph CSI plugin pod was
 restarted. This can be mitigated by running the `rbd map`/`mount -t` commands
 in a different network namespace which does not get deleted when the CSI
-CephFS/RBD plugin is restarted or terminated.
+CephFS/RBD/NFS plugin is restarted or terminated.
 
 If someone wants to run the CephCSI with the pod networking they can still do
 by setting the `netNamespaceFilePath`. If this path is set CephCSI will execute

examples/csi-config-map-sample.yaml

@@ -24,6 +24,10 @@ kind: ConfigMap
 # path for the Ceph cluster identified by the <cluster-id>, This will be used
 # by the CephFS CSI plugin to execute the mount -t in the
 # network namespace specified by the "cephFS.netNamespaceFilePath".
+# The "nfs.netNamespaceFilePath" fields are the various network namespace
+# path for the Ceph cluster identified by the <cluster-id>, This will be used
+# by the NFS CSI plugin to execute the mount -t in the
+# network namespace specified by the "nfs.netNamespaceFilePath".
 # The "rbd.netNamespaceFilePath" fields are the various network namespace
 # path for the Ceph cluster identified by the <cluster-id>, This will be used
 # by the RBD CSI plugin to execute the rbd map/unmap in the
@@ -60,6 +64,9 @@ data:
           "subvolumeGroup": "<subvolumegroup for cephFS volumes>"
           "netNamespaceFilePath": "<kubeletRootPath>/plugins/cephfs.csi.ceph.com/net",
         }
+        "nfs": {
+          "netNamespaceFilePath": "<kubeletRootPath>/plugins/nfs.csi.ceph.com/net",
+        }
       }
     ]
   cluster-mapping.json: |-