Merge pull request #139 from rootfs/node-reg

switch to node registrar

deploy/cephfs/kubernetes/csi-attacher-rbac.yaml

@@ -1,13 +1,13 @@
 apiVersion: v1
 kind: ServiceAccount
 metadata:
-  name: csi-attacher
+  name: cephfs-csi-attacher
 
 ---
 kind: ClusterRole
 apiVersion: rbac.authorization.k8s.io/v1
 metadata:
-  name: external-attacher-runner
+  name: cephfs-external-attacher-runner
 rules:
   - apiGroups: [""]
     resources: ["events"]
@@ -26,12 +26,12 @@ rules:
 kind: ClusterRoleBinding
 apiVersion: rbac.authorization.k8s.io/v1
 metadata:
-  name: csi-attacher-role
+  name: cephfs-csi-attacher-role
 subjects:
   - kind: ServiceAccount
-    name: csi-attacher
+    name: cephfs-csi-attacher
     namespace: default
 roleRef:
   kind: ClusterRole
-  name: external-attacher-runner
+  name: cephfs-external-attacher-runner
   apiGroup: rbac.authorization.k8s.io

deploy/cephfs/kubernetes/csi-cephfsplugin-attacher.yaml

@@ -24,7 +24,7 @@ spec:
       labels:
         app: csi-cephfsplugin-attacher
     spec:
-      serviceAccount: csi-attacher
+      serviceAccount: cephfs-csi-attacher
       containers:
         - name: csi-cephfsplugin-attacher
           image: quay.io/k8scsi/csi-attacher:v1.0.0
@@ -33,13 +33,13 @@ spec:
             - "--csi-address=$(ADDRESS)"
           env:
             - name: ADDRESS
-              value: /var/lib/kubelet/plugins_registry/csi-cephfsplugin/csi.sock
+              value: /var/lib/kubelet/plugins/csi-cephfsplugin/csi.sock
           imagePullPolicy: "IfNotPresent"
           volumeMounts:
             - name: socket-dir
-              mountPath: /var/lib/kubelet/plugins_registry/csi-cephfsplugin
+              mountPath: /var/lib/kubelet/plugins/csi-cephfsplugin
       volumes:
         - name: socket-dir
           hostPath:
-            path: /var/lib/kubelet/plugins_registry/csi-cephfsplugin
+            path: /var/lib/kubelet/plugins/csi-cephfsplugin
             type: DirectoryOrCreate

deploy/cephfs/kubernetes/csi-cephfsplugin-provisioner.yaml

@@ -24,7 +24,7 @@ spec:
       labels:
         app: csi-cephfsplugin-provisioner
     spec:
-      serviceAccount: csi-provisioner
+      serviceAccount: cephfs-csi-provisioner
       containers:
         - name: csi-provisioner
           image: quay.io/k8scsi/csi-provisioner:v1.0.0
@@ -34,13 +34,56 @@ spec:
             - "--v=5"
           env:
             - name: ADDRESS
-              value: /var/lib/kubelet/plugins_registry/csi-cephfsplugin/csi.sock
+              value: /var/lib/kubelet/plugins/csi-cephfsplugin/csi-provisioner.sock
           imagePullPolicy: "IfNotPresent"
           volumeMounts:
             - name: socket-dir
-              mountPath: /var/lib/kubelet/plugins_registry/csi-cephfsplugin
+              mountPath: /var/lib/kubelet/plugins/csi-cephfsplugin
+        - name: csi-cephfsplugin
+          securityContext:
+            privileged: true
+            capabilities:
+              add: ["SYS_ADMIN"]
+          image: quay.io/cephcsi/cephfsplugin:v1.0.0
+          args :
+            - "--nodeid=$(NODE_ID)"
+            - "--endpoint=$(CSI_ENDPOINT)"
+            - "--v=5"
+            - "--drivername=csi-cephfsplugin"
+            - "--metadatastorage=k8s_configmap"
+          env:
+            - name: NODE_ID
+              valueFrom:
+                fieldRef:
+                  fieldPath: spec.nodeName
+            - name: POD_NAMESPACE
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.namespace
+            - name: CSI_ENDPOINT
+              value: unix://var/lib/kubelet/plugins/csi-cephfsplugin/csi-provisioner.sock
+          imagePullPolicy: "IfNotPresent"
+          volumeMounts:
+            - name: socket-dir
+              mountPath: /var/lib/kubelet/plugins/csi-cephfsplugin            
+            - name: host-sys
+              mountPath: /sys
+            - name: lib-modules
+              mountPath: /lib/modules
+              readOnly: true
+            - name: host-dev
+              mountPath: /dev              
       volumes:
         - name: socket-dir
           hostPath:
-            path: /var/lib/kubelet/plugins_registry/csi-cephfsplugin
+            path: /var/lib/kubelet/plugins/csi-cephfsplugin
             type: DirectoryOrCreate
+        - name: host-sys
+          hostPath:
+            path: /sys
+        - name: lib-modules
+          hostPath:
+            path: /lib/modules
+        - name: host-dev
+          hostPath:
+            path: /dev

deploy/cephfs/kubernetes/csi-cephfsplugin.yaml

@@ -11,30 +11,30 @@ spec:
       labels:
         app: csi-cephfsplugin
     spec:
-      serviceAccount: csi-nodeplugin
+      serviceAccount: cephfs-csi-nodeplugin
       hostNetwork: true
       # to use e.g. Rook orchestrated cluster, and mons' FQDN is
       # resolved through k8s service, set dns policy to cluster first
       dnsPolicy: ClusterFirstWithHostNet      
       containers:
         - name: driver-registrar
-          image: quay.io/k8scsi/driver-registrar:canary
+          image: quay.io/k8scsi/csi-node-driver-registrar:v1.0.2
           args:
             - "--v=5"
-            - "--csi-address=$(ADDRESS)"
-            - "--kubelet-registration-path=$(DRIVER_REG_SOCK_PATH)"
+            - "--csi-address=/csi/csi.sock"
+            - "--kubelet-registration-path=/var/lib/kubelet/plugins/csi-cephfsplugin/csi.sock"
+          lifecycle:
+            preStop:
+              exec:
+                  command: ["/bin/sh", "-c", "rm -rf /registration/csi-cephfsplugin /registration/csi-cephfsplugin-reg.sock"]          
           env:
-            - name: ADDRESS
-              value: /var/lib/kubelet/plugins_registry/csi-cephfsplugin/csi.sock
-            - name: DRIVER_REG_SOCK_PATH
-              value: /var/lib/kubelet/plugins_registry/csi-cephfsplugin/csi.sock
             - name: KUBE_NODE_NAME
               valueFrom:
                 fieldRef:
                   fieldPath: spec.nodeName
           volumeMounts:
-            - name: socket-dir
-              mountPath: /var/lib/kubelet/plugins_registry/csi-cephfsplugin
+            - name: plugin-dir
+              mountPath: /csi
             - name: registration-dir
               mountPath: /registration
         - name: csi-cephfsplugin
@@ -60,11 +60,11 @@ spec:
                 fieldRef:
                   fieldPath: metadata.namespace
             - name: CSI_ENDPOINT
-              value: unix://var/lib/kubelet/plugins_registry/csi-cephfsplugin/csi.sock
+              value: unix://var/lib/kubelet/plugins/csi-cephfsplugin/csi.sock
           imagePullPolicy: "IfNotPresent"
           volumeMounts:
             - name: plugin-dir
-              mountPath: /var/lib/kubelet/plugins_registry/csi-cephfsplugin
+              mountPath: /var/lib/kubelet/plugins/csi-cephfsplugin
             - name: csi-plugins-dir
               mountPath: /var/lib/kubelet/plugins/kubernetes.io/csi
               mountPropagation: "Bidirectional"
@@ -81,7 +81,7 @@ spec:
       volumes:
         - name: plugin-dir
           hostPath:
-            path: /var/lib/kubelet/plugins_registry/csi-cephfsplugin
+            path: /var/lib/kubelet/plugins/csi-cephfsplugin/
             type: DirectoryOrCreate
         - name: csi-plugins-dir
           hostPath:
@@ -95,10 +95,6 @@ spec:
           hostPath:
             path: /var/lib/kubelet/pods
             type: Directory
-        - name: socket-dir
-          hostPath:
-            path: /var/lib/kubelet/plugins_registry/csi-cephfsplugin
-            type: DirectoryOrCreate
         - name: host-sys
           hostPath:
             path: /sys

deploy/cephfs/kubernetes/csi-nodeplugin-rbac.yaml

@@ -1,13 +1,13 @@
 apiVersion: v1
 kind: ServiceAccount
 metadata:
-  name: csi-nodeplugin
+  name: cephfs-csi-nodeplugin
 
 ---
 kind: ClusterRole
 apiVersion: rbac.authorization.k8s.io/v1
 metadata:
-  name: csi-nodeplugin
+  name: cephfs-csi-nodeplugin
 rules:
   - apiGroups: [""]
     resources: ["nodes"]
@@ -23,18 +23,18 @@ rules:
     verbs: ["get", "list", "watch", "update"]
   - apiGroups: [""]
     resources: ["configmaps"]
-    verbs: ["get", "list", "create", "delete"]
+    verbs: ["get", "list"]
 
 ---
 kind: ClusterRoleBinding
 apiVersion: rbac.authorization.k8s.io/v1
 metadata:
-  name: csi-nodeplugin
+  name: cephfs-csi-nodeplugin
 subjects:
   - kind: ServiceAccount
-    name: csi-nodeplugin
+    name: cephfs-csi-nodeplugin
     namespace: default
 roleRef:
   kind: ClusterRole
-  name: csi-nodeplugin
+  name: cephfs-csi-nodeplugin
   apiGroup: rbac.authorization.k8s.io          

deploy/cephfs/kubernetes/csi-provisioner-rbac.yaml

@@ -1,13 +1,13 @@
 apiVersion: v1
 kind: ServiceAccount
 metadata:
-  name: csi-provisioner
+  name: cephfs-csi-provisioner
 
 ---
 kind: ClusterRole
 apiVersion: rbac.authorization.k8s.io/v1
 metadata:
-  name: external-provisioner-runner
+  name: cephfs-external-provisioner-runner
 rules:
   - apiGroups: [""]
     resources: ["secrets"]
@@ -24,17 +24,20 @@ rules:
   - apiGroups: [""]
     resources: ["events"]
     verbs: ["list", "watch", "create", "update", "patch"]
+  - apiGroups: [""]
+    resources: ["configmaps"]
+    verbs: ["get", "list", "create", "delete"]
     
 ---
 kind: ClusterRoleBinding
 apiVersion: rbac.authorization.k8s.io/v1
 metadata:
-  name: csi-provisioner-role
+  name: cephfs-csi-provisioner-role
 subjects:
   - kind: ServiceAccount
-    name: csi-provisioner
+    name: cephfs-csi-provisioner
     namespace: default
 roleRef:
   kind: ClusterRole
-  name: external-provisioner-runner
+  name: cephfs-external-provisioner-runner
   apiGroup: rbac.authorization.k8s.io

deploy/rbd/kubernetes/csi-attacher-rbac.yaml

@@ -1,13 +1,13 @@
 apiVersion: v1
 kind: ServiceAccount
 metadata:
-  name: csi-attacher
+  name: rbd-csi-attacher
 
 ---
 kind: ClusterRole
 apiVersion: rbac.authorization.k8s.io/v1
 metadata:
-  name: external-attacher-runner
+  name: rbd-external-attacher-runner
 rules:
   - apiGroups: [""]
     resources: ["events"]
@@ -26,12 +26,12 @@ rules:
 kind: ClusterRoleBinding
 apiVersion: rbac.authorization.k8s.io/v1
 metadata:
-  name: csi-attacher-role
+  name: rbd-csi-attacher-role
 subjects:
   - kind: ServiceAccount
-    name: csi-attacher
+    name: rbd-csi-attacher
     namespace: default
 roleRef:
   kind: ClusterRole
-  name: external-attacher-runner
+  name: rbd-external-attacher-runner
   apiGroup: rbac.authorization.k8s.io

deploy/rbd/kubernetes/csi-nodeplugin-rbac.yaml

@@ -1,13 +1,13 @@
 apiVersion: v1
 kind: ServiceAccount
 metadata:
-  name: csi-nodeplugin
+  name: rbd-csi-nodeplugin
 
 ---
 kind: ClusterRole
 apiVersion: rbac.authorization.k8s.io/v1
 metadata:
-  name: csi-nodeplugin
+  name: rbd-csi-nodeplugin
 rules:
   - apiGroups: [""]
     resources: ["nodes"]
@@ -23,18 +23,18 @@ rules:
     verbs: ["get", "list", "watch", "update"]
   - apiGroups: [""]
     resources: ["configmaps"]
-    verbs: ["get", "list", "create", "delete"]
+    verbs: ["get", "list"]
 
 ---
 kind: ClusterRoleBinding
 apiVersion: rbac.authorization.k8s.io/v1
 metadata:
-  name: csi-nodeplugin
+  name: rbd-csi-nodeplugin
 subjects:
   - kind: ServiceAccount
-    name: csi-nodeplugin
+    name: rbd-csi-nodeplugin
     namespace: default
 roleRef:
   kind: ClusterRole
-  name: csi-nodeplugin
+  name: rbd-csi-nodeplugin
   apiGroup: rbac.authorization.k8s.io          

deploy/rbd/kubernetes/csi-provisioner-rbac.yaml

@@ -1,13 +1,13 @@
 apiVersion: v1
 kind: ServiceAccount
 metadata:
-  name: csi-provisioner
+  name: rbd-csi-provisioner
 
 ---
 kind: ClusterRole
 apiVersion: rbac.authorization.k8s.io/v1
 metadata:
-  name: external-provisioner-runner
+  name: rbd-external-provisioner-runner
 rules:
   - apiGroups: [""]
     resources: ["secrets"]
@@ -27,17 +27,20 @@ rules:
   - apiGroups: [""]
     resources: ["endpoints"]
     verbs: ["get", "create", "update"]
+  - apiGroups: [""]
+    resources: ["configmaps"]
+    verbs: ["get", "list", "create", "delete"]
     
 ---
 kind: ClusterRoleBinding
 apiVersion: rbac.authorization.k8s.io/v1
 metadata:
-  name: csi-provisioner-role
+  name: rbd-csi-provisioner-role
 subjects:
   - kind: ServiceAccount
-    name: csi-provisioner
+    name: rbd-csi-provisioner
     namespace: default
 roleRef:
   kind: ClusterRole
-  name: external-provisioner-runner
+  name: rbd-external-provisioner-runner
   apiGroup: rbac.authorization.k8s.io

deploy/rbd/kubernetes/csi-rbdplugin-attacher.yaml

@@ -24,7 +24,7 @@ spec:
       labels:
         app: csi-rbdplugin-attacher
     spec:
-      serviceAccount: csi-attacher
+      serviceAccount: rbd-csi-attacher
       containers:
         - name: csi-rbdplugin-attacher
           image: quay.io/k8scsi/csi-attacher:v1.0.0
@@ -33,13 +33,13 @@ spec:
             - "--csi-address=$(ADDRESS)"
           env:
             - name: ADDRESS
-              value: /var/lib/kubelet/plugins_registry/csi-rbdplugin/csi.sock
+              value: /var/lib/kubelet/plugins/csi-rbdplugin/csi.sock
           imagePullPolicy: "IfNotPresent"
           volumeMounts:
             - name: socket-dir
-              mountPath: /var/lib/kubelet/plugins_registry/csi-rbdplugin
+              mountPath: /var/lib/kubelet/plugins/csi-rbdplugin
       volumes:
         - name: socket-dir
           hostPath:
-            path: /var/lib/kubelet/plugins_registry/csi-rbdplugin
+            path: /var/lib/kubelet/plugins/csi-rbdplugin
             type: DirectoryOrCreate

deploy/rbd/kubernetes/csi-rbdplugin-provisioner.yaml

@@ -24,7 +24,7 @@ spec:
       labels:
         app: csi-rbdplugin-provisioner
     spec:
-      serviceAccount: csi-provisioner
+      serviceAccount: rbd-csi-provisioner
       containers:
         - name: csi-provisioner
           image: quay.io/k8scsi/csi-provisioner:canary
@@ -33,13 +33,64 @@ spec:
             - "--v=5"
           env:
             - name: ADDRESS
-              value: /var/lib/kubelet/plugins_registry/csi-rbdplugin/csi.sock
+              value: /var/lib/kubelet/plugins/csi-rbdplugin/csi-provisioner.sock
           imagePullPolicy: "IfNotPresent"
           volumeMounts:
             - name: socket-dir
-              mountPath: /var/lib/kubelet/plugins_registry/csi-rbdplugin
+              mountPath: /var/lib/kubelet/plugins/csi-rbdplugin
+        - name: csi-rbdplugin
+          securityContext:
+            privileged: true
+            capabilities:
+              add: ["SYS_ADMIN"]
+          image: quay.io/cephcsi/rbdplugin:v1.0.0
+          args :
+            - "--nodeid=$(NODE_ID)"
+            - "--endpoint=$(CSI_ENDPOINT)"
+            - "--v=5"
+            - "--drivername=csi-rbdplugin"
+            - "--containerized=true"
+            - "--metadatastorage=k8s_configmap"
+          env:
+            - name: HOST_ROOTFS
+              value: "/rootfs" 
+            - name: NODE_ID
+              valueFrom:
+                fieldRef:
+                  fieldPath: spec.nodeName
+            - name: POD_NAMESPACE
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.namespace
+            - name: CSI_ENDPOINT
+              value: unix://var/lib/kubelet/plugins/csi-rbdplugin/csi-provisioner.sock
+          imagePullPolicy: "IfNotPresent"
+          volumeMounts:
+            - name: socket-dir
+              mountPath: /var/lib/kubelet/plugins/csi-rbdplugin
+            - mountPath: /dev
+              name: host-dev
+            - mountPath: /rootfs
+              name: host-rootfs            
+            - mountPath: /sys
+              name: host-sys
+            - mountPath: /lib/modules
+              name: lib-modules
+              readOnly: true
       volumes:
+        - name: host-dev
+          hostPath:
+            path: /dev
+        - name: host-rootfs
+          hostPath:
+            path: /            
+        - name: host-sys
+          hostPath:
+            path: /sys
+        - name: lib-modules
+          hostPath:
+            path: /lib/modules              
         - name: socket-dir
           hostPath:
-            path: /var/lib/kubelet/plugins_registry/csi-rbdplugin
+            path: /var/lib/kubelet/plugins/csi-rbdplugin
             type: DirectoryOrCreate

deploy/rbd/kubernetes/csi-rbdplugin.yaml

@@ -11,7 +11,7 @@ spec:
       labels:
         app: csi-rbdplugin
     spec:
-      serviceAccount: csi-nodeplugin
+      serviceAccount: rbd-csi-nodeplugin
       hostNetwork: true
       hostPID: true      
       # to use e.g. Rook orchestrated cluster, and mons' FQDN is
@@ -19,23 +19,23 @@ spec:
       dnsPolicy: ClusterFirstWithHostNet      
       containers:
         - name: driver-registrar
-          image: quay.io/k8scsi/driver-registrar:canary
+          image: quay.io/k8scsi/csi-node-driver-registrar:v1.0.2
           args:
             - "--v=5"
-            - "--csi-address=$(ADDRESS)"
-            - "--kubelet-registration-path=$(DRIVER_REG_SOCK_PATH)"
+            - "--csi-address=/csi/csi.sock"
+            - "--kubelet-registration-path=/var/lib/kubelet/plugins/csi-rbdplugin/csi.sock"
+          lifecycle:
+            preStop:
+              exec:
+                  command: ["/bin/sh", "-c", "rm -rf /registration/csi-rbdplugin /registration/csi-rbdplugin-reg.sock"]          
           env:
-            - name: ADDRESS
-              value: /var/lib/kubelet/plugins_registry/csi-rbdplugin/csi.sock
-            - name: DRIVER_REG_SOCK_PATH
-              value: /var/lib/kubelet/plugins_registry/csi-rbdplugin/csi.sock
             - name: KUBE_NODE_NAME
               valueFrom:
                 fieldRef:
                   fieldPath: spec.nodeName
           volumeMounts:
-            - name: socket-dir
-              mountPath: /var/lib/kubelet/plugins_registry/csi-rbdplugin
+            - name: plugin-dir
+              mountPath: /csi
             - name: registration-dir
               mountPath: /registration
         - name: csi-rbdplugin
@@ -87,7 +87,7 @@ spec:
       volumes:
         - name: plugin-dir
           hostPath:
-            path: /var/lib/kubelet/plugins_registry/csi-rbdplugin
+            path: /var/lib/kubelet/plugins/csi-rbdplugin
             type: DirectoryOrCreate
         - name: plugin-mount-dir
           hostPath: 
@@ -101,10 +101,6 @@ spec:
           hostPath:
             path: /var/lib/kubelet/pods
             type: Directory
-        - name: socket-dir
-          hostPath:
-            path: /var/lib/kubelet/plugins_registry/csi-rbdplugin
-            type: DirectoryOrCreate
         - name: host-dev
           hostPath:
             path: /dev

pkg/cephfs/driver.go

@@ -26,7 +26,7 @@ import (
 )
 
 const (
-	PluginFolder = "/var/lib/kubelet/plugins_registry/csi-cephfsplugin"
+	PluginFolder = "/var/lib/kubelet/plugins/csi-cephfsplugin"
 	Version      = "1.0.0"
 )
 

pkg/cephfs/nodeserver.go

@@ -241,3 +241,7 @@ func (ns *nodeServer) NodeGetCapabilities(ctx context.Context, req *csi.NodeGetC
 		},
 	}, nil
 }
+
+func (ns *nodeServer) NodeGetInfo(ctx context.Context, req *csi.NodeGetInfoRequest) (*csi.NodeGetInfoResponse, error) {
+	return ns.DefaultNodeServer.NodeGetInfo(ctx, req)
+}

pkg/rbd/nodeserver.go

@@ -218,6 +218,10 @@ func (ns *nodeServer) NodeUnstageVolume(
 	return nil, status.Error(codes.Unimplemented, "")
 }
 
+func (ns *nodeServer) NodeGetInfo(ctx context.Context, req *csi.NodeGetInfoRequest) (*csi.NodeGetInfoResponse, error) {
+	return ns.DefaultNodeServer.NodeGetInfo(ctx, req)
+}
+
 func resolveBindMountedBlockDevice(mountPath string) (string, error) {
 	cmd := exec.Command("findmnt", "-n", "-o", "SOURCE", "--first-only", "--target", mountPath)
 	out, err := cmd.CombinedOutput()

pkg/rbd/rbd.go

@@ -30,7 +30,7 @@ import (
 
 // PluginFolder defines the location of rbdplugin
 const (
-	PluginFolder      = "/var/lib/kubelet/plugins_registry/csi-rbdplugin"
+	PluginFolder      = "/var/lib/kubelet/plugins/csi-rbdplugin"
 	rbdDefaultAdminId = "admin"
 	rbdDefaultUserId  = rbdDefaultAdminId
 )