Merge pull request #89 from rootfs/containerized

support nsmounter when running in containerized mode

deploy/rbd/kubernetes/csi-rbdplugin.yaml

@@ -13,6 +13,7 @@ spec:
     spec:
       serviceAccount: csi-nodeplugin
       hostNetwork: true
+      hostPID: true      
       # to use e.g. Rook orchestrated cluster, and mons' FQDN is
       # resolved through k8s service, set dns policy to cluster first
       dnsPolicy: ClusterFirstWithHostNet      
@@ -49,9 +50,10 @@ spec:
             - "--endpoint=$(CSI_ENDPOINT)"
             - "--v=5"
             - "--drivername=csi-rbdplugin"
+            - "--containerized=true"            
           env:
             - name: HOST_ROOTFS
-              value: "/host"
+              value: "/rootfs"
             - name: NODE_ID
               valueFrom:
                 fieldRef:
@@ -67,7 +69,7 @@ spec:
               mountPropagation: "Bidirectional"
             - mountPath: /dev
               name: host-dev
-            - mountPath: /host
+            - mountPath: /rootfs
               name: host-rootfs            
             - mountPath: /sys
               name: host-sys

docs/deploy-rbd.md

@@ -25,6 +25,8 @@ Option | Default value | Description
 `--endpoint` | `unix://tmp/csi.sock` | CSI endpoint, must be a UNIX socket
 `--drivername` | `csi-cephfsplugin` | name of the driver (Kubernetes: `provisioner` field in StorageClass must correspond to this value)
 `--nodeid` | _empty_ | This node's ID
+`--containerized` | true | Whether running in containerized mode
+
 
 **Available environmental variables:**
 `HOST_ROOTFS`: rbdplugin searches `/proc` directory under the directory set by `HOST_ROOTFS`.

pkg/rbd/nodeserver.go

@@ -35,6 +35,7 @@ import (
 
 type nodeServer struct {
 	*csicommon.DefaultNodeServer
+	mounter mount.Interface
 }
 
 func (ns *nodeServer) NodePublishVolume(ctx context.Context, req *csi.NodePublishVolumeRequest) (*csi.NodePublishVolumeResponse, error) {
@@ -46,7 +47,7 @@ func (ns *nodeServer) NodePublishVolume(ctx context.Context, req *csi.NodePublis
 	s := strings.Split(strings.TrimSuffix(targetPath, "/mount"), "/")
 	volName := s[len(s)-1]
 
-	notMnt, err := mount.New("").IsLikelyNotMountPoint(targetPath)
+	notMnt, err := ns.mounter.IsLikelyNotMountPoint(targetPath)
 	if err != nil {
 		if os.IsNotExist(err) {
 			if err = os.MkdirAll(targetPath, 0750); err != nil {
@@ -86,7 +87,7 @@ func (ns *nodeServer) NodePublishVolume(ctx context.Context, req *csi.NodePublis
 		options = append(options, "ro")
 	}
 
-	diskMounter := &mount.SafeFormatAndMount{Interface: mount.New(""), Exec: mount.NewOsExec()}
+	diskMounter := &mount.SafeFormatAndMount{Interface: ns.mounter, Exec: mount.NewOsExec()}
 	if err := diskMounter.FormatAndMount(devicePath, targetPath, fsType, options); err != nil {
 		return nil, err
 	}
@@ -96,9 +97,8 @@ func (ns *nodeServer) NodePublishVolume(ctx context.Context, req *csi.NodePublis
 
 func (ns *nodeServer) NodeUnpublishVolume(ctx context.Context, req *csi.NodeUnpublishVolumeRequest) (*csi.NodeUnpublishVolumeResponse, error) {
 	targetPath := req.GetTargetPath()
-	mounter := mount.New("")
 
-	notMnt, err := mounter.IsLikelyNotMountPoint(targetPath)
+	notMnt, err := ns.mounter.IsLikelyNotMountPoint(targetPath)
 	if err != nil {
 		return nil, status.Error(codes.Internal, err.Error())
 	}
@@ -106,13 +106,13 @@ func (ns *nodeServer) NodeUnpublishVolume(ctx context.Context, req *csi.NodeUnpu
 		return nil, status.Error(codes.NotFound, "Volume not mounted")
 	}
 
-	devicePath, cnt, err := mount.GetDeviceNameFromMount(mounter, targetPath)
+	devicePath, cnt, err := mount.GetDeviceNameFromMount(ns.mounter, targetPath)
 	if err != nil {
 		return nil, status.Error(codes.Internal, err.Error())
 	}
 
 	// Unmounting the image
-	err = mounter.Unmount(targetPath)
+	err = ns.mounter.Unmount(targetPath)
 	if err != nil {
 		return nil, status.Error(codes.Internal, err.Error())
 	}

pkg/rbd/rbd.go

@@ -27,6 +27,10 @@ import (
 
 	"github.com/container-storage-interface/spec/lib/go/csi/v0"
 	"github.com/kubernetes-csi/drivers/pkg/csi-common"
+
+	"k8s.io/kubernetes/pkg/util/mount"
+	"k8s.io/kubernetes/pkg/util/nsenter"
+	"k8s.io/utils/exec"
 )
 
 // PluginFolder defines the location of rbdplugin
@@ -156,13 +160,23 @@ func NewControllerServer(d *csicommon.CSIDriver) *controllerServer {
 	}
 }
 
-func NewNodeServer(d *csicommon.CSIDriver) *nodeServer {
+func NewNodeServer(d *csicommon.CSIDriver, containerized bool) (*nodeServer, error) {
+	mounter := mount.New("")
+	if containerized {
+		ne, err := nsenter.NewNsenter(nsenter.DefaultHostRootFsPath, exec.New())
+		if err != nil {
+			return nil, err
+		}
+		mounter = mount.NewNsenterMounter("", ne)
+	}
 	return &nodeServer{
 		DefaultNodeServer: csicommon.NewDefaultNodeServer(d),
-	}
+		mounter:           mounter,
+	}, nil
 }
 
-func (rbd *rbd) Run(driverName, nodeID, endpoint string) {
+func (rbd *rbd) Run(driverName, nodeID, endpoint string, containerized bool) {
+	var err error
 	glog.Infof("Driver: %v version: %v", driverName, version)
 
 	// Initialize default library driver
@@ -180,7 +194,10 @@ func (rbd *rbd) Run(driverName, nodeID, endpoint string) {
 
 	// Create GRPC servers
 	rbd.ids = NewIdentityServer(rbd.driver)
-	rbd.ns = NewNodeServer(rbd.driver)
+	rbd.ns, err = NewNodeServer(rbd.driver, containerized)
+	if err != nil {
+		glog.Fatalln("failed to start node server, err %v", err)
+	}
 	rbd.cs = NewControllerServer(rbd.driver)
 	s := csicommon.NewNonBlockingGRPCServer()
 	s.Start(endpoint, rbd.ids, rbd.cs, rbd.ns)

rbd/main.go

@@ -30,9 +30,10 @@ func init() {
 }
 
 var (
-	endpoint   = flag.String("endpoint", "unix://tmp/csi.sock", "CSI endpoint")
+	endpoint      = flag.String("endpoint", "unix://tmp/csi.sock", "CSI endpoint")
-	driverName = flag.String("drivername", "csi-rbdplugin", "name of the driver")
+	driverName    = flag.String("drivername", "csi-rbdplugin", "name of the driver")
-	nodeID     = flag.String("nodeid", "", "node id")
+	nodeID        = flag.String("nodeid", "", "node id")
+	containerized = flag.Bool("containerized", true, "whether run as containerized")
 )
 
 func main() {
@@ -53,7 +54,7 @@ func main() {
 
 func handle() {
 	driver := rbd.GetRBDDriver()
-	driver.Run(*driverName, *nodeID, *endpoint)
+	driver.Run(*driverName, *nodeID, *endpoint, *containerized)
 }
 
 func createPersistentStorage(persistentStoragePath string) error {