From 3436a094f753471f0bd495b4601c0b7ef31f9470 Mon Sep 17 00:00:00 2001 From: Huamin Chen Date: Mon, 15 Oct 2018 14:59:41 +0000 Subject: [PATCH] support nsmounter when running in containerized mode Signed-off-by: Huamin Chen --- deploy/rbd/kubernetes/csi-rbdplugin.yaml | 6 ++++-- docs/deploy-rbd.md | 2 ++ pkg/rbd/nodeserver.go | 12 ++++++------ pkg/rbd/rbd.go | 25 ++++++++++++++++++++---- rbd/main.go | 9 +++++---- 5 files changed, 38 insertions(+), 16 deletions(-) diff --git a/deploy/rbd/kubernetes/csi-rbdplugin.yaml b/deploy/rbd/kubernetes/csi-rbdplugin.yaml index e68681f72..d641a7832 100644 --- a/deploy/rbd/kubernetes/csi-rbdplugin.yaml +++ b/deploy/rbd/kubernetes/csi-rbdplugin.yaml @@ -13,6 +13,7 @@ spec: spec: serviceAccount: csi-nodeplugin hostNetwork: true + hostPID: true # to use e.g. Rook orchestrated cluster, and mons' FQDN is # resolved through k8s service, set dns policy to cluster first dnsPolicy: ClusterFirstWithHostNet @@ -49,9 +50,10 @@ spec: - "--endpoint=$(CSI_ENDPOINT)" - "--v=5" - "--drivername=csi-rbdplugin" + - "--containerized=true" env: - name: HOST_ROOTFS - value: "/host" + value: "/rootfs" - name: NODE_ID valueFrom: fieldRef: @@ -67,7 +69,7 @@ spec: mountPropagation: "Bidirectional" - mountPath: /dev name: host-dev - - mountPath: /host + - mountPath: /rootfs name: host-rootfs - mountPath: /sys name: host-sys diff --git a/docs/deploy-rbd.md b/docs/deploy-rbd.md index ffab3f4c8..4f876655f 100644 --- a/docs/deploy-rbd.md +++ b/docs/deploy-rbd.md @@ -25,6 +25,8 @@ Option | Default value | Description `--endpoint` | `unix://tmp/csi.sock` | CSI endpoint, must be a UNIX socket `--drivername` | `csi-cephfsplugin` | name of the driver (Kubernetes: `provisioner` field in StorageClass must correspond to this value) `--nodeid` | _empty_ | This node's ID +`--containerized` | true | Whether running in containerized mode + **Available environmental variables:** `HOST_ROOTFS`: rbdplugin searches `/proc` directory under the directory set by `HOST_ROOTFS`. diff --git a/pkg/rbd/nodeserver.go b/pkg/rbd/nodeserver.go index ca309b18e..acf178672 100644 --- a/pkg/rbd/nodeserver.go +++ b/pkg/rbd/nodeserver.go @@ -35,6 +35,7 @@ import ( type nodeServer struct { *csicommon.DefaultNodeServer + mounter mount.Interface } func (ns *nodeServer) NodePublishVolume(ctx context.Context, req *csi.NodePublishVolumeRequest) (*csi.NodePublishVolumeResponse, error) { @@ -46,7 +47,7 @@ func (ns *nodeServer) NodePublishVolume(ctx context.Context, req *csi.NodePublis s := strings.Split(strings.TrimSuffix(targetPath, "/mount"), "/") volName := s[len(s)-1] - notMnt, err := mount.New("").IsLikelyNotMountPoint(targetPath) + notMnt, err := ns.mounter.IsLikelyNotMountPoint(targetPath) if err != nil { if os.IsNotExist(err) { if err = os.MkdirAll(targetPath, 0750); err != nil { @@ -86,7 +87,7 @@ func (ns *nodeServer) NodePublishVolume(ctx context.Context, req *csi.NodePublis options = append(options, "ro") } - diskMounter := &mount.SafeFormatAndMount{Interface: mount.New(""), Exec: mount.NewOsExec()} + diskMounter := &mount.SafeFormatAndMount{Interface: ns.mounter, Exec: mount.NewOsExec()} if err := diskMounter.FormatAndMount(devicePath, targetPath, fsType, options); err != nil { return nil, err } @@ -96,9 +97,8 @@ func (ns *nodeServer) NodePublishVolume(ctx context.Context, req *csi.NodePublis func (ns *nodeServer) NodeUnpublishVolume(ctx context.Context, req *csi.NodeUnpublishVolumeRequest) (*csi.NodeUnpublishVolumeResponse, error) { targetPath := req.GetTargetPath() - mounter := mount.New("") - notMnt, err := mounter.IsLikelyNotMountPoint(targetPath) + notMnt, err := ns.mounter.IsLikelyNotMountPoint(targetPath) if err != nil { return nil, status.Error(codes.Internal, err.Error()) } @@ -106,13 +106,13 @@ func (ns *nodeServer) NodeUnpublishVolume(ctx context.Context, req *csi.NodeUnpu return nil, status.Error(codes.NotFound, "Volume not mounted") } - devicePath, cnt, err := mount.GetDeviceNameFromMount(mounter, targetPath) + devicePath, cnt, err := mount.GetDeviceNameFromMount(ns.mounter, targetPath) if err != nil { return nil, status.Error(codes.Internal, err.Error()) } // Unmounting the image - err = mounter.Unmount(targetPath) + err = ns.mounter.Unmount(targetPath) if err != nil { return nil, status.Error(codes.Internal, err.Error()) } diff --git a/pkg/rbd/rbd.go b/pkg/rbd/rbd.go index 3f0a04f70..7ef552338 100644 --- a/pkg/rbd/rbd.go +++ b/pkg/rbd/rbd.go @@ -27,6 +27,10 @@ import ( "github.com/container-storage-interface/spec/lib/go/csi/v0" "github.com/kubernetes-csi/drivers/pkg/csi-common" + + "k8s.io/kubernetes/pkg/util/mount" + "k8s.io/kubernetes/pkg/util/nsenter" + "k8s.io/utils/exec" ) // PluginFolder defines the location of rbdplugin @@ -156,13 +160,23 @@ func NewControllerServer(d *csicommon.CSIDriver) *controllerServer { } } -func NewNodeServer(d *csicommon.CSIDriver) *nodeServer { +func NewNodeServer(d *csicommon.CSIDriver, containerized bool) (*nodeServer, error) { + mounter := mount.New("") + if containerized { + ne, err := nsenter.NewNsenter(nsenter.DefaultHostRootFsPath, exec.New()) + if err != nil { + return nil, err + } + mounter = mount.NewNsenterMounter("", ne) + } return &nodeServer{ DefaultNodeServer: csicommon.NewDefaultNodeServer(d), - } + mounter: mounter, + }, nil } -func (rbd *rbd) Run(driverName, nodeID, endpoint string) { +func (rbd *rbd) Run(driverName, nodeID, endpoint string, containerized bool) { + var err error glog.Infof("Driver: %v version: %v", driverName, version) // Initialize default library driver @@ -180,7 +194,10 @@ func (rbd *rbd) Run(driverName, nodeID, endpoint string) { // Create GRPC servers rbd.ids = NewIdentityServer(rbd.driver) - rbd.ns = NewNodeServer(rbd.driver) + rbd.ns, err = NewNodeServer(rbd.driver, containerized) + if err != nil { + glog.Fatalln("failed to start node server, err %v", err) + } rbd.cs = NewControllerServer(rbd.driver) s := csicommon.NewNonBlockingGRPCServer() s.Start(endpoint, rbd.ids, rbd.cs, rbd.ns) diff --git a/rbd/main.go b/rbd/main.go index 6a2adcca0..649589441 100644 --- a/rbd/main.go +++ b/rbd/main.go @@ -30,9 +30,10 @@ func init() { } var ( - endpoint = flag.String("endpoint", "unix://tmp/csi.sock", "CSI endpoint") - driverName = flag.String("drivername", "csi-rbdplugin", "name of the driver") - nodeID = flag.String("nodeid", "", "node id") + endpoint = flag.String("endpoint", "unix://tmp/csi.sock", "CSI endpoint") + driverName = flag.String("drivername", "csi-rbdplugin", "name of the driver") + nodeID = flag.String("nodeid", "", "node id") + containerized = flag.Bool("containerized", true, "whether run as containerized") ) func main() { @@ -53,7 +54,7 @@ func main() { func handle() { driver := rbd.GetRBDDriver() - driver.Run(*driverName, *nodeID, *endpoint) + driver.Run(*driverName, *nodeID, *endpoint, *containerized) } func createPersistentStorage(persistentStoragePath string) error {