Use "rbd device list" to list and find rbd images and their device paths

This change also starts mapping nbd based access using ther rbd CLI
as, it is a prerequisite to get device listing for nbd as well.

Signed-off-by: ShyamsundarR <srangana@redhat.com>

deploy/cephfs/kubernetes/v1.13/helm/templates/nodeplugin-daemonset.yaml

@@ -73,8 +73,6 @@ spec:
             - "--metadatastorage=k8s_configmap"
             - "--mountcachedir=/mount-cache-dir"
           env:
-            - name: HOST_ROOTFS
-              value: "/rootfs"
             - name: DRIVER_NAME
               value: {{ .Values.driverName }}
             - name: NODE_ID

deploy/cephfs/kubernetes/v1.13/helm/templates/provisioner-statefulset.yaml

@@ -72,8 +72,6 @@ spec:
             - "--drivername=$(DRIVER_NAME)"
             - "--metadatastorage=k8s_configmap"
           env:
-            - name: HOST_ROOTFS
-              value: "/rootfs"
             - name: DRIVER_NAME
               value: {{ .Values.driverName }}
             - name: NODE_ID

deploy/cephfs/kubernetes/v1.14+/helm/templates/nodeplugin-daemonset.yaml

@@ -73,8 +73,6 @@ spec:
             - "--metadatastorage=k8s_configmap"
             - "--mountcachedir=/mount-cache-dir"
           env:
-            - name: HOST_ROOTFS
-              value: "/rootfs"
             - name: DRIVER_NAME
               value: {{ .Values.driverName }}
             - name: NODE_ID

deploy/cephfs/kubernetes/v1.14+/helm/templates/provisioner-deployment.yaml

@@ -75,8 +75,6 @@ spec:
             - "--drivername=$(DRIVER_NAME)"
             - "--metadatastorage=k8s_configmap"
           env:
-            - name: HOST_ROOTFS
-              value: "/rootfs"
             - name: DRIVER_NAME
               value: {{ .Values.driverName }}
             - name: NODE_ID

deploy/rbd/kubernetes/v1.13/csi-rbdplugin-provisioner.yaml

@@ -86,8 +86,6 @@ spec:
             - "--drivername=rbd.csi.ceph.com"
             - "--containerized=true"
           env:
-            - name: HOST_ROOTFS
-              value: "/rootfs"
             - name: NODE_ID
               valueFrom:
                 fieldRef:

deploy/rbd/kubernetes/v1.13/csi-rbdplugin.yaml

@@ -61,8 +61,6 @@ spec:
             - "--drivername=rbd.csi.ceph.com"
             - "--containerized=true"
           env:
-            - name: HOST_ROOTFS
-              value: "/rootfs"
             - name: NODE_ID
               valueFrom:
                 fieldRef:

deploy/rbd/kubernetes/v1.13/helm/templates/nodeplugin-daemonset.yaml

@@ -74,8 +74,6 @@ spec:
             - "--drivername=$(DRIVER_NAME)"
             - "--containerized=true"
           env:
-            - name: HOST_ROOTFS
-              value: "/rootfs"
             - name: DRIVER_NAME
               value: {{ .Values.driverName }}
             - name: NODE_ID

deploy/rbd/kubernetes/v1.13/helm/templates/provisioner-statefulset.yaml

@@ -89,8 +89,6 @@ spec:
             - "--drivername=$(DRIVER_NAME)"
             - "--containerized=true"
           env:
-            - name: HOST_ROOTFS
-              value: "/rootfs"
             - name: DRIVER_NAME
               value: {{ .Values.driverName }}
             - name: NODE_ID

deploy/rbd/kubernetes/v1.14+/csi-rbdplugin-provisioner.yaml

@@ -76,8 +76,6 @@ spec:
             - "--drivername=rbd.csi.ceph.com"
             - "--containerized=true"
           env:
-            - name: HOST_ROOTFS
-              value: "/rootfs"
             - name: NODE_ID
               valueFrom:
                 fieldRef:

deploy/rbd/kubernetes/v1.14+/csi-rbdplugin.yaml

@@ -61,8 +61,6 @@ spec:
             - "--drivername=rbd.csi.ceph.com"
             - "--containerized=true"
           env:
-            - name: HOST_ROOTFS
-              value: "/rootfs"
             - name: NODE_ID
               valueFrom:
                 fieldRef:

deploy/rbd/kubernetes/v1.14+/helm/templates/nodeplugin-daemonset.yaml

@@ -74,8 +74,6 @@ spec:
             - "--drivername=$(DRIVER_NAME)"
             - "--containerized=true"
           env:
-            - name: HOST_ROOTFS
-              value: "/rootfs"
             - name: DRIVER_NAME
               value: {{ .Values.driverName }}
             - name: NODE_ID

deploy/rbd/kubernetes/v1.14+/helm/templates/provisioner-deployment.yaml

@@ -93,8 +93,6 @@ spec:
             - "--drivername=$(DRIVER_NAME)"
             - "--containerized=true"
           env:
-            - name: HOST_ROOTFS
-              value: "/rootfs"
             - name: DRIVER_NAME
               value: {{ .Values.driverName }}
             - name: NODE_ID

docs/deploy-rbd.md

@@ -36,10 +36,6 @@ make image-cephcsi
 | `--instanceid`      | "default"             | Unique ID distinguishing this instance of Ceph CSI among other instances, when sharing Ceph clusters across CSI instances for provisioning                                   |
 | `--metadatastorage` | _empty_               | Points to where legacy (1.0.0 or older plugin versions) metadata about provisioned volumes are kept, as file or in as k8s configmap (`node` or `k8s_configmap` respectively) |
 
-**Available environmental variables:**
-
-`HOST_ROOTFS`: rbdplugin searches `/proc` directory under the directory set by `HOST_ROOTFS`.
-
 **Available volume parameters:**
 
 | Parameter                                                                                             | Required             | Description                                                                                                                                                                                                             |

pkg/rbd/rbd_attach.go

@@ -17,10 +17,8 @@ limitations under the License.
 package rbd
 
 import (
+	"encoding/json"
 	"fmt"
-	"io/ioutil"
-	"os"
-	"path"
 	"strconv"
 	"strings"
 	"time"
@@ -32,189 +30,121 @@ import (
 )
 
 const (
-	envHostRootFS = "HOST_ROOTFS"
+	rbdTonbd  = "rbd-nbd"
-	rbdTonbd      = "rbd-nbd"
+	moduleNbd = "nbd"
-	rbd           = "rbd"
+
-	nbd           = "nbd"
+	accessTypeKRbd = "krbd"
+	accessTypeNbd  = "nbd"
+
+	rbd = "rbd"
 )
 
-var (
+var hasNBD = false
-	hostRootFS = "/"
-	hasNBD     = false
-)
 
 func init() {
-	host := os.Getenv(envHostRootFS)
-	if len(host) > 0 {
-		hostRootFS = host
-	}
 	hasNBD = checkRbdNbdTools()
 }
 
-// Search /sys/bus for rbd device that matches given pool and image.
+// rbdDeviceInfo strongly typed JSON spec for rbd device list output (of type krbd)
-func getRbdDevFromImageAndPool(pool, image string) (string, bool) {
+type rbdDeviceInfo struct {
-	// /sys/bus/rbd/devices/X/name and /sys/bus/rbd/devices/X/pool
+	ID     string `json:"id"`
-	sysPath := "/sys/bus/rbd/devices"
+	Pool   string `json:"pool"`
-	if dirs, err := ioutil.ReadDir(sysPath); err == nil {
+	Name   string `json:"name"`
-		for _, f := range dirs {
+	Device string `json:"device"`
-			// Pool and name format:
+}
-			// see rbd_pool_show() and rbd_name_show() at
+
-			// https://github.com/torvalds/linux/blob/master/drivers/block/rbd.c
+// nbdDeviceInfo strongly typed JSON spec for rbd-nbd device list output (of type nbd)
-			name := f.Name()
+// NOTE: There is a bug in rbd output that returns id as number for nbd, and string for krbd, thus
-			// First match pool, then match name.
+// requiring 2 different JSON structures to unmarshal the output.
-			poolFile := path.Join(sysPath, name, "pool")
+// NOTE: image key is "name" in krbd output and "image" in nbd output, which is another difference
-			// #nosec
+type nbdDeviceInfo struct {
-			poolBytes, err := ioutil.ReadFile(poolFile)
+	ID     int64  `json:"id"`
-			if err != nil {
+	Pool   string `json:"pool"`
-				klog.V(4).Infof("error reading %s: %v", poolFile, err)
+	Name   string `json:"image"`
-				continue
+	Device string `json:"device"`
-			}
+}
-			if strings.TrimSpace(string(poolBytes)) != pool {
+
-				klog.V(4).Infof("device %s is not %q: %q", name, pool, string(poolBytes))
+// rbdGetDeviceList queries rbd about mapped devices and returns a list of rbdDeviceInfo
-				continue
+// It will selectively list devices mapped using krbd or nbd as specified by accessType
-			}
+func rbdGetDeviceList(accessType string) ([]rbdDeviceInfo, error) {
-			imgFile := path.Join(sysPath, name, "name")
+	// rbd device list --format json --device-type [krbd|nbd]
-			// #nosec
+	var (
-			imgBytes, err := ioutil.ReadFile(imgFile)
+		rbdDeviceList []rbdDeviceInfo
-			if err != nil {
+		nbdDeviceList []nbdDeviceInfo
-				klog.V(4).Infof("error reading %s: %v", imgFile, err)
+	)
-				continue
+
-			}
+	stdout, _, err := util.ExecCommand(rbd, "device", "list", "--format="+"json", "--device-type", accessType)
-			if strings.TrimSpace(string(imgBytes)) != image {
+	if err != nil {
-				klog.V(4).Infof("device %s is not %q: %q", name, image, string(imgBytes))
+		return nil, fmt.Errorf("error getting device list from rbd for devices of type (%s): (%v)", accessType, err)
-				continue
+	}
-			}
+
-			// Found a match, check if device exists.
+	if accessType == accessTypeKRbd {
-			devicePath := "/dev/rbd" + name
+		err = json.Unmarshal(stdout, &rbdDeviceList)
-			if _, err := os.Lstat(devicePath); err == nil {
+	} else {
-				return devicePath, true
+		err = json.Unmarshal(stdout, &nbdDeviceList)
-			}
+	}
+	if err != nil {
+		return nil, fmt.Errorf("error to parse JSON output of device list for devices of type (%s): (%v)", accessType, err)
+	}
+
+	// convert output to a rbdDeviceInfo list for consumers
+	if accessType == accessTypeNbd {
+		for _, device := range nbdDeviceList {
+			rbdDeviceList = append(
+				rbdDeviceList,
+				rbdDeviceInfo{
+					ID:     strconv.FormatInt(device.ID, 10),
+					Pool:   device.Pool,
+					Name:   device.Name,
+					Device: device.Device,
+				})
 		}
 	}
-	return "", false
+
+	return rbdDeviceList, nil
 }
 
-func getMaxNbds() (int, error) {
+// findDeviceMappingImage finds a devicePath, if available, based on image spec (pool/image) on the node.
-
+func findDeviceMappingImage(pool, image string, useNbdDriver bool) (string, bool) {
-	// the max number of nbd devices may be found in maxNbdsPath
+	accessType := accessTypeKRbd
-	// we will check sysfs for possible nbd devices even if this is not available
+	if useNbdDriver {
-	maxNbdsPath := "/sys/module/nbd/parameters/nbds_max"
+		accessType = accessTypeNbd
-	_, err := os.Lstat(maxNbdsPath)
-	if err != nil {
-		return 0, fmt.Errorf("rbd-nbd: failed to retrieve max_nbds from %s err: %q", maxNbdsPath, err)
 	}
 
-	klog.V(4).Infof("found nbds max parameters file at %s", maxNbdsPath)
+	rbdDeviceList, err := rbdGetDeviceList(accessType)
-
-	maxNbdBytes, err := ioutil.ReadFile(maxNbdsPath)
 	if err != nil {
-		return 0, fmt.Errorf("rbd-nbd: failed to read max_nbds from %s err: %q", maxNbdsPath, err)
+		klog.Warningf("failed to determine if image (%s/%s) is mapped to a device (%v)", pool, image, err)
-	}
-
-	maxNbds, err := strconv.Atoi(strings.TrimSpace(string(maxNbdBytes)))
-	if err != nil {
-		return 0, fmt.Errorf("rbd-nbd: failed to read max_nbds err: %q", err)
-	}
-
-	klog.V(4).Infof("rbd-nbd: max_nbds: %d", maxNbds)
-	return maxNbds, nil
-}
-
-// Locate any existing rbd-nbd process mapping given a <pool, image>.
-// Recent versions of rbd-nbd tool can correctly provide this info using list-mapped
-// but older versions of list-mapped don't.
-// The implementation below peeks at the command line of nbd bound processes
-// to figure out any mapped images.
-func getNbdDevFromImageAndPool(pool, image string) (string, bool) {
-	// nbd module exports the pid of serving process in sysfs
-	basePath := "/sys/block/nbd"
-	// Do not change imgPath format - some tools like rbd-nbd are strict about it.
-	imgPath := fmt.Sprintf("%s/%s", pool, image)
-
-	maxNbds, maxNbdsErr := getMaxNbds()
-	if maxNbdsErr != nil {
-		klog.V(4).Infof("error reading nbds_max %v", maxNbdsErr)
 		return "", false
 	}
 
-	for i := 0; i < maxNbds; i++ {
+	for _, device := range rbdDeviceList {
-		nbdPath := basePath + strconv.Itoa(i)
+		if device.Name == image && device.Pool == pool {
-		devicePath, err := getnbdDevicePath(nbdPath, imgPath, i)
+			return device.Device, true
-		if err != nil {
-			continue
 		}
-		return devicePath, true
 	}
+
 	return "", false
 }
 
-func getnbdDevicePath(nbdPath, imgPath string, count int) (string, error) {
-
-	_, err := os.Lstat(nbdPath)
-	if err != nil {
-		klog.V(4).Infof("error reading nbd info directory %s: %v", nbdPath, err)
-		return "", err
-	}
-	// #nosec
-	pidBytes, err := ioutil.ReadFile(path.Join(nbdPath, "pid"))
-	if err != nil {
-		klog.V(5).Infof("did not find valid pid file in dir %s: %v", nbdPath, err)
-		return "", err
-	}
-	cmdlineFileName := path.Join(hostRootFS, "/proc", strings.TrimSpace(string(pidBytes)), "cmdline")
-	// #nosec
-	rawCmdline, err := ioutil.ReadFile(cmdlineFileName)
-	if err != nil {
-		klog.V(4).Infof("failed to read cmdline file %s: %v", cmdlineFileName, err)
-		return "", err
-	}
-	cmdlineArgs := strings.FieldsFunc(string(rawCmdline), func(r rune) bool {
-		return r == '\u0000'
-	})
-	// Check if this process is mapping a rbd device.
-	// Only accepted pattern of cmdline is from execRbdMap:
-	// rbd-nbd map pool/image ...
-	if len(cmdlineArgs) < 3 || cmdlineArgs[0] != rbdTonbd || cmdlineArgs[1] != "map" {
-		klog.V(4).Infof("nbd device %s is not used by rbd", nbdPath)
-		return "", fmt.Errorf("nbd device %s is not used by rbd", nbdPath)
-
-	}
-	if cmdlineArgs[2] != imgPath {
-		klog.V(4).Infof("rbd-nbd device %s did not match expected image path: %s with path found: %s",
-			nbdPath, imgPath, cmdlineArgs[2])
-		return "", fmt.Errorf("rbd-nbd device %s did not match expected image path: %s with path found: %s",
-			nbdPath, imgPath, cmdlineArgs[2])
-	}
-	devicePath := path.Join("/dev", "nbd"+strconv.Itoa(count))
-	if _, err := os.Lstat(devicePath); err != nil {
-		klog.Warningf("Stat device %s for imgpath %s failed %v", devicePath, imgPath, err)
-		return "", err
-	}
-	return devicePath, nil
-}
-
 // Stat a path, if it doesn't exist, retry maxRetries times.
 func waitForPath(pool, image string, maxRetries int, useNbdDriver bool) (string, bool) {
 	for i := 0; i < maxRetries; i++ {
 		if i != 0 {
 			time.Sleep(time.Second)
 		}
-		if useNbdDriver {
+
-			if devicePath, found := getNbdDevFromImageAndPool(pool, image); found {
+		device, found := findDeviceMappingImage(pool, image, useNbdDriver)
-				return devicePath, true
+		if found {
-			}
+			return device, found
-		} else {
-			if devicePath, found := getRbdDevFromImageAndPool(pool, image); found {
-				return devicePath, true
-			}
 		}
 	}
+
 	return "", false
 }
 
 // Check if rbd-nbd tools are installed.
 func checkRbdNbdTools() bool {
-	_, err := execCommand("modprobe", []string{"nbd"})
+	_, err := execCommand("modprobe", []string{moduleNbd})
 	if err != nil {
 		klog.V(3).Infof("rbd-nbd: nbd modprobe failed with error %v", err)
 		return false
@@ -232,20 +162,12 @@ func attachRBDImage(volOptions *rbdVolume, cr *util.Credentials) (string, error)
 
 	image := volOptions.RbdImageName
 	useNBD := false
-	moduleName := rbd
 	if volOptions.Mounter == rbdTonbd && hasNBD {
 		useNBD = true
-		moduleName = nbd
 	}
 
 	devicePath, found := waitForPath(volOptions.Pool, image, 1, useNBD)
 	if !found {
-		_, err = execCommand("modprobe", []string{moduleName})
-		if err != nil {
-			klog.Warningf("rbd: failed to load rbd kernel module:%v", err)
-			return "", err
-		}
-
 		backoff := wait.Backoff{
 			Duration: rbdImageWatcherInitDelay,
 			Factor:   rbdImageWatcherFactor,
@@ -269,18 +191,36 @@ func createPath(volOpt *rbdVolume, cr *util.Credentials) (string, error) {
 
 	klog.V(5).Infof("rbd: map mon %s", volOpt.Monitors)
 
-	cmdName := rbd
+	// Map options
+	mapOptions := []string{
+		"--id", cr.ID,
+		"-m", volOpt.Monitors,
+		"--keyfile=" + cr.KeyFile}
+
+	// Construct map and unmap variants for the command
+	mapOptions = append(mapOptions, "map", imagePath)
+	unmapOptions := []string{"unmap", imagePath}
+
+	// Choose access protocol
+	useNBD := false
+	accessType := accessTypeKRbd
 	if volOpt.Mounter == rbdTonbd && hasNBD {
-		cmdName = rbdTonbd
+		accessType = accessTypeNbd
+		useNBD = true
 	}
 
-	output, err := execCommand(cmdName, []string{
+	// Update options with device type selection
-		"map", imagePath, "--id", cr.ID, "-m", volOpt.Monitors, "--keyfile=" + cr.KeyFile})
+	mapOptions = append(mapOptions, "--device-type", accessType)
+	unmapOptions = append(unmapOptions, "--device-type", accessType)
+
+	// Execute map
+	output, err := execCommand(rbd, mapOptions)
 	if err != nil {
 		klog.Warningf("rbd: map error %v, rbd output: %s", err, string(output))
 		return "", fmt.Errorf("rbd: map failed %v, rbd output: %s", err, string(output))
 	}
 	devicePath := strings.TrimSuffix(string(output), "\n")
+
 	return devicePath, nil
 }
 
@@ -313,12 +253,13 @@ func detachRBDDevice(devicePath string) error {
 
 	klog.V(3).Infof("rbd: unmap device %s", devicePath)
 
-	cmdName := rbd
+	accessType := accessTypeKRbd
 	if strings.HasPrefix(devicePath, "/dev/nbd") {
-		cmdName = rbdTonbd
+		accessType = accessTypeNbd
 	}
+	options := []string{"unmap", "--device-type", accessType, devicePath}
 
-	output, err = execCommand(cmdName, []string{"unmap", devicePath})
+	output, err = execCommand(rbd, options)
 	if err != nil {
 		return fmt.Errorf("rbd: unmap failed %v, rbd output: %s", err, string(output))
 	}