From 8955eb03bc8c7a2db914a848fac2309feb0f6125 Mon Sep 17 00:00:00 2001 From: Huamin Chen Date: Mon, 17 Sep 2018 18:12:22 +0000 Subject: [PATCH 1/4] support rbd-nbd Signed-off-by: Huamin Chen --- deploy/rbd/docker/Dockerfile | 2 +- deploy/rbd/kubernetes/csi-rbdplugin.yaml | 9 +- pkg/rbd/rbd_attach.go | 303 +++++++++++++++++++++++ pkg/rbd/rbd_util.go | 125 ---------- 4 files changed, 312 insertions(+), 127 deletions(-) create mode 100644 pkg/rbd/rbd_attach.go diff --git a/deploy/rbd/docker/Dockerfile b/deploy/rbd/docker/Dockerfile index 84693ae28..00d0bca70 100644 --- a/deploy/rbd/docker/Dockerfile +++ b/deploy/rbd/docker/Dockerfile @@ -4,7 +4,7 @@ LABEL description="RBD CSI Plugin" ENV CEPH_VERSION "mimic" RUN yum install -y centos-release-ceph && \ - yum install -y ceph-common e2fsprogs && \ + yum install -y ceph-common e2fsprogs rbd-nbd && \ yum clean all COPY rbdplugin /rbdplugin diff --git a/deploy/rbd/kubernetes/csi-rbdplugin.yaml b/deploy/rbd/kubernetes/csi-rbdplugin.yaml index 0ea72f9ed..f0872e886 100644 --- a/deploy/rbd/kubernetes/csi-rbdplugin.yaml +++ b/deploy/rbd/kubernetes/csi-rbdplugin.yaml @@ -34,7 +34,7 @@ spec: mountPath: /var/lib/kubelet/plugins/csi-rbdplugin - name: registration-dir mountPath: /registration - - name: csi-rbdplugin + - name: csi-rbdplugin securityContext: privileged: true capabilities: @@ -47,6 +47,8 @@ spec: - "--v=5" - "--drivername=csi-rbdplugin" env: + - name: HOST_ROOTFS + value: "/host" - name: NODE_ID valueFrom: fieldRef: @@ -62,6 +64,8 @@ spec: mountPropagation: "Bidirectional" - mountPath: /dev name: host-dev + - mountPath: /host + name: host-rootfs - mountPath: /sys name: host-sys - mountPath: /lib/modules @@ -87,6 +91,9 @@ spec: - name: host-dev hostPath: path: /dev + - name: host-rootfs + hostPath: + path: / - name: host-sys hostPath: path: /sys diff --git a/pkg/rbd/rbd_attach.go b/pkg/rbd/rbd_attach.go new file mode 100644 index 000000000..707d23e81 --- /dev/null +++ b/pkg/rbd/rbd_attach.go @@ -0,0 +1,303 @@ +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package rbd + +import ( + "fmt" + "io/ioutil" + "os" + "path" + "strconv" + "strings" + "time" + + "github.com/golang/glog" + "k8s.io/apimachinery/pkg/util/wait" +) + +const ( + envHostRootFS = "HOST_ROOTFS" +) + +var ( + hostRootFS = "/" + useNBD = false +) + +func init() { + host := os.Getenv(envHostRootFS) + if len(host) > 0 { + hostRootFS = host + } + useNBD = checkRbdNbdTools() +} + +func getDevFromImageAndPool(pool, image string) (string, bool) { + device, found := getRbdDevFromImageAndPool(pool, image) + if found { + return device, true + } + device, found = getNbdDevFromImageAndPool(pool, image) + if found { + return device, true + } + return "", false +} + +// Search /sys/bus for rbd device that matches given pool and image. +func getRbdDevFromImageAndPool(pool string, image string) (string, bool) { + // /sys/bus/rbd/devices/X/name and /sys/bus/rbd/devices/X/pool + sys_path := "/sys/bus/rbd/devices" + if dirs, err := ioutil.ReadDir(sys_path); err == nil { + for _, f := range dirs { + // Pool and name format: + // see rbd_pool_show() and rbd_name_show() at + // https://github.com/torvalds/linux/blob/master/drivers/block/rbd.c + name := f.Name() + // First match pool, then match name. + poolFile := path.Join(sys_path, name, "pool") + poolBytes, err := ioutil.ReadFile(poolFile) + if err != nil { + glog.V(4).Infof("error reading %s: %v", poolFile, err) + continue + } + if strings.TrimSpace(string(poolBytes)) != pool { + glog.V(4).Infof("device %s is not %q: %q", name, pool, string(poolBytes)) + continue + } + imgFile := path.Join(sys_path, name, "name") + imgBytes, err := ioutil.ReadFile(imgFile) + if err != nil { + glog.V(4).Infof("error reading %s: %v", imgFile, err) + continue + } + if strings.TrimSpace(string(imgBytes)) != image { + glog.V(4).Infof("device %s is not %q: %q", name, image, string(imgBytes)) + continue + } + // Found a match, check if device exists. + devicePath := "/dev/rbd" + name + if _, err := os.Lstat(devicePath); err == nil { + return devicePath, true + } + } + } + return "", false +} + +func getMaxNbds() (int, error) { + + // the max number of nbd devices may be found in maxNbdsPath + // we will check sysfs for possible nbd devices even if this is not available + maxNbdsPath := "/sys/module/nbd/parameters/nbds_max" + _, err := os.Lstat(maxNbdsPath) + if err != nil { + return 0, fmt.Errorf("rbd-nbd: failed to retrieve max_nbds from %s err: %q", maxNbdsPath, err) + } + + glog.V(4).Infof("found nbds max parameters file at %s", maxNbdsPath) + + maxNbdBytes, err := ioutil.ReadFile(maxNbdsPath) + if err != nil { + return 0, fmt.Errorf("rbd-nbd: failed to read max_nbds from %s err: %q", maxNbdsPath, err) + } + + maxNbds, err := strconv.Atoi(strings.TrimSpace(string(maxNbdBytes))) + if err != nil { + return 0, fmt.Errorf("rbd-nbd: failed to read max_nbds err: %q", err) + } + + glog.V(4).Infof("rbd-nbd: max_nbds: %d", maxNbds) + return maxNbds, nil +} + +// Locate any existing rbd-nbd process mapping given a . +// Recent versions of rbd-nbd tool can correctly provide this info using list-mapped +// but older versions of list-mapped don't. +// The implementation below peeks at the command line of nbd bound processes +// to figure out any mapped images. +func getNbdDevFromImageAndPool(pool string, image string) (string, bool) { + // nbd module exports the pid of serving process in sysfs + basePath := "/sys/block/nbd" + // Do not change imgPath format - some tools like rbd-nbd are strict about it. + imgPath := fmt.Sprintf("%s/%s", pool, image) + + maxNbds, maxNbdsErr := getMaxNbds() + if maxNbdsErr != nil { + glog.V(4).Infof("error reading nbds_max %v", maxNbdsErr) + return "", false + } + + for i := 0; i < maxNbds; i++ { + nbdPath := basePath + strconv.Itoa(i) + _, err := os.Lstat(nbdPath) + if err != nil { + glog.V(4).Infof("error reading nbd info directory %s: %v", nbdPath, err) + continue + } + pidBytes, err := ioutil.ReadFile(path.Join(nbdPath, "pid")) + if err != nil { + glog.V(5).Infof("did not find valid pid file in dir %s: %v", nbdPath, err) + continue + } + cmdlineFileName := path.Join(hostRootFS, "/proc", strings.TrimSpace(string(pidBytes)), "cmdline") + rawCmdline, err := ioutil.ReadFile(cmdlineFileName) + if err != nil { + glog.V(4).Infof("failed to read cmdline file %s: %v", cmdlineFileName, err) + continue + } + cmdlineArgs := strings.FieldsFunc(string(rawCmdline), func(r rune) bool { + return r == '\u0000' + }) + // Check if this process is mapping a rbd device. + // Only accepted pattern of cmdline is from execRbdMap: + // rbd-nbd map pool/image ... + if len(cmdlineArgs) < 3 || cmdlineArgs[0] != "rbd-nbd" || cmdlineArgs[1] != "map" { + glog.V(4).Infof("nbd device %s is not used by rbd", nbdPath) + continue + } + if cmdlineArgs[2] != imgPath { + glog.V(4).Infof("rbd-nbd device %s did not match expected image path: %s with path found: %s", + nbdPath, imgPath, cmdlineArgs[2]) + continue + } + devicePath := path.Join("/dev", "nbd"+strconv.Itoa(i)) + if _, err := os.Lstat(devicePath); err != nil { + glog.Warningf("Stat device %s for imgpath %s failed %v", devicePath, imgPath, err) + continue + } + return devicePath, true + } + return "", false +} + +// Stat a path, if it doesn't exist, retry maxRetries times. +func waitForPath(pool, image string, maxRetries int, useNbdDriver bool) (string, bool) { + for i := 0; i < maxRetries; i++ { + if i != 0 { + time.Sleep(time.Second) + } + if useNbdDriver { + if devicePath, found := getNbdDevFromImageAndPool(pool, image); found { + return devicePath, true + } + } else { + if devicePath, found := getRbdDevFromImageAndPool(pool, image); found { + return devicePath, true + } + } + } + return "", false +} + +// Check if rbd-nbd tools are installed. +func checkRbdNbdTools() bool { + _, err := execCommand("modprobe", []string{"nbd"}) + if err != nil { + glog.V(3).Infof("rbd-nbd: nbd modprobe failed with error %v", err) + return false + } + if _, err := execCommand("rbd-nbd", []string{"--version"}); err != nil { + glog.V(3).Infof("rbd-nbd: running rbd-nbd -h failed with error %v", err) + return false + } + glog.V(3).Infof("rbd-nbd tools were found.") + return true +} + +func attachRBDImage(volOptions *rbdVolume, userId string, credentials map[string]string) (string, error) { + var err error + var output []byte + + image := volOptions.VolName + imagePath := fmt.Sprintf("%s/%s", volOptions.Pool, image) + + cmdName := "rbd" + moduleName := "rbd" + if useNBD { + cmdName = "rbd-nbd" + moduleName = "nbd" + } + devicePath, found := waitForPath(volOptions.Pool, image, 1, useNBD) + if !found { + attachdetachMutex.LockKey(string(volOptions.Pool + image)) + defer attachdetachMutex.UnlockKey(string(imagePath)) + + _, err = execCommand("modprobe", []string{moduleName}) + if err != nil { + glog.Warningf("rbd: failed to load rbd kernel module:%v", err) + } + + backoff := wait.Backoff{ + Duration: rbdImageWatcherInitDelay, + Factor: rbdImageWatcherFactor, + Steps: rbdImageWatcherSteps, + } + err := wait.ExponentialBackoff(backoff, func() (bool, error) { + used, rbdOutput, err := rbdStatus(volOptions, userId, credentials) + if err != nil { + return false, fmt.Errorf("fail to check rbd image status with: (%v), rbd output: (%s)", err, rbdOutput) + } + return !used, nil + }) + // return error if rbd image has not become available for the specified timeout + if err == wait.ErrWaitTimeout { + return "", fmt.Errorf("rbd image %s is still being used", imagePath) + } + // return error if any other errors were encountered during wating for the image to becme avialble + if err != nil { + return "", err + } + + glog.V(3).Infof("rbd: map mon %s", volOptions.Monitors) + key, err := getRBDKey(userId, credentials) + if err != nil { + return "", err + } + output, err = execCommand(cmdName, []string{ + "map", imagePath, "--id", userId, "-m", volOptions.Monitors, "--key=" + key}) + if err != nil { + glog.Warningf("rbd: map error %v, rbd output: %s", err, string(output)) + return "", fmt.Errorf("rbd: map failed %v, rbd output: %s", err, string(output)) + } + devicePath, found = waitForPath(volOptions.Pool, image, 10, useNBD) + if !found { + return "", fmt.Errorf("Could not map image %s, Timeout after 10s", imagePath) + } + } + + return devicePath, nil +} + +func detachRBDDevice(devicePath string) error { + var err error + var output []byte + + glog.V(3).Infof("rbd: unmap device %s", devicePath) + + cmdName := "rbd" + if strings.HasPrefix(devicePath, "/dev/nbd") { + cmdName = "rbd-nbd" + } + + output, err = execCommand(cmdName, []string{"unmap", devicePath}) + if err != nil { + return fmt.Errorf("rbd: unmap failed %v, rbd output: %s", err, string(output)) + } + + return nil +} diff --git a/pkg/rbd/rbd_util.go b/pkg/rbd/rbd_util.go index e2682b032..93ff3c431 100644 --- a/pkg/rbd/rbd_util.go +++ b/pkg/rbd/rbd_util.go @@ -19,7 +19,6 @@ package rbd import ( "encoding/json" "fmt" - "io/ioutil" "os" "os/exec" "path" @@ -28,7 +27,6 @@ import ( "github.com/golang/glog" "k8s.io/apimachinery/pkg/util/sets" - "k8s.io/apimachinery/pkg/util/wait" "k8s.io/kubernetes/pkg/util/keymutex" ) @@ -264,129 +262,6 @@ func hasSnapshotFeature(imageFeatures string) bool { return false } -func attachRBDImage(volOptions *rbdVolume, userId string, credentials map[string]string) (string, error) { - var err error - var output []byte - - image := volOptions.VolName - devicePath, found := waitForPath(volOptions.Pool, image, 1) - if !found { - attachdetachMutex.LockKey(string(volOptions.Pool + image)) - defer attachdetachMutex.UnlockKey(string(volOptions.Pool + image)) - - _, err = execCommand("modprobe", []string{"rbd"}) - if err != nil { - glog.Warningf("rbd: failed to load rbd kernel module:%v", err) - } - - backoff := wait.Backoff{ - Duration: rbdImageWatcherInitDelay, - Factor: rbdImageWatcherFactor, - Steps: rbdImageWatcherSteps, - } - err := wait.ExponentialBackoff(backoff, func() (bool, error) { - used, rbdOutput, err := rbdStatus(volOptions, userId, credentials) - if err != nil { - return false, fmt.Errorf("fail to check rbd image status with: (%v), rbd output: (%s)", err, rbdOutput) - } - return !used, nil - }) - // return error if rbd image has not become available for the specified timeout - if err == wait.ErrWaitTimeout { - return "", fmt.Errorf("rbd image %s/%s is still being used", volOptions.Pool, image) - } - // return error if any other errors were encountered during wating for the image to becme avialble - if err != nil { - return "", err - } - - glog.V(1).Infof("rbd: map mon %s", volOptions.Monitors) - key, err := getRBDKey(userId, credentials) - if err != nil { - return "", err - } - - output, err = execCommand("rbd", []string{ - "map", image, "--pool", volOptions.Pool, "--id", userId, "-m", volOptions.Monitors, "--key=" + key}) - if err != nil { - glog.V(1).Infof("rbd: map error %v, rbd output: %s", err, string(output)) - return "", fmt.Errorf("rbd: map failed %v, rbd output: %s", err, string(output)) - } - devicePath, found = waitForPath(volOptions.Pool, image, 10) - if !found { - return "", fmt.Errorf("Could not map image %s/%s, Timeout after 10s", volOptions.Pool, image) - } - } - - return devicePath, nil -} - -func detachRBDDevice(devicePath string) error { - var err error - var output []byte - - glog.V(3).Infof("rbd: unmap device %s", devicePath) - - output, err = execCommand("rbd", []string{"unmap", devicePath}) - if err != nil { - return fmt.Errorf("rbd: unmap failed %v, rbd output: %s", err, string(output)) - } - - return nil -} - -func getDevFromImageAndPool(pool, image string) (string, bool) { - // /sys/bus/rbd/devices/X/name and /sys/bus/rbd/devices/X/pool - sys_path := "/sys/bus/rbd/devices" - if dirs, err := ioutil.ReadDir(sys_path); err == nil { - for _, f := range dirs { - name := f.Name() - // first match pool, then match name - poolFile := path.Join(sys_path, name, "pool") - poolBytes, err := ioutil.ReadFile(poolFile) - if err != nil { - glog.V(4).Infof("Error reading %s: %v", poolFile, err) - continue - } - if strings.TrimSpace(string(poolBytes)) != pool { - glog.V(4).Infof("Device %s is not %q: %q", name, pool, string(poolBytes)) - continue - } - imgFile := path.Join(sys_path, name, "name") - imgBytes, err := ioutil.ReadFile(imgFile) - if err != nil { - glog.V(4).Infof("Error reading %s: %v", imgFile, err) - continue - } - if strings.TrimSpace(string(imgBytes)) != image { - glog.V(4).Infof("Device %s is not %q: %q", name, image, string(imgBytes)) - continue - } - // found a match, check if device exists - devicePath := "/dev/rbd" + name - if _, err := os.Lstat(devicePath); err == nil { - return devicePath, true - } - } - } - return "", false -} - -// stat a path, if not exists, retry maxRetries times -func waitForPath(pool, image string, maxRetries int) (string, bool) { - for i := 0; i < maxRetries; i++ { - devicePath, found := getDevFromImageAndPool(pool, image) - if found { - return devicePath, true - } - if i == maxRetries-1 { - break - } - time.Sleep(time.Second) - } - return "", false -} - func persistVolInfo(image string, persistentStoragePath string, volInfo *rbdVolume) error { file := path.Join(persistentStoragePath, image+".json") fp, err := os.Create(file) From 6f3625b11e030304959cd3144f761ecb45e011b7 Mon Sep 17 00:00:00 2001 From: Huamin Chen Date: Tue, 18 Sep 2018 13:10:28 +0000 Subject: [PATCH 2/4] review feedback Signed-off-by: Huamin Chen --- pkg/rbd/rbd_attach.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/rbd/rbd_attach.go b/pkg/rbd/rbd_attach.go index 707d23e81..9418af2eb 100644 --- a/pkg/rbd/rbd_attach.go +++ b/pkg/rbd/rbd_attach.go @@ -212,7 +212,7 @@ func checkRbdNbdTools() bool { return false } if _, err := execCommand("rbd-nbd", []string{"--version"}); err != nil { - glog.V(3).Infof("rbd-nbd: running rbd-nbd -h failed with error %v", err) + glog.V(3).Infof("rbd-nbd: running rbd-nbd --version failed with error %v", err) return false } glog.V(3).Infof("rbd-nbd tools were found.") @@ -234,7 +234,7 @@ func attachRBDImage(volOptions *rbdVolume, userId string, credentials map[string } devicePath, found := waitForPath(volOptions.Pool, image, 1, useNBD) if !found { - attachdetachMutex.LockKey(string(volOptions.Pool + image)) + attachdetachMutex.LockKey(string(imagePath)) defer attachdetachMutex.UnlockKey(string(imagePath)) _, err = execCommand("modprobe", []string{moduleName}) From 30a5d9a6e7a28cdfadb51d1926d48eb3991fc176 Mon Sep 17 00:00:00 2001 From: Huamin Chen Date: Tue, 18 Sep 2018 14:09:12 +0000 Subject: [PATCH 3/4] add rbd-nbd mounter in storage class Signed-off-by: Huamin Chen --- examples/rbd/storageclass.yaml | 2 ++ pkg/rbd/rbd_attach.go | 11 +++++++---- pkg/rbd/rbd_util.go | 6 ++++++ 3 files changed, 15 insertions(+), 4 deletions(-) diff --git a/examples/rbd/storageclass.yaml b/examples/rbd/storageclass.yaml index e1a8a06ba..7535e2e01 100644 --- a/examples/rbd/storageclass.yaml +++ b/examples/rbd/storageclass.yaml @@ -25,4 +25,6 @@ parameters: # Ceph users for operating RBD adminid: admin userid: kubernetes + # uncomment the following to use rbd-nbd as mounter on supported nodes + #mounter: rbd-nbd reclaimPolicy: Delete diff --git a/pkg/rbd/rbd_attach.go b/pkg/rbd/rbd_attach.go index 9418af2eb..6385db81a 100644 --- a/pkg/rbd/rbd_attach.go +++ b/pkg/rbd/rbd_attach.go @@ -35,7 +35,7 @@ const ( var ( hostRootFS = "/" - useNBD = false + hasNBD = false ) func init() { @@ -43,7 +43,7 @@ func init() { if len(host) > 0 { hostRootFS = host } - useNBD = checkRbdNbdTools() + hasNBD = checkRbdNbdTools() } func getDevFromImageAndPool(pool, image string) (string, bool) { @@ -226,12 +226,15 @@ func attachRBDImage(volOptions *rbdVolume, userId string, credentials map[string image := volOptions.VolName imagePath := fmt.Sprintf("%s/%s", volOptions.Pool, image) + useNBD := false cmdName := "rbd" moduleName := "rbd" - if useNBD { + if volOptions.Mounter == "rbd-nbd" && hasNBD { + useNBD = true cmdName = "rbd-nbd" moduleName = "nbd" } + devicePath, found := waitForPath(volOptions.Pool, image, 1, useNBD) if !found { attachdetachMutex.LockKey(string(imagePath)) @@ -258,7 +261,7 @@ func attachRBDImage(volOptions *rbdVolume, userId string, credentials map[string if err == wait.ErrWaitTimeout { return "", fmt.Errorf("rbd image %s is still being used", imagePath) } - // return error if any other errors were encountered during wating for the image to becme avialble + // return error if any other errors were encountered during wating for the image to become available if err != nil { return "", err } diff --git a/pkg/rbd/rbd_util.go b/pkg/rbd/rbd_util.go index 93ff3c431..eaea663d7 100644 --- a/pkg/rbd/rbd_util.go +++ b/pkg/rbd/rbd_util.go @@ -42,6 +42,7 @@ const ( rbdImageWatcherInitDelay = 1 * time.Second rbdImageWatcherFactor = 1.4 rbdImageWatcherSteps = 10 + rbdDefaultMounter = "rbd" ) type rbdVolume struct { @@ -54,6 +55,7 @@ type rbdVolume struct { VolSize int64 `json:"volSize"` AdminId string `json:"adminId"` UserId string `json:"userId"` + Mounter string `json:"mounter"` } type rbdSnapshot struct { @@ -226,6 +228,10 @@ func getRBDVolumeOptions(volOptions map[string]string) (*rbdVolume, error) { if !ok { rbdVol.UserId = rbdDefaultUserId } + rbdVol.Mounter, ok = volOptions["mounter"] + if !ok { + rbdVol.Mounter = rbdDefaultMounter + } return rbdVol, nil } From c311bfcad338887cc1996869a9e3292c8a28c77e Mon Sep 17 00:00:00 2001 From: Huamin Chen Date: Tue, 18 Sep 2018 14:28:55 +0000 Subject: [PATCH 4/4] update rbd doc --- docs/deploy-rbd.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docs/deploy-rbd.md b/docs/deploy-rbd.md index 94da193af..ca9468707 100644 --- a/docs/deploy-rbd.md +++ b/docs/deploy-rbd.md @@ -26,6 +26,9 @@ Option | Default value | Description `--drivername` | `csi-cephfsplugin` | name of the driver (Kubernetes: `provisioner` field in StorageClass must correspond to this value) `--nodeid` | _empty_ | This node's ID +**Available environmental variables:** +`HOST_ROOTFS`: rbdplugin searches `/proc` directory under the directory set by `HOST_ROOTFS`. + **Available volume parameters:** Parameter | Required | Description @@ -36,6 +39,7 @@ Parameter | Required | Description `imageFeatures` | no | RBD image features. Available for `imageFormat=2`. CSI RBD currently supports only `layering` feature. See [man pages](http://docs.ceph.com/docs/mimic/man/8/rbd/#cmdoption-rbd-image-feature) `csiProvisionerSecretName`, `csiNodePublishSecretName` | for Kubernetes | name of the Kubernetes Secret object containing Ceph client credentials. Both parameters should have the same value `csiProvisionerSecretNamespace`, `csiNodePublishSecretNamespace` | for Kubernetes | namespaces of the above Secret objects +`mounter`| no | if set to `rbd-nbd`, use `rbd-nbd` on nodes that have `rbd-nbd` and `nbd` kernel modules to map rbd images **Required secrets:** Admin credentials are required for provisioning new RBD images