mirror of
https://github.com/ceph/ceph-csi.git
synced 2024-12-22 04:50:23 +00:00
649aeb7aaf
if the PVC access mode is ReadOnlyMany or single node readonly, mounting the rbd device path to the staging path as readonly to avoid the write operation. If the PVC acccess mode is readonly, mapping rbd images as readonly. Signed-off-by: Madhu Rajanna <madhupr007@gmail.com>
327 lines
9.9 KiB
Go
327 lines
9.9 KiB
Go
/*
|
|
Copyright 2018 The Ceph-CSI Authors.
|
|
|
|
Licensed under the Apache License, Version 2.0 (the "License");
|
|
you may not use this file except in compliance with the License.
|
|
You may obtain a copy of the License at
|
|
|
|
http://www.apache.org/licenses/LICENSE-2.0
|
|
|
|
Unless required by applicable law or agreed to in writing, software
|
|
distributed under the License is distributed on an "AS IS" BASIS,
|
|
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
See the License for the specific language governing permissions and
|
|
limitations under the License.
|
|
*/
|
|
|
|
package rbd
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"errors"
|
|
"fmt"
|
|
"os"
|
|
"strconv"
|
|
"strings"
|
|
"time"
|
|
|
|
"github.com/ceph/ceph-csi/internal/util"
|
|
|
|
"k8s.io/apimachinery/pkg/util/wait"
|
|
"k8s.io/klog"
|
|
)
|
|
|
|
const (
|
|
rbdTonbd = "rbd-nbd"
|
|
moduleNbd = "nbd"
|
|
|
|
accessTypeKRbd = "krbd"
|
|
accessTypeNbd = "nbd"
|
|
|
|
rbd = "rbd"
|
|
|
|
// Output strings returned during invocation of "rbd unmap --device-type... <imageSpec>" when
|
|
// image is not found to be mapped. Used to ignore errors when attempting to unmap such images.
|
|
// The %s format specifier should contain the <imageSpec> string
|
|
// NOTE: When using devicePath instead of imageSpec, the error strings are different
|
|
rbdUnmapCmdkRbdMissingMap = "rbd: %s: not a mapped image or snapshot"
|
|
rbdUnmapCmdNbdMissingMap = "rbd-nbd: %s is not mapped"
|
|
rbdMapConnectionTimeout = "Connection timed out"
|
|
)
|
|
|
|
var hasNBD = false
|
|
|
|
func init() {
|
|
hasNBD = checkRbdNbdTools()
|
|
}
|
|
|
|
// rbdDeviceInfo strongly typed JSON spec for rbd device list output (of type krbd)
|
|
type rbdDeviceInfo struct {
|
|
ID string `json:"id"`
|
|
Pool string `json:"pool"`
|
|
Name string `json:"name"`
|
|
Device string `json:"device"`
|
|
}
|
|
|
|
// nbdDeviceInfo strongly typed JSON spec for rbd-nbd device list output (of type nbd)
|
|
// NOTE: There is a bug in rbd output that returns id as number for nbd, and string for krbd, thus
|
|
// requiring 2 different JSON structures to unmarshal the output.
|
|
// NOTE: image key is "name" in krbd output and "image" in nbd output, which is another difference
|
|
type nbdDeviceInfo struct {
|
|
ID int64 `json:"id"`
|
|
Pool string `json:"pool"`
|
|
Name string `json:"image"`
|
|
Device string `json:"device"`
|
|
}
|
|
|
|
// rbdGetDeviceList queries rbd about mapped devices and returns a list of rbdDeviceInfo
|
|
// It will selectively list devices mapped using krbd or nbd as specified by accessType
|
|
func rbdGetDeviceList(accessType string) ([]rbdDeviceInfo, error) {
|
|
// rbd device list --format json --device-type [krbd|nbd]
|
|
var (
|
|
rbdDeviceList []rbdDeviceInfo
|
|
nbdDeviceList []nbdDeviceInfo
|
|
)
|
|
|
|
stdout, _, err := util.ExecCommand(rbd, "device", "list", "--format="+"json", "--device-type", accessType)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("error getting device list from rbd for devices of type (%s): (%v)", accessType, err)
|
|
}
|
|
|
|
if accessType == accessTypeKRbd {
|
|
err = json.Unmarshal(stdout, &rbdDeviceList)
|
|
} else {
|
|
err = json.Unmarshal(stdout, &nbdDeviceList)
|
|
}
|
|
if err != nil {
|
|
return nil, fmt.Errorf("error to parse JSON output of device list for devices of type (%s): (%v)", accessType, err)
|
|
}
|
|
|
|
// convert output to a rbdDeviceInfo list for consumers
|
|
if accessType == accessTypeNbd {
|
|
for _, device := range nbdDeviceList {
|
|
rbdDeviceList = append(
|
|
rbdDeviceList,
|
|
rbdDeviceInfo{
|
|
ID: strconv.FormatInt(device.ID, 10),
|
|
Pool: device.Pool,
|
|
Name: device.Name,
|
|
Device: device.Device,
|
|
})
|
|
}
|
|
}
|
|
|
|
return rbdDeviceList, nil
|
|
}
|
|
|
|
// findDeviceMappingImage finds a devicePath, if available, based on image spec (pool/image) on the node.
|
|
func findDeviceMappingImage(ctx context.Context, pool, image string, useNbdDriver bool) (string, bool) {
|
|
accessType := accessTypeKRbd
|
|
if useNbdDriver {
|
|
accessType = accessTypeNbd
|
|
}
|
|
|
|
rbdDeviceList, err := rbdGetDeviceList(accessType)
|
|
if err != nil {
|
|
klog.Warningf(util.Log(ctx, "failed to determine if image (%s/%s) is mapped to a device (%v)"), pool, image, err)
|
|
return "", false
|
|
}
|
|
|
|
for _, device := range rbdDeviceList {
|
|
if device.Name == image && device.Pool == pool {
|
|
return device.Device, true
|
|
}
|
|
}
|
|
|
|
return "", false
|
|
}
|
|
|
|
// Stat a path, if it doesn't exist, retry maxRetries times.
|
|
func waitForPath(ctx context.Context, pool, image string, maxRetries int, useNbdDriver bool) (string, bool) {
|
|
for i := 0; i < maxRetries; i++ {
|
|
if i != 0 {
|
|
time.Sleep(time.Second)
|
|
}
|
|
|
|
device, found := findDeviceMappingImage(ctx, pool, image, useNbdDriver)
|
|
if found {
|
|
return device, found
|
|
}
|
|
}
|
|
|
|
return "", false
|
|
}
|
|
|
|
// Check if rbd-nbd tools are installed.
|
|
func checkRbdNbdTools() bool {
|
|
// check if the module is loaded or compiled in
|
|
_, err := os.Stat(fmt.Sprintf("/sys/module/%s", moduleNbd))
|
|
if os.IsNotExist(err) {
|
|
// try to load the module
|
|
_, err = execCommand("modprobe", []string{moduleNbd})
|
|
if err != nil {
|
|
klog.V(3).Infof("rbd-nbd: nbd modprobe failed with error %v", err)
|
|
return false
|
|
}
|
|
}
|
|
if _, err := execCommand(rbdTonbd, []string{"--version"}); err != nil {
|
|
klog.V(3).Infof("rbd-nbd: running rbd-nbd --version failed with error %v", err)
|
|
return false
|
|
}
|
|
klog.V(3).Infof("rbd-nbd tools were found.")
|
|
return true
|
|
}
|
|
|
|
func attachRBDImage(ctx context.Context, volOptions *rbdVolume, cr *util.Credentials) (string, error) {
|
|
var err error
|
|
|
|
image := volOptions.RbdImageName
|
|
useNBD := false
|
|
if volOptions.Mounter == rbdTonbd && hasNBD {
|
|
useNBD = true
|
|
}
|
|
|
|
devicePath, found := waitForPath(ctx, volOptions.Pool, image, 1, useNBD)
|
|
if !found {
|
|
backoff := wait.Backoff{
|
|
Duration: rbdImageWatcherInitDelay,
|
|
Factor: rbdImageWatcherFactor,
|
|
Steps: rbdImageWatcherSteps,
|
|
}
|
|
|
|
err = waitForrbdImage(ctx, backoff, volOptions, cr)
|
|
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
devicePath, err = createPath(ctx, volOptions, cr)
|
|
}
|
|
|
|
return devicePath, err
|
|
}
|
|
|
|
func createPath(ctx context.Context, volOpt *rbdVolume, cr *util.Credentials) (string, error) {
|
|
isNbd := false
|
|
imagePath := volOpt.String()
|
|
|
|
klog.V(5).Infof(util.Log(ctx, "rbd: map mon %s"), volOpt.Monitors)
|
|
|
|
// Map options
|
|
mapOptions := []string{
|
|
"--id", cr.ID,
|
|
"-m", volOpt.Monitors,
|
|
"--keyfile=" + cr.KeyFile,
|
|
"map", imagePath,
|
|
}
|
|
|
|
// Choose access protocol
|
|
accessType := accessTypeKRbd
|
|
if volOpt.Mounter == rbdTonbd && hasNBD {
|
|
isNbd = true
|
|
accessType = accessTypeNbd
|
|
}
|
|
|
|
// Update options with device type selection
|
|
mapOptions = append(mapOptions, "--device-type", accessType)
|
|
|
|
if volOpt.readOnly {
|
|
mapOptions = append(mapOptions, "--read-only")
|
|
}
|
|
// Execute map
|
|
output, err := execCommand(rbd, mapOptions)
|
|
if err != nil {
|
|
klog.Warningf(util.Log(ctx, "rbd: map error %v, rbd output: %s"), err, string(output))
|
|
// unmap rbd image if connection timeout
|
|
if strings.Contains(err.Error(), rbdMapConnectionTimeout) {
|
|
detErr := detachRBDImageOrDeviceSpec(ctx, imagePath, true, isNbd, volOpt.Encrypted, volOpt.VolID)
|
|
if detErr != nil {
|
|
klog.Warningf(util.Log(ctx, "rbd: %s unmap error %v"), imagePath, detErr)
|
|
}
|
|
}
|
|
return "", fmt.Errorf("rbd: map failed %v, rbd output: %s", err, string(output))
|
|
}
|
|
devicePath := strings.TrimSuffix(string(output), "\n")
|
|
|
|
return devicePath, nil
|
|
}
|
|
|
|
func waitForrbdImage(ctx context.Context, backoff wait.Backoff, volOptions *rbdVolume, cr *util.Credentials) error {
|
|
imagePath := volOptions.String()
|
|
|
|
err := wait.ExponentialBackoff(backoff, func() (bool, error) {
|
|
used, rbdOutput, err := rbdStatus(ctx, volOptions, cr)
|
|
if err != nil {
|
|
return false, fmt.Errorf("fail to check rbd image status with: (%v), rbd output: (%s)", err, rbdOutput)
|
|
}
|
|
if (volOptions.DisableInUseChecks) && (used) {
|
|
klog.V(2).Info(util.Log(ctx, "valid multi-node attach requested, ignoring watcher in-use result"))
|
|
return used, nil
|
|
}
|
|
return !used, nil
|
|
})
|
|
// return error if rbd image has not become available for the specified timeout
|
|
if errors.Is(err, wait.ErrWaitTimeout) {
|
|
return fmt.Errorf("rbd image %s is still being used", imagePath)
|
|
}
|
|
// return error if any other errors were encountered during waiting for the image to become available
|
|
return err
|
|
}
|
|
|
|
func detachRBDDevice(ctx context.Context, devicePath, volumeID string, encrypted bool) error {
|
|
nbdType := false
|
|
if strings.HasPrefix(devicePath, "/dev/nbd") {
|
|
nbdType = true
|
|
}
|
|
|
|
return detachRBDImageOrDeviceSpec(ctx, devicePath, false, nbdType, encrypted, volumeID)
|
|
}
|
|
|
|
// detachRBDImageOrDeviceSpec detaches an rbd imageSpec or devicePath, with additional checking
|
|
// when imageSpec is used to decide if image is already unmapped
|
|
func detachRBDImageOrDeviceSpec(ctx context.Context, imageOrDeviceSpec string, isImageSpec, ndbType, encrypted bool, volumeID string) error {
|
|
var output []byte
|
|
|
|
if encrypted {
|
|
mapperFile, mapperPath := util.VolumeMapper(volumeID)
|
|
mappedDevice, mapper, err := util.DeviceEncryptionStatus(ctx, mapperPath)
|
|
if err != nil {
|
|
klog.Errorf(util.Log(ctx, "error determining LUKS device on %s, %s: %s"),
|
|
mapperPath, imageOrDeviceSpec, err)
|
|
return err
|
|
}
|
|
if len(mapper) > 0 {
|
|
// mapper found, so it is open Luks device
|
|
err = util.CloseEncryptedVolume(ctx, mapperFile)
|
|
if err != nil {
|
|
klog.Errorf(util.Log(ctx, "error closing LUKS device on %s, %s: %s"),
|
|
mapperPath, imageOrDeviceSpec, err)
|
|
return err
|
|
}
|
|
imageOrDeviceSpec = mappedDevice
|
|
}
|
|
}
|
|
|
|
accessType := accessTypeKRbd
|
|
if ndbType {
|
|
accessType = accessTypeNbd
|
|
}
|
|
options := []string{"unmap", "--device-type", accessType, imageOrDeviceSpec}
|
|
|
|
output, err := execCommand(rbd, options)
|
|
if err != nil {
|
|
// Messages for krbd and nbd differ, hence checking either of them for missing mapping
|
|
// This is not applicable when a device path is passed in
|
|
if isImageSpec &&
|
|
(strings.Contains(string(output), fmt.Sprintf(rbdUnmapCmdkRbdMissingMap, imageOrDeviceSpec)) ||
|
|
strings.Contains(string(output), fmt.Sprintf(rbdUnmapCmdNbdMissingMap, imageOrDeviceSpec))) {
|
|
// Devices found not to be mapped are treated as a successful detach
|
|
klog.V(5).Infof(util.Log(ctx, "image or device spec (%s) not mapped"), imageOrDeviceSpec)
|
|
return nil
|
|
}
|
|
return fmt.Errorf("rbd: unmap for spec (%s) failed (%v): (%s)", imageOrDeviceSpec, err, string(output))
|
|
}
|
|
|
|
return nil
|
|
}
|