mirror of
https://github.com/ceph/ceph-csi.git
synced 2025-06-14 10:53:34 +00:00
cleanup: move pkg/ to internal/
The internal/ directory in Go has a special meaning, and indicates that those packages are not meant for external consumption. Ceph-CSI does provide public APIs for other projects to consume. There is no plan to keep the API of the internally used packages stable. Closes: #903 Signed-off-by: Niels de Vos <ndevos@redhat.com>
This commit is contained in:
Niels de Vos
committed by
mergify[bot]
mergify[bot]
parent
d0abc3f5e6
commit
32839948ef
324
internal/rbd/rbd_attach.go
Normal file
324
internal/rbd/rbd_attach.go
Normal file
@ -0,0 +1,324 @@
|
||||
/*
|
||||
Copyright 2018 The Ceph-CSI Authors.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package rbd
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/ceph/ceph-csi/internal/util"
|
||||
|
||||
"k8s.io/apimachinery/pkg/util/wait"
|
||||
"k8s.io/klog"
|
||||
)
|
||||
|
||||
const (
|
||||
rbdTonbd = "rbd-nbd"
|
||||
moduleNbd = "nbd"
|
||||
|
||||
accessTypeKRbd = "krbd"
|
||||
accessTypeNbd = "nbd"
|
||||
|
||||
rbd = "rbd"
|
||||
|
||||
// Output strings returned during invocation of "rbd unmap --device-type... <imageSpec>" when
|
||||
// image is not found to be mapped. Used to ignore errors when attempting to unmap such images.
|
||||
// The %s format specifier should contain the <imageSpec> string
|
||||
// NOTE: When using devicePath instead of imageSpec, the error strings are different
|
||||
rbdUnmapCmdkRbdMissingMap = "rbd: %s: not a mapped image or snapshot"
|
||||
rbdUnmapCmdNbdMissingMap = "rbd-nbd: %s is not mapped"
|
||||
rbdMapConnectionTimeout = "Connection timed out"
|
||||
)
|
||||
|
||||
var hasNBD = false
|
||||
|
||||
func init() {
|
||||
hasNBD = checkRbdNbdTools()
|
||||
}
|
||||
|
||||
// rbdDeviceInfo strongly typed JSON spec for rbd device list output (of type krbd)
|
||||
type rbdDeviceInfo struct {
|
||||
ID string `json:"id"`
|
||||
Pool string `json:"pool"`
|
||||
Name string `json:"name"`
|
||||
Device string `json:"device"`
|
||||
}
|
||||
|
||||
// nbdDeviceInfo strongly typed JSON spec for rbd-nbd device list output (of type nbd)
|
||||
// NOTE: There is a bug in rbd output that returns id as number for nbd, and string for krbd, thus
|
||||
// requiring 2 different JSON structures to unmarshal the output.
|
||||
// NOTE: image key is "name" in krbd output and "image" in nbd output, which is another difference
|
||||
type nbdDeviceInfo struct {
|
||||
ID int64 `json:"id"`
|
||||
Pool string `json:"pool"`
|
||||
Name string `json:"image"`
|
||||
Device string `json:"device"`
|
||||
}
|
||||
|
||||
// rbdGetDeviceList queries rbd about mapped devices and returns a list of rbdDeviceInfo
|
||||
// It will selectively list devices mapped using krbd or nbd as specified by accessType
|
||||
func rbdGetDeviceList(accessType string) ([]rbdDeviceInfo, error) {
|
||||
// rbd device list --format json --device-type [krbd|nbd]
|
||||
var (
|
||||
rbdDeviceList []rbdDeviceInfo
|
||||
nbdDeviceList []nbdDeviceInfo
|
||||
)
|
||||
|
||||
stdout, _, err := util.ExecCommand(rbd, "device", "list", "--format="+"json", "--device-type", accessType)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("error getting device list from rbd for devices of type (%s): (%v)", accessType, err)
|
||||
}
|
||||
|
||||
if accessType == accessTypeKRbd {
|
||||
err = json.Unmarshal(stdout, &rbdDeviceList)
|
||||
} else {
|
||||
err = json.Unmarshal(stdout, &nbdDeviceList)
|
||||
}
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("error to parse JSON output of device list for devices of type (%s): (%v)", accessType, err)
|
||||
}
|
||||
|
||||
// convert output to a rbdDeviceInfo list for consumers
|
||||
if accessType == accessTypeNbd {
|
||||
for _, device := range nbdDeviceList {
|
||||
rbdDeviceList = append(
|
||||
rbdDeviceList,
|
||||
rbdDeviceInfo{
|
||||
ID: strconv.FormatInt(device.ID, 10),
|
||||
Pool: device.Pool,
|
||||
Name: device.Name,
|
||||
Device: device.Device,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
return rbdDeviceList, nil
|
||||
}
|
||||
|
||||
// findDeviceMappingImage finds a devicePath, if available, based on image spec (pool/image) on the node.
|
||||
func findDeviceMappingImage(ctx context.Context, pool, image string, useNbdDriver bool) (string, bool) {
|
||||
accessType := accessTypeKRbd
|
||||
if useNbdDriver {
|
||||
accessType = accessTypeNbd
|
||||
}
|
||||
|
||||
rbdDeviceList, err := rbdGetDeviceList(accessType)
|
||||
if err != nil {
|
||||
klog.Warningf(util.Log(ctx, "failed to determine if image (%s/%s) is mapped to a device (%v)"), pool, image, err)
|
||||
return "", false
|
||||
}
|
||||
|
||||
for _, device := range rbdDeviceList {
|
||||
if device.Name == image && device.Pool == pool {
|
||||
return device.Device, true
|
||||
}
|
||||
}
|
||||
|
||||
return "", false
|
||||
}
|
||||
|
||||
// Stat a path, if it doesn't exist, retry maxRetries times.
|
||||
func waitForPath(ctx context.Context, pool, image string, maxRetries int, useNbdDriver bool) (string, bool) {
|
||||
for i := 0; i < maxRetries; i++ {
|
||||
if i != 0 {
|
||||
time.Sleep(time.Second)
|
||||
}
|
||||
|
||||
device, found := findDeviceMappingImage(ctx, pool, image, useNbdDriver)
|
||||
if found {
|
||||
return device, found
|
||||
}
|
||||
}
|
||||
|
||||
return "", false
|
||||
}
|
||||
|
||||
// Check if rbd-nbd tools are installed.
|
||||
func checkRbdNbdTools() bool {
|
||||
// check if the module is loaded or compiled in
|
||||
_, err := os.Stat(fmt.Sprintf("/sys/module/%s", moduleNbd))
|
||||
if os.IsNotExist(err) {
|
||||
// try to load the module
|
||||
_, err = execCommand("modprobe", []string{moduleNbd})
|
||||
if err != nil {
|
||||
klog.V(3).Infof("rbd-nbd: nbd modprobe failed with error %v", err)
|
||||
return false
|
||||
}
|
||||
}
|
||||
if _, err := execCommand(rbdTonbd, []string{"--version"}); err != nil {
|
||||
klog.V(3).Infof("rbd-nbd: running rbd-nbd --version failed with error %v", err)
|
||||
return false
|
||||
}
|
||||
klog.V(3).Infof("rbd-nbd tools were found.")
|
||||
return true
|
||||
}
|
||||
|
||||
func attachRBDImage(ctx context.Context, volOptions *rbdVolume, cr *util.Credentials) (string, error) {
|
||||
var err error
|
||||
|
||||
image := volOptions.RbdImageName
|
||||
useNBD := false
|
||||
if volOptions.Mounter == rbdTonbd && hasNBD {
|
||||
useNBD = true
|
||||
}
|
||||
|
||||
devicePath, found := waitForPath(ctx, volOptions.Pool, image, 1, useNBD)
|
||||
if !found {
|
||||
backoff := wait.Backoff{
|
||||
Duration: rbdImageWatcherInitDelay,
|
||||
Factor: rbdImageWatcherFactor,
|
||||
Steps: rbdImageWatcherSteps,
|
||||
}
|
||||
|
||||
err = waitForrbdImage(ctx, backoff, volOptions, cr)
|
||||
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
devicePath, err = createPath(ctx, volOptions, cr)
|
||||
}
|
||||
|
||||
return devicePath, err
|
||||
}
|
||||
|
||||
func createPath(ctx context.Context, volOpt *rbdVolume, cr *util.Credentials) (string, error) {
|
||||
isNbd := false
|
||||
image := volOpt.RbdImageName
|
||||
imagePath := fmt.Sprintf("%s/%s", volOpt.Pool, image)
|
||||
|
||||
klog.V(5).Infof(util.Log(ctx, "rbd: map mon %s"), volOpt.Monitors)
|
||||
|
||||
// Map options
|
||||
mapOptions := []string{
|
||||
"--id", cr.ID,
|
||||
"-m", volOpt.Monitors,
|
||||
"--keyfile=" + cr.KeyFile,
|
||||
"map", imagePath,
|
||||
}
|
||||
|
||||
// Choose access protocol
|
||||
accessType := accessTypeKRbd
|
||||
if volOpt.Mounter == rbdTonbd && hasNBD {
|
||||
isNbd = true
|
||||
accessType = accessTypeNbd
|
||||
}
|
||||
|
||||
// Update options with device type selection
|
||||
mapOptions = append(mapOptions, "--device-type", accessType)
|
||||
|
||||
// Execute map
|
||||
output, err := execCommand(rbd, mapOptions)
|
||||
if err != nil {
|
||||
klog.Warningf(util.Log(ctx, "rbd: map error %v, rbd output: %s"), err, string(output))
|
||||
// unmap rbd image if connection timeout
|
||||
if strings.Contains(err.Error(), rbdMapConnectionTimeout) {
|
||||
detErr := detachRBDImageOrDeviceSpec(ctx, imagePath, true, isNbd, volOpt.Encrypted, volOpt.VolID)
|
||||
if detErr != nil {
|
||||
klog.Warningf(util.Log(ctx, "rbd: %s unmap error %v"), imagePath, detErr)
|
||||
}
|
||||
}
|
||||
return "", fmt.Errorf("rbd: map failed %v, rbd output: %s", err, string(output))
|
||||
}
|
||||
devicePath := strings.TrimSuffix(string(output), "\n")
|
||||
|
||||
return devicePath, nil
|
||||
}
|
||||
|
||||
func waitForrbdImage(ctx context.Context, backoff wait.Backoff, volOptions *rbdVolume, cr *util.Credentials) error {
|
||||
image := volOptions.RbdImageName
|
||||
imagePath := fmt.Sprintf("%s/%s", volOptions.Pool, image)
|
||||
|
||||
err := wait.ExponentialBackoff(backoff, func() (bool, error) {
|
||||
used, rbdOutput, err := rbdStatus(ctx, volOptions, cr)
|
||||
if err != nil {
|
||||
return false, fmt.Errorf("fail to check rbd image status with: (%v), rbd output: (%s)", err, rbdOutput)
|
||||
}
|
||||
if (volOptions.DisableInUseChecks) && (used) {
|
||||
klog.V(2).Info(util.Log(ctx, "valid multi-node attach requested, ignoring watcher in-use result"))
|
||||
return used, nil
|
||||
}
|
||||
return !used, nil
|
||||
})
|
||||
// return error if rbd image has not become available for the specified timeout
|
||||
if err == wait.ErrWaitTimeout {
|
||||
return fmt.Errorf("rbd image %s is still being used", imagePath)
|
||||
}
|
||||
// return error if any other errors were encountered during waiting for the image to become available
|
||||
return err
|
||||
}
|
||||
|
||||
func detachRBDDevice(ctx context.Context, devicePath, volumeID string, encrypted bool) error {
|
||||
nbdType := false
|
||||
if strings.HasPrefix(devicePath, "/dev/nbd") {
|
||||
nbdType = true
|
||||
}
|
||||
|
||||
return detachRBDImageOrDeviceSpec(ctx, devicePath, false, nbdType, encrypted, volumeID)
|
||||
}
|
||||
|
||||
// detachRBDImageOrDeviceSpec detaches an rbd imageSpec or devicePath, with additional checking
|
||||
// when imageSpec is used to decide if image is already unmapped
|
||||
func detachRBDImageOrDeviceSpec(ctx context.Context, imageOrDeviceSpec string, isImageSpec, ndbType, encrypted bool, volumeID string) error {
|
||||
var output []byte
|
||||
|
||||
if encrypted {
|
||||
mapperFile, mapperPath := util.VolumeMapper(volumeID)
|
||||
mappedDevice, mapper, err := util.DeviceEncryptionStatus(ctx, mapperPath)
|
||||
if err != nil {
|
||||
klog.Errorf(util.Log(ctx, "error determining LUKS device on %s, %s: %s"),
|
||||
mapperPath, imageOrDeviceSpec, err)
|
||||
return err
|
||||
}
|
||||
if len(mapper) > 0 {
|
||||
// mapper found, so it is open Luks device
|
||||
err = util.CloseEncryptedVolume(ctx, mapperFile)
|
||||
if err != nil {
|
||||
klog.Errorf(util.Log(ctx, "error closing LUKS device on %s, %s: %s"),
|
||||
mapperPath, imageOrDeviceSpec, err)
|
||||
return err
|
||||
}
|
||||
imageOrDeviceSpec = mappedDevice
|
||||
}
|
||||
}
|
||||
|
||||
accessType := accessTypeKRbd
|
||||
if ndbType {
|
||||
accessType = accessTypeNbd
|
||||
}
|
||||
options := []string{"unmap", "--device-type", accessType, imageOrDeviceSpec}
|
||||
|
||||
output, err := execCommand(rbd, options)
|
||||
if err != nil {
|
||||
// Messages for krbd and nbd differ, hence checking either of them for missing mapping
|
||||
// This is not applicable when a device path is passed in
|
||||
if isImageSpec &&
|
||||
(strings.Contains(string(output), fmt.Sprintf(rbdUnmapCmdkRbdMissingMap, imageOrDeviceSpec)) ||
|
||||
strings.Contains(string(output), fmt.Sprintf(rbdUnmapCmdNbdMissingMap, imageOrDeviceSpec))) {
|
||||
// Devices found not to be mapped are treated as a successful detach
|
||||
klog.V(5).Infof(util.Log(ctx, "image or device spec (%s) not mapped"), imageOrDeviceSpec)
|
||||
return nil
|
||||
}
|
||||
return fmt.Errorf("rbd: unmap for spec (%s) failed (%v): (%s)", imageOrDeviceSpec, err, string(output))
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
Reference in New Issue
Block a user