mirror of
https://github.com/ceph/ceph-csi.git
synced 2024-12-18 02:50:30 +00:00
cleanup: move mount functions to new pkg
moved fuse and kernel mount functions to a new package. Signed-off-by: Madhu Rajanna <madhupr007@gmail.com>
This commit is contained in:
parent
b1ef842640
commit
34a21cdbe3
@ -40,8 +40,6 @@ import (
|
|||||||
var clusterAdditionalInfo = make(map[string]*localClusterState)
|
var clusterAdditionalInfo = make(map[string]*localClusterState)
|
||||||
|
|
||||||
const (
|
const (
|
||||||
cephEntityClientPrefix = "client."
|
|
||||||
|
|
||||||
// modeAllRWX can be used for setting permissions to Read-Write-eXecute
|
// modeAllRWX can be used for setting permissions to Read-Write-eXecute
|
||||||
// for User, Group and Other.
|
// for User, Group and Other.
|
||||||
modeAllRWX = 0o777
|
modeAllRWX = 0o777
|
||||||
|
@ -1,310 +0,0 @@
|
|||||||
/*
|
|
||||||
Copyright 2018 The Ceph-CSI Authors.
|
|
||||||
|
|
||||||
Licensed under the Apache License, Version 2.0 (the "License");
|
|
||||||
you may not use this file except in compliance with the License.
|
|
||||||
You may obtain a copy of the License at
|
|
||||||
|
|
||||||
http://www.apache.org/licenses/LICENSE-2.0
|
|
||||||
|
|
||||||
Unless required by applicable law or agreed to in writing, software
|
|
||||||
distributed under the License is distributed on an "AS IS" BASIS,
|
|
||||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
||||||
See the License for the specific language governing permissions and
|
|
||||||
limitations under the License.
|
|
||||||
*/
|
|
||||||
|
|
||||||
package core
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"errors"
|
|
||||||
"fmt"
|
|
||||||
"os"
|
|
||||||
"os/exec"
|
|
||||||
"regexp"
|
|
||||||
"strconv"
|
|
||||||
"strings"
|
|
||||||
"sync"
|
|
||||||
|
|
||||||
"github.com/ceph/ceph-csi/internal/util"
|
|
||||||
"github.com/ceph/ceph-csi/internal/util/log"
|
|
||||||
)
|
|
||||||
|
|
||||||
const (
|
|
||||||
volumeMounterFuse = "fuse"
|
|
||||||
volumeMounterKernel = "kernel"
|
|
||||||
netDev = "_netdev"
|
|
||||||
)
|
|
||||||
|
|
||||||
var (
|
|
||||||
availableMounters []string
|
|
||||||
|
|
||||||
// maps a mountpoint to PID of its FUSE daemon.
|
|
||||||
fusePidMap = make(map[string]int)
|
|
||||||
fusePidMapMtx sync.Mutex
|
|
||||||
|
|
||||||
fusePidRx = regexp.MustCompile(`(?m)^ceph-fuse\[(.+)\]: starting fuse$`)
|
|
||||||
|
|
||||||
// nolint:gomnd // numbers specify Kernel versions.
|
|
||||||
quotaSupport = []util.KernelVersion{
|
|
||||||
{
|
|
||||||
Version: 4,
|
|
||||||
PatchLevel: 17,
|
|
||||||
SubLevel: 0,
|
|
||||||
ExtraVersion: 0, Distribution: "",
|
|
||||||
Backport: false,
|
|
||||||
}, // standard 4.17+ versions
|
|
||||||
{
|
|
||||||
Version: 3,
|
|
||||||
PatchLevel: 10,
|
|
||||||
SubLevel: 0,
|
|
||||||
ExtraVersion: 1062,
|
|
||||||
Distribution: ".el7",
|
|
||||||
Backport: true,
|
|
||||||
}, // RHEL-7.7
|
|
||||||
}
|
|
||||||
)
|
|
||||||
|
|
||||||
func execCommandErr(ctx context.Context, program string, args ...string) error {
|
|
||||||
_, _, err := util.ExecCommand(ctx, program, args...)
|
|
||||||
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
// Load available ceph mounters installed on system into availableMounters
|
|
||||||
// Called from driver.go's Run().
|
|
||||||
func LoadAvailableMounters(conf *util.Config) error {
|
|
||||||
// #nosec
|
|
||||||
fuseMounterProbe := exec.Command("ceph-fuse", "--version")
|
|
||||||
// #nosec
|
|
||||||
kernelMounterProbe := exec.Command("mount.ceph")
|
|
||||||
|
|
||||||
err := kernelMounterProbe.Run()
|
|
||||||
if err != nil {
|
|
||||||
log.ErrorLogMsg("failed to run mount.ceph %v", err)
|
|
||||||
} else {
|
|
||||||
// fetch the current running kernel info
|
|
||||||
release, kvErr := util.GetKernelVersion()
|
|
||||||
if kvErr != nil {
|
|
||||||
return kvErr
|
|
||||||
}
|
|
||||||
|
|
||||||
if conf.ForceKernelCephFS || util.CheckKernelSupport(release, quotaSupport) {
|
|
||||||
log.DefaultLog("loaded mounter: %s", volumeMounterKernel)
|
|
||||||
availableMounters = append(availableMounters, volumeMounterKernel)
|
|
||||||
} else {
|
|
||||||
log.DefaultLog("kernel version < 4.17 might not support quota feature, hence not loading kernel client")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
err = fuseMounterProbe.Run()
|
|
||||||
if err != nil {
|
|
||||||
log.ErrorLogMsg("failed to run ceph-fuse %v", err)
|
|
||||||
} else {
|
|
||||||
log.DefaultLog("loaded mounter: %s", volumeMounterFuse)
|
|
||||||
availableMounters = append(availableMounters, volumeMounterFuse)
|
|
||||||
}
|
|
||||||
|
|
||||||
if len(availableMounters) == 0 {
|
|
||||||
return errors.New("no ceph mounters found on system")
|
|
||||||
}
|
|
||||||
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
type VolumeMounter interface {
|
|
||||||
Mount(ctx context.Context, mountPoint string, cr *util.Credentials, volOptions *VolumeOptions) error
|
|
||||||
Name() string
|
|
||||||
}
|
|
||||||
|
|
||||||
func NewMounter(volOptions *VolumeOptions) (VolumeMounter, error) {
|
|
||||||
// Get the mounter from the configuration
|
|
||||||
|
|
||||||
wantMounter := volOptions.Mounter
|
|
||||||
|
|
||||||
// Verify that it's available
|
|
||||||
|
|
||||||
var chosenMounter string
|
|
||||||
|
|
||||||
for _, availMounter := range availableMounters {
|
|
||||||
if availMounter == wantMounter {
|
|
||||||
chosenMounter = wantMounter
|
|
||||||
|
|
||||||
break
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if chosenMounter == "" {
|
|
||||||
// Otherwise pick whatever is left
|
|
||||||
chosenMounter = availableMounters[0]
|
|
||||||
log.DebugLogMsg("requested mounter: %s, chosen mounter: %s", wantMounter, chosenMounter)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Create the mounter
|
|
||||||
|
|
||||||
switch chosenMounter {
|
|
||||||
case volumeMounterFuse:
|
|
||||||
return &FuseMounter{}, nil
|
|
||||||
case volumeMounterKernel:
|
|
||||||
return &KernelMounter{}, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
return nil, fmt.Errorf("unknown mounter '%s'", chosenMounter)
|
|
||||||
}
|
|
||||||
|
|
||||||
type FuseMounter struct{}
|
|
||||||
|
|
||||||
func mountFuse(ctx context.Context, mountPoint string, cr *util.Credentials, volOptions *VolumeOptions) error {
|
|
||||||
args := []string{
|
|
||||||
mountPoint,
|
|
||||||
"-m", volOptions.Monitors,
|
|
||||||
"-c", util.CephConfigPath,
|
|
||||||
"-n", cephEntityClientPrefix + cr.ID, "--keyfile=" + cr.KeyFile,
|
|
||||||
"-r", volOptions.RootPath,
|
|
||||||
}
|
|
||||||
|
|
||||||
fmo := "nonempty"
|
|
||||||
if volOptions.FuseMountOptions != "" {
|
|
||||||
fmo += "," + strings.TrimSpace(volOptions.FuseMountOptions)
|
|
||||||
}
|
|
||||||
args = append(args, "-o", fmo)
|
|
||||||
|
|
||||||
if volOptions.FsName != "" {
|
|
||||||
args = append(args, "--client_mds_namespace="+volOptions.FsName)
|
|
||||||
}
|
|
||||||
|
|
||||||
_, stderr, err := util.ExecCommand(ctx, "ceph-fuse", args[:]...)
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("%w stderr: %s", err, stderr)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Parse the output:
|
|
||||||
// We need "starting fuse" meaning the mount is ok
|
|
||||||
// and PID of the ceph-fuse daemon for unmount
|
|
||||||
|
|
||||||
match := fusePidRx.FindSubmatch([]byte(stderr))
|
|
||||||
// validMatchLength is set to 2 as match is expected
|
|
||||||
// to have 2 items, starting fuse and PID of the fuse daemon
|
|
||||||
const validMatchLength = 2
|
|
||||||
if len(match) != validMatchLength {
|
|
||||||
return fmt.Errorf("ceph-fuse failed: %s", stderr)
|
|
||||||
}
|
|
||||||
|
|
||||||
pid, err := strconv.Atoi(string(match[1]))
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("failed to parse FUSE daemon PID: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
fusePidMapMtx.Lock()
|
|
||||||
fusePidMap[mountPoint] = pid
|
|
||||||
fusePidMapMtx.Unlock()
|
|
||||||
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (m *FuseMounter) Mount(
|
|
||||||
ctx context.Context,
|
|
||||||
mountPoint string,
|
|
||||||
cr *util.Credentials,
|
|
||||||
volOptions *VolumeOptions) error {
|
|
||||||
if err := util.CreateMountPoint(mountPoint); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
return mountFuse(ctx, mountPoint, cr, volOptions)
|
|
||||||
}
|
|
||||||
|
|
||||||
func (m *FuseMounter) Name() string { return "Ceph FUSE driver" }
|
|
||||||
|
|
||||||
type KernelMounter struct{}
|
|
||||||
|
|
||||||
func mountKernel(ctx context.Context, mountPoint string, cr *util.Credentials, volOptions *VolumeOptions) error {
|
|
||||||
if err := execCommandErr(ctx, "modprobe", "ceph"); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
args := []string{
|
|
||||||
"-t", "ceph",
|
|
||||||
fmt.Sprintf("%s:%s", volOptions.Monitors, volOptions.RootPath),
|
|
||||||
mountPoint,
|
|
||||||
}
|
|
||||||
|
|
||||||
optionsStr := fmt.Sprintf("name=%s,secretfile=%s", cr.ID, cr.KeyFile)
|
|
||||||
mdsNamespace := ""
|
|
||||||
if volOptions.FsName != "" {
|
|
||||||
mdsNamespace = fmt.Sprintf("mds_namespace=%s", volOptions.FsName)
|
|
||||||
}
|
|
||||||
optionsStr = util.MountOptionsAdd(optionsStr, mdsNamespace, volOptions.KernelMountOptions, netDev)
|
|
||||||
|
|
||||||
args = append(args, "-o", optionsStr)
|
|
||||||
|
|
||||||
_, stderr, err := util.ExecCommand(ctx, "mount", args[:]...)
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("%w stderr: %s", err, stderr)
|
|
||||||
}
|
|
||||||
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
func (m *KernelMounter) Mount(
|
|
||||||
ctx context.Context,
|
|
||||||
mountPoint string,
|
|
||||||
cr *util.Credentials,
|
|
||||||
volOptions *VolumeOptions) error {
|
|
||||||
if err := util.CreateMountPoint(mountPoint); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
return mountKernel(ctx, mountPoint, cr, volOptions)
|
|
||||||
}
|
|
||||||
|
|
||||||
func (m *KernelMounter) Name() string { return "Ceph kernel client" }
|
|
||||||
|
|
||||||
func BindMount(ctx context.Context, from, to string, readOnly bool, mntOptions []string) error {
|
|
||||||
mntOptionSli := strings.Join(mntOptions, ",")
|
|
||||||
if err := execCommandErr(ctx, "mount", "-o", mntOptionSli, from, to); err != nil {
|
|
||||||
return fmt.Errorf("failed to bind-mount %s to %s: %w", from, to, err)
|
|
||||||
}
|
|
||||||
|
|
||||||
if readOnly {
|
|
||||||
mntOptionSli = util.MountOptionsAdd(mntOptionSli, "remount")
|
|
||||||
if err := execCommandErr(ctx, "mount", "-o", mntOptionSli, to); err != nil {
|
|
||||||
return fmt.Errorf("failed read-only remount of %s: %w", to, err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func UnmountVolume(ctx context.Context, mountPoint string) error {
|
|
||||||
if _, stderr, err := util.ExecCommand(ctx, "umount", mountPoint); err != nil {
|
|
||||||
err = fmt.Errorf("%w stderr: %s", err, stderr)
|
|
||||||
if strings.Contains(err.Error(), fmt.Sprintf("umount: %s: not mounted", mountPoint)) ||
|
|
||||||
strings.Contains(err.Error(), "No such file or directory") {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
fusePidMapMtx.Lock()
|
|
||||||
pid, ok := fusePidMap[mountPoint]
|
|
||||||
if ok {
|
|
||||||
delete(fusePidMap, mountPoint)
|
|
||||||
}
|
|
||||||
fusePidMapMtx.Unlock()
|
|
||||||
|
|
||||||
if ok {
|
|
||||||
p, err := os.FindProcess(pid)
|
|
||||||
if err != nil {
|
|
||||||
log.WarningLog(ctx, "failed to find process %d: %v", pid, err)
|
|
||||||
} else {
|
|
||||||
if _, err = p.Wait(); err != nil {
|
|
||||||
log.WarningLog(ctx, "%d is not a child process: %v", pid, err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return nil
|
|
||||||
}
|
|
@ -123,8 +123,8 @@ func extractOption(dest *string, optionLabel string, options map[string]string)
|
|||||||
|
|
||||||
func validateMounter(m string) error {
|
func validateMounter(m string) error {
|
||||||
switch m {
|
switch m {
|
||||||
case volumeMounterFuse:
|
case "fuse":
|
||||||
case volumeMounterKernel:
|
case "kernel":
|
||||||
default:
|
default:
|
||||||
return fmt.Errorf("unknown mounter '%s'. Valid options are 'fuse' and 'kernel'", m)
|
return fmt.Errorf("unknown mounter '%s'. Valid options are 'fuse' and 'kernel'", m)
|
||||||
}
|
}
|
||||||
|
@ -18,6 +18,7 @@ package cephfs
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"github.com/ceph/ceph-csi/internal/cephfs/core"
|
"github.com/ceph/ceph-csi/internal/cephfs/core"
|
||||||
|
"github.com/ceph/ceph-csi/internal/cephfs/mounter"
|
||||||
fsutil "github.com/ceph/ceph-csi/internal/cephfs/util"
|
fsutil "github.com/ceph/ceph-csi/internal/cephfs/util"
|
||||||
csicommon "github.com/ceph/ceph-csi/internal/csi-common"
|
csicommon "github.com/ceph/ceph-csi/internal/csi-common"
|
||||||
"github.com/ceph/ceph-csi/internal/journal"
|
"github.com/ceph/ceph-csi/internal/journal"
|
||||||
@ -36,11 +37,9 @@ type Driver struct {
|
|||||||
cs *ControllerServer
|
cs *ControllerServer
|
||||||
}
|
}
|
||||||
|
|
||||||
var (
|
// CSIInstanceID is the instance ID that is unique to an instance of CSI, used when sharing
|
||||||
// CSIInstanceID is the instance ID that is unique to an instance of CSI, used when sharing
|
// ceph clusters across CSI instances, to differentiate omap names per CSI instance.
|
||||||
// ceph clusters across CSI instances, to differentiate omap names per CSI instance.
|
var CSIInstanceID = "default"
|
||||||
CSIInstanceID = "default"
|
|
||||||
)
|
|
||||||
|
|
||||||
// NewDriver returns new ceph driver.
|
// NewDriver returns new ceph driver.
|
||||||
func NewDriver() *Driver {
|
func NewDriver() *Driver {
|
||||||
@ -79,7 +78,7 @@ func (fs *Driver) Run(conf *util.Config) {
|
|||||||
var topology map[string]string
|
var topology map[string]string
|
||||||
|
|
||||||
// Configuration
|
// Configuration
|
||||||
if err = core.LoadAvailableMounters(conf); err != nil {
|
if err = mounter.LoadAvailableMounters(conf); err != nil {
|
||||||
log.FatalLogMsg("cephfs: failed to load ceph mounters: %v", err)
|
log.FatalLogMsg("cephfs: failed to load ceph mounters: %v", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
142
internal/cephfs/mounter/fuse.go
Normal file
142
internal/cephfs/mounter/fuse.go
Normal file
@ -0,0 +1,142 @@
|
|||||||
|
/*
|
||||||
|
Copyright 2021 The Ceph-CSI Authors.
|
||||||
|
|
||||||
|
Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
you may not use this file except in compliance with the License.
|
||||||
|
You may obtain a copy of the License at
|
||||||
|
|
||||||
|
http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
|
||||||
|
Unless required by applicable law or agreed to in writing, software
|
||||||
|
distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
See the License for the specific language governing permissions and
|
||||||
|
limitations under the License.
|
||||||
|
*/
|
||||||
|
|
||||||
|
package mounter
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"fmt"
|
||||||
|
"os"
|
||||||
|
"regexp"
|
||||||
|
"strconv"
|
||||||
|
"strings"
|
||||||
|
"sync"
|
||||||
|
|
||||||
|
"github.com/ceph/ceph-csi/internal/cephfs/core"
|
||||||
|
"github.com/ceph/ceph-csi/internal/util"
|
||||||
|
"github.com/ceph/ceph-csi/internal/util/log"
|
||||||
|
)
|
||||||
|
|
||||||
|
const (
|
||||||
|
volumeMounterFuse = "fuse"
|
||||||
|
|
||||||
|
cephEntityClientPrefix = "client."
|
||||||
|
)
|
||||||
|
|
||||||
|
var (
|
||||||
|
|
||||||
|
// maps a mountpoint to PID of its FUSE daemon.
|
||||||
|
fusePidMap = make(map[string]int)
|
||||||
|
fusePidMapMtx sync.Mutex
|
||||||
|
|
||||||
|
fusePidRx = regexp.MustCompile(`(?m)^ceph-fuse\[(.+)\]: starting fuse$`)
|
||||||
|
)
|
||||||
|
|
||||||
|
type FuseMounter struct{}
|
||||||
|
|
||||||
|
func mountFuse(ctx context.Context, mountPoint string, cr *util.Credentials, volOptions *core.VolumeOptions) error {
|
||||||
|
args := []string{
|
||||||
|
mountPoint,
|
||||||
|
"-m", volOptions.Monitors,
|
||||||
|
"-c", util.CephConfigPath,
|
||||||
|
"-n", cephEntityClientPrefix + cr.ID, "--keyfile=" + cr.KeyFile,
|
||||||
|
"-r", volOptions.RootPath,
|
||||||
|
}
|
||||||
|
|
||||||
|
fmo := "nonempty"
|
||||||
|
if volOptions.FuseMountOptions != "" {
|
||||||
|
fmo += "," + strings.TrimSpace(volOptions.FuseMountOptions)
|
||||||
|
}
|
||||||
|
args = append(args, "-o", fmo)
|
||||||
|
|
||||||
|
if volOptions.FsName != "" {
|
||||||
|
args = append(args, "--client_mds_namespace="+volOptions.FsName)
|
||||||
|
}
|
||||||
|
|
||||||
|
_, stderr, err := util.ExecCommand(ctx, "ceph-fuse", args[:]...)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("%w stderr: %s", err, stderr)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Parse the output:
|
||||||
|
// We need "starting fuse" meaning the mount is ok
|
||||||
|
// and PID of the ceph-fuse daemon for unmount
|
||||||
|
|
||||||
|
match := fusePidRx.FindSubmatch([]byte(stderr))
|
||||||
|
// validMatchLength is set to 2 as match is expected
|
||||||
|
// to have 2 items, starting fuse and PID of the fuse daemon
|
||||||
|
const validMatchLength = 2
|
||||||
|
if len(match) != validMatchLength {
|
||||||
|
return fmt.Errorf("ceph-fuse failed: %s", stderr)
|
||||||
|
}
|
||||||
|
|
||||||
|
pid, err := strconv.Atoi(string(match[1]))
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("failed to parse FUSE daemon PID: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
fusePidMapMtx.Lock()
|
||||||
|
fusePidMap[mountPoint] = pid
|
||||||
|
fusePidMapMtx.Unlock()
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (m *FuseMounter) Mount(
|
||||||
|
ctx context.Context,
|
||||||
|
mountPoint string,
|
||||||
|
cr *util.Credentials,
|
||||||
|
volOptions *core.VolumeOptions) error {
|
||||||
|
if err := util.CreateMountPoint(mountPoint); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
return mountFuse(ctx, mountPoint, cr, volOptions)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (m *FuseMounter) Name() string { return "Ceph FUSE driver" }
|
||||||
|
|
||||||
|
func UnmountVolume(ctx context.Context, mountPoint string) error {
|
||||||
|
if _, stderr, err := util.ExecCommand(ctx, "umount", mountPoint); err != nil {
|
||||||
|
err = fmt.Errorf("%w stderr: %s", err, stderr)
|
||||||
|
if strings.Contains(err.Error(), fmt.Sprintf("umount: %s: not mounted", mountPoint)) ||
|
||||||
|
strings.Contains(err.Error(), "No such file or directory") {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
fusePidMapMtx.Lock()
|
||||||
|
pid, ok := fusePidMap[mountPoint]
|
||||||
|
if ok {
|
||||||
|
delete(fusePidMap, mountPoint)
|
||||||
|
}
|
||||||
|
fusePidMapMtx.Unlock()
|
||||||
|
|
||||||
|
if ok {
|
||||||
|
p, err := os.FindProcess(pid)
|
||||||
|
if err != nil {
|
||||||
|
log.WarningLog(ctx, "failed to find process %d: %v", pid, err)
|
||||||
|
} else {
|
||||||
|
if _, err = p.Wait(); err != nil {
|
||||||
|
log.WarningLog(ctx, "%d is not a child process: %v", pid, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
74
internal/cephfs/mounter/kernel.go
Normal file
74
internal/cephfs/mounter/kernel.go
Normal file
@ -0,0 +1,74 @@
|
|||||||
|
/*
|
||||||
|
Copyright 2021 The Ceph-CSI Authors.
|
||||||
|
|
||||||
|
Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
you may not use this file except in compliance with the License.
|
||||||
|
You may obtain a copy of the License at
|
||||||
|
|
||||||
|
http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
|
||||||
|
Unless required by applicable law or agreed to in writing, software
|
||||||
|
distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
See the License for the specific language governing permissions and
|
||||||
|
limitations under the License.
|
||||||
|
*/
|
||||||
|
|
||||||
|
package mounter
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"fmt"
|
||||||
|
|
||||||
|
"github.com/ceph/ceph-csi/internal/cephfs/core"
|
||||||
|
"github.com/ceph/ceph-csi/internal/util"
|
||||||
|
)
|
||||||
|
|
||||||
|
const (
|
||||||
|
volumeMounterKernel = "kernel"
|
||||||
|
netDev = "_netdev"
|
||||||
|
)
|
||||||
|
|
||||||
|
type KernelMounter struct{}
|
||||||
|
|
||||||
|
func mountKernel(ctx context.Context, mountPoint string, cr *util.Credentials, volOptions *core.VolumeOptions) error {
|
||||||
|
if err := execCommandErr(ctx, "modprobe", "ceph"); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
args := []string{
|
||||||
|
"-t", "ceph",
|
||||||
|
fmt.Sprintf("%s:%s", volOptions.Monitors, volOptions.RootPath),
|
||||||
|
mountPoint,
|
||||||
|
}
|
||||||
|
|
||||||
|
optionsStr := fmt.Sprintf("name=%s,secretfile=%s", cr.ID, cr.KeyFile)
|
||||||
|
mdsNamespace := ""
|
||||||
|
if volOptions.FsName != "" {
|
||||||
|
mdsNamespace = fmt.Sprintf("mds_namespace=%s", volOptions.FsName)
|
||||||
|
}
|
||||||
|
optionsStr = util.MountOptionsAdd(optionsStr, mdsNamespace, volOptions.KernelMountOptions, netDev)
|
||||||
|
|
||||||
|
args = append(args, "-o", optionsStr)
|
||||||
|
|
||||||
|
_, stderr, err := util.ExecCommand(ctx, "mount", args[:]...)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("%w stderr: %s", err, stderr)
|
||||||
|
}
|
||||||
|
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
func (m *KernelMounter) Mount(
|
||||||
|
ctx context.Context,
|
||||||
|
mountPoint string,
|
||||||
|
cr *util.Credentials,
|
||||||
|
volOptions *core.VolumeOptions) error {
|
||||||
|
if err := util.CreateMountPoint(mountPoint); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
return mountKernel(ctx, mountPoint, cr, volOptions)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (m *KernelMounter) Name() string { return "Ceph kernel client" }
|
154
internal/cephfs/mounter/volumemounter.go
Normal file
154
internal/cephfs/mounter/volumemounter.go
Normal file
@ -0,0 +1,154 @@
|
|||||||
|
/*
|
||||||
|
Copyright 2018 The Ceph-CSI Authors.
|
||||||
|
|
||||||
|
Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
you may not use this file except in compliance with the License.
|
||||||
|
You may obtain a copy of the License at
|
||||||
|
|
||||||
|
http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
|
||||||
|
Unless required by applicable law or agreed to in writing, software
|
||||||
|
distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
See the License for the specific language governing permissions and
|
||||||
|
limitations under the License.
|
||||||
|
*/
|
||||||
|
|
||||||
|
package mounter
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"os/exec"
|
||||||
|
"strings"
|
||||||
|
|
||||||
|
"github.com/ceph/ceph-csi/internal/cephfs/core"
|
||||||
|
"github.com/ceph/ceph-csi/internal/util"
|
||||||
|
"github.com/ceph/ceph-csi/internal/util/log"
|
||||||
|
)
|
||||||
|
|
||||||
|
var (
|
||||||
|
availableMounters []string
|
||||||
|
|
||||||
|
// nolint:gomnd // numbers specify Kernel versions.
|
||||||
|
quotaSupport = []util.KernelVersion{
|
||||||
|
{
|
||||||
|
Version: 4,
|
||||||
|
PatchLevel: 17,
|
||||||
|
SubLevel: 0,
|
||||||
|
ExtraVersion: 0, Distribution: "",
|
||||||
|
Backport: false,
|
||||||
|
}, // standard 4.17+ versions
|
||||||
|
{
|
||||||
|
Version: 3,
|
||||||
|
PatchLevel: 10,
|
||||||
|
SubLevel: 0,
|
||||||
|
ExtraVersion: 1062,
|
||||||
|
Distribution: ".el7",
|
||||||
|
Backport: true,
|
||||||
|
}, // RHEL-7.7
|
||||||
|
}
|
||||||
|
)
|
||||||
|
|
||||||
|
func execCommandErr(ctx context.Context, program string, args ...string) error {
|
||||||
|
_, _, err := util.ExecCommand(ctx, program, args...)
|
||||||
|
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
// Load available ceph mounters installed on system into availableMounters
|
||||||
|
// Called from driver.go's Run().
|
||||||
|
func LoadAvailableMounters(conf *util.Config) error {
|
||||||
|
// #nosec
|
||||||
|
fuseMounterProbe := exec.Command("ceph-fuse", "--version")
|
||||||
|
// #nosec
|
||||||
|
kernelMounterProbe := exec.Command("mount.ceph")
|
||||||
|
|
||||||
|
err := kernelMounterProbe.Run()
|
||||||
|
if err != nil {
|
||||||
|
log.ErrorLogMsg("failed to run mount.ceph %v", err)
|
||||||
|
} else {
|
||||||
|
// fetch the current running kernel info
|
||||||
|
release, kvErr := util.GetKernelVersion()
|
||||||
|
if kvErr != nil {
|
||||||
|
return kvErr
|
||||||
|
}
|
||||||
|
|
||||||
|
if conf.ForceKernelCephFS || util.CheckKernelSupport(release, quotaSupport) {
|
||||||
|
log.DefaultLog("loaded mounter: %s", volumeMounterKernel)
|
||||||
|
availableMounters = append(availableMounters, volumeMounterKernel)
|
||||||
|
} else {
|
||||||
|
log.DefaultLog("kernel version < 4.17 might not support quota feature, hence not loading kernel client")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
err = fuseMounterProbe.Run()
|
||||||
|
if err != nil {
|
||||||
|
log.ErrorLogMsg("failed to run ceph-fuse %v", err)
|
||||||
|
} else {
|
||||||
|
log.DefaultLog("loaded mounter: %s", volumeMounterFuse)
|
||||||
|
availableMounters = append(availableMounters, volumeMounterFuse)
|
||||||
|
}
|
||||||
|
|
||||||
|
if len(availableMounters) == 0 {
|
||||||
|
return errors.New("no ceph mounters found on system")
|
||||||
|
}
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
type VolumeMounter interface {
|
||||||
|
Mount(ctx context.Context, mountPoint string, cr *util.Credentials, volOptions *core.VolumeOptions) error
|
||||||
|
Name() string
|
||||||
|
}
|
||||||
|
|
||||||
|
func New(volOptions *core.VolumeOptions) (VolumeMounter, error) {
|
||||||
|
// Get the mounter from the configuration
|
||||||
|
|
||||||
|
wantMounter := volOptions.Mounter
|
||||||
|
|
||||||
|
// Verify that it's available
|
||||||
|
|
||||||
|
var chosenMounter string
|
||||||
|
|
||||||
|
for _, availMounter := range availableMounters {
|
||||||
|
if availMounter == wantMounter {
|
||||||
|
chosenMounter = wantMounter
|
||||||
|
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if chosenMounter == "" {
|
||||||
|
// Otherwise pick whatever is left
|
||||||
|
chosenMounter = availableMounters[0]
|
||||||
|
log.DebugLogMsg("requested mounter: %s, chosen mounter: %s", wantMounter, chosenMounter)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Create the mounter
|
||||||
|
switch chosenMounter {
|
||||||
|
case volumeMounterFuse:
|
||||||
|
return &FuseMounter{}, nil
|
||||||
|
case volumeMounterKernel:
|
||||||
|
return &KernelMounter{}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
return nil, fmt.Errorf("unknown mounter '%s'", chosenMounter)
|
||||||
|
}
|
||||||
|
|
||||||
|
func BindMount(ctx context.Context, from, to string, readOnly bool, mntOptions []string) error {
|
||||||
|
mntOptionSli := strings.Join(mntOptions, ",")
|
||||||
|
if err := execCommandErr(ctx, "mount", "-o", mntOptionSli, from, to); err != nil {
|
||||||
|
return fmt.Errorf("failed to bind-mount %s to %s: %w", from, to, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if readOnly {
|
||||||
|
mntOptionSli = util.MountOptionsAdd(mntOptionSli, "remount")
|
||||||
|
if err := execCommandErr(ctx, "mount", "-o", mntOptionSli, to); err != nil {
|
||||||
|
return fmt.Errorf("failed read-only remount of %s: %w", to, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
@ -25,6 +25,7 @@ import (
|
|||||||
|
|
||||||
"github.com/ceph/ceph-csi/internal/cephfs/core"
|
"github.com/ceph/ceph-csi/internal/cephfs/core"
|
||||||
cerrors "github.com/ceph/ceph-csi/internal/cephfs/errors"
|
cerrors "github.com/ceph/ceph-csi/internal/cephfs/errors"
|
||||||
|
"github.com/ceph/ceph-csi/internal/cephfs/mounter"
|
||||||
fsutil "github.com/ceph/ceph-csi/internal/cephfs/util"
|
fsutil "github.com/ceph/ceph-csi/internal/cephfs/util"
|
||||||
csicommon "github.com/ceph/ceph-csi/internal/csi-common"
|
csicommon "github.com/ceph/ceph-csi/internal/csi-common"
|
||||||
"github.com/ceph/ceph-csi/internal/util"
|
"github.com/ceph/ceph-csi/internal/util"
|
||||||
@ -153,7 +154,7 @@ func (*NodeServer) mount(ctx context.Context, volOptions *core.VolumeOptions, re
|
|||||||
}
|
}
|
||||||
defer cr.DeleteCredentials()
|
defer cr.DeleteCredentials()
|
||||||
|
|
||||||
m, err := core.NewMounter(volOptions)
|
m, err := mounter.New(volOptions)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
log.ErrorLog(ctx, "failed to create mounter for volume %s: %v", volID, err)
|
log.ErrorLog(ctx, "failed to create mounter for volume %s: %v", volID, err)
|
||||||
|
|
||||||
@ -169,12 +170,12 @@ func (*NodeServer) mount(ctx context.Context, volOptions *core.VolumeOptions, re
|
|||||||
if req.VolumeCapability.AccessMode.Mode == csi.VolumeCapability_AccessMode_MULTI_NODE_READER_ONLY ||
|
if req.VolumeCapability.AccessMode.Mode == csi.VolumeCapability_AccessMode_MULTI_NODE_READER_ONLY ||
|
||||||
req.VolumeCapability.AccessMode.Mode == csi.VolumeCapability_AccessMode_SINGLE_NODE_READER_ONLY {
|
req.VolumeCapability.AccessMode.Mode == csi.VolumeCapability_AccessMode_SINGLE_NODE_READER_ONLY {
|
||||||
switch m.(type) {
|
switch m.(type) {
|
||||||
case *core.FuseMounter:
|
case *mounter.FuseMounter:
|
||||||
if !csicommon.MountOptionContains(strings.Split(volOptions.FuseMountOptions, ","), readOnly) {
|
if !csicommon.MountOptionContains(strings.Split(volOptions.FuseMountOptions, ","), readOnly) {
|
||||||
volOptions.FuseMountOptions = util.MountOptionsAdd(volOptions.FuseMountOptions, readOnly)
|
volOptions.FuseMountOptions = util.MountOptionsAdd(volOptions.FuseMountOptions, readOnly)
|
||||||
fuseMountOptions = append(fuseMountOptions, readOnly)
|
fuseMountOptions = append(fuseMountOptions, readOnly)
|
||||||
}
|
}
|
||||||
case *core.KernelMounter:
|
case *mounter.KernelMounter:
|
||||||
if !csicommon.MountOptionContains(strings.Split(volOptions.KernelMountOptions, ","), readOnly) {
|
if !csicommon.MountOptionContains(strings.Split(volOptions.KernelMountOptions, ","), readOnly) {
|
||||||
volOptions.KernelMountOptions = util.MountOptionsAdd(volOptions.KernelMountOptions, readOnly)
|
volOptions.KernelMountOptions = util.MountOptionsAdd(volOptions.KernelMountOptions, readOnly)
|
||||||
kernelMountOptions = append(kernelMountOptions, readOnly)
|
kernelMountOptions = append(kernelMountOptions, readOnly)
|
||||||
@ -201,7 +202,7 @@ func (*NodeServer) mount(ctx context.Context, volOptions *core.VolumeOptions, re
|
|||||||
stagingTargetPath,
|
stagingTargetPath,
|
||||||
volID,
|
volID,
|
||||||
err)
|
err)
|
||||||
uErr := core.UnmountVolume(ctx, stagingTargetPath)
|
uErr := mounter.UnmountVolume(ctx, stagingTargetPath)
|
||||||
if uErr != nil {
|
if uErr != nil {
|
||||||
log.ErrorLog(
|
log.ErrorLog(
|
||||||
ctx,
|
ctx,
|
||||||
@ -263,7 +264,7 @@ func (ns *NodeServer) NodePublishVolume(
|
|||||||
|
|
||||||
// It's not, mount now
|
// It's not, mount now
|
||||||
|
|
||||||
if err = core.BindMount(
|
if err = mounter.BindMount(
|
||||||
ctx,
|
ctx,
|
||||||
req.GetStagingTargetPath(),
|
req.GetStagingTargetPath(),
|
||||||
req.GetTargetPath(),
|
req.GetTargetPath(),
|
||||||
@ -310,7 +311,7 @@ func (ns *NodeServer) NodeUnpublishVolume(
|
|||||||
}
|
}
|
||||||
|
|
||||||
// Unmount the bind-mount
|
// Unmount the bind-mount
|
||||||
if err = core.UnmountVolume(ctx, targetPath); err != nil {
|
if err = mounter.UnmountVolume(ctx, targetPath); err != nil {
|
||||||
return nil, status.Error(codes.Internal, err.Error())
|
return nil, status.Error(codes.Internal, err.Error())
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -358,7 +359,7 @@ func (ns *NodeServer) NodeUnstageVolume(
|
|||||||
return &csi.NodeUnstageVolumeResponse{}, nil
|
return &csi.NodeUnstageVolumeResponse{}, nil
|
||||||
}
|
}
|
||||||
// Unmount the volume
|
// Unmount the volume
|
||||||
if err = core.UnmountVolume(ctx, stagingTargetPath); err != nil {
|
if err = mounter.UnmountVolume(ctx, stagingTargetPath); err != nil {
|
||||||
return nil, status.Error(codes.Internal, err.Error())
|
return nil, status.Error(codes.Internal, err.Error())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
Loading…
Reference in New Issue
Block a user