2018-03-05 11:59:47 +00:00
|
|
|
/*
|
2019-04-03 08:46:15 +00:00
|
|
|
Copyright 2018 The Ceph-CSI Authors.
|
2018-03-05 11:59:47 +00:00
|
|
|
|
|
|
|
Licensed under the Apache License, Version 2.0 (the "License");
|
|
|
|
you may not use this file except in compliance with the License.
|
|
|
|
You may obtain a copy of the License at
|
|
|
|
|
|
|
|
http://www.apache.org/licenses/LICENSE-2.0
|
|
|
|
|
|
|
|
Unless required by applicable law or agreed to in writing, software
|
|
|
|
distributed under the License is distributed on an "AS IS" BASIS,
|
|
|
|
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
|
|
See the License for the specific language governing permissions and
|
|
|
|
limitations under the License.
|
|
|
|
*/
|
|
|
|
|
|
|
|
package cephfs
|
|
|
|
|
|
|
|
import (
|
2019-08-22 17:19:06 +00:00
|
|
|
"context"
|
2019-07-10 10:50:04 +00:00
|
|
|
"path"
|
2019-06-08 05:06:03 +00:00
|
|
|
"strconv"
|
|
|
|
"strings"
|
2019-01-28 13:59:16 +00:00
|
|
|
|
2020-04-17 09:23:49 +00:00
|
|
|
"github.com/ceph/ceph-csi/internal/util"
|
2019-06-01 21:26:42 +00:00
|
|
|
|
2020-07-09 05:14:10 +00:00
|
|
|
klog "k8s.io/klog/v2"
|
2018-03-09 16:05:19 +00:00
|
|
|
)
|
|
|
|
|
2019-06-08 05:06:03 +00:00
|
|
|
var (
|
2020-06-25 13:48:48 +00:00
|
|
|
// clusterAdditionalInfo contains information regarding if resize is
|
|
|
|
// supported in the particular cluster and subvolumegroup is
|
|
|
|
// created or not.
|
|
|
|
// Subvolumegroup creation and volume resize decisions are
|
|
|
|
// taken through this additional cluster information.
|
|
|
|
clusterAdditionalInfo = make(map[string]*localClusterState)
|
2019-06-08 05:06:03 +00:00
|
|
|
)
|
2018-03-22 13:01:10 +00:00
|
|
|
|
2020-07-10 10:44:59 +00:00
|
|
|
const (
|
|
|
|
cephEntityClientPrefix = "client."
|
|
|
|
)
|
2019-07-10 10:50:04 +00:00
|
|
|
|
2020-08-04 04:21:31 +00:00
|
|
|
// Subvolume holds subvolume information.
|
|
|
|
type Subvolume struct {
|
|
|
|
BytesQuota int `json:"bytes_quota"`
|
|
|
|
DataPool string `json:"data_pool"`
|
|
|
|
GID int `json:"gid"`
|
|
|
|
Mode int `json:"mode"`
|
|
|
|
MonAddrs []string `json:"mon_addrs"`
|
|
|
|
Path string `json:"path"`
|
|
|
|
PoolNamespace string `json:"pool_namespace"`
|
|
|
|
Type string `json:"type"`
|
|
|
|
UID int `json:"uid"`
|
|
|
|
}
|
|
|
|
|
2019-07-10 10:50:04 +00:00
|
|
|
func getVolumeRootPathCephDeprecated(volID volumeID) string {
|
|
|
|
return path.Join("/", "csi-volumes", string(volID))
|
|
|
|
}
|
|
|
|
|
2019-08-22 17:19:06 +00:00
|
|
|
func getVolumeRootPathCeph(ctx context.Context, volOptions *volumeOptions, cr *util.Credentials, volID volumeID) (string, error) {
|
2020-01-23 02:48:46 +00:00
|
|
|
stdout, stderr, err := util.ExecCommand(
|
2020-07-22 12:11:41 +00:00
|
|
|
ctx,
|
2019-06-08 05:06:03 +00:00
|
|
|
"ceph",
|
|
|
|
"fs",
|
|
|
|
"subvolume",
|
|
|
|
"getpath",
|
|
|
|
volOptions.FsName,
|
|
|
|
string(volID),
|
|
|
|
"--group_name",
|
2020-04-29 10:03:08 +00:00
|
|
|
volOptions.SubvolumeGroup,
|
2019-06-08 05:06:03 +00:00
|
|
|
"-m", volOptions.Monitors,
|
|
|
|
"-c", util.CephConfigPath,
|
|
|
|
"-n", cephEntityClientPrefix+cr.ID,
|
2019-06-25 19:29:17 +00:00
|
|
|
"--keyfile="+cr.KeyFile)
|
2018-03-26 13:00:28 +00:00
|
|
|
|
2019-06-08 05:06:03 +00:00
|
|
|
if err != nil {
|
2020-07-22 12:53:22 +00:00
|
|
|
klog.Errorf(util.Log(ctx, "failed to get the rootpath for the vol %s(%s) stdError %s"), string(volID), err, stderr)
|
2020-08-04 04:23:26 +00:00
|
|
|
if strings.Contains(stderr, ErrVolumeNotFound.Error()) {
|
2020-07-10 00:14:39 +00:00
|
|
|
return "", util.JoinErrors(ErrVolumeNotFound, err)
|
2020-01-23 02:48:46 +00:00
|
|
|
}
|
|
|
|
|
2019-06-08 05:06:03 +00:00
|
|
|
return "", err
|
|
|
|
}
|
2020-07-22 12:53:22 +00:00
|
|
|
return strings.TrimSuffix(stdout, "\n"), nil
|
2018-04-13 12:49:49 +00:00
|
|
|
}
|
|
|
|
|
2020-08-04 04:21:31 +00:00
|
|
|
func getSubVolumeInfo(ctx context.Context, volOptions *volumeOptions, cr *util.Credentials, volID volumeID) (Subvolume, error) {
|
|
|
|
info := Subvolume{}
|
|
|
|
err := execCommandJSON(
|
|
|
|
ctx,
|
|
|
|
&info,
|
|
|
|
"ceph",
|
|
|
|
"fs",
|
|
|
|
"subvolume",
|
|
|
|
"info",
|
|
|
|
volOptions.FsName,
|
|
|
|
string(volID),
|
|
|
|
"--group_name",
|
|
|
|
volOptions.SubvolumeGroup,
|
|
|
|
"-m", volOptions.Monitors,
|
|
|
|
"-c", util.CephConfigPath,
|
|
|
|
"-n", cephEntityClientPrefix+cr.ID,
|
|
|
|
"--keyfile="+cr.KeyFile)
|
|
|
|
if err != nil {
|
|
|
|
klog.Errorf(util.Log(ctx, "failed to get subvolume info for the vol %s(%s)"), string(volID), err)
|
|
|
|
if strings.HasPrefix(err.Error(), ErrVolumeNotFound.Error()) {
|
|
|
|
return info, ErrVolumeNotFound
|
|
|
|
}
|
|
|
|
// Incase the error is other than invalid command return error to the caller.
|
|
|
|
if !strings.Contains(err.Error(), ErrInvalidCommand.Error()) {
|
|
|
|
return info, ErrInvalidCommand
|
|
|
|
}
|
|
|
|
|
|
|
|
return info, err
|
|
|
|
}
|
|
|
|
return info, nil
|
|
|
|
}
|
|
|
|
|
2020-06-25 13:48:48 +00:00
|
|
|
type localClusterState struct {
|
|
|
|
// set true if cluster supports resize functionality.
|
|
|
|
resizeSupported bool
|
|
|
|
// set true once a subvolumegroup is created
|
|
|
|
// for corresponding cluster.
|
|
|
|
subVolumeGroupCreated bool
|
|
|
|
}
|
|
|
|
|
2019-08-22 17:19:06 +00:00
|
|
|
func createVolume(ctx context.Context, volOptions *volumeOptions, cr *util.Credentials, volID volumeID, bytesQuota int64) error {
|
2020-06-25 13:48:48 +00:00
|
|
|
// verify if corresponding ClusterID key is present in the map,
|
|
|
|
// and if not, initialize with default values(false).
|
|
|
|
if _, keyPresent := clusterAdditionalInfo[volOptions.ClusterID]; !keyPresent {
|
|
|
|
clusterAdditionalInfo[volOptions.ClusterID] = &localClusterState{}
|
|
|
|
}
|
|
|
|
|
|
|
|
// create subvolumegroup if not already created for the cluster.
|
|
|
|
if !clusterAdditionalInfo[volOptions.ClusterID].subVolumeGroupCreated {
|
2019-06-08 05:06:03 +00:00
|
|
|
err := execCommandErr(
|
2019-08-22 17:19:06 +00:00
|
|
|
ctx,
|
2019-06-08 05:06:03 +00:00
|
|
|
"ceph",
|
|
|
|
"fs",
|
|
|
|
"subvolumegroup",
|
|
|
|
"create",
|
|
|
|
volOptions.FsName,
|
2020-04-29 10:03:08 +00:00
|
|
|
volOptions.SubvolumeGroup,
|
2019-06-08 05:06:03 +00:00
|
|
|
"-m", volOptions.Monitors,
|
|
|
|
"-c", util.CephConfigPath,
|
|
|
|
"-n", cephEntityClientPrefix+cr.ID,
|
2019-06-25 19:29:17 +00:00
|
|
|
"--keyfile="+cr.KeyFile)
|
2019-06-08 05:06:03 +00:00
|
|
|
if err != nil {
|
2020-04-29 10:03:08 +00:00
|
|
|
klog.Errorf(util.Log(ctx, "failed to create subvolume group %s, for the vol %s(%s)"), volOptions.SubvolumeGroup, string(volID), err)
|
2018-12-01 09:39:09 +00:00
|
|
|
return err
|
|
|
|
}
|
2020-07-09 14:48:24 +00:00
|
|
|
util.DebugLog(ctx, "cephfs: created subvolume group %s", volOptions.SubvolumeGroup)
|
2020-06-25 13:48:48 +00:00
|
|
|
clusterAdditionalInfo[volOptions.ClusterID].subVolumeGroupCreated = true
|
2018-04-13 12:49:49 +00:00
|
|
|
}
|
2019-08-05 15:40:48 +00:00
|
|
|
|
|
|
|
args := []string{
|
2019-06-08 05:06:03 +00:00
|
|
|
"fs",
|
|
|
|
"subvolume",
|
|
|
|
"create",
|
|
|
|
volOptions.FsName,
|
|
|
|
string(volID),
|
|
|
|
strconv.FormatInt(bytesQuota, 10),
|
|
|
|
"--group_name",
|
2020-04-29 10:03:08 +00:00
|
|
|
volOptions.SubvolumeGroup,
|
2019-07-25 12:39:52 +00:00
|
|
|
"--mode", "777",
|
2019-06-08 05:06:03 +00:00
|
|
|
"-m", volOptions.Monitors,
|
|
|
|
"-c", util.CephConfigPath,
|
2019-08-05 15:40:48 +00:00
|
|
|
"-n", cephEntityClientPrefix + cr.ID,
|
|
|
|
"--keyfile=" + cr.KeyFile,
|
|
|
|
}
|
|
|
|
|
|
|
|
if volOptions.Pool != "" {
|
|
|
|
args = append(args, "--pool_layout", volOptions.Pool)
|
|
|
|
}
|
|
|
|
|
|
|
|
err := execCommandErr(
|
2019-08-22 17:19:06 +00:00
|
|
|
ctx,
|
2019-08-05 15:40:48 +00:00
|
|
|
"ceph",
|
|
|
|
args[:]...)
|
2019-06-08 05:06:03 +00:00
|
|
|
if err != nil {
|
2019-08-22 17:19:06 +00:00
|
|
|
klog.Errorf(util.Log(ctx, "failed to create subvolume %s(%s) in fs %s"), string(volID), err, volOptions.FsName)
|
2019-02-26 10:06:16 +00:00
|
|
|
return err
|
|
|
|
}
|
2018-04-13 12:49:49 +00:00
|
|
|
|
2019-02-26 10:06:16 +00:00
|
|
|
return nil
|
|
|
|
}
|
|
|
|
|
2020-05-06 13:47:46 +00:00
|
|
|
// resizeVolume will try to use ceph fs subvolume resize command to resize the
|
|
|
|
// subvolume. If the command is not available as a fallback it will use
|
|
|
|
// CreateVolume to resize the subvolume.
|
|
|
|
func resizeVolume(ctx context.Context, volOptions *volumeOptions, cr *util.Credentials, volID volumeID, bytesQuota int64) error {
|
2020-06-25 13:48:48 +00:00
|
|
|
// keyPresent checks whether corresponding clusterID key is present in clusterAdditionalInfo
|
|
|
|
var keyPresent bool
|
|
|
|
// verify if corresponding ClusterID key is present in the map,
|
|
|
|
// and if not, initialize with default values(false).
|
|
|
|
if _, keyPresent = clusterAdditionalInfo[volOptions.ClusterID]; !keyPresent {
|
|
|
|
clusterAdditionalInfo[volOptions.ClusterID] = &localClusterState{}
|
|
|
|
}
|
|
|
|
// resize subvolume when either it's supported, or when corresponding
|
|
|
|
// clusterID key was not present.
|
|
|
|
if clusterAdditionalInfo[volOptions.ClusterID].resizeSupported || !keyPresent {
|
2020-05-06 13:47:46 +00:00
|
|
|
args := []string{
|
|
|
|
"fs",
|
|
|
|
"subvolume",
|
|
|
|
"resize",
|
|
|
|
volOptions.FsName,
|
|
|
|
string(volID),
|
|
|
|
strconv.FormatInt(bytesQuota, 10),
|
|
|
|
"--group_name",
|
|
|
|
volOptions.SubvolumeGroup,
|
|
|
|
"-m", volOptions.Monitors,
|
|
|
|
"-c", util.CephConfigPath,
|
|
|
|
"-n", cephEntityClientPrefix + cr.ID,
|
|
|
|
"--keyfile=" + cr.KeyFile,
|
|
|
|
}
|
|
|
|
|
|
|
|
err := execCommandErr(
|
|
|
|
ctx,
|
|
|
|
"ceph",
|
|
|
|
args[:]...)
|
|
|
|
|
|
|
|
if err == nil {
|
2020-06-25 13:48:48 +00:00
|
|
|
clusterAdditionalInfo[volOptions.ClusterID].resizeSupported = true
|
2020-05-06 13:47:46 +00:00
|
|
|
return nil
|
|
|
|
}
|
|
|
|
// Incase the error is other than invalid command return error to the caller.
|
2020-08-04 04:23:26 +00:00
|
|
|
if !strings.Contains(err.Error(), ErrInvalidCommand.Error()) {
|
2020-05-06 13:47:46 +00:00
|
|
|
klog.Errorf(util.Log(ctx, "failed to resize subvolume %s(%s) in fs %s"), string(volID), err, volOptions.FsName)
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
}
|
2020-06-25 13:48:48 +00:00
|
|
|
clusterAdditionalInfo[volOptions.ClusterID].resizeSupported = false
|
2020-05-06 13:47:46 +00:00
|
|
|
return createVolume(ctx, volOptions, cr, volID, bytesQuota)
|
|
|
|
}
|
|
|
|
|
2020-08-04 04:23:26 +00:00
|
|
|
func purgeVolume(ctx context.Context, volID volumeID, cr *util.Credentials, volOptions *volumeOptions, force bool) error {
|
|
|
|
arg := []string{
|
2019-06-08 05:06:03 +00:00
|
|
|
"fs",
|
|
|
|
"subvolume",
|
|
|
|
"rm",
|
|
|
|
volOptions.FsName,
|
|
|
|
string(volID),
|
|
|
|
"--group_name",
|
2020-04-29 10:03:08 +00:00
|
|
|
volOptions.SubvolumeGroup,
|
2019-06-08 05:06:03 +00:00
|
|
|
"-m", volOptions.Monitors,
|
|
|
|
"-c", util.CephConfigPath,
|
2020-08-04 04:23:26 +00:00
|
|
|
"-n", cephEntityClientPrefix + cr.ID,
|
|
|
|
"--keyfile=" + cr.KeyFile,
|
|
|
|
}
|
|
|
|
if force {
|
|
|
|
arg = append(arg, "--force")
|
|
|
|
}
|
|
|
|
|
|
|
|
err := execCommandErr(ctx, "ceph", arg...)
|
2018-08-14 09:19:41 +00:00
|
|
|
if err != nil {
|
2019-08-22 17:19:06 +00:00
|
|
|
klog.Errorf(util.Log(ctx, "failed to purge subvolume %s(%s) in fs %s"), string(volID), err, volOptions.FsName)
|
2020-08-04 04:23:26 +00:00
|
|
|
if strings.HasPrefix(err.Error(), ErrVolumeNotFound.Error()) {
|
2020-07-10 00:14:39 +00:00
|
|
|
return util.JoinErrors(ErrVolumeNotFound, err)
|
2020-01-23 02:48:46 +00:00
|
|
|
}
|
2019-06-08 05:06:03 +00:00
|
|
|
return err
|
2018-04-13 12:49:49 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
return nil
|
2018-03-09 16:05:19 +00:00
|
|
|
}
|