2018-03-05 11:59:47 +00:00
|
|
|
/*
|
2019-04-03 08:46:15 +00:00
|
|
|
Copyright 2018 The Ceph-CSI Authors.
|
2018-03-05 11:59:47 +00:00
|
|
|
|
|
|
|
Licensed under the Apache License, Version 2.0 (the "License");
|
|
|
|
you may not use this file except in compliance with the License.
|
|
|
|
You may obtain a copy of the License at
|
|
|
|
|
|
|
|
http://www.apache.org/licenses/LICENSE-2.0
|
|
|
|
|
|
|
|
Unless required by applicable law or agreed to in writing, software
|
|
|
|
distributed under the License is distributed on an "AS IS" BASIS,
|
|
|
|
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
|
|
See the License for the specific language governing permissions and
|
|
|
|
limitations under the License.
|
|
|
|
*/
|
|
|
|
|
|
|
|
package cephfs
|
|
|
|
|
|
|
|
import (
|
2019-08-22 17:19:06 +00:00
|
|
|
"context"
|
2020-10-10 16:17:08 +00:00
|
|
|
"fmt"
|
2019-07-10 10:50:04 +00:00
|
|
|
"path"
|
2019-06-08 05:06:03 +00:00
|
|
|
"strconv"
|
|
|
|
"strings"
|
2019-01-28 13:59:16 +00:00
|
|
|
|
2020-04-17 09:23:49 +00:00
|
|
|
"github.com/ceph/ceph-csi/internal/util"
|
2020-08-05 14:31:22 +00:00
|
|
|
|
|
|
|
fsAdmin "github.com/ceph/go-ceph/cephfs/admin"
|
2018-03-09 16:05:19 +00:00
|
|
|
)
|
|
|
|
|
2019-06-08 05:06:03 +00:00
|
|
|
var (
|
2020-06-25 13:48:48 +00:00
|
|
|
// clusterAdditionalInfo contains information regarding if resize is
|
|
|
|
// supported in the particular cluster and subvolumegroup is
|
|
|
|
// created or not.
|
|
|
|
// Subvolumegroup creation and volume resize decisions are
|
|
|
|
// taken through this additional cluster information.
|
|
|
|
clusterAdditionalInfo = make(map[string]*localClusterState)
|
2019-06-08 05:06:03 +00:00
|
|
|
)
|
2018-03-22 13:01:10 +00:00
|
|
|
|
2020-07-10 10:44:59 +00:00
|
|
|
const (
|
|
|
|
cephEntityClientPrefix = "client."
|
2020-10-01 07:18:58 +00:00
|
|
|
|
|
|
|
// modeAllRWX can be used for setting permissions to Read-Write-eXecute
|
|
|
|
// for User, Group and Other.
|
|
|
|
modeAllRWX = 0777
|
2020-07-10 10:44:59 +00:00
|
|
|
)
|
2019-07-10 10:50:04 +00:00
|
|
|
|
2020-10-10 16:17:08 +00:00
|
|
|
// Subvolume holds subvolume information. This includes only the needed members
|
|
|
|
// from fsAdmin.SubVolumeInfo.
|
2020-08-04 04:21:31 +00:00
|
|
|
type Subvolume struct {
|
2020-10-10 16:17:08 +00:00
|
|
|
BytesQuota int64
|
|
|
|
Path string
|
|
|
|
Features []string
|
2020-08-04 04:21:31 +00:00
|
|
|
}
|
|
|
|
|
2019-07-10 10:50:04 +00:00
|
|
|
func getVolumeRootPathCephDeprecated(volID volumeID) string {
|
|
|
|
return path.Join("/", "csi-volumes", string(volID))
|
|
|
|
}
|
|
|
|
|
2019-08-22 17:19:06 +00:00
|
|
|
func getVolumeRootPathCeph(ctx context.Context, volOptions *volumeOptions, cr *util.Credentials, volID volumeID) (string, error) {
|
2020-01-23 02:48:46 +00:00
|
|
|
stdout, stderr, err := util.ExecCommand(
|
2020-07-22 12:11:41 +00:00
|
|
|
ctx,
|
2019-06-08 05:06:03 +00:00
|
|
|
"ceph",
|
|
|
|
"fs",
|
|
|
|
"subvolume",
|
|
|
|
"getpath",
|
|
|
|
volOptions.FsName,
|
|
|
|
string(volID),
|
|
|
|
"--group_name",
|
2020-04-29 10:03:08 +00:00
|
|
|
volOptions.SubvolumeGroup,
|
2019-06-08 05:06:03 +00:00
|
|
|
"-m", volOptions.Monitors,
|
|
|
|
"-c", util.CephConfigPath,
|
|
|
|
"-n", cephEntityClientPrefix+cr.ID,
|
2019-06-25 19:29:17 +00:00
|
|
|
"--keyfile="+cr.KeyFile)
|
2018-03-26 13:00:28 +00:00
|
|
|
|
2019-06-08 05:06:03 +00:00
|
|
|
if err != nil {
|
2020-10-01 11:38:07 +00:00
|
|
|
util.ErrorLog(ctx, "failed to get the rootpath for the vol %s: %s (stdError: %s)", string(volID), err, stderr)
|
2020-08-17 05:58:59 +00:00
|
|
|
if strings.Contains(stderr, volumeNotFound) {
|
2020-07-10 00:14:39 +00:00
|
|
|
return "", util.JoinErrors(ErrVolumeNotFound, err)
|
2020-01-23 02:48:46 +00:00
|
|
|
}
|
|
|
|
|
2019-06-08 05:06:03 +00:00
|
|
|
return "", err
|
|
|
|
}
|
2020-07-22 12:53:22 +00:00
|
|
|
return strings.TrimSuffix(stdout, "\n"), nil
|
2018-04-13 12:49:49 +00:00
|
|
|
}
|
|
|
|
|
2020-10-10 16:17:08 +00:00
|
|
|
func (vo *volumeOptions) getSubVolumeInfo(ctx context.Context, volID volumeID) (*Subvolume, error) {
|
|
|
|
fsa, err := vo.conn.GetFSAdmin()
|
|
|
|
if err != nil {
|
|
|
|
util.ErrorLog(ctx, "could not get FSAdmin, can not fetch metadata pool for %s:", vo.FsName, err)
|
|
|
|
return nil, err
|
|
|
|
}
|
|
|
|
|
|
|
|
info, err := fsa.SubVolumeInfo(vo.FsName, vo.SubvolumeGroup, string(volID))
|
2020-08-04 04:21:31 +00:00
|
|
|
if err != nil {
|
2020-10-01 11:38:07 +00:00
|
|
|
util.ErrorLog(ctx, "failed to get subvolume info for the vol %s: %s", string(volID), err)
|
2020-08-17 05:58:59 +00:00
|
|
|
if strings.HasPrefix(err.Error(), volumeNotFound) {
|
2020-10-10 16:17:08 +00:00
|
|
|
return nil, ErrVolumeNotFound
|
2020-08-04 04:21:31 +00:00
|
|
|
}
|
|
|
|
// Incase the error is other than invalid command return error to the caller.
|
2020-08-17 05:58:59 +00:00
|
|
|
if !strings.Contains(err.Error(), invalidCommand) {
|
2020-10-10 16:17:08 +00:00
|
|
|
return nil, ErrInvalidCommand
|
2020-08-04 04:21:31 +00:00
|
|
|
}
|
|
|
|
|
2020-10-10 16:17:08 +00:00
|
|
|
return nil, err
|
|
|
|
}
|
|
|
|
|
|
|
|
bc, ok := info.BytesQuota.(fsAdmin.ByteCount)
|
|
|
|
if !ok {
|
|
|
|
// info.BytesQuota == Infinite
|
|
|
|
return nil, fmt.Errorf("subvolume %s has unsupported quota: %v", string(volID), info.BytesQuota)
|
2020-08-04 04:21:31 +00:00
|
|
|
}
|
2020-10-10 16:17:08 +00:00
|
|
|
|
|
|
|
subvol := Subvolume{
|
|
|
|
BytesQuota: int64(bc),
|
|
|
|
Path: info.Path,
|
|
|
|
Features: make([]string, len(info.Features)),
|
|
|
|
}
|
|
|
|
for i, feature := range info.Features {
|
|
|
|
subvol.Features[i] = string(feature)
|
|
|
|
}
|
|
|
|
|
|
|
|
return &subvol, nil
|
2020-08-04 04:21:31 +00:00
|
|
|
}
|
|
|
|
|
2020-06-25 13:48:48 +00:00
|
|
|
type localClusterState struct {
|
|
|
|
// set true if cluster supports resize functionality.
|
|
|
|
resizeSupported bool
|
|
|
|
// set true once a subvolumegroup is created
|
|
|
|
// for corresponding cluster.
|
|
|
|
subVolumeGroupCreated bool
|
|
|
|
}
|
|
|
|
|
2020-10-01 07:08:09 +00:00
|
|
|
func createVolume(ctx context.Context, volOptions *volumeOptions, volID volumeID, bytesQuota int64) error {
|
2020-06-25 13:48:48 +00:00
|
|
|
// verify if corresponding ClusterID key is present in the map,
|
|
|
|
// and if not, initialize with default values(false).
|
|
|
|
if _, keyPresent := clusterAdditionalInfo[volOptions.ClusterID]; !keyPresent {
|
|
|
|
clusterAdditionalInfo[volOptions.ClusterID] = &localClusterState{}
|
|
|
|
}
|
|
|
|
|
2020-08-05 14:31:22 +00:00
|
|
|
ca, err := volOptions.conn.GetFSAdmin()
|
|
|
|
if err != nil {
|
|
|
|
util.ErrorLog(ctx, "could not get FSAdmin, can not create subvolume %s: %s", string(volID), err)
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
|
2020-06-25 13:48:48 +00:00
|
|
|
// create subvolumegroup if not already created for the cluster.
|
|
|
|
if !clusterAdditionalInfo[volOptions.ClusterID].subVolumeGroupCreated {
|
2020-08-05 14:31:22 +00:00
|
|
|
opts := fsAdmin.SubVolumeGroupOptions{}
|
|
|
|
err = ca.CreateSubVolumeGroup(volOptions.FsName, volOptions.SubvolumeGroup, &opts)
|
2019-06-08 05:06:03 +00:00
|
|
|
if err != nil {
|
2020-10-01 11:38:07 +00:00
|
|
|
util.ErrorLog(ctx, "failed to create subvolume group %s, for the vol %s: %s", volOptions.SubvolumeGroup, string(volID), err)
|
2018-12-01 09:39:09 +00:00
|
|
|
return err
|
|
|
|
}
|
2020-07-09 14:48:24 +00:00
|
|
|
util.DebugLog(ctx, "cephfs: created subvolume group %s", volOptions.SubvolumeGroup)
|
2020-06-25 13:48:48 +00:00
|
|
|
clusterAdditionalInfo[volOptions.ClusterID].subVolumeGroupCreated = true
|
2018-04-13 12:49:49 +00:00
|
|
|
}
|
2019-08-05 15:40:48 +00:00
|
|
|
|
2020-08-05 14:31:22 +00:00
|
|
|
opts := fsAdmin.SubVolumeOptions{
|
|
|
|
Size: fsAdmin.ByteCount(bytesQuota),
|
2020-10-01 07:18:58 +00:00
|
|
|
Mode: modeAllRWX,
|
2019-08-05 15:40:48 +00:00
|
|
|
}
|
|
|
|
if volOptions.Pool != "" {
|
2020-08-05 14:31:22 +00:00
|
|
|
opts.PoolLayout = volOptions.Pool
|
2019-08-05 15:40:48 +00:00
|
|
|
}
|
|
|
|
|
2020-08-05 14:31:22 +00:00
|
|
|
// FIXME: check if the right credentials are used ("-n", cephEntityClientPrefix + cr.ID)
|
|
|
|
err = ca.CreateSubVolume(volOptions.FsName, volOptions.SubvolumeGroup, string(volID), &opts)
|
2019-06-08 05:06:03 +00:00
|
|
|
if err != nil {
|
2020-10-01 11:38:07 +00:00
|
|
|
util.ErrorLog(ctx, "failed to create subvolume %s in fs %s: %s", string(volID), volOptions.FsName, err)
|
2019-02-26 10:06:16 +00:00
|
|
|
return err
|
|
|
|
}
|
2018-04-13 12:49:49 +00:00
|
|
|
|
2019-02-26 10:06:16 +00:00
|
|
|
return nil
|
|
|
|
}
|
|
|
|
|
2020-05-06 13:47:46 +00:00
|
|
|
// resizeVolume will try to use ceph fs subvolume resize command to resize the
|
|
|
|
// subvolume. If the command is not available as a fallback it will use
|
|
|
|
// CreateVolume to resize the subvolume.
|
|
|
|
func resizeVolume(ctx context.Context, volOptions *volumeOptions, cr *util.Credentials, volID volumeID, bytesQuota int64) error {
|
2020-06-25 13:48:48 +00:00
|
|
|
// keyPresent checks whether corresponding clusterID key is present in clusterAdditionalInfo
|
|
|
|
var keyPresent bool
|
|
|
|
// verify if corresponding ClusterID key is present in the map,
|
|
|
|
// and if not, initialize with default values(false).
|
|
|
|
if _, keyPresent = clusterAdditionalInfo[volOptions.ClusterID]; !keyPresent {
|
|
|
|
clusterAdditionalInfo[volOptions.ClusterID] = &localClusterState{}
|
|
|
|
}
|
|
|
|
// resize subvolume when either it's supported, or when corresponding
|
|
|
|
// clusterID key was not present.
|
|
|
|
if clusterAdditionalInfo[volOptions.ClusterID].resizeSupported || !keyPresent {
|
2020-05-06 13:47:46 +00:00
|
|
|
args := []string{
|
|
|
|
"fs",
|
|
|
|
"subvolume",
|
|
|
|
"resize",
|
|
|
|
volOptions.FsName,
|
|
|
|
string(volID),
|
|
|
|
strconv.FormatInt(bytesQuota, 10),
|
|
|
|
"--group_name",
|
|
|
|
volOptions.SubvolumeGroup,
|
|
|
|
"-m", volOptions.Monitors,
|
|
|
|
"-c", util.CephConfigPath,
|
|
|
|
"-n", cephEntityClientPrefix + cr.ID,
|
|
|
|
"--keyfile=" + cr.KeyFile,
|
|
|
|
}
|
|
|
|
|
|
|
|
err := execCommandErr(
|
|
|
|
ctx,
|
|
|
|
"ceph",
|
|
|
|
args[:]...)
|
|
|
|
|
|
|
|
if err == nil {
|
2020-06-25 13:48:48 +00:00
|
|
|
clusterAdditionalInfo[volOptions.ClusterID].resizeSupported = true
|
2020-05-06 13:47:46 +00:00
|
|
|
return nil
|
|
|
|
}
|
|
|
|
// Incase the error is other than invalid command return error to the caller.
|
2020-08-17 05:58:59 +00:00
|
|
|
if !strings.Contains(err.Error(), invalidCommand) {
|
2020-10-01 11:38:07 +00:00
|
|
|
util.ErrorLog(ctx, "failed to resize subvolume %s in fs %s: %s", string(volID), volOptions.FsName, err)
|
2020-05-06 13:47:46 +00:00
|
|
|
return err
|
|
|
|
}
|
|
|
|
}
|
2020-06-25 13:48:48 +00:00
|
|
|
clusterAdditionalInfo[volOptions.ClusterID].resizeSupported = false
|
2020-10-01 07:08:09 +00:00
|
|
|
return createVolume(ctx, volOptions, volID, bytesQuota)
|
2020-05-06 13:47:46 +00:00
|
|
|
}
|
|
|
|
|
2020-08-04 04:23:26 +00:00
|
|
|
func purgeVolume(ctx context.Context, volID volumeID, cr *util.Credentials, volOptions *volumeOptions, force bool) error {
|
|
|
|
arg := []string{
|
2019-06-08 05:06:03 +00:00
|
|
|
"fs",
|
|
|
|
"subvolume",
|
|
|
|
"rm",
|
|
|
|
volOptions.FsName,
|
|
|
|
string(volID),
|
|
|
|
"--group_name",
|
2020-04-29 10:03:08 +00:00
|
|
|
volOptions.SubvolumeGroup,
|
2019-06-08 05:06:03 +00:00
|
|
|
"-m", volOptions.Monitors,
|
|
|
|
"-c", util.CephConfigPath,
|
2020-08-04 04:23:26 +00:00
|
|
|
"-n", cephEntityClientPrefix + cr.ID,
|
|
|
|
"--keyfile=" + cr.KeyFile,
|
|
|
|
}
|
|
|
|
if force {
|
|
|
|
arg = append(arg, "--force")
|
|
|
|
}
|
2020-09-10 12:04:51 +00:00
|
|
|
if checkSubvolumeHasFeature("snapshot-retention", volOptions.Features) {
|
|
|
|
arg = append(arg, "--retain-snapshots")
|
|
|
|
}
|
2020-08-04 04:23:26 +00:00
|
|
|
|
|
|
|
err := execCommandErr(ctx, "ceph", arg...)
|
2018-08-14 09:19:41 +00:00
|
|
|
if err != nil {
|
2020-10-01 11:38:07 +00:00
|
|
|
util.ErrorLog(ctx, "failed to purge subvolume %s in fs %s: %s", string(volID), volOptions.FsName, err)
|
2020-09-21 05:38:14 +00:00
|
|
|
if strings.Contains(err.Error(), volumeNotEmpty) {
|
|
|
|
return util.JoinErrors(ErrVolumeHasSnapshots, err)
|
|
|
|
}
|
2020-08-17 05:58:59 +00:00
|
|
|
if strings.Contains(err.Error(), volumeNotFound) {
|
2020-07-10 00:14:39 +00:00
|
|
|
return util.JoinErrors(ErrVolumeNotFound, err)
|
2020-01-23 02:48:46 +00:00
|
|
|
}
|
2019-06-08 05:06:03 +00:00
|
|
|
return err
|
2018-04-13 12:49:49 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
return nil
|
2018-03-09 16:05:19 +00:00
|
|
|
}
|
2020-09-10 12:05:53 +00:00
|
|
|
|
|
|
|
// checkSubvolumeHasFeature verifies if the referred subvolume has
|
|
|
|
// the required feature.
|
|
|
|
func checkSubvolumeHasFeature(feature string, subVolFeatures []string) bool {
|
|
|
|
// The subvolume "features" are based on the internal version of the subvolume.
|
|
|
|
// Verify if subvolume supports the required feature.
|
|
|
|
for _, subvolFeature := range subVolFeatures {
|
|
|
|
if subvolFeature == feature {
|
|
|
|
return true
|
|
|
|
}
|
|
|
|
}
|
|
|
|
return false
|
|
|
|
}
|