mirrors/ceph-csi
1
0
Fork 0
mirror of https://github.com/ceph/ceph-csi.git synced 2025-06-13 02:33:34 +00:00
Code Issues Packages Projects Releases Wiki Activity

cephfs: added support for snapshot-backed volumes

Browse Source 
This commit implements most of
docs/design/proposals/cephfs-snapshot-shallow-ro-vol.md design document;
specifically (de-)provisioning of snapshot-backed volumes, mounting such
volumes as well as mounting pre-provisioned snapshot-backed volumes.

Signed-off-by: Robert Vasek <robert.vasek@cern.ch>
This commit is contained in:
Robert Vasek
2022-04-06 15:26:07 +02:00
committed by mergify[bot]
parent 0807fd2e6c
commit fd7559a903
7 changed files with 710 additions and 107 deletions
Download Patch File Download Diff File Expand all files Collapse all files

168
internal/cephfs/store/backingsnapshot.go Normal file
View File

@ -0,0 +1,168 @@
/*
Copyright 2022 The Ceph-CSI Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package store
import (
"context"
"fmt"
fsutil "github.com/ceph/ceph-csi/internal/cephfs/util"
"github.com/ceph/ceph-csi/internal/util/log"
"github.com/ceph/ceph-csi/internal/util/reftracker"
"github.com/ceph/ceph-csi/internal/util/reftracker/radoswrapper"
"github.com/ceph/ceph-csi/internal/util/reftracker/reftype"
)
func fmtBackingSnapshotReftrackerName(backingSnapID string) string {
return fmt.Sprintf("rt-backingsnapshot-%s", backingSnapID)
}
func AddSnapshotBackedVolumeRef(
ctx context.Context,
volOptions *VolumeOptions,
) error {
ioctx, err := volOptions.conn.GetIoctx(volOptions.MetadataPool)
if err != nil {
log.ErrorLog(ctx, "failed to create RADOS ioctx: %s", err)
return err
}
defer ioctx.Destroy()
ioctx.SetNamespace(fsutil.RadosNamespace)
var (
backingSnapID = volOptions.BackingSnapshotID
ioctxW = radoswrapper.NewIOContext(ioctx)
)
created, err := reftracker.Add(
ioctxW,
fmtBackingSnapshotReftrackerName(backingSnapID),
map[string]struct{}{
backingSnapID: {},
volOptions.VolID: {},
},
)
if err != nil {
log.ErrorLog(ctx, "failed to add refs for backing snapshot %s: %v",
backingSnapID, err)
return err
}
defer func() {
if err == nil {
return
}
// Clean up after failure.
var deleted bool
deleted, err = reftracker.Remove(
ioctxW,
fmtBackingSnapshotReftrackerName(backingSnapID),
map[string]reftype.RefType{
backingSnapID: reftype.Normal,
volOptions.VolID: reftype.Normal,
},
)
if err != nil {
log.ErrorLog(ctx, "failed to remove refs in cleanup procedure for backing snapshot %s: %v",
backingSnapID, err)
}
if created && !deleted {
log.ErrorLog(ctx, "orphaned reftracker object %s (pool %s, namespace %s)",
backingSnapID, volOptions.MetadataPool, fsutil.RadosNamespace)
}
}()
// There may have been a race between adding a ref to the reftracker and
// deleting the backing snapshot. Make sure the snapshot still exists by
// trying to retrieve it again.
_, _, _, err = NewSnapshotOptionsFromID(ctx, volOptions.BackingSnapshotID, volOptions.conn.Creds)
if err != nil {
log.ErrorLog(ctx, "failed to get backing snapshot %s: %v", volOptions.BackingSnapshotID, err)
}
return err
}
func UnrefSnapshotBackedVolume(
ctx context.Context,
volOptions *VolumeOptions,
) (bool, error) {
ioctx, err := volOptions.conn.GetIoctx(volOptions.MetadataPool)
if err != nil {
log.ErrorLog(ctx, "failed to create RADOS ioctx: %s", err)
return false, err
}
defer ioctx.Destroy()
ioctx.SetNamespace(fsutil.RadosNamespace)
var (
backingSnapID = volOptions.BackingSnapshotID
ioctxW = radoswrapper.NewIOContext(ioctx)
)
deleted, err := reftracker.Remove(
ioctxW,
fmtBackingSnapshotReftrackerName(backingSnapID),
map[string]reftype.RefType{
volOptions.VolID: reftype.Normal,
},
)
if err != nil {
log.ErrorLog(ctx, "failed to remove refs for backing snapshot %s: %v",
backingSnapID, err)
return false, err
}
return deleted, err
}
// UnrefSelfInSnapshotBackedVolumes removes (masks) snapshot ID in the
// reftracker for volumes backed by this snapshot. The returned boolean
// value signals whether the snapshot is not referenced by any such volumes
// and needs to be removed.
func UnrefSelfInSnapshotBackedVolumes(
ctx context.Context,
snapParentVolOptions *VolumeOptions,
snapshotID string,
) (bool, error) {
ioctx, err := snapParentVolOptions.conn.GetIoctx(snapParentVolOptions.MetadataPool)
if err != nil {
log.ErrorLog(ctx, "failed to create RADOS ioctx: %s", err)
return false, err
}
defer ioctx.Destroy()
ioctx.SetNamespace(fsutil.RadosNamespace)
return reftracker.Remove(
radoswrapper.NewIOContext(ioctx),
fmtBackingSnapshotReftrackerName(snapshotID),
map[string]reftype.RefType{
snapshotID: reftype.Mask,
},
)
}

189
internal/cephfs/store/volumeoptions.go
View File

@ -20,6 +20,7 @@ import (
"context"
"errors"
"fmt"
"path"
"strconv"
"strings"
@ -34,27 +35,31 @@ import (
type VolumeOptions struct {
core.SubVolume
TopologyPools *[]util.TopologyConstrainedPool
TopologyRequirement *csi.TopologyRequirement
Topology map[string]string
RequestName string
NamePrefix string
ClusterID string
FscID int64
MetadataPool string
RequestName string
NamePrefix string
ClusterID string
MetadataPool string
// ReservedID represents the ID reserved for a subvolume
ReservedID string
Monitors string `json:"monitors"`
RootPath string `json:"rootPath"`
Mounter string `json:"mounter"`
ProvisionVolume bool `json:"provisionVolume"`
KernelMountOptions string `json:"kernelMountOptions"`
FuseMountOptions string `json:"fuseMountOptions"`
// Network namespace file path to execute nsenter command
ReservedID string
Monitors string `json:"monitors"`
RootPath string `json:"rootPath"`
Mounter string `json:"mounter"`
BackingSnapshotRoot string // Snapshot root relative to RootPath.
BackingSnapshotID string
KernelMountOptions string `json:"kernelMountOptions"`
FuseMountOptions string `json:"fuseMountOptions"`
NetNamespaceFilePath string
TopologyPools *[]util.TopologyConstrainedPool
TopologyRequirement *csi.TopologyRequirement
Topology map[string]string
FscID int64
// conn is a connection to the Ceph cluster obtained from a ConnPool
conn *util.ClusterConnection
ProvisionVolume bool `json:"provisionVolume"`
BackingSnapshot bool `json:"backingSnapshot"`
}
// Connect a CephFS volume to the Ceph cluster.
@ -184,14 +189,20 @@ func (vo *VolumeOptions) GetConnection() *util.ClusterConnection {
return vo.conn
}
func fmtBackingSnapshotOptionMismatch(optName, expected, actual string) error {
return fmt.Errorf("%s option mismatch with backing snapshot: got %s, expected %s",
optName, actual, expected)
}
// NewVolumeOptions generates a new instance of volumeOptions from the provided
// CSI request parameters.
func NewVolumeOptions(ctx context.Context, requestName string, req *csi.CreateVolumeRequest,
cr *util.Credentials,
) (*VolumeOptions, error) {
var (
opts VolumeOptions
err error
opts VolumeOptions
backingSnapshotBool string
err error
)
volOptions := req.GetParameters()
@ -228,6 +239,16 @@ func NewVolumeOptions(ctx context.Context, requestName string, req *csi.CreateVo
return nil, err
}
if err = extractOptionalOption(&backingSnapshotBool, "backingSnapshot", volOptions); err != nil {
return nil, err
}
if backingSnapshotBool != "" {
if opts.BackingSnapshot, err = strconv.ParseBool(backingSnapshotBool); err != nil {
return nil, fmt.Errorf("failed to parse backingSnapshot: %w", err)
}
}
opts.RequestName = requestName
err = opts.Connect(cr)
@ -261,6 +282,19 @@ func NewVolumeOptions(ctx context.Context, requestName string, req *csi.CreateVo
opts.ProvisionVolume = true
if opts.BackingSnapshot {
if req.GetVolumeContentSource() == nil || req.GetVolumeContentSource().GetSnapshot() == nil {
return nil, errors.New("backingSnapshot option requires snapshot volume source")
}
opts.BackingSnapshotID = req.GetVolumeContentSource().GetSnapshot().GetSnapshotId()
err = opts.populateVolumeOptionsFromBackingSnapshot(ctx, cr)
if err != nil {
return nil, err
}
}
return &opts, nil
}
@ -364,23 +398,109 @@ func NewVolumeOptionsFromVolID(
}
}
volOptions.ProvisionVolume = true
volOptions.SubVolume.VolID = vid.FsSubvolName
vol := core.NewSubVolume(volOptions.conn, &volOptions.SubVolume, volOptions.ClusterID)
info, err := vol.GetSubVolumeInfo(ctx)
if err == nil {
volOptions.RootPath = info.Path
volOptions.Features = info.Features
volOptions.Size = info.BytesQuota
if imageAttributes.BackingSnapshotID != "" || volOptions.BackingSnapshotID != "" {
volOptions.BackingSnapshot = true
volOptions.BackingSnapshotID = imageAttributes.BackingSnapshotID
}
if errors.Is(err, cerrors.ErrInvalidCommand) {
volOptions.RootPath, err = vol.GetVolumeRootPathCeph(ctx)
volOptions.ProvisionVolume = true
volOptions.SubVolume.VolID = vid.FsSubvolName
if volOptions.BackingSnapshot {
err = volOptions.populateVolumeOptionsFromBackingSnapshot(ctx, cr)
} else {
err = volOptions.populateVolumeOptionsFromSubvolume(ctx)
}
return &volOptions, &vid, err
}
func (vo *VolumeOptions) populateVolumeOptionsFromSubvolume(ctx context.Context) error {
vol := core.NewSubVolume(vo.conn, &vo.SubVolume, vo.ClusterID)
var info *core.Subvolume
info, err := vol.GetSubVolumeInfo(ctx)
if err == nil {
vo.RootPath = info.Path
vo.Features = info.Features
vo.Size = info.BytesQuota
}
if errors.Is(err, cerrors.ErrInvalidCommand) {
vo.RootPath, err = vol.GetVolumeRootPathCeph(ctx)
}
return err
}
func (vo *VolumeOptions) populateVolumeOptionsFromBackingSnapshot(
ctx context.Context,
cr *util.Credentials,
) error {
// As of CephFS snapshot v2 API, snapshots may be found in two locations:
//
// (a) /volumes/<volume group>/<subvolume>/.snap/<snapshot>/<UUID>
// (b) /volumes/<volume group>/<subvolume>/<UUID>/.snap/_<snapshot>_<snapshot inode number>
if !vo.ProvisionVolume {
// Case (b)
//
// If the volume is not provisioned by us, we assume that we have access only
// to snapshot's parent volume root. In this case, o.RootPath is expected to
// be already set in the volume context.
// BackingSnapshotRoot cannot be determined at this stage, because the
// full directory name is not known (see snapshot path format for case
// (b) above). RootPath/.snap must be traversed in order to find out
// the snapshot directory name.
return nil
}
parentBackingSnapVolOpts, _, snapID, err := NewSnapshotOptionsFromID(ctx, vo.BackingSnapshotID, cr)
if err != nil {
return fmt.Errorf("failed to retrieve backing snapshot %s: %w", vo.BackingSnapshotID, err)
}
// Ensure that backing snapshot parent's volume options match the context.
// Snapshot-backed volume inherits all its parent's (parent of the snapshot) options.
if vo.ClusterID != parentBackingSnapVolOpts.ClusterID {
return fmtBackingSnapshotOptionMismatch("clusterID", vo.ClusterID, parentBackingSnapVolOpts.ClusterID)
}
if vo.Pool != "" {
return errors.New("cannot set pool for snapshot-backed volume")
}
if vo.MetadataPool != parentBackingSnapVolOpts.MetadataPool {
return fmtBackingSnapshotOptionMismatch("MetadataPool", vo.MetadataPool, parentBackingSnapVolOpts.MetadataPool)
}
if vo.FsName != parentBackingSnapVolOpts.FsName {
return fmtBackingSnapshotOptionMismatch("fsName", vo.FsName, parentBackingSnapVolOpts.FsName)
}
if vo.SubvolumeGroup != parentBackingSnapVolOpts.SubvolumeGroup {
return fmtBackingSnapshotOptionMismatch("SubvolumeGroup", vo.SubvolumeGroup, parentBackingSnapVolOpts.SubvolumeGroup)
}
vo.Features = parentBackingSnapVolOpts.Features
vo.Size = parentBackingSnapVolOpts.Size
// For case (a) (o.ProvisionVolume==true is assumed), snapshot root path
// can be built out of subvolume root path, which is in following format:
//
// /volumes/<volume group>/<subvolume>/<subvolume UUID>
subvolRoot, subvolUUID := path.Split(parentBackingSnapVolOpts.RootPath)
vo.RootPath = subvolRoot
vo.BackingSnapshotRoot = path.Join(".snap", snapID.FsSnapshotName, subvolUUID)
return nil
}
// NewVolumeOptionsFromMonitorList generates a new instance of VolumeOptions and
// VolumeIdentifier from the provided CSI volume context.
func NewVolumeOptionsFromMonitorList(
@ -438,9 +558,17 @@ func NewVolumeOptionsFromMonitorList(
return nil, nil, err
}
if err = extractOptionalOption(&opts.BackingSnapshotID, "backingSnapshotID", options); err != nil {
return nil, nil, err
}
vid.FsSubvolName = volID
vid.VolumeID = volID
if opts.BackingSnapshotID != "" {
opts.BackingSnapshot = true
}
return &opts, &vid, nil
}
@ -511,6 +639,10 @@ func NewVolumeOptionsFromStaticVolume(
vid.FsSubvolName = opts.RootPath
vid.VolumeID = volID
if opts.BackingSnapshotID != "" {
opts.BackingSnapshot = true
}
return &opts, &vid, nil
}
@ -599,6 +731,7 @@ func NewSnapshotOptionsFromID(
}
volOptions.Features = subvolInfo.Features
volOptions.Size = subvolInfo.BytesQuota
volOptions.RootPath = subvolInfo.Path
snap := core.NewSnapshot(volOptions.conn, sid.FsSnapshotName, &volOptions.SubVolume)
info, err := snap.GetSnapshotInfo(ctx)
if err != nil {