From e489413dbdf6b0b0bc6a817d3bea4326d3a72e14 Mon Sep 17 00:00:00 2001 From: Niels de Vos Date: Fri, 21 Mar 2025 15:14:14 +0100 Subject: [PATCH] rbd: introduce functions for comparing Volumes in a VolumeGroup CompareVolumesInGroup() verifies that all the volumes are part of the given VolumeGroup. It does so by obtaining the VolumeGroupID for each volume with GetVolumeGroupByID(). The helper VolumesInSameGroup() verifies that all volumes belong to the same (or no) VolumeGroup. It can be called by CSI(-Addons) procedures before acting on a VolumeGroup. Signed-off-by: Niels de Vos --- internal/rbd/group.go | 23 +++++++++++ internal/rbd/manager.go | 73 +++++++++++++++++++++++++++++++++++ internal/rbd/types/manager.go | 16 ++++++-- internal/rbd/types/volume.go | 5 +++ 4 files changed, 113 insertions(+), 4 deletions(-) diff --git a/internal/rbd/group.go b/internal/rbd/group.go index 701e3b47d..a1a546501 100644 --- a/internal/rbd/group.go +++ b/internal/rbd/group.go @@ -22,6 +22,7 @@ import ( librbd "github.com/ceph/go-ceph/rbd" + rbderrors "github.com/ceph/ceph-csi/internal/rbd/errors" "github.com/ceph/ceph-csi/internal/rbd/types" ) @@ -79,6 +80,28 @@ func (rv *rbdVolume) RemoveFromGroup(ctx context.Context, vg types.VolumeGroup) return librbd.GroupImageRemove(ioctx, name, rv.ioctx, rv.RbdImageName) } +// GetVolumeGroupID returns the ID of the VolumeGroup where this rbdVolume +// belongs to. If the rbdVolume does not belong to a VolumeGroup, a +// rbderrors.ErrGroupNotFound is returned. +func (rv *rbdVolume) GetVolumeGroupID(ctx context.Context, resolver types.VolumeGroupResolver) (string, error) { + image, err := rv.open() + if err != nil { + return "", fmt.Errorf("failed to open image %q: %w", rv, err) + } + defer image.Close() + + info, err := image.GetGroup() + if err != nil { + return "", fmt.Errorf("could not get group information for image %q: %w", rv, err) + } + + if info.Name == "" { + return "", fmt.Errorf("%w: image %q is not part of a volume group", rbderrors.ErrGroupNotFound, rv) + } + + return resolver.MakeVolumeGroupID(ctx, info.PoolID, info.Name) +} + func (rv *rbdVolume) ToMirror() (types.Mirror, error) { return rv, nil } diff --git a/internal/rbd/manager.go b/internal/rbd/manager.go index 48693e64f..966564507 100644 --- a/internal/rbd/manager.go +++ b/internal/rbd/manager.go @@ -652,3 +652,76 @@ func (mgr *rbdManager) RegenerateVolumeGroupJournal( return groupHandle, nil } + +// CompareVolumesInGroup returns 'true' when the list of volumes matches the +// volumes in the group. In case a volume belongs to no group, or an other +// group than the VolumeGroup, 'false' is returned. +func (mgr *rbdManager) CompareVolumesInGroup( + ctx context.Context, + volumes []types.Volume, + vg types.VolumeGroup, +) (bool, error) { + vgVols, err := vg.ListVolumes(ctx) + if err != nil { + return false, fmt.Errorf("failed to list volumes in group %q: %w", vg, err) + } + + // the vg is allowed to be empty, or have the exact number of volumes + if !(len(vgVols) == 0 || len(vgVols) == len(volumes)) { + return false, fmt.Errorf( + "volume group %q has more or less volumes (%d) than expected (%d)", + vg, + len(vgVols), + len(volumes)) + } + + vgID, err := vg.GetID(ctx) + if err != nil { + return false, fmt.Errorf("failed to get name for volume group %q: %w", vg, err) + } + + // verify that all volumes are part of the vg, or do not have a group at all + matchingGroup, err := mgr.VolumesInSameGroup(ctx, volumes) + if err != nil { + return false, err + } else if !matchingGroup { + return false, nil + } + + // all volumes are in the same group + groupID, err := volumes[0].GetVolumeGroupID(ctx, mgr) + if err != nil && !errors.Is(err, rbderrors.ErrGroupNotFound) { + return false, fmt.Errorf("failed to get group for volume %q: %w", volumes[0], err) + } + + // if none of the volumes is in a group, groupID will be "" + if groupID != "" && vgID != groupID { + log.DebugLog(ctx, "expecting group %q but volume %q has group %q", vgID, volumes[0], groupID) + + return false, nil + } + + return true, nil +} + +// VolumesInSameGroup returns 'true' when all volumes are in the same group, or +// in no group at all. +func (mgr *rbdManager) VolumesInSameGroup(ctx context.Context, volumes []types.Volume) (bool, error) { + var lastID *string + for _, v := range volumes { + id, err := v.GetVolumeGroupID(ctx, mgr) + if err != nil && !errors.Is(err, rbderrors.ErrGroupNotFound) { + return false, fmt.Errorf("failed to get group name for volume %q: %w", v, err) + } + + // all volumes should be part of the same group + // lastID == nil in the 1st loop + if lastID != nil && *lastID != id { + return false, fmt.Errorf("volume %q belongs to group %q, but expected %q", v, id, *lastID) + } + + lastID = &id + } + + return true, nil +} diff --git a/internal/rbd/types/manager.go b/internal/rbd/types/manager.go index 5416c9d28..206dab83d 100644 --- a/internal/rbd/types/manager.go +++ b/internal/rbd/types/manager.go @@ -40,6 +40,18 @@ type VolumeGroupResolver interface { // The poolID and name are details of the Ceph RBD-group for which the // CSI VolumeGroupId should get constructed. MakeVolumeGroupID(ctx context.Context, poolID int64, name string) (string, error) + + // GetVolumeGroupByID uses the CSI-Addons VolumeGroupId to resolve the + // returned VolumeGroup. + GetVolumeGroupByID(ctx context.Context, id string) (VolumeGroup, error) + + // CompareVolumesInGroup verifies that all the volumes are part of the + // given VolumeGroup. + CompareVolumesInGroup(ctx context.Context, volumes []Volume, vg VolumeGroup) (bool, error) + + // VolumesInSameGroup verifies that all volumes belong to the same (or + // no) VolumeGroup. + VolumesInSameGroup(ctx context.Context, volumes []Volume) (bool, error) } // Manager provides a way for other packages to get Volumes and VolumeGroups. @@ -58,10 +70,6 @@ type Manager interface { // Destroy frees all resources that the Manager allocated. Destroy(ctx context.Context) - // GetVolumeGroupByID uses the CSI-Addons VolumeGroupId to resolve the - // returned VolumeGroup. - GetVolumeGroupByID(ctx context.Context, id string) (VolumeGroup, error) - // CreateVolumeGroup allocates a new VolumeGroup in the backend storage // and records details about it in the journal. CreateVolumeGroup(ctx context.Context, name string) (VolumeGroup, error) diff --git a/internal/rbd/types/volume.go b/internal/rbd/types/volume.go index ce0fff684..b2984de7b 100644 --- a/internal/rbd/types/volume.go +++ b/internal/rbd/types/volume.go @@ -36,6 +36,11 @@ type snapshottableVolume interface { } type csiAddonsVolume interface { + // GetVolumeGroupID returns the name of the VolumeGroup where this + // Volume belongs to. If the rbdVolume does not belong to a + // VolumeGroup, a ErrGroupNotFound is returned. + GetVolumeGroupID(ctx context.Context, resolver VolumeGroupResolver) (string, error) + // AddToGroup adds the Volume to the VolumeGroup. AddToGroup(ctx context.Context, vg VolumeGroup) error