diff --git a/internal/rbd/group/util.go b/internal/rbd/group/util.go new file mode 100644 index 000000000..8e7f76eb3 --- /dev/null +++ b/internal/rbd/group/util.go @@ -0,0 +1,248 @@ +/* +Copyright 2024 The Ceph-CSI Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package group + +import ( + "context" + "errors" + "fmt" + "time" + + "github.com/ceph/go-ceph/rados" + + "github.com/ceph/ceph-csi/internal/journal" + "github.com/ceph/ceph-csi/internal/util" + "github.com/ceph/ceph-csi/internal/util/log" +) + +type commonVolumeGroup struct { + // id is a unique value for this volume group in the Ceph cluster, it + // is used to find the group in the journal. + id string + + // name is used in RBD API calls as the name of this object + name string + + // creationTime is the time the group was created + creationTime *time.Time + + clusterID string + objectUUID string + + credentials *util.Credentials + + // temporary connection attributes + conn *util.ClusterConnection + ioctx *rados.IOContext + + // required details to perform operations on the group + monitors string + pool string + namespace string + + journal journal.VolumeGroupJournal +} + +func (cvg *commonVolumeGroup) initCommonVolumeGroup( + ctx context.Context, + id string, + j journal.VolumeGroupJournal, + creds *util.Credentials, +) error { + csiID := util.CSIIdentifier{} + err := csiID.DecomposeCSIID(id) + if err != nil { + return fmt.Errorf("failed to decompose volume group id %q: %w", id, err) + } + + mons, err := util.Mons(util.CsiConfigFile, csiID.ClusterID) + if err != nil { + return fmt.Errorf("failed to get MONs for cluster id %q: %w", csiID.ClusterID, err) + } + + namespace, err := util.GetRadosNamespace(util.CsiConfigFile, csiID.ClusterID) + if err != nil { + return fmt.Errorf("failed to get RADOS namespace for cluster id %q: %w", csiID.ClusterID, err) + } + + pool, err := util.GetPoolName(mons, creds, csiID.LocationID) + if err != nil { + return fmt.Errorf("failed to get pool for volume group id %q: %w", id, err) + } + + cvg.journal = j + cvg.credentials = creds + cvg.id = id + cvg.clusterID = csiID.ClusterID + cvg.objectUUID = csiID.ObjectUUID + cvg.monitors = mons + cvg.pool = pool + cvg.namespace = namespace + + log.DebugLog(ctx, "object for volume group %q has been initialized", cvg.id) + + return nil +} + +func (cvg *commonVolumeGroup) Destroy(ctx context.Context) { + if cvg.ioctx != nil { + cvg.ioctx.Destroy() + cvg.ioctx = nil + } + + if cvg.conn != nil { + cvg.conn.Destroy() + cvg.conn = nil + } + + if cvg.credentials != nil { + cvg.credentials.DeleteCredentials() + cvg.credentials = nil + } + + log.DebugLog(ctx, "destroyed volume group instance with id %q", cvg.id) +} + +// getVolumeGroupAttributes fetches the attributes from the journal, sets some +// of the common values for the VolumeGroup and returns the attributes struct +// for further consumption (like checking the VolumeMap). +func (cvg *commonVolumeGroup) getVolumeGroupAttributes(ctx context.Context) (*journal.VolumeGroupAttributes, error) { + attrs, err := cvg.journal.GetVolumeGroupAttributes(ctx, cvg.pool, cvg.objectUUID) + if err != nil { + if !errors.Is(err, util.ErrKeyNotFound) && !errors.Is(err, util.ErrPoolNotFound) { + return nil, fmt.Errorf("failed to get attributes for volume group id %q: %w", cvg.id, err) + } + + attrs = &journal.VolumeGroupAttributes{} + } + + cvg.name = attrs.GroupName + cvg.creationTime = attrs.CreationTime + + return attrs, nil +} + +// String returns the image-spec (pool/{namespace}/{name}) format of the group. +func (cvg *commonVolumeGroup) String() string { + if cvg.namespace != "" && cvg.pool != "" && cvg.name != "" { + return fmt.Sprintf("%s/%s/%s", cvg.pool, cvg.namespace, cvg.name) + } + + if cvg.name != "" && cvg.pool != "" { + return fmt.Sprintf("%s/%s", cvg.pool, cvg.name) + } + + return fmt.Sprintf("", *cvg) +} + +// GetID returns the CSI-Addons VolumeGroupId of the VolumeGroup. +func (cvg *commonVolumeGroup) GetID(ctx context.Context) (string, error) { + if cvg.id == "" { + return "", errors.New("BUG: ID is not set") + } + + return cvg.id, nil +} + +// GetName returns the name in the backend storage for the VolumeGroup. +func (cvg *commonVolumeGroup) GetName(ctx context.Context) (string, error) { + if cvg.name == "" { + return "", errors.New("BUG: name is not set") + } + + return cvg.name, nil +} + +// GetPool returns the name of the pool that holds the VolumeGroup. +func (cvg *commonVolumeGroup) GetPool(ctx context.Context) (string, error) { + if cvg.pool == "" { + return "", errors.New("BUG: pool is not set") + } + + return cvg.pool, nil +} + +// GetClusterID returns the name of the pool that holds the VolumeGroup. +func (cvg *commonVolumeGroup) GetClusterID(ctx context.Context) (string, error) { + if cvg.clusterID == "" { + return "", errors.New("BUG: clusterID is not set") + } + + return cvg.clusterID, nil +} + +// getConnection returns the ClusterConnection for the volume group if it +// exists, otherwise it will open a new one. +// Destroy should be used to close the ClusterConnection. +func (cvg *commonVolumeGroup) getConnection(ctx context.Context) (*util.ClusterConnection, error) { + if cvg.conn != nil { + return cvg.conn, nil + } + + conn := &util.ClusterConnection{} + err := conn.Connect(cvg.monitors, cvg.credentials) + if err != nil { + return nil, fmt.Errorf("failed to connect to MONs %q: %w", cvg.monitors, err) + } + + cvg.conn = conn + log.DebugLog(ctx, "connection established for volume group %q", cvg.id) + + return conn, nil +} + +// GetIOContext returns the IOContext for the volume group if it exists, +// otherwise it will allocate a new one. +// Destroy should be used to free the IOContext. +func (cvg *commonVolumeGroup) GetIOContext(ctx context.Context) (*rados.IOContext, error) { + if cvg.ioctx != nil { + return cvg.ioctx, nil + } + + conn, err := cvg.getConnection(ctx) + if err != nil { + return nil, fmt.Errorf("%w: failed to connect: %w", ErrRBDGroupNotConnected, err) + } + + ioctx, err := conn.GetIoctx(cvg.pool) + if err != nil { + return nil, fmt.Errorf("%w: failed to get IOContext: %w", ErrRBDGroupNotConnected, err) + } + + if cvg.namespace != "" { + ioctx.SetNamespace(cvg.namespace) + } + + cvg.ioctx = ioctx + log.DebugLog(ctx, "iocontext created for volume group %q in pool %q", cvg.id, cvg.pool) + + return ioctx, nil +} + +// GetCreationTime fetches the creation time of the volume group from the +// journal and returns it. +func (cvg *commonVolumeGroup) GetCreationTime(ctx context.Context) (*time.Time, error) { + if cvg.creationTime == nil { + // getVolumeGroupAttributes sets .creationTime (and a few other attributes) + _, err := cvg.getVolumeGroupAttributes(ctx) + if err != nil { + return nil, fmt.Errorf("failed to get volume attributes for id %q: %w", cvg, err) + } + } + + return cvg.creationTime, nil +} diff --git a/internal/rbd/group/volume_group.go b/internal/rbd/group/volume_group.go index 6e6e59019..337c974ef 100644 --- a/internal/rbd/group/volume_group.go +++ b/internal/rbd/group/volume_group.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package rbd_group +package group import ( "context" @@ -37,27 +37,7 @@ var ErrRBDGroupNotConnected = errors.New("RBD group is not connected") // volumeGroup handles all requests for 'rbd group' operations. type volumeGroup struct { - // id is a unique value for this volume group in the Ceph cluster, it - // is used to find the group in the journal. - id string - - // name is used in RBD API calls as the name of this object - name string - - clusterID string - - credentials *util.Credentials - - // temporary connection attributes - conn *util.ClusterConnection - ioctx *rados.IOContext - - // required details to perform operations on the group - monitors string - pool string - namespace string - - journal journal.VolumeGroupJournal + *commonVolumeGroup // volumes is a list of rbd-images that are part of the group. The ID // of each volume is stored in the journal. @@ -87,119 +67,47 @@ func GetVolumeGroup( creds *util.Credentials, volumeResolver types.VolumeResolver, ) (types.VolumeGroup, error) { - csiID := util.CSIIdentifier{} - err := csiID.DecomposeCSIID(id) + vg := &volumeGroup{} + err := vg.initCommonVolumeGroup(ctx, id, j, creds) if err != nil { - return nil, fmt.Errorf("failed to decompose volume group id %q: %w", id, err) + return nil, fmt.Errorf("failed to initialize volume group with id %q: %w", id, err) } - mons, err := util.Mons(util.CsiConfigFile, csiID.ClusterID) + attrs, err := vg.getVolumeGroupAttributes(ctx) if err != nil { - return nil, fmt.Errorf("failed to get MONs for cluster id %q: %w", csiID.ClusterID, err) - } - - namespace, err := util.GetRadosNamespace(util.CsiConfigFile, csiID.ClusterID) - if err != nil { - return nil, fmt.Errorf("failed to get RADOS namespace for cluster id %q: %w", csiID.ClusterID, err) - } - - pool, err := util.GetPoolName(mons, creds, csiID.LocationID) - if err != nil { - return nil, fmt.Errorf("failed to get pool for volume group id %q: %w", id, err) - } - - attrs, err := j.GetVolumeGroupAttributes(ctx, pool, csiID.ObjectUUID) - if err != nil { - if !errors.Is(err, util.ErrKeyNotFound) && !errors.Is(err, util.ErrPoolNotFound) { - return nil, fmt.Errorf("failed to get attributes for volume group id %q: %w", id, err) - } - - attrs = &journal.VolumeGroupAttributes{} + return nil, fmt.Errorf("failed to get volume attributes for id %q: %w", vg, err) } var volumes []types.Volume + // it is needed to free the previously allocated volumes + defer func() { + // volumesToFree is empty in case of an error, let .Destroy() handle it otherwise + if len(vg.volumesToFree) > 0 { + return + } + + for _, v := range volumes { + v.Destroy(ctx) + } + }() for volID := range attrs.VolumeMap { vol, err := volumeResolver.GetVolumeByID(ctx, volID) if err != nil { - // free the previously allocated volumes - for _, v := range volumes { - v.Destroy(ctx) - } - return nil, fmt.Errorf("failed to get attributes for volume group id %q: %w", id, err) } volumes = append(volumes, vol) } - vg := &volumeGroup{ - journal: j, - credentials: creds, - id: id, - name: attrs.GroupName, - clusterID: csiID.ClusterID, - monitors: mons, - pool: pool, - namespace: namespace, - volumes: volumes, - // all allocated volumes need to be free'd at Destroy() time - volumesToFree: volumes, - } + vg.volumes = volumes + // all allocated volumes need to be free'd at Destroy() time + vg.volumesToFree = volumes log.DebugLog(ctx, "GetVolumeGroup(%s) returns %+v", id, *vg) return vg, nil } -// String returns the image-spec (pool/{namespace}/{name}) format of the group. -func (vg *volumeGroup) String() string { - if vg.namespace != "" && vg.pool != "" && vg.name != "" { - return fmt.Sprintf("%s/%s/%s", vg.pool, vg.namespace, vg.name) - } - - if vg.name != "" && vg.pool != "" { - return fmt.Sprintf("%s/%s", vg.pool, vg.name) - } - - return fmt.Sprintf("", *vg) -} - -// GetID returns the CSI-Addons VolumeGroupId of the VolumeGroup. -func (vg *volumeGroup) GetID(ctx context.Context) (string, error) { - if vg.id == "" { - return "", errors.New("BUG: ID is not set") - } - - return vg.id, nil -} - -// GetName returns the name in the backend storage for the VolumeGroup. -func (vg *volumeGroup) GetName(ctx context.Context) (string, error) { - if vg.name == "" { - return "", errors.New("BUG: name is not set") - } - - return vg.name, nil -} - -// GetPool returns the name of the pool that holds the VolumeGroup. -func (vg *volumeGroup) GetPool(ctx context.Context) (string, error) { - if vg.pool == "" { - return "", errors.New("BUG: pool is not set") - } - - return vg.pool, nil -} - -// GetClusterID returns the name of the pool that holds the VolumeGroup. -func (vg *volumeGroup) GetClusterID(ctx context.Context) (string, error) { - if vg.clusterID == "" { - return "", errors.New("BUG: clusterID is not set") - } - - return vg.clusterID, nil -} - // ToCSI creates a CSI-Addons type for the VolumeGroup. func (vg *volumeGroup) ToCSI(ctx context.Context) (*volumegroup.VolumeGroup, error) { volumes, err := vg.ListVolumes(ctx) @@ -230,54 +138,6 @@ func (vg *volumeGroup) ToCSI(ctx context.Context) (*volumegroup.VolumeGroup, err }, nil } -// getConnection returns the ClusterConnection for the volume group if it -// exists, otherwise it will open a new one. -// Destroy should be used to close the ClusterConnection. -func (vg *volumeGroup) getConnection(ctx context.Context) (*util.ClusterConnection, error) { - if vg.conn != nil { - return vg.conn, nil - } - - conn := &util.ClusterConnection{} - err := conn.Connect(vg.monitors, vg.credentials) - if err != nil { - return nil, fmt.Errorf("failed to connect to MONs %q: %w", vg.monitors, err) - } - - vg.conn = conn - log.DebugLog(ctx, "connection established for volume group %q", vg.id) - - return conn, nil -} - -// GetIOContext returns the IOContext for the volume group if it exists, -// otherwise it will allocate a new one. -// Destroy should be used to free the IOContext. -func (vg *volumeGroup) GetIOContext(ctx context.Context) (*rados.IOContext, error) { - if vg.ioctx != nil { - return vg.ioctx, nil - } - - conn, err := vg.getConnection(ctx) - if err != nil { - return nil, fmt.Errorf("%w: failed to connect: %w", ErrRBDGroupNotConnected, err) - } - - ioctx, err := conn.GetIoctx(vg.pool) - if err != nil { - return nil, fmt.Errorf("%w: failed to get IOContext: %w", ErrRBDGroupNotConnected, err) - } - - if vg.namespace != "" { - ioctx.SetNamespace(vg.namespace) - } - - vg.ioctx = ioctx - log.DebugLog(ctx, "iocontext created for volume group %q in pool %q", vg.id, vg.pool) - - return ioctx, nil -} - // Destroy frees the resources used by the volumeGroup. func (vg *volumeGroup) Destroy(ctx context.Context) { // free the volumes that were allocated in GetVolumeGroup() @@ -288,22 +148,7 @@ func (vg *volumeGroup) Destroy(ctx context.Context) { vg.volumesToFree = make([]types.Volume, 0) } - if vg.ioctx != nil { - vg.ioctx.Destroy() - vg.ioctx = nil - } - - if vg.conn != nil { - vg.conn.Destroy() - vg.conn = nil - } - - if vg.credentials != nil { - vg.credentials.DeleteCredentials() - vg.credentials = nil - } - - log.DebugLog(ctx, "destroyed volume group instance with id %q", vg.id) + vg.commonVolumeGroup.Destroy(ctx) } func (vg *volumeGroup) Create(ctx context.Context) error {