rbd: refractor to use mirror interface

Refractoring code to use mirror
interface.

Signed-off-by: Madhu Rajanna <madhupr007@gmail.com>

internal/rbd/controllerserver.go

@@ -988,7 +988,7 @@ func (cs *ControllerServer) DeleteVolume(
 func cleanupRBDImage(ctx context.Context,
 	rbdVol *rbdVolume, cr *util.Credentials,
 ) (*csi.DeleteVolumeResponse, error) {
-	mirroringInfo, err := rbdVol.GetImageMirroringInfo()
+	info, err := rbdVol.GetMirroringInfo()
 	if err != nil {
 		log.ErrorLog(ctx, err.Error())
 
@@ -998,7 +998,7 @@ func cleanupRBDImage(ctx context.Context,
 	// Mirroring is enabled on the image
 	// Local image is secondary
 	// Local image is in up+replaying state
-	if mirroringInfo.State == librbd.MirrorImageEnabled && !mirroringInfo.Primary {
+	if info.GetState() == librbd.MirrorImageEnabled.String() && !info.IsPrimary() {
 		// If the image is in a secondary state and its up+replaying means its
 		// an healthy secondary and the image is primary somewhere in the
 		// remote cluster and the local image is getting replayed. Delete the
@@ -1007,11 +1007,18 @@ func cleanupRBDImage(ctx context.Context,
 		// the image on all the remote (secondary) clusters will get
 		// auto-deleted. This helps in garbage collecting the OMAP, PVC and PV
 		// objects after failback operation.
-		localStatus, rErr := rbdVol.GetLocalState()
+		sts, rErr := rbdVol.GetGlobalMirroringStatus()
 		if rErr != nil {
 			return nil, status.Error(codes.Internal, rErr.Error())
 		}
-		if localStatus.Up && localStatus.State == librbd.MirrorImageStatusStateReplaying {
+
+		localStatus, rErr := sts.GetLocalSiteStatus()
+		if rErr != nil {
+			log.ErrorLog(ctx, "failed to get local status for volume %s: %w", rbdVol.RbdImageName, rErr)
+
+			return nil, status.Error(codes.Internal, rErr.Error())
+		}
+		if localStatus.IsUP() && localStatus.GetState() == librbd.MirrorImageStatusStateReplaying.String() {
 			if err = undoVolReservation(ctx, rbdVol, cr); err != nil {
 				log.ErrorLog(ctx, "failed to remove reservation for volume (%s) with backing image (%s) (%s)",
 					rbdVol.RequestName, rbdVol.RbdImageName, err)
@@ -1023,8 +1030,8 @@ func cleanupRBDImage(ctx context.Context,
 		}
 		log.ErrorLog(ctx,
 			"secondary image status is up=%t and state=%s",
-			localStatus.Up,
-			localStatus.State)
+			localStatus.IsUP(),
+			localStatus.GetState())
 	}
 
 	inUse, err := rbdVol.isInUse()

internal/rbd/driver/driver.go

@@ -219,7 +219,7 @@ func (r *Driver) setupCSIAddonsServer(conf *util.Config) error {
 		fcs := casrbd.NewFenceControllerServer()
 		r.cas.RegisterService(fcs)
 
-		rcs := casrbd.NewReplicationServer(NewControllerServer(r.cd))
+		rcs := casrbd.NewReplicationServer(rbd.CSIInstanceID, NewControllerServer(r.cd))
 		r.cas.RegisterService(rcs)
 
 		vgcs := casrbd.NewVolumeGroupServer(conf.InstanceID)

internal/rbd/group.go

@@ -77,3 +77,7 @@ func (rv *rbdVolume) RemoveFromGroup(ctx context.Context, vg types.VolumeGroup)
 
 	return librbd.GroupImageRemove(ioctx, name, rv.ioctx, rv.RbdImageName)
 }
+
+func (rv *rbdVolume) ToMirror() (types.Mirror, error) {
+	return rv, nil
+}

internal/rbd/mirror.go

@@ -20,21 +20,13 @@ import (
 	"fmt"
 	"time"
 
+	"github.com/ceph/ceph-csi/internal/rbd/types"
 	"github.com/ceph/ceph-csi/internal/util"
+	"github.com/ceph/ceph-csi/internal/util/log"
 
 	librbd "github.com/ceph/go-ceph/rbd"
 )
 
-// FlattenMode is used to indicate the flatten mode for an RBD image.
-type FlattenMode string
-
-const (
-	// FlattenModeNever indicates that the image should never be flattened.
-	FlattenModeNever FlattenMode = "never"
-	// FlattenModeForce indicates that the image with the parent must be flattened.
-	FlattenModeForce FlattenMode = "force"
-)
-
 // HandleParentImageExistence checks the image's parent.
 // if the parent image does not exist and is not in trash, it returns nil.
 // if the flattenMode is FlattenModeForce, it flattens the image itself.
@@ -42,13 +34,12 @@ const (
 // if the parent image exists and is not enabled for mirroring, it returns an error.
 func (rv *rbdVolume) HandleParentImageExistence(
 	ctx context.Context,
-	flattenMode FlattenMode,
+	mode types.FlattenMode,
 ) error {
 	if rv.ParentName == "" && !rv.ParentInTrash {
 		return nil
 	}
-
-	if flattenMode == FlattenModeForce {
+	if mode == types.FlattenModeForce {
 		// Delete temp image that exists for volume datasource since
 		// it is no longer required when the live image is flattened.
 		err := rv.DeleteTempImage(ctx)
@@ -72,14 +63,13 @@ func (rv *rbdVolume) HandleParentImageExistence(
 	if err != nil {
 		return err
 	}
-	parentMirroringInfo, err := parent.GetImageMirroringInfo()
+	parentMirroringInfo, err := parent.GetMirroringInfo()
 	if err != nil {
 		return fmt.Errorf(
 			"failed to get mirroring info of parent %q of image %q: %w",
 			parent, rv, err)
 	}
-
-	if parentMirroringInfo.State != librbd.MirrorImageEnabled {
+	if parentMirroringInfo.GetState() != librbd.MirrorImageEnabled.String() {
 		return fmt.Errorf("%w: failed to enable mirroring on image %q: "+
 			"parent image %q is not enabled for mirroring",
 			ErrFailedPrecondition, rv, parent)
@@ -88,8 +78,11 @@ func (rv *rbdVolume) HandleParentImageExistence(
 	return nil
 }
 
-// EnableImageMirroring enables mirroring on an image.
-func (ri *rbdImage) EnableImageMirroring(mode librbd.ImageMirrorMode) error {
+// check that rbdVolume implements the types.Mirror interface.
+var _ types.Mirror = &rbdVolume{}
+
+// EnableMirroring enables mirroring on an image.
+func (ri *rbdImage) EnableMirroring(mode librbd.ImageMirrorMode) error {
 	image, err := ri.open()
 	if err != nil {
 		return fmt.Errorf("failed to open image %q with error: %w", ri, err)
@@ -104,8 +97,8 @@ func (ri *rbdImage) EnableImageMirroring(mode librbd.ImageMirrorMode) error {
 	return nil
 }
 
-// DisableImageMirroring disables mirroring on an image.
-func (ri *rbdImage) DisableImageMirroring(force bool) error {
+// DisableMirroring disables mirroring on an image.
+func (ri *rbdImage) DisableMirroring(force bool) error {
 	image, err := ri.open()
 	if err != nil {
 		return fmt.Errorf("failed to open image %q with error: %w", ri, err)
@@ -120,8 +113,8 @@ func (ri *rbdImage) DisableImageMirroring(force bool) error {
 	return nil
 }
 
-// GetImageMirroringInfo gets mirroring information of an image.
-func (ri *rbdImage) GetImageMirroringInfo() (*librbd.MirrorImageInfo, error) {
+// GetMirroringInfo gets mirroring information of an image.
+func (ri *rbdImage) GetMirroringInfo() (types.MirrorInfo, error) {
 	image, err := ri.open()
 	if err != nil {
 		return nil, fmt.Errorf("failed to open image %q with error: %w", ri, err)
@@ -133,11 +126,11 @@ func (ri *rbdImage) GetImageMirroringInfo() (*librbd.MirrorImageInfo, error) {
 		return nil, fmt.Errorf("failed to get mirroring info of %q with error: %w", ri, err)
 	}
 
-	return info, nil
+	return ImageStatus{MirrorImageInfo: info}, nil
 }
 
-// PromoteImage promotes image to primary.
-func (ri *rbdImage) PromoteImage(force bool) error {
+// Promote promotes image to primary.
+func (ri *rbdImage) Promote(force bool) error {
 	image, err := ri.open()
 	if err != nil {
 		return fmt.Errorf("failed to open image %q with error: %w", ri, err)
@@ -151,10 +144,10 @@ func (ri *rbdImage) PromoteImage(force bool) error {
 	return nil
 }
 
-// ForcePromoteImage promotes image to primary with force option with 2 minutes
+// ForcePromote promotes image to primary with force option with 2 minutes
 // timeout. If there is no response within 2 minutes,the rbd CLI process will be
 // killed and an error is returned.
-func (rv *rbdVolume) ForcePromoteImage(cr *util.Credentials) error {
+func (rv *rbdVolume) ForcePromote(cr *util.Credentials) error {
 	promoteArgs := []string{
 		"mirror", "image", "promote",
 		rv.String(),
@@ -181,8 +174,8 @@ func (rv *rbdVolume) ForcePromoteImage(cr *util.Credentials) error {
 	return nil
 }
 
-// DemoteImage demotes image to secondary.
-func (ri *rbdImage) DemoteImage() error {
+// Demote demotes image to secondary.
+func (ri *rbdImage) Demote() error {
 	image, err := ri.open()
 	if err != nil {
 		return fmt.Errorf("failed to open image %q with error: %w", ri, err)
@@ -196,8 +189,8 @@ func (ri *rbdImage) DemoteImage() error {
 	return nil
 }
 
-// resyncImage resync image to correct the split-brain.
-func (ri *rbdImage) resyncImage() error {
+// Resync resync image to correct the split-brain.
+func (ri *rbdImage) Resync() error {
 	image, err := ri.open()
 	if err != nil {
 		return fmt.Errorf("failed to open image %q with error: %w", ri, err)
@@ -208,11 +201,14 @@ func (ri *rbdImage) resyncImage() error {
 		return fmt.Errorf("failed to resync image %q with error: %w", ri, err)
 	}
 
-	return nil
+	// If we issued a resync, return a non-final error as image needs to be recreated
+	// locally. Caller retries till RBD syncs an initial version of the image to
+	// report its status in the resync request.
+	return fmt.Errorf("%w: awaiting initial resync due to split brain", ErrUnavailable)
 }
 
-// GetImageMirroringStatus get the mirroring status of an image.
-func (ri *rbdImage) GetImageMirroringStatus() (*librbd.GlobalMirrorImageStatus, error) {
+// GetGlobalMirroringStatus get the mirroring status of an image.
+func (ri *rbdImage) GetGlobalMirroringStatus() (types.GlobalStatus, error) {
 	image, err := ri.open()
 	if err != nil {
 		return nil, fmt.Errorf("failed to open image %q with error: %w", ri, err)
@@ -223,26 +219,110 @@ func (ri *rbdImage) GetImageMirroringStatus() (*librbd.GlobalMirrorImageStatus,
 		return nil, fmt.Errorf("failed to get image mirroring status %q with error: %w", ri, err)
 	}
 
-	return &statusInfo, nil
+	return GlobalMirrorStatus{GlobalMirrorImageStatus: statusInfo}, nil
 }
 
-// GetLocalState returns the local state of the image.
-func (ri *rbdImage) GetLocalState() (librbd.SiteMirrorImageStatus, error) {
-	localStatus := librbd.SiteMirrorImageStatus{}
-	image, err := ri.open()
-	if err != nil {
-		return localStatus, fmt.Errorf("failed to open image %q with error: %w", ri, err)
-	}
-	defer image.Close()
-
-	statusInfo, err := image.GetGlobalMirrorStatus()
-	if err != nil {
-		return localStatus, fmt.Errorf("failed to get image mirroring status %q with error: %w", ri, err)
-	}
-	localStatus, err = statusInfo.LocalStatus()
-	if err != nil {
-		return localStatus, fmt.Errorf("failed to get local status: %w", err)
-	}
-
-	return localStatus, nil
+// ImageStatus is a wrapper around librbd.MirrorImageInfo that contains the
+// image mirror status.
+type ImageStatus struct {
+	*librbd.MirrorImageInfo
+}
+
+func (status ImageStatus) GetState() string {
+	return status.State.String()
+}
+
+func (status ImageStatus) IsPrimary() bool {
+	return status.Primary
+}
+
+// GlobalMirrorStatus is a wrapper around librbd.GlobalMirrorImageStatus that contains the
+// global mirror image status.
+type GlobalMirrorStatus struct {
+	librbd.GlobalMirrorImageStatus
+}
+
+func (status GlobalMirrorStatus) GetState() string {
+	return status.GlobalMirrorImageStatus.Info.State.String()
+}
+
+func (status GlobalMirrorStatus) IsPrimary() bool {
+	return status.GlobalMirrorImageStatus.Info.Primary
+}
+
+func (status GlobalMirrorStatus) GetLocalSiteStatus() (types.SiteStatus, error) {
+	s, err := status.GlobalMirrorImageStatus.LocalStatus()
+	if err != nil {
+		err = fmt.Errorf("failed to get local site status: %w", err)
+	}
+
+	return SiteMirrorImageStatus{
+		SiteMirrorImageStatus: s,
+	}, err
+}
+
+func (status GlobalMirrorStatus) GetAllSitesStatus() []types.SiteStatus {
+	var siteStatuses []types.SiteStatus
+	for _, ss := range status.SiteStatuses {
+		siteStatuses = append(siteStatuses, SiteMirrorImageStatus{SiteMirrorImageStatus: ss})
+	}
+
+	return siteStatuses
+}
+
+// RemoteStatus returns one SiteMirrorImageStatus item from the SiteStatuses
+// slice that corresponds to the remote site's status. If the remote status
+// is not found than the error ErrNotExist will be returned.
+func (status GlobalMirrorStatus) GetRemoteSiteStatus(ctx context.Context) (types.SiteStatus, error) {
+	var (
+		ss  librbd.SiteMirrorImageStatus
+		err error = librbd.ErrNotExist
+	)
+
+	for i := range status.SiteStatuses {
+		log.DebugLog(
+			ctx,
+			"Site status of MirrorUUID: %s, state: %s, description: %s, lastUpdate: %v, up: %t",
+			status.SiteStatuses[i].MirrorUUID,
+			status.SiteStatuses[i].State,
+			status.SiteStatuses[i].Description,
+			status.SiteStatuses[i].LastUpdate,
+			status.SiteStatuses[i].Up)
+
+		if status.SiteStatuses[i].MirrorUUID != "" {
+			ss = status.SiteStatuses[i]
+			err = nil
+
+			break
+		}
+	}
+
+	return SiteMirrorImageStatus{SiteMirrorImageStatus: ss}, err
+}
+
+// SiteMirrorImageStatus is a wrapper around librbd.SiteMirrorImageStatus that contains the
+// site mirror image status.
+type SiteMirrorImageStatus struct {
+	librbd.SiteMirrorImageStatus
+}
+
+func (status SiteMirrorImageStatus) GetMirrorUUID() string {
+	return status.MirrorUUID
+}
+
+func (status SiteMirrorImageStatus) GetState() string {
+	return status.State.String()
+}
+
+func (status SiteMirrorImageStatus) GetDescription() string {
+	return status.Description
+}
+
+func (status SiteMirrorImageStatus) IsUP() bool {
+	return status.Up
+}
+
+func (status SiteMirrorImageStatus) GetLastUpdate() time.Time {
+	// convert the last update time to UTC
+	return time.Unix(status.LastUpdate, 0).UTC()
 }

internal/rbd/rbd_util.go

@@ -413,6 +413,10 @@ func (ri *rbdImage) String() string {
 	return fmt.Sprintf("%s/%s", ri.Pool, ri.RbdImageName)
 }
 
+func (ri *rbdImage) GetPoolName() string {
+	return ri.Pool
+}
+
 // String returns the snap-spec (pool/{namespace/}image@snap) format of the snapshot.
 func (rs *rbdSnapshot) String() string {
 	if rs.RadosNamespace != "" {
@@ -1594,9 +1598,9 @@ func (rv *rbdVolume) setImageOptions(ctx context.Context, options *librbd.ImageO
 	return nil
 }
 
-// GetImageCreationTime returns the creation time of the image. if the image
+// GetCreationTime returns the creation time of the image. if the image
 // creation time is not set, it queries the image info and returns the creation time.
-func (ri *rbdImage) GetImageCreationTime() (*timestamppb.Timestamp, error) {
+func (ri *rbdImage) GetCreationTime() (*timestamppb.Timestamp, error) {
 	if ri.CreatedAt != nil {
 		return ri.CreatedAt, nil
 	}

internal/rbd/replication.go

@@ -20,20 +20,11 @@ import (
 	"context"
 	"fmt"
 
+	"github.com/ceph/ceph-csi/internal/rbd/types"
+
 	librbd "github.com/ceph/go-ceph/rbd"
 )
 
-func (rv *rbdVolume) ResyncVol(localStatus librbd.SiteMirrorImageStatus) error {
-	if err := rv.resyncImage(); err != nil {
-		return fmt.Errorf("failed to resync image: %w", err)
-	}
-
-	// If we issued a resync, return a non-final error as image needs to be recreated
-	// locally. Caller retries till RBD syncs an initial version of the image to
-	// report its status in the resync request.
-	return fmt.Errorf("%w: awaiting initial resync due to split brain", ErrUnavailable)
-}
-
 // repairResyncedImageID updates the existing image ID with new one.
 func (rv *rbdVolume) RepairResyncedImageID(ctx context.Context, ready bool) error {
 	// During resync operation the local image will get deleted and a new
@@ -54,11 +45,11 @@ func (rv *rbdVolume) RepairResyncedImageID(ctx context.Context, ready bool) erro
 	return rv.repairImageID(ctx, j, true)
 }
 
-func (rv *rbdVolume) DisableVolumeReplication(
-	mirroringInfo *librbd.MirrorImageInfo,
+func DisableVolumeReplication(mirror types.Mirror,
+	primary,
 	force bool,
 ) error {
-	if !mirroringInfo.Primary {
+	if !primary {
 		// Return success if the below condition is met
 		// Local image is secondary
 		// Local image is in up+replaying state
@@ -71,29 +62,35 @@ func (rv *rbdVolume) DisableVolumeReplication(
 		// disabled the image on all the remote (secondary) clusters will get
 		// auto-deleted. This helps in garbage collecting the volume
 		// replication Kubernetes artifacts after failback operation.
-		localStatus, rErr := rv.GetLocalState()
+		sts, rErr := mirror.GetGlobalMirroringStatus()
 		if rErr != nil {
-			return fmt.Errorf("failed to get local state: %w", rErr)
+			return fmt.Errorf("failed to get global state: %w", rErr)
 		}
-		if localStatus.Up && localStatus.State == librbd.MirrorImageStatusStateReplaying {
+
+		localStatus, err := sts.GetLocalSiteStatus()
+		if err != nil {
+			return fmt.Errorf("failed to get local state: %w", ErrInvalidArgument)
+		}
+		if localStatus.IsUP() && localStatus.GetState() == librbd.MirrorImageStatusStateReplaying.String() {
 			return nil
 		}
 
 		return fmt.Errorf("%w: secondary image status is up=%t and state=%s",
-			ErrInvalidArgument, localStatus.Up, localStatus.State)
+			ErrInvalidArgument, localStatus.IsUP(), localStatus.GetState())
 	}
-	err := rv.DisableImageMirroring(force)
+	err := mirror.DisableMirroring(force)
 	if err != nil {
 		return fmt.Errorf("failed to disable image mirroring: %w", err)
 	}
 	// the image state can be still disabling once we disable the mirroring
 	// check the mirroring is disabled or not
-	mirroringInfo, err = rv.GetImageMirroringInfo()
+	info, err := mirror.GetMirroringInfo()
 	if err != nil {
 		return fmt.Errorf("failed to get mirroring info of image: %w", err)
 	}
-	if mirroringInfo.State == librbd.MirrorImageDisabling {
-		return fmt.Errorf("%w: %q is in disabling state", ErrAborted, rv.VolID)
+
+	if info.GetState() == librbd.MirrorImageDisabling.String() {
+		return fmt.Errorf("%w: image is in disabling state", ErrAborted)
 	}
 
 	return nil

internal/rbd/types/volume.go

@@ -59,4 +59,7 @@ type Volume interface {
 	// if the parent image is in trash, it returns an error.
 	// if the parent image exists and is not enabled for mirroring, it returns an error.
 	HandleParentImageExistence(ctx context.Context, flattenMode FlattenMode) error
+
+	// ToMirror converts the Volume to a Mirror.
+	ToMirror() (Mirror, error)
 }