From fc0d6f6b8b1461ddec596a090719172224856bfe Mon Sep 17 00:00:00 2001
From: Madhu Rajanna <madhupr007@gmail.com>
Date: Thu, 12 Aug 2021 12:35:56 +0530
Subject: [PATCH] rbd: return succuss if image is healthy secondary

If the image is in secondary state and its
up+replaying means its an healthy secondary
and the image is primary somewhere in the remote
cluster and the local image is getting replayed.
Return success for the Disabling mirroring as
we cannot disable the mirroring on the secondary
state, when the image on the remote site gets
disabled the image on all the remote (secondary)
will get auto deleted. This helps in garbage
collecting the volume replication kuberentes
artifacts

Signed-off-by: Madhu Rajanna <madhupr007@gmail.com>
---
 internal/rbd/replicationcontrollerserver.go | 69 ++++++++++++++-------
 1 file changed, 47 insertions(+), 22 deletions(-)

diff --git a/internal/rbd/replicationcontrollerserver.go b/internal/rbd/replicationcontrollerserver.go
index 5a177ad4f..feb656cb4 100644
--- a/internal/rbd/replicationcontrollerserver.go
+++ b/internal/rbd/replicationcontrollerserver.go
@@ -321,28 +321,7 @@ func (rs *ReplicationServer) DisableVolumeReplication(ctx context.Context,
 	case librbd.MirrorImageDisabling:
 		return nil, status.Errorf(codes.Aborted, "%s is in disabling state", volumeID)
 	case librbd.MirrorImageEnabled:
-		if !force && !mirroringInfo.Primary {
-			return nil, status.Error(codes.InvalidArgument, "image is in non-primary state")
-		}
-		err = rbdVol.disableImageMirroring(force)
-		if err != nil {
-			util.ErrorLog(ctx, err.Error())
-
-			return nil, status.Error(codes.Internal, err.Error())
-		}
-		// the image state can be still disabling once we disable the mirroring
-		// check the mirroring is disabled or not
-		mirroringInfo, err = rbdVol.getImageMirroringInfo()
-		if err != nil {
-			util.ErrorLog(ctx, err.Error())
-
-			return nil, status.Error(codes.Internal, err.Error())
-		}
-		if mirroringInfo.State == librbd.MirrorImageDisabling {
-			return nil, status.Errorf(codes.Aborted, "%s is in disabling state", volumeID)
-		}
-
-		return &replication.DisableVolumeReplicationResponse{}, nil
+		return disableVolumeReplication(rbdVol, mirroringInfo, force)
 	default:
 		// TODO: use string instead of int for returning valid error message
 		return nil, status.Errorf(codes.InvalidArgument, "image is in %d Mode", mirroringInfo.State)
@@ -351,6 +330,52 @@ func (rs *ReplicationServer) DisableVolumeReplication(ctx context.Context,
 	return &replication.DisableVolumeReplicationResponse{}, nil
 }
 
+func disableVolumeReplication(rbdVol *rbdVolume,
+	mirroringInfo *librbd.MirrorImageInfo,
+	force bool) (*replication.DisableVolumeReplicationResponse, error) {
+	if !mirroringInfo.Primary {
+		// Return success if the below condition is met
+		// Local image is secondary
+		// Local image is in up+replaying state
+
+		// If the image is in a secondary and its state is  up+replaying means
+		// its an healthy secondary and the image is primary somewhere in the
+		// remote cluster and the local image is getting replayed. Return
+		// success for the Disabling mirroring as we cannot disable mirroring
+		// on the secondary image, when the image on the primary site gets
+		// disabled the image on all the remote (secondary) clusters will get
+		// auto-deleted. This helps in garbage collecting the volume
+		// replication Kubernetes artifacts after failback operation.
+		localStatus, rErr := rbdVol.getLocalState()
+		if rErr != nil {
+			return nil, status.Error(codes.Internal, rErr.Error())
+		}
+		if localStatus.Up && localStatus.State == librbd.MirrorImageStatusStateReplaying {
+			return &replication.DisableVolumeReplicationResponse{}, nil
+		}
+
+		return nil, status.Errorf(codes.InvalidArgument,
+			"secondary image status is up=%t and state=%s",
+			localStatus.Up,
+			localStatus.State)
+	}
+	err := rbdVol.disableImageMirroring(force)
+	if err != nil {
+		return nil, status.Error(codes.Internal, err.Error())
+	}
+	// the image state can be still disabling once we disable the mirroring
+	// check the mirroring is disabled or not
+	mirroringInfo, err = rbdVol.getImageMirroringInfo()
+	if err != nil {
+		return nil, status.Error(codes.Internal, err.Error())
+	}
+	if mirroringInfo.State == librbd.MirrorImageDisabling {
+		return nil, status.Errorf(codes.Aborted, "%s is in disabling state", rbdVol.VolID)
+	}
+
+	return &replication.DisableVolumeReplicationResponse{}, nil
+}
+
 // PromoteVolume extracts the RBD volume information from the volumeID, If the
 // image is present, mirroring is enabled and the image is in demoted state it
 // will promote the volume as primary.