From 64ce5e0949da0e1684456f56d18f89f5850076b1 Mon Sep 17 00:00:00 2001
From: Madhu Rajanna <madhupr007@gmail.com>
Date: Tue, 30 Nov 2021 11:46:34 +0530
Subject: [PATCH] rbd: check local image state during promote operation

rbd mirroring CLI calls are async and it doesn't wait
for the operation to be completed. ex:- `rbd mirror image enable`
it will enable the mirroring on the image but it doesn't
ensure that the image is mirroring enabled and healthy
primary. The same goes for the promote volume also.
This commits adds a check-in PromoteVolume to make sure
the image in a healthy state i.e `up+stopped`.

note:- not considering any intermediate states to make
sure the image is completely healthy before responding
success to the RPC call.

Signed-off-by: Madhu Rajanna <madhupr007@gmail.com>
---
 internal/rbd/errors.go                      |  2 ++
 internal/rbd/replicationcontrollerserver.go | 34 +++++++++++++++++++++
 2 files changed, 36 insertions(+)

diff --git a/internal/rbd/errors.go b/internal/rbd/errors.go
index 2443b8797..32937c058 100644
--- a/internal/rbd/errors.go
+++ b/internal/rbd/errors.go
@@ -42,4 +42,6 @@ var (
 	ErrMissingImageNameInVolID = errors.New("rbd image name information can not be empty in volID")
 	// ErrDecodeClusterIDFromMonsInVolID is returned when mons hash decoding on migration volID.
 	ErrDecodeClusterIDFromMonsInVolID = errors.New("failed to get clusterID from monitors hash in volID")
+	// ErrUnHealthyMirroredImage is returned when mirrored image is not healthy.
+	ErrUnHealthyMirroredImage = errors.New("mirrored image is not healthy")
 )
diff --git a/internal/rbd/replicationcontrollerserver.go b/internal/rbd/replicationcontrollerserver.go
index 8def27e00..11a8bd906 100644
--- a/internal/rbd/replicationcontrollerserver.go
+++ b/internal/rbd/replicationcontrollerserver.go
@@ -536,6 +536,11 @@ func (rs *ReplicationServer) PromoteVolume(ctx context.Context,
 		}
 	}
 
+	err = checkHealthyPrimary(ctx, rbdVol)
+	if err != nil {
+		return nil, status.Error(codes.Internal, err.Error())
+	}
+
 	var mode librbd.ImageMirrorMode
 	mode, err = getMirroringMode(ctx, req.GetParameters())
 	if err != nil {
@@ -565,6 +570,35 @@ func (rs *ReplicationServer) PromoteVolume(ctx context.Context,
 	return &replication.PromoteVolumeResponse{}, nil
 }
 
+// checkHealthyPrimary checks if the image is a healhty primary or not.
+// healthy primary image will be in up+stopped state, for states other
+// than this it returns an error message.
+func checkHealthyPrimary(ctx context.Context, rbdVol *rbdVolume) error {
+	mirrorStatus, err := rbdVol.getImageMirroringStatus()
+	if err != nil {
+		return err
+	}
+	localStatus, err := mirrorStatus.LocalStatus()
+	if err != nil {
+		// LocalStatus can fail if the local site status is not found in
+		// mirroring status. Log complete sites status to debug why getting
+		// local status failed
+		log.ErrorLog(ctx, "mirroring status is %+v", mirrorStatus)
+
+		return fmt.Errorf("failed to get local status: %w", err)
+	}
+
+	if !localStatus.Up && localStatus.State != librbd.MirrorImageStatusStateStopped {
+		return fmt.Errorf("%s %w. State is up=%t, state=%q",
+			rbdVol,
+			ErrUnHealthyMirroredImage,
+			localStatus.Up,
+			localStatus.State)
+	}
+
+	return nil
+}
+
 // DemoteVolume extracts the RBD volume information from the
 // volumeID, If the image is present, mirroring is enabled and the
 // image is in promoted state it will demote the volume as secondary.