mirror of
https://github.com/ceph/ceph-csi.git
synced 2024-11-09 16:00:22 +00:00
rbd: mark image ready when image state is up+unknown
To recover from split brain (up+error) state the image need to be demoted and requested for resync on site-a and then the image on site-b should gets demoted.The volume should be marked to ready=true when the image state on both the clusters are up+unknown because during the last snapshot syncing the data gets copied first and then image state on the site-a changes to up+unknown. If the image state on both the sites are up+unknown consider that complete data is synced as the last snapshot gets exchanged between the clusters. * create 10 GB of file and validate the data after resync * Do Failover when the site-a goes down * Force promote the image and write data in GiB * Once the site-a comes back, Demote the image and issue resync * Demote the image on site-b * The status will get reflected on the other site when the last snapshot sync happens * The image will go to up+unknown state. and complete data will be copied to site a * Promote the image on site-a and use it ```bash csi-vol-5633715e-a7eb-11eb-bebb-0242ac110006: global_id: e7f9ec55-06ab-46cb-a1ae-784be75ed96d state: up+unknown description: remote image demoted service: a on minicluster1 last_update: 2021-04-28 07:11:56 peer_sites: name: e47e29f4-96e8-44ed-b6c6-edf15c5a91d6-rook-ceph state: up+unknown description: remote image demoted last_update: 2021-04-28 07:11:41 ``` * Do Failover when the site-a goes down * Force promote the image on site-b and write data in GiB * Demote the image on site-b * Once the site-a comes back, Demote the image on site-a * The images on the both site will go to split brain state ```bash csi-vol-37effcb5-a7f1-11eb-bebb-0242ac110006: global_id: 115c3df9-3d4f-4c04-93a7-531b82155ddf state: up+error description: split-brain service: a on minicluster2 last_update: 2021-04-28 07:25:41 peer_sites: name: abbda0f0-0117-4425-8cb2-deb4c853da47-rook-ceph state: up+error description: split-brain last_update: 2021-04-28 07:25:26 ``` * Issue resync * The images cannot be resynced because when we issue resync on site a the image on site-b was in demoted state * To recover from this state (promote and then demote the image on site-b after sometime) ```bash csi-vol-37effcb5-a7f1-11eb-bebb-0242ac110006: global_id: 115c3df9-3d4f-4c04-93a7-531b82155ddf state: up+unknown description: remote image demoted service: a on minicluster1 last_update: 2021-04-28 07:32:56 peer_sites: name: e47e29f4-96e8-44ed-b6c6-edf15c5a91d6-rook-ceph state: up+unknown description: remote image demoted last_update: 2021-04-28 07:32:41 ``` * Once the data is copied we can see that the image state is moved to up+unknown on both sites * Promote the image on site-a and use it Signed-off-by: Madhu Rajanna <madhupr007@gmail.com>
This commit is contained in:
parent
90fd12629f
commit
07a916b84d
@ -43,15 +43,6 @@ const (
|
||||
type imageMirroringState string
|
||||
|
||||
const (
|
||||
// If the state is up+replaying, then mirroring is functioning properly.
|
||||
// up means the rbd-mirror daemon is running, and replaying means
|
||||
// this image is the target for replication from another storage cluster.
|
||||
upAndReplaying imageMirroringState = "up+replaying"
|
||||
// If the state is up+stopped means the rbd-mirror daemon is
|
||||
// running and stopped means the image is not a target for replication from
|
||||
// another cluster
|
||||
upAndStopped imageMirroringState = "up+stopped"
|
||||
|
||||
// If the state is up+unknown means the rbd-mirror daemon is
|
||||
// running and the image is demoted on both the clusters.
|
||||
upAndUnknown imageMirroringState = "up+unknown"
|
||||
@ -445,20 +436,16 @@ func (rs *ReplicationServer) ResyncVolume(ctx context.Context,
|
||||
}
|
||||
ready := false
|
||||
state := imageMirroringState(mirrorStatus.State)
|
||||
if state == upAndStopped || state == upAndReplaying {
|
||||
// Make sure the peer site image state is up and stopped
|
||||
ready = true
|
||||
for _, s := range mirrorStatus.PeerSites {
|
||||
if imageMirroringState(s.State) != upAndStopped {
|
||||
util.UsefulLog(ctx, "peer site name=%s, mirroring state=%s, description=%s and lastUpdate=%s", s.SiteName, s.State, s.Description, s.LastUpdate)
|
||||
ready = false
|
||||
}
|
||||
}
|
||||
}
|
||||
// To recover from split brain (up+error) state the image need to be
|
||||
// demoted and requested for resync on site-a and then the image on site-b
|
||||
// should be demoted. The volume should be marked to ready=true when the
|
||||
// image state on both the clusters are up+unknown because during the last
|
||||
// snapshot syncing the data gets copied first and then image state on the
|
||||
// site-a changes to up+unknown.
|
||||
|
||||
// when the images are demoted on both clusters and user requests for the
|
||||
// resync of the image, the image mirror state will be unknown state in
|
||||
// both clusters.
|
||||
// If the image state on both the sites are up+unknown consider that
|
||||
// complete data is synced as the last snapshot
|
||||
// gets exchanged between the clusters.
|
||||
if state == upAndUnknown {
|
||||
ready = true
|
||||
for _, s := range mirrorStatus.PeerSites {
|
||||
|
Loading…
Reference in New Issue
Block a user