mirrors/ceph-csi
1
0
Fork 0
mirror of https://github.com/ceph/ceph-csi.git synced 2025-06-13 02:33:34 +00:00
Code Issues Packages Projects Releases Wiki Activity

rbd: add new controller to regenerate omap data

Browse Source 
In the case of Disaster Recovery failover, the
user expected to create the static PVC's. We have
planned not to go with the PVC name and namespace
for many reasons (as in kubernetes it's planned to
support PVC transfer to a new namespace with a
different name and with new features coming in
like data populator etc). For now, we are
planning to go with static PVC's to support
async mirroring.

During Async mirroring only the RBD images are
mirrored to the secondary site, and when the
user creates the static PVC's on the failover
we need to regenerate the omap data. The
volumeHandler in PV spec is an encoded string
which contains clusterID and poolID and image UUID,
The clusterID and poolID won't remain same on both
the clusters, for that cephcsi need to generate the
new volume handler and its to create a mapping
between new volume handler and old volume handler
with that whenever cephcsi gets csi requests it
check if the mapping exists it will pull the new
volume handler and continues other operations.

The new controller watches for the PVs created,
It checks if the omap exists if it doesn't it
will regenerate the entire omap data.

Signed-off-by: Madhu Rajanna <madhupr007@gmail.com>
This commit is contained in:
Madhu Rajanna
2020-10-21 18:19:45 +05:30
committed by mergify[bot]
parent 5af3fe5deb
commit 68bd44beba
5 changed files with 471 additions and 21 deletions
Download Patch File Download Diff File Expand all files Collapse all files

148
internal/rbd/rbd_journal.go
View File

@ -21,6 +21,7 @@ import (
"errors"
"fmt"
"github.com/ceph/ceph-csi/internal/journal"
"github.com/ceph/ceph-csi/internal/util"
)
@ -466,3 +467,150 @@ func undoVolReservation(ctx context.Context, rbdVol *rbdVolume, cr *util.Credent
return err
}
// RegenerateJournal regenerates the omap data for the static volumes, the
// input parameters imageName, volumeID, pool, journalPool, requestName will be
// present in the PV.Spec.CSI object based on that we can regenerate the
// complete omap mapping between imageName and volumeID.
// RegenerateJournal performs below operations
// Extract information from volumeID
// Get pool ID from pool name
// Extract uuid from volumeID
// Reserve omap data
// Generate new volume Handler
// Create old volumeHandler to new handler mapping
// The volume handler wont remain same as its contains poolID,clusterID etc
// which are not same across clusters.
func RegenerateJournal(imageName, volumeID, pool, journalPool, requestName string, cr *util.Credentials) error {
ctx := context.Background()
var (
options map[string]string
vi util.CSIIdentifier
rbdVol *rbdVolume
)
options = make(map[string]string)
rbdVol = &rbdVolume{VolID: volumeID}
err := vi.DecomposeCSIID(rbdVol.VolID)
if err != nil {
return fmt.Errorf("%w: error decoding volume ID (%s) (%s)",
ErrInvalidVolID, err, rbdVol.VolID)
}
// TODO check clusterID mapping exists
rbdVol.ClusterID = vi.ClusterID
options["clusterID"] = rbdVol.ClusterID
rbdVol.Monitors, _, err = util.GetMonsAndClusterID(options)
if err != nil {
util.ErrorLog(ctx, "failed getting mons (%s)", err)
return err
}
rbdVol.Pool = pool
err = rbdVol.Connect(cr)
if err != nil {
return err
}
rbdVol.JournalPool = journalPool
if rbdVol.JournalPool == "" {
rbdVol.JournalPool = rbdVol.Pool
}
volJournal = journal.NewCSIVolumeJournal("default")
j, err := volJournal.Connect(rbdVol.Monitors, rbdVol.RadosNamespace, cr)
if err != nil {
return err
}
defer j.Destroy()
journalPoolID, imagePoolID, err := util.GetPoolIDs(ctx, rbdVol.Monitors, rbdVol.JournalPool, rbdVol.Pool, cr)
if err != nil {
return err
}
rbdVol.RequestName = requestName
// TODO add Nameprefix also
kmsID := ""
imageData, err := j.CheckReservation(
ctx, rbdVol.JournalPool, rbdVol.RequestName, rbdVol.NamePrefix, "", kmsID)
if err != nil {
return err
}
if imageData != nil {
rbdVol.ReservedID = imageData.ImageUUID
rbdVol.ImageID = imageData.ImageAttributes.ImageID
if rbdVol.ImageID == "" {
err = rbdVol.getImageID()
if err != nil {
util.ErrorLog(ctx, "failed to get image id %s: %v", rbdVol, err)
return err
}
err = j.StoreImageID(ctx, rbdVol.JournalPool, rbdVol.ReservedID, rbdVol.ImageID)
if err != nil {
util.ErrorLog(ctx, "failed to store volume id %s: %v", rbdVol, err)
return err
}
}
err = rbdVol.addNewUUIDMapping(ctx, imagePoolID, j)
if err != nil {
util.ErrorLog(ctx, "failed to add UUID mapping %s: %v", rbdVol, err)
return err
}
}
rbdVol.ReservedID, rbdVol.RbdImageName, err = j.ReserveName(
ctx, rbdVol.JournalPool, journalPoolID, rbdVol.Pool, imagePoolID,
rbdVol.RequestName, rbdVol.NamePrefix, "", kmsID, vi.ObjectUUID)
if err != nil {
return err
}
rbdVol.VolID, err = util.GenerateVolID(ctx, rbdVol.Monitors, cr, imagePoolID, rbdVol.Pool,
rbdVol.ClusterID, rbdVol.ReservedID, volIDVersion)
if err != nil {
return err
}
util.DebugLog(ctx, "re-generated Volume ID (%s) and image name (%s) for request name (%s)",
rbdVol.VolID, rbdVol.RbdImageName, rbdVol.RequestName)
if rbdVol.ImageID == "" {
err = rbdVol.getImageID()
if err != nil {
util.ErrorLog(ctx, "failed to get image id %s: %v", rbdVol, err)
return err
}
err = j.StoreImageID(ctx, rbdVol.JournalPool, rbdVol.ReservedID, rbdVol.ImageID)
if err != nil {
util.ErrorLog(ctx, "failed to store volume id %s: %v", rbdVol, err)
return err
}
}
if volumeID != rbdVol.VolID {
return j.ReserveNewUUIDMapping(ctx, rbdVol.JournalPool, volumeID, rbdVol.VolID)
}
return nil
}
// addNewUUIDMapping creates the mapping between two volumeID.
func (rv *rbdVolume) addNewUUIDMapping(ctx context.Context, imagePoolID int64, j *journal.Connection) error {
var err error
volID := ""
id, err := j.CheckNewUUIDMapping(ctx, rv.JournalPool, rv.VolID)
if err == nil && id == "" {
volID, err = util.GenerateVolID(ctx, rv.Monitors, rv.conn.Creds, imagePoolID, rv.Pool,
rv.ClusterID, rv.ReservedID, volIDVersion)
if err != nil {
return err
}
if rv.VolID == volID {
return nil
}
return j.ReserveNewUUIDMapping(ctx, rv.JournalPool, rv.VolID, volID)
}
return err
}

46
internal/rbd/rbd_util.go
View File

@ -663,6 +663,7 @@ func genVolFromVolID(ctx context.Context, volumeID string, cr *util.Credentials,
options map[string]string
vi util.CSIIdentifier
rbdVol *rbdVolume
err error
)
options = make(map[string]string)
@ -670,12 +671,14 @@ func genVolFromVolID(ctx context.Context, volumeID string, cr *util.Credentials,
// Mounter, MultiNodeWritable
rbdVol = &rbdVolume{VolID: volumeID}
err := vi.DecomposeCSIID(rbdVol.VolID)
err = vi.DecomposeCSIID(rbdVol.VolID)
if err != nil {
return rbdVol, fmt.Errorf("%w: error decoding volume ID (%s) (%s)",
ErrInvalidVolID, err, rbdVol.VolID)
}
// TODO check clusterID mapping exists
rbdVol.ClusterID = vi.ClusterID
options["clusterID"] = rbdVol.ClusterID
@ -685,17 +688,6 @@ func genVolFromVolID(ctx context.Context, volumeID string, cr *util.Credentials,
return rbdVol, err
}
rbdVol.Pool, err = util.GetPoolName(rbdVol.Monitors, cr, vi.LocationID)
if err != nil {
return rbdVol, err
}
err = rbdVol.Connect(cr)
if err != nil {
return rbdVol, err
}
rbdVol.JournalPool = rbdVol.Pool
rbdVol.RadosNamespace, err = util.RadosNamespace(util.CsiConfigFile, rbdVol.ClusterID)
if err != nil {
return nil, err
@ -707,6 +699,31 @@ func genVolFromVolID(ctx context.Context, volumeID string, cr *util.Credentials,
}
defer j.Destroy()
// check is there any volumeID mapping exists.
id, err := j.CheckNewUUIDMapping(ctx, rbdVol.JournalPool, volumeID)
if err != nil {
return rbdVol, fmt.Errorf("failed to get volume id %s mapping %w",
volumeID, err)
}
if id != "" {
rbdVol.VolID = id
err = vi.DecomposeCSIID(rbdVol.VolID)
if err != nil {
return rbdVol, fmt.Errorf("%w: error decoding volume ID (%s) (%s)",
ErrInvalidVolID, err, rbdVol.VolID)
}
}
rbdVol.Pool, err = util.GetPoolName(rbdVol.Monitors, cr, vi.LocationID)
if err != nil {
return rbdVol, err
}
err = rbdVol.Connect(cr)
if err != nil {
return rbdVol, err
}
rbdVol.JournalPool = rbdVol.Pool
imageAttributes, err := j.GetImageAttributes(
ctx, rbdVol.Pool, vi.ObjectUUID, false)
if err != nil {
@ -982,10 +999,7 @@ type imageInfo struct {
// parentInfo spec for parent volume info.
type mirroring struct {
Mode string `json:"mode"`
State string `json:"state"`
GlobalID string `json:"global_id"`
Primary bool `json:"primary"`
Primary bool `json:"primary"`
}
// updateVolWithImageInfo updates provided rbdVolume with information from on-disk data