mirror of
https://github.com/ceph/ceph-csi.git
synced 2025-06-13 02:33:34 +00:00
Make CephFS plugin stateless reusing RADOS based journal scheme
This is a part of the stateless set of commits for CephCSI. This commit removes the dependency on config maps to store cephFS provisioned volumes, and instead relies on RADOS based objects and keys, and required CSI VolumeID encoding to detect the provisioned volumes. Changes: - Provide backward compatibility to provisioned volumes by older plugin versions (1.0.0 or older) - Remove Create/Delete support for statically provisioned volumes (fixes #382) - Added namespace support to RADOS OMaps and used the same to store RADOS CSI objects and keys in the CephFS metadata pool - Added support to mention fsname for CephFS provisioning (fixes #359) - Changed field name in CSI Identifier to 'location', to denote a pool or fscid - Updated mounter cache to use new scheme - Required Helm manifests are updated - Required documentation and other manifests are updated - Made driver option 'metadatastorage' as optional, as fresh installs do not need to specify the same Testing done: - Create/Mount/Delete PVC - Create/Delete 5 PVCs - Mount version 1.0.0 PVC - Delete version 1.0.0 PV - Mount Statically defined PV/PVC/Pod - Mount Statically defined version 1.0.0 PV/PVC/Pod - Delete Statically defined version 1.0.0 PV/PVC/Pod - Node restart when mounted to test mountcache - Use InstanceID other than 'default' - RBD basic round of tests, as namespace is added to OMaps - csitest against ceph-fs plugin - NOTE: CephFS plugin still does not detect and address already created volumes but of a different size - Test not providing any value to the metadata storage parameter Signed-off-by: ShyamsundarR <srangana@redhat.com>
This commit is contained in:
ShyamsundarR
@ -123,17 +123,22 @@ func GetPoolName(monitors string, adminID string, key string, poolID int64) (str
|
||||
}
|
||||
|
||||
// SetOMapKeyValue sets the given key and value into the provided Ceph omap name
|
||||
func SetOMapKeyValue(monitors, adminID, key, poolName, oMapName, oMapKey, keyValue string) error {
|
||||
func SetOMapKeyValue(monitors, adminID, key, poolName, namespace, oMapName, oMapKey, keyValue string) error {
|
||||
// Command: "rados <options> setomapval oMapName oMapKey keyValue"
|
||||
|
||||
_, _, err := ExecCommand(
|
||||
"rados",
|
||||
args := []string{
|
||||
"-m", monitors,
|
||||
"--id", adminID,
|
||||
"--key="+key,
|
||||
"--key=" + key,
|
||||
"-c", CephConfigPath,
|
||||
"-p", poolName,
|
||||
"setomapval", oMapName, oMapKey, keyValue)
|
||||
"setomapval", oMapName, oMapKey, keyValue,
|
||||
}
|
||||
|
||||
if namespace != "" {
|
||||
args = append(args, "--namespace="+namespace)
|
||||
}
|
||||
|
||||
_, _, err := ExecCommand("rados", args[:]...)
|
||||
if err != nil {
|
||||
klog.Errorf("failed adding key (%s with value %s), to omap (%s) in "+
|
||||
"pool (%s): (%v)", oMapKey, keyValue, oMapName, poolName, err)
|
||||
@ -144,7 +149,7 @@ func SetOMapKeyValue(monitors, adminID, key, poolName, oMapName, oMapKey, keyVal
|
||||
}
|
||||
|
||||
// GetOMapValue gets the value for the given key from the named omap
|
||||
func GetOMapValue(monitors, adminID, key, poolName, oMapName, oMapKey string) (string, error) {
|
||||
func GetOMapValue(monitors, adminID, key, poolName, namespace, oMapName, oMapKey string) (string, error) {
|
||||
// Command: "rados <options> getomapval oMapName oMapKey <outfile>"
|
||||
// No such key: replicapool/csi.volumes.directory.default/csi.volname
|
||||
tmpFile, err := ioutil.TempFile("", "omap-get-")
|
||||
@ -155,14 +160,20 @@ func GetOMapValue(monitors, adminID, key, poolName, oMapName, oMapKey string) (s
|
||||
defer tmpFile.Close()
|
||||
defer os.Remove(tmpFile.Name())
|
||||
|
||||
stdout, stderr, err := ExecCommand(
|
||||
"rados",
|
||||
args := []string{
|
||||
"-m", monitors,
|
||||
"--id", adminID,
|
||||
"--key="+key,
|
||||
"--key=" + key,
|
||||
"-c", CephConfigPath,
|
||||
"-p", poolName,
|
||||
"getomapval", oMapName, oMapKey, tmpFile.Name())
|
||||
"getomapval", oMapName, oMapKey, tmpFile.Name(),
|
||||
}
|
||||
|
||||
if namespace != "" {
|
||||
args = append(args, "--namespace="+namespace)
|
||||
}
|
||||
|
||||
stdout, stderr, err := ExecCommand("rados", args[:]...)
|
||||
if err != nil {
|
||||
// no logs, as attempting to check for non-existent key/value is done even on
|
||||
// regular call sequences
|
||||
@ -189,17 +200,22 @@ func GetOMapValue(monitors, adminID, key, poolName, oMapName, oMapKey string) (s
|
||||
}
|
||||
|
||||
// RemoveOMapKey removes the omap key from the given omap name
|
||||
func RemoveOMapKey(monitors, adminID, key, poolName, oMapName, oMapKey string) error {
|
||||
func RemoveOMapKey(monitors, adminID, key, poolName, namespace, oMapName, oMapKey string) error {
|
||||
// Command: "rados <options> rmomapkey oMapName oMapKey"
|
||||
|
||||
_, _, err := ExecCommand(
|
||||
"rados",
|
||||
args := []string{
|
||||
"-m", monitors,
|
||||
"--id", adminID,
|
||||
"--key="+key,
|
||||
"--key=" + key,
|
||||
"-c", CephConfigPath,
|
||||
"-p", poolName,
|
||||
"rmomapkey", oMapName, oMapKey)
|
||||
"rmomapkey", oMapName, oMapKey,
|
||||
}
|
||||
|
||||
if namespace != "" {
|
||||
args = append(args, "--namespace="+namespace)
|
||||
}
|
||||
|
||||
_, _, err := ExecCommand("rados", args[:]...)
|
||||
if err != nil {
|
||||
// NOTE: Missing omap key removal does not return an error
|
||||
klog.Errorf("failed removing key (%s), from omap (%s) in "+
|
||||
@ -212,17 +228,22 @@ func RemoveOMapKey(monitors, adminID, key, poolName, oMapName, oMapKey string) e
|
||||
|
||||
// CreateObject creates the object name passed in and returns ErrObjectExists if the provided object
|
||||
// is already present in rados
|
||||
func CreateObject(monitors, adminID, key, poolName, objectName string) error {
|
||||
func CreateObject(monitors, adminID, key, poolName, namespace, objectName string) error {
|
||||
// Command: "rados <options> create objectName"
|
||||
|
||||
stdout, _, err := ExecCommand(
|
||||
"rados",
|
||||
args := []string{
|
||||
"-m", monitors,
|
||||
"--id", adminID,
|
||||
"--key="+key,
|
||||
"--key=" + key,
|
||||
"-c", CephConfigPath,
|
||||
"-p", poolName,
|
||||
"create", objectName)
|
||||
"create", objectName,
|
||||
}
|
||||
|
||||
if namespace != "" {
|
||||
args = append(args, "--namespace="+namespace)
|
||||
}
|
||||
|
||||
stdout, _, err := ExecCommand("rados", args[:]...)
|
||||
if err != nil {
|
||||
klog.Errorf("failed creating omap (%s) in pool (%s): (%v)", objectName, poolName, err)
|
||||
if strings.Contains(string(stdout), "error creating "+poolName+"/"+objectName+
|
||||
@ -237,17 +258,22 @@ func CreateObject(monitors, adminID, key, poolName, objectName string) error {
|
||||
|
||||
// RemoveObject removes the entire omap name passed in and returns ErrObjectNotFound is provided omap
|
||||
// is not found in rados
|
||||
func RemoveObject(monitors, adminID, key, poolName, oMapName string) error {
|
||||
func RemoveObject(monitors, adminID, key, poolName, namespace, oMapName string) error {
|
||||
// Command: "rados <options> rm oMapName"
|
||||
|
||||
stdout, _, err := ExecCommand(
|
||||
"rados",
|
||||
args := []string{
|
||||
"-m", monitors,
|
||||
"--id", adminID,
|
||||
"--key="+key,
|
||||
"--key=" + key,
|
||||
"-c", CephConfigPath,
|
||||
"-p", poolName,
|
||||
"rm", oMapName)
|
||||
"rm", oMapName,
|
||||
}
|
||||
|
||||
if namespace != "" {
|
||||
args = append(args, "--namespace="+namespace)
|
||||
}
|
||||
|
||||
stdout, _, err := ExecCommand("rados", args[:]...)
|
||||
if err != nil {
|
||||
klog.Errorf("failed removing omap (%s) in pool (%s): (%v)", oMapName, poolName, err)
|
||||
if strings.Contains(string(stdout), "error removing "+poolName+">"+oMapName+
|
||||
|
||||
@ -102,7 +102,7 @@ func GenerateVolID(monitors, id, key, pool, clusterID, objUUID string, volIDVers
|
||||
|
||||
// generate the volume ID to return to the CO system
|
||||
vi := CSIIdentifier{
|
||||
PoolID: poolID,
|
||||
LocationID: poolID,
|
||||
EncodingVersion: volIDVersion,
|
||||
ClusterID: clusterID,
|
||||
ObjectUUID: objUUID,
|
||||
|
||||
@ -32,8 +32,8 @@ The CSI identifier is composed as elaborated in the comment against ComposeCSIID
|
||||
DecomposeCSIID is the inverse of the same function.
|
||||
|
||||
The CSIIdentifier structure carries the following fields,
|
||||
- PoolID: 64 bit integer of the pool that the volume belongs to, where the ID comes from Ceph pool
|
||||
identifier for the corresponding pool name.
|
||||
- LocationID: 64 bit integer identifier determining the location of the volume on the Ceph cluster.
|
||||
It is the ID of the poolname or fsname, for RBD or CephFS backed volumes respectively.
|
||||
- EncodingVersion: Carries the version number of the encoding scheme used to encode the CSI ID,
|
||||
and is preserved for any future proofing w.r.t changes in the encoding scheme, and to retain
|
||||
ability to parse backward compatible encodings.
|
||||
@ -43,7 +43,7 @@ The CSIIdentifier structure carries the following fields,
|
||||
corresponds to this CSI ID.
|
||||
*/
|
||||
type CSIIdentifier struct {
|
||||
PoolID int64 // TODO: Name appropriately when reused for CephFS
|
||||
LocationID int64
|
||||
EncodingVersion uint16
|
||||
ClusterID string
|
||||
ObjectUUID string
|
||||
@ -87,7 +87,7 @@ func (ci CSIIdentifier) ComposeCSIID() (string, error) {
|
||||
binary.BigEndian.PutUint16(buf16, uint16(len(ci.ClusterID)))
|
||||
clusterIDLength := hex.EncodeToString(buf16)
|
||||
|
||||
binary.BigEndian.PutUint64(buf64, uint64(ci.PoolID))
|
||||
binary.BigEndian.PutUint64(buf64, uint64(ci.LocationID))
|
||||
poolIDEncodedHex := hex.EncodeToString(buf64)
|
||||
|
||||
return strings.Join([]string{versionEncodedHex, clusterIDLength, ci.ClusterID,
|
||||
@ -136,7 +136,7 @@ func (ci *CSIIdentifier) DecomposeCSIID(composedCSIID string) (err error) {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
ci.PoolID = int64(binary.BigEndian.Uint64(buf64))
|
||||
ci.LocationID = int64(binary.BigEndian.Uint64(buf64))
|
||||
// 16 for poolID encoding and 1 for '-' separator
|
||||
bytesToProcess -= 17
|
||||
nextFieldStartIdx = nextFieldStartIdx + 17
|
||||
|
||||
@ -33,7 +33,7 @@ type testTuple struct {
|
||||
var testData = []testTuple{
|
||||
{
|
||||
vID: CSIIdentifier{
|
||||
PoolID: 0xffff,
|
||||
LocationID: 0xffff,
|
||||
EncodingVersion: 0xffff,
|
||||
ClusterID: "01616094-9d93-4178-bf45-c7eac19e8b15",
|
||||
ObjectUUID: "00000000-1111-2222-bbbb-cacacacacaca",
|
||||
|
||||
@ -114,6 +114,9 @@ type CSIJournal struct {
|
||||
|
||||
// volume name prefix for naming on Ceph rbd or FS, suffix is a uuid generated per volume
|
||||
namingPrefix string
|
||||
|
||||
// namespace in which the RADOS objects are stored, default is no namespace
|
||||
namespace string
|
||||
}
|
||||
|
||||
// CSIVolumeJournal returns an instance of volume keys
|
||||
@ -125,6 +128,7 @@ func NewCSIVolumeJournal() *CSIJournal {
|
||||
csiNameKey: "csi.volname",
|
||||
namingPrefix: "csi-vol-",
|
||||
cephSnapSourceKey: "",
|
||||
namespace: "",
|
||||
}
|
||||
}
|
||||
|
||||
@ -137,6 +141,7 @@ func NewCSISnapshotJournal() *CSIJournal {
|
||||
csiNameKey: "csi.snapname",
|
||||
namingPrefix: "csi-snap-",
|
||||
cephSnapSourceKey: "csi.source",
|
||||
namespace: "",
|
||||
}
|
||||
}
|
||||
|
||||
@ -150,6 +155,11 @@ func (cj *CSIJournal) SetCSIDirectorySuffix(suffix string) {
|
||||
cj.csiDirectory = cj.csiDirectory + "." + suffix
|
||||
}
|
||||
|
||||
// SetNamespace sets the namespace in which all RADOS objects would be created
|
||||
func (cj *CSIJournal) SetNamespace(ns string) {
|
||||
cj.namespace = ns
|
||||
}
|
||||
|
||||
/*
|
||||
CheckReservation checks if given request name contains a valid reservation
|
||||
- If there is a valid reservation, then the corresponding UUID for the volume/snapshot is returned
|
||||
@ -177,7 +187,7 @@ func (cj *CSIJournal) CheckReservation(monitors, id, key, pool, reqName, parentN
|
||||
}
|
||||
|
||||
// check if request name is already part of the directory omap
|
||||
objUUID, err := GetOMapValue(monitors, id, key, pool, cj.csiDirectory,
|
||||
objUUID, err := GetOMapValue(monitors, id, key, pool, cj.namespace, cj.csiDirectory,
|
||||
cj.csiNameKeyPrefix+reqName)
|
||||
if err != nil {
|
||||
// error should specifically be not found, for volume to be absent, any other error
|
||||
@ -237,7 +247,7 @@ func (cj *CSIJournal) UndoReservation(monitors, id, key, pool, volName, reqName
|
||||
// delete volume UUID omap (first, inverse of create order)
|
||||
// TODO: Check cases where volName can be empty, and we need to just cleanup the reqName
|
||||
imageUUID := strings.TrimPrefix(volName, cj.namingPrefix)
|
||||
err := RemoveObject(monitors, id, key, pool, cj.cephUUIDDirectoryPrefix+imageUUID)
|
||||
err := RemoveObject(monitors, id, key, pool, cj.namespace, cj.cephUUIDDirectoryPrefix+imageUUID)
|
||||
if err != nil {
|
||||
if _, ok := err.(ErrObjectNotFound); !ok {
|
||||
klog.Errorf("failed removing oMap %s (%s)", cj.cephUUIDDirectoryPrefix+imageUUID, err)
|
||||
@ -246,7 +256,7 @@ func (cj *CSIJournal) UndoReservation(monitors, id, key, pool, volName, reqName
|
||||
}
|
||||
|
||||
// delete the request name key (last, inverse of create order)
|
||||
err = RemoveOMapKey(monitors, id, key, pool, cj.csiDirectory,
|
||||
err = RemoveOMapKey(monitors, id, key, pool, cj.namespace, cj.csiDirectory,
|
||||
cj.csiNameKeyPrefix+reqName)
|
||||
if err != nil {
|
||||
klog.Errorf("failed removing oMap key %s (%s)", cj.csiNameKeyPrefix+reqName, err)
|
||||
@ -259,7 +269,7 @@ func (cj *CSIJournal) UndoReservation(monitors, id, key, pool, volName, reqName
|
||||
// reserveOMapName creates an omap with passed in oMapNamePrefix and a generated <uuid>.
|
||||
// It ensures generated omap name does not already exist and if conflicts are detected, a set
|
||||
// number of retires with newer uuids are attempted before returning an error
|
||||
func reserveOMapName(monitors, id, key, pool, oMapNamePrefix string) (string, error) {
|
||||
func reserveOMapName(monitors, id, key, pool, namespace, oMapNamePrefix string) (string, error) {
|
||||
var iterUUID string
|
||||
|
||||
maxAttempts := 5
|
||||
@ -268,7 +278,7 @@ func reserveOMapName(monitors, id, key, pool, oMapNamePrefix string) (string, er
|
||||
// generate a uuid for the image name
|
||||
iterUUID = uuid.NewUUID().String()
|
||||
|
||||
err := CreateObject(monitors, id, key, pool, oMapNamePrefix+iterUUID)
|
||||
err := CreateObject(monitors, id, key, pool, namespace, oMapNamePrefix+iterUUID)
|
||||
if err != nil {
|
||||
if _, ok := err.(ErrObjectExists); ok {
|
||||
attempt++
|
||||
@ -315,15 +325,15 @@ func (cj *CSIJournal) ReserveName(monitors, id, key, pool, reqName, parentName s
|
||||
// NOTE: If any service loss occurs post creation of the UUID directory, and before
|
||||
// setting the request name key (csiNameKey) to point back to the UUID directory, the
|
||||
// UUID directory key will be leaked
|
||||
volUUID, err := reserveOMapName(monitors, id, key, pool, cj.cephUUIDDirectoryPrefix)
|
||||
volUUID, err := reserveOMapName(monitors, id, key, pool, cj.namespace, cj.cephUUIDDirectoryPrefix)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
// Create request name (csiNameKey) key in csiDirectory and store the UUId based
|
||||
// volume name into it
|
||||
err = SetOMapKeyValue(monitors, id, key, pool, cj.csiDirectory, cj.csiNameKeyPrefix+reqName,
|
||||
volUUID)
|
||||
err = SetOMapKeyValue(monitors, id, key, pool, cj.namespace, cj.csiDirectory,
|
||||
cj.csiNameKeyPrefix+reqName, volUUID)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
@ -339,7 +349,7 @@ func (cj *CSIJournal) ReserveName(monitors, id, key, pool, reqName, parentName s
|
||||
}()
|
||||
|
||||
// Update UUID directory to store CSI request name
|
||||
err = SetOMapKeyValue(monitors, id, key, pool, cj.cephUUIDDirectoryPrefix+volUUID,
|
||||
err = SetOMapKeyValue(monitors, id, key, pool, cj.namespace, cj.cephUUIDDirectoryPrefix+volUUID,
|
||||
cj.csiNameKey, reqName)
|
||||
if err != nil {
|
||||
return "", err
|
||||
@ -347,7 +357,7 @@ func (cj *CSIJournal) ReserveName(monitors, id, key, pool, reqName, parentName s
|
||||
|
||||
if snapSource {
|
||||
// Update UUID directory to store source volume UUID in case of snapshots
|
||||
err = SetOMapKeyValue(monitors, id, key, pool, cj.cephUUIDDirectoryPrefix+volUUID,
|
||||
err = SetOMapKeyValue(monitors, id, key, pool, cj.namespace, cj.cephUUIDDirectoryPrefix+volUUID,
|
||||
cj.cephSnapSourceKey, parentName)
|
||||
if err != nil {
|
||||
return "", err
|
||||
@ -373,14 +383,14 @@ func (cj *CSIJournal) GetObjectUUIDData(monitors, id, key, pool, objectUUID stri
|
||||
}
|
||||
|
||||
// TODO: fetch all omap vals in one call, than make multiple listomapvals
|
||||
requestName, err := GetOMapValue(monitors, id, key, pool,
|
||||
requestName, err := GetOMapValue(monitors, id, key, pool, cj.namespace,
|
||||
cj.cephUUIDDirectoryPrefix+objectUUID, cj.csiNameKey)
|
||||
if err != nil {
|
||||
return "", "", err
|
||||
}
|
||||
|
||||
if snapSource {
|
||||
sourceName, err = GetOMapValue(monitors, id, key, pool,
|
||||
sourceName, err = GetOMapValue(monitors, id, key, pool, cj.namespace,
|
||||
cj.cephUUIDDirectoryPrefix+objectUUID, cj.cephSnapSourceKey)
|
||||
if err != nil {
|
||||
return "", "", err
|
||||
|
||||
Reference in New Issue
Block a user