rbd: detect migration volID in DeleteVolume() and delete rbd image

This commit adds the logic to detect a passed in volumeID
is a migrated volume ID and if yes, the driver connect to the
backend cluster and clean/delete the image. The logic
only applied if its a migration volume ID. The migration volume ID
carry the information like mons, pool and image name which is
good enough for the driver to identify and connect to the backend
cluster for its operations.

migration volID format:
<mig>_mons-<monsHash>_image-<imageUID>_<poolHash>

Details on the hash values:

* MonsHash: this carry a hash value (md5sum) which will be acted as the
`clusterID` for the operations in this context.

* ImageUID: this is the unique UUID generated by kubernetes for the created
volume.

* PoolHash: this is an encoded string of pool name.

Signed-off-by: Humble Chirammal <hchiramm@redhat.com>
This commit is contained in:
Humble Chirammal 2021-10-01 09:48:19 +05:30 committed by mergify[bot]
parent b778fe51a4
commit 3c9d7e3cd5
6 changed files with 333 additions and 5 deletions

View File

@ -833,6 +833,16 @@ func (cs *ControllerServer) DeleteVolume(
}
defer cs.OperationLocks.ReleaseDeleteLock(volumeID)
if isMigrationVolID(volumeID) {
log.DebugLog(ctx, "migration volume ID : %s", volumeID)
err = parseAndDeleteMigratedVolume(ctx, volumeID, cr)
if err != nil && !errors.Is(err, ErrImageNotFound) {
return nil, status.Error(codes.Internal, err.Error())
}
return &csi.DeleteVolumeResponse{}, nil
}
rbdVol, err := genVolFromVolID(ctx, volumeID, cr, req.GetSecrets())
defer rbdVol.Destroy()
if err != nil {

View File

@ -34,4 +34,12 @@ var (
ErrMissingStash = errors.New("missing stash")
// ErrFlattenInProgress is returned when flatten is in progress for an image.
ErrFlattenInProgress = errors.New("flatten in progress")
// ErrMissingMonitorsInVolID is returned when monitor information is missing in migration volID.
ErrMissingMonitorsInVolID = errors.New("monitor information can not be empty in volID")
// ErrMissingPoolNameInVolID is returned when pool information is missing in migration volID.
ErrMissingPoolNameInVolID = errors.New("pool information can not be empty in volID")
// ErrMissingImageNameInVolID is returned when image name information is missing in migration volID.
ErrMissingImageNameInVolID = errors.New("rbd image name information can not be empty in volID")
// ErrDecodeClusterIDFromMonsInVolID is returned when mons hash decoding on migration volID.
ErrDecodeClusterIDFromMonsInVolID = errors.New("failed to get clusterID from monitors hash in volID")
)

116
internal/rbd/migration.go Normal file
View File

@ -0,0 +1,116 @@
/*
Copyright 2021 The Ceph-CSI Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package rbd
import (
"context"
"encoding/hex"
"strings"
"github.com/ceph/ceph-csi/internal/util"
"github.com/ceph/ceph-csi/internal/util/log"
)
// isMigrationVolID validates if the passed in volID is a volumeID
// of a migrated volume.
func isMigrationVolID(volHash string) bool {
return strings.Contains(volHash, migIdentifier) &&
strings.Contains(volHash, migImageNamePrefix) && strings.Contains(volHash, migMonPrefix)
}
// parseMigrationVolID decodes the volume ID and generates a migrationVolID
// struct which consists of mon, image name, pool and clusterID information.
func parseMigrationVolID(vh string) (*migrationVolID, error) {
mh := &migrationVolID{}
handSlice := strings.Split(vh, migVolIDFieldSep)
if len(handSlice) < migVolIDTotalLength {
// its short of length in this case, so return error
return nil, ErrInvalidVolID
}
// Store pool
poolHash := strings.Join(handSlice[migVolIDSplitLength:], migVolIDFieldSep)
poolByte, dErr := hex.DecodeString(poolHash)
if dErr != nil {
return nil, ErrMissingPoolNameInVolID
}
mh.poolName = string(poolByte)
// Parse migration mons( for clusterID) and image
for _, field := range handSlice[:migVolIDSplitLength] {
switch {
case strings.Contains(field, migImageNamePrefix):
imageSli := strings.Split(field, migImageNamePrefix)
if len(imageSli) > 0 {
mh.imageName = migInTreeImagePrefix + imageSli[1]
}
case strings.Contains(field, migMonPrefix):
// ex: mons-7982de6a23b77bce50b1ba9f2e879cce
mh.clusterID = strings.Trim(field, migMonPrefix)
}
}
if mh.imageName == "" {
return nil, ErrMissingImageNameInVolID
}
if mh.poolName == "" {
return nil, ErrMissingPoolNameInVolID
}
if mh.clusterID == "" {
return nil, ErrDecodeClusterIDFromMonsInVolID
}
return mh, nil
}
// parseAndDeleteMigratedVolume get rbd volume details from the migration volID
// and delete the volume from the cluster, return err if there was an error on the process.
func parseAndDeleteMigratedVolume(ctx context.Context, volumeID string, cr *util.Credentials) error {
parsedMigHandle, err := parseMigrationVolID(volumeID)
if err != nil {
log.ErrorLog(ctx, "failed to parse migration volumeID: %s , err: %v", volumeID, err)
return err
}
rv := &rbdVolume{}
// fill details to rv struct from parsed migration handle
rv.RbdImageName = parsedMigHandle.imageName
rv.Pool = parsedMigHandle.poolName
rv.ClusterID = parsedMigHandle.clusterID
rv.Monitors, err = util.Mons(util.CsiConfigFile, rv.ClusterID)
if err != nil {
log.ErrorLog(ctx, "failed to fetch monitors using clusterID: %s, err: %v", rv.ClusterID, err)
return err
}
// connect to the volume.
err = rv.Connect(cr)
if err != nil {
log.ErrorLog(ctx, "failed to get connected to the rbd image : %s, err: %v", rv.RbdImageName, err)
return err
}
defer rv.Destroy()
// if connected , delete it
err = deleteImage(ctx, rv, cr)
if err != nil {
log.ErrorLog(ctx, "failed to delete rbd image : %s, err: %v", rv.RbdImageName, err)
return err
}
return nil
}

View File

@ -0,0 +1,173 @@
/*
Copyright 2021 The Ceph-CSI Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package rbd
import (
"reflect"
"testing"
)
func TestIsMigrationVolID(t *testing.T) {
t.Parallel()
tests := []struct {
name string
args string
migVolID bool
}{
{
"correct volume ID",
"mig_mons-b7f67366bb43f32e07d8a261a7840da9_image-e0b45b52-7e09-47d3-8f1b-806995fa4412_706f6f6c5f7265706c6963615f706f6f6c", //nolint:lll // migration volID
true,
},
{
"Wrong volume ID",
"wrong_volume_ID",
false,
},
{
"wrong mons prefixed volume ID",
"mig_mon-b7f67366bb43f32e07d8a261a7840da9_image-e0b45b52-7e09-47d3-8f1b-806995fa4412_706f6f6c5f7265706c6963615f706f6f6c", //nolint:lll // migration volID
false,
},
{
"wrong image prefixed volume ID",
"mig_imae-e0b45b52-7e09-47d3-8f1b-806995fa4412_pool_replica_pool",
false,
},
{
"wrong volume ID",
"mig_image-e0b45b52-7e09-47d3-8f1b-806995fa4412_pool_replica_pool",
false,
},
}
for _, tt := range tests {
tt := tt
t.Run(tt.name, func(t *testing.T) {
t.Parallel()
got := isMigrationVolID(tt.args)
if got != tt.migVolID {
t.Errorf("isMigrationVolID() = %v, want %v", got, tt.migVolID)
}
})
}
}
func TestParseMigrationVolID(t *testing.T) {
t.Parallel()
tests := []struct {
name string
args string
want *migrationVolID
wantErr bool
}{
{
"correct volume ID",
"mig_mons-b7f67366bb43f32e07d8a261a7840da9_image-e0b45b52-7e09-47d3-8f1b-806995fa4412_706f6f6c5f7265706c6963615f706f6f6c", //nolint:lll // migration volID
&migrationVolID{
// monitors: "10.70.53.126:6789",
imageName: "kubernetes-dynamic-pvc-e0b45b52-7e09-47d3-8f1b-806995fa4412",
poolName: "pool_replica_pool",
clusterID: "b7f67366bb43f32e07d8a261a7840da9",
},
false,
},
{
"volume ID without mons",
"mig_kubernetes-dynamic-pvc-e0b45b52-7e09-47d3-8f1b-806995fa4412_706f6f6c5f7265706c6963615f706f6f6c",
nil,
true,
},
{
"volume ID without image",
"mig_pool-706f6f6c5f7265706c6963615f706f6f6c",
nil,
true,
},
{
"volume ID without pool",
"mig",
nil,
true,
},
{
"correct volume ID with single mon",
"mig_mons-7982de6a23b77bce50b1ba9f2e879cce_image-e0b45b52-7e09-47d3-8f1b-806995fa4412_706f6f6c5f7265706c6963615f706f6f6c", //nolint:lll // migration volID
&migrationVolID{
// monitors: "10.70.53.126:6789,10.70.53.156:6789",
imageName: "kubernetes-dynamic-pvc-e0b45b52-7e09-47d3-8f1b-806995fa4412",
poolName: "pool_replica_pool",
clusterID: "7982de6a23b77bce50b1ba9f2e879cce",
},
false,
},
{
"correct volume ID with more than one mon",
"mig_mons-7982de6a23b77bce50b1ba9f2e879cce_image-e0b45b52-7e09-47d3-8f1b-806995fa4412_706f6f6c5f7265706c6963615f706f6f6c", //nolint:lll // migration volID
&migrationVolID{
// monitors: "10.70.53.126:6789,10.70.53.156:6789",
imageName: "kubernetes-dynamic-pvc-e0b45b52-7e09-47d3-8f1b-806995fa4412",
poolName: "pool_replica_pool",
clusterID: "7982de6a23b77bce50b1ba9f2e879cce",
},
false,
},
{
"correct volume ID with '_' pool name",
"mig_mons-7982de6a23b77bce50b1ba9f2e879cce_image-e0b45b52-7e09-47d3-8f1b-806995fa4412_706f6f6c5f7265706c6963615f706f6f6c", //nolint:lll // migration volID
&migrationVolID{
// monitors: "10.70.53.126:6789,10.70.53.156:6789",
imageName: "kubernetes-dynamic-pvc-e0b45b52-7e09-47d3-8f1b-806995fa4412",
poolName: "pool_replica_pool",
clusterID: "7982de6a23b77bce50b1ba9f2e879cce",
},
false,
},
{
"volume ID with unallowed migration version string",
"migrate-beta_mons-b7f67366bb43f32e07d8a261a7840da9_kubernetes-pvc-e0b45b52-7e09-47d3-8f1b-806995fa4412_706f6f6c5f7265706c6963615f706f6f6c", //nolint:lll // migration volID
nil,
true,
},
{
"volume ID with unallowed image name",
"mig_mons-b7f67366bb43f32e07d8a261a7840da9_kubernetes-pvc-e0b45b52-7e09-47d3-8f1b-806995fa4412_706f6f6c5f7265706c6963615f706f6f6c", //nolint:lll // migration volID
nil,
true,
},
{
"volume ID without 'mon-' prefix string",
"mig_b7f67366bb43f32e07d8a261a7840da9_kubernetes-pvc-e0b45b52-7e09-47d3-8f1b-806995fa4412_706f6f6c5f7265706c6963615f706f6f6c", //nolint:lll // migration volID
nil,
true,
},
}
for _, tt := range tests {
tt := tt
t.Run(tt.name, func(t *testing.T) {
t.Parallel()
got, err := parseMigrationVolID(tt.args)
if (err != nil) != tt.wantErr {
t.Errorf("ParseMigrationVolID() error = %v, wantErr %v", err, tt.wantErr)
return
}
if !reflect.DeepEqual(got, tt.want) {
t.Errorf("ParseMigrationVolID() got = %v, want %v", got, tt.want)
}
})
}
}

View File

@ -150,11 +150,10 @@ func healerStageTransaction(ctx context.Context, cr *util.Credentials, volOps *r
}
// getClusterIDFromMigrationVolume fills the clusterID for the passed in monitors.
func getClusterIDFromMigrationVolume(parameters map[string]string) (string, error) {
func getClusterIDFromMigrationVolume(monitors string) (string, error) {
var err error
var rclusterID string
mons := parameters["monitors"]
for _, m := range strings.Split(mons, ",") {
for _, m := range strings.Split(monitors, ",") {
rclusterID, err = util.GetClusterIDFromMon(m)
if err != nil && !errors.Is(err, util.ErrMissingConfigForMonitor) {
return "", err
@ -294,7 +293,7 @@ func (ns *NodeServer) NodeStageVolume(
// Check this is a migration request because in that case, unlike other node stage requests
// it will be missing the clusterID, so fill it by fetching it from config file using mon.
if req.GetVolumeContext()[intreeMigrationKey] == intreeMigrationLabel && req.VolumeContext[util.ClusterIDKey] == "" {
cID, cErr := getClusterIDFromMigrationVolume(req.GetVolumeContext())
cID, cErr := getClusterIDFromMigrationVolume(req.GetVolumeContext()["monitors"])
if cErr != nil {
return nil, status.Error(codes.Internal, cErr.Error())
}

View File

@ -74,9 +74,23 @@ const (
thickProvisionMetaData = "true"
thinProvisionMetaData = "false"
// these are the migration label key and value for parameters in volume context.
// migration label key and value for parameters in volume context.
intreeMigrationKey = "migration"
intreeMigrationLabel = "true"
migInTreeImagePrefix = "kubernetes-dynamic-pvc-"
// migration volume handle identifiers.
// total length of fields in the migration volume handle.
migVolIDTotalLength = 4
// split boundary length of fields.
migVolIDSplitLength = 3
// separator for migration handle fields.
migVolIDFieldSep = "_"
// identifier of a migration vol handle.
migIdentifier = "mig"
// prefix of image field.
migImageNamePrefix = "image-"
// prefix in the handle for monitors field.
migMonPrefix = "mons-"
)
// rbdImage contains common attributes and methods for the rbdVolume and
@ -175,6 +189,14 @@ type imageFeature struct {
dependsOn []string
}
// migrationvolID is a struct which consists of required fields of a rbd volume
// from migrated volumeID.
type migrationVolID struct {
imageName string
poolName string
clusterID string
}
var supportedFeatures = map[string]imageFeature{
librbd.FeatureNameLayering: {
needRbdNbd: false,