rbd: detect migration volID in DeleteVolume() and delete rbd image

This commit adds the logic to detect a passed in volumeID
is a migrated volume ID and if yes, the driver connect to the
backend cluster and clean/delete the image. The logic
only applied if its a migration volume ID. The migration volume ID
carry the information like mons, pool and image name which is
good enough for the driver to identify and connect to the backend
cluster for its operations.

migration volID format:
<mig>_mons-<monsHash>_image-<imageUID>_<poolHash>

Details on the hash values:

* MonsHash: this carry a hash value (md5sum) which will be acted as the
`clusterID` for the operations in this context.

* ImageUID: this is the unique UUID generated by kubernetes for the created
volume.

* PoolHash: this is an encoded string of pool name.

Signed-off-by: Humble Chirammal <hchiramm@redhat.com>

internal/rbd/controllerserver.go

@@ -833,6 +833,16 @@ func (cs *ControllerServer) DeleteVolume(
 	}
 	defer cs.OperationLocks.ReleaseDeleteLock(volumeID)
 
+	if isMigrationVolID(volumeID) {
+		log.DebugLog(ctx, "migration volume ID : %s", volumeID)
+		err = parseAndDeleteMigratedVolume(ctx, volumeID, cr)
+		if err != nil && !errors.Is(err, ErrImageNotFound) {
+			return nil, status.Error(codes.Internal, err.Error())
+		}
+
+		return &csi.DeleteVolumeResponse{}, nil
+	}
+
 	rbdVol, err := genVolFromVolID(ctx, volumeID, cr, req.GetSecrets())
 	defer rbdVol.Destroy()
 	if err != nil {

internal/rbd/errors.go

@@ -34,4 +34,12 @@ var (
 	ErrMissingStash = errors.New("missing stash")
 	// ErrFlattenInProgress is returned when flatten is in progress for an image.
 	ErrFlattenInProgress = errors.New("flatten in progress")
+	// ErrMissingMonitorsInVolID is returned when monitor information is missing in migration volID.
+	ErrMissingMonitorsInVolID = errors.New("monitor information can not be empty in volID")
+	// ErrMissingPoolNameInVolID is returned when pool information is missing in migration volID.
+	ErrMissingPoolNameInVolID = errors.New("pool information can not be empty in volID")
+	// ErrMissingImageNameInVolID is returned when image name information is missing in migration volID.
+	ErrMissingImageNameInVolID = errors.New("rbd image name information can not be empty in volID")
+	// ErrDecodeClusterIDFromMonsInVolID is returned when mons hash decoding on migration volID.
+	ErrDecodeClusterIDFromMonsInVolID = errors.New("failed to get clusterID from monitors hash in volID")
 )

internal/rbd/migration.go

@@ -0,0 +1,116 @@
+/*
+Copyright 2021 The Ceph-CSI Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package rbd
+
+import (
+	"context"
+	"encoding/hex"
+	"strings"
+
+	"github.com/ceph/ceph-csi/internal/util"
+	"github.com/ceph/ceph-csi/internal/util/log"
+)
+
+// isMigrationVolID validates if the passed in volID is a volumeID
+// of a migrated volume.
+func isMigrationVolID(volHash string) bool {
+	return strings.Contains(volHash, migIdentifier) &&
+		strings.Contains(volHash, migImageNamePrefix) && strings.Contains(volHash, migMonPrefix)
+}
+
+// parseMigrationVolID decodes the volume ID and generates a migrationVolID
+// struct which consists of mon, image name, pool and clusterID information.
+func parseMigrationVolID(vh string) (*migrationVolID, error) {
+	mh := &migrationVolID{}
+	handSlice := strings.Split(vh, migVolIDFieldSep)
+	if len(handSlice) < migVolIDTotalLength {
+		// its short of length in this case, so return error
+		return nil, ErrInvalidVolID
+	}
+	// Store pool
+	poolHash := strings.Join(handSlice[migVolIDSplitLength:], migVolIDFieldSep)
+	poolByte, dErr := hex.DecodeString(poolHash)
+	if dErr != nil {
+		return nil, ErrMissingPoolNameInVolID
+	}
+	mh.poolName = string(poolByte)
+	// Parse migration mons( for clusterID) and image
+	for _, field := range handSlice[:migVolIDSplitLength] {
+		switch {
+		case strings.Contains(field, migImageNamePrefix):
+			imageSli := strings.Split(field, migImageNamePrefix)
+			if len(imageSli) > 0 {
+				mh.imageName = migInTreeImagePrefix + imageSli[1]
+			}
+		case strings.Contains(field, migMonPrefix):
+			// ex: mons-7982de6a23b77bce50b1ba9f2e879cce
+			mh.clusterID = strings.Trim(field, migMonPrefix)
+		}
+	}
+	if mh.imageName == "" {
+		return nil, ErrMissingImageNameInVolID
+	}
+	if mh.poolName == "" {
+		return nil, ErrMissingPoolNameInVolID
+	}
+	if mh.clusterID == "" {
+		return nil, ErrDecodeClusterIDFromMonsInVolID
+	}
+
+	return mh, nil
+}
+
+// parseAndDeleteMigratedVolume get rbd volume details from the migration volID
+// and delete the volume from the cluster, return err if there was an error on the process.
+func parseAndDeleteMigratedVolume(ctx context.Context, volumeID string, cr *util.Credentials) error {
+	parsedMigHandle, err := parseMigrationVolID(volumeID)
+	if err != nil {
+		log.ErrorLog(ctx, "failed to parse migration volumeID: %s , err: %v", volumeID, err)
+
+		return err
+	}
+	rv := &rbdVolume{}
+
+	// fill details to rv struct from parsed migration handle
+	rv.RbdImageName = parsedMigHandle.imageName
+	rv.Pool = parsedMigHandle.poolName
+	rv.ClusterID = parsedMigHandle.clusterID
+	rv.Monitors, err = util.Mons(util.CsiConfigFile, rv.ClusterID)
+	if err != nil {
+		log.ErrorLog(ctx, "failed to fetch monitors using clusterID: %s, err: %v", rv.ClusterID, err)
+
+		return err
+	}
+
+	// connect to the volume.
+	err = rv.Connect(cr)
+	if err != nil {
+		log.ErrorLog(ctx, "failed to get connected to the rbd image : %s, err: %v", rv.RbdImageName, err)
+
+		return err
+	}
+	defer rv.Destroy()
+	// if connected , delete it
+	err = deleteImage(ctx, rv, cr)
+	if err != nil {
+		log.ErrorLog(ctx, "failed to delete rbd image : %s, err: %v", rv.RbdImageName, err)
+
+		return err
+	}
+
+	return nil
+}

internal/rbd/migration_test.go

@@ -0,0 +1,173 @@
+/*
+Copyright 2021 The Ceph-CSI Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+package rbd
+
+import (
+	"reflect"
+	"testing"
+)
+
+func TestIsMigrationVolID(t *testing.T) {
+	t.Parallel()
+	tests := []struct {
+		name     string
+		args     string
+		migVolID bool
+	}{
+		{
+			"correct volume ID",
+			"mig_mons-b7f67366bb43f32e07d8a261a7840da9_image-e0b45b52-7e09-47d3-8f1b-806995fa4412_706f6f6c5f7265706c6963615f706f6f6c", //nolint:lll // migration volID
+			true,
+		},
+		{
+			"Wrong volume ID",
+			"wrong_volume_ID",
+			false,
+		},
+		{
+			"wrong mons prefixed volume ID",
+			"mig_mon-b7f67366bb43f32e07d8a261a7840da9_image-e0b45b52-7e09-47d3-8f1b-806995fa4412_706f6f6c5f7265706c6963615f706f6f6c", //nolint:lll // migration volID
+			false,
+		},
+		{
+			"wrong image prefixed volume ID",
+			"mig_imae-e0b45b52-7e09-47d3-8f1b-806995fa4412_pool_replica_pool",
+			false,
+		},
+		{
+			"wrong volume ID",
+			"mig_image-e0b45b52-7e09-47d3-8f1b-806995fa4412_pool_replica_pool",
+			false,
+		},
+	}
+	for _, tt := range tests {
+		tt := tt
+		t.Run(tt.name, func(t *testing.T) {
+			t.Parallel()
+			got := isMigrationVolID(tt.args)
+			if got != tt.migVolID {
+				t.Errorf("isMigrationVolID() = %v, want %v", got, tt.migVolID)
+			}
+		})
+	}
+}
+
+func TestParseMigrationVolID(t *testing.T) {
+	t.Parallel()
+	tests := []struct {
+		name    string
+		args    string
+		want    *migrationVolID
+		wantErr bool
+	}{
+		{
+			"correct volume ID",
+			"mig_mons-b7f67366bb43f32e07d8a261a7840da9_image-e0b45b52-7e09-47d3-8f1b-806995fa4412_706f6f6c5f7265706c6963615f706f6f6c", //nolint:lll // migration volID
+			&migrationVolID{
+				// monitors:  "10.70.53.126:6789",
+				imageName: "kubernetes-dynamic-pvc-e0b45b52-7e09-47d3-8f1b-806995fa4412",
+				poolName:  "pool_replica_pool",
+				clusterID: "b7f67366bb43f32e07d8a261a7840da9",
+			},
+			false,
+		},
+		{
+			"volume ID without mons",
+			"mig_kubernetes-dynamic-pvc-e0b45b52-7e09-47d3-8f1b-806995fa4412_706f6f6c5f7265706c6963615f706f6f6c",
+			nil,
+			true,
+		},
+		{
+			"volume ID without image",
+			"mig_pool-706f6f6c5f7265706c6963615f706f6f6c",
+			nil,
+			true,
+		},
+		{
+			"volume ID without pool",
+			"mig",
+			nil,
+			true,
+		},
+		{
+			"correct volume ID with single mon",
+			"mig_mons-7982de6a23b77bce50b1ba9f2e879cce_image-e0b45b52-7e09-47d3-8f1b-806995fa4412_706f6f6c5f7265706c6963615f706f6f6c", //nolint:lll // migration volID
+			&migrationVolID{
+				// monitors:  "10.70.53.126:6789,10.70.53.156:6789",
+				imageName: "kubernetes-dynamic-pvc-e0b45b52-7e09-47d3-8f1b-806995fa4412",
+				poolName:  "pool_replica_pool",
+				clusterID: "7982de6a23b77bce50b1ba9f2e879cce",
+			},
+			false,
+		},
+		{
+			"correct volume ID with more than one mon",
+			"mig_mons-7982de6a23b77bce50b1ba9f2e879cce_image-e0b45b52-7e09-47d3-8f1b-806995fa4412_706f6f6c5f7265706c6963615f706f6f6c", //nolint:lll // migration volID
+			&migrationVolID{
+				// monitors:  "10.70.53.126:6789,10.70.53.156:6789",
+				imageName: "kubernetes-dynamic-pvc-e0b45b52-7e09-47d3-8f1b-806995fa4412",
+				poolName:  "pool_replica_pool",
+				clusterID: "7982de6a23b77bce50b1ba9f2e879cce",
+			},
+			false,
+		},
+		{
+			"correct volume ID with '_' pool name",
+			"mig_mons-7982de6a23b77bce50b1ba9f2e879cce_image-e0b45b52-7e09-47d3-8f1b-806995fa4412_706f6f6c5f7265706c6963615f706f6f6c", //nolint:lll // migration volID
+			&migrationVolID{
+				// monitors:  "10.70.53.126:6789,10.70.53.156:6789",
+				imageName: "kubernetes-dynamic-pvc-e0b45b52-7e09-47d3-8f1b-806995fa4412",
+				poolName:  "pool_replica_pool",
+				clusterID: "7982de6a23b77bce50b1ba9f2e879cce",
+			},
+			false,
+		},
+		{
+			"volume ID with unallowed migration version string",
+			"migrate-beta_mons-b7f67366bb43f32e07d8a261a7840da9_kubernetes-pvc-e0b45b52-7e09-47d3-8f1b-806995fa4412_706f6f6c5f7265706c6963615f706f6f6c", //nolint:lll // migration volID
+			nil,
+			true,
+		},
+		{
+			"volume ID with unallowed image name",
+			"mig_mons-b7f67366bb43f32e07d8a261a7840da9_kubernetes-pvc-e0b45b52-7e09-47d3-8f1b-806995fa4412_706f6f6c5f7265706c6963615f706f6f6c", //nolint:lll // migration volID
+			nil,
+			true,
+		},
+
+		{
+			"volume ID without 'mon-' prefix string",
+			"mig_b7f67366bb43f32e07d8a261a7840da9_kubernetes-pvc-e0b45b52-7e09-47d3-8f1b-806995fa4412_706f6f6c5f7265706c6963615f706f6f6c", //nolint:lll // migration volID
+			nil,
+			true,
+		},
+	}
+	for _, tt := range tests {
+		tt := tt
+		t.Run(tt.name, func(t *testing.T) {
+			t.Parallel()
+			got, err := parseMigrationVolID(tt.args)
+			if (err != nil) != tt.wantErr {
+				t.Errorf("ParseMigrationVolID() error = %v, wantErr %v", err, tt.wantErr)
+
+				return
+			}
+			if !reflect.DeepEqual(got, tt.want) {
+				t.Errorf("ParseMigrationVolID() got = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}

internal/rbd/nodeserver.go

@@ -150,11 +150,10 @@ func healerStageTransaction(ctx context.Context, cr *util.Credentials, volOps *r
 }
 
 // getClusterIDFromMigrationVolume fills the clusterID for the passed in monitors.
-func getClusterIDFromMigrationVolume(parameters map[string]string) (string, error) {
+func getClusterIDFromMigrationVolume(monitors string) (string, error) {
 	var err error
 	var rclusterID string
-	mons := parameters["monitors"]
+	for _, m := range strings.Split(monitors, ",") {
-	for _, m := range strings.Split(mons, ",") {
 		rclusterID, err = util.GetClusterIDFromMon(m)
 		if err != nil && !errors.Is(err, util.ErrMissingConfigForMonitor) {
 			return "", err
@@ -294,7 +293,7 @@ func (ns *NodeServer) NodeStageVolume(
 	// Check this is a migration request because in that case, unlike other node stage requests
 	// it will be missing the clusterID, so fill it by fetching it from config file using mon.
 	if req.GetVolumeContext()[intreeMigrationKey] == intreeMigrationLabel && req.VolumeContext[util.ClusterIDKey] == "" {
-		cID, cErr := getClusterIDFromMigrationVolume(req.GetVolumeContext())
+		cID, cErr := getClusterIDFromMigrationVolume(req.GetVolumeContext()["monitors"])
 		if cErr != nil {
 			return nil, status.Error(codes.Internal, cErr.Error())
 		}

internal/rbd/rbd_util.go

@@ -74,9 +74,23 @@ const (
 	thickProvisionMetaData = "true"
 	thinProvisionMetaData  = "false"
 
-	// these are the migration label key and value for parameters in volume context.
+	// migration label key and value for parameters in volume context.
 	intreeMigrationKey   = "migration"
 	intreeMigrationLabel = "true"
+	migInTreeImagePrefix = "kubernetes-dynamic-pvc-"
+	// migration volume handle identifiers.
+	// total length of fields in the migration volume handle.
+	migVolIDTotalLength = 4
+	// split boundary length of fields.
+	migVolIDSplitLength = 3
+	// separator for migration handle fields.
+	migVolIDFieldSep = "_"
+	// identifier of a migration vol handle.
+	migIdentifier = "mig"
+	// prefix of image field.
+	migImageNamePrefix = "image-"
+	// prefix in the handle for monitors field.
+	migMonPrefix = "mons-"
 )
 
 // rbdImage contains common attributes and methods for the rbdVolume and
@@ -175,6 +189,14 @@ type imageFeature struct {
 	dependsOn []string
 }
 
+// migrationvolID is a struct which consists of required fields of a rbd volume
+// from migrated volumeID.
+type migrationVolID struct {
+	imageName string
+	poolName  string
+	clusterID string
+}
+
 var supportedFeatures = map[string]imageFeature{
 	librbd.FeatureNameLayering: {
 		needRbdNbd: false,