ceph-csi/e2e/migration.go

package e2e

import (
	"context"
	"encoding/hex"
	"fmt"
	"strings"

	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/client-go/kubernetes"
	"k8s.io/kubernetes/test/e2e/framework"
)

// composeIntreeMigVolID create a volID similar to intree migration volID
// the migration volID format looks like below
// mig-mons-<hash>-image-<UUID_<poolhash>
// nolint:lll    // ex: "mig_mons-b7f67366bb43f32e07d8a261a7840da9_image-e0b45b52-7e09-47d3-8f1b-806995fa4412_706f6f6c5f7265706c6963615f706f6f6c
func composeIntreeMigVolID(mons, rbdImageName string) string {
	poolField := hex.EncodeToString([]byte(defaultRBDPool))
	monsField := monsPrefix + getMonsHash(mons)
	imageUID := strings.Split(rbdImageName, intreeVolPrefix)[1:]
	imageField := imagePrefix + imageUID[0]
	vhSlice := []string{migIdentifier, monsField, imageField, poolField}

	return strings.Join(vhSlice, "_")
}

// generateClusterIDConfigMapForMigration retrieve monitors and generate a hash value which
// is used as a clusterID in the custom configmap, this function also recreate RBD CSI pods
// once the custom config map has been recreated.
func generateClusterIDConfigMapForMigration(f *framework.Framework, c kubernetes.Interface) error {
	// create monitors hash by fetching monitors from the cluster.
	mons, err := getMons(rookNamespace, c)
	if err != nil {
		return fmt.Errorf("failed to get monitors %w", err)
	}
	mon := strings.Join(mons, ",")
	inClusterID := getMonsHash(mon)

	clusterInfo := map[string]map[string]string{}
	clusterInfo[inClusterID] = map[string]string{}

	// create custom configmap
	err = createCustomConfigMap(f.ClientSet, rbdDirPath, clusterInfo)
	if err != nil {
		return fmt.Errorf("failed to create configmap: %w", err)
	}
	// restart csi pods for the configmap to take effect.
	err = recreateCSIRBDPods(f)
	if err != nil {
		return fmt.Errorf("failed to recreate rbd csi pods: %w", err)
	}

	return nil
}

// createRBDMigrationSecret creates a migration secret with the passed in user name.
// this secret differs from csi secret data on below aspects.
// equivalent to the `UserKey` field, migration secret has `key` field.
// if 'userName' has passed and if it is not admin, the passed in userName will be
// set as the `adminId` field in the secret.
func createRBDMigrationSecret(f *framework.Framework, secretName, userName, userKey string) error {
	secPath := fmt.Sprintf("%s/%s", rbdExamplePath, "secret.yaml")
	sec, err := getSecret(secPath)
	if err != nil {
		return err
	}
	if secretName != "" {
		sec.Name = secretName
	}
	// if its admin, we dont need to change anything in the migration secret, the CSI driver
	// will use the key from existing secret and continue.
	if userName != "admin" {
		sec.StringData["adminId"] = userName
	}
	sec.StringData["key"] = userKey
	sec.Namespace = cephCSINamespace
	_, err = f.ClientSet.CoreV1().Secrets(cephCSINamespace).Create(context.TODO(), &sec, metav1.CreateOptions{})

	return err
}

// createMigrationUserSecretAndSC creates migration user and a secret associated with this user first,
// then create SC based on the same.
func createMigrationUserSecretAndSC(f *framework.Framework, scName string) error {
	if scName == "" {
		scName = defaultSCName
	}
	err := createProvNodeCephUserAndSecret(f, true, true)
	if err != nil {
		return err
	}

	err = createMigrationSC(f, scName)
	if err != nil {
		return err
	}

	return nil
}

// createMigrationSC create a SC with migration specific secrets and clusterid.
func createMigrationSC(f *framework.Framework, scName string) error {
	err := deleteResource(rbdExamplePath + "storageclass.yaml")
	if err != nil {
		return fmt.Errorf("failed to delete storageclass: %w", err)
	}
	param := make(map[string]string)
	// add new secrets to the SC parameters
	param["csi.storage.k8s.io/provisioner-secret-namespace"] = cephCSINamespace
	param["csi.storage.k8s.io/provisioner-secret-name"] = rbdMigrationProvisionerSecretName
	param["csi.storage.k8s.io/controller-expand-secret-namespace"] = cephCSINamespace
	param["csi.storage.k8s.io/controller-expand-secret-name"] = rbdMigrationProvisionerSecretName
	param["csi.storage.k8s.io/node-stage-secret-namespace"] = cephCSINamespace
	param["csi.storage.k8s.io/node-stage-secret-name"] = rbdMigrationNodePluginSecretName
	mons, err := getMons(rookNamespace, f.ClientSet)
	if err != nil {
		return fmt.Errorf("failed to get mons: %w", err)
	}
	mon := strings.Join(mons, ",")
	param["migration"] = "true"
	param["clusterID"] = getMonsHash(mon)
	err = createRBDStorageClass(f.ClientSet, f, scName, nil, param, deletePolicy)
	if err != nil {
		return fmt.Errorf("failed to create storageclass: %w", err)
	}

	return nil
}

// createProvNodeCephUserAndSecret fetches the ceph migration user's key and create migration secret
// with it based on the arg values of 'provSecret' and 'nodeSecret'.
func createProvNodeCephUserAndSecret(f *framework.Framework, provisionerSecret, nodeSecret bool) error {
	if provisionerSecret {
		// Fetch the key.
		key, err := createCephUser(
			f,
			keyringRBDProvisionerUsername,
			rbdProvisionerCaps(defaultRBDPool, radosNamespace),
		)
		if err != nil {
			return fmt.Errorf("failed to create user %q: %w", keyringRBDProvisionerUsername, err)
		}
		err = createRBDMigrationSecret(f, rbdMigrationProvisionerSecretName, keyringRBDProvisionerUsername, key)
		if err != nil {
			return fmt.Errorf("failed to create provisioner secret: %w", err)
		}
	}

	if nodeSecret {
		// Fetch the key.
		key, err := createCephUser(
			f,
			keyringRBDNodePluginUsername,
			rbdNodePluginCaps(defaultRBDPool, radosNamespace))
		if err != nil {
			return fmt.Errorf("failed to create user %q: %w", keyringRBDNodePluginUsername, err)
		}
		err = createRBDMigrationSecret(f, rbdMigrationNodePluginSecretName, keyringRBDNodePluginUsername, key)
		if err != nil {
			return fmt.Errorf("failed to create node secret: %w", err)
		}
	}

	return nil
}

// deleteProvNodeMigrationSecret deletes ceph migration secrets based on the
// arg values of 'provisionerSecret' and 'nodeSecret'.
func deleteProvNodeMigrationSecret(f *framework.Framework, provisionerSecret, nodeSecret bool) error {
	c := f.ClientSet
	if provisionerSecret {
		// delete RBD provisioner secret.
		err := c.CoreV1().
			Secrets(cephCSINamespace).
			Delete(context.TODO(), rbdMigrationProvisionerSecretName, metav1.DeleteOptions{})
		if err != nil {
			return fmt.Errorf("failed to delete provisioner secret: %w", err)
		}
	}

	if nodeSecret {
		// delete RBD node secret.
		err := c.CoreV1().
			Secrets(cephCSINamespace).
			Delete(context.TODO(), rbdMigrationNodePluginSecretName, metav1.DeleteOptions{})
		if err != nil {
			return fmt.Errorf("failed to delete node secret: %w", err)
		}
	}

	return nil
}

// setupMigrationCMSecretAndSC create custom configmap, secret and SC for migration tests.
func setupMigrationCMSecretAndSC(f *framework.Framework, scName string) error {
	c := f.ClientSet
	if scName == "" {
		scName = defaultSCName
	}
	err := generateClusterIDConfigMapForMigration(f, c)
	if err != nil {
		return fmt.Errorf("failed to generate clusterID configmap: %w", err)
	}

	err = createMigrationUserSecretAndSC(f, scName)
	if err != nil {
		return fmt.Errorf("failed to create storageclass: %w", err)
	}

	return nil
}

// tearDownMigrationSetup deletes custom configmap and secret.
func tearDownMigrationSetup(f *framework.Framework) error {
	err := deleteConfigMap(rbdDirPath)
	if err != nil {
		return fmt.Errorf("failed to delete configmap: %w", err)
	}
	err = createConfigMap(rbdDirPath, f.ClientSet, f)
	if err != nil {
		return fmt.Errorf("failed to create configmap: %w", err)
	}
	err = deleteProvNodeMigrationSecret(f, true, true)
	if err != nil {
		return fmt.Errorf("failed to delete migration users and Secrets associated: %w", err)
	}

	return nil
}