ceph-csi/e2e/rbd_helper.go

577 lines
17 KiB
Go
Raw Normal View History

package e2e
import (
"context"
"encoding/json"
"fmt"
"regexp"
"strings"
v1 "k8s.io/api/core/v1"
scv1 "k8s.io/api/storage/v1"
"k8s.io/apimachinery/pkg/api/resource"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/client-go/kubernetes"
"k8s.io/kubernetes/test/e2e/framework"
e2elog "k8s.io/kubernetes/test/e2e/framework/log"
)
func imageSpec(pool, image string) string {
if radosNamespace != "" {
return pool + "/" + radosNamespace + "/" + image
}
return pool + "/" + image
}
func rbdOptions(pool string) string {
if radosNamespace != "" {
return "--pool=" + pool + " --namespace " + radosNamespace
}
return "--pool=" + pool
}
func createRBDStorageClass(c kubernetes.Interface, f *framework.Framework, scOptions, parameters map[string]string, policy v1.PersistentVolumeReclaimPolicy) error {
scPath := fmt.Sprintf("%s/%s", rbdExamplePath, "storageclass.yaml")
sc, err := getStorageClass(scPath)
if err != nil {
return nil
}
sc.Parameters["pool"] = defaultRBDPool
sc.Parameters["csi.storage.k8s.io/provisioner-secret-namespace"] = cephCSINamespace
sc.Parameters["csi.storage.k8s.io/provisioner-secret-name"] = rbdProvisionerSecretName
sc.Parameters["csi.storage.k8s.io/controller-expand-secret-namespace"] = cephCSINamespace
sc.Parameters["csi.storage.k8s.io/controller-expand-secret-name"] = rbdProvisionerSecretName
sc.Parameters["csi.storage.k8s.io/node-stage-secret-namespace"] = cephCSINamespace
sc.Parameters["csi.storage.k8s.io/node-stage-secret-name"] = rbdNodePluginSecretName
fsID, stdErr, err := execCommandInToolBoxPod(f, "ceph fsid", rookNamespace)
if err != nil {
return err
}
if stdErr != "" {
return fmt.Errorf("error getting fsid %v", stdErr)
}
// remove new line present in fsID
fsID = strings.Trim(fsID, "\n")
sc.Parameters["clusterID"] = fsID
for k, v := range parameters {
sc.Parameters[k] = v
}
sc.Namespace = cephCSINamespace
if scOptions["volumeBindingMode"] == "WaitForFirstConsumer" {
value := scv1.VolumeBindingWaitForFirstConsumer
sc.VolumeBindingMode = &value
}
// comma separated mount options
if opt, ok := scOptions[rbdmountOptions]; ok {
mOpt := strings.Split(opt, ",")
sc.MountOptions = append(sc.MountOptions, mOpt...)
}
sc.ReclaimPolicy = &policy
_, err = c.StorageV1().StorageClasses().Create(context.TODO(), &sc, metav1.CreateOptions{})
return err
}
func createRadosNamespace(f *framework.Framework) error {
stdOut, stdErr, err := execCommandInToolBoxPod(f,
fmt.Sprintf("rbd namespace ls --pool=%s", defaultRBDPool), rookNamespace)
if err != nil {
return err
}
if stdErr != "" {
return fmt.Errorf("error listing rbd namespace %v", stdErr)
}
if !strings.Contains(stdOut, radosNamespace) {
_, stdErr, err = execCommandInToolBoxPod(f,
fmt.Sprintf("rbd namespace create %s", rbdOptions(defaultRBDPool)), rookNamespace)
if err != nil {
return err
}
if stdErr != "" {
return fmt.Errorf("error creating rbd namespace %v", stdErr)
}
}
stdOut, stdErr, err = execCommandInToolBoxPod(f,
fmt.Sprintf("rbd namespace ls --pool=%s", rbdTopologyPool), rookNamespace)
if err != nil {
return err
}
if stdErr != "" {
return fmt.Errorf("error listing rbd namespace %v", stdErr)
}
if !strings.Contains(stdOut, radosNamespace) {
_, stdErr, err = execCommandInToolBoxPod(f,
fmt.Sprintf("rbd namespace create %s", rbdOptions(rbdTopologyPool)), rookNamespace)
if err != nil {
return err
}
if stdErr != "" {
return fmt.Errorf("error creating rbd namespace %v", stdErr)
}
}
return nil
}
func createRBDSecret(f *framework.Framework, secretName, userName, userKey string) error {
scPath := fmt.Sprintf("%s/%s", rbdExamplePath, "secret.yaml")
sc, err := getSecret(scPath)
if err != nil {
return err
}
if secretName != "" {
sc.Name = secretName
}
sc.StringData["userID"] = userName
sc.StringData["userKey"] = userKey
sc.Namespace = cephCSINamespace
_, err = f.ClientSet.CoreV1().Secrets(cephCSINamespace).Create(context.TODO(), &sc, metav1.CreateOptions{})
return err
}
type imageInfoFromPVC struct {
imageID string
imageName string
csiVolumeHandle string
pvName string
}
// getImageInfoFromPVC reads volume handle of the bound PV to the passed in PVC,
// and returns imageInfoFromPVC or error.
func getImageInfoFromPVC(pvcNamespace, pvcName string, f *framework.Framework) (imageInfoFromPVC, error) {
var imageData imageInfoFromPVC
c := f.ClientSet.CoreV1()
pvc, err := c.PersistentVolumeClaims(pvcNamespace).Get(context.TODO(), pvcName, metav1.GetOptions{})
if err != nil {
return imageData, err
}
pv, err := c.PersistentVolumes().Get(context.TODO(), pvc.Spec.VolumeName, metav1.GetOptions{})
if err != nil {
return imageData, err
}
imageIDRegex := regexp.MustCompile(`(\w+\-?){5}$`)
imageID := imageIDRegex.FindString(pv.Spec.CSI.VolumeHandle)
imageData = imageInfoFromPVC{
imageID: imageID,
imageName: fmt.Sprintf("csi-vol-%s", imageID),
csiVolumeHandle: pv.Spec.CSI.VolumeHandle,
pvName: pv.Name,
}
return imageData, nil
}
func getImageMeta(rbdImageSpec, metaKey string, f *framework.Framework) (string, error) {
cmd := fmt.Sprintf("rbd image-meta get %s %s", rbdImageSpec, metaKey)
stdOut, stdErr, err := execCommandInToolBoxPod(f, cmd, rookNamespace)
if err != nil {
return "", err
}
if stdErr != "" {
return strings.TrimSpace(stdOut), fmt.Errorf(stdErr)
}
return strings.TrimSpace(stdOut), nil
}
// validateImageOwner checks the "csi.volume.owner" key on the image journal
// and verifies that the owner is set to the namespace where the PVC is
// created.
func validateImageOwner(pvcPath string, f *framework.Framework) error {
const ownerKey = "csi.volume.owner"
pvc, err := loadPVC(pvcPath)
if err != nil {
return err
}
pvc.Namespace = f.UniqueName
pvc.Name = f.UniqueName
err = createPVCAndvalidatePV(f.ClientSet, pvc, deployTimeout)
if err != nil {
return err
}
imageData, err := getImageInfoFromPVC(pvc.Namespace, pvc.Name, f)
if err != nil {
return err
}
stdOut, stdErr, err := execCommandInToolBoxPod(f,
fmt.Sprintf("rados %s getomapval csi.volume.%s %s", rbdOptions(defaultRBDPool), imageData.imageID, ownerKey), rookNamespace)
if err != nil {
return err
}
if stdErr != "" {
return fmt.Errorf("failed to getomapval %v", stdErr)
}
if radosNamespace != "" {
e2elog.Logf("found image journal %s in pool %s namespace %s", "csi.volume."+imageData.imageID, defaultRBDPool, radosNamespace)
} else {
e2elog.Logf("found image journal %s in pool %s", "csi.volume."+imageData.imageID, defaultRBDPool)
}
if !strings.Contains(stdOut, pvc.Namespace) {
return fmt.Errorf("%q does not contain %q: %s", ownerKey, pvc.Namespace, stdOut)
}
return deletePVCAndValidatePV(f.ClientSet, pvc, deployTimeout)
}
func validateEncryptedPVCAndAppBinding(pvcPath, appPath, kms string, f *framework.Framework) error {
pvc, app, err := createPVCAndAppBinding(pvcPath, appPath, f, deployTimeout)
if err != nil {
return err
}
imageData, err := getImageInfoFromPVC(pvc.Namespace, pvc.Name, f)
if err != nil {
return err
}
rbdImageSpec := imageSpec(defaultRBDPool, imageData.imageName)
encryptedState, err := getImageMeta(rbdImageSpec, ".rbd.csi.ceph.com/encrypted", f)
if err != nil {
return err
}
if encryptedState != "encrypted" {
return fmt.Errorf("%v not equal to encrypted", encryptedState)
}
volumeMountPath := app.Spec.Containers[0].VolumeMounts[0].MountPath
mountType, err := getMountType(app.Name, app.Namespace, volumeMountPath, f)
if err != nil {
return err
}
if mountType != "crypt" {
return fmt.Errorf("%v not equal to crypt", mountType)
}
if kms == "vault" {
// check new passphrase created
_, stdErr := readVaultSecret(imageData.csiVolumeHandle, f)
if stdErr != "" {
return fmt.Errorf("failed to read passphrase from vault: %s", stdErr)
}
}
err = deletePVCAndApp("", f, pvc, app)
if err != nil {
return err
}
if kms == "vault" {
// check new passphrase created
stdOut, _ := readVaultSecret(imageData.csiVolumeHandle, f)
if stdOut != "" {
return fmt.Errorf("passphrase found in vault while should be deleted: %s", stdOut)
}
}
return nil
}
func listRBDImages(f *framework.Framework) ([]string, error) {
var imgInfos []string
stdout, stdErr, err := execCommandInToolBoxPod(f,
fmt.Sprintf("rbd ls --format=json %s", rbdOptions(defaultRBDPool)), rookNamespace)
if err != nil {
return imgInfos, err
}
if stdErr != "" {
return imgInfos, fmt.Errorf("failed to list images %v", stdErr)
}
err = json.Unmarshal([]byte(stdout), &imgInfos)
if err != nil {
return imgInfos, err
}
return imgInfos, nil
}
func deleteBackingRBDImage(f *framework.Framework, pvc *v1.PersistentVolumeClaim) error {
imageData, err := getImageInfoFromPVC(pvc.Namespace, pvc.Name, f)
if err != nil {
return err
}
cmd := fmt.Sprintf("rbd rm %s %s", rbdOptions(defaultRBDPool), imageData.imageName)
_, _, err = execCommandInToolBoxPod(f, cmd, rookNamespace)
return err
}
// rbdDuImage contains the disk-usage statistics of an RBD image.
type rbdDuImage struct {
Name string `json:"name"`
ProvisionedSize uint64 `json:"provisioned_size"`
UsedSize uint64 `json:"used_size"`
}
// rbdDuImageList contains the list of images returned by 'rbd du'.
type rbdDuImageList struct {
Images []*rbdDuImage `json:"images"`
}
// getRbdDu runs 'rbd du' on the RBD image and returns a rbdDuImage struct with
// the result.
func getRbdDu(f *framework.Framework, pvc *v1.PersistentVolumeClaim) (*rbdDuImage, error) {
rdil := rbdDuImageList{}
imageData, err := getImageInfoFromPVC(pvc.Namespace, pvc.Name, f)
if err != nil {
return nil, err
}
cmd := fmt.Sprintf("rbd du --format=json %s %s", rbdOptions(defaultRBDPool), imageData.imageName)
stdout, _, err := execCommandInToolBoxPod(f, cmd, rookNamespace)
if err != nil {
return nil, err
}
err = json.Unmarshal([]byte(stdout), &rdil)
if err != nil {
return nil, err
}
for _, image := range rdil.Images {
if image.Name == imageData.imageName {
return image, nil
}
}
return nil, fmt.Errorf("image %s not found", imageData.imageName)
}
// sparsifyBackingRBDImage runs `rbd sparsify` on the RBD image. Once done, all
// data blocks that contain zeros are discarded/trimmed/unmapped and do not
// take up any space anymore. This can be used to verify that an empty, but
// allocated (with zerofill) extents have been released.
func sparsifyBackingRBDImage(f *framework.Framework, pvc *v1.PersistentVolumeClaim) error {
imageData, err := getImageInfoFromPVC(pvc.Namespace, pvc.Name, f)
if err != nil {
return err
}
cmd := fmt.Sprintf("rbd sparsify %s %s", rbdOptions(defaultRBDPool), imageData.imageName)
_, _, err = execCommandInToolBoxPod(f, cmd, rookNamespace)
return err
}
func deletePool(name string, cephfs bool, f *framework.Framework) error {
var cmds = []string{}
if cephfs {
// ceph fs fail
// ceph fs rm myfs --yes-i-really-mean-it
// ceph osd pool delete myfs-metadata myfs-metadata
// --yes-i-really-mean-it
// ceph osd pool delete myfs-data0 myfs-data0
// --yes-i-really-mean-it
cmds = append(cmds, fmt.Sprintf("ceph fs fail %s", name),
fmt.Sprintf("ceph fs rm %s --yes-i-really-mean-it", name),
fmt.Sprintf("ceph osd pool delete %s-metadata %s-metadata --yes-i-really-really-mean-it", name, name),
fmt.Sprintf("ceph osd pool delete %s-data0 %s-data0 --yes-i-really-really-mean-it", name, name))
} else {
// ceph osd pool delete replicapool replicapool
// --yes-i-really-mean-it
cmds = append(cmds, fmt.Sprintf("ceph osd pool delete %s %s --yes-i-really-really-mean-it", name, name))
}
for _, cmd := range cmds {
// discard stdErr as some commands prints warning in strErr
_, _, err := execCommandInToolBoxPod(f, cmd, rookNamespace)
if err != nil {
return err
}
}
return nil
}
func getPVCImageInfoInPool(f *framework.Framework, pvc *v1.PersistentVolumeClaim, pool string) (string, error) {
imageData, err := getImageInfoFromPVC(pvc.Namespace, pvc.Name, f)
if err != nil {
return "", err
}
stdOut, stdErr, err := execCommandInToolBoxPod(f,
fmt.Sprintf("rbd info %s", imageSpec(pool, imageData.imageName)), rookNamespace)
if err != nil {
return "", err
}
if stdErr != "" {
return "", fmt.Errorf("failed to get rbd info %v", stdErr)
}
if radosNamespace != "" {
e2elog.Logf("found image %s in pool %s namespace %s", imageData.imageName, pool, radosNamespace)
} else {
e2elog.Logf("found image %s in pool %s", imageData.imageName, pool)
}
return stdOut, nil
}
func checkPVCImageInPool(f *framework.Framework, pvc *v1.PersistentVolumeClaim, pool string) error {
_, err := getPVCImageInfoInPool(f, pvc, pool)
return err
}
func checkPVCDataPoolForImageInPool(f *framework.Framework, pvc *v1.PersistentVolumeClaim, pool, dataPool string) error {
stdOut, err := getPVCImageInfoInPool(f, pvc, pool)
if err != nil {
return err
}
if !strings.Contains(stdOut, "data_pool: "+dataPool) {
return fmt.Errorf("missing data pool value in image info, got info (%s)", stdOut)
}
return nil
}
func checkPVCImageJournalInPool(f *framework.Framework, pvc *v1.PersistentVolumeClaim, pool string) error {
imageData, err := getImageInfoFromPVC(pvc.Namespace, pvc.Name, f)
if err != nil {
return err
}
_, stdErr, err := execCommandInToolBoxPod(f,
fmt.Sprintf("rados listomapkeys %s csi.volume.%s", rbdOptions(pool), imageData.imageID), rookNamespace)
if err != nil {
return err
}
if stdErr != "" {
return fmt.Errorf("failed to listomapkeys %v", stdErr)
}
if radosNamespace != "" {
e2elog.Logf("found image journal %s in pool %s namespace %s", "csi.volume."+imageData.imageID, pool, radosNamespace)
} else {
e2elog.Logf("found image journal %s in pool %s", "csi.volume."+imageData.imageID, pool)
}
return nil
}
func checkPVCCSIJournalInPool(f *framework.Framework, pvc *v1.PersistentVolumeClaim, pool string) error {
imageData, err := getImageInfoFromPVC(pvc.Namespace, pvc.Name, f)
if err != nil {
return err
}
_, stdErr, err := execCommandInToolBoxPod(f,
fmt.Sprintf("rados getomapval %s csi.volumes.default csi.volume.%s", rbdOptions(pool), imageData.pvName), rookNamespace)
if err != nil {
return err
}
if stdErr != "" {
return fmt.Errorf("error getting fsid %v", stdErr)
}
if radosNamespace != "" {
e2elog.Logf("found CSI journal entry %s in pool %s namespace %s", "csi.volume."+imageData.pvName, pool, radosNamespace)
} else {
e2elog.Logf("found CSI journal entry %s in pool %s", "csi.volume."+imageData.pvName, pool)
}
return nil
}
func deletePVCImageJournalInPool(f *framework.Framework, pvc *v1.PersistentVolumeClaim, pool string) error {
imageData, err := getImageInfoFromPVC(pvc.Namespace, pvc.Name, f)
if err != nil {
return err
}
_, stdErr, err := execCommandInToolBoxPod(f,
fmt.Sprintf("rados rm %s csi.volume.%s", rbdOptions(pool), imageData.imageID), rookNamespace)
if err != nil {
return err
}
if stdErr != "" {
return fmt.Errorf("failed to remove omap %s csi.volume.%s with error %v", rbdOptions(pool), imageData.imageID, stdErr)
}
return nil
}
func deletePVCCSIJournalInPool(f *framework.Framework, pvc *v1.PersistentVolumeClaim, pool string) error {
imageData, err := getImageInfoFromPVC(pvc.Namespace, pvc.Name, f)
if err != nil {
return err
}
_, stdErr, err := execCommandInToolBoxPod(f,
fmt.Sprintf("rados rmomapkey %s csi.volumes.default csi.volume.%s", rbdOptions(pool), imageData.pvName), rookNamespace)
if err != nil {
return err
}
if stdErr != "" {
return fmt.Errorf("failed to remove %s csi.volumes.default csi.volume.%s with error %v", rbdOptions(pool), imageData.imageID, stdErr)
}
return nil
}
func validateThickPVC(f *framework.Framework, pvc *v1.PersistentVolumeClaim, size string) error {
pvc.Namespace = f.UniqueName
pvc.Spec.Resources.Requests[v1.ResourceStorage] = resource.MustParse(size)
err := createPVCAndvalidatePV(f.ClientSet, pvc, deployTimeout)
if err != nil {
return fmt.Errorf("failed to create PVC with error %w", err)
}
validateRBDImageCount(f, 1)
// nothing has been written, but the image should be allocated
du, err := getRbdDu(f, pvc)
if err != nil {
return fmt.Errorf("failed to get allocations of RBD image: %w", err)
} else if du.UsedSize == 0 || du.UsedSize != du.ProvisionedSize {
return fmt.Errorf("backing RBD image is not thick-provisioned (%d/%d)", du.UsedSize, du.ProvisionedSize)
}
// expanding the PVC should thick-allocate the expansion
// nolint:mnd // we want 2x the size so that extending is done
newSize := du.ProvisionedSize * 2
err = expandPVCSize(f.ClientSet, pvc, fmt.Sprintf("%d", newSize), deployTimeout)
if err != nil {
return fmt.Errorf("failed to expand PVC: %w", err)
}
// after expansion, the updated 'du' should be larger
du, err = getRbdDu(f, pvc)
if err != nil {
return fmt.Errorf("failed to get allocations of RBD image: %w", err)
} else if du.UsedSize != newSize {
return fmt.Errorf("backing RBD image is not extended thick-provisioned (%d/%d)", du.UsedSize, newSize)
}
// thick provisioning allows for sparsifying
err = sparsifyBackingRBDImage(f, pvc)
if err != nil {
return fmt.Errorf("failed to sparsify RBD image: %w", err)
}
// after sparsifying the image should not have any allocations
du, err = getRbdDu(f, pvc)
if err != nil {
return fmt.Errorf("backing RBD image is not thick-provisioned: %w", err)
} else if du.UsedSize != 0 {
return fmt.Errorf("backing RBD image was not sparsified (%d bytes allocated)", du.UsedSize)
}
err = deletePVCAndValidatePV(f.ClientSet, pvc, deployTimeout)
if err != nil {
return fmt.Errorf("failed to delete PVC with error: %w", err)
}
validateRBDImageCount(f, 0)
return nil
}