ceph-csi/e2e/rbd.go
ShyamsundarR 1a8f8e3c24 Add support for erasure coded pools
This commit adds support to mention dataPool parameter for the
topology constrained pools in the StorageClass, that can be
leveraged to mention erasure coded pool names to use for RBD
data instead of the replica pools.

Signed-off-by: ShyamsundarR <srangana@redhat.com>
2020-04-14 14:14:29 +00:00

576 lines
18 KiB
Go

package e2e
import (
"fmt"
"strings"
. "github.com/onsi/ginkgo" // nolint
clientset "k8s.io/client-go/kubernetes"
"k8s.io/kubernetes/test/e2e/framework"
e2elog "k8s.io/kubernetes/test/e2e/framework/log"
)
var (
rbdProvisioner = "csi-rbdplugin-provisioner.yaml"
rbdProvisionerRBAC = "csi-provisioner-rbac.yaml"
rbdProvisionerPSP = "csi-provisioner-psp.yaml"
rbdNodePlugin = "csi-rbdplugin.yaml"
rbdNodePluginRBAC = "csi-nodeplugin-rbac.yaml"
rbdNodePluginPSP = "csi-nodeplugin-psp.yaml"
configMap = "csi-config-map.yaml"
rbdDirPath = "../deploy/rbd/kubernetes/"
rbdExamplePath = "../examples/rbd/"
rbdDeploymentName = "csi-rbdplugin-provisioner"
rbdDaemonsetName = "csi-rbdplugin"
// Topology related variables
nodeRegionLabel = "test.failure-domain/region"
regionValue = "testregion"
nodeZoneLabel = "test.failure-domain/zone"
zoneValue = "testzone"
nodeCSIRegionLabel = "topology.rbd.csi.ceph.com/region"
nodeCSIZoneLabel = "topology.rbd.csi.ceph.com/zone"
rbdTopologyPool = "newrbdpool"
rbdTopologyDataPool = "replicapool" // NOTE: should be different than rbdTopologyPool for test to be effective
)
func deployRBDPlugin() {
// delete objects deployed by rook
data, err := replaceNamespaceInTemplate(rbdDirPath + rbdProvisionerRBAC)
if err != nil {
e2elog.Logf("failed to read content from %s %v", rbdDirPath+rbdProvisionerRBAC, err)
}
_, err = framework.RunKubectlInput(cephCSINamespace, data, "--ignore-not-found=true", ns, "delete", "-f", "-")
if err != nil {
e2elog.Logf("failed to delete provisioner rbac %s %v", rbdDirPath+rbdProvisionerRBAC, err)
}
data, err = replaceNamespaceInTemplate(rbdDirPath + rbdNodePluginRBAC)
if err != nil {
e2elog.Logf("failed to read content from %s %v", rbdDirPath+rbdNodePluginRBAC, err)
}
_, err = framework.RunKubectlInput(cephCSINamespace, data, "delete", "--ignore-not-found=true", ns, "-f", "-")
if err != nil {
e2elog.Logf("failed to delete nodeplugin rbac %s %v", rbdDirPath+rbdNodePluginRBAC, err)
}
createORDeleteRbdResouces("create")
}
func deleteRBDPlugin() {
createORDeleteRbdResouces("delete")
}
func createORDeleteRbdResouces(action string) {
data, err := replaceNamespaceInTemplate(rbdDirPath + rbdProvisioner)
if err != nil {
e2elog.Logf("failed to read content from %s %v", rbdDirPath+rbdProvisioner, err)
}
_, err = framework.RunKubectlInput(cephCSINamespace, data, action, ns, "-f", "-")
if err != nil {
e2elog.Logf("failed to %s rbd provisioner %v", action, err)
}
data, err = replaceNamespaceInTemplate(rbdDirPath + rbdProvisionerRBAC)
if err != nil {
e2elog.Logf("failed to read content from %s %v", rbdDirPath+rbdProvisionerRBAC, err)
}
_, err = framework.RunKubectlInput(cephCSINamespace, data, action, ns, "-f", "-")
if err != nil {
e2elog.Logf("failed to %s provisioner rbac %v", action, err)
}
data, err = replaceNamespaceInTemplate(rbdDirPath + rbdProvisionerPSP)
if err != nil {
e2elog.Logf("failed to read content from %s %v", rbdDirPath+rbdProvisionerPSP, err)
}
_, err = framework.RunKubectlInput(cephCSINamespace, data, action, "-f", "-")
if err != nil {
e2elog.Logf("failed to %s provisioner psp %v", action, err)
}
data, err = replaceNamespaceInTemplate(rbdDirPath + rbdNodePlugin)
if err != nil {
e2elog.Logf("failed to read content from %s %v", rbdDirPath+rbdNodePlugin, err)
}
domainLabel := nodeRegionLabel + "," + nodeZoneLabel
data = addTopologyDomainsToDSYaml(data, domainLabel)
_, err = framework.RunKubectlInput(cephCSINamespace, data, action, ns, "-f", "-")
if err != nil {
e2elog.Logf("failed to %s nodeplugin %v", action, err)
Fail(err.Error())
}
data, err = replaceNamespaceInTemplate(rbdDirPath + rbdNodePluginRBAC)
if err != nil {
e2elog.Logf("failed to read content from %s %v", rbdDirPath+rbdNodePluginRBAC, err)
}
_, err = framework.RunKubectlInput(cephCSINamespace, data, action, ns, "-f", "-")
if err != nil {
e2elog.Logf("failed to %s nodeplugin rbac %v", action, err)
}
data, err = replaceNamespaceInTemplate(rbdDirPath + rbdNodePluginPSP)
if err != nil {
e2elog.Logf("failed to read content from %s %v", rbdDirPath+rbdNodePluginPSP, err)
}
_, err = framework.RunKubectlInput(cephCSINamespace, data, action, ns, "-f", "-")
if err != nil {
e2elog.Logf("failed to %s nodeplugin psp %v", action, err)
}
}
var _ = Describe("RBD", func() {
f := framework.NewDefaultFramework("rbd")
var c clientset.Interface
// deploy RBD CSI
BeforeEach(func() {
c = f.ClientSet
if deployRBD {
createNodeLabel(f, nodeRegionLabel, regionValue)
createNodeLabel(f, nodeZoneLabel, zoneValue)
if cephCSINamespace != defaultNs {
err := createNamespace(c, cephCSINamespace)
if err != nil {
Fail(err.Error())
}
}
deployRBDPlugin()
}
createConfigMap(rbdDirPath, f.ClientSet, f)
createRBDStorageClass(f.ClientSet, f, nil, nil)
createRBDSecret(f.ClientSet, f)
deployVault(f.ClientSet, deployTimeout)
})
AfterEach(func() {
if CurrentGinkgoTestDescription().Failed {
// log pods created by helm chart
logsCSIPods("app=ceph-csi-rbd", c)
// log provisoner
logsCSIPods("app=csi-rbdplugin-provisioner", c)
// log node plugin
logsCSIPods("app=csi-rbdplugin", c)
}
deleteConfigMap(rbdDirPath)
deleteResource(rbdExamplePath + "secret.yaml")
deleteResource(rbdExamplePath + "storageclass.yaml")
// deleteResource(rbdExamplePath + "snapshotclass.yaml")
deleteVault()
if deployRBD {
deleteRBDPlugin()
if cephCSINamespace != defaultNs {
err := deleteNamespace(c, cephCSINamespace)
if err != nil {
Fail(err.Error())
}
}
}
deleteNodeLabel(c, nodeRegionLabel)
deleteNodeLabel(c, nodeZoneLabel)
// Remove the CSI labels that get added
deleteNodeLabel(c, nodeCSIRegionLabel)
deleteNodeLabel(c, nodeCSIZoneLabel)
})
Context("Test RBD CSI", func() {
It("Test RBD CSI", func() {
pvcPath := rbdExamplePath + "pvc.yaml"
appPath := rbdExamplePath + "pod.yaml"
rawPvcPath := rbdExamplePath + "raw-block-pvc.yaml"
rawAppPath := rbdExamplePath + "raw-block-pod.yaml"
// pvcClonePath := rbdExamplePath + "pvc-restore.yaml"
// appClonePath := rbdExamplePath + "pod-restore.yaml"
// snapshotPath := rbdExamplePath + "snapshot.yaml"
By("checking provisioner deployment is running", func() {
err := waitForDeploymentComplete(rbdDeploymentName, cephCSINamespace, f.ClientSet, deployTimeout)
if err != nil {
Fail(err.Error())
}
})
By("checking nodeplugin deamonsets is running", func() {
err := waitForDaemonSets(rbdDaemonsetName, cephCSINamespace, f.ClientSet, deployTimeout)
if err != nil {
Fail(err.Error())
}
})
By("create a PVC and Bind it to an app", func() {
validatePVCAndAppBinding(pvcPath, appPath, f)
})
By("create a PVC and Bind it to an app with normal user", func() {
validateNormalUserPVCAccess(pvcPath, f)
})
By("create a PVC and Bind it to an app with ext4 as the FS ", func() {
deleteResource(rbdExamplePath + "storageclass.yaml")
createRBDStorageClass(f.ClientSet, f, nil, map[string]string{"csi.storage.k8s.io/fstype": "ext4"})
validatePVCAndAppBinding(pvcPath, appPath, f)
deleteResource(rbdExamplePath + "storageclass.yaml")
createRBDStorageClass(f.ClientSet, f, nil, nil)
})
By("create a PVC and Bind it to an app with encrypted RBD volume", func() {
deleteResource(rbdExamplePath + "storageclass.yaml")
createRBDStorageClass(f.ClientSet, f, nil, map[string]string{"encrypted": "true"})
validateEncryptedPVCAndAppBinding(pvcPath, appPath, "", f)
deleteResource(rbdExamplePath + "storageclass.yaml")
createRBDStorageClass(f.ClientSet, f, nil, nil)
})
By("create a PVC and Bind it to an app with encrypted RBD volume with Vault KMS", func() {
deleteResource(rbdExamplePath + "storageclass.yaml")
scOpts := map[string]string{
"encrypted": "true",
"encryptionKMSID": "vault-test",
}
createRBDStorageClass(f.ClientSet, f, nil, scOpts)
validateEncryptedPVCAndAppBinding(pvcPath, appPath, "vault", f)
deleteResource(rbdExamplePath + "storageclass.yaml")
createRBDStorageClass(f.ClientSet, f, nil, nil)
})
// skipping snapshot testing
// By("create a PVC clone and Bind it to an app", func() {
// createRBDSnapshotClass(f)
// pvc, err := loadPVC(pvcPath)
// if err != nil {
// Fail(err.Error())
// }
// pvc.Namespace = f.UniqueName
// e2elog.Logf("The PVC template %+v", pvc)
// err = createPVCAndvalidatePV(f.ClientSet, pvc, deployTimeout)
// if err != nil {
// Fail(err.Error())
// }
// // validate created backend rbd images
// images := listRBDImages(f)
// if len(images) != 1 {
// e2elog.Logf("backend image count %d expected image count %d", len(images), 1)
// Fail("validate backend image failed")
// }
// snap := getSnapshot(snapshotPath)
// snap.Namespace = f.UniqueName
// snap.Spec.Source.Name = pvc.Name
// snap.Spec.Source.Kind = "PersistentVolumeClaim"
// err = createSnapshot(&snap, deployTimeout)
// if err != nil {
// Fail(err.Error())
// }
// pool := "replicapool"
// snapList, err := listSnapshots(f, pool, images[0])
// if err != nil {
// Fail(err.Error())
// }
// if len(snapList) != 1 {
// e2elog.Logf("backend snapshot not matching kube snap count,snap count = % kube snap count %d", len(snapList), 1)
// Fail("validate backend snapshot failed")
// }
// validatePVCAndAppBinding(pvcClonePath, appClonePath, f)
// err = deleteSnapshot(&snap, deployTimeout)
// if err != nil {
// Fail(err.Error())
// }
// err = deletePVCAndValidatePV(f.ClientSet, pvc, deployTimeout)
// if err != nil {
// Fail(err.Error())
// }
// })
By("create a block type PVC and Bind it to an app", func() {
validatePVCAndAppBinding(rawPvcPath, rawAppPath, f)
})
By("create/delete multiple PVCs and Apps", func() {
totalCount := 2
pvc, err := loadPVC(pvcPath)
if err != nil {
Fail(err.Error())
}
pvc.Namespace = f.UniqueName
app, err := loadApp(appPath)
if err != nil {
Fail(err.Error())
}
app.Namespace = f.UniqueName
// create pvc and app
for i := 0; i < totalCount; i++ {
name := fmt.Sprintf("%s%d", f.UniqueName, i)
err := createPVCAndApp(name, f, pvc, app, deployTimeout)
if err != nil {
Fail(err.Error())
}
}
// validate created backend rbd images
images := listRBDImages(f)
if len(images) != totalCount {
e2elog.Logf("backend image creation not matching pvc count, image count = % pvc count %d", len(images), totalCount)
Fail("validate multiple pvc failed")
}
// delete pvc and app
for i := 0; i < totalCount; i++ {
name := fmt.Sprintf("%s%d", f.UniqueName, i)
err := deletePVCAndApp(name, f, pvc, app)
if err != nil {
Fail(err.Error())
}
}
// validate created backend rbd images
images = listRBDImages(f)
if len(images) > 0 {
e2elog.Logf("left out rbd backend images count %d", len(images))
Fail("validate multiple pvc failed")
}
})
By("check data persist after recreating pod with same pvc", func() {
err := checkDataPersist(pvcPath, appPath, f)
if err != nil {
Fail(err.Error())
}
})
By("Resize Filesystem PVC and check application directory size", func() {
v, err := f.ClientSet.Discovery().ServerVersion()
if err != nil {
e2elog.Logf("failed to get server version with error %v", err)
Fail(err.Error())
}
// Resize 0.3.0 is only supported from v1.15+
if v.Major > "1" || (v.Major == "1" && v.Minor >= "15") {
err := resizePVCAndValidateSize(pvcPath, appPath, f)
if err != nil {
e2elog.Logf("failed to resize filesystem PVC %v", err)
Fail(err.Error())
}
deleteResource(rbdExamplePath + "storageclass.yaml")
createRBDStorageClass(f.ClientSet, f, nil, map[string]string{"csi.storage.k8s.io/fstype": "xfs"})
err = resizePVCAndValidateSize(pvcPath, appPath, f)
if err != nil {
e2elog.Logf("failed to resize filesystem PVC %v", err)
Fail(err.Error())
}
}
})
By("Resize Block PVC and check Device size", func() {
v, err := f.ClientSet.Discovery().ServerVersion()
if err != nil {
e2elog.Logf("failed to get server version with error %v", err)
Fail(err.Error())
}
// Block PVC resize is supported in kubernetes 1.16+
if v.Major > "1" || (v.Major == "1" && v.Minor >= "16") {
err := resizePVCAndValidateSize(rawPvcPath, rawAppPath, f)
if err != nil {
e2elog.Logf("failed to resize block PVC %v", err)
Fail(err.Error())
}
}
})
By("Test unmount after nodeplugin restart", func() {
pvc, err := loadPVC(pvcPath)
if err != nil {
Fail(err.Error())
}
pvc.Namespace = f.UniqueName
app, err := loadApp(appPath)
if err != nil {
Fail(err.Error())
}
app.Namespace = f.UniqueName
err = createPVCAndApp("", f, pvc, app, deployTimeout)
if err != nil {
Fail(err.Error())
}
// delete rbd nodeplugin pods
err = deletePodWithLabel("app=csi-rbdplugin", cephCSINamespace, false)
if err != nil {
Fail(err.Error())
}
// wait for nodeplugin pods to come up
err = waitForDaemonSets(rbdDaemonsetName, cephCSINamespace, f.ClientSet, deployTimeout)
if err != nil {
Fail(err.Error())
}
err = deletePVCAndApp("", f, pvc, app)
if err != nil {
Fail(err.Error())
}
})
By("create PVC in storageClass with volumeNamePrefix", func() {
volumeNamePrefix := "foo-bar-"
deleteResource(rbdExamplePath + "storageclass.yaml")
createRBDStorageClass(f.ClientSet, f, nil, map[string]string{"volumeNamePrefix": volumeNamePrefix})
// set up PVC
pvc, err := loadPVC(pvcPath)
if err != nil {
Fail(err.Error())
}
pvc.Namespace = f.UniqueName
err = createPVCAndvalidatePV(f.ClientSet, pvc, deployTimeout)
if err != nil {
Fail(err.Error())
}
// list RBD images and check if one of them has the same prefix
foundIt := false
for _, imgName := range listRBDImages(f) {
fmt.Printf("Checking prefix on %s\n", imgName)
if strings.HasPrefix(imgName, volumeNamePrefix) {
foundIt = true
break
}
}
// clean up after ourselves
err = deletePVCAndValidatePV(f.ClientSet, pvc, deployTimeout)
if err != nil {
Fail(err.Error())
}
deleteResource(rbdExamplePath + "storageclass.yaml")
createRBDStorageClass(f.ClientSet, f, nil, nil)
if !foundIt {
Fail(fmt.Sprintf("could not find image with prefix %s", volumeNamePrefix))
}
})
By("validate RBD static FileSystem PVC", func() {
err := validateRBDStaticPV(f, appPath, false)
if err != nil {
Fail(err.Error())
}
})
By("validate RBD static Block PVC", func() {
err := validateRBDStaticPV(f, rawAppPath, true)
if err != nil {
Fail(err.Error())
}
})
By("validate mount options in app pod", func() {
mountFlags := []string{"discard"}
err := checkMountOptions(pvcPath, appPath, f, mountFlags)
if err != nil {
Fail(err.Error())
}
})
By("creating an app with a PVC, using a topology constrained StorageClass", func() {
By("checking node has required CSI topology labels set", func() {
checkNodeHasLabel(f.ClientSet, nodeCSIRegionLabel, regionValue)
checkNodeHasLabel(f.ClientSet, nodeCSIZoneLabel, zoneValue)
})
By("creating a StorageClass with delayed binding mode and CSI topology parameter")
deleteResource(rbdExamplePath + "storageclass.yaml")
topologyConstraint := "[{\"poolName\":\"" + rbdTopologyPool + "\",\"domainSegments\":" +
"[{\"domainLabel\":\"region\",\"value\":\"" + regionValue + "\"}," +
"{\"domainLabel\":\"zone\",\"value\":\"" + zoneValue + "\"}]}]"
createRBDStorageClass(f.ClientSet, f,
map[string]string{"volumeBindingMode": "WaitForFirstConsumer"},
map[string]string{"topologyConstrainedPools": topologyConstraint})
By("creating an app using a PV from the delayed binding mode StorageClass")
pvc, app := createPVCAndAppBinding(pvcPath, appPath, f, 0)
By("ensuring created PV has required node selector values populated")
checkPVSelectorValuesForPVC(f, pvc)
By("ensuring created PV has its image in the topology specific pool")
err := checkPVCImageInPool(f, pvc, rbdTopologyPool)
if err != nil {
Fail(err.Error())
}
By("ensuring created PV has its image journal in the topology specific pool")
err = checkPVCImageJournalInPool(f, pvc, rbdTopologyPool)
if err != nil {
Fail(err.Error())
}
By("ensuring created PV has its CSI journal in the CSI journal specific pool")
err = checkPVCCSIJournalInPool(f, pvc, "replicapool")
if err != nil {
Fail(err.Error())
}
err = deletePVCAndApp("", f, pvc, app)
if err != nil {
Fail(err.Error())
}
By("checking if data pool parameter is honored", func() {
deleteResource(rbdExamplePath + "storageclass.yaml")
topologyConstraint := "[{\"poolName\":\"" + rbdTopologyPool + "\",\"dataPool\":\"" + rbdTopologyDataPool +
"\",\"domainSegments\":" +
"[{\"domainLabel\":\"region\",\"value\":\"" + regionValue + "\"}," +
"{\"domainLabel\":\"zone\",\"value\":\"" + zoneValue + "\"}]}]"
createRBDStorageClass(f.ClientSet, f,
map[string]string{"volumeBindingMode": "WaitForFirstConsumer"},
map[string]string{"topologyConstrainedPools": topologyConstraint})
By("creating an app using a PV from the delayed binding mode StorageClass with a data pool")
pvc, app = createPVCAndAppBinding(pvcPath, appPath, f, 0)
By("ensuring created PV has its image in the topology specific pool")
err = checkPVCImageInPool(f, pvc, rbdTopologyPool)
if err != nil {
Fail(err.Error())
}
By("ensuring created image has the right data pool parameter set")
err = checkPVCDataPoolForImageInPool(f, pvc, rbdTopologyPool, rbdTopologyDataPool)
if err != nil {
Fail(err.Error())
}
// cleanup and undo changes made by the test
err = deletePVCAndApp("", f, pvc, app)
if err != nil {
Fail(err.Error())
}
})
// cleanup and undo changes made by the test
deleteResource(rbdExamplePath + "storageclass.yaml")
createRBDStorageClass(f.ClientSet, f, nil, nil)
})
// Make sure this should be last testcase in this file, because
// it deletes pool
By("Create a PVC and Delete PVC when backend pool deleted", func() {
err := pvcDeleteWhenPoolNotFound(pvcPath, false, f)
if err != nil {
Fail(err.Error())
}
})
})
})
})