ceph-csi/e2e/cephfs.go

package e2e

import (
	"fmt"
	"time"

	. "github.com/onsi/ginkgo" // nolint

	v1 "k8s.io/api/core/v1"
	apierrs "k8s.io/apimachinery/pkg/api/errors"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	clientset "k8s.io/client-go/kubernetes"
	"k8s.io/kubernetes/test/e2e/framework"
	e2elog "k8s.io/kubernetes/test/e2e/framework/log"
)

var (
	cephfsProvisioner     = "csi-cephfsplugin-provisioner.yaml"
	cephfsProvisionerRBAC = "csi-provisioner-rbac.yaml"
	cephfsProvisionerPSP  = "csi-provisioner-psp.yaml"
	cephfsNodePlugin      = "csi-cephfsplugin.yaml"
	cephfsNodePluginRBAC  = "csi-nodeplugin-rbac.yaml"
	cephfsNodePluginPSP   = "csi-nodeplugin-psp.yaml"
	cephfsDeploymentName  = "csi-cephfsplugin-provisioner"
	cephfsDeamonSetName   = "csi-cephfsplugin"
	cephfsDirPath         = "../deploy/cephfs/kubernetes/"
	cephfsExamplePath     = "../examples/cephfs/"
)

func deployCephfsPlugin() {
	// delete objects deployed by rook
	framework.RunKubectlOrDie("delete", "--ignore-not-found=true", "-f", cephfsDirPath+cephfsProvisionerRBAC, fmt.Sprintf("--namespace=%s", cephCSINamespace))
	framework.RunKubectlOrDie("delete", "--ignore-not-found=true", "-f", cephfsDirPath+cephfsNodePluginRBAC, fmt.Sprintf("--namespace=%s", cephCSINamespace))
	// deploy provisioner
	framework.RunKubectlOrDie("create", "-f", cephfsDirPath+cephfsProvisioner, fmt.Sprintf("--namespace=%s", cephCSINamespace))
	framework.RunKubectlOrDie("create", "-f", cephfsDirPath+cephfsProvisionerRBAC, fmt.Sprintf("--namespace=%s", cephCSINamespace))
	framework.RunKubectlOrDie("create", "-f", cephfsDirPath+cephfsProvisionerPSP, fmt.Sprintf("--namespace=%s", cephCSINamespace))
	// deploy nodeplugin
	framework.RunKubectlOrDie("create", "-f", cephfsDirPath+cephfsNodePlugin, fmt.Sprintf("--namespace=%s", cephCSINamespace))
	framework.RunKubectlOrDie("create", "-f", cephfsDirPath+cephfsNodePluginRBAC, fmt.Sprintf("--namespace=%s", cephCSINamespace))
	framework.RunKubectlOrDie("create", "-f", cephfsDirPath+cephfsNodePluginPSP, fmt.Sprintf("--namespace=%s", cephCSINamespace))
}

func deleteCephfsPlugin() {
	_, err := framework.RunKubectl("delete", "-f", cephfsDirPath+cephfsProvisioner, fmt.Sprintf("--namespace=%s", cephCSINamespace))
	if err != nil {
		e2elog.Logf("failed to delete cephfs provisioner %v", err)
	}
	_, err = framework.RunKubectl("delete", "-f", cephfsDirPath+cephfsProvisionerRBAC, fmt.Sprintf("--namespace=%s", cephCSINamespace))
	if err != nil {
		e2elog.Logf("failed to delete cephfs provisioner rbac %v", err)
	}
	_, err = framework.RunKubectl("delete", "-f", cephfsDirPath+cephfsProvisionerPSP, fmt.Sprintf("--namespace=%s", cephCSINamespace))
	if err != nil {
		e2elog.Logf("failed to delete cephfs provisioner psp %v", err)
	}
	_, err = framework.RunKubectl("delete", "-f", cephfsDirPath+cephfsNodePlugin, fmt.Sprintf("--namespace=%s", cephCSINamespace))
	if err != nil {
		e2elog.Logf("failed to delete cephfs nodeplugin %v", err)
	}
	_, err = framework.RunKubectl("delete", "-f", cephfsDirPath+cephfsNodePluginRBAC, fmt.Sprintf("--namespace=%s", cephCSINamespace))
	if err != nil {
		e2elog.Logf("failed to delete cephfs nodeplugin rbac %v", err)
	}
	_, err = framework.RunKubectl("delete", "-f", cephfsDirPath+cephfsNodePluginPSP, fmt.Sprintf("--namespace=%s", cephCSINamespace))
	if err != nil {
		e2elog.Logf("failed to delete cephfs nodeplugin psp %v", err)
	}
}

var _ = Describe("cephfs", func() {
	f := framework.NewDefaultFramework("cephfs")
	var c clientset.Interface
	// deploy cephfs CSI
	BeforeEach(func() {
		c = f.ClientSet
		createConfigMap(cephfsDirPath, f.ClientSet, f)
		if deployCephFS {
			if cephCSINamespace != defaultNs {
				// create namespace
				ns := &v1.Namespace{
					ObjectMeta: metav1.ObjectMeta{
						Name: cephCSINamespace,
					},
				}
				_, err := c.CoreV1().Namespaces().Create(ns)
				if err != nil && !apierrs.IsAlreadyExists(err) {
					Fail(err.Error())
				}
			}

			deployCephfsPlugin()
		}
		createCephfsSecret(f.ClientSet, f)
	})

	AfterEach(func() {
		if CurrentGinkgoTestDescription().Failed {
			// log provisoner
			logsCSIPods("app=csi-cephfsplugin-provisioner", c)
			// log node plugin
			logsCSIPods("app=csi-cephfsplugin", c)
		}
		deleteConfigMap(cephfsDirPath)
		deleteResource(cephfsExamplePath + "secret.yaml")
		deleteResource(cephfsExamplePath + "storageclass.yaml")
		if deployCephFS {
			deleteCephfsPlugin()
			if cephCSINamespace != defaultNs {
				err := c.CoreV1().Namespaces().Delete(cephCSINamespace, nil)
				if err != nil && !apierrs.IsNotFound(err) {
					Fail(err.Error())
				}
				err = framework.WaitForNamespacesDeleted(c, []string{cephCSINamespace}, time.Duration(deployTimeout)*time.Minute)
				if err != nil {
					Fail(err.Error())
				}
			}
		}
	})

	Context("Test cephfs CSI", func() {
		It("Test cephfs CSI", func() {
			pvcPath := cephfsExamplePath + "pvc.yaml"
			appPath := cephfsExamplePath + "pod.yaml"

			By("checking provisioner deployment is running")
			var err error
			err = waitForDeploymentComplete(cephfsDeploymentName, cephCSINamespace, f.ClientSet, deployTimeout)
			if err != nil {
				Fail(err.Error())
			}

			By("checking nodeplugin deamonsets is running")
			err = waitForDaemonSets(cephfsDeamonSetName, cephCSINamespace, f.ClientSet, deployTimeout)
			if err != nil {
				Fail(err.Error())
			}

			By("create a storage class with pool and a PVC then Bind it to an app", func() {
				createCephfsStorageClass(f.ClientSet, f, true)
				validatePVCAndAppBinding(pvcPath, appPath, f)
				deleteResource(cephfsExamplePath + "storageclass.yaml")
			})

			createCephfsStorageClass(f.ClientSet, f, false)

			By("create and delete a PVC", func() {
				By("create a PVC and Bind it to an app", func() {
					validatePVCAndAppBinding(pvcPath, appPath, f)

				})

				By("create a PVC and Bind it to an app with normal user", func() {
					validateNormalUserPVCAccess(pvcPath, f)
				})

				By("create/delete multiple PVCs and Apps", func() {
					totalCount := 2
					pvc, err := loadPVC(pvcPath)
					if err != nil {
						Fail(err.Error())
					}
					pvc.Namespace = f.UniqueName

					app, err := loadApp(appPath)
					if err != nil {
						Fail(err.Error())
					}
					app.Namespace = f.UniqueName
					// create pvc and app
					for i := 0; i < totalCount; i++ {
						name := fmt.Sprintf("%s%d", f.UniqueName, i)
						err := createPVCAndApp(name, f, pvc, app)
						if err != nil {
							Fail(err.Error())
						}

					}
					// TODO add cephfs backend validation

					// delete pvc and app
					for i := 0; i < totalCount; i++ {
						name := fmt.Sprintf("%s%d", f.UniqueName, i)
						err := deletePVCAndApp(name, f, pvc, app)
						if err != nil {
							Fail(err.Error())
						}

					}
				})

				By("check data persist after recreating pod with same pvc", func() {
					err := checkDataPersist(pvcPath, appPath, f)
					if err != nil {
						Fail(err.Error())
					}
				})

				By("creating a PVC, deleting backing subvolume, and checking successful PV deletion", func() {
					pvc, err := loadPVC(pvcPath)
					if pvc == nil {
						Fail(err.Error())
					}
					pvc.Namespace = f.UniqueName

					err = createPVCAndvalidatePV(f.ClientSet, pvc, deployTimeout)
					if err != nil {
						Fail(err.Error())
					}

					err = deleteBackingCephFSVolume(f, pvc)
					if err != nil {
						Fail(err.Error())
					}

					err = deletePVCAndValidatePV(f.ClientSet, pvc, deployTimeout)
					if err != nil {
						Fail(err.Error())
					}
				})

				By("Resize PVC and check application directory size", func() {
					v, err := f.ClientSet.Discovery().ServerVersion()
					if err != nil {
						e2elog.Logf("failed to get server version with error %v", err)
						Fail(err.Error())
					}

					// Resize 0.3.0 is only supported from v1.15+
					if v.Major > "1" || (v.Major == "1" && v.Minor >= "15") {
						err := resizePVCAndValidateSize(pvcPath, appPath, f)
						if err != nil {
							e2elog.Logf("failed to resize PVC %v", err)
							Fail(err.Error())
						}
					}

				})

				// Make sure this should be last testcase in this file, because
				// it deletes pool
				By("Create a PVC and Delete PVC when backend pool deleted", func() {
					err := pvcDeleteWhenPoolNotFound(pvcPath, true, f)
					if err != nil {
						Fail(err.Error())
					}
				})

			})

		})
	})

})