diff --git a/e2e/cephfs.go b/e2e/cephfs.go index 85685fa05..4f4e65184 100644 --- a/e2e/cephfs.go +++ b/e2e/cephfs.go @@ -23,6 +23,7 @@ var ( cephfsDeamonSetName = "csi-cephfsplugin" cephfsDirPath = "../deploy/cephfs/kubernetes/" cephfsExamplePath = "../examples/cephfs/" + subvolumegroup = "e2e" ) func deployCephfsPlugin() { @@ -115,7 +116,7 @@ var _ = Describe("cephfs", func() { var c clientset.Interface // deploy cephfs CSI BeforeEach(func() { - if !testCephFS { + if !testCephFS || upgradeTesting { Skip("Skipping CephFS E2E") } c = f.ClientSet @@ -133,7 +134,7 @@ var _ = Describe("cephfs", func() { }) AfterEach(func() { - if !testCephFS { + if !testCephFS || upgradeTesting { Skip("Skipping CephFS E2E") } if CurrentGinkgoTestDescription().Failed { diff --git a/e2e/e2e_test.go b/e2e/e2e_test.go index def681cdc..9dfea337c 100644 --- a/e2e/e2e_test.go +++ b/e2e/e2e_test.go @@ -22,6 +22,8 @@ func init() { flag.BoolVar(&deployRBD, "deploy-rbd", true, "deploy rbd csi driver") flag.BoolVar(&testCephFS, "test-cephfs", true, "test cephfs csi driver") flag.BoolVar(&testRBD, "test-rbd", true, "test rbd csi driver") + flag.BoolVar(&upgradeTesting, "upgrade-testing", false, "perform upgrade testing") + flag.StringVar(&upgradeVersion, "upgrade-version", "v2.1.2", "target version for upgrade testing") flag.StringVar(&cephCSINamespace, "cephcsi-namespace", defaultNs, "namespace in which cephcsi deployed") flag.StringVar(&rookNamespace, "rook-namespace", "rook-ceph", "namespace in which rook is deployed") setDefaultKubeconfig() diff --git a/e2e/upgrade-cephfs.go b/e2e/upgrade-cephfs.go new file mode 100644 index 000000000..cf6fba978 --- /dev/null +++ b/e2e/upgrade-cephfs.go @@ -0,0 +1,185 @@ +package e2e + +import ( + "context" + "os" + + . "github.com/onsi/ginkgo" // nolint + v1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/version" + clientset "k8s.io/client-go/kubernetes" + "k8s.io/kubernetes/test/e2e/framework" + e2elog "k8s.io/kubernetes/test/e2e/framework/log" +) + +var _ = Describe("CephFS Upgrade Testing", func() { + f := framework.NewDefaultFramework("upgrade-test-cephfs") + var ( + c clientset.Interface + pvc *v1.PersistentVolumeClaim + app *v1.Pod + // cwd stores the initial working directory. + cwd string + ) + // deploy cephfs CSI + BeforeEach(func() { + if !upgradeTesting || !testCephFS { + Skip("Skipping CephFS Upgrade Test") + } + c = f.ClientSet + if cephCSINamespace != defaultNs { + err := createNamespace(c, cephCSINamespace) + if err != nil { + Fail(err.Error()) + } + } + + // fetch current working directory to switch back + // when we are done upgrading. + var err error + cwd, err = os.Getwd() + if err != nil { + Fail(err.Error()) + } + err = upgradeAndDeployCSI(upgradeVersion, "cephfs") + if err != nil { + Fail(err.Error()) + } + createConfigMap(cephfsDirPath, f.ClientSet, f) + createCephfsSecret(f.ClientSet, f) + createCephfsStorageClass(f.ClientSet, f, true, "") + }) + AfterEach(func() { + if !testCephFS || !upgradeTesting { + Skip("Skipping CephFS Upgrade Test") + } + if CurrentGinkgoTestDescription().Failed { + // log pods created by helm chart + logsCSIPods("app=ceph-csi-cephfs", c) + // log provisoner + logsCSIPods("app=csi-cephfsplugin-provisioner", c) + // log node plugin + logsCSIPods("app=csi-cephfsplugin", c) + } + deleteConfigMap(cephfsDirPath) + deleteResource(cephfsExamplePath + "secret.yaml") + deleteResource(cephfsExamplePath + "storageclass.yaml") + if deployCephFS { + deleteCephfsPlugin() + if cephCSINamespace != defaultNs { + err := deleteNamespace(c, cephCSINamespace) + if err != nil { + Fail(err.Error()) + } + } + } + }) + + Context("Cephfs Upgrade Test", func() { + It("Cephfs Upgrade Test", func() { + + By("checking provisioner deployment is running") + err := waitForDeploymentComplete(cephfsDeploymentName, cephCSINamespace, f.ClientSet, deployTimeout) + if err != nil { + Fail(err.Error()) + } + + By("checking nodeplugin deamonsets is running") + err = waitForDaemonSets(cephfsDeamonSetName, cephCSINamespace, f.ClientSet, deployTimeout) + if err != nil { + Fail(err.Error()) + } + + By("upgrade to latest changes and verify app re-mount", func() { + // TODO: fetch pvc size from spec. + pvcSize := "2Gi" + pvcPath := cephfsExamplePath + "pvc.yaml" + appPath := cephfsExamplePath + "pod.yaml" + + pvc, err = loadPVC(pvcPath) + if pvc == nil { + Fail(err.Error()) + } + pvc.Namespace = f.UniqueName + e2elog.Logf("The PVC template %+v", pvc) + + app, err = loadApp(appPath) + if err != nil { + Fail(err.Error()) + } + app.Namespace = f.UniqueName + app.Labels = map[string]string{"app": "upgrade-testing"} + pvc.Spec.Resources.Requests[v1.ResourceStorage] = resource.MustParse(pvcSize) + err = createPVCAndApp("", f, pvc, app, deployTimeout) + if err != nil { + Fail(err.Error()) + } + err = deletePod(app.Name, app.Namespace, f.ClientSet, deployTimeout) + if err != nil { + Fail(err.Error()) + } + deleteCephfsPlugin() + + // switch back to current changes. + err = os.Chdir(cwd) + if err != nil { + Fail(err.Error()) + } + deployCephfsPlugin() + + app.Labels = map[string]string{"app": "upgrade-testing"} + // validate if the app gets bound to a pvc created by + // an earlier release. + err = createApp(f.ClientSet, app, deployTimeout) + if err != nil { + Fail(err.Error()) + } + }) + + By("Resize pvc and verify expansion", func() { + var v *version.Info + pvcExpandSize := "5Gi" + v, err = f.ClientSet.Discovery().ServerVersion() + if err != nil { + e2elog.Logf("failed to get server version with error %v", err) + Fail(err.Error()) + } + // Resize 0.3.0 is only supported from v1.15+ + if v.Major > "1" || (v.Major == "1" && v.Minor >= "15") { + opt := metav1.ListOptions{ + LabelSelector: "app=upgrade-testing", + } + pvc, err = f.ClientSet.CoreV1().PersistentVolumeClaims(pvc.Namespace).Get(context.TODO(), pvc.Name, metav1.GetOptions{}) + if err != nil { + Fail(err.Error()) + } + + // resize PVC + err = expandPVCSize(f.ClientSet, pvc, pvcExpandSize, deployTimeout) + if err != nil { + Fail(err.Error()) + } + // wait for application pod to come up after resize + err = waitForPodInRunningState(app.Name, app.Namespace, f.ClientSet, deployTimeout) + if err != nil { + Fail(err.Error()) + } + // validate if resize is successful. + err = checkDirSize(app, f, &opt, pvcExpandSize) + if err != nil { + Fail(err.Error()) + } + } + + }) + + By("delete pvc and app") + err = deletePVCAndApp("", f, pvc, app) + if err != nil { + Fail(err.Error()) + } + }) + }) +}) diff --git a/e2e/upgrade.go b/e2e/upgrade.go new file mode 100644 index 000000000..d9a971697 --- /dev/null +++ b/e2e/upgrade.go @@ -0,0 +1,43 @@ +package e2e + +import ( + "errors" + "fmt" + "os" + "os/exec" +) + +// upgradeCSI deploys a desired ceph-csi release version. +func upgradeCSI(version string) error { + tempDir := "/tmp/ceph-csi" + gitRepo := "https://github.com/ceph/ceph-csi.git" + // clone the desired release branch inside a temporary directory. + cmd := exec.Command("git", "clone", "--single-branch", "--branch", version, gitRepo, tempDir) + cmd.Stderr = os.Stderr + cmd.Stdout = os.Stdout + if err := cmd.Run(); err != nil { + return fmt.Errorf("unable to clone repo %s : %w", gitRepo, err) + } + err := os.Chdir(tempDir + "/e2e") + if err != nil { + return fmt.Errorf("unable to switch directory : %w", err) + } + return nil +} + +// upgradeAndDeployCSI upgrades the CSI to a specific release. +func upgradeAndDeployCSI(version, testtype string) error { + err := upgradeCSI(version) + if err != nil { + return fmt.Errorf("failed to upgrade driver %w", err) + } + switch testtype { + case "cephfs": + deployCephfsPlugin() + case "rbd": + deployRBDPlugin() + default: + return errors.New("incorrect test type, can be cephfs/rbd") + } + return nil +} diff --git a/e2e/utils.go b/e2e/utils.go index 53ee0a15c..6f1aa423e 100644 --- a/e2e/utils.go +++ b/e2e/utils.go @@ -55,6 +55,8 @@ var ( deployRBD bool testCephFS bool testRBD bool + upgradeTesting bool + upgradeVersion string cephCSINamespace string rookNamespace string ns string @@ -353,7 +355,10 @@ func createConfigMap(pluginPath string, c kubernetes.Interface, f *framework.Fra ClusterID: fsID, Monitors: mons, }} - conmap[0].CephFS.SubvolumeGroup = "e2e" + if upgradeTesting { + subvolumegroup = "csi" + } + conmap[0].CephFS.SubvolumeGroup = subvolumegroup data, err := json.Marshal(conmap) Expect(err).Should(BeNil()) cm.Data["config.json"] = string(data) @@ -902,7 +907,7 @@ func deleteBackingCephFSVolume(f *framework.Framework, pvc *v1.PersistentVolumeC return err } - _, stdErr := execCommandInToolBoxPod(f, "ceph fs subvolume rm myfs "+imageData.imageName+" e2e", rookNamespace) + _, stdErr := execCommandInToolBoxPod(f, "ceph fs subvolume rm myfs "+imageData.imageName+" "+subvolumegroup, rookNamespace) Expect(stdErr).Should(BeEmpty()) if stdErr != "" {