e2e: rework on E2E framework

rework of E2E framework for better code
organization and add more helpful logs for
debugging.

Signed-off-by: Madhu Rajanna <madhupr007@gmail.com>
This commit is contained in:
Madhu Rajanna 2020-09-03 15:04:29 +05:30 committed by mergify[bot]
parent 3ea22bc5a8
commit b4693dcffe
15 changed files with 2344 additions and 1957 deletions

File diff suppressed because it is too large Load Diff

131
e2e/cephfs_helper.go Normal file
View File

@ -0,0 +1,131 @@
package e2e
import (
"context"
"encoding/json"
"fmt"
"strings"
v1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/client-go/kubernetes"
"k8s.io/kubernetes/test/e2e/framework"
)
const (
adminUser = "admin"
)
// validateSubvolumegroup validates whether subvolumegroup is present.
func validateSubvolumegroup(f *framework.Framework, subvolgrp string) error {
cmd := fmt.Sprintf("ceph fs subvolumegroup getpath myfs %s", subvolgrp)
stdOut, stdErr, err := execCommandInToolBoxPod(f, cmd, rookNamespace)
if err != nil {
return err
}
if stdErr != "" {
return fmt.Errorf("failed to getpath for subvolumegroup %s with error %v", subvolgrp, stdErr)
}
expectedGrpPath := "/volumes/" + subvolgrp
stdOut = strings.TrimSpace(stdOut)
if stdOut != expectedGrpPath {
return fmt.Errorf("error unexpected group path. Found: %s", stdOut)
}
return nil
}
func createCephfsStorageClass(c kubernetes.Interface, f *framework.Framework, enablePool bool, clusterID string) error {
scPath := fmt.Sprintf("%s/%s", cephfsExamplePath, "storageclass.yaml")
sc, err := getStorageClass(scPath)
if err != nil {
return err
}
sc.Parameters["fsName"] = "myfs"
sc.Parameters["csi.storage.k8s.io/provisioner-secret-namespace"] = rookNamespace
sc.Parameters["csi.storage.k8s.io/provisioner-secret-name"] = cephfsProvisionerSecretName
sc.Parameters["csi.storage.k8s.io/controller-expand-secret-namespace"] = rookNamespace
sc.Parameters["csi.storage.k8s.io/controller-expand-secret-name"] = cephfsProvisionerSecretName
sc.Parameters["csi.storage.k8s.io/node-stage-secret-namespace"] = rookNamespace
sc.Parameters["csi.storage.k8s.io/node-stage-secret-name"] = cephfsNodePluginSecretName
if enablePool {
sc.Parameters["pool"] = "myfs-data0"
}
fsID, stdErr, err := execCommandInToolBoxPod(f, "ceph fsid", rookNamespace)
if err != nil {
return err
}
if stdErr != "" {
return fmt.Errorf("error getting fsid %v", stdErr)
}
// remove new line present in fsID
fsID = strings.Trim(fsID, "\n")
if clusterID != "" {
fsID = clusterID
}
sc.Namespace = cephCSINamespace
sc.Parameters["clusterID"] = fsID
_, err = c.StorageV1().StorageClasses().Create(context.TODO(), &sc, metav1.CreateOptions{})
return err
}
func createCephfsSecret(c kubernetes.Interface, f *framework.Framework) error {
scPath := fmt.Sprintf("%s/%s", cephfsExamplePath, "secret.yaml")
sc, err := getSecret(scPath)
if err != nil {
return err
}
adminKey, stdErr, err := execCommandInToolBoxPod(f, "ceph auth get-key client.admin", rookNamespace)
if err != nil {
return err
}
if stdErr != "" {
return fmt.Errorf("error getting admin key %v", stdErr)
}
sc.StringData["adminID"] = adminUser
sc.StringData["adminKey"] = adminKey
delete(sc.StringData, "userID")
delete(sc.StringData, "userKey")
sc.Namespace = cephCSINamespace
_, err = c.CoreV1().Secrets(cephCSINamespace).Create(context.TODO(), &sc, metav1.CreateOptions{})
return err
}
func deleteBackingCephFSVolume(f *framework.Framework, pvc *v1.PersistentVolumeClaim) error {
imageData, err := getImageInfoFromPVC(pvc.Namespace, pvc.Name, f)
if err != nil {
return err
}
_, stdErr, err := execCommandInToolBoxPod(f, "ceph fs subvolume rm myfs "+imageData.imageName+" "+subvolumegroup, rookNamespace)
if err != nil {
return err
}
if stdErr != "" {
return fmt.Errorf("error deleting backing volume %s %v", imageData.imageName, stdErr)
}
return nil
}
type cephfsSubVolume struct {
Name string `json:"name"`
}
func listCephFSSubVolumes(f *framework.Framework, filesystem, groupname string) ([]cephfsSubVolume, error) {
var subVols []cephfsSubVolume
stdout, stdErr, err := execCommandInToolBoxPod(f, fmt.Sprintf("ceph fs subvolume ls %s --group_name=%s --format=json", filesystem, groupname), rookNamespace)
if err != nil {
return subVols, err
}
if stdErr != "" {
return subVols, fmt.Errorf("error listing subolumes %v", stdErr)
}
err = json.Unmarshal([]byte(stdout), &subVols)
if err != nil {
return subVols, err
}
return subVols, nil
}

120
e2e/configmap.go Normal file
View File

@ -0,0 +1,120 @@
package e2e
import (
"context"
"encoding/json"
"fmt"
"strings"
"github.com/ceph/ceph-csi/internal/util"
v1 "k8s.io/api/core/v1"
apierrs "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/client-go/kubernetes"
"k8s.io/kubernetes/test/e2e/framework"
)
func deleteConfigMap(pluginPath string) error {
path := pluginPath + configMap
_, err := framework.RunKubectl(cephCSINamespace, "delete", "-f", path, ns)
if err != nil {
return err
}
return nil
}
func createConfigMap(pluginPath string, c kubernetes.Interface, f *framework.Framework) error {
path := pluginPath + configMap
cm := v1.ConfigMap{}
err := unmarshal(path, &cm)
if err != nil {
return err
}
fsID, stdErr, err := execCommandInToolBoxPod(f, "ceph fsid", rookNamespace)
if err != nil {
return err
}
if stdErr != "" {
return fmt.Errorf("error getting fsid %v", stdErr)
}
// remove new line present in fsID
fsID = strings.Trim(fsID, "\n")
// get mon list
mons, err := getMons(rookNamespace, c)
if err != nil {
return err
}
conmap := []util.ClusterInfo{{
ClusterID: fsID,
Monitors: mons,
RadosNamespace: radosNamespace,
}}
if upgradeTesting {
subvolumegroup = "csi"
}
conmap[0].CephFS.SubvolumeGroup = subvolumegroup
data, err := json.Marshal(conmap)
if err != nil {
return err
}
cm.Data["config.json"] = string(data)
cm.Namespace = cephCSINamespace
// if the configmap is present update it,during cephcsi helm charts
// deployment empty configmap gets created we need to override it
_, err = c.CoreV1().ConfigMaps(cephCSINamespace).Get(context.TODO(), cm.Name, metav1.GetOptions{})
if err == nil {
_, updateErr := c.CoreV1().ConfigMaps(cephCSINamespace).Update(context.TODO(), &cm, metav1.UpdateOptions{})
if updateErr != nil {
return updateErr
}
}
if apierrs.IsNotFound(err) {
_, err = c.CoreV1().ConfigMaps(cephCSINamespace).Create(context.TODO(), &cm, metav1.CreateOptions{})
}
return err
}
// createCustomConfigMap provides multiple clusters information.
func createCustomConfigMap(c kubernetes.Interface, pluginPath string, subvolgrpInfo map[string]string) error {
path := pluginPath + configMap
cm := v1.ConfigMap{}
err := unmarshal(path, &cm)
if err != nil {
return err
}
// get mon list
mons, err := getMons(rookNamespace, c)
if err != nil {
return err
}
// get clusterIDs
var clusterID []string
for key := range subvolgrpInfo {
clusterID = append(clusterID, key)
}
conmap := []util.ClusterInfo{
{
ClusterID: clusterID[0],
Monitors: mons,
},
{
ClusterID: clusterID[1],
Monitors: mons,
}}
for i := 0; i < len(subvolgrpInfo); i++ {
conmap[i].CephFS.SubvolumeGroup = subvolgrpInfo[clusterID[i]]
}
data, err := json.Marshal(conmap)
if err != nil {
return err
}
cm.Data["config.json"] = string(data)
cm.Namespace = cephCSINamespace
// since a configmap is already created, update the existing configmap
_, err = c.CoreV1().ConfigMaps(cephCSINamespace).Update(context.TODO(), &cm, metav1.UpdateOptions{})
return err
}

75
e2e/namespace.go Normal file
View File

@ -0,0 +1,75 @@
package e2e
import (
"context"
"fmt"
"io/ioutil"
"strings"
"time"
v1 "k8s.io/api/core/v1"
apierrs "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/util/wait"
"k8s.io/client-go/kubernetes"
e2elog "k8s.io/kubernetes/test/e2e/framework/log"
testutils "k8s.io/kubernetes/test/utils"
)
func createNamespace(c kubernetes.Interface, name string) error {
timeout := time.Duration(deployTimeout) * time.Minute
ns := &v1.Namespace{
ObjectMeta: metav1.ObjectMeta{
Name: name,
},
}
_, err := c.CoreV1().Namespaces().Create(context.TODO(), ns, metav1.CreateOptions{})
if err != nil && !apierrs.IsAlreadyExists(err) {
return err
}
return wait.PollImmediate(poll, timeout, func() (bool, error) {
_, err := c.CoreV1().Namespaces().Get(context.TODO(), name, metav1.GetOptions{})
if err != nil {
e2elog.Logf("Error getting namespace: '%s': %v", name, err)
if apierrs.IsNotFound(err) {
return false, nil
}
if testutils.IsRetryableAPIError(err) {
return false, nil
}
return false, err
}
return true, nil
})
}
func deleteNamespace(c kubernetes.Interface, name string) error {
timeout := time.Duration(deployTimeout) * time.Minute
err := c.CoreV1().Namespaces().Delete(context.TODO(), name, metav1.DeleteOptions{})
if err != nil && !apierrs.IsNotFound(err) {
return err
}
return wait.PollImmediate(poll, timeout, func() (bool, error) {
_, err = c.CoreV1().Namespaces().Get(context.TODO(), name, metav1.GetOptions{})
if err != nil {
if apierrs.IsNotFound(err) {
return true, nil
}
e2elog.Logf("Error getting namespace: '%s': %v", name, err)
if testutils.IsRetryableAPIError(err) {
return false, nil
}
return false, err
}
return false, nil
})
}
func replaceNamespaceInTemplate(filePath string) (string, error) {
read, err := ioutil.ReadFile(filePath)
if err != nil {
return "", err
}
return strings.ReplaceAll(string(read), "namespace: default", fmt.Sprintf("namespace: %s", cephCSINamespace)), nil
}

44
e2e/node.go Normal file
View File

@ -0,0 +1,44 @@
package e2e
import (
"context"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/client-go/kubernetes"
"k8s.io/kubernetes/test/e2e/framework"
)
func createNodeLabel(f *framework.Framework, labelKey, labelValue string) error {
// NOTE: This makes all nodes (in a multi-node setup) in the test take
// the same label values, which is fine for the test
nodes, err := f.ClientSet.CoreV1().Nodes().List(context.TODO(), metav1.ListOptions{})
if err != nil {
return err
}
for i := range nodes.Items {
framework.AddOrUpdateLabelOnNode(f.ClientSet, nodes.Items[i].Name, labelKey, labelValue)
}
return nil
}
func deleteNodeLabel(c kubernetes.Interface, labelKey string) error {
nodes, err := c.CoreV1().Nodes().List(context.TODO(), metav1.ListOptions{})
if err != nil {
return err
}
for i := range nodes.Items {
framework.RemoveLabelOffNode(c, nodes.Items[i].Name, labelKey)
}
return nil
}
func checkNodeHasLabel(c kubernetes.Interface, labelKey, labelValue string) error {
nodes, err := c.CoreV1().Nodes().List(context.TODO(), metav1.ListOptions{})
if err != nil {
return err
}
for i := range nodes.Items {
framework.ExpectNodeHasLabel(c, nodes.Items[i].Name, labelKey, labelValue)
}
return nil
}

212
e2e/pod.go Normal file
View File

@ -0,0 +1,212 @@
package e2e
import (
"context"
"errors"
"fmt"
"strings"
"time"
appsv1 "k8s.io/api/apps/v1"
v1 "k8s.io/api/core/v1"
apierrs "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/util/wait"
"k8s.io/client-go/kubernetes"
"k8s.io/kubernetes/pkg/client/conditions"
"k8s.io/kubernetes/test/e2e/framework"
e2elog "k8s.io/kubernetes/test/e2e/framework/log"
testutils "k8s.io/kubernetes/test/utils"
)
func waitForDaemonSets(name, ns string, c kubernetes.Interface, t int) error {
timeout := time.Duration(t) * time.Minute
start := time.Now()
e2elog.Logf("Waiting up to %v for all daemonsets in namespace '%s' to start", timeout, ns)
return wait.PollImmediate(poll, timeout, func() (bool, error) {
ds, err := c.AppsV1().DaemonSets(ns).Get(context.TODO(), name, metav1.GetOptions{})
if err != nil {
e2elog.Logf("Error getting daemonsets in namespace: '%s': %v", ns, err)
if strings.Contains(err.Error(), "not found") {
return false, nil
}
if testutils.IsRetryableAPIError(err) {
return false, nil
}
return false, err
}
dNum := ds.Status.DesiredNumberScheduled
ready := ds.Status.NumberReady
e2elog.Logf("%d / %d pods ready in namespace '%s' in daemonset '%s' (%d seconds elapsed)", ready, dNum, ns, ds.ObjectMeta.Name, int(time.Since(start).Seconds()))
if ready != dNum {
return false, nil
}
return true, nil
})
}
// Waits for the deployment to complete.
func waitForDeploymentComplete(name, ns string, c kubernetes.Interface, t int) error {
var (
deployment *appsv1.Deployment
reason string
err error
)
timeout := time.Duration(t) * time.Minute
err = wait.PollImmediate(poll, timeout, func() (bool, error) {
deployment, err = c.AppsV1().Deployments(ns).Get(context.TODO(), name, metav1.GetOptions{})
if err != nil {
return false, err
}
// TODO need to check rolling update
// When the deployment status and its underlying resources reach the
// desired state, we're done
if deployment.Status.Replicas == deployment.Status.ReadyReplicas {
return true, nil
}
e2elog.Logf("deployment status: expected replica count %d running replica count %d", deployment.Status.Replicas, deployment.Status.ReadyReplicas)
reason = fmt.Sprintf("deployment status: %#v", deployment.Status.String())
return false, nil
})
if errors.Is(err, wait.ErrWaitTimeout) {
err = fmt.Errorf("%s", reason)
}
if err != nil {
return fmt.Errorf("error waiting for deployment %q status to match expectation: %w", name, err)
}
return nil
}
func getCommandInPodOpts(f *framework.Framework, c, ns string, opt *metav1.ListOptions) (framework.ExecOptions, error) {
cmd := []string{"/bin/sh", "-c", c}
podList, err := f.PodClientNS(ns).List(context.TODO(), *opt)
framework.ExpectNoError(err)
if len(podList.Items) == 0 {
return framework.ExecOptions{}, errors.New("podlist is empty")
}
if err != nil {
return framework.ExecOptions{}, err
}
return framework.ExecOptions{
Command: cmd,
PodName: podList.Items[0].Name,
Namespace: ns,
ContainerName: podList.Items[0].Spec.Containers[0].Name,
Stdin: nil,
CaptureStdout: true,
CaptureStderr: true,
PreserveWhitespace: true,
}, nil
}
func execCommandInPod(f *framework.Framework, c, ns string, opt *metav1.ListOptions) (string, string, error) {
podPot, err := getCommandInPodOpts(f, c, ns, opt)
if err != nil {
return "", "", err
}
stdOut, stdErr, err := f.ExecWithOptions(podPot)
if stdErr != "" {
e2elog.Logf("stdErr occurred: %v", stdErr)
}
return stdOut, stdErr, err
}
func execCommandInToolBoxPod(f *framework.Framework, c, ns string) (string, string, error) {
opt := &metav1.ListOptions{
LabelSelector: rookTolBoxPodLabel,
}
podPot, err := getCommandInPodOpts(f, c, ns, opt)
if err != nil {
return "", "", err
}
stdOut, stdErr, err := f.ExecWithOptions(podPot)
if stdErr != "" {
e2elog.Logf("stdErr occurred: %v", stdErr)
}
return stdOut, stdErr, err
}
func execCommandInPodAndAllowFail(f *framework.Framework, c, ns string, opt *metav1.ListOptions) (string, string) {
podPot, err := getCommandInPodOpts(f, c, ns, opt)
if err != nil {
return "", err.Error()
}
stdOut, stdErr, err := f.ExecWithOptions(podPot)
if err != nil {
e2elog.Logf("command %s failed: %v", c, err)
}
return stdOut, stdErr
}
func loadApp(path string) (*v1.Pod, error) {
app := v1.Pod{}
err := unmarshal(path, &app)
if err != nil {
return nil, err
}
return &app, nil
}
func createApp(c kubernetes.Interface, app *v1.Pod, timeout int) error {
_, err := c.CoreV1().Pods(app.Namespace).Create(context.TODO(), app, metav1.CreateOptions{})
if err != nil {
return err
}
return waitForPodInRunningState(app.Name, app.Namespace, c, timeout)
}
func waitForPodInRunningState(name, ns string, c kubernetes.Interface, t int) error {
timeout := time.Duration(t) * time.Minute
start := time.Now()
e2elog.Logf("Waiting up to %v to be in Running state", name)
return wait.PollImmediate(poll, timeout, func() (bool, error) {
pod, err := c.CoreV1().Pods(ns).Get(context.TODO(), name, metav1.GetOptions{})
if err != nil {
return false, err
}
switch pod.Status.Phase {
case v1.PodRunning:
return true, nil
case v1.PodFailed, v1.PodSucceeded:
return false, conditions.ErrPodCompleted
}
e2elog.Logf("%s app is in %s phase expected to be in Running state (%d seconds elapsed)", name, pod.Status.Phase, int(time.Since(start).Seconds()))
return false, nil
})
}
func deletePod(name, ns string, c kubernetes.Interface, t int) error {
timeout := time.Duration(t) * time.Minute
err := c.CoreV1().Pods(ns).Delete(context.TODO(), name, metav1.DeleteOptions{})
if err != nil {
return err
}
start := time.Now()
e2elog.Logf("Waiting for pod %v to be deleted", name)
return wait.PollImmediate(poll, timeout, func() (bool, error) {
_, err := c.CoreV1().Pods(ns).Get(context.TODO(), name, metav1.GetOptions{})
if apierrs.IsNotFound(err) {
return true, nil
}
e2elog.Logf("%s app to be deleted (%d seconds elapsed)", name, int(time.Since(start).Seconds()))
if err != nil {
return false, err
}
return false, nil
})
}
func deletePodWithLabel(label, ns string, skipNotFound bool) error {
_, err := framework.RunKubectl(cephCSINamespace, "delete", "po", "-l", label, fmt.Sprintf("--ignore-not-found=%t", skipNotFound), fmt.Sprintf("--namespace=%s", ns))
if err != nil {
e2elog.Logf("failed to delete pod %v", err)
}
return err
}

171
e2e/pvc.go Normal file
View File

@ -0,0 +1,171 @@
package e2e
import (
"context"
"errors"
"fmt"
"time"
v1 "k8s.io/api/core/v1"
apierrs "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/util/wait"
"k8s.io/client-go/kubernetes"
"k8s.io/kubernetes/test/e2e/framework"
e2elog "k8s.io/kubernetes/test/e2e/framework/log"
e2epv "k8s.io/kubernetes/test/e2e/framework/pv"
testutils "k8s.io/kubernetes/test/utils"
)
func loadPVC(path string) (*v1.PersistentVolumeClaim, error) {
pvc := &v1.PersistentVolumeClaim{}
err := unmarshal(path, &pvc)
if err != nil {
return nil, err
}
return pvc, err
}
func createPVCAndvalidatePV(c kubernetes.Interface, pvc *v1.PersistentVolumeClaim, t int) error {
timeout := time.Duration(t) * time.Minute
pv := &v1.PersistentVolume{}
var err error
_, err = c.CoreV1().PersistentVolumeClaims(pvc.Namespace).Create(context.TODO(), pvc, metav1.CreateOptions{})
if err != nil {
return err
}
if timeout == 0 {
return nil
}
name := pvc.Name
start := time.Now()
e2elog.Logf("Waiting up to %v to be in Bound state", pvc)
return wait.PollImmediate(poll, timeout, func() (bool, error) {
e2elog.Logf("waiting for PVC %s (%d seconds elapsed)", pvc.Name, int(time.Since(start).Seconds()))
pvc, err = c.CoreV1().PersistentVolumeClaims(pvc.Namespace).Get(context.TODO(), name, metav1.GetOptions{})
if err != nil {
e2elog.Logf("Error getting pvc in namespace: '%s': %v", pvc.Namespace, err)
if testutils.IsRetryableAPIError(err) {
return false, nil
}
if apierrs.IsNotFound(err) {
return false, nil
}
return false, err
}
if pvc.Spec.VolumeName == "" {
return false, nil
}
pv, err = c.CoreV1().PersistentVolumes().Get(context.TODO(), pvc.Spec.VolumeName, metav1.GetOptions{})
if err != nil {
return false, err
}
if apierrs.IsNotFound(err) {
return false, nil
}
err = e2epv.WaitOnPVandPVC(c, pvc.Namespace, pv, pvc)
if err != nil {
return false, nil
}
return true, nil
})
}
func deletePVCAndValidatePV(c kubernetes.Interface, pvc *v1.PersistentVolumeClaim, t int) error {
timeout := time.Duration(t) * time.Minute
nameSpace := pvc.Namespace
name := pvc.Name
var err error
e2elog.Logf("Deleting PersistentVolumeClaim %v on namespace %v", name, nameSpace)
pvc, err = c.CoreV1().PersistentVolumeClaims(nameSpace).Get(context.TODO(), name, metav1.GetOptions{})
if err != nil {
return err
}
pv, err := c.CoreV1().PersistentVolumes().Get(context.TODO(), pvc.Spec.VolumeName, metav1.GetOptions{})
if err != nil {
return err
}
err = c.CoreV1().PersistentVolumeClaims(nameSpace).Delete(context.TODO(), name, metav1.DeleteOptions{})
if err != nil {
return fmt.Errorf("delete of PVC %v failed: %w", name, err)
}
start := time.Now()
return wait.PollImmediate(poll, timeout, func() (bool, error) {
// Check that the PVC is really deleted.
e2elog.Logf("waiting for PVC %s in state %s to be deleted (%d seconds elapsed)", name, pvc.Status.String(), int(time.Since(start).Seconds()))
pvc, err = c.CoreV1().PersistentVolumeClaims(nameSpace).Get(context.TODO(), name, metav1.GetOptions{})
if err == nil {
return false, nil
}
if !apierrs.IsNotFound(err) {
return false, fmt.Errorf("get on deleted PVC %v failed with error other than \"not found\": %w", name, err)
}
// Examine the pv.ClaimRef and UID. Expect nil values.
_, err = c.CoreV1().PersistentVolumes().Get(context.TODO(), pv.Name, metav1.GetOptions{})
if err == nil {
return false, nil
}
if !apierrs.IsNotFound(err) {
return false, fmt.Errorf("delete PV %v failed with error other than \"not found\": %w", pv.Name, err)
}
return true, nil
})
}
// getBoundPV returns a PV details.
func getBoundPV(client kubernetes.Interface, pvc *v1.PersistentVolumeClaim) (*v1.PersistentVolume, error) {
// Get new copy of the claim
claim, err := client.CoreV1().PersistentVolumeClaims(pvc.Namespace).Get(context.TODO(), pvc.Name, metav1.GetOptions{})
if err != nil {
return nil, err
}
// Get the bound PV
pv, err := client.CoreV1().PersistentVolumes().Get(context.TODO(), claim.Spec.VolumeName, metav1.GetOptions{})
return pv, err
}
func checkPVSelectorValuesForPVC(f *framework.Framework, pvc *v1.PersistentVolumeClaim) error {
pv, err := getBoundPV(f.ClientSet, pvc)
if err != nil {
return err
}
if len(pv.Spec.NodeAffinity.Required.NodeSelectorTerms) == 0 {
return errors.New("found empty NodeSelectorTerms in PV")
}
rFound := false
zFound := false
for _, expression := range pv.Spec.NodeAffinity.Required.NodeSelectorTerms[0].MatchExpressions {
switch expression.Key {
case nodeCSIRegionLabel:
if rFound {
return errors.New("found multiple occurrences of topology key for region")
}
rFound = true
if expression.Values[0] != regionValue {
return errors.New("topology value for region label mismatch")
}
case nodeCSIZoneLabel:
if zFound {
return errors.New("found multiple occurrences of topology key for zone")
}
zFound = true
if expression.Values[0] != zoneValue {
return errors.New("topology value for zone label mismatch")
}
default:
return errors.New("unexpected key in node selector terms found in PV")
}
}
return nil
}

File diff suppressed because it is too large Load Diff

390
e2e/rbd_helper.go Normal file
View File

@ -0,0 +1,390 @@
package e2e
import (
"context"
"encoding/json"
"fmt"
"regexp"
"strings"
v1 "k8s.io/api/core/v1"
scv1 "k8s.io/api/storage/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/client-go/kubernetes"
"k8s.io/kubernetes/test/e2e/framework"
e2elog "k8s.io/kubernetes/test/e2e/framework/log"
)
func imageSpec(pool, image string) string {
if radosNamespace != "" {
return pool + "/" + radosNamespace + "/" + image
}
return pool + "/" + image
}
func rbdOptions(pool string) string {
if radosNamespace != "" {
return "--pool=" + pool + " --namespace " + radosNamespace
}
return "--pool=" + pool
}
func createRBDStorageClass(c kubernetes.Interface, f *framework.Framework, scOptions, parameters map[string]string) error {
scPath := fmt.Sprintf("%s/%s", rbdExamplePath, "storageclass.yaml")
sc, err := getStorageClass(scPath)
if err != nil {
return nil
}
sc.Parameters["pool"] = defaultRBDPool
sc.Parameters["csi.storage.k8s.io/provisioner-secret-namespace"] = rookNamespace
sc.Parameters["csi.storage.k8s.io/provisioner-secret-name"] = rbdProvisionerSecretName
sc.Parameters["csi.storage.k8s.io/controller-expand-secret-namespace"] = rookNamespace
sc.Parameters["csi.storage.k8s.io/controller-expand-secret-name"] = rbdProvisionerSecretName
sc.Parameters["csi.storage.k8s.io/node-stage-secret-namespace"] = rookNamespace
sc.Parameters["csi.storage.k8s.io/node-stage-secret-name"] = rbdNodePluginSecretName
fsID, stdErr, err := execCommandInToolBoxPod(f, "ceph fsid", rookNamespace)
if err != nil {
return err
}
if stdErr != "" {
return fmt.Errorf("error getting fsid %v", stdErr)
}
// remove new line present in fsID
fsID = strings.Trim(fsID, "\n")
sc.Parameters["clusterID"] = fsID
for k, v := range parameters {
sc.Parameters[k] = v
}
sc.Namespace = cephCSINamespace
if scOptions["volumeBindingMode"] == "WaitForFirstConsumer" {
value := scv1.VolumeBindingWaitForFirstConsumer
sc.VolumeBindingMode = &value
}
// comma separated mount options
if opt, ok := scOptions[rbdmountOptions]; ok {
mOpt := strings.Split(opt, ",")
sc.MountOptions = append(sc.MountOptions, mOpt...)
}
_, err = c.StorageV1().StorageClasses().Create(context.TODO(), &sc, metav1.CreateOptions{})
return err
}
func createRadosNamespace(f *framework.Framework) error {
stdOut, stdErr, err := execCommandInToolBoxPod(f,
fmt.Sprintf("rbd namespace ls --pool=%s", defaultRBDPool), rookNamespace)
if err != nil {
return err
}
if stdErr != "" {
return fmt.Errorf("error listing rbd namespace %v", stdErr)
}
if !strings.Contains(stdOut, radosNamespace) {
_, stdErr, err = execCommandInToolBoxPod(f,
fmt.Sprintf("rbd namespace create %s", rbdOptions(defaultRBDPool)), rookNamespace)
if err != nil {
return err
}
if stdErr != "" {
return fmt.Errorf("error creating rbd namespace %v", stdErr)
}
}
stdOut, stdErr, err = execCommandInToolBoxPod(f,
fmt.Sprintf("rbd namespace ls --pool=%s", rbdTopologyPool), rookNamespace)
if err != nil {
return err
}
if stdErr != "" {
return fmt.Errorf("error listing rbd namespace %v", stdErr)
}
if !strings.Contains(stdOut, radosNamespace) {
_, stdErr, err = execCommandInToolBoxPod(f,
fmt.Sprintf("rbd namespace create %s", rbdOptions(rbdTopologyPool)), rookNamespace)
if err != nil {
return err
}
if stdErr != "" {
return fmt.Errorf("error creating rbd namespace %v", stdErr)
}
}
return nil
}
func createRBDSecret(c kubernetes.Interface, f *framework.Framework) error {
scPath := fmt.Sprintf("%s/%s", rbdExamplePath, "secret.yaml")
sc, err := getSecret(scPath)
if err != nil {
return err
}
adminKey, stdErr, err := execCommandInToolBoxPod(f, "ceph auth get-key client.admin", rookNamespace)
if err != nil {
return err
}
if stdErr != "" {
return fmt.Errorf("error getting admin key %v", stdErr)
}
sc.StringData["userID"] = adminUser
sc.StringData["userKey"] = adminKey
sc.Namespace = cephCSINamespace
_, err = c.CoreV1().Secrets(cephCSINamespace).Create(context.TODO(), &sc, metav1.CreateOptions{})
if err != nil {
return err
}
err = updateSecretForEncryption(c)
return err
}
type imageInfoFromPVC struct {
imageID string
imageName string
csiVolumeHandle string
pvName string
}
// getImageInfoFromPVC reads volume handle of the bound PV to the passed in PVC,
// and returns imageInfoFromPVC or error.
func getImageInfoFromPVC(pvcNamespace, pvcName string, f *framework.Framework) (imageInfoFromPVC, error) {
var imageData imageInfoFromPVC
c := f.ClientSet.CoreV1()
pvc, err := c.PersistentVolumeClaims(pvcNamespace).Get(context.TODO(), pvcName, metav1.GetOptions{})
if err != nil {
return imageData, err
}
pv, err := c.PersistentVolumes().Get(context.TODO(), pvc.Spec.VolumeName, metav1.GetOptions{})
if err != nil {
return imageData, err
}
imageIDRegex := regexp.MustCompile(`(\w+\-?){5}$`)
imageID := imageIDRegex.FindString(pv.Spec.CSI.VolumeHandle)
imageData = imageInfoFromPVC{
imageID: imageID,
imageName: fmt.Sprintf("csi-vol-%s", imageID),
csiVolumeHandle: pv.Spec.CSI.VolumeHandle,
pvName: pv.Name,
}
return imageData, nil
}
func getImageMeta(rbdImageSpec, metaKey string, f *framework.Framework) (string, error) {
cmd := fmt.Sprintf("rbd image-meta get %s %s", rbdImageSpec, metaKey)
stdOut, stdErr, err := execCommandInToolBoxPod(f, cmd, rookNamespace)
if err != nil {
return "", err
}
if stdErr != "" {
return strings.TrimSpace(stdOut), fmt.Errorf(stdErr)
}
return strings.TrimSpace(stdOut), nil
}
func validateEncryptedPVCAndAppBinding(pvcPath, appPath, kms string, f *framework.Framework) error {
pvc, app, err := createPVCAndAppBinding(pvcPath, appPath, f, deployTimeout)
if err != nil {
return err
}
imageData, err := getImageInfoFromPVC(pvc.Namespace, pvc.Name, f)
if err != nil {
return err
}
rbdImageSpec := imageSpec(defaultRBDPool, imageData.imageName)
encryptedState, err := getImageMeta(rbdImageSpec, ".rbd.csi.ceph.com/encrypted", f)
if err != nil {
return err
}
if encryptedState != "encrypted" {
return fmt.Errorf("%v not equal to encrypted", encryptedState)
}
volumeMountPath := app.Spec.Containers[0].VolumeMounts[0].MountPath
mountType, err := getMountType(app.Name, app.Namespace, volumeMountPath, f)
if err != nil {
return err
}
if mountType != "crypt" {
return fmt.Errorf("%v not equal to crypt", mountType)
}
if kms == "vault" {
// check new passphrase created
_, stdErr := readVaultSecret(imageData.csiVolumeHandle, f)
if stdErr != "" {
return fmt.Errorf("failed to read passphrase from vault: %s", stdErr)
}
}
err = deletePVCAndApp("", f, pvc, app)
if err != nil {
return err
}
if kms == "vault" {
// check new passphrase created
stdOut, _ := readVaultSecret(imageData.csiVolumeHandle, f)
if stdOut != "" {
return fmt.Errorf("passphrase found in vault while should be deleted: %s", stdOut)
}
}
return nil
}
func listRBDImages(f *framework.Framework) ([]string, error) {
var imgInfos []string
stdout, stdErr, err := execCommandInToolBoxPod(f,
fmt.Sprintf("rbd ls --format=json %s", rbdOptions(defaultRBDPool)), rookNamespace)
if err != nil {
return imgInfos, err
}
if stdErr != "" {
return imgInfos, fmt.Errorf("failed to list images %v", stdErr)
}
err = json.Unmarshal([]byte(stdout), &imgInfos)
if err != nil {
return imgInfos, err
}
return imgInfos, nil
}
func deleteBackingRBDImage(f *framework.Framework, pvc *v1.PersistentVolumeClaim) error {
imageData, err := getImageInfoFromPVC(pvc.Namespace, pvc.Name, f)
if err != nil {
return err
}
cmd := fmt.Sprintf("rbd rm %s %s", rbdOptions(defaultRBDPool), imageData.imageName)
_, _, err = execCommandInToolBoxPod(f, cmd, rookNamespace)
return err
}
func deletePool(name string, cephfs bool, f *framework.Framework) error {
var cmds = []string{}
if cephfs {
// ceph fs fail
// ceph fs rm myfs --yes-i-really-mean-it
// ceph osd pool delete myfs-metadata myfs-metadata
// --yes-i-really-mean-it
// ceph osd pool delete myfs-data0 myfs-data0
// --yes-i-really-mean-it
cmds = append(cmds, fmt.Sprintf("ceph fs fail %s", name),
fmt.Sprintf("ceph fs rm %s --yes-i-really-mean-it", name),
fmt.Sprintf("ceph osd pool delete %s-metadata %s-metadata --yes-i-really-really-mean-it", name, name),
fmt.Sprintf("ceph osd pool delete %s-data0 %s-data0 --yes-i-really-really-mean-it", name, name))
} else {
// ceph osd pool delete replicapool replicapool
// --yes-i-really-mean-it
cmds = append(cmds, fmt.Sprintf("ceph osd pool delete %s %s --yes-i-really-really-mean-it", name, name))
}
for _, cmd := range cmds {
// discard stdErr as some commands prints warning in strErr
_, _, err := execCommandInToolBoxPod(f, cmd, rookNamespace)
if err != nil {
return err
}
}
return nil
}
func getPVCImageInfoInPool(f *framework.Framework, pvc *v1.PersistentVolumeClaim, pool string) (string, error) {
imageData, err := getImageInfoFromPVC(pvc.Namespace, pvc.Name, f)
if err != nil {
return "", err
}
stdOut, stdErr, err := execCommandInToolBoxPod(f,
fmt.Sprintf("rbd info %s", imageSpec(pool, imageData.imageName)), rookNamespace)
if err != nil {
return "", err
}
if stdErr != "" {
return "", fmt.Errorf("failed to get rbd info %v", stdErr)
}
if radosNamespace != "" {
e2elog.Logf("found image %s in pool %s namespace %s", imageData.imageName, pool, radosNamespace)
} else {
e2elog.Logf("found image %s in pool %s", imageData.imageName, pool)
}
return stdOut, nil
}
func checkPVCImageInPool(f *framework.Framework, pvc *v1.PersistentVolumeClaim, pool string) error {
_, err := getPVCImageInfoInPool(f, pvc, pool)
return err
}
func checkPVCDataPoolForImageInPool(f *framework.Framework, pvc *v1.PersistentVolumeClaim, pool, dataPool string) error {
stdOut, err := getPVCImageInfoInPool(f, pvc, pool)
if err != nil {
return err
}
if !strings.Contains(stdOut, "data_pool: "+dataPool) {
return fmt.Errorf("missing data pool value in image info, got info (%s)", stdOut)
}
return nil
}
func checkPVCImageJournalInPool(f *framework.Framework, pvc *v1.PersistentVolumeClaim, pool string) error {
imageData, err := getImageInfoFromPVC(pvc.Namespace, pvc.Name, f)
if err != nil {
return err
}
_, stdErr, err := execCommandInToolBoxPod(f,
fmt.Sprintf("rados listomapkeys %s csi.volume.%s", rbdOptions(pool), imageData.imageID), rookNamespace)
if err != nil {
return err
}
if stdErr != "" {
return fmt.Errorf("failed to listomapkeys %v", stdErr)
}
if radosNamespace != "" {
e2elog.Logf("found image journal %s in pool %s namespace %s", "csi.volume."+imageData.imageID, pool, radosNamespace)
} else {
e2elog.Logf("found image journal %s in pool %s", "csi.volume."+imageData.imageID, pool)
}
return nil
}
func checkPVCCSIJournalInPool(f *framework.Framework, pvc *v1.PersistentVolumeClaim, pool string) error {
imageData, err := getImageInfoFromPVC(pvc.Namespace, pvc.Name, f)
if err != nil {
return err
}
_, stdErr, err := execCommandInToolBoxPod(f,
fmt.Sprintf("rados getomapval %s csi.volumes.default csi.volume.%s", rbdOptions(pool), imageData.pvName), rookNamespace)
if err != nil {
return err
}
if stdErr != "" {
return fmt.Errorf("error getting fsid %v", stdErr)
}
if radosNamespace != "" {
e2elog.Logf("found CSI journal entry %s in pool %s namespace %s", "csi.volume."+imageData.pvName, pool, radosNamespace)
} else {
e2elog.Logf("found CSI journal entry %s in pool %s", "csi.volume."+imageData.pvName, pool)
}
return nil
}

View File

@ -161,8 +161,10 @@ func checkAppMntSize(f *framework.Framework, opt *metav1.ListOptions, size, cmd,
return wait.PollImmediate(poll, timeout, func() (bool, error) {
e2elog.Logf("executing cmd %s (%d seconds elapsed)", cmd, int(time.Since(start).Seconds()))
output, stdErr := execCommandInPod(f, cmd, ns, opt)
output, stdErr, err := execCommandInPod(f, cmd, ns, opt)
if err != nil {
return false, err
}
if stdErr != "" {
e2elog.Logf("failed to execute command in app pod %v", stdErr)
return false, nil

View File

@ -113,32 +113,46 @@ func deleteSnapshot(snap *snapapi.VolumeSnapshot, t int) error {
})
}
func createRBDSnapshotClass(f *framework.Framework) {
func createRBDSnapshotClass(f *framework.Framework) error {
scPath := fmt.Sprintf("%s/%s", rbdExamplePath, "snapshotclass.yaml")
sc := getSnapshotClass(scPath)
sc.Parameters["csi.storage.k8s.io/snapshotter-secret-namespace"] = cephCSINamespace
fsID, stdErr := execCommandInToolBoxPod(f, "ceph fsid", rookNamespace)
Expect(stdErr).Should(BeEmpty())
fsID, stdErr, err := execCommandInToolBoxPod(f, "ceph fsid", rookNamespace)
if err != nil {
return err
}
if stdErr != "" {
return fmt.Errorf("failed to get fsid from ceph cluster %s", stdErr)
}
fsID = strings.Trim(fsID, "\n")
sc.Parameters["clusterID"] = fsID
sclient, err := newSnapshotClient()
Expect(err).Should(BeNil())
if err != nil {
return err
}
_, err = sclient.SnapshotV1beta1().VolumeSnapshotClasses().Create(context.TODO(), &sc, metav1.CreateOptions{})
Expect(err).Should(BeNil())
return err
}
func createCephFSSnapshotClass(f *framework.Framework) {
func createCephFSSnapshotClass(f *framework.Framework) error {
scPath := fmt.Sprintf("%s/%s", cephfsExamplePath, "snapshotclass.yaml")
sc := getSnapshotClass(scPath)
sc.Parameters["csi.storage.k8s.io/snapshotter-secret-namespace"] = cephCSINamespace
fsID, stdErr := execCommandInToolBoxPod(f, "ceph fsid", rookNamespace)
Expect(stdErr).Should(BeEmpty())
fsID, stdErr, err := execCommandInToolBoxPod(f, "ceph fsid", rookNamespace)
if err != nil {
return err
}
if stdErr != "" {
return fmt.Errorf("failed to get fsid from ceph cluster %s", stdErr)
}
fsID = strings.Trim(fsID, "\n")
sc.Parameters["clusterID"] = fsID
sclient, err := newSnapshotClient()
Expect(err).Should(BeNil())
if err != nil {
return err
}
_, err = sclient.SnapshotV1beta1().VolumeSnapshotClasses().Create(context.TODO(), &sc, metav1.CreateOptions{})
Expect(err).Should(BeNil())
return err
}

View File

@ -91,7 +91,10 @@ func validateRBDStaticPV(f *framework.Framework, appPath string, isBlock bool) e
c := f.ClientSet
fsID, e := execCommandInToolBoxPod(f, "ceph fsid", rookNamespace)
fsID, e, err := execCommandInToolBoxPod(f, "ceph fsid", rookNamespace)
if err != nil {
return err
}
if e != "" {
return fmt.Errorf("failed to get fsid from ceph cluster %s", e)
}
@ -101,7 +104,10 @@ func validateRBDStaticPV(f *framework.Framework, appPath string, isBlock bool) e
// create rbd image
cmd := fmt.Sprintf("rbd create %s --size=%d --image-feature=layering %s", rbdImageName, 4096, rbdOptions(defaultRBDPool))
_, e = execCommandInToolBoxPod(f, cmd, rookNamespace)
_, e, err = execCommandInToolBoxPod(f, cmd, rookNamespace)
if err != nil {
return err
}
if e != "" {
return fmt.Errorf("failed to create rbd image %s", e)
}
@ -115,7 +121,7 @@ func validateRBDStaticPV(f *framework.Framework, appPath string, isBlock bool) e
pv := getStaticPV(pvName, rbdImageName, size, "csi-rbd-secret", cephCSINamespace, sc, "rbd.csi.ceph.com", isBlock, opt)
_, err := c.CoreV1().PersistentVolumes().Create(context.TODO(), pv, metav1.CreateOptions{})
_, err = c.CoreV1().PersistentVolumes().Create(context.TODO(), pv, metav1.CreateOptions{})
if err != nil {
return fmt.Errorf("PV Create API error: %w", err)
}
@ -155,10 +161,11 @@ func validateRBDStaticPV(f *framework.Framework, appPath string, isBlock bool) e
}
cmd = fmt.Sprintf("rbd rm %s %s", rbdImageName, rbdOptions(defaultRBDPool))
execCommandInToolBoxPod(f, cmd, rookNamespace)
return nil
_, _, err = execCommandInToolBoxPod(f, cmd, rookNamespace)
return err
}
// nolint:gocyclo // reduce complexity
func validateCephFsStaticPV(f *framework.Framework, appPath, scPath string) error {
opt := make(map[string]string)
var (
@ -180,7 +187,10 @@ func validateCephFsStaticPV(f *framework.Framework, appPath, scPath string) erro
LabelSelector: "app=rook-ceph-tools",
}
fsID, e := execCommandInPod(f, "ceph fsid", rookNamespace, &listOpt)
fsID, e, err := execCommandInPod(f, "ceph fsid", rookNamespace, &listOpt)
if err != nil {
return err
}
if e != "" {
return fmt.Errorf("failed to get fsid from ceph cluster %s", e)
}
@ -193,21 +203,30 @@ func validateCephFsStaticPV(f *framework.Framework, appPath, scPath string) erro
// create subvolumegroup, command will work even if group is already present.
cmd := fmt.Sprintf("ceph fs subvolumegroup create %s %s", fsName, groupName)
_, e = execCommandInPod(f, cmd, rookNamespace, &listOpt)
_, e, err = execCommandInPod(f, cmd, rookNamespace, &listOpt)
if err != nil {
return err
}
if e != "" {
return fmt.Errorf("failed to create subvolumegroup %s", e)
}
// create subvolume
cmd = fmt.Sprintf("ceph fs subvolume create %s %s %s --size %s", fsName, cephFsVolName, groupName, size)
_, e = execCommandInPod(f, cmd, rookNamespace, &listOpt)
_, e, err = execCommandInPod(f, cmd, rookNamespace, &listOpt)
if err != nil {
return err
}
if e != "" {
return fmt.Errorf("failed to create subvolume %s", e)
}
// get rootpath
cmd = fmt.Sprintf("ceph fs subvolume getpath %s %s %s", fsName, cephFsVolName, groupName)
rootPath, e := execCommandInPod(f, cmd, rookNamespace, &listOpt)
rootPath, e, err := execCommandInPod(f, cmd, rookNamespace, &listOpt)
if err != nil {
return err
}
if e != "" {
return fmt.Errorf("failed to get rootpath %s", e)
}
@ -215,17 +234,22 @@ func validateCephFsStaticPV(f *framework.Framework, appPath, scPath string) erro
rootPath = strings.Trim(rootPath, "\n")
// create secret
userID := "admin" // nolint
secret := getSecret(scPath)
adminKey, e := execCommandInPod(f, "ceph auth get-key client.admin", rookNamespace, &listOpt)
secret, err := getSecret(scPath)
if err != nil {
return err
}
adminKey, e, err := execCommandInPod(f, "ceph auth get-key client.admin", rookNamespace, &listOpt)
if err != nil {
return err
}
if e != "" {
return fmt.Errorf("failed to get adminKey %s", e)
}
secret.StringData["userID"] = userID
secret.StringData["userID"] = adminUser
secret.StringData["userKey"] = adminKey
secret.Name = secretName
secret.Namespace = cephCSINamespace
_, err := c.CoreV1().Secrets(cephCSINamespace).Create(context.TODO(), &secret, metav1.CreateOptions{})
_, err = c.CoreV1().Secrets(cephCSINamespace).Create(context.TODO(), &secret, metav1.CreateOptions{})
if err != nil {
return fmt.Errorf("failed to create secret, error %w", err)
}
@ -280,14 +304,20 @@ func validateCephFsStaticPV(f *framework.Framework, appPath, scPath string) erro
// delete subvolume
cmd = fmt.Sprintf("ceph fs subvolume rm %s %s %s", fsName, cephFsVolName, groupName)
_, e = execCommandInPod(f, cmd, rookNamespace, &listOpt)
_, e, err = execCommandInPod(f, cmd, rookNamespace, &listOpt)
if err != nil {
return err
}
if e != "" {
return fmt.Errorf("failed to remove sub-volume %s", e)
}
// delete subvolume group
cmd = fmt.Sprintf("ceph fs subvolumegroup rm %s %s", fsName, groupName)
_, e = execCommandInPod(f, cmd, rookNamespace, &listOpt)
_, e, err = execCommandInPod(f, cmd, rookNamespace, &listOpt)
if err != nil {
return err
}
if e != "" {
return fmt.Errorf("failed to remove subvolume group %s", e)
}

View File

@ -22,6 +22,7 @@ var _ = Describe("CephFS Upgrade Testing", func() {
app *v1.Pod
// cwd stores the initial working directory.
cwd string
err error
)
// deploy cephfs CSI
BeforeEach(func() {
@ -30,26 +31,34 @@ var _ = Describe("CephFS Upgrade Testing", func() {
}
c = f.ClientSet
if cephCSINamespace != defaultNs {
err := createNamespace(c, cephCSINamespace)
err = createNamespace(c, cephCSINamespace)
if err != nil {
Fail(err.Error())
e2elog.Failf("failed to create namespace with error %v", err)
}
}
// fetch current working directory to switch back
// when we are done upgrading.
var err error
cwd, err = os.Getwd()
if err != nil {
Fail(err.Error())
e2elog.Failf("failed to getwd with error %v", err)
}
err = upgradeAndDeployCSI(upgradeVersion, "cephfs")
if err != nil {
Fail(err.Error())
e2elog.Failf("failed to upgrade csi with error %v", err)
}
err = createConfigMap(cephfsDirPath, f.ClientSet, f)
if err != nil {
e2elog.Failf("failed to create configmap with error %v", err)
}
err = createCephfsSecret(f.ClientSet, f)
if err != nil {
e2elog.Failf("failed to create secret with error %v", err)
}
err = createCephfsStorageClass(f.ClientSet, f, true, "")
if err != nil {
e2elog.Failf("failed to create storageclass with error %v", err)
}
createConfigMap(cephfsDirPath, f.ClientSet, f)
createCephfsSecret(f.ClientSet, f)
createCephfsStorageClass(f.ClientSet, f, true, "")
})
AfterEach(func() {
if !testCephFS || !upgradeTesting {
@ -63,15 +72,26 @@ var _ = Describe("CephFS Upgrade Testing", func() {
// log node plugin
logsCSIPods("app=csi-cephfsplugin", c)
}
deleteConfigMap(cephfsDirPath)
deleteResource(cephfsExamplePath + "secret.yaml")
deleteResource(cephfsExamplePath + "storageclass.yaml")
err = deleteConfigMap(cephfsDirPath)
if err != nil {
e2elog.Failf("failed to delete configmap with error %v", err)
}
err = deleteResource(cephfsExamplePath + "secret.yaml")
if err != nil {
e2elog.Failf("failed to delete secret with error %v", err)
}
err = deleteResource(cephfsExamplePath + "storageclass.yaml")
if err != nil {
e2elog.Failf("failed to delete storageclass with error %v", err)
}
if deployCephFS {
deleteCephfsPlugin()
if cephCSINamespace != defaultNs {
err := deleteNamespace(c, cephCSINamespace)
if err != nil {
Fail(err.Error())
if err != nil {
e2elog.Failf("failed to delete namespace with error %v", err)
}
}
}
}
@ -83,13 +103,13 @@ var _ = Describe("CephFS Upgrade Testing", func() {
By("checking provisioner deployment is running")
err := waitForDeploymentComplete(cephfsDeploymentName, cephCSINamespace, f.ClientSet, deployTimeout)
if err != nil {
Fail(err.Error())
e2elog.Failf("timeout waiting for deployment %s with error %v", cephfsDeploymentName, err)
}
By("checking nodeplugin deamonsets is running")
By("checking nodeplugin deamonset pods are running")
err = waitForDaemonSets(cephfsDeamonSetName, cephCSINamespace, f.ClientSet, deployTimeout)
if err != nil {
Fail(err.Error())
e2elog.Failf("timeout waiting for daemonset %s with error%v", cephfsDeamonSetName, err)
}
By("upgrade to latest changes and verify app re-mount", func() {
@ -100,32 +120,31 @@ var _ = Describe("CephFS Upgrade Testing", func() {
pvc, err = loadPVC(pvcPath)
if pvc == nil {
Fail(err.Error())
e2elog.Failf("failed to load pvc with error %v", err)
}
pvc.Namespace = f.UniqueName
e2elog.Logf("The PVC template %+v", pvc)
app, err = loadApp(appPath)
if err != nil {
Fail(err.Error())
e2elog.Failf("failed to load application with error %v", err)
}
app.Namespace = f.UniqueName
app.Labels = map[string]string{"app": "upgrade-testing"}
pvc.Spec.Resources.Requests[v1.ResourceStorage] = resource.MustParse(pvcSize)
err = createPVCAndApp("", f, pvc, app, deployTimeout)
if err != nil {
Fail(err.Error())
e2elog.Failf("failed to create pvc and application with error %v", err)
}
err = deletePod(app.Name, app.Namespace, f.ClientSet, deployTimeout)
if err != nil {
Fail(err.Error())
e2elog.Failf("failed to delete application with error %v", err)
}
deleteCephfsPlugin()
// switch back to current changes.
err = os.Chdir(cwd)
if err != nil {
Fail(err.Error())
e2elog.Failf("failed to d chdir with error %v", err)
}
deployCephfsPlugin()
@ -134,7 +153,7 @@ var _ = Describe("CephFS Upgrade Testing", func() {
// an earlier release.
err = createApp(f.ClientSet, app, deployTimeout)
if err != nil {
Fail(err.Error())
e2elog.Failf("failed to create application with error %v", err)
}
})
@ -144,7 +163,6 @@ var _ = Describe("CephFS Upgrade Testing", func() {
v, err = f.ClientSet.Discovery().ServerVersion()
if err != nil {
e2elog.Logf("failed to get server version with error %v", err)
Fail(err.Error())
}
// Resize 0.3.0 is only supported from v1.15+
if v.Major > "1" || (v.Major == "1" && v.Minor >= "15") {
@ -153,23 +171,23 @@ var _ = Describe("CephFS Upgrade Testing", func() {
}
pvc, err = f.ClientSet.CoreV1().PersistentVolumeClaims(pvc.Namespace).Get(context.TODO(), pvc.Name, metav1.GetOptions{})
if err != nil {
Fail(err.Error())
e2elog.Failf("failed to get pvc with error %v", err)
}
// resize PVC
err = expandPVCSize(f.ClientSet, pvc, pvcExpandSize, deployTimeout)
if err != nil {
Fail(err.Error())
e2elog.Failf("failed to expand pvc with error %v", err)
}
// wait for application pod to come up after resize
err = waitForPodInRunningState(app.Name, app.Namespace, f.ClientSet, deployTimeout)
if err != nil {
Fail(err.Error())
e2elog.Failf("timout waiting for pod to be in running state with error %v", err)
}
// validate if resize is successful.
err = checkDirSize(app, f, &opt, pvcExpandSize)
if err != nil {
Fail(err.Error())
e2elog.Failf("failed to check directory size with error %v", err)
}
}
@ -178,7 +196,7 @@ var _ = Describe("CephFS Upgrade Testing", func() {
By("delete pvc and app")
err = deletePVCAndApp("", f, pvc, app)
if err != nil {
Fail(err.Error())
e2elog.Failf("failed to delete pvc and application with error %v", err)
}
})
})

View File

@ -32,28 +32,44 @@ var _ = Describe("RBD Upgrade Testing", func() {
if cephCSINamespace != defaultNs {
err := createNamespace(c, cephCSINamespace)
if err != nil {
Fail(err.Error())
e2elog.Failf("failed to create namespace with error %v", err)
}
}
createNodeLabel(f, nodeRegionLabel, regionValue)
createNodeLabel(f, nodeZoneLabel, zoneValue)
// fetch current working directory to switch back
// when we are done upgrading.
var err error
cwd, err = os.Getwd()
if err != nil {
Fail(err.Error())
e2elog.Failf("failed to do getwd with error %v", err)
}
deployVault(f.ClientSet, deployTimeout)
err = upgradeAndDeployCSI(upgradeVersion, "rbd")
if err != nil {
Fail(err.Error())
e2elog.Failf("failed to upgrade and deploy CSI with error %v", err)
}
err = createConfigMap(rbdDirPath, f.ClientSet, f)
if err != nil {
e2elog.Failf("failed to create configmap with error %v", err)
}
err = createRBDStorageClass(f.ClientSet, f, nil, nil)
if err != nil {
e2elog.Failf("failed to create storageclass with error %v", err)
}
err = createRBDSecret(f.ClientSet, f)
if err != nil {
e2elog.Failf("failed to create secret with error %v", err)
}
err = createNodeLabel(f, nodeRegionLabel, regionValue)
if err != nil {
e2elog.Failf("failed to create node label with error %v", err)
}
err = createNodeLabel(f, nodeZoneLabel, zoneValue)
if err != nil {
e2elog.Failf("failed to create node label with error %v", err)
}
createConfigMap(rbdDirPath, f.ClientSet, f)
createRBDStorageClass(f.ClientSet, f, nil, nil)
createRBDSecret(f.ClientSet, f)
})
AfterEach(func() {
if !testRBD || !upgradeTesting {
@ -68,21 +84,36 @@ var _ = Describe("RBD Upgrade Testing", func() {
logsCSIPods("app=csi-rbdplugin", c)
}
deleteConfigMap(rbdDirPath)
deleteResource(rbdExamplePath + "secret.yaml")
deleteResource(rbdExamplePath + "storageclass.yaml")
err := deleteConfigMap(rbdDirPath)
if err != nil {
e2elog.Failf("failed to delete configmap with error %v", err)
}
err = deleteResource(rbdExamplePath + "secret.yaml")
if err != nil {
e2elog.Failf("failed to delete secret with error %v", err)
}
err = deleteResource(rbdExamplePath + "storageclass.yaml")
if err != nil {
e2elog.Failf("failed to delete storageclass with error %v", err)
}
deleteVault()
if deployRBD {
deleteRBDPlugin()
if cephCSINamespace != defaultNs {
err := deleteNamespace(c, cephCSINamespace)
err = deleteNamespace(c, cephCSINamespace)
if err != nil {
Fail(err.Error())
e2elog.Failf("failed to delete namespace with error %v", err)
}
}
}
deleteNodeLabel(c, nodeRegionLabel)
deleteNodeLabel(c, nodeZoneLabel)
err = deleteNodeLabel(c, nodeRegionLabel)
if err != nil {
e2elog.Failf("failed to delete node label with error %v", err)
}
err = deleteNodeLabel(c, nodeZoneLabel)
if err != nil {
e2elog.Failf("failed to delete node label with error %v", err)
}
})
Context("Test RBD CSI", func() {
@ -93,14 +124,14 @@ var _ = Describe("RBD Upgrade Testing", func() {
By("checking provisioner deployment is running", func() {
err := waitForDeploymentComplete(rbdDeploymentName, cephCSINamespace, f.ClientSet, deployTimeout)
if err != nil {
Fail(err.Error())
e2elog.Failf("timeout waiting for deployment %s with error %v", rbdDeploymentName, err)
}
})
By("checking nodeplugin deamonsets is running", func() {
By("checking nodeplugin deamonset pods are running", func() {
err := waitForDaemonSets(rbdDaemonsetName, cephCSINamespace, f.ClientSet, deployTimeout)
if err != nil {
Fail(err.Error())
e2elog.Failf("timeout waiting for daemonset %s with error %v", rbdDaemonsetName, err)
}
})
@ -110,31 +141,30 @@ var _ = Describe("RBD Upgrade Testing", func() {
var err error
pvc, err = loadPVC(pvcPath)
if pvc == nil {
Fail(err.Error())
e2elog.Failf("failed to load pvc with error %v", err)
}
pvc.Namespace = f.UniqueName
e2elog.Logf("The PVC template %+v", pvc)
app, err = loadApp(appPath)
if err != nil {
Fail(err.Error())
e2elog.Failf("failed to load application with error %v", err)
}
app.Namespace = f.UniqueName
app.Labels = map[string]string{"app": "upgrade-testing"}
pvc.Spec.Resources.Requests[v1.ResourceStorage] = resource.MustParse(pvcSize)
err = createPVCAndApp("", f, pvc, app, deployTimeout)
if err != nil {
Fail(err.Error())
e2elog.Failf("failed to create pvc with error %v", err)
}
err = deletePod(app.Name, app.Namespace, f.ClientSet, deployTimeout)
if err != nil {
Fail(err.Error())
e2elog.Failf("failed to delete application with error %v", err)
}
deleteRBDPlugin()
err = os.Chdir(cwd)
if err != nil {
Fail(err.Error())
e2elog.Failf("failed to change directory with error %v", err)
}
deployRBDPlugin()
@ -143,7 +173,7 @@ var _ = Describe("RBD Upgrade Testing", func() {
app.Labels = map[string]string{"app": "upgrade-testing"}
err = createApp(f.ClientSet, app, deployTimeout)
if err != nil {
Fail(err.Error())
e2elog.Failf("failed to create application with error %v", err)
}
})
@ -153,7 +183,6 @@ var _ = Describe("RBD Upgrade Testing", func() {
v, err := f.ClientSet.Discovery().ServerVersion()
if err != nil {
e2elog.Logf("failed to get server version with error %v", err)
Fail(err.Error())
}
// Resize 0.3.0 is only supported from v1.15+
if v.Major > "1" || (v.Major == "1" && v.Minor >= "15") {
@ -162,23 +191,23 @@ var _ = Describe("RBD Upgrade Testing", func() {
}
pvc, err = f.ClientSet.CoreV1().PersistentVolumeClaims(pvc.Namespace).Get(context.TODO(), pvc.Name, metav1.GetOptions{})
if err != nil {
Fail(err.Error())
e2elog.Failf("failed to get pvc with error %v", err)
}
// resize PVC
err = expandPVCSize(f.ClientSet, pvc, pvcExpandSize, deployTimeout)
if err != nil {
Fail(err.Error())
e2elog.Failf("failed to expand pvc with error %v", err)
}
// wait for application pod to come up after resize
err = waitForPodInRunningState(app.Name, app.Namespace, f.ClientSet, deployTimeout)
if err != nil {
Fail(err.Error())
e2elog.Failf("timeout waiting for pod to be in running state with error %v", err)
}
// validate if resize is successful.
err = checkDirSize(app, f, &opt, pvcExpandSize)
if err != nil {
Fail(err.Error())
e2elog.Failf("failed to check directory size with error %v", err)
}
}
@ -187,7 +216,7 @@ var _ = Describe("RBD Upgrade Testing", func() {
By("delete pvc and app", func() {
err := deletePVCAndApp("", f, pvc, app)
if err != nil {
Fail(err.Error())
e2elog.Failf("failed to delete pvc and application with error %v", err)
}
})
})

File diff suppressed because it is too large Load Diff