From 2c66dfc3e42382dab4f717c0fe9aeae10a79ad32 Mon Sep 17 00:00:00 2001 From: Madhu Rajanna Date: Wed, 28 Jul 2021 09:21:18 +0530 Subject: [PATCH] e2e: retry running kubectl on known errors By using retryKubectl helper function, a retry will be done, and the known error messages will be skipped. Signed-off-by: Madhu Rajanna --- e2e/configmap.go | 6 +----- e2e/deploy-vault.go | 8 +++----- e2e/pod.go | 9 ++++++++- e2e/rbd.go | 32 +++++++++++++++++++------------- e2e/utils.go | 2 +- 5 files changed, 32 insertions(+), 25 deletions(-) diff --git a/e2e/configmap.go b/e2e/configmap.go index 71b385781..de1d4b790 100644 --- a/e2e/configmap.go +++ b/e2e/configmap.go @@ -17,12 +17,8 @@ import ( func deleteConfigMap(pluginPath string) error { path := pluginPath + configMap - _, err := framework.RunKubectl(cephCSINamespace, "delete", "-f", path, ns) - if err != nil { - return err - } - return nil + return retryKubectlFile(cephCSINamespace, kubectlDelete, path, deployTimeout) } func createConfigMap(pluginPath string, c kubernetes.Interface, f *framework.Framework) error { diff --git a/e2e/deploy-vault.go b/e2e/deploy-vault.go index 27c363106..88283323a 100644 --- a/e2e/deploy-vault.go +++ b/e2e/deploy-vault.go @@ -8,7 +8,6 @@ import ( . "github.com/onsi/gomega" // nolint metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes" - "k8s.io/kubernetes/test/e2e/framework" e2elog "k8s.io/kubernetes/test/e2e/framework/log" ) @@ -25,13 +24,12 @@ var ( func deployVault(c kubernetes.Interface, deployTimeout int) { // hack to make helm E2E pass as helm charts creates this configmap as part // of cephcsi deployment - _, err := framework.RunKubectl( + err := retryKubectlArgs( cephCSINamespace, - "delete", + kubectlDelete, + deployTimeout, "cm", "ceph-csi-encryption-kms-config", - "--namespace", - cephCSINamespace, "--ignore-not-found=true") Expect(err).Should(BeNil()) diff --git a/e2e/pod.go b/e2e/pod.go index 2ea30cfbb..940bdc261 100644 --- a/e2e/pod.go +++ b/e2e/pod.go @@ -375,7 +375,14 @@ func deletePod(name, ns string, c kubernetes.Interface, t int) error { } func deletePodWithLabel(label, ns string, skipNotFound bool) error { - _, err := framework.RunKubectl(ns, "delete", "po", "-l", label, fmt.Sprintf("--ignore-not-found=%t", skipNotFound)) + err := retryKubectlArgs( + ns, + kubectlDelete, + deployTimeout, + "po", + "-l", + label, + fmt.Sprintf("--ignore-not-found=%t", skipNotFound)) if err != nil { e2elog.Logf("failed to delete pod %v", err) } diff --git a/e2e/rbd.go b/e2e/rbd.go index 9481474aa..d37f2f6f7 100644 --- a/e2e/rbd.go +++ b/e2e/rbd.go @@ -63,7 +63,7 @@ func deployRBDPlugin() { if err != nil { e2elog.Failf("failed to read content from %s with error %v", rbdDirPath+rbdProvisionerRBAC, err) } - _, err = framework.RunKubectlInput(cephCSINamespace, data, "--ignore-not-found=true", ns, "delete", "-f", "-") + err = retryKubectlInput(cephCSINamespace, kubectlDelete, data, deployTimeout, "--ignore-not-found=true") if err != nil { e2elog.Failf("failed to delete provisioner rbac %s with error %v", rbdDirPath+rbdProvisionerRBAC, err) } @@ -72,19 +72,19 @@ func deployRBDPlugin() { if err != nil { e2elog.Failf("failed to read content from %s with error %v", rbdDirPath+rbdNodePluginRBAC, err) } - _, err = framework.RunKubectlInput(cephCSINamespace, data, "delete", "--ignore-not-found=true", ns, "-f", "-") + err = retryKubectlInput(cephCSINamespace, kubectlDelete, data, deployTimeout, "--ignore-not-found=true") if err != nil { e2elog.Failf("failed to delete nodeplugin rbac %s with error %v", rbdDirPath+rbdNodePluginRBAC, err) } - createORDeleteRbdResources("create") + createORDeleteRbdResources(kubectlCreate) } func deleteRBDPlugin() { - createORDeleteRbdResources("delete") + createORDeleteRbdResources(kubectlDelete) } -func createORDeleteRbdResources(action string) { +func createORDeleteRbdResources(action kubectlAction) { csiDriver, err := ioutil.ReadFile(rbdDirPath + csiDriverObject) if err != nil { // createORDeleteRbdResources is used for upgrade testing as csidriverObject is @@ -93,7 +93,7 @@ func createORDeleteRbdResources(action string) { e2elog.Failf("failed to read content from %s with error %v", rbdDirPath+csiDriverObject, err) } } else { - _, err = framework.RunKubectlInput(cephCSINamespace, string(csiDriver), action, "-f", "-") + err = retryKubectlInput(cephCSINamespace, action, string(csiDriver), deployTimeout) if err != nil { e2elog.Failf("failed to %s CSIDriver object with error %v", action, err) } @@ -104,7 +104,7 @@ func createORDeleteRbdResources(action string) { } data = oneReplicaDeployYaml(data) data = enableTopologyInTemplate(data) - _, err = framework.RunKubectlInput(cephCSINamespace, data, action, ns, "-f", "-") + err = retryKubectlInput(cephCSINamespace, action, data, deployTimeout) if err != nil { e2elog.Failf("failed to %s rbd provisioner with error %v", action, err) } @@ -113,7 +113,7 @@ func createORDeleteRbdResources(action string) { if err != nil { e2elog.Failf("failed to read content from %s with error %v", rbdDirPath+rbdProvisionerRBAC, err) } - _, err = framework.RunKubectlInput(cephCSINamespace, data, action, ns, "-f", "-") + err = retryKubectlInput(cephCSINamespace, action, data, deployTimeout) if err != nil { e2elog.Failf("failed to %s provisioner rbac with error %v", action, err) } @@ -122,7 +122,7 @@ func createORDeleteRbdResources(action string) { if err != nil { e2elog.Failf("failed to read content from %s with error %v", rbdDirPath+rbdProvisionerPSP, err) } - _, err = framework.RunKubectlInput(cephCSINamespace, data, action, "-f", "-") + err = retryKubectlInput(cephCSINamespace, action, data, deployTimeout) if err != nil { e2elog.Failf("failed to %s provisioner psp with error %v", action, err) } @@ -134,7 +134,7 @@ func createORDeleteRbdResources(action string) { domainLabel := nodeRegionLabel + "," + nodeZoneLabel data = addTopologyDomainsToDSYaml(data, domainLabel) - _, err = framework.RunKubectlInput(cephCSINamespace, data, action, ns, "-f", "-") + err = retryKubectlInput(cephCSINamespace, action, data, deployTimeout) if err != nil { e2elog.Failf("failed to %s nodeplugin with error %v", action, err) } @@ -143,7 +143,7 @@ func createORDeleteRbdResources(action string) { if err != nil { e2elog.Failf("failed to read content from %s with error %v", rbdDirPath+rbdNodePluginRBAC, err) } - _, err = framework.RunKubectlInput(cephCSINamespace, data, action, ns, "-f", "-") + err = retryKubectlInput(cephCSINamespace, action, data, deployTimeout) if err != nil { e2elog.Failf("failed to %s nodeplugin rbac with error %v", action, err) } @@ -152,7 +152,7 @@ func createORDeleteRbdResources(action string) { if err != nil { e2elog.Failf("failed to read content from %s with error %v", rbdDirPath+rbdNodePluginPSP, err) } - _, err = framework.RunKubectlInput(cephCSINamespace, data, action, ns, "-f", "-") + err = retryKubectlInput(cephCSINamespace, action, data, deployTimeout) if err != nil { e2elog.Failf("failed to %s nodeplugin psp with error %v", action, err) } @@ -1529,7 +1529,13 @@ var _ = Describe("RBD", func() { e2elog.Failf("failed to validate clones in different pool with error %v", err) } - _, err = framework.RunKubectl(cephCSINamespace, "delete", "sc", cloneSC, "--ignore-not-found=true") + err = retryKubectlArgs( + cephCSINamespace, + kubectlDelete, + deployTimeout, + "sc", + cloneSC, + "--ignore-not-found=true") if err != nil { e2elog.Failf("failed to delete storageclass %s with error %v", cloneSC, err) } diff --git a/e2e/utils.go b/e2e/utils.go index d649dbc31..1996b330d 100644 --- a/e2e/utils.go +++ b/e2e/utils.go @@ -127,7 +127,7 @@ func deleteResource(scPath string) error { if err != nil { e2elog.Logf("failed to read content from %s %v", scPath, err) } - _, err = framework.RunKubectlInput(cephCSINamespace, data, ns, "delete", "-f", "-") + err = retryKubectlInput(cephCSINamespace, kubectlDelete, data, deployTimeout) if err != nil { e2elog.Logf("failed to delete %s %v", scPath, err) }