e2e: retry running kubectl on known errors

By using retryKubectl helper function,
a retry will be done, and the known error
messages will be skipped.

Signed-off-by: Madhu Rajanna <madhupr007@gmail.com>

e2e/configmap.go

@@ -17,12 +17,8 @@ import (
 
 func deleteConfigMap(pluginPath string) error {
 	path := pluginPath + configMap
-	_, err := framework.RunKubectl(cephCSINamespace, "delete", "-f", path, ns)
-	if err != nil {
-		return err
-	}
 
-	return nil
+	return retryKubectlFile(cephCSINamespace, kubectlDelete, path, deployTimeout)
 }
 
 func createConfigMap(pluginPath string, c kubernetes.Interface, f *framework.Framework) error {

e2e/deploy-vault.go

@@ -8,7 +8,6 @@ import (
 	. "github.com/onsi/gomega" // nolint
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/client-go/kubernetes"
-	"k8s.io/kubernetes/test/e2e/framework"
 	e2elog "k8s.io/kubernetes/test/e2e/framework/log"
 )
 
@@ -25,13 +24,12 @@ var (
 func deployVault(c kubernetes.Interface, deployTimeout int) {
 	// hack to make helm E2E pass as helm charts creates this configmap as part
 	// of cephcsi deployment
-	_, err := framework.RunKubectl(
+	err := retryKubectlArgs(
 		cephCSINamespace,
-		"delete",
+		kubectlDelete,
+		deployTimeout,
 		"cm",
 		"ceph-csi-encryption-kms-config",
-		"--namespace",
-		cephCSINamespace,
 		"--ignore-not-found=true")
 	Expect(err).Should(BeNil())
 

e2e/pod.go

@@ -375,7 +375,14 @@ func deletePod(name, ns string, c kubernetes.Interface, t int) error {
 }
 
 func deletePodWithLabel(label, ns string, skipNotFound bool) error {
-	_, err := framework.RunKubectl(ns, "delete", "po", "-l", label, fmt.Sprintf("--ignore-not-found=%t", skipNotFound))
+	err := retryKubectlArgs(
+		ns,
+		kubectlDelete,
+		deployTimeout,
+		"po",
+		"-l",
+		label,
+		fmt.Sprintf("--ignore-not-found=%t", skipNotFound))
 	if err != nil {
 		e2elog.Logf("failed to delete pod %v", err)
 	}

e2e/rbd.go

@@ -63,7 +63,7 @@ func deployRBDPlugin() {
 	if err != nil {
 		e2elog.Failf("failed to read content from %s with error %v", rbdDirPath+rbdProvisionerRBAC, err)
 	}
-	_, err = framework.RunKubectlInput(cephCSINamespace, data, "--ignore-not-found=true", ns, "delete", "-f", "-")
+	err = retryKubectlInput(cephCSINamespace, kubectlDelete, data, deployTimeout, "--ignore-not-found=true")
 	if err != nil {
 		e2elog.Failf("failed to delete provisioner rbac %s with error %v", rbdDirPath+rbdProvisionerRBAC, err)
 	}
@@ -72,19 +72,19 @@ func deployRBDPlugin() {
 	if err != nil {
 		e2elog.Failf("failed to read content from %s with error %v", rbdDirPath+rbdNodePluginRBAC, err)
 	}
-	_, err = framework.RunKubectlInput(cephCSINamespace, data, "delete", "--ignore-not-found=true", ns, "-f", "-")
+	err = retryKubectlInput(cephCSINamespace, kubectlDelete, data, deployTimeout, "--ignore-not-found=true")
 	if err != nil {
 		e2elog.Failf("failed to delete nodeplugin rbac %s with error %v", rbdDirPath+rbdNodePluginRBAC, err)
 	}
 
-	createORDeleteRbdResources("create")
+	createORDeleteRbdResources(kubectlCreate)
 }
 
 func deleteRBDPlugin() {
-	createORDeleteRbdResources("delete")
+	createORDeleteRbdResources(kubectlDelete)
 }
 
-func createORDeleteRbdResources(action string) {
+func createORDeleteRbdResources(action kubectlAction) {
 	csiDriver, err := ioutil.ReadFile(rbdDirPath + csiDriverObject)
 	if err != nil {
 		// createORDeleteRbdResources is used for upgrade testing as csidriverObject is
@@ -93,7 +93,7 @@ func createORDeleteRbdResources(action string) {
 			e2elog.Failf("failed to read content from %s with error %v", rbdDirPath+csiDriverObject, err)
 		}
 	} else {
-		_, err = framework.RunKubectlInput(cephCSINamespace, string(csiDriver), action, "-f", "-")
+		err = retryKubectlInput(cephCSINamespace, action, string(csiDriver), deployTimeout)
 		if err != nil {
 			e2elog.Failf("failed to %s CSIDriver object with error %v", action, err)
 		}
@@ -104,7 +104,7 @@ func createORDeleteRbdResources(action string) {
 	}
 	data = oneReplicaDeployYaml(data)
 	data = enableTopologyInTemplate(data)
-	_, err = framework.RunKubectlInput(cephCSINamespace, data, action, ns, "-f", "-")
+	err = retryKubectlInput(cephCSINamespace, action, data, deployTimeout)
 	if err != nil {
 		e2elog.Failf("failed to %s rbd provisioner with error %v", action, err)
 	}
@@ -113,7 +113,7 @@ func createORDeleteRbdResources(action string) {
 	if err != nil {
 		e2elog.Failf("failed to read content from %s with error %v", rbdDirPath+rbdProvisionerRBAC, err)
 	}
-	_, err = framework.RunKubectlInput(cephCSINamespace, data, action, ns, "-f", "-")
+	err = retryKubectlInput(cephCSINamespace, action, data, deployTimeout)
 	if err != nil {
 		e2elog.Failf("failed to %s provisioner rbac with error %v", action, err)
 	}
@@ -122,7 +122,7 @@ func createORDeleteRbdResources(action string) {
 	if err != nil {
 		e2elog.Failf("failed to read content from %s with error %v", rbdDirPath+rbdProvisionerPSP, err)
 	}
-	_, err = framework.RunKubectlInput(cephCSINamespace, data, action, "-f", "-")
+	err = retryKubectlInput(cephCSINamespace, action, data, deployTimeout)
 	if err != nil {
 		e2elog.Failf("failed to %s provisioner psp with error %v", action, err)
 	}
@@ -134,7 +134,7 @@ func createORDeleteRbdResources(action string) {
 
 	domainLabel := nodeRegionLabel + "," + nodeZoneLabel
 	data = addTopologyDomainsToDSYaml(data, domainLabel)
-	_, err = framework.RunKubectlInput(cephCSINamespace, data, action, ns, "-f", "-")
+	err = retryKubectlInput(cephCSINamespace, action, data, deployTimeout)
 	if err != nil {
 		e2elog.Failf("failed to %s nodeplugin with error %v", action, err)
 	}
@@ -143,7 +143,7 @@ func createORDeleteRbdResources(action string) {
 	if err != nil {
 		e2elog.Failf("failed to read content from %s with error %v", rbdDirPath+rbdNodePluginRBAC, err)
 	}
-	_, err = framework.RunKubectlInput(cephCSINamespace, data, action, ns, "-f", "-")
+	err = retryKubectlInput(cephCSINamespace, action, data, deployTimeout)
 	if err != nil {
 		e2elog.Failf("failed to %s nodeplugin rbac with error %v", action, err)
 	}
@@ -152,7 +152,7 @@ func createORDeleteRbdResources(action string) {
 	if err != nil {
 		e2elog.Failf("failed to read content from %s with error %v", rbdDirPath+rbdNodePluginPSP, err)
 	}
-	_, err = framework.RunKubectlInput(cephCSINamespace, data, action, ns, "-f", "-")
+	err = retryKubectlInput(cephCSINamespace, action, data, deployTimeout)
 	if err != nil {
 		e2elog.Failf("failed to %s nodeplugin psp with error %v", action, err)
 	}
@@ -1529,7 +1529,13 @@ var _ = Describe("RBD", func() {
 					e2elog.Failf("failed to validate clones in different pool with error %v", err)
 				}
 
-				_, err = framework.RunKubectl(cephCSINamespace, "delete", "sc", cloneSC, "--ignore-not-found=true")
+				err = retryKubectlArgs(
+					cephCSINamespace,
+					kubectlDelete,
+					deployTimeout,
+					"sc",
+					cloneSC,
+					"--ignore-not-found=true")
 				if err != nil {
 					e2elog.Failf("failed to delete storageclass %s with error %v", cloneSC, err)
 				}

e2e/utils.go

@@ -127,7 +127,7 @@ func deleteResource(scPath string) error {
 	if err != nil {
 		e2elog.Logf("failed to read content from %s %v", scPath, err)
 	}
-	_, err = framework.RunKubectlInput(cephCSINamespace, data, ns, "delete", "-f", "-")
+	err = retryKubectlInput(cephCSINamespace, kubectlDelete, data, deployTimeout)
 	if err != nil {
 		e2elog.Logf("failed to delete %s %v", scPath, err)
 	}