e2e: check isRetryableAPIError during polling

check isRetryableAPIError error when the
function is polling for the resouce.

updates: #1969

Signed-off-by: Madhu Rajanna <madhupr007@gmail.com>

e2e/errors.go

@@ -27,6 +27,7 @@ func isRetryableAPIError(err error) bool {
 		apierrors.IsTooManyRequests(err) || utilnet.IsProbableEOF(err) || utilnet.IsConnectionReset(err) {
 		return true
 	}
+
 	// If the error sends the Retry-After header, we respect it as an explicit confirmation we should retry.
 	if _, shouldRetry := apierrors.SuggestsClientDelay(err); shouldRetry {
 		return true

e2e/pod.go

@@ -81,12 +81,10 @@ func waitForDeploymentComplete(name, ns string, c kubernetes.Interface, t int) e
 	err = wait.PollImmediate(poll, timeout, func() (bool, error) {
 		deployment, err = c.AppsV1().Deployments(ns).Get(context.TODO(), name, metav1.GetOptions{})
 		if err != nil {
-			// a StatusError is not marked as 'retryable', but we want to retry anyway
+			if isRetryableAPIError(err) {
-			if isRetryableAPIError(err) || strings.Contains(err.Error(), "etcdserver: request timed out") {
-				// hide API-server timeouts, so that PollImmediate() retries
-				e2elog.Logf("deployment error: %v", err)
 				return false, nil
 			}
+			e2elog.Logf("deployment error: %v", err)
 			return false, err
 		}
 
@@ -293,6 +291,9 @@ func waitForPodInRunningState(name, ns string, c kubernetes.Interface, t int, ex
 	return wait.PollImmediate(poll, timeout, func() (bool, error) {
 		pod, err := c.CoreV1().Pods(ns).Get(context.TODO(), name, metav1.GetOptions{})
 		if err != nil {
+			if isRetryableAPIError(err) {
+				return false, nil
+			}
 			return false, fmt.Errorf("failed to get app: %w", err)
 		}
 		switch pod.Status.Phase {
@@ -333,12 +334,14 @@ func deletePod(name, ns string, c kubernetes.Interface, t int) error {
 	e2elog.Logf("Waiting for pod %v to be deleted", name)
 	return wait.PollImmediate(poll, timeout, func() (bool, error) {
 		_, err := c.CoreV1().Pods(ns).Get(context.TODO(), name, metav1.GetOptions{})
-
-		if apierrs.IsNotFound(err) {
-			return true, nil
-		}
-		e2elog.Logf("%s app  to be deleted (%d seconds elapsed)", name, int(time.Since(start).Seconds()))
 		if err != nil {
+			if isRetryableAPIError(err) {
+				return false, nil
+			}
+			if apierrs.IsNotFound(err) {
+				return true, nil
+			}
+			e2elog.Logf("%s app  to be deleted (%d seconds elapsed)", name, int(time.Since(start).Seconds()))
 			return false, fmt.Errorf("failed to get app: %w", err)
 		}
 		return false, nil

e2e/pvc.go

@@ -63,6 +63,9 @@ func createPVCAndvalidatePV(c kubernetes.Interface, pvc *v1.PersistentVolumeClai
 		if err != nil {
 			return false, fmt.Errorf("failed to get pv: %w", err)
 		}
+		if isRetryableAPIError(err) {
+			return false, nil
+		}
 		if apierrs.IsNotFound(err) {
 			return false, nil
 		}
@@ -118,6 +121,9 @@ func deletePVCAndPV(c kubernetes.Interface, pvc *v1.PersistentVolumeClaim, pv *v
 			}
 			return false, nil
 		}
+		if isRetryableAPIError(err) {
+			return false, nil
+		}
 		if !apierrs.IsNotFound(err) {
 			return false, fmt.Errorf(
 				"get on deleted PVC %v failed with error other than \"not found\": %w",
@@ -145,7 +151,9 @@ func deletePVCAndPV(c kubernetes.Interface, pvc *v1.PersistentVolumeClaim, pv *v
 		if err == nil {
 			return false, nil
 		}
-
+		if isRetryableAPIError(err) {
+			return false, nil
+		}
 		if !apierrs.IsNotFound(err) {
 			return false, fmt.Errorf("delete PV %v failed with error other than \"not found\": %w", pv.Name, err)
 		}
@@ -200,6 +208,9 @@ func deletePVCAndValidatePV(c kubernetes.Interface, pvc *v1.PersistentVolumeClai
 		if err == nil {
 			return false, nil
 		}
+		if isRetryableAPIError(err) {
+			return false, nil
+		}
 		if !apierrs.IsNotFound(err) {
 			return false, fmt.Errorf("get on deleted PVC %v failed with error other than \"not found\": %w", name, err)
 		}
@@ -209,7 +220,9 @@ func deletePVCAndValidatePV(c kubernetes.Interface, pvc *v1.PersistentVolumeClai
 		if err == nil {
 			return false, nil
 		}
-
+		if isRetryableAPIError(err) {
+			return false, nil
+		}
 		if !apierrs.IsNotFound(err) {
 			return false, fmt.Errorf("delete PV %v failed with error other than \"not found\": %w", pv.Name, err)
 		}

e2e/snapshot.go

@@ -114,6 +114,9 @@ func deleteSnapshot(snap *snapapi.VolumeSnapshot, t int) error {
 			return false, nil
 		}
 
+		if isRetryableAPIError(err) {
+			return false, nil
+		}
 		if !apierrs.IsNotFound(err) {
 			return false, fmt.Errorf(
 				"get on deleted snapshot %v failed with error other than \"not found\": %v",