diff --git a/e2e/errors.go b/e2e/errors.go index e7ec26824..04d4e7bc8 100644 --- a/e2e/errors.go +++ b/e2e/errors.go @@ -27,6 +27,7 @@ func isRetryableAPIError(err error) bool { apierrors.IsTooManyRequests(err) || utilnet.IsProbableEOF(err) || utilnet.IsConnectionReset(err) { return true } + // If the error sends the Retry-After header, we respect it as an explicit confirmation we should retry. if _, shouldRetry := apierrors.SuggestsClientDelay(err); shouldRetry { return true diff --git a/e2e/pod.go b/e2e/pod.go index 627acbae9..e96b84e33 100644 --- a/e2e/pod.go +++ b/e2e/pod.go @@ -81,12 +81,10 @@ func waitForDeploymentComplete(name, ns string, c kubernetes.Interface, t int) e err = wait.PollImmediate(poll, timeout, func() (bool, error) { deployment, err = c.AppsV1().Deployments(ns).Get(context.TODO(), name, metav1.GetOptions{}) if err != nil { - // a StatusError is not marked as 'retryable', but we want to retry anyway - if isRetryableAPIError(err) || strings.Contains(err.Error(), "etcdserver: request timed out") { - // hide API-server timeouts, so that PollImmediate() retries - e2elog.Logf("deployment error: %v", err) + if isRetryableAPIError(err) { return false, nil } + e2elog.Logf("deployment error: %v", err) return false, err } @@ -293,6 +291,9 @@ func waitForPodInRunningState(name, ns string, c kubernetes.Interface, t int, ex return wait.PollImmediate(poll, timeout, func() (bool, error) { pod, err := c.CoreV1().Pods(ns).Get(context.TODO(), name, metav1.GetOptions{}) if err != nil { + if isRetryableAPIError(err) { + return false, nil + } return false, fmt.Errorf("failed to get app: %w", err) } switch pod.Status.Phase { @@ -333,12 +334,14 @@ func deletePod(name, ns string, c kubernetes.Interface, t int) error { e2elog.Logf("Waiting for pod %v to be deleted", name) return wait.PollImmediate(poll, timeout, func() (bool, error) { _, err := c.CoreV1().Pods(ns).Get(context.TODO(), name, metav1.GetOptions{}) - - if apierrs.IsNotFound(err) { - return true, nil - } - e2elog.Logf("%s app to be deleted (%d seconds elapsed)", name, int(time.Since(start).Seconds())) if err != nil { + if isRetryableAPIError(err) { + return false, nil + } + if apierrs.IsNotFound(err) { + return true, nil + } + e2elog.Logf("%s app to be deleted (%d seconds elapsed)", name, int(time.Since(start).Seconds())) return false, fmt.Errorf("failed to get app: %w", err) } return false, nil diff --git a/e2e/pvc.go b/e2e/pvc.go index d64eca391..c99052ba0 100644 --- a/e2e/pvc.go +++ b/e2e/pvc.go @@ -63,6 +63,9 @@ func createPVCAndvalidatePV(c kubernetes.Interface, pvc *v1.PersistentVolumeClai if err != nil { return false, fmt.Errorf("failed to get pv: %w", err) } + if isRetryableAPIError(err) { + return false, nil + } if apierrs.IsNotFound(err) { return false, nil } @@ -118,6 +121,9 @@ func deletePVCAndPV(c kubernetes.Interface, pvc *v1.PersistentVolumeClaim, pv *v } return false, nil } + if isRetryableAPIError(err) { + return false, nil + } if !apierrs.IsNotFound(err) { return false, fmt.Errorf( "get on deleted PVC %v failed with error other than \"not found\": %w", @@ -145,7 +151,9 @@ func deletePVCAndPV(c kubernetes.Interface, pvc *v1.PersistentVolumeClaim, pv *v if err == nil { return false, nil } - + if isRetryableAPIError(err) { + return false, nil + } if !apierrs.IsNotFound(err) { return false, fmt.Errorf("delete PV %v failed with error other than \"not found\": %w", pv.Name, err) } @@ -200,6 +208,9 @@ func deletePVCAndValidatePV(c kubernetes.Interface, pvc *v1.PersistentVolumeClai if err == nil { return false, nil } + if isRetryableAPIError(err) { + return false, nil + } if !apierrs.IsNotFound(err) { return false, fmt.Errorf("get on deleted PVC %v failed with error other than \"not found\": %w", name, err) } @@ -209,7 +220,9 @@ func deletePVCAndValidatePV(c kubernetes.Interface, pvc *v1.PersistentVolumeClai if err == nil { return false, nil } - + if isRetryableAPIError(err) { + return false, nil + } if !apierrs.IsNotFound(err) { return false, fmt.Errorf("delete PV %v failed with error other than \"not found\": %w", pv.Name, err) } diff --git a/e2e/snapshot.go b/e2e/snapshot.go index 6ce9d8ccf..3ba860616 100644 --- a/e2e/snapshot.go +++ b/e2e/snapshot.go @@ -114,6 +114,9 @@ func deleteSnapshot(snap *snapapi.VolumeSnapshot, t int) error { return false, nil } + if isRetryableAPIError(err) { + return false, nil + } if !apierrs.IsNotFound(err) { return false, fmt.Errorf( "get on deleted snapshot %v failed with error other than \"not found\": %v",