diff --git a/e2e/errors.go b/e2e/errors.go
index da414bfd8..a9a5d6b5a 100644
--- a/e2e/errors.go
+++ b/e2e/errors.go
@@ -127,3 +127,33 @@ func isAlreadyExistsCLIError(err error) bool {
 
 	return true
 }
+
+// isNotFoundCLIError checks for "is not found" error from kubectl CLI.
+func isNotFoundCLIError(err error) bool {
+	if err == nil {
+		return false
+	}
+	// if multiple resources already exists. each error is separated by newline
+	stdErr := getStdErr(err.Error())
+	if stdErr == "" {
+		return false
+	}
+
+	stdErrs := strings.Split(stdErr, "\n")
+	for _, s := range stdErrs {
+		// If the string is just a new line continue
+		if strings.TrimSuffix(s, "\n") == "" {
+			continue
+		}
+		// Ignore warnings
+		if strings.Contains(s, "Warning") {
+			continue
+		}
+		// Resource not found error message
+		if !strings.Contains(s, "Error from server (NotFound)") {
+			return false
+		}
+	}
+
+	return true
+}
diff --git a/e2e/utils.go b/e2e/utils.go
index d3e296477..34ca6a849 100644
--- a/e2e/utils.go
+++ b/e2e/utils.go
@@ -1415,7 +1415,10 @@ func retryKubectlInput(namespace string, action kubectlAction, data string, t in
 			if isRetryableAPIError(err) {
 				return false, nil
 			}
-			if isAlreadyExistsCLIError(err) {
+			if action == kubectlCreate && isAlreadyExistsCLIError(err) {
+				return true, nil
+			}
+			if action == kubectlDelete && isNotFoundCLIError(err) {
 				return true, nil
 			}
 			e2elog.Logf(
@@ -1451,7 +1454,10 @@ func retryKubectlFile(namespace string, action kubectlAction, filename string, t
 			if isRetryableAPIError(err) {
 				return false, nil
 			}
-			if isAlreadyExistsCLIError(err) {
+			if action == kubectlCreate && isAlreadyExistsCLIError(err) {
+				return true, nil
+			}
+			if action == kubectlDelete && isNotFoundCLIError(err) {
 				return true, nil
 			}
 			e2elog.Logf(
@@ -1484,7 +1490,10 @@ func retryKubectlArgs(namespace string, action kubectlAction, t int, args ...str
 			if isRetryableAPIError(err) {
 				return false, nil
 			}
-			if isAlreadyExistsCLIError(err) {
+			if action == kubectlCreate && isAlreadyExistsCLIError(err) {
+				return true, nil
+			}
+			if action == kubectlDelete && isNotFoundCLIError(err) {
 				return true, nil
 			}
 			e2elog.Logf(