From d3beaeb014fbc3a7f605054ab03006d127ac0430 Mon Sep 17 00:00:00 2001 From: Niels de Vos Date: Mon, 26 Jul 2021 17:48:49 +0200 Subject: [PATCH] e2e: retry deploying CephFS components on failure There are reports where CephFS deploying failed with etcdserver timeouts: INFO: Running '/usr/bin/kubectl --server=https://192.168.39.187:8443 --kubeconfig=/root/.kube/config --namespace=cephcsi-e2e-ea434921 create --namespace=cephcsi-e2e-ea434921 -f -' INFO: rc: 1 FAIL: failed to create CephFS provisioner rbac with error error running /usr/bin/kubectl --server=https://192.168.39.187:8443 --kubeconfig=/root/.kube/config --namespace=cephcsi-e2e-ea434921 create --namespace=cephcsi-e2e-ea434921 -f -: Command stdout: role.rbac.authorization.k8s.io/cephfs-external-provisioner-cfg created rolebinding.rbac.authorization.k8s.io/cephfs-csi-provisioner-role-cfg created stderr: Error from server: error when creating "STDIN": etcdserver: request timed out Error from server: error when creating "STDIN": etcdserver: request timed out Error from server: error when creating "STDIN": etcdserver: request timed out error: exit status 1 By using retryKubectlInput() helper function, a retry will be done, and the failure should not be fatal any longer. Signed-off-by: Niels de Vos --- e2e/cephfs.go | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/e2e/cephfs.go b/e2e/cephfs.go index 7044aec64..d8271e7c1 100644 --- a/e2e/cephfs.go +++ b/e2e/cephfs.go @@ -55,14 +55,14 @@ func deployCephfsPlugin() { e2elog.Failf("failed to delete nodeplugin rbac %s with error %v", cephfsDirPath+cephfsNodePluginRBAC, err) } - createORDeleteCephfsResources("create") + createORDeleteCephfsResources(kubectlCreate) } func deleteCephfsPlugin() { - createORDeleteCephfsResources("delete") + createORDeleteCephfsResources(kubectlDelete) } -func createORDeleteCephfsResources(action string) { +func createORDeleteCephfsResources(action kubectlAction) { csiDriver, err := ioutil.ReadFile(cephfsDirPath + csiDriverObject) if err != nil { // createORDeleteRbdResources is used for upgrade testing as csidriverObject is @@ -71,7 +71,7 @@ func createORDeleteCephfsResources(action string) { e2elog.Failf("failed to read content from %s with error %v", cephfsDirPath+csiDriverObject, err) } } else { - _, err = framework.RunKubectlInput(cephCSINamespace, string(csiDriver), action, "-f", "-") + err = retryKubectlInput(cephCSINamespace, action, string(csiDriver), deployTimeout) if err != nil { e2elog.Failf("failed to %s CSIDriver object with error %v", action, err) } @@ -81,7 +81,7 @@ func createORDeleteCephfsResources(action string) { e2elog.Failf("failed to read content from %s with error %v", cephfsDirPath+cephfsProvisioner, err) } data = oneReplicaDeployYaml(data) - _, err = framework.RunKubectlInput(cephCSINamespace, data, action, ns, "-f", "-") + err = retryKubectlInput(cephCSINamespace, action, data, deployTimeout) if err != nil { e2elog.Failf("failed to %s CephFS provisioner with error %v", action, err) } @@ -90,7 +90,7 @@ func createORDeleteCephfsResources(action string) { if err != nil { e2elog.Failf("failed to read content from %s with error %v", cephfsDirPath+cephfsProvisionerRBAC, err) } - _, err = framework.RunKubectlInput(cephCSINamespace, data, action, ns, "-f", "-") + err = retryKubectlInput(cephCSINamespace, action, data, deployTimeout) if err != nil { e2elog.Failf("failed to %s CephFS provisioner rbac with error %v", action, err) } @@ -99,7 +99,7 @@ func createORDeleteCephfsResources(action string) { if err != nil { e2elog.Failf("failed to read content from %s with error %v", cephfsDirPath+cephfsProvisionerPSP, err) } - _, err = framework.RunKubectlInput(cephCSINamespace, data, action, ns, "-f", "-") + err = retryKubectlInput(cephCSINamespace, action, data, deployTimeout) if err != nil { e2elog.Failf("failed to %s CephFS provisioner psp with error %v", action, err) } @@ -108,7 +108,7 @@ func createORDeleteCephfsResources(action string) { if err != nil { e2elog.Failf("failed to read content from %s with error %v", cephfsDirPath+cephfsNodePlugin, err) } - _, err = framework.RunKubectlInput(cephCSINamespace, data, action, ns, "-f", "-") + err = retryKubectlInput(cephCSINamespace, action, data, deployTimeout) if err != nil { e2elog.Failf("failed to %s CephFS nodeplugin with error %v", action, err) } @@ -117,7 +117,7 @@ func createORDeleteCephfsResources(action string) { if err != nil { e2elog.Failf("failed to read content from %s with error %v", cephfsDirPath+cephfsNodePluginRBAC, err) } - _, err = framework.RunKubectlInput(cephCSINamespace, data, action, ns, "-f", "-") + err = retryKubectlInput(cephCSINamespace, action, data, deployTimeout) if err != nil { e2elog.Failf("failed to %s CephFS nodeplugin rbac with error %v", action, err) } @@ -126,7 +126,7 @@ func createORDeleteCephfsResources(action string) { if err != nil { e2elog.Failf("failed to read content from %s with error %v", cephfsDirPath+cephfsNodePluginPSP, err) } - _, err = framework.RunKubectlInput(cephCSINamespace, data, action, ns, "-f", "-") + err = retryKubectlInput(cephCSINamespace, action, data, deployTimeout) if err != nil { e2elog.Failf("failed to %s CephFS nodeplugin psp with error %v", action, err) }