mirrors/ceph-csi
1
0
Fork 0
mirror of https://github.com/ceph/ceph-csi.git synced 2024-12-18 11:00:25 +00:00
Code Issues Packages Projects Releases Wiki Activity

e2e: retry deploying CephFS components on failure

Browse Source 
There are reports where CephFS deploying failed with etcdserver
timeouts:

    INFO: Running '/usr/bin/kubectl --server=https://192.168.39.187:8443 --kubeconfig=/root/.kube/config --namespace=cephcsi-e2e-ea434921 create --namespace=cephcsi-e2e-ea434921 -f -'
    INFO: rc: 1
    FAIL: failed to create CephFS provisioner rbac with error error running /usr/bin/kubectl --server=https://192.168.39.187:8443 --kubeconfig=/root/.kube/config --namespace=cephcsi-e2e-ea434921 create --namespace=cephcsi-e2e-ea434921 -f -:
    Command stdout:
    role.rbac.authorization.k8s.io/cephfs-external-provisioner-cfg created
    rolebinding.rbac.authorization.k8s.io/cephfs-csi-provisioner-role-cfg created

    stderr:
    Error from server: error when creating "STDIN": etcdserver: request timed out
    Error from server: error when creating "STDIN": etcdserver: request timed out
    Error from server: error when creating "STDIN": etcdserver: request timed out

    error:
    exit status 1

By using retryKubectlInput() helper function, a retry will be done, and
the failure should not be fatal any longer.

Signed-off-by: Niels de Vos <ndevos@redhat.com>
This commit is contained in:
Niels de Vos 2021-07-26 17:48:49 +02:00 committed by mergify[bot]
parent d2def71944
commit d3beaeb014
1 changed files with 10 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

20
e2e/cephfs.go
View File

@ -55,14 +55,14 @@ func deployCephfsPlugin() {
e2elog.Failf("failed to delete nodeplugin rbac %s with error %v", cephfsDirPath+cephfsNodePluginRBAC, err) e2elog.Failf("failed to delete nodeplugin rbac %s with error %v", cephfsDirPath+cephfsNodePluginRBAC, err)
} }
createORDeleteCephfsResources("create") createORDeleteCephfsResources(kubectlCreate)
} }
func deleteCephfsPlugin() { func deleteCephfsPlugin() {
createORDeleteCephfsResources("delete") createORDeleteCephfsResources(kubectlDelete)
} }
func createORDeleteCephfsResources(action string) { func createORDeleteCephfsResources(action kubectlAction) {
csiDriver, err := ioutil.ReadFile(cephfsDirPath + csiDriverObject) csiDriver, err := ioutil.ReadFile(cephfsDirPath + csiDriverObject)
if err != nil { if err != nil {
// createORDeleteRbdResources is used for upgrade testing as csidriverObject is // createORDeleteRbdResources is used for upgrade testing as csidriverObject is
@ -71,7 +71,7 @@ func createORDeleteCephfsResources(action string) {
e2elog.Failf("failed to read content from %s with error %v", cephfsDirPath+csiDriverObject, err) e2elog.Failf("failed to read content from %s with error %v", cephfsDirPath+csiDriverObject, err)
} }
} else { } else {
_, err = framework.RunKubectlInput(cephCSINamespace, string(csiDriver), action, "-f", "-") err = retryKubectlInput(cephCSINamespace, action, string(csiDriver), deployTimeout)
if err != nil { if err != nil {
e2elog.Failf("failed to %s CSIDriver object with error %v", action, err) e2elog.Failf("failed to %s CSIDriver object with error %v", action, err)
} }
@ -81,7 +81,7 @@ func createORDeleteCephfsResources(action string) {
e2elog.Failf("failed to read content from %s with error %v", cephfsDirPath+cephfsProvisioner, err) e2elog.Failf("failed to read content from %s with error %v", cephfsDirPath+cephfsProvisioner, err)
} }
data = oneReplicaDeployYaml(data) data = oneReplicaDeployYaml(data)
_, err = framework.RunKubectlInput(cephCSINamespace, data, action, ns, "-f", "-") err = retryKubectlInput(cephCSINamespace, action, data, deployTimeout)
if err != nil { if err != nil {
e2elog.Failf("failed to %s CephFS provisioner with error %v", action, err) e2elog.Failf("failed to %s CephFS provisioner with error %v", action, err)
} }
@ -90,7 +90,7 @@ func createORDeleteCephfsResources(action string) {
if err != nil { if err != nil {
e2elog.Failf("failed to read content from %s with error %v", cephfsDirPath+cephfsProvisionerRBAC, err) e2elog.Failf("failed to read content from %s with error %v", cephfsDirPath+cephfsProvisionerRBAC, err)
} }
_, err = framework.RunKubectlInput(cephCSINamespace, data, action, ns, "-f", "-") err = retryKubectlInput(cephCSINamespace, action, data, deployTimeout)
if err != nil { if err != nil {
e2elog.Failf("failed to %s CephFS provisioner rbac with error %v", action, err) e2elog.Failf("failed to %s CephFS provisioner rbac with error %v", action, err)
} }
@ -99,7 +99,7 @@ func createORDeleteCephfsResources(action string) {
if err != nil { if err != nil {
e2elog.Failf("failed to read content from %s with error %v", cephfsDirPath+cephfsProvisionerPSP, err) e2elog.Failf("failed to read content from %s with error %v", cephfsDirPath+cephfsProvisionerPSP, err)
} }
_, err = framework.RunKubectlInput(cephCSINamespace, data, action, ns, "-f", "-") err = retryKubectlInput(cephCSINamespace, action, data, deployTimeout)
if err != nil { if err != nil {
e2elog.Failf("failed to %s CephFS provisioner psp with error %v", action, err) e2elog.Failf("failed to %s CephFS provisioner psp with error %v", action, err)
} }
@ -108,7 +108,7 @@ func createORDeleteCephfsResources(action string) {
if err != nil { if err != nil {
e2elog.Failf("failed to read content from %s with error %v", cephfsDirPath+cephfsNodePlugin, err) e2elog.Failf("failed to read content from %s with error %v", cephfsDirPath+cephfsNodePlugin, err)
} }
_, err = framework.RunKubectlInput(cephCSINamespace, data, action, ns, "-f", "-") err = retryKubectlInput(cephCSINamespace, action, data, deployTimeout)
if err != nil { if err != nil {
e2elog.Failf("failed to %s CephFS nodeplugin with error %v", action, err) e2elog.Failf("failed to %s CephFS nodeplugin with error %v", action, err)
} }
@ -117,7 +117,7 @@ func createORDeleteCephfsResources(action string) {
if err != nil { if err != nil {
e2elog.Failf("failed to read content from %s with error %v", cephfsDirPath+cephfsNodePluginRBAC, err) e2elog.Failf("failed to read content from %s with error %v", cephfsDirPath+cephfsNodePluginRBAC, err)
} }
_, err = framework.RunKubectlInput(cephCSINamespace, data, action, ns, "-f", "-") err = retryKubectlInput(cephCSINamespace, action, data, deployTimeout)
if err != nil { if err != nil {
e2elog.Failf("failed to %s CephFS nodeplugin rbac with error %v", action, err) e2elog.Failf("failed to %s CephFS nodeplugin rbac with error %v", action, err)
} }
@ -126,7 +126,7 @@ func createORDeleteCephfsResources(action string) {
if err != nil { if err != nil {
e2elog.Failf("failed to read content from %s with error %v", cephfsDirPath+cephfsNodePluginPSP, err) e2elog.Failf("failed to read content from %s with error %v", cephfsDirPath+cephfsNodePluginPSP, err)
} }
_, err = framework.RunKubectlInput(cephCSINamespace, data, action, ns, "-f", "-") err = retryKubectlInput(cephCSINamespace, action, data, deployTimeout)
if err != nil { if err != nil {
e2elog.Failf("failed to %s CephFS nodeplugin psp with error %v", action, err) e2elog.Failf("failed to %s CephFS nodeplugin psp with error %v", action, err)
} }