From 96eafecad89b2075edcb274ba6a626e59c0a7825 Mon Sep 17 00:00:00 2001 From: Niels de Vos Date: Wed, 4 Nov 2020 15:45:06 +0100 Subject: [PATCH] e2e: do not use Failf() to abort tests in a go-routine There are several go-routines where Failf() is called, which will cause a Golang panic inside the Ginko test framework. Instead of aborting the go-routine, capture the error and check for failures once all go-routines have finished. Updates: #1359 Signed-off-by: Niels de Vos --- e2e/cephfs.go | 132 ++++++++++++++++++++++++++++++++++++++------------ 1 file changed, 100 insertions(+), 32 deletions(-) diff --git a/e2e/cephfs.go b/e2e/cephfs.go index 4de74acde..f9df1eee7 100644 --- a/e2e/cephfs.go +++ b/e2e/cephfs.go @@ -482,6 +482,7 @@ var _ = Describe("cephfs", func() { if k8sVersionGreaterEquals(f.ClientSet, 1, 17) { var wg sync.WaitGroup totalCount := 3 + wgErrs := make([]error, totalCount) // totalSubvolumes represents the subvolumes in backend // always totalCount+parentPVC totalSubvolumes := totalCount + 1 @@ -520,15 +521,24 @@ var _ = Describe("cephfs", func() { for i := 0; i < totalCount; i++ { go func(w *sync.WaitGroup, n int, s vs.VolumeSnapshot) { s.Name = fmt.Sprintf("%s%d", f.UniqueName, n) - err = createSnapshot(&s, deployTimeout) - if err != nil { - e2elog.Failf("failed to create snapshot with error %v", err) - } + wgErrs[n] = createSnapshot(&s, deployTimeout) w.Done() }(&wg, i, snap) } wg.Wait() + failed := 0 + for i, err := range wgErrs { + if err != nil { + // not using Failf() as it aborts the test and does not log other errors + e2elog.Logf("failed to create snapshot (%s%d): %v", f.UniqueName, i, err) + failed++ + } + } + if failed != 0 { + e2elog.Failf("creating snapshots failed, %d errors were logged", failed) + } + pvcClone, err := loadPVC(pvcClonePath) if err != nil { e2elog.Failf("failed to load PVC with error %v", err) @@ -547,15 +557,23 @@ var _ = Describe("cephfs", func() { go func(w *sync.WaitGroup, n int, p v1.PersistentVolumeClaim, a v1.Pod) { name := fmt.Sprintf("%s%d", f.UniqueName, n) - err = createPVCAndApp(name, f, &p, &a, deployTimeout) - if err != nil { - e2elog.Failf("failed to create PVC and app with error %v", err) - } + wgErrs[n] = createPVCAndApp(name, f, &p, &a, deployTimeout) w.Done() }(&wg, i, *pvcClone, *appClone) } wg.Wait() + for i, err := range wgErrs { + if err != nil { + // not using Failf() as it aborts the test and does not log other errors + e2elog.Logf("failed to create PVC and app (%s%d): %v", f.UniqueName, i, err) + failed++ + } + } + if failed != 0 { + e2elog.Failf("creating PVCs and apps failed, %d errors were logged", failed) + } + validateSubvolumeCount(f, totalSubvolumes, fileSystemName, subvolumegroup) wg.Add(totalCount) @@ -564,15 +582,23 @@ var _ = Describe("cephfs", func() { go func(w *sync.WaitGroup, n int, p v1.PersistentVolumeClaim, a v1.Pod) { name := fmt.Sprintf("%s%d", f.UniqueName, n) p.Spec.DataSource.Name = name - err = deletePVCAndApp(name, f, &p, &a) - if err != nil { - e2elog.Failf("failed to delete PVC and app with error %v", err) - } + wgErrs[n] = deletePVCAndApp(name, f, &p, &a) w.Done() }(&wg, i, *pvcClone, *appClone) } wg.Wait() + for i, err := range wgErrs { + if err != nil { + // not using Failf() as it aborts the test and does not log other errors + e2elog.Logf("failed to delete PVC and app (%s%d): %v", f.UniqueName, i, err) + failed++ + } + } + if failed != 0 { + e2elog.Failf("deleting PVCs and apps failed, %d errors were logged", failed) + } + parentPVCCount := totalSubvolumes - totalCount validateSubvolumeCount(f, parentPVCCount, fileSystemName, subvolumegroup) // create clones from different snapshosts and bind it to an @@ -582,15 +608,23 @@ var _ = Describe("cephfs", func() { go func(w *sync.WaitGroup, n int, p v1.PersistentVolumeClaim, a v1.Pod) { name := fmt.Sprintf("%s%d", f.UniqueName, n) p.Spec.DataSource.Name = name - err = createPVCAndApp(name, f, &p, &a, deployTimeout) - if err != nil { - e2elog.Failf("failed to create PVC and app with error %v", err) - } + wgErrs[n] = createPVCAndApp(name, f, &p, &a, deployTimeout) w.Done() }(&wg, i, *pvcClone, *appClone) } wg.Wait() + for i, err := range wgErrs { + if err != nil { + // not using Failf() as it aborts the test and does not log other errors + e2elog.Logf("failed to create PVC and app (%s%d): %v", f.UniqueName, i, err) + failed++ + } + } + if failed != 0 { + e2elog.Failf("creating PVCs and apps failed, %d errors were logged", failed) + } + validateSubvolumeCount(f, totalSubvolumes, fileSystemName, subvolumegroup) wg.Add(totalCount) @@ -598,30 +632,46 @@ var _ = Describe("cephfs", func() { for i := 0; i < totalCount; i++ { go func(w *sync.WaitGroup, n int, s vs.VolumeSnapshot) { s.Name = fmt.Sprintf("%s%d", f.UniqueName, n) - err = deleteSnapshot(&s, deployTimeout) - if err != nil { - e2elog.Failf("failed to delete snapshot with error %v", err) - } + wgErrs[n] = deleteSnapshot(&s, deployTimeout) w.Done() }(&wg, i, snap) } wg.Wait() + for i, err := range wgErrs { + if err != nil { + // not using Failf() as it aborts the test and does not log other errors + e2elog.Logf("failed to delete snapshot (%s%d): %v", f.UniqueName, i, err) + failed++ + } + } + if failed != 0 { + e2elog.Failf("deleting snapshots failed, %d errors were logged", failed) + } + wg.Add(totalCount) // delete clone and app for i := 0; i < totalCount; i++ { go func(w *sync.WaitGroup, n int, p v1.PersistentVolumeClaim, a v1.Pod) { name := fmt.Sprintf("%s%d", f.UniqueName, n) p.Spec.DataSource.Name = name - err = deletePVCAndApp(name, f, &p, &a) - if err != nil { - e2elog.Failf("failed to delete PVC and app with error %v", err) - } + wgErrs[n] = deletePVCAndApp(name, f, &p, &a) w.Done() }(&wg, i, *pvcClone, *appClone) } wg.Wait() + for i, err := range wgErrs { + if err != nil { + // not using Failf() as it aborts the test and does not log other errors + e2elog.Logf("failed to delete PVC and app (%s%d): %v", f.UniqueName, i, err) + failed++ + } + } + if failed != 0 { + e2elog.Failf("deleting PVCs and apps failed, %d errors were logged", failed) + } + validateSubvolumeCount(f, parentPVCCount, fileSystemName, subvolumegroup) // delete parent pvc err = deletePVCAndApp("", f, pvc, app) @@ -638,6 +688,7 @@ var _ = Describe("cephfs", func() { if k8sVersionGreaterEquals(f.ClientSet, 1, 16) { var wg sync.WaitGroup totalCount := 3 + wgErrs := make([]error, totalCount) // totalSubvolumes represents the subvolumes in backend // always totalCount+parentPVC totalSubvolumes := totalCount + 1 @@ -678,15 +729,24 @@ var _ = Describe("cephfs", func() { for i := 0; i < totalCount; i++ { go func(w *sync.WaitGroup, n int, p v1.PersistentVolumeClaim, a v1.Pod) { name := fmt.Sprintf("%s%d", f.UniqueName, n) - err = createPVCAndApp(name, f, &p, &a, deployTimeout) - if err != nil { - e2elog.Failf("failed to create PVC or application with error %v", err) - } + wgErrs[n] = createPVCAndApp(name, f, &p, &a, deployTimeout) w.Done() }(&wg, i, *pvcClone, *appClone) } wg.Wait() + failed := 0 + for i, err := range wgErrs { + if err != nil { + // not using Failf() as it aborts the test and does not log other errors + e2elog.Logf("failed to create PVC or application (%s%d): %v", f.UniqueName, i, err) + failed++ + } + } + if failed != 0 { + e2elog.Failf("deleting PVCs and apps failed, %d errors were logged", failed) + } + validateSubvolumeCount(f, totalSubvolumes, fileSystemName, subvolumegroup) // delete parent pvc @@ -701,15 +761,23 @@ var _ = Describe("cephfs", func() { go func(w *sync.WaitGroup, n int, p v1.PersistentVolumeClaim, a v1.Pod) { name := fmt.Sprintf("%s%d", f.UniqueName, n) p.Spec.DataSource.Name = name - err = deletePVCAndApp(name, f, &p, &a) - if err != nil { - e2elog.Failf("failed to delete PVC or application with error %v", err) - } + wgErrs[n] = deletePVCAndApp(name, f, &p, &a) w.Done() }(&wg, i, *pvcClone, *appClone) } wg.Wait() + for i, err := range wgErrs { + if err != nil { + // not using Failf() as it aborts the test and does not log other errors + e2elog.Logf("failed to delete PVC or application (%s%d): %v", f.UniqueName, i, err) + failed++ + } + } + if failed != 0 { + e2elog.Failf("deleting PVCs and apps failed, %d errors were logged", failed) + } + validateSubvolumeCount(f, 0, fileSystemName, subvolumegroup) } })