e2e: do not use Failf() to abort tests in a go-routine

There are several go-routines where Failf() is called, which will cause
a Golang panic inside the Ginko test framework. Instead of aborting the
go-routine, capture the error and check for failures once all
go-routines have finished.

Updates: #1359
Signed-off-by: Niels de Vos <ndevos@redhat.com>

e2e/cephfs.go

@@ -482,6 +482,7 @@ var _ = Describe("cephfs", func() {
 				if k8sVersionGreaterEquals(f.ClientSet, 1, 17) {
 					var wg sync.WaitGroup
 					totalCount := 3
+					wgErrs := make([]error, totalCount)
 					// totalSubvolumes represents the subvolumes in backend
 					// always totalCount+parentPVC
 					totalSubvolumes := totalCount + 1
@@ -520,15 +521,24 @@ var _ = Describe("cephfs", func() {
 					for i := 0; i < totalCount; i++ {
 						go func(w *sync.WaitGroup, n int, s vs.VolumeSnapshot) {
 							s.Name = fmt.Sprintf("%s%d", f.UniqueName, n)
-							err = createSnapshot(&s, deployTimeout)
+							wgErrs[n] = createSnapshot(&s, deployTimeout)
-							if err != nil {
-								e2elog.Failf("failed to create snapshot with error %v", err)
-							}
 							w.Done()
 						}(&wg, i, snap)
 					}
 					wg.Wait()
 
+					failed := 0
+					for i, err := range wgErrs {
+						if err != nil {
+							// not using Failf() as it aborts the test and does not log other errors
+							e2elog.Logf("failed to create snapshot (%s%d): %v", f.UniqueName, i, err)
+							failed++
+						}
+					}
+					if failed != 0 {
+						e2elog.Failf("creating snapshots failed, %d errors were logged", failed)
+					}
+
 					pvcClone, err := loadPVC(pvcClonePath)
 					if err != nil {
 						e2elog.Failf("failed to load PVC with error %v", err)
@@ -547,15 +557,23 @@ var _ = Describe("cephfs", func() {
 
 						go func(w *sync.WaitGroup, n int, p v1.PersistentVolumeClaim, a v1.Pod) {
 							name := fmt.Sprintf("%s%d", f.UniqueName, n)
-							err = createPVCAndApp(name, f, &p, &a, deployTimeout)
+							wgErrs[n] = createPVCAndApp(name, f, &p, &a, deployTimeout)
-							if err != nil {
-								e2elog.Failf("failed to create PVC and app with error %v", err)
-							}
 							w.Done()
 						}(&wg, i, *pvcClone, *appClone)
 					}
 					wg.Wait()
 
+					for i, err := range wgErrs {
+						if err != nil {
+							// not using Failf() as it aborts the test and does not log other errors
+							e2elog.Logf("failed to create PVC and app (%s%d): %v", f.UniqueName, i, err)
+							failed++
+						}
+					}
+					if failed != 0 {
+						e2elog.Failf("creating PVCs and apps failed, %d errors were logged", failed)
+					}
+
 					validateSubvolumeCount(f, totalSubvolumes, fileSystemName, subvolumegroup)
 
 					wg.Add(totalCount)
@@ -564,15 +582,23 @@ var _ = Describe("cephfs", func() {
 						go func(w *sync.WaitGroup, n int, p v1.PersistentVolumeClaim, a v1.Pod) {
 							name := fmt.Sprintf("%s%d", f.UniqueName, n)
 							p.Spec.DataSource.Name = name
-							err = deletePVCAndApp(name, f, &p, &a)
+							wgErrs[n] = deletePVCAndApp(name, f, &p, &a)
-							if err != nil {
-								e2elog.Failf("failed to delete PVC and app with error %v", err)
-							}
 							w.Done()
 						}(&wg, i, *pvcClone, *appClone)
 					}
 					wg.Wait()
 
+					for i, err := range wgErrs {
+						if err != nil {
+							// not using Failf() as it aborts the test and does not log other errors
+							e2elog.Logf("failed to delete PVC and app (%s%d): %v", f.UniqueName, i, err)
+							failed++
+						}
+					}
+					if failed != 0 {
+						e2elog.Failf("deleting PVCs and apps failed, %d errors were logged", failed)
+					}
+
 					parentPVCCount := totalSubvolumes - totalCount
 					validateSubvolumeCount(f, parentPVCCount, fileSystemName, subvolumegroup)
 					// create clones from different snapshosts and bind it to an
@@ -582,15 +608,23 @@ var _ = Describe("cephfs", func() {
 						go func(w *sync.WaitGroup, n int, p v1.PersistentVolumeClaim, a v1.Pod) {
 							name := fmt.Sprintf("%s%d", f.UniqueName, n)
 							p.Spec.DataSource.Name = name
-							err = createPVCAndApp(name, f, &p, &a, deployTimeout)
+							wgErrs[n] = createPVCAndApp(name, f, &p, &a, deployTimeout)
-							if err != nil {
-								e2elog.Failf("failed to create PVC and app with error %v", err)
-							}
 							w.Done()
 						}(&wg, i, *pvcClone, *appClone)
 					}
 					wg.Wait()
 
+					for i, err := range wgErrs {
+						if err != nil {
+							// not using Failf() as it aborts the test and does not log other errors
+							e2elog.Logf("failed to create PVC and app (%s%d): %v", f.UniqueName, i, err)
+							failed++
+						}
+					}
+					if failed != 0 {
+						e2elog.Failf("creating PVCs and apps failed, %d errors were logged", failed)
+					}
+
 					validateSubvolumeCount(f, totalSubvolumes, fileSystemName, subvolumegroup)
 
 					wg.Add(totalCount)
@@ -598,30 +632,46 @@ var _ = Describe("cephfs", func() {
 					for i := 0; i < totalCount; i++ {
 						go func(w *sync.WaitGroup, n int, s vs.VolumeSnapshot) {
 							s.Name = fmt.Sprintf("%s%d", f.UniqueName, n)
-							err = deleteSnapshot(&s, deployTimeout)
+							wgErrs[n] = deleteSnapshot(&s, deployTimeout)
-							if err != nil {
-								e2elog.Failf("failed to delete snapshot with error %v", err)
-							}
 							w.Done()
 						}(&wg, i, snap)
 					}
 					wg.Wait()
 
+					for i, err := range wgErrs {
+						if err != nil {
+							// not using Failf() as it aborts the test and does not log other errors
+							e2elog.Logf("failed to delete snapshot (%s%d): %v", f.UniqueName, i, err)
+							failed++
+						}
+					}
+					if failed != 0 {
+						e2elog.Failf("deleting snapshots failed, %d errors were logged", failed)
+					}
+
 					wg.Add(totalCount)
 					// delete clone and app
 					for i := 0; i < totalCount; i++ {
 						go func(w *sync.WaitGroup, n int, p v1.PersistentVolumeClaim, a v1.Pod) {
 							name := fmt.Sprintf("%s%d", f.UniqueName, n)
 							p.Spec.DataSource.Name = name
-							err = deletePVCAndApp(name, f, &p, &a)
+							wgErrs[n] = deletePVCAndApp(name, f, &p, &a)
-							if err != nil {
-								e2elog.Failf("failed to delete PVC and app with error %v", err)
-							}
 							w.Done()
 						}(&wg, i, *pvcClone, *appClone)
 					}
 					wg.Wait()
 
+					for i, err := range wgErrs {
+						if err != nil {
+							// not using Failf() as it aborts the test and does not log other errors
+							e2elog.Logf("failed to delete PVC and app (%s%d): %v", f.UniqueName, i, err)
+							failed++
+						}
+					}
+					if failed != 0 {
+						e2elog.Failf("deleting PVCs and apps failed, %d errors were logged", failed)
+					}
+
 					validateSubvolumeCount(f, parentPVCCount, fileSystemName, subvolumegroup)
 					// delete parent pvc
 					err = deletePVCAndApp("", f, pvc, app)
@@ -638,6 +688,7 @@ var _ = Describe("cephfs", func() {
 				if k8sVersionGreaterEquals(f.ClientSet, 1, 16) {
 					var wg sync.WaitGroup
 					totalCount := 3
+					wgErrs := make([]error, totalCount)
 					// totalSubvolumes represents the subvolumes in backend
 					// always totalCount+parentPVC
 					totalSubvolumes := totalCount + 1
@@ -678,15 +729,24 @@ var _ = Describe("cephfs", func() {
 					for i := 0; i < totalCount; i++ {
 						go func(w *sync.WaitGroup, n int, p v1.PersistentVolumeClaim, a v1.Pod) {
 							name := fmt.Sprintf("%s%d", f.UniqueName, n)
-							err = createPVCAndApp(name, f, &p, &a, deployTimeout)
+							wgErrs[n] = createPVCAndApp(name, f, &p, &a, deployTimeout)
-							if err != nil {
-								e2elog.Failf("failed to create PVC or application with error %v", err)
-							}
 							w.Done()
 						}(&wg, i, *pvcClone, *appClone)
 					}
 					wg.Wait()
 
+					failed := 0
+					for i, err := range wgErrs {
+						if err != nil {
+							// not using Failf() as it aborts the test and does not log other errors
+							e2elog.Logf("failed to create PVC or application (%s%d): %v", f.UniqueName, i, err)
+							failed++
+						}
+					}
+					if failed != 0 {
+						e2elog.Failf("deleting PVCs and apps failed, %d errors were logged", failed)
+					}
+
 					validateSubvolumeCount(f, totalSubvolumes, fileSystemName, subvolumegroup)
 
 					// delete parent pvc
@@ -701,15 +761,23 @@ var _ = Describe("cephfs", func() {
 						go func(w *sync.WaitGroup, n int, p v1.PersistentVolumeClaim, a v1.Pod) {
 							name := fmt.Sprintf("%s%d", f.UniqueName, n)
 							p.Spec.DataSource.Name = name
-							err = deletePVCAndApp(name, f, &p, &a)
+							wgErrs[n] = deletePVCAndApp(name, f, &p, &a)
-							if err != nil {
-								e2elog.Failf("failed to delete PVC or application with error %v", err)
-							}
 							w.Done()
 						}(&wg, i, *pvcClone, *appClone)
 					}
 					wg.Wait()
 
+					for i, err := range wgErrs {
+						if err != nil {
+							// not using Failf() as it aborts the test and does not log other errors
+							e2elog.Logf("failed to delete PVC or application (%s%d): %v", f.UniqueName, i, err)
+							failed++
+						}
+					}
+					if failed != 0 {
+						e2elog.Failf("deleting PVCs and apps failed, %d errors were logged", failed)
+					}
+
 					validateSubvolumeCount(f, 0, fileSystemName, subvolumegroup)
 				}
 			})