vendor files

vendor/k8s.io/kubernetes/pkg/util/goroutinemap/BUILD

@@ -0,0 +1,42 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_library",
+    "go_test",
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = ["goroutinemap.go"],
+    importpath = "k8s.io/kubernetes/pkg/util/goroutinemap",
+    deps = [
+        "//pkg/util/goroutinemap/exponentialbackoff:go_default_library",
+        "//vendor/github.com/golang/glog:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/util/runtime:go_default_library",
+    ],
+)
+
+go_test(
+    name = "go_default_test",
+    srcs = ["goroutinemap_test.go"],
+    importpath = "k8s.io/kubernetes/pkg/util/goroutinemap",
+    library = ":go_default_library",
+    deps = ["//vendor/k8s.io/apimachinery/pkg/util/wait:go_default_library"],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [
+        ":package-srcs",
+        "//pkg/util/goroutinemap/exponentialbackoff:all-srcs",
+    ],
+    tags = ["automanaged"],
+)

vendor/k8s.io/kubernetes/pkg/util/goroutinemap/OWNERS

@@ -0,0 +1,4 @@
+approvers:
+- saad-ali
+reviewers:
+- saad-ali

vendor/k8s.io/kubernetes/pkg/util/goroutinemap/exponentialbackoff/BUILD

@@ -0,0 +1,25 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_library",
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = ["exponential_backoff.go"],
+    importpath = "k8s.io/kubernetes/pkg/util/goroutinemap/exponentialbackoff",
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+)

vendor/k8s.io/kubernetes/pkg/util/goroutinemap/exponentialbackoff/exponential_backoff.go

@@ -0,0 +1,121 @@
+/*
+Copyright 2016 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// Package exponentialbackoff contains logic for implementing exponential
+// backoff for GoRoutineMap and NestedPendingOperations.
+package exponentialbackoff
+
+import (
+	"fmt"
+	"time"
+)
+
+const (
+	// initialDurationBeforeRetry is the amount of time after an error occurs
+	// that GoroutineMap will refuse to allow another operation to start with
+	// the same target (if exponentialBackOffOnError is enabled). Each
+	// successive error results in a wait 2x times the previous.
+	initialDurationBeforeRetry time.Duration = 500 * time.Millisecond
+
+	// maxDurationBeforeRetry is the maximum amount of time that
+	// durationBeforeRetry will grow to due to exponential backoff.
+	// Value is slightly offset from 2 minutes to make timeouts due to this
+	// constant recognizable.
+	maxDurationBeforeRetry time.Duration = 2*time.Minute + 2*time.Second
+)
+
+// ExponentialBackoff contains the last occurrence of an error and the duration
+// that retries are not permitted.
+type ExponentialBackoff struct {
+	lastError           error
+	lastErrorTime       time.Time
+	durationBeforeRetry time.Duration
+}
+
+// SafeToRetry returns an error if the durationBeforeRetry period for the given
+// lastErrorTime has not yet expired. Otherwise it returns nil.
+func (expBackoff *ExponentialBackoff) SafeToRetry(operationName string) error {
+	if time.Since(expBackoff.lastErrorTime) <= expBackoff.durationBeforeRetry {
+		return NewExponentialBackoffError(operationName, *expBackoff)
+	}
+
+	return nil
+}
+
+func (expBackoff *ExponentialBackoff) Update(err *error) {
+	if expBackoff.durationBeforeRetry == 0 {
+		expBackoff.durationBeforeRetry = initialDurationBeforeRetry
+	} else {
+		expBackoff.durationBeforeRetry = 2 * expBackoff.durationBeforeRetry
+		if expBackoff.durationBeforeRetry > maxDurationBeforeRetry {
+			expBackoff.durationBeforeRetry = maxDurationBeforeRetry
+		}
+	}
+
+	expBackoff.lastError = *err
+	expBackoff.lastErrorTime = time.Now()
+}
+
+func (expBackoff *ExponentialBackoff) GenerateNoRetriesPermittedMsg(operationName string) string {
+	return fmt.Sprintf("Operation for %q failed. No retries permitted until %v (durationBeforeRetry %v). Error: %q",
+		operationName,
+		expBackoff.lastErrorTime.Add(expBackoff.durationBeforeRetry),
+		expBackoff.durationBeforeRetry,
+		expBackoff.lastError)
+}
+
+// NewExponentialBackoffError returns a new instance of ExponentialBackoff error.
+func NewExponentialBackoffError(
+	operationName string, expBackoff ExponentialBackoff) error {
+	return exponentialBackoffError{
+		operationName: operationName,
+		expBackoff:    expBackoff,
+	}
+}
+
+// IsExponentialBackoff returns true if an error returned from GoroutineMap
+// indicates that a new operation can not be started because
+// exponentialBackOffOnError is enabled and a previous operation with the same
+// operation failed within the durationBeforeRetry period.
+func IsExponentialBackoff(err error) bool {
+	switch err.(type) {
+	case exponentialBackoffError:
+		return true
+	default:
+		return false
+	}
+}
+
+// exponentialBackoffError is the error returned returned from GoroutineMap when
+// a new operation can not be started because exponentialBackOffOnError is
+// enabled and a previous operation with the same operation failed within the
+// durationBeforeRetry period.
+type exponentialBackoffError struct {
+	operationName string
+	expBackoff    ExponentialBackoff
+}
+
+var _ error = exponentialBackoffError{}
+
+func (err exponentialBackoffError) Error() string {
+	return fmt.Sprintf(
+		"Failed to create operation with name %q. An operation with that name failed at %v. No retries permitted until %v (%v). Last error: %q.",
+		err.operationName,
+		err.expBackoff.lastErrorTime,
+		err.expBackoff.lastErrorTime.Add(err.expBackoff.durationBeforeRetry),
+		err.expBackoff.durationBeforeRetry,
+		err.expBackoff.lastError)
+}

vendor/k8s.io/kubernetes/pkg/util/goroutinemap/goroutinemap.go

@@ -0,0 +1,245 @@
+/*
+Copyright 2016 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+/*
+Package goroutinemap implements a data structure for managing go routines
+by name. It prevents the creation of new go routines if an existing go routine
+with the same name exists.
+*/
+package goroutinemap
+
+import (
+	"fmt"
+	"sync"
+	"time"
+
+	"github.com/golang/glog"
+	k8sRuntime "k8s.io/apimachinery/pkg/util/runtime"
+	"k8s.io/kubernetes/pkg/util/goroutinemap/exponentialbackoff"
+)
+
+const (
+	// initialDurationBeforeRetry is the amount of time after an error occurs
+	// that GoRoutineMap will refuse to allow another operation to start with
+	// the same operation name (if exponentialBackOffOnError is enabled). Each
+	// successive error results in a wait 2x times the previous.
+	initialDurationBeforeRetry = 500 * time.Millisecond
+
+	// maxDurationBeforeRetry is the maximum amount of time that
+	// durationBeforeRetry will grow to due to exponential backoff.
+	// Value is slightly offset from 2 minutes to make timeouts due to this
+	// constant recognizable.
+	maxDurationBeforeRetry = 2*time.Minute + 1*time.Second
+)
+
+// GoRoutineMap defines a type that can run named goroutines and track their
+// state.  It prevents the creation of multiple goroutines with the same name
+// and may prevent recreation of a goroutine until after the a backoff time
+// has elapsed after the last goroutine with that name finished.
+type GoRoutineMap interface {
+	// Run adds operation name to the list of running operations and spawns a
+	// new go routine to execute the operation.
+	// If an operation with the same operation name already exists, an
+	// AlreadyExists or ExponentialBackoff error is returned.
+	// Once the operation is complete, the go routine is terminated and the
+	// operation name is removed from the list of executing operations allowing
+	// a new operation to be started with the same operation name without error.
+	Run(operationName string, operationFunc func() error) error
+
+	// Wait blocks until operations map is empty. This is typically
+	// necessary during tests - the test should wait until all operations finish
+	// and evaluate results after that.
+	Wait()
+
+	// WaitForCompletion blocks until either all operations have successfully completed
+	// or have failed but are not pending. The test should wait until operations are either
+	// complete or have failed.
+	WaitForCompletion()
+
+	// IsOperationPending returns true if the operation is pending (currently
+	// running), otherwise returns false.
+	IsOperationPending(operationName string) bool
+}
+
+// NewGoRoutineMap returns a new instance of GoRoutineMap.
+func NewGoRoutineMap(exponentialBackOffOnError bool) GoRoutineMap {
+	g := &goRoutineMap{
+		operations:                make(map[string]operation),
+		exponentialBackOffOnError: exponentialBackOffOnError,
+	}
+
+	g.cond = sync.NewCond(&g.lock)
+	return g
+}
+
+type goRoutineMap struct {
+	operations                map[string]operation
+	exponentialBackOffOnError bool
+	cond                      *sync.Cond
+	lock                      sync.RWMutex
+}
+
+// operation holds the state of a single goroutine.
+type operation struct {
+	operationPending bool
+	expBackoff       exponentialbackoff.ExponentialBackoff
+}
+
+func (grm *goRoutineMap) Run(
+	operationName string,
+	operationFunc func() error) error {
+	grm.lock.Lock()
+	defer grm.lock.Unlock()
+
+	existingOp, exists := grm.operations[operationName]
+	if exists {
+		// Operation with name exists
+		if existingOp.operationPending {
+			return NewAlreadyExistsError(operationName)
+		}
+
+		if err := existingOp.expBackoff.SafeToRetry(operationName); err != nil {
+			return err
+		}
+	}
+
+	grm.operations[operationName] = operation{
+		operationPending: true,
+		expBackoff:       existingOp.expBackoff,
+	}
+	go func() (err error) {
+		// Handle unhandled panics (very unlikely)
+		defer k8sRuntime.HandleCrash()
+		// Handle completion of and error, if any, from operationFunc()
+		defer grm.operationComplete(operationName, &err)
+		// Handle panic, if any, from operationFunc()
+		defer k8sRuntime.RecoverFromPanic(&err)
+		return operationFunc()
+	}()
+
+	return nil
+}
+
+// operationComplete handles the completion of a goroutine run in the
+// goRoutineMap.
+func (grm *goRoutineMap) operationComplete(
+	operationName string, err *error) {
+	// Defer operations are executed in Last-In is First-Out order. In this case
+	// the lock is acquired first when operationCompletes begins, and is
+	// released when the method finishes, after the lock is released cond is
+	// signaled to wake waiting goroutine.
+	defer grm.cond.Signal()
+	grm.lock.Lock()
+	defer grm.lock.Unlock()
+
+	if *err == nil || !grm.exponentialBackOffOnError {
+		// Operation completed without error, or exponentialBackOffOnError disabled
+		delete(grm.operations, operationName)
+		if *err != nil {
+			// Log error
+			glog.Errorf("operation for %q failed with: %v",
+				operationName,
+				*err)
+		}
+	} else {
+		// Operation completed with error and exponentialBackOffOnError Enabled
+		existingOp := grm.operations[operationName]
+		existingOp.expBackoff.Update(err)
+		existingOp.operationPending = false
+		grm.operations[operationName] = existingOp
+
+		// Log error
+		glog.Errorf("%v",
+			existingOp.expBackoff.GenerateNoRetriesPermittedMsg(operationName))
+	}
+}
+
+func (grm *goRoutineMap) IsOperationPending(operationName string) bool {
+	grm.lock.RLock()
+	defer grm.lock.RUnlock()
+	existingOp, exists := grm.operations[operationName]
+	if exists && existingOp.operationPending {
+		return true
+	}
+	return false
+}
+
+func (grm *goRoutineMap) Wait() {
+	grm.lock.Lock()
+	defer grm.lock.Unlock()
+
+	for len(grm.operations) > 0 {
+		grm.cond.Wait()
+	}
+}
+
+func (grm *goRoutineMap) WaitForCompletion() {
+	grm.lock.Lock()
+	defer grm.lock.Unlock()
+
+	for {
+		if len(grm.operations) == 0 || grm.nothingPending() {
+			break
+		} else {
+			grm.cond.Wait()
+		}
+	}
+}
+
+// Check if any operation is pending. Already assumes caller has the
+// necessary locks
+func (grm *goRoutineMap) nothingPending() bool {
+	nothingIsPending := true
+	for _, operation := range grm.operations {
+		if operation.operationPending {
+			nothingIsPending = false
+			break
+		}
+	}
+	return nothingIsPending
+}
+
+// NewAlreadyExistsError returns a new instance of AlreadyExists error.
+func NewAlreadyExistsError(operationName string) error {
+	return alreadyExistsError{operationName}
+}
+
+// IsAlreadyExists returns true if an error returned from GoRoutineMap indicates
+// a new operation can not be started because an operation with the same
+// operation name is already executing.
+func IsAlreadyExists(err error) bool {
+	switch err.(type) {
+	case alreadyExistsError:
+		return true
+	default:
+		return false
+	}
+}
+
+// alreadyExistsError is the error returned by GoRoutineMap when a new operation
+// can not be started because an operation with the same operation name is
+// already executing.
+type alreadyExistsError struct {
+	operationName string
+}
+
+var _ error = alreadyExistsError{}
+
+func (err alreadyExistsError) Error() string {
+	return fmt.Sprintf(
+		"Failed to create operation with name %q. An operation with that name is already executing.",
+		err.operationName)
+}

vendor/k8s.io/kubernetes/pkg/util/goroutinemap/goroutinemap_test.go

@@ -0,0 +1,531 @@
+/*
+Copyright 2016 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package goroutinemap
+
+import (
+	"fmt"
+	"testing"
+	"time"
+
+	"k8s.io/apimachinery/pkg/util/wait"
+)
+
+const (
+	// testTimeout is a timeout of goroutines to finish. This _should_ be just a
+	// "context switch" and it should take several ms, however, Clayton says "We
+	// have had flakes due to tests that assumed that 15s is long enough to sleep")
+	testTimeout time.Duration = 1 * time.Minute
+
+	// initialOperationWaitTimeShort is the initial amount of time the test will
+	// wait for an operation to complete (each successive failure results in
+	// exponential backoff).
+	initialOperationWaitTimeShort time.Duration = 20 * time.Millisecond
+
+	// initialOperationWaitTimeLong is the initial amount of time the test will
+	// wait for an operation to complete (each successive failure results in
+	// exponential backoff).
+	initialOperationWaitTimeLong time.Duration = 500 * time.Millisecond
+)
+
+func Test_NewGoRoutineMap_Positive_SingleOp(t *testing.T) {
+	// Arrange
+	grm := NewGoRoutineMap(false /* exponentialBackOffOnError */)
+	operationName := "operation-name"
+	operation := func() error { return nil }
+
+	// Act
+	err := grm.Run(operationName, operation)
+
+	// Assert
+	if err != nil {
+		t.Fatalf("NewGoRoutine failed. Expected: <no error> Actual: <%v>", err)
+	}
+}
+
+func Test_NewGoRoutineMap_Positive_TwoOps(t *testing.T) {
+	// Arrange
+	grm := NewGoRoutineMap(false /* exponentialBackOffOnError */)
+	operation1Name := "operation1-name"
+	operation2Name := "operation2-name"
+	operation := func() error { return nil }
+
+	// Act
+	err1 := grm.Run(operation1Name, operation)
+	err2 := grm.Run(operation2Name, operation)
+
+	// Assert
+	if err1 != nil {
+		t.Fatalf("NewGoRoutine %q failed. Expected: <no error> Actual: <%v>", operation1Name, err1)
+	}
+
+	if err2 != nil {
+		t.Fatalf("NewGoRoutine %q failed. Expected: <no error> Actual: <%v>", operation2Name, err2)
+	}
+}
+
+func Test_NewGoRoutineMap_Positive_SingleOpWithExpBackoff(t *testing.T) {
+	// Arrange
+	grm := NewGoRoutineMap(true /* exponentialBackOffOnError */)
+	operationName := "operation-name"
+	operation := func() error { return nil }
+
+	// Act
+	err := grm.Run(operationName, operation)
+
+	// Assert
+	if err != nil {
+		t.Fatalf("NewGoRoutine failed. Expected: <no error> Actual: <%v>", err)
+	}
+}
+
+func Test_NewGoRoutineMap_Positive_SecondOpAfterFirstCompletes(t *testing.T) {
+	// Arrange
+	grm := NewGoRoutineMap(false /* exponentialBackOffOnError */)
+	operationName := "operation-name"
+	operation1DoneCh := make(chan interface{}, 0 /* bufferSize */)
+	operation1 := generateCallbackFunc(operation1DoneCh)
+	err1 := grm.Run(operationName, operation1)
+	if err1 != nil {
+		t.Fatalf("NewGoRoutine failed. Expected: <no error> Actual: <%v>", err1)
+	}
+	operation2 := generateNoopFunc()
+	<-operation1DoneCh // Force operation1 to complete
+
+	// Act
+	err2 := retryWithExponentialBackOff(
+		time.Duration(initialOperationWaitTimeShort),
+		func() (bool, error) {
+			err := grm.Run(operationName, operation2)
+			if err != nil {
+				t.Logf("Warning: NewGoRoutine failed with %v. Will retry.", err)
+				return false, nil
+			}
+			return true, nil
+		},
+	)
+
+	// Assert
+	if err2 != nil {
+		t.Fatalf("NewGoRoutine failed. Expected: <no error> Actual: <%v>", err2)
+	}
+}
+
+func Test_NewGoRoutineMap_Positive_SecondOpAfterFirstCompletesWithExpBackoff(t *testing.T) {
+	// Arrange
+	grm := NewGoRoutineMap(true /* exponentialBackOffOnError */)
+	operationName := "operation-name"
+	operation1DoneCh := make(chan interface{}, 0 /* bufferSize */)
+	operation1 := generateCallbackFunc(operation1DoneCh)
+	err1 := grm.Run(operationName, operation1)
+	if err1 != nil {
+		t.Fatalf("NewGoRoutine failed. Expected: <no error> Actual: <%v>", err1)
+	}
+	operation2 := generateNoopFunc()
+	<-operation1DoneCh // Force operation1 to complete
+
+	// Act
+	err2 := retryWithExponentialBackOff(
+		time.Duration(initialOperationWaitTimeShort),
+		func() (bool, error) {
+			err := grm.Run(operationName, operation2)
+			if err != nil {
+				t.Logf("Warning: NewGoRoutine failed with %v. Will retry.", err)
+				return false, nil
+			}
+			return true, nil
+		},
+	)
+
+	// Assert
+	if err2 != nil {
+		t.Fatalf("NewGoRoutine failed. Expected: <no error> Actual: <%v>", err2)
+	}
+}
+
+func Test_NewGoRoutineMap_Positive_SecondOpAfterFirstPanics(t *testing.T) {
+	// Arrange
+	grm := NewGoRoutineMap(false /* exponentialBackOffOnError */)
+	operationName := "operation-name"
+	operation1 := generatePanicFunc()
+	err1 := grm.Run(operationName, operation1)
+	if err1 != nil {
+		t.Fatalf("NewGoRoutine failed. Expected: <no error> Actual: <%v>", err1)
+	}
+	operation2 := generateNoopFunc()
+
+	// Act
+	err2 := retryWithExponentialBackOff(
+		time.Duration(initialOperationWaitTimeShort),
+		func() (bool, error) {
+			err := grm.Run(operationName, operation2)
+			if err != nil {
+				t.Logf("Warning: NewGoRoutine failed with %v. Will retry.", err)
+				return false, nil
+			}
+			return true, nil
+		},
+	)
+
+	// Assert
+	if err2 != nil {
+		t.Fatalf("NewGoRoutine failed. Expected: <no error> Actual: <%v>", err2)
+	}
+}
+
+func Test_NewGoRoutineMap_Positive_SecondOpAfterFirstPanicsWithExpBackoff(t *testing.T) {
+	// Arrange
+	grm := NewGoRoutineMap(true /* exponentialBackOffOnError */)
+	operationName := "operation-name"
+	operation1 := generatePanicFunc()
+	err1 := grm.Run(operationName, operation1)
+	if err1 != nil {
+		t.Fatalf("NewGoRoutine failed. Expected: <no error> Actual: <%v>", err1)
+	}
+	operation2 := generateNoopFunc()
+
+	// Act
+	err2 := retryWithExponentialBackOff(
+		time.Duration(initialOperationWaitTimeLong), // Longer duration to accommodate for backoff
+		func() (bool, error) {
+			err := grm.Run(operationName, operation2)
+			if err != nil {
+				t.Logf("Warning: NewGoRoutine failed with %v. Will retry.", err)
+				return false, nil
+			}
+			return true, nil
+		},
+	)
+
+	// Assert
+	if err2 != nil {
+		t.Fatalf("NewGoRoutine failed. Expected: <no error> Actual: <%v>", err2)
+	}
+}
+
+func Test_NewGoRoutineMap_Negative_SecondOpBeforeFirstCompletes(t *testing.T) {
+	// Arrange
+	grm := NewGoRoutineMap(false /* exponentialBackOffOnError */)
+	operationName := "operation-name"
+	operation1DoneCh := make(chan interface{}, 0 /* bufferSize */)
+	operation1 := generateWaitFunc(operation1DoneCh)
+	err1 := grm.Run(operationName, operation1)
+	if err1 != nil {
+		t.Fatalf("NewGoRoutine failed. Expected: <no error> Actual: <%v>", err1)
+	}
+	operation2 := generateNoopFunc()
+
+	// Act
+	err2 := grm.Run(operationName, operation2)
+
+	// Assert
+	if err2 == nil {
+		t.Fatalf("NewGoRoutine did not fail. Expected: <Failed to create operation with name \"%s\". An operation with that name already exists.> Actual: <no error>", operationName)
+	}
+	if !IsAlreadyExists(err2) {
+		t.Fatalf("NewGoRoutine did not return alreadyExistsError, got: %v", err2)
+	}
+}
+
+func Test_NewGoRoutineMap_Negative_SecondOpBeforeFirstCompletesWithExpBackoff(t *testing.T) {
+	// Arrange
+	grm := NewGoRoutineMap(true /* exponentialBackOffOnError */)
+	operationName := "operation-name"
+	operation1DoneCh := make(chan interface{}, 0 /* bufferSize */)
+	operation1 := generateWaitFunc(operation1DoneCh)
+	err1 := grm.Run(operationName, operation1)
+	if err1 != nil {
+		t.Fatalf("NewGoRoutine failed. Expected: <no error> Actual: <%v>", err1)
+	}
+	operation2 := generateNoopFunc()
+
+	// Act
+	err2 := grm.Run(operationName, operation2)
+
+	// Assert
+	if err2 == nil {
+		t.Fatalf("NewGoRoutine did not fail. Expected: <Failed to create operation with name \"%s\". An operation with that name already exists.> Actual: <no error>", operationName)
+	}
+	if !IsAlreadyExists(err2) {
+		t.Fatalf("NewGoRoutine did not return alreadyExistsError, got: %v", err2)
+	}
+}
+
+func Test_NewGoRoutineMap_Positive_ThirdOpAfterFirstCompletes(t *testing.T) {
+	// Arrange
+	grm := NewGoRoutineMap(false /* exponentialBackOffOnError */)
+	operationName := "operation-name"
+	operation1DoneCh := make(chan interface{}, 0 /* bufferSize */)
+	operation1 := generateWaitFunc(operation1DoneCh)
+	err1 := grm.Run(operationName, operation1)
+	if err1 != nil {
+		t.Fatalf("NewGoRoutine failed. Expected: <no error> Actual: <%v>", err1)
+	}
+	operation2 := generateNoopFunc()
+	operation3 := generateNoopFunc()
+
+	// Act
+	err2 := grm.Run(operationName, operation2)
+
+	// Assert
+	if err2 == nil {
+		t.Fatalf("NewGoRoutine did not fail. Expected: <Failed to create operation with name \"%s\". An operation with that name already exists.> Actual: <no error>", operationName)
+	}
+	if !IsAlreadyExists(err2) {
+		t.Fatalf("NewGoRoutine did not return alreadyExistsError, got: %v", err2)
+	}
+
+	// Act
+	operation1DoneCh <- true // Force operation1 to complete
+	err3 := retryWithExponentialBackOff(
+		time.Duration(initialOperationWaitTimeShort),
+		func() (bool, error) {
+			err := grm.Run(operationName, operation3)
+			if err != nil {
+				t.Logf("Warning: NewGoRoutine failed with %v. Will retry.", err)
+				return false, nil
+			}
+			return true, nil
+		},
+	)
+
+	// Assert
+	if err3 != nil {
+		t.Fatalf("NewGoRoutine failed. Expected: <no error> Actual: <%v>", err3)
+	}
+}
+
+func Test_NewGoRoutineMap_Positive_ThirdOpAfterFirstCompletesWithExpBackoff(t *testing.T) {
+	// Arrange
+	grm := NewGoRoutineMap(true /* exponentialBackOffOnError */)
+	operationName := "operation-name"
+	operation1DoneCh := make(chan interface{}, 0 /* bufferSize */)
+	operation1 := generateWaitFunc(operation1DoneCh)
+	err1 := grm.Run(operationName, operation1)
+	if err1 != nil {
+		t.Fatalf("NewGoRoutine failed. Expected: <no error> Actual: <%v>", err1)
+	}
+	operation2 := generateNoopFunc()
+	operation3 := generateNoopFunc()
+
+	// Act
+	err2 := grm.Run(operationName, operation2)
+
+	// Assert
+	if err2 == nil {
+		t.Fatalf("NewGoRoutine did not fail. Expected: <Failed to create operation with name \"%s\". An operation with that name already exists.> Actual: <no error>", operationName)
+	}
+	if !IsAlreadyExists(err2) {
+		t.Fatalf("NewGoRoutine did not return alreadyExistsError, got: %v", err2)
+	}
+
+	// Act
+	operation1DoneCh <- true // Force operation1 to complete
+	err3 := retryWithExponentialBackOff(
+		time.Duration(initialOperationWaitTimeShort),
+		func() (bool, error) {
+			err := grm.Run(operationName, operation3)
+			if err != nil {
+				t.Logf("Warning: NewGoRoutine failed with %v. Will retry.", err)
+				return false, nil
+			}
+			return true, nil
+		},
+	)
+
+	// Assert
+	if err3 != nil {
+		t.Fatalf("NewGoRoutine failed. Expected: <no error> Actual: <%v>", err3)
+	}
+}
+
+func Test_NewGoRoutineMap_Positive_WaitEmpty(t *testing.T) {
+	// Test than Wait() on empty GoRoutineMap always succeeds without blocking
+	// Arrange
+	grm := NewGoRoutineMap(false /* exponentialBackOffOnError */)
+
+	// Act
+	waitDoneCh := make(chan interface{}, 1)
+	go func() {
+		grm.Wait()
+		waitDoneCh <- true
+	}()
+
+	// Assert
+	err := waitChannelWithTimeout(waitDoneCh, testTimeout)
+	if err != nil {
+		t.Errorf("Error waiting for GoRoutineMap.Wait: %v", err)
+	}
+}
+
+func Test_NewGoRoutineMap_Positive_WaitEmptyWithExpBackoff(t *testing.T) {
+	// Test than Wait() on empty GoRoutineMap always succeeds without blocking
+	// Arrange
+	grm := NewGoRoutineMap(true /* exponentialBackOffOnError */)
+
+	// Act
+	waitDoneCh := make(chan interface{}, 1)
+	go func() {
+		grm.Wait()
+		waitDoneCh <- true
+	}()
+
+	// Assert
+	err := waitChannelWithTimeout(waitDoneCh, testTimeout)
+	if err != nil {
+		t.Errorf("Error waiting for GoRoutineMap.Wait: %v", err)
+	}
+}
+
+func Test_NewGoRoutineMap_Positive_Wait(t *testing.T) {
+	// Test that Wait() really blocks until the last operation succeeds
+	// Arrange
+	grm := NewGoRoutineMap(false /* exponentialBackOffOnError */)
+	operationName := "operation-name"
+	operation1DoneCh := make(chan interface{}, 0 /* bufferSize */)
+	operation1 := generateWaitFunc(operation1DoneCh)
+	err := grm.Run(operationName, operation1)
+	if err != nil {
+		t.Fatalf("NewGoRoutine failed. Expected: <no error> Actual: <%v>", err)
+	}
+
+	// Act
+	waitDoneCh := make(chan interface{}, 1)
+	go func() {
+		grm.Wait()
+		waitDoneCh <- true
+	}()
+
+	// Finish the operation
+	operation1DoneCh <- true
+
+	// Assert
+	err = waitChannelWithTimeout(waitDoneCh, testTimeout)
+	if err != nil {
+		t.Fatalf("Error waiting for GoRoutineMap.Wait: %v", err)
+	}
+}
+
+func Test_NewGoRoutineMap_Positive_WaitWithExpBackoff(t *testing.T) {
+	// Test that Wait() really blocks until the last operation succeeds
+	// Arrange
+	grm := NewGoRoutineMap(true /* exponentialBackOffOnError */)
+	operationName := "operation-name"
+	operation1DoneCh := make(chan interface{}, 0 /* bufferSize */)
+	operation1 := generateWaitFunc(operation1DoneCh)
+	err := grm.Run(operationName, operation1)
+	if err != nil {
+		t.Fatalf("NewGoRoutine failed. Expected: <no error> Actual: <%v>", err)
+	}
+
+	// Act
+	waitDoneCh := make(chan interface{}, 1)
+	go func() {
+		grm.Wait()
+		waitDoneCh <- true
+	}()
+
+	// Finish the operation
+	operation1DoneCh <- true
+
+	// Assert
+	err = waitChannelWithTimeout(waitDoneCh, testTimeout)
+	if err != nil {
+		t.Fatalf("Error waiting for GoRoutineMap.Wait: %v", err)
+	}
+}
+
+func Test_NewGoRoutineMap_WaitForCompletionWithExpBackoff(t *testing.T) {
+	grm := NewGoRoutineMap(true /* exponentialBackOffOnError */)
+	operationName := "operation-err"
+
+	operation1DoneCh := make(chan interface{}, 0 /* bufferSize */)
+	operation1 := generateErrorFunc(operation1DoneCh)
+	err := grm.Run(operationName, operation1)
+	if err != nil {
+		t.Fatalf("NewGoRoutine failed. Expected: <no error> Actual: <%v>", err)
+	}
+
+	// Act
+	waitDoneCh := make(chan interface{}, 1)
+	go func() {
+		grm.WaitForCompletion()
+		waitDoneCh <- true
+	}()
+
+	// Finish the operation
+	operation1DoneCh <- true
+
+	// Assert that WaitForCompletion returns even if scheduled op had error
+	err = waitChannelWithTimeout(waitDoneCh, testTimeout)
+	if err != nil {
+		t.Fatalf("Error waiting for GoRoutineMap.Wait: %v", err)
+	}
+}
+
+func generateCallbackFunc(done chan<- interface{}) func() error {
+	return func() error {
+		done <- true
+		return nil
+	}
+}
+
+func generateErrorFunc(done <-chan interface{}) func() error {
+	return func() error {
+		<-done
+		return fmt.Errorf("Generic error")
+	}
+}
+
+func generateWaitFunc(done <-chan interface{}) func() error {
+	return func() error {
+		<-done
+		return nil
+	}
+}
+
+func generatePanicFunc() func() error {
+	return func() error {
+		panic("testing panic")
+	}
+}
+
+func generateNoopFunc() func() error {
+	return func() error { return nil }
+}
+
+func retryWithExponentialBackOff(initialDuration time.Duration, fn wait.ConditionFunc) error {
+	backoff := wait.Backoff{
+		Duration: initialDuration,
+		Factor:   3,
+		Jitter:   0,
+		Steps:    4,
+	}
+	return wait.ExponentialBackoff(backoff, fn)
+}
+
+func waitChannelWithTimeout(ch <-chan interface{}, timeout time.Duration) error {
+	timer := time.NewTimer(timeout)
+	defer timer.Stop()
+
+	select {
+	case <-ch:
+		// Success!
+		return nil
+	case <-timer.C:
+		return fmt.Errorf("timeout after %v", timeout)
+	}
+}