vendor files

vendor/k8s.io/kubernetes/pkg/volume/util/nestedpendingoperations/BUILD

@@ -0,0 +1,45 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_library",
+    "go_test",
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = ["nestedpendingoperations.go"],
+    importpath = "k8s.io/kubernetes/pkg/volume/util/nestedpendingoperations",
+    deps = [
+        "//pkg/util/goroutinemap/exponentialbackoff:go_default_library",
+        "//pkg/volume/util/types:go_default_library",
+        "//vendor/github.com/golang/glog:go_default_library",
+        "//vendor/k8s.io/api/core/v1:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/util/runtime:go_default_library",
+    ],
+)
+
+go_test(
+    name = "go_default_test",
+    srcs = ["nestedpendingoperations_test.go"],
+    importpath = "k8s.io/kubernetes/pkg/volume/util/nestedpendingoperations",
+    library = ":go_default_library",
+    deps = [
+        "//pkg/volume/util/types:go_default_library",
+        "//vendor/k8s.io/api/core/v1:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/util/wait:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+)

vendor/k8s.io/kubernetes/pkg/volume/util/nestedpendingoperations/OWNERS

@@ -0,0 +1,2 @@
+approvers:
+- saad-ali

vendor/k8s.io/kubernetes/pkg/volume/util/nestedpendingoperations/nestedpendingoperations.go

@@ -0,0 +1,318 @@
+/*
+Copyright 2016 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+/*
+Package nestedpendingoperations is a modified implementation of
+pkg/util/goroutinemap. It implements a data structure for managing go routines
+by volume/pod name. It prevents the creation of new go routines if an existing
+go routine for the volume already exists. It also allows multiple operations to
+execute in parallel for the same volume as long as they are operating on
+different pods.
+*/
+package nestedpendingoperations
+
+import (
+	"fmt"
+	"sync"
+
+	"github.com/golang/glog"
+	"k8s.io/api/core/v1"
+	k8sRuntime "k8s.io/apimachinery/pkg/util/runtime"
+	"k8s.io/kubernetes/pkg/util/goroutinemap/exponentialbackoff"
+	"k8s.io/kubernetes/pkg/volume/util/types"
+)
+
+const (
+	// EmptyUniquePodName is a UniquePodName for empty string.
+	EmptyUniquePodName types.UniquePodName = types.UniquePodName("")
+
+	// EmptyUniqueVolumeName is a UniqueVolumeName for empty string
+	EmptyUniqueVolumeName v1.UniqueVolumeName = v1.UniqueVolumeName("")
+)
+
+// NestedPendingOperations defines the supported set of operations.
+type NestedPendingOperations interface {
+	// Run adds the concatenation of volumeName and podName to the list of
+	// running operations and spawns a new go routine to execute operationFunc.
+	// If an operation with the same volumeName and same or empty podName
+	// exists, an AlreadyExists or ExponentialBackoff error is returned.
+	// This enables multiple operations to execute in parallel for the same
+	// volumeName as long as they have different podName.
+	// Once the operation is complete, the go routine is terminated and the
+	// concatenation of volumeName and podName is removed from the list of
+	// executing operations allowing a new operation to be started with the
+	// volumeName without error.
+	Run(volumeName v1.UniqueVolumeName, podName types.UniquePodName, operationFunc func() error, operationCompleteFunc func(error)) error
+
+	// Wait blocks until all operations are completed. This is typically
+	// necessary during tests - the test should wait until all operations finish
+	// and evaluate results after that.
+	Wait()
+
+	// IsOperationPending returns true if an operation for the given volumeName and podName is pending,
+	// otherwise it returns false
+	IsOperationPending(volumeName v1.UniqueVolumeName, podName types.UniquePodName) bool
+}
+
+// NewNestedPendingOperations returns a new instance of NestedPendingOperations.
+func NewNestedPendingOperations(exponentialBackOffOnError bool) NestedPendingOperations {
+	g := &nestedPendingOperations{
+		operations:                []operation{},
+		exponentialBackOffOnError: exponentialBackOffOnError,
+	}
+	g.cond = sync.NewCond(&g.lock)
+	return g
+}
+
+type nestedPendingOperations struct {
+	operations                []operation
+	exponentialBackOffOnError bool
+	cond                      *sync.Cond
+	lock                      sync.RWMutex
+}
+
+type operation struct {
+	volumeName       v1.UniqueVolumeName
+	podName          types.UniquePodName
+	operationPending bool
+	expBackoff       exponentialbackoff.ExponentialBackoff
+}
+
+func (grm *nestedPendingOperations) Run(
+	volumeName v1.UniqueVolumeName,
+	podName types.UniquePodName,
+	operationFunc func() error,
+	operationCompleteFunc func(error)) error {
+	grm.lock.Lock()
+	defer grm.lock.Unlock()
+	opExists, previousOpIndex := grm.isOperationExists(volumeName, podName)
+	if opExists {
+		previousOp := grm.operations[previousOpIndex]
+		// Operation already exists
+		if previousOp.operationPending {
+			// Operation is pending
+			operationName := getOperationName(volumeName, podName)
+			return NewAlreadyExistsError(operationName)
+		}
+
+		operationName := getOperationName(volumeName, podName)
+		if err := previousOp.expBackoff.SafeToRetry(operationName); err != nil {
+			return err
+		}
+
+		// Update existing operation to mark as pending.
+		grm.operations[previousOpIndex].operationPending = true
+		grm.operations[previousOpIndex].volumeName = volumeName
+		grm.operations[previousOpIndex].podName = podName
+	} else {
+		// Create a new operation
+		grm.operations = append(grm.operations,
+			operation{
+				operationPending: true,
+				volumeName:       volumeName,
+				podName:          podName,
+				expBackoff:       exponentialbackoff.ExponentialBackoff{},
+			})
+	}
+
+	go func() (err error) {
+		// Handle unhandled panics (very unlikely)
+		defer k8sRuntime.HandleCrash()
+		// Handle completion of and error, if any, from operationFunc()
+		defer grm.operationComplete(volumeName, podName, &err)
+		defer operationCompleteFunc(err)
+		// Handle panic, if any, from operationFunc()
+		defer k8sRuntime.RecoverFromPanic(&err)
+		return operationFunc()
+	}()
+
+	return nil
+}
+
+func (grm *nestedPendingOperations) IsOperationPending(
+	volumeName v1.UniqueVolumeName,
+	podName types.UniquePodName) bool {
+
+	grm.lock.RLock()
+	defer grm.lock.RUnlock()
+
+	exist, previousOpIndex := grm.isOperationExists(volumeName, podName)
+	if exist && grm.operations[previousOpIndex].operationPending {
+		return true
+	}
+	return false
+}
+
+// This is an internal function and caller should acquire and release the lock
+func (grm *nestedPendingOperations) isOperationExists(
+	volumeName v1.UniqueVolumeName,
+	podName types.UniquePodName) (bool, int) {
+
+	// If volumeName is empty, operation can be executed concurrently
+	if volumeName == EmptyUniqueVolumeName {
+		return false, -1
+	}
+
+	for previousOpIndex, previousOp := range grm.operations {
+		if previousOp.volumeName != volumeName {
+			// No match, keep searching
+			continue
+		}
+
+		if previousOp.podName != EmptyUniquePodName &&
+			podName != EmptyUniquePodName &&
+			previousOp.podName != podName {
+			// No match, keep searching
+			continue
+		}
+
+		// Match
+		return true, previousOpIndex
+	}
+	return false, -1
+}
+
+func (grm *nestedPendingOperations) getOperation(
+	volumeName v1.UniqueVolumeName,
+	podName types.UniquePodName) (uint, error) {
+	// Assumes lock has been acquired by caller.
+
+	for i, op := range grm.operations {
+		if op.volumeName == volumeName &&
+			op.podName == podName {
+			return uint(i), nil
+		}
+	}
+
+	logOperationName := getOperationName(volumeName, podName)
+	return 0, fmt.Errorf("Operation %q not found", logOperationName)
+}
+
+func (grm *nestedPendingOperations) deleteOperation(
+	// Assumes lock has been acquired by caller.
+	volumeName v1.UniqueVolumeName,
+	podName types.UniquePodName) {
+
+	opIndex := -1
+	for i, op := range grm.operations {
+		if op.volumeName == volumeName &&
+			op.podName == podName {
+			opIndex = i
+			break
+		}
+	}
+
+	// Delete index without preserving order
+	grm.operations[opIndex] = grm.operations[len(grm.operations)-1]
+	grm.operations = grm.operations[:len(grm.operations)-1]
+}
+
+func (grm *nestedPendingOperations) operationComplete(
+	volumeName v1.UniqueVolumeName, podName types.UniquePodName, err *error) {
+	// Defer operations are executed in Last-In is First-Out order. In this case
+	// the lock is acquired first when operationCompletes begins, and is
+	// released when the method finishes, after the lock is released cond is
+	// signaled to wake waiting goroutine.
+	defer grm.cond.Signal()
+	grm.lock.Lock()
+	defer grm.lock.Unlock()
+
+	if *err == nil || !grm.exponentialBackOffOnError {
+		// Operation completed without error, or exponentialBackOffOnError disabled
+		grm.deleteOperation(volumeName, podName)
+		if *err != nil {
+			// Log error
+			logOperationName := getOperationName(volumeName, podName)
+			glog.Errorf("operation %s failed with: %v",
+				logOperationName,
+				*err)
+		}
+		return
+	}
+
+	// Operation completed with error and exponentialBackOffOnError Enabled
+	existingOpIndex, getOpErr := grm.getOperation(volumeName, podName)
+	if getOpErr != nil {
+		// Failed to find existing operation
+		logOperationName := getOperationName(volumeName, podName)
+		glog.Errorf("Operation %s completed. error: %v. exponentialBackOffOnError is enabled, but failed to get operation to update.",
+			logOperationName,
+			*err)
+		return
+	}
+
+	grm.operations[existingOpIndex].expBackoff.Update(err)
+	grm.operations[existingOpIndex].operationPending = false
+
+	// Log error
+	operationName :=
+		getOperationName(volumeName, podName)
+	glog.Errorf("%v", grm.operations[existingOpIndex].expBackoff.
+		GenerateNoRetriesPermittedMsg(operationName))
+}
+
+func (grm *nestedPendingOperations) Wait() {
+	grm.lock.Lock()
+	defer grm.lock.Unlock()
+
+	for len(grm.operations) > 0 {
+		grm.cond.Wait()
+	}
+}
+
+func getOperationName(
+	volumeName v1.UniqueVolumeName, podName types.UniquePodName) string {
+	podNameStr := ""
+	if podName != EmptyUniquePodName {
+		podNameStr = fmt.Sprintf(" (%q)", podName)
+	}
+
+	return fmt.Sprintf("%q%s",
+		volumeName,
+		podNameStr)
+}
+
+// NewAlreadyExistsError returns a new instance of AlreadyExists error.
+func NewAlreadyExistsError(operationName string) error {
+	return alreadyExistsError{operationName}
+}
+
+// IsAlreadyExists returns true if an error returned from
+// NestedPendingOperations indicates a new operation can not be started because
+// an operation with the same operation name is already executing.
+func IsAlreadyExists(err error) bool {
+	switch err.(type) {
+	case alreadyExistsError:
+		return true
+	default:
+		return false
+	}
+}
+
+// alreadyExistsError is the error returned by NestedPendingOperations when a
+// new operation can not be started because an operation with the same operation
+// name is already executing.
+type alreadyExistsError struct {
+	operationName string
+}
+
+var _ error = alreadyExistsError{}
+
+func (err alreadyExistsError) Error() string {
+	return fmt.Sprintf(
+		"Failed to create operation with name %q. An operation with that name is already executing.",
+		err.operationName)
+}

vendor/k8s.io/kubernetes/pkg/volume/util/nestedpendingoperations/nestedpendingoperations_test.go

@@ -0,0 +1,570 @@
+/*
+Copyright 2016 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package nestedpendingoperations
+
+import (
+	"fmt"
+	"testing"
+	"time"
+
+	"k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/util/wait"
+	"k8s.io/kubernetes/pkg/volume/util/types"
+)
+
+const (
+	// testTimeout is a timeout of goroutines to finish. This _should_ be just a
+	// "context switch" and it should take several ms, however, Clayton says "We
+	// have had flakes due to tests that assumed that 15s is long enough to sleep")
+	testTimeout time.Duration = 1 * time.Minute
+
+	// initialOperationWaitTimeShort is the initial amount of time the test will
+	// wait for an operation to complete (each successive failure results in
+	// exponential backoff).
+	initialOperationWaitTimeShort time.Duration = 20 * time.Millisecond
+
+	// initialOperationWaitTimeLong is the initial amount of time the test will
+	// wait for an operation to complete (each successive failure results in
+	// exponential backoff).
+	initialOperationWaitTimeLong time.Duration = 500 * time.Millisecond
+)
+
+func Test_NewGoRoutineMap_Positive_SingleOp(t *testing.T) {
+	// Arrange
+	grm := NewNestedPendingOperations(false /* exponentialBackOffOnError */)
+	volumeName := v1.UniqueVolumeName("volume-name")
+	operation := func() error { return nil }
+
+	// Act
+	err := grm.Run(volumeName, "" /* operationSubName */, operation, func(error) {})
+
+	// Assert
+	if err != nil {
+		t.Fatalf("NewGoRoutine failed. Expected: <no error> Actual: <%v>", err)
+	}
+}
+
+func Test_NewGoRoutineMap_Positive_TwoOps(t *testing.T) {
+	// Arrange
+	grm := NewNestedPendingOperations(false /* exponentialBackOffOnError */)
+	volume1Name := v1.UniqueVolumeName("volume1-name")
+	volume2Name := v1.UniqueVolumeName("volume2-name")
+	operation := func() error { return nil }
+
+	// Act
+	err1 := grm.Run(volume1Name, "" /* operationSubName */, operation, func(error) {})
+	err2 := grm.Run(volume2Name, "" /* operationSubName */, operation, func(error) {})
+
+	// Assert
+	if err1 != nil {
+		t.Fatalf("NewGoRoutine %q failed. Expected: <no error> Actual: <%v>", volume1Name, err1)
+	}
+
+	if err2 != nil {
+		t.Fatalf("NewGoRoutine %q failed. Expected: <no error> Actual: <%v>", volume2Name, err2)
+	}
+}
+
+func Test_NewGoRoutineMap_Positive_TwoSubOps(t *testing.T) {
+	// Arrange
+	grm := NewNestedPendingOperations(false /* exponentialBackOffOnError */)
+	volumeName := v1.UniqueVolumeName("volume-name")
+	operation1PodName := types.UniquePodName("operation1-podname")
+	operation2PodName := types.UniquePodName("operation2-podname")
+	operation := func() error { return nil }
+
+	// Act
+	err1 := grm.Run(volumeName, operation1PodName, operation, func(error) {})
+	err2 := grm.Run(volumeName, operation2PodName, operation, func(error) {})
+
+	// Assert
+	if err1 != nil {
+		t.Fatalf("NewGoRoutine %q failed. Expected: <no error> Actual: <%v>", operation1PodName, err1)
+	}
+
+	if err2 != nil {
+		t.Fatalf("NewGoRoutine %q failed. Expected: <no error> Actual: <%v>", operation2PodName, err2)
+	}
+}
+
+func Test_NewGoRoutineMap_Positive_SingleOpWithExpBackoff(t *testing.T) {
+	// Arrange
+	grm := NewNestedPendingOperations(true /* exponentialBackOffOnError */)
+	volumeName := v1.UniqueVolumeName("volume-name")
+	operation := func() error { return nil }
+
+	// Act
+	err := grm.Run(volumeName, "" /* operationSubName */, operation, func(error) {})
+
+	// Assert
+	if err != nil {
+		t.Fatalf("NewGoRoutine failed. Expected: <no error> Actual: <%v>", err)
+	}
+}
+
+func Test_NewGoRoutineMap_Positive_SecondOpAfterFirstCompletes(t *testing.T) {
+	// Arrange
+	grm := NewNestedPendingOperations(false /* exponentialBackOffOnError */)
+	volumeName := v1.UniqueVolumeName("volume-name")
+	operation1DoneCh := make(chan interface{}, 0 /* bufferSize */)
+	operation1 := generateCallbackFunc(operation1DoneCh)
+	err1 := grm.Run(volumeName, "" /* operationSubName */, operation1, func(error) {})
+	if err1 != nil {
+		t.Fatalf("NewGoRoutine failed. Expected: <no error> Actual: <%v>", err1)
+	}
+	operation2 := generateNoopFunc()
+	<-operation1DoneCh // Force operation1 to complete
+
+	// Act
+	err2 := retryWithExponentialBackOff(
+		time.Duration(initialOperationWaitTimeShort),
+		func() (bool, error) {
+			err := grm.Run(volumeName, "" /* operationSubName */, operation2, func(error) {})
+			if err != nil {
+				t.Logf("Warning: NewGoRoutine failed with %v. Will retry.", err)
+				return false, nil
+			}
+			return true, nil
+		},
+	)
+
+	// Assert
+	if err2 != nil {
+		t.Fatalf("NewGoRoutine failed. Expected: <no error> Actual: <%v>", err2)
+	}
+}
+
+func Test_NewGoRoutineMap_Positive_SecondOpAfterFirstCompletesWithExpBackoff(t *testing.T) {
+	// Arrange
+	grm := NewNestedPendingOperations(true /* exponentialBackOffOnError */)
+	volumeName := v1.UniqueVolumeName("volume-name")
+	operation1DoneCh := make(chan interface{}, 0 /* bufferSize */)
+	operation1 := generateCallbackFunc(operation1DoneCh)
+	err1 := grm.Run(volumeName, "" /* operationSubName */, operation1, func(error) {})
+	if err1 != nil {
+		t.Fatalf("NewGoRoutine failed. Expected: <no error> Actual: <%v>", err1)
+	}
+	operation2 := generateNoopFunc()
+	<-operation1DoneCh // Force operation1 to complete
+
+	// Act
+	err2 := retryWithExponentialBackOff(
+		time.Duration(initialOperationWaitTimeShort),
+		func() (bool, error) {
+			err := grm.Run(volumeName, "" /* operationSubName */, operation2, func(error) {})
+			if err != nil {
+				t.Logf("Warning: NewGoRoutine failed with %v. Will retry.", err)
+				return false, nil
+			}
+			return true, nil
+		},
+	)
+
+	// Assert
+	if err2 != nil {
+		t.Fatalf("NewGoRoutine failed. Expected: <no error> Actual: <%v>", err2)
+	}
+}
+
+func Test_NewGoRoutineMap_Positive_SecondOpAfterFirstPanics(t *testing.T) {
+	// Arrange
+	grm := NewNestedPendingOperations(false /* exponentialBackOffOnError */)
+	volumeName := v1.UniqueVolumeName("volume-name")
+	operation1 := generatePanicFunc()
+	err1 := grm.Run(volumeName, "" /* operationSubName */, operation1, func(error) {})
+	if err1 != nil {
+		t.Fatalf("NewGoRoutine failed. Expected: <no error> Actual: <%v>", err1)
+	}
+	operation2 := generateNoopFunc()
+
+	// Act
+	err2 := retryWithExponentialBackOff(
+		time.Duration(initialOperationWaitTimeShort),
+		func() (bool, error) {
+			err := grm.Run(volumeName, "" /* operationSubName */, operation2, func(error) {})
+			if err != nil {
+				t.Logf("Warning: NewGoRoutine failed with %v. Will retry.", err)
+				return false, nil
+			}
+			return true, nil
+		},
+	)
+
+	// Assert
+	if err2 != nil {
+		t.Fatalf("NewGoRoutine failed. Expected: <no error> Actual: <%v>", err2)
+	}
+}
+
+func Test_NewGoRoutineMap_Positive_SecondOpAfterFirstPanicsWithExpBackoff(t *testing.T) {
+	// Arrange
+	grm := NewNestedPendingOperations(true /* exponentialBackOffOnError */)
+	volumeName := v1.UniqueVolumeName("volume-name")
+	operation1 := generatePanicFunc()
+	err1 := grm.Run(volumeName, "" /* operationSubName */, operation1, func(error) {})
+	if err1 != nil {
+		t.Fatalf("NewGoRoutine failed. Expected: <no error> Actual: <%v>", err1)
+	}
+	operation2 := generateNoopFunc()
+
+	// Act
+	err2 := retryWithExponentialBackOff(
+		time.Duration(initialOperationWaitTimeLong), // Longer duration to accommodate for backoff
+		func() (bool, error) {
+			err := grm.Run(volumeName, "" /* operationSubName */, operation2, func(error) {})
+			if err != nil {
+				t.Logf("Warning: NewGoRoutine failed with %v. Will retry.", err)
+				return false, nil
+			}
+			return true, nil
+		},
+	)
+
+	// Assert
+	if err2 != nil {
+		t.Fatalf("NewGoRoutine failed. Expected: <no error> Actual: <%v>", err2)
+	}
+}
+
+func Test_NewGoRoutineMap_Negative_SecondOpBeforeFirstCompletes(t *testing.T) {
+	// Arrange
+	grm := NewNestedPendingOperations(false /* exponentialBackOffOnError */)
+	volumeName := v1.UniqueVolumeName("volume-name")
+	operation1DoneCh := make(chan interface{}, 0 /* bufferSize */)
+	operation1 := generateWaitFunc(operation1DoneCh)
+	err1 := grm.Run(volumeName, "" /* operationSubName */, operation1, func(error) {})
+	if err1 != nil {
+		t.Fatalf("NewGoRoutine failed. Expected: <no error> Actual: <%v>", err1)
+	}
+	operation2 := generateNoopFunc()
+
+	// Act
+	err2 := grm.Run(volumeName, "" /* operationSubName */, operation2, func(error) {})
+
+	// Assert
+	if err2 == nil {
+		t.Fatalf("NewGoRoutine did not fail. Expected: <Failed to create operation with name \"%s\". An operation with that name already exists.> Actual: <no error>", volumeName)
+	}
+	if !IsAlreadyExists(err2) {
+		t.Fatalf("NewGoRoutine did not return alreadyExistsError, got: %v", err2)
+	}
+}
+
+func Test_NewGoRoutineMap_Negative_SecondSubOpBeforeFirstCompletes2(t *testing.T) {
+	// Arrange
+	grm := NewNestedPendingOperations(false /* exponentialBackOffOnError */)
+	volumeName := v1.UniqueVolumeName("volume-name")
+	operationPodName := types.UniquePodName("operation-podname")
+	operation1DoneCh := make(chan interface{}, 0 /* bufferSize */)
+	operation1 := generateWaitFunc(operation1DoneCh)
+	err1 := grm.Run(volumeName, operationPodName, operation1, func(error) {})
+	if err1 != nil {
+		t.Fatalf("NewGoRoutine failed. Expected: <no error> Actual: <%v>", err1)
+	}
+	operation2 := generateNoopFunc()
+
+	// Act
+	err2 := grm.Run(volumeName, operationPodName, operation2, func(error) {})
+
+	// Assert
+	if err2 == nil {
+		t.Fatalf("NewGoRoutine did not fail. Expected: <Failed to create operation with name \"%s\". An operation with that name already exists.> Actual: <no error>", volumeName)
+	}
+	if !IsAlreadyExists(err2) {
+		t.Fatalf("NewGoRoutine did not return alreadyExistsError, got: %v", err2)
+	}
+}
+
+func Test_NewGoRoutineMap_Negative_SecondSubOpBeforeFirstCompletes(t *testing.T) {
+	// Arrange
+	grm := NewNestedPendingOperations(false /* exponentialBackOffOnError */)
+	volumeName := v1.UniqueVolumeName("volume-name")
+	operationPodName := types.UniquePodName("operation-podname")
+	operation1DoneCh := make(chan interface{}, 0 /* bufferSize */)
+	operation1 := generateWaitFunc(operation1DoneCh)
+	err1 := grm.Run(volumeName, operationPodName, operation1, func(error) {})
+	if err1 != nil {
+		t.Fatalf("NewGoRoutine failed. Expected: <no error> Actual: <%v>", err1)
+	}
+	operation2 := generateNoopFunc()
+
+	// Act
+	err2 := grm.Run(volumeName, operationPodName, operation2, func(error) {})
+
+	// Assert
+	if err2 == nil {
+		t.Fatalf("NewGoRoutine did not fail. Expected: <Failed to create operation with name \"%s\". An operation with that name already exists.> Actual: <no error>", volumeName)
+	}
+	if !IsAlreadyExists(err2) {
+		t.Fatalf("NewGoRoutine did not return alreadyExistsError, got: %v", err2)
+	}
+}
+
+func Test_NewGoRoutineMap_Negative_SecondOpBeforeFirstCompletesWithExpBackoff(t *testing.T) {
+	// Arrange
+	grm := NewNestedPendingOperations(true /* exponentialBackOffOnError */)
+	volumeName := v1.UniqueVolumeName("volume-name")
+	operation1DoneCh := make(chan interface{}, 0 /* bufferSize */)
+	operation1 := generateWaitFunc(operation1DoneCh)
+	err1 := grm.Run(volumeName, "" /* operationSubName */, operation1, func(error) {})
+	if err1 != nil {
+		t.Fatalf("NewGoRoutine failed. Expected: <no error> Actual: <%v>", err1)
+	}
+	operation2 := generateNoopFunc()
+
+	// Act
+	err2 := grm.Run(volumeName, "" /* operationSubName */, operation2, func(error) {})
+
+	// Assert
+	if err2 == nil {
+		t.Fatalf("NewGoRoutine did not fail. Expected: <Failed to create operation with name \"%s\". An operation with that name already exists.> Actual: <no error>", volumeName)
+	}
+	if !IsAlreadyExists(err2) {
+		t.Fatalf("NewGoRoutine did not return alreadyExistsError, got: %v", err2)
+	}
+}
+
+func Test_NewGoRoutineMap_Positive_ThirdOpAfterFirstCompletes(t *testing.T) {
+	// Arrange
+	grm := NewNestedPendingOperations(false /* exponentialBackOffOnError */)
+	volumeName := v1.UniqueVolumeName("volume-name")
+	operation1DoneCh := make(chan interface{}, 0 /* bufferSize */)
+	operation1 := generateWaitFunc(operation1DoneCh)
+	err1 := grm.Run(volumeName, "" /* operationSubName */, operation1, func(error) {})
+	if err1 != nil {
+		t.Fatalf("NewGoRoutine failed. Expected: <no error> Actual: <%v>", err1)
+	}
+	operation2 := generateNoopFunc()
+	operation3 := generateNoopFunc()
+
+	// Act
+	err2 := grm.Run(volumeName, "" /* operationSubName */, operation2, func(error) {})
+
+	// Assert
+	if err2 == nil {
+		t.Fatalf("NewGoRoutine did not fail. Expected: <Failed to create operation with name \"%s\". An operation with that name already exists.> Actual: <no error>", volumeName)
+	}
+	if !IsAlreadyExists(err2) {
+		t.Fatalf("NewGoRoutine did not return alreadyExistsError, got: %v", err2)
+	}
+
+	// Act
+	operation1DoneCh <- true // Force operation1 to complete
+	err3 := retryWithExponentialBackOff(
+		time.Duration(initialOperationWaitTimeShort),
+		func() (bool, error) {
+			err := grm.Run(volumeName, "" /* operationSubName */, operation3, func(error) {})
+			if err != nil {
+				t.Logf("Warning: NewGoRoutine failed with %v. Will retry.", err)
+				return false, nil
+			}
+			return true, nil
+		},
+	)
+
+	// Assert
+	if err3 != nil {
+		t.Fatalf("NewGoRoutine failed. Expected: <no error> Actual: <%v>", err3)
+	}
+}
+
+func Test_NewGoRoutineMap_Positive_ThirdOpAfterFirstCompletesWithExpBackoff(t *testing.T) {
+	// Arrange
+	grm := NewNestedPendingOperations(true /* exponentialBackOffOnError */)
+	volumeName := v1.UniqueVolumeName("volume-name")
+	operation1DoneCh := make(chan interface{}, 0 /* bufferSize */)
+	operation1 := generateWaitFunc(operation1DoneCh)
+	err1 := grm.Run(volumeName, "" /* operationSubName */, operation1, func(error) {})
+	if err1 != nil {
+		t.Fatalf("NewGoRoutine failed. Expected: <no error> Actual: <%v>", err1)
+	}
+	operation2 := generateNoopFunc()
+	operation3 := generateNoopFunc()
+
+	// Act
+	err2 := grm.Run(volumeName, "" /* operationSubName */, operation2, func(error) {})
+
+	// Assert
+	if err2 == nil {
+		t.Fatalf("NewGoRoutine did not fail. Expected: <Failed to create operation with name \"%s\". An operation with that name already exists.> Actual: <no error>", volumeName)
+	}
+	if !IsAlreadyExists(err2) {
+		t.Fatalf("NewGoRoutine did not return alreadyExistsError, got: %v", err2)
+	}
+
+	// Act
+	operation1DoneCh <- true // Force operation1 to complete
+	err3 := retryWithExponentialBackOff(
+		time.Duration(initialOperationWaitTimeShort),
+		func() (bool, error) {
+			err := grm.Run(volumeName, "" /* operationSubName */, operation3, func(error) {})
+			if err != nil {
+				t.Logf("Warning: NewGoRoutine failed with %v. Will retry.", err)
+				return false, nil
+			}
+			return true, nil
+		},
+	)
+
+	// Assert
+	if err3 != nil {
+		t.Fatalf("NewGoRoutine failed. Expected: <no error> Actual: <%v>", err3)
+	}
+}
+
+func Test_NewGoRoutineMap_Positive_WaitEmpty(t *testing.T) {
+	// Test than Wait() on empty GoRoutineMap always succeeds without blocking
+	// Arrange
+	grm := NewNestedPendingOperations(false /* exponentialBackOffOnError */)
+
+	// Act
+	waitDoneCh := make(chan interface{}, 1)
+	go func() {
+		grm.Wait()
+		waitDoneCh <- true
+	}()
+
+	// Assert
+	err := waitChannelWithTimeout(waitDoneCh, testTimeout)
+	if err != nil {
+		t.Errorf("Error waiting for GoRoutineMap.Wait: %v", err)
+	}
+}
+
+func Test_NewGoRoutineMap_Positive_WaitEmptyWithExpBackoff(t *testing.T) {
+	// Test than Wait() on empty GoRoutineMap always succeeds without blocking
+	// Arrange
+	grm := NewNestedPendingOperations(true /* exponentialBackOffOnError */)
+
+	// Act
+	waitDoneCh := make(chan interface{}, 1)
+	go func() {
+		grm.Wait()
+		waitDoneCh <- true
+	}()
+
+	// Assert
+	err := waitChannelWithTimeout(waitDoneCh, testTimeout)
+	if err != nil {
+		t.Errorf("Error waiting for GoRoutineMap.Wait: %v", err)
+	}
+}
+
+func Test_NewGoRoutineMap_Positive_Wait(t *testing.T) {
+	// Test that Wait() really blocks until the last operation succeeds
+	// Arrange
+	grm := NewNestedPendingOperations(false /* exponentialBackOffOnError */)
+	volumeName := v1.UniqueVolumeName("volume-name")
+	operation1DoneCh := make(chan interface{}, 0 /* bufferSize */)
+	operation1 := generateWaitFunc(operation1DoneCh)
+	err := grm.Run(volumeName, "" /* operationSubName */, operation1, func(error) {})
+	if err != nil {
+		t.Fatalf("NewGoRoutine failed. Expected: <no error> Actual: <%v>", err)
+	}
+
+	// Act
+	waitDoneCh := make(chan interface{}, 1)
+	go func() {
+		grm.Wait()
+		waitDoneCh <- true
+	}()
+
+	// Finish the operation
+	operation1DoneCh <- true
+
+	// Assert
+	err = waitChannelWithTimeout(waitDoneCh, testTimeout)
+	if err != nil {
+		t.Fatalf("Error waiting for GoRoutineMap.Wait: %v", err)
+	}
+}
+
+func Test_NewGoRoutineMap_Positive_WaitWithExpBackoff(t *testing.T) {
+	// Test that Wait() really blocks until the last operation succeeds
+	// Arrange
+	grm := NewNestedPendingOperations(true /* exponentialBackOffOnError */)
+	volumeName := v1.UniqueVolumeName("volume-name")
+	operation1DoneCh := make(chan interface{}, 0 /* bufferSize */)
+	operation1 := generateWaitFunc(operation1DoneCh)
+	err := grm.Run(volumeName, "" /* operationSubName */, operation1, func(error) {})
+	if err != nil {
+		t.Fatalf("NewGoRoutine failed. Expected: <no error> Actual: <%v>", err)
+	}
+
+	// Act
+	waitDoneCh := make(chan interface{}, 1)
+	go func() {
+		grm.Wait()
+		waitDoneCh <- true
+	}()
+
+	// Finish the operation
+	operation1DoneCh <- true
+
+	// Assert
+	err = waitChannelWithTimeout(waitDoneCh, testTimeout)
+	if err != nil {
+		t.Fatalf("Error waiting for GoRoutineMap.Wait: %v", err)
+	}
+}
+
+func generateCallbackFunc(done chan<- interface{}) func() error {
+	return func() error {
+		done <- true
+		return nil
+	}
+}
+
+func generateWaitFunc(done <-chan interface{}) func() error {
+	return func() error {
+		<-done
+		return nil
+	}
+}
+
+func generatePanicFunc() func() error {
+	return func() error {
+		panic("testing panic")
+	}
+}
+
+func generateNoopFunc() func() error {
+	return func() error { return nil }
+}
+
+func retryWithExponentialBackOff(initialDuration time.Duration, fn wait.ConditionFunc) error {
+	backoff := wait.Backoff{
+		Duration: initialDuration,
+		Factor:   3,
+		Jitter:   0,
+		Steps:    4,
+	}
+	return wait.ExponentialBackoff(backoff, fn)
+}
+
+func waitChannelWithTimeout(ch <-chan interface{}, timeout time.Duration) error {
+	timer := time.NewTimer(timeout)
+	defer timer.Stop()
+
+	select {
+	case <-ch:
+		// Success!
+		return nil
+	case <-timer.C:
+		return fmt.Errorf("timeout after %v", timeout)
+	}
+}